Page MenuHomeSoftware Heritage
Differential D726

swh.objstorage: Align config 'cls' key class for memory objstorage
ClosedPublic
Actions

Authored by ardumont on Nov 28 2018, 5:49 PM.

Details

Reviewers
olasd
Group Reviewers
Reviewers
Commits
rDOBJS994cf36e7541: swh.objstorage: Align config 'cls' key class for memory objstorage
Summary
  • storage uses 'memory'
  • indexer-storage starts as well

Might as well unify.

Did not see any use yet.

Indexers might start using this soon.

Test Plan

tox

Diff Detail

Repository
rDOBJS Object storage
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

ardumont created this revision.Nov 28 2018, 5:49 PM
swh-public-ci added a subscriber: swh-public-ci.Nov 28 2018, 5:51 PM

Build is green
See https://jenkins.softwareheritage.org/job/DOBJS/job/tox/6/ for more details.

Harbormaster completed remote builds in B2662: Diff 2285.Nov 28 2018, 5:51 PM
olasd accepted this revision.Nov 28 2018, 5:52 PM
olasd added a subscriber: olasd.

You could update swh/storage/tests/test_in_memory.py to use get_objstorage() with the class argument, that way it'd have at least one user ;)

This revision is now accepted and ready to land.Nov 28 2018, 5:52 PM
ardumont updated this revision to Diff 2286.Nov 28 2018, 5:59 PM

Use get_objstorage method to instantiate the memory objstorage

ardumont added a comment.Nov 28 2018, 5:59 PM

You could update swh/storage/tests/test_in_memory.py to use get_objstorage() with the class argument, that way it'd have at least one user ;)

Good idea!

swh-public-ci added a comment.Nov 28 2018, 6:01 PM

Build is green
See https://jenkins.softwareheritage.org/job/DOBJS/job/tox/7/ for more details.

Harbormaster completed remote builds in B2663: Diff 2286.Nov 28 2018, 6:01 PM
Closed by commit rDOBJS994cf36e7541: swh.objstorage: Align config 'cls' key class for memory objstorage (authored by ardumont). · Explain WhyNov 28 2018, 6:03 PM
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

PathSize
swh/
objstorage/
2 lines
tests/
6 lines

Diff 2287

View Options

swh/objstorage/__init__.py

# Copyright (C) 2015-2021 The Software Heritage developers # Copyright (C) 2015-2022 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution # See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version # License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information # See top-level LICENSE file for more information
import os import os
import shutil import shutil
import stat import stat
from subprocess import run from subprocess import run
import tarfile import tarfile
import zipfile import zipfile
import magic import magic
from . import utils from . import utils
MIMETYPE_TO_ARCHIVE_FORMAT = {
"application/x-compress": "tar.Z|x",
"application/x-tar": "tar",
"application/x-bzip2": "bztar",
"application/gzip": "gztar",
"application/x-lzip": "tar.lz",
"application/zip": "zip",
"application/java-archive": "jar",
"application/zstd": "tar.zst",
"application/x-zstd": "tar.zst",
}
def _unpack_tar(tarpath: str, extract_dir: str) -> str: def _unpack_tar(tarpath: str, extract_dir: str) -> str:
"""Unpack tarballs unsupported by the standard python library. Examples """Unpack tarballs unsupported by the standard python library. Examples
include tar.Z, tar.lz, tar.x, etc.... include tar.Z, tar.lz, tar.x, etc....
As this implementation relies on the `tar` command, this function supports As this implementation relies on the `tar` command, this function supports
the same compression the tar command supports. the same compression the tar command supports.
▲ Show 20 Lines • Show All 87 Lines • ▼ Show 20 Linesdef register_new_archive_formats():
"""Register new archive formats to uncompress""" """Register new archive formats to uncompress"""
registered_formats = [f[0] for f in shutil.get_unpack_formats()] registered_formats = [f[0] for f in shutil.get_unpack_formats()]
for name, extensions, function in ADDITIONAL_ARCHIVE_FORMATS: for name, extensions, function in ADDITIONAL_ARCHIVE_FORMATS:
if name in registered_formats: if name in registered_formats:
continue continue
shutil.register_unpack_format(name, extensions, function) shutil.register_unpack_format(name, extensions, function)
_mime_to_archive_format = {
"application/x-compress": "tar.Z|x",
"application/x-tar": "tar",
"application/x-bzip2": "bztar",
"application/gzip": "gztar",
"application/x-lzip": "tar.lz",
"application/zip": "zip",
"application/java-archive": "jar",
"application/zstd": "tar.zst",
"application/x-zstd": "tar.zst",
}
def uncompress(tarpath: str, dest: str): def uncompress(tarpath: str, dest: str):
"""Uncompress tarpath to dest folder if tarball is supported. """Uncompress tarpath to dest folder if tarball is supported.
Note that this fixes permissions after successfully Note that this fixes permissions after successfully
uncompressing the archive. uncompressing the archive.
Args: Args:
tarpath: path to tarball to uncompress tarpath: path to tarball to uncompress
Show All 11 Lines try:
for format_, exts, _ in shutil.get_unpack_formats(): for format_, exts, _ in shutil.get_unpack_formats():
if any([tarpath.lower().endswith(ext.lower()) for ext in exts]): if any([tarpath.lower().endswith(ext.lower()) for ext in exts]):
format = format_ format = format_
break break
# try to get archive format from file mimetype # try to get archive format from file mimetype
if format is None: if format is None:
m = magic.Magic(mime=True) m = magic.Magic(mime=True)
mime = m.from_file(tarpath) mime = m.from_file(tarpath)
format = _mime_to_archive_format.get(mime) format = MIMETYPE_TO_ARCHIVE_FORMAT.get(mime)
shutil.unpack_archive(tarpath, extract_dir=dest, format=format) shutil.unpack_archive(tarpath, extract_dir=dest, format=format)
except shutil.ReadError as e: except shutil.ReadError as e:
raise ValueError(f"Problem during unpacking {tarpath}. Reason: {e}") raise ValueError(f"Problem during unpacking {tarpath}. Reason: {e}")
except NotImplementedError: except NotImplementedError:
if tarpath.lower().endswith(".zip") or format == "zip": if tarpath.lower().endswith(".zip") or format == "zip":
_unpack_zip(tarpath, dest) _unpack_zip(tarpath, dest)
else: else:
raise raise
▲ Show 20 Lines • Show All 82 Lines • Show Last 20 Lines
View Options

swh/objstorage/tests/test_objstorage_in_memory.py

# Copyright (C) 2015-2021 The Software Heritage developers # Copyright (C) 2015-2022 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution # See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version # License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information # See top-level LICENSE file for more information
import os import os
import shutil import shutil
import stat import stat
from subprocess import run from subprocess import run
import tarfile import tarfile
import zipfile import zipfile
import magic import magic
from . import utils from . import utils
MIMETYPE_TO_ARCHIVE_FORMAT = {
"application/x-compress": "tar.Z|x",
"application/x-tar": "tar",
"application/x-bzip2": "bztar",
"application/gzip": "gztar",
"application/x-lzip": "tar.lz",
"application/zip": "zip",
"application/java-archive": "jar",
"application/zstd": "tar.zst",
"application/x-zstd": "tar.zst",
}
def _unpack_tar(tarpath: str, extract_dir: str) -> str: def _unpack_tar(tarpath: str, extract_dir: str) -> str:
"""Unpack tarballs unsupported by the standard python library. Examples """Unpack tarballs unsupported by the standard python library. Examples
include tar.Z, tar.lz, tar.x, etc.... include tar.Z, tar.lz, tar.x, etc....
As this implementation relies on the `tar` command, this function supports As this implementation relies on the `tar` command, this function supports
the same compression the tar command supports. the same compression the tar command supports.
▲ Show 20 Lines • Show All 87 Lines • ▼ Show 20 Linesdef register_new_archive_formats():
"""Register new archive formats to uncompress""" """Register new archive formats to uncompress"""
registered_formats = [f[0] for f in shutil.get_unpack_formats()] registered_formats = [f[0] for f in shutil.get_unpack_formats()]
for name, extensions, function in ADDITIONAL_ARCHIVE_FORMATS: for name, extensions, function in ADDITIONAL_ARCHIVE_FORMATS:
if name in registered_formats: if name in registered_formats:
continue continue
shutil.register_unpack_format(name, extensions, function) shutil.register_unpack_format(name, extensions, function)
_mime_to_archive_format = {
"application/x-compress": "tar.Z|x",
"application/x-tar": "tar",
"application/x-bzip2": "bztar",
"application/gzip": "gztar",
"application/x-lzip": "tar.lz",
"application/zip": "zip",
"application/java-archive": "jar",
"application/zstd": "tar.zst",
"application/x-zstd": "tar.zst",
}
def uncompress(tarpath: str, dest: str): def uncompress(tarpath: str, dest: str):
"""Uncompress tarpath to dest folder if tarball is supported. """Uncompress tarpath to dest folder if tarball is supported.
Note that this fixes permissions after successfully Note that this fixes permissions after successfully
uncompressing the archive. uncompressing the archive.
Args: Args:
tarpath: path to tarball to uncompress tarpath: path to tarball to uncompress
Show All 11 Lines try:
for format_, exts, _ in shutil.get_unpack_formats(): for format_, exts, _ in shutil.get_unpack_formats():
if any([tarpath.lower().endswith(ext.lower()) for ext in exts]): if any([tarpath.lower().endswith(ext.lower()) for ext in exts]):
format = format_ format = format_
break break
# try to get archive format from file mimetype # try to get archive format from file mimetype
if format is None: if format is None:
m = magic.Magic(mime=True) m = magic.Magic(mime=True)
mime = m.from_file(tarpath) mime = m.from_file(tarpath)
format = _mime_to_archive_format.get(mime) format = MIMETYPE_TO_ARCHIVE_FORMAT.get(mime)
shutil.unpack_archive(tarpath, extract_dir=dest, format=format) shutil.unpack_archive(tarpath, extract_dir=dest, format=format)
except shutil.ReadError as e: except shutil.ReadError as e:
raise ValueError(f"Problem during unpacking {tarpath}. Reason: {e}") raise ValueError(f"Problem during unpacking {tarpath}. Reason: {e}")
except NotImplementedError: except NotImplementedError:
if tarpath.lower().endswith(".zip") or format == "zip": if tarpath.lower().endswith(".zip") or format == "zip":
_unpack_zip(tarpath, dest) _unpack_zip(tarpath, dest)
else: else:
raise raise
▲ Show 20 Lines • Show All 82 Lines • Show Last 20 Lines
About · Developer information · Legal