diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 380c658..69b3349 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,46 +1,40 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v2.4.0 hooks: - id: trailing-whitespace - id: flake8 - id: check-json - id: check-yaml - repo: https://github.com/codespell-project/codespell rev: v1.16.0 hooks: - id: codespell - repo: local hooks: - id: mypy name: mypy entry: mypy args: [swh] pass_filenames: false language: system types: [python] +- repo: https://github.com/python/black + rev: 19.10b0 + hooks: + - id: black + # unfortunately, we are far from being able to enable this... # - repo: https://github.com/PyCQA/pydocstyle.git # rev: 4.0.0 # hooks: # - id: pydocstyle # name: pydocstyle # description: pydocstyle is a static analysis tool for checking compliance with Python docstring conventions. # entry: pydocstyle --convention=google # language: python # types: [python] -# black requires py3.6+ -#- repo: https://github.com/python/black -# rev: 19.3b0 -# hooks: -# - id: black -# language_version: python3 -#- repo: https://github.com/asottile/blacken-docs -# rev: v1.0.0-1 -# hooks: -# - id: blacken-docs -# additional_dependencies: [black==19.3b0] diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..8d79b7e --- /dev/null +++ b/setup.cfg @@ -0,0 +1,6 @@ +[flake8] +# E203: whitespaces before ':' +# E231: missing whitespace after ',' +# W503: line break before binary operator +ignore = E203,E231,W503 +max-line-length = 88 diff --git a/setup.py b/setup.py index 0f78fc4..3442131 100755 --- a/setup.py +++ b/setup.py @@ -1,69 +1,69 @@ #!/usr/bin/env python3 # Copyright (C) 2015-2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from setuptools import setup, find_packages from os import path from io import open here = path.abspath(path.dirname(__file__)) # Get the long description from the README file -with open(path.join(here, 'README.md'), encoding='utf-8') as f: +with open(path.join(here, "README.md"), encoding="utf-8") as f: long_description = f.read() def parse_requirements(name=None): if name: - reqf = 'requirements-%s.txt' % name + reqf = "requirements-%s.txt" % name else: - reqf = 'requirements.txt' + reqf = "requirements.txt" requirements = [] if not path.exists(reqf): return requirements with open(reqf) as f: for line in f.readlines(): line = line.strip() - if not line or line.startswith('#'): + if not line or line.startswith("#"): continue requirements.append(line) return requirements setup( - name='swh.loader.git', - description='Software Heritage git loader', + name="swh.loader.git", + description="Software Heritage git loader", long_description=long_description, - long_description_content_type='text/markdown', - author='Software Heritage developers', - author_email='swh-devel@inria.fr', - url='https://forge.softwareheritage.org/diffusion/DLDG/', + long_description_content_type="text/markdown", + author="Software Heritage developers", + author_email="swh-devel@inria.fr", + url="https://forge.softwareheritage.org/diffusion/DLDG/", packages=find_packages(), scripts=[], - install_requires=parse_requirements() + parse_requirements('swh'), - setup_requires=['vcversioner'], - extras_require={'testing': parse_requirements('test')}, + install_requires=parse_requirements() + parse_requirements("swh"), + setup_requires=["vcversioner"], + extras_require={"testing": parse_requirements("test")}, vcversioner={}, include_package_data=True, - entry_points=''' + entry_points=""" [swh.workers] loader.git=swh.loader.git:register - ''', + """, classifiers=[ "Programming Language :: Python :: 3", "Intended Audience :: Developers", "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", "Operating System :: OS Independent", "Development Status :: 5 - Production/Stable", ], project_urls={ - 'Bug Reports': 'https://forge.softwareheritage.org/maniphest', - 'Funding': 'https://www.softwareheritage.org/donate', - 'Source': 'https://forge.softwareheritage.org/source/swh-loader-git', + "Bug Reports": "https://forge.softwareheritage.org/maniphest", + "Funding": "https://www.softwareheritage.org/donate", + "Source": "https://forge.softwareheritage.org/source/swh-loader-git", }, ) diff --git a/swh/loader/git/__init__.py b/swh/loader/git/__init__.py index 6203a37..2a444a1 100644 --- a/swh/loader/git/__init__.py +++ b/swh/loader/git/__init__.py @@ -1,14 +1,15 @@ # Copyright (C) 2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from typing import Any, Dict def register() -> Dict[str, Any]: from swh.loader.git.loader import GitLoader + return { - 'task_modules': ['%s.tasks' % __name__], - 'loader': GitLoader, + "task_modules": ["%s.tasks" % __name__], + "loader": GitLoader, } diff --git a/swh/loader/git/converters.py b/swh/loader/git/converters.py index 6f611b2..0cfb46e 100644 --- a/swh/loader/git/converters.py +++ b/swh/loader/git/converters.py @@ -1,198 +1,189 @@ # Copyright (C) 2015-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information """Convert dulwich objects to dictionaries suitable for swh.storage""" from typing import Any, Dict, Optional -from swh.model.hashutil import ( - DEFAULT_ALGORITHMS, hash_to_bytes, MultiHash -) +from swh.model.hashutil import DEFAULT_ALGORITHMS, hash_to_bytes, MultiHash from swh.model.model import ( - BaseContent, Content, Directory, DirectoryEntry, - ObjectType, Person, Release, Revision, RevisionType, - SkippedContent, TargetType, Timestamp, TimestampWithTimezone, + BaseContent, + Content, + Directory, + DirectoryEntry, + ObjectType, + Person, + Release, + Revision, + RevisionType, + SkippedContent, + TargetType, + Timestamp, + TimestampWithTimezone, ) -HASH_ALGORITHMS = DEFAULT_ALGORITHMS - {'sha1_git'} +HASH_ALGORITHMS = DEFAULT_ALGORITHMS - {"sha1_git"} def dulwich_blob_to_content_id(blob) -> Dict[str, Any]: """Convert a dulwich blob to a Software Heritage content id""" - if blob.type_name != b'blob': - raise ValueError('Argument is not a blob.') + if blob.type_name != b"blob": + raise ValueError("Argument is not a blob.") size = blob.raw_length() data = blob.as_raw_string() hashes = MultiHash.from_data(data, HASH_ALGORITHMS).digest() - hashes['sha1_git'] = blob.sha().digest() - hashes['length'] = size + hashes["sha1_git"] = blob.sha().digest() + hashes["length"] = size return hashes def dulwich_blob_to_content(blob, max_content_size=None) -> BaseContent: """Convert a dulwich blob to a Software Heritage content """ - if blob.type_name != b'blob': - raise ValueError('Argument is not a blob.') + if blob.type_name != b"blob": + raise ValueError("Argument is not a blob.") hashes = dulwich_blob_to_content_id(blob) - if max_content_size is not None and hashes['length'] >= max_content_size: - return SkippedContent( - status='absent', - reason='Content too large', - **hashes, - ) + if max_content_size is not None and hashes["length"] >= max_content_size: + return SkippedContent(status="absent", reason="Content too large", **hashes,) else: - return Content( - data=blob.as_raw_string(), - status='visible', - **hashes, - ) + return Content(data=blob.as_raw_string(), status="visible", **hashes,) def dulwich_tree_to_directory(tree, log=None) -> Directory: """Format a tree as a directory""" - if tree.type_name != b'tree': - raise ValueError('Argument is not a tree.') + if tree.type_name != b"tree": + raise ValueError("Argument is not a tree.") entries = [] entry_mode_map = { - 0o040000: 'dir', - 0o160000: 'rev', - 0o100644: 'file', - 0o100755: 'file', - 0o120000: 'file', + 0o040000: "dir", + 0o160000: "rev", + 0o100644: "file", + 0o100755: "file", + 0o120000: "file", } for entry in tree.iteritems(): - entries.append(DirectoryEntry( - type=entry_mode_map.get(entry.mode, 'file'), - perms=entry.mode, - name=entry.path, - target=hash_to_bytes(entry.sha.decode('ascii')), - )) - - return Directory( - id=tree.sha().digest(), - entries=entries, - ) + entries.append( + DirectoryEntry( + type=entry_mode_map.get(entry.mode, "file"), + perms=entry.mode, + name=entry.path, + target=hash_to_bytes(entry.sha.decode("ascii")), + ) + ) + + return Directory(id=tree.sha().digest(), entries=entries,) def parse_author(name_email: bytes) -> Person: """Parse an author line""" return Person.from_fullname(name_email) def dulwich_tsinfo_to_timestamp( - timestamp, timezone, timezone_neg_utc) -> TimestampWithTimezone: + timestamp, timezone, timezone_neg_utc +) -> TimestampWithTimezone: """Convert the dulwich timestamp information to a structure compatible with Software Heritage""" return TimestampWithTimezone( - timestamp=Timestamp( - seconds=int(timestamp), - microseconds=0, - ), + timestamp=Timestamp(seconds=int(timestamp), microseconds=0,), offset=timezone // 60, negative_utc=timezone_neg_utc if timezone == 0 else False, ) def dulwich_commit_to_revision(commit, log=None) -> Revision: - if commit.type_name != b'commit': - raise ValueError('Argument is not a commit.') + if commit.type_name != b"commit": + raise ValueError("Argument is not a commit.") git_metadata = [] if commit.encoding is not None: - git_metadata.append(['encoding', commit.encoding]) + git_metadata.append(["encoding", commit.encoding]) if commit.mergetag: for mergetag in commit.mergetag: raw_string = mergetag.as_raw_string() - assert raw_string.endswith(b'\n') - git_metadata.append(['mergetag', raw_string[:-1]]) + assert raw_string.endswith(b"\n") + git_metadata.append(["mergetag", raw_string[:-1]]) if commit.extra: - git_metadata.extend([k.decode('utf-8'), v] for k, v in commit.extra) + git_metadata.extend([k.decode("utf-8"), v] for k, v in commit.extra) if commit.gpgsig: - git_metadata.append(['gpgsig', commit.gpgsig]) + git_metadata.append(["gpgsig", commit.gpgsig]) if git_metadata: metadata: Optional[Dict[str, Any]] = { - 'extra_headers': git_metadata, + "extra_headers": git_metadata, } else: metadata = None return Revision( id=commit.sha().digest(), author=parse_author(commit.author), date=dulwich_tsinfo_to_timestamp( - commit.author_time, - commit.author_timezone, - commit._author_timezone_neg_utc, + commit.author_time, commit.author_timezone, commit._author_timezone_neg_utc, ), committer=parse_author(commit.committer), committer_date=dulwich_tsinfo_to_timestamp( - commit.commit_time, - commit.commit_timezone, - commit._commit_timezone_neg_utc, + commit.commit_time, commit.commit_timezone, commit._commit_timezone_neg_utc, ), type=RevisionType.GIT, directory=bytes.fromhex(commit.tree.decode()), message=commit.message, metadata=metadata, synthetic=False, parents=[bytes.fromhex(p.decode()) for p in commit.parents], ) DULWICH_TARGET_TYPES = { - b'blob': TargetType.CONTENT, - b'tree': TargetType.DIRECTORY, - b'commit': TargetType.REVISION, - b'tag': TargetType.RELEASE, + b"blob": TargetType.CONTENT, + b"tree": TargetType.DIRECTORY, + b"commit": TargetType.REVISION, + b"tag": TargetType.RELEASE, } DULWICH_OBJECT_TYPES = { - b'blob': ObjectType.CONTENT, - b'tree': ObjectType.DIRECTORY, - b'commit': ObjectType.REVISION, - b'tag': ObjectType.RELEASE, + b"blob": ObjectType.CONTENT, + b"tree": ObjectType.DIRECTORY, + b"commit": ObjectType.REVISION, + b"tag": ObjectType.RELEASE, } def dulwich_tag_to_release(tag, log=None) -> Release: - if tag.type_name != b'tag': - raise ValueError('Argument is not a tag.') + if tag.type_name != b"tag": + raise ValueError("Argument is not a tag.") target_type, target = tag.object if tag.tagger: author: Optional[Person] = parse_author(tag.tagger) if not tag.tag_time: date = None else: date = dulwich_tsinfo_to_timestamp( - tag.tag_time, - tag.tag_timezone, - tag._tag_timezone_neg_utc, + tag.tag_time, tag.tag_timezone, tag._tag_timezone_neg_utc, ) else: author = date = None return Release( id=tag.sha().digest(), author=author, date=date, name=tag.name, target=bytes.fromhex(target.decode()), target_type=DULWICH_OBJECT_TYPES[target_type.type_name], message=tag._message, metadata=None, synthetic=False, ) diff --git a/swh/loader/git/from_disk.py b/swh/loader/git/from_disk.py index 7571957..627c6c1 100644 --- a/swh/loader/git/from_disk.py +++ b/swh/loader/git/from_disk.py @@ -1,371 +1,384 @@ # Copyright (C) 2015-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import datetime import dulwich.repo import os import shutil from typing import Dict, Optional from dulwich.errors import ObjectFormatException, EmptyFileException from collections import defaultdict from swh.model import hashutil -from swh.model.model import ( - Origin, Snapshot, SnapshotBranch, TargetType) +from swh.model.model import Origin, Snapshot, SnapshotBranch, TargetType from swh.loader.core.loader import DVCSLoader from . import converters, utils class GitLoaderFromDisk(DVCSLoader): """Load a git repository from a directory. """ - CONFIG_BASE_FILENAME = 'loader/git-disk' + CONFIG_BASE_FILENAME = "loader/git-disk" - visit_type = 'git' + visit_type = "git" def __init__(self, url, visit_date=None, directory=None, config=None): - super().__init__(logging_class='swh.loader.git.Loader', config=config) + super().__init__(logging_class="swh.loader.git.Loader", config=config) self.origin_url = url self.visit_date = visit_date self.directory = directory def prepare_origin_visit(self, *args, **kwargs): self.origin = Origin(url=self.origin_url) def prepare(self, *args, **kwargs): self.repo = dulwich.repo.Repo(self.directory) def iter_objects(self): object_store = self.repo.object_store for pack in object_store.packs: objs = list(pack.index.iterentries()) objs.sort(key=lambda x: x[1]) for sha, offset, crc32 in objs: yield hashutil.hash_to_bytehex(sha) yield from object_store._iter_loose_objects() yield from object_store._iter_alternate_objects() def _check(self, obj): """Check the object's repository representation. If any errors in check exists, an ObjectFormatException is raised. Args: obj (object): Dulwich object read from the repository. """ obj.check() from dulwich.objects import Commit, Tag + try: # For additional checks on dulwich objects with date # for now, only checks on *time if isinstance(obj, Commit): commit_time = obj._commit_time utils.check_date_time(commit_time) author_time = obj._author_time utils.check_date_time(author_time) elif isinstance(obj, Tag): tag_time = obj._tag_time utils.check_date_time(tag_time) except Exception as e: raise ObjectFormatException(e) def get_object(self, oid): """Given an object id, return the object if it is found and not malformed in some way. Args: oid (bytes): the object's identifier Returns: The object if found without malformation """ try: # some errors are raised when reading the object obj = self.repo[oid] # some we need to check ourselves self._check(obj) except KeyError: - _id = oid.decode('utf-8') - self.log.warn('object %s not found, skipping' % _id, - extra={ - 'swh_type': 'swh_loader_git_missing_object', - 'swh_object_id': _id, - 'origin_url': self.origin.url, - }) + _id = oid.decode("utf-8") + self.log.warn( + "object %s not found, skipping" % _id, + extra={ + "swh_type": "swh_loader_git_missing_object", + "swh_object_id": _id, + "origin_url": self.origin.url, + }, + ) return None except ObjectFormatException: - _id = oid.decode('utf-8') - self.log.warn('object %s malformed, skipping' % _id, - extra={ - 'swh_type': 'swh_loader_git_missing_object', - 'swh_object_id': _id, - 'origin_url': self.origin.url, - }) + _id = oid.decode("utf-8") + self.log.warn( + "object %s malformed, skipping" % _id, + extra={ + "swh_type": "swh_loader_git_missing_object", + "swh_object_id": _id, + "origin_url": self.origin.url, + }, + ) return None except EmptyFileException: - _id = oid.decode('utf-8') - self.log.warn('object %s corrupted (empty file), skipping' % _id, - extra={ - 'swh_type': 'swh_loader_git_missing_object', - 'swh_object_id': _id, - 'origin_url': self.origin.url, - }) + _id = oid.decode("utf-8") + self.log.warn( + "object %s corrupted (empty file), skipping" % _id, + extra={ + "swh_type": "swh_loader_git_missing_object", + "swh_object_id": _id, + "origin_url": self.origin.url, + }, + ) else: return obj def fetch_data(self): """Fetch the data from the data source""" previous_visit = self.storage.origin_visit_get_latest( - self.origin.url, require_snapshot=True) + self.origin.url, require_snapshot=True + ) if previous_visit: - self.previous_snapshot_id = previous_visit['snapshot'] + self.previous_snapshot_id = previous_visit["snapshot"] else: self.previous_snapshot_id = None type_to_ids = defaultdict(list) for oid in self.iter_objects(): obj = self.get_object(oid) if not obj: continue type_name = obj.type_name type_to_ids[type_name].append(oid) self.type_to_ids = type_to_ids def has_contents(self): """Checks whether we need to load contents""" - return bool(self.type_to_ids[b'blob']) + return bool(self.type_to_ids[b"blob"]) def get_content_ids(self): """Get the content identifiers from the git repository""" - for oid in self.type_to_ids[b'blob']: + for oid in self.type_to_ids[b"blob"]: yield converters.dulwich_blob_to_content_id(self.repo[oid]) def get_contents(self): """Get the contents that need to be loaded""" - missing_contents = set(self.storage.content_missing( - self.get_content_ids(), 'sha1_git')) + missing_contents = set( + self.storage.content_missing(self.get_content_ids(), "sha1_git") + ) for oid in missing_contents: yield converters.dulwich_blob_to_content( - self.repo[hashutil.hash_to_bytehex(oid)]) + self.repo[hashutil.hash_to_bytehex(oid)] + ) def has_directories(self): """Checks whether we need to load directories""" - return bool(self.type_to_ids[b'tree']) + return bool(self.type_to_ids[b"tree"]) def get_directory_ids(self): """Get the directory identifiers from the git repository""" - return (hashutil.hash_to_bytes(id.decode()) - for id in self.type_to_ids[b'tree']) + return (hashutil.hash_to_bytes(id.decode()) for id in self.type_to_ids[b"tree"]) def get_directories(self): """Get the directories that need to be loaded""" - missing_dirs = set(self.storage.directory_missing( - sorted(self.get_directory_ids()))) + missing_dirs = set( + self.storage.directory_missing(sorted(self.get_directory_ids())) + ) for oid in missing_dirs: yield converters.dulwich_tree_to_directory( - self.repo[hashutil.hash_to_bytehex(oid)], log=self.log) + self.repo[hashutil.hash_to_bytehex(oid)], log=self.log + ) def has_revisions(self): """Checks whether we need to load revisions""" - return bool(self.type_to_ids[b'commit']) + return bool(self.type_to_ids[b"commit"]) def get_revision_ids(self): """Get the revision identifiers from the git repository""" - return (hashutil.hash_to_bytes(id.decode()) - for id in self.type_to_ids[b'commit']) + return ( + hashutil.hash_to_bytes(id.decode()) for id in self.type_to_ids[b"commit"] + ) def get_revisions(self): """Get the revisions that need to be loaded""" - missing_revs = set(self.storage.revision_missing( - sorted(self.get_revision_ids()))) + missing_revs = set( + self.storage.revision_missing(sorted(self.get_revision_ids())) + ) for oid in missing_revs: yield converters.dulwich_commit_to_revision( - self.repo[hashutil.hash_to_bytehex(oid)], log=self.log) + self.repo[hashutil.hash_to_bytehex(oid)], log=self.log + ) def has_releases(self): """Checks whether we need to load releases""" - return bool(self.type_to_ids[b'tag']) + return bool(self.type_to_ids[b"tag"]) def get_release_ids(self): """Get the release identifiers from the git repository""" - return (hashutil.hash_to_bytes(id.decode()) - for id in self.type_to_ids[b'tag']) + return (hashutil.hash_to_bytes(id.decode()) for id in self.type_to_ids[b"tag"]) def get_releases(self): """Get the releases that need to be loaded""" - missing_rels = set(self.storage.release_missing( - sorted(self.get_release_ids()))) + missing_rels = set(self.storage.release_missing(sorted(self.get_release_ids()))) for oid in missing_rels: yield converters.dulwich_tag_to_release( - self.repo[hashutil.hash_to_bytehex(oid)], log=self.log) + self.repo[hashutil.hash_to_bytehex(oid)], log=self.log + ) def get_snapshot(self): """Turn the list of branches into a snapshot to load""" branches: Dict[bytes, Optional[SnapshotBranch]] = {} for ref, target in self.repo.refs.as_dict().items(): obj = self.get_object(target) if obj: target_type = converters.DULWICH_TARGET_TYPES[obj.type_name] branches[ref] = SnapshotBranch( - target=hashutil.bytehex_to_hash(target), - target_type=target_type, + target=hashutil.bytehex_to_hash(target), target_type=target_type, ) else: branches[ref] = None for ref, target in self.repo.refs.get_symrefs().items(): - branches[ref] = SnapshotBranch( - target=target, - target_type=TargetType.ALIAS, - ) + branches[ref] = SnapshotBranch(target=target, target_type=TargetType.ALIAS,) self.snapshot = Snapshot(branches=branches) return self.snapshot def get_fetch_history_result(self): """Return the data to store in fetch_history for the current loader""" return { - 'contents': len(self.type_to_ids[b'blob']), - 'directories': len(self.type_to_ids[b'tree']), - 'revisions': len(self.type_to_ids[b'commit']), - 'releases': len(self.type_to_ids[b'tag']), + "contents": len(self.type_to_ids[b"blob"]), + "directories": len(self.type_to_ids[b"tree"]), + "revisions": len(self.type_to_ids[b"commit"]), + "releases": len(self.type_to_ids[b"tag"]), } def save_data(self): """We already have the data locally, no need to save it""" pass def load_status(self): """The load was eventful if the current occurrences are different to the ones we retrieved at the beginning of the run""" eventful = False if self.previous_snapshot_id: eventful = self.snapshot.id != self.previous_snapshot_id else: eventful = bool(self.snapshot.branches) - return {'status': ('eventful' if eventful else 'uneventful')} + return {"status": ("eventful" if eventful else "uneventful")} class GitLoaderFromArchive(GitLoaderFromDisk): """Load a git repository from an archive. This loader ingests a git repository compressed into an archive. The supported archive formats are ``.zip`` and ``.tar.gz``. From an input tarball named ``my-git-repo.zip``, the following layout is expected in it:: my-git-repo/ ├── .git │ ├── branches │ ├── COMMIT_EDITMSG │ ├── config │ ├── description │ ├── HEAD ... Nevertheless, the loader is able to ingest tarballs with the following layouts too:: . ├── .git │ ├── branches │ ├── COMMIT_EDITMSG │ ├── config │ ├── description │ ├── HEAD ... or:: other-repo-name/ ├── .git │ ├── branches │ ├── COMMIT_EDITMSG │ ├── config │ ├── description │ ├── HEAD ... """ + def __init__(self, *args, archive_path, **kwargs): super().__init__(*args, **kwargs) self.temp_dir = self.repo_path = None self.archive_path = archive_path def project_name_from_archive(self, archive_path): """Compute the project name from the archive's path. """ archive_name = os.path.basename(archive_path) - for ext in ('.zip', '.tar.gz', '.tgz'): + for ext in (".zip", ".tar.gz", ".tgz"): if archive_name.lower().endswith(ext): - archive_name = archive_name[:-len(ext)] + archive_name = archive_name[: -len(ext)] break return archive_name def prepare(self, *args, **kwargs): """1. Uncompress the archive in temporary location. 2. Prepare as the GitLoaderFromDisk does 3. Load as GitLoaderFromDisk does """ project_name = self.project_name_from_archive(self.archive_path) self.temp_dir, self.repo_path = utils.init_git_repo_from_archive( - project_name, self.archive_path) - - self.log.info('Project %s - Uncompressing archive %s at %s', - self.origin_url, os.path.basename(self.archive_path), - self.repo_path) + project_name, self.archive_path + ) + + self.log.info( + "Project %s - Uncompressing archive %s at %s", + self.origin_url, + os.path.basename(self.archive_path), + self.repo_path, + ) self.directory = self.repo_path super().prepare(*args, **kwargs) def cleanup(self): """Cleanup the temporary location (if it exists). """ if self.temp_dir and os.path.exists(self.temp_dir): shutil.rmtree(self.temp_dir) - self.log.info('Project %s - Done injecting %s' % ( - self.origin_url, self.repo_path)) + self.log.info( + "Project %s - Done injecting %s" % (self.origin_url, self.repo_path) + ) -if __name__ == '__main__': +if __name__ == "__main__": import click import logging logging.basicConfig( - level=logging.DEBUG, - format='%(asctime)s %(process)d %(message)s' + level=logging.DEBUG, format="%(asctime)s %(process)d %(message)s" ) @click.command() - @click.option('--origin-url', help='origin url') - @click.option('--git-directory', help='Path to git repository to load') - @click.option('--visit-date', default=None, help='Visit date') + @click.option("--origin-url", help="origin url") + @click.option("--git-directory", help="Path to git repository to load") + @click.option("--visit-date", default=None, help="Visit date") def main(origin_url, git_directory, visit_date): if not visit_date: visit_date = datetime.datetime.now(tz=datetime.timezone.utc) return GitLoaderFromDisk().load(origin_url, git_directory, visit_date) main() diff --git a/swh/loader/git/loader.py b/swh/loader/git/loader.py index 2be7f71..e56a5b7 100644 --- a/swh/loader/git/loader.py +++ b/swh/loader/git/loader.py @@ -1,558 +1,566 @@ # Copyright (C) 2016-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from collections import defaultdict import datetime from io import BytesIO import logging import os import pickle import sys from typing import Any, Dict, Iterable, List, Optional, Union import dulwich.client from dulwich.object_store import ObjectStoreGraphWalker from dulwich.pack import PackData, PackInflater from swh.model import hashutil from swh.model.model import ( - BaseContent, Directory, Origin, Revision, - Release, Snapshot, SnapshotBranch, TargetType, Sha1Git) + BaseContent, + Directory, + Origin, + Revision, + Release, + Snapshot, + SnapshotBranch, + TargetType, + Sha1Git, +) from swh.loader.core.loader import DVCSLoader from swh.storage.algos.snapshot import snapshot_get_all_branches from . import converters class RepoRepresentation: """Repository representation for a Software Heritage origin.""" - def __init__(self, storage, base_snapshot=None, - ignore_history=False): + + def __init__(self, storage, base_snapshot=None, ignore_history=False): self.storage = storage self._parents_cache = {} self._type_cache: Dict[bytes, TargetType] = {} self.ignore_history = ignore_history if base_snapshot and not ignore_history: self.heads = set(self._cache_heads(base_snapshot)) else: self.heads = set() def _fill_parents_cache(self, commits): """When querying for a commit's parents, we fill the cache to a depth of 1000 commits.""" root_revs = self._encode_for_storage(commits) for rev, parents in self.storage.revision_shortlog(root_revs, 1000): rev_id = hashutil.hash_to_bytehex(rev) if rev_id not in self._parents_cache: self._parents_cache[rev_id] = [ hashutil.hash_to_bytehex(parent) for parent in parents ] for rev in commits: if rev not in self._parents_cache: self._parents_cache[rev] = [] def _cache_heads(self, base_snapshot): """Return all the known head commits for the given snapshot""" _git_types = list(converters.DULWICH_TARGET_TYPES.values()) if not base_snapshot: return [] snapshot_targets = set() for branch in base_snapshot.branches.values(): if branch and branch.target_type in _git_types: snapshot_targets.add(branch.target) decoded_targets = self._decode_from_storage(snapshot_targets) for id, objs in self.get_stored_objects(decoded_targets).items(): if not objs: - logging.warn('Missing head: %s' % hashutil.hash_to_hex(id)) + logging.warn("Missing head: %s" % hashutil.hash_to_hex(id)) return [] return decoded_targets def get_parents(self, commit): """Bogus method to prevent expensive recursion, at the expense of less efficient downloading""" return [] def get_heads(self): return self.heads @staticmethod def _encode_for_storage(objects): return [hashutil.bytehex_to_hash(object) for object in objects] @staticmethod def _decode_from_storage(objects): return set(hashutil.hash_to_bytehex(object) for object in objects) def graph_walker(self): return ObjectStoreGraphWalker(self.get_heads(), self.get_parents) @staticmethod def filter_unwanted_refs(refs): """Filter the unwanted references from refs""" ret = {} for ref, val in refs.items(): - if ref.endswith(b'^{}'): + if ref.endswith(b"^{}"): # Peeled refs make the git protocol explode continue - elif ref.startswith(b'refs/pull/') and ref.endswith(b'/merge'): + elif ref.startswith(b"refs/pull/") and ref.endswith(b"/merge"): # We filter-out auto-merged GitHub pull requests continue else: ret[ref] = val return ret def determine_wants(self, refs): """Filter the remote references to figure out which ones Software Heritage needs. """ if not refs: return [] # Find what objects Software Heritage has refs = self.find_remote_ref_types_in_swh(refs) # Cache the objects found in swh as existing heads for target in refs.values(): - if target['target_type'] is not None: - self.heads.add(target['target']) + if target["target_type"] is not None: + self.heads.add(target["target"]) ret = set() for target in self.filter_unwanted_refs(refs).values(): - if target['target_type'] is None: + if target["target_type"] is None: # The target doesn't exist in Software Heritage, let's retrieve # it. - ret.add(target['target']) + ret.add(target["target"]) return list(ret) def _get_stored_objects_batch( - self, query - ) -> Dict[bytes, List[Dict[str, Union[bytes, TargetType]]]]: - results = self.storage.object_find_by_sha1_git( - self._encode_for_storage(query) - ) + self, query + ) -> Dict[bytes, List[Dict[str, Union[bytes, TargetType]]]]: + results = self.storage.object_find_by_sha1_git(self._encode_for_storage(query)) ret: Dict[bytes, List[Dict[str, Union[bytes, TargetType]]]] = {} for (id, objects) in results.items(): assert id not in ret ret[id] = [ - { - 'sha1_git': obj['sha1_git'], - 'type': TargetType(obj['type']), - } + {"sha1_git": obj["sha1_git"], "type": TargetType(obj["type"]),} for obj in objects ] return ret def get_stored_objects( - self, objects - ) -> Dict[bytes, List[Dict[str, Union[bytes, TargetType]]]]: + self, objects + ) -> Dict[bytes, List[Dict[str, Union[bytes, TargetType]]]]: """Find which of these objects were stored in the archive. Do the request in packets to avoid a server timeout. """ if self.ignore_history: return {} packet_size = 1000 ret: Dict[bytes, List[Dict[str, Union[bytes, TargetType]]]] = {} query = [] for object in objects: query.append(object) if len(query) >= packet_size: ret.update(self._get_stored_objects_batch(query)) query = [] if query: ret.update(self._get_stored_objects_batch(query)) return ret - def find_remote_ref_types_in_swh( - self, remote_refs) -> Dict[bytes, Dict[str, Any]]: + def find_remote_ref_types_in_swh(self, remote_refs) -> Dict[bytes, Dict[str, Any]]: """Parse the remote refs information and list the objects that exist in Software Heritage. Returns: dict whose keys are branch names, and values are tuples `(target, target_type)`. """ all_objs = set(remote_refs.values()) - set(self._type_cache) type_by_id: Dict[bytes, TargetType] = {} for id, objs in self.get_stored_objects(all_objs).items(): id = hashutil.hash_to_bytehex(id) if objs: - type_ = objs[0]['type'] + type_ = objs[0]["type"] assert isinstance(type_, TargetType) type_by_id[id] = type_ self._type_cache.update(type_by_id) ret = {} for ref, id in remote_refs.items(): ret[ref] = { - 'target': id, - 'target_type': self._type_cache.get(id), + "target": id, + "target_type": self._type_cache.get(id), } return ret class GitLoader(DVCSLoader): """A bulk loader for a git repository""" - CONFIG_BASE_FILENAME = 'loader/git' + + CONFIG_BASE_FILENAME = "loader/git" ADDITIONAL_CONFIG = { - 'pack_size_bytes': ('int', 4 * 1024 * 1024 * 1024), + "pack_size_bytes": ("int", 4 * 1024 * 1024 * 1024), } - visit_type = 'git' + visit_type = "git" - def __init__(self, url, base_url=None, ignore_history=False, - repo_representation=RepoRepresentation, config=None): + def __init__( + self, + url, + base_url=None, + ignore_history=False, + repo_representation=RepoRepresentation, + config=None, + ): """Initialize the bulk updater. Args: repo_representation: swh's repository representation which is in charge of filtering between known and remote data. """ - super().__init__(logging_class='swh.loader.git.BulkLoader', - config=config) + super().__init__(logging_class="swh.loader.git.BulkLoader", config=config) self.origin_url = url self.base_url = base_url self.ignore_history = ignore_history self.repo_representation = repo_representation - def fetch_pack_from_origin(self, origin_url, - base_snapshot, do_activity): + def fetch_pack_from_origin(self, origin_url, base_snapshot, do_activity): """Fetch a pack from the origin""" pack_buffer = BytesIO() base_repo = self.repo_representation( storage=self.storage, base_snapshot=base_snapshot, ignore_history=self.ignore_history, ) - client, path = dulwich.client.get_transport_and_path(origin_url, - thin_packs=False) + client, path = dulwich.client.get_transport_and_path( + origin_url, thin_packs=False + ) - size_limit = self.config['pack_size_bytes'] + size_limit = self.config["pack_size_bytes"] def do_pack(data): cur_size = pack_buffer.tell() would_write = len(data) if cur_size + would_write > size_limit: - raise IOError('Pack file too big for repository %s, ' - 'limit is %d bytes, current size is %d, ' - 'would write %d' % - (origin_url, size_limit, cur_size, would_write)) + raise IOError( + "Pack file too big for repository %s, " + "limit is %d bytes, current size is %d, " + "would write %d" % (origin_url, size_limit, cur_size, would_write) + ) pack_buffer.write(data) - pack_result = client.fetch_pack(path, - base_repo.determine_wants, - base_repo.graph_walker(), - do_pack, - progress=do_activity) + pack_result = client.fetch_pack( + path, + base_repo.determine_wants, + base_repo.graph_walker(), + do_pack, + progress=do_activity, + ) remote_refs = pack_result.refs symbolic_refs = pack_result.symrefs if remote_refs: local_refs = base_repo.find_remote_ref_types_in_swh(remote_refs) else: local_refs = remote_refs = {} pack_buffer.flush() pack_size = pack_buffer.tell() pack_buffer.seek(0) return { - 'remote_refs': base_repo.filter_unwanted_refs(remote_refs), - 'local_refs': local_refs, - 'symbolic_refs': symbolic_refs, - 'pack_buffer': pack_buffer, - 'pack_size': pack_size, + "remote_refs": base_repo.filter_unwanted_refs(remote_refs), + "local_refs": local_refs, + "symbolic_refs": symbolic_refs, + "pack_buffer": pack_buffer, + "pack_size": pack_size, } def list_pack(self, pack_data, pack_size): id_to_type = {} type_to_ids = defaultdict(set) inflater = self.get_inflater() for obj in inflater: type, id = obj.type_name, obj.id id_to_type[id] = type type_to_ids[type].add(id) return id_to_type, type_to_ids def prepare_origin_visit(self, *args, **kwargs): self.visit_date = datetime.datetime.now(tz=datetime.timezone.utc) self.origin = Origin(url=self.origin_url) def get_full_snapshot(self, origin_url) -> Optional[Snapshot]: - visit = self.storage.origin_visit_get_latest( - origin_url, require_snapshot=True) - if visit and visit['snapshot']: - snapshot = snapshot_get_all_branches( - self.storage, visit['snapshot']) + visit = self.storage.origin_visit_get_latest(origin_url, require_snapshot=True) + if visit and visit["snapshot"]: + snapshot = snapshot_get_all_branches(self.storage, visit["snapshot"]) else: snapshot = None if snapshot is None: return None return Snapshot.from_dict(snapshot) def prepare(self, *args, **kwargs): base_origin_url = origin_url = self.origin.url prev_snapshot = None if not self.ignore_history: prev_snapshot = self.get_full_snapshot(origin_url) if self.base_url and not prev_snapshot: base_origin = Origin(url=self.base_url) base_origin = self.storage.origin_get(base_origin) if base_origin: - base_origin_url = base_origin['url'] + base_origin_url = base_origin["url"] prev_snapshot = self.get_full_snapshot(base_origin_url) self.base_snapshot = prev_snapshot self.base_origin_url = base_origin_url def fetch_data(self): def do_progress(msg): sys.stderr.buffer.write(msg) sys.stderr.flush() fetch_info = self.fetch_pack_from_origin( - self.origin.url, self.base_snapshot, - do_progress) + self.origin.url, self.base_snapshot, do_progress + ) - self.pack_buffer = fetch_info['pack_buffer'] - self.pack_size = fetch_info['pack_size'] + self.pack_buffer = fetch_info["pack_buffer"] + self.pack_size = fetch_info["pack_size"] - self.remote_refs = fetch_info['remote_refs'] - self.local_refs = fetch_info['local_refs'] - self.symbolic_refs = fetch_info['symbolic_refs'] + self.remote_refs = fetch_info["remote_refs"] + self.local_refs = fetch_info["local_refs"] + self.symbolic_refs = fetch_info["symbolic_refs"] origin_url = self.origin.url - self.log.info('Listed %d refs for repo %s' % ( - len(self.remote_refs), origin_url), extra={ - 'swh_type': 'git_repo_list_refs', - 'swh_repo': origin_url, - 'swh_num_refs': len(self.remote_refs), - }) + self.log.info( + "Listed %d refs for repo %s" % (len(self.remote_refs), origin_url), + extra={ + "swh_type": "git_repo_list_refs", + "swh_repo": origin_url, + "swh_num_refs": len(self.remote_refs), + }, + ) # We want to load the repository, walk all the objects - id_to_type, type_to_ids = self.list_pack(self.pack_buffer, - self.pack_size) + id_to_type, type_to_ids = self.list_pack(self.pack_buffer, self.pack_size) self.id_to_type = id_to_type self.type_to_ids = type_to_ids def save_data(self): """Store a pack for archival""" write_size = 8192 pack_dir = self.get_save_data_path() pack_name = "%s.pack" % self.visit_date.isoformat() refs_name = "%s.refs" % self.visit_date.isoformat() - with open(os.path.join(pack_dir, pack_name), 'xb') as f: + with open(os.path.join(pack_dir, pack_name), "xb") as f: self.pack_buffer.seek(0) while True: r = self.pack_buffer.read(write_size) if not r: break f.write(r) self.pack_buffer.seek(0) - with open(os.path.join(pack_dir, refs_name), 'xb') as f: + with open(os.path.join(pack_dir, refs_name), "xb") as f: pickle.dump(self.remote_refs, f) def get_inflater(self): """Reset the pack buffer and get an object inflater from it""" self.pack_buffer.seek(0) return PackInflater.for_pack_data( - PackData.from_file(self.pack_buffer, self.pack_size)) + PackData.from_file(self.pack_buffer, self.pack_size) + ) def has_contents(self): - return bool(self.type_to_ids[b'blob']) + return bool(self.type_to_ids[b"blob"]) def get_content_ids(self) -> Iterable[Dict[str, Any]]: """Get the content identifiers from the git repository""" for raw_obj in self.get_inflater(): - if raw_obj.type_name != b'blob': + if raw_obj.type_name != b"blob": continue yield converters.dulwich_blob_to_content_id(raw_obj) def get_contents(self) -> Iterable[BaseContent]: """Format the blobs from the git repository as swh contents""" - missing_contents = set(self.storage.content_missing( - self.get_content_ids(), 'sha1_git')) + missing_contents = set( + self.storage.content_missing(self.get_content_ids(), "sha1_git") + ) for raw_obj in self.get_inflater(): - if raw_obj.type_name != b'blob': + if raw_obj.type_name != b"blob": continue if raw_obj.sha().digest() not in missing_contents: continue yield converters.dulwich_blob_to_content( - raw_obj, max_content_size=self.max_content_size) + raw_obj, max_content_size=self.max_content_size + ) def has_directories(self) -> bool: - return bool(self.type_to_ids[b'tree']) + return bool(self.type_to_ids[b"tree"]) def get_directory_ids(self) -> Iterable[Sha1Git]: """Get the directory identifiers from the git repository""" - return (hashutil.hash_to_bytes(id.decode()) - for id in self.type_to_ids[b'tree']) + return (hashutil.hash_to_bytes(id.decode()) for id in self.type_to_ids[b"tree"]) def get_directories(self) -> Iterable[Directory]: """Format the trees as swh directories""" - missing_dirs = set(self.storage.directory_missing( - sorted(self.get_directory_ids()))) + missing_dirs = set( + self.storage.directory_missing(sorted(self.get_directory_ids())) + ) for raw_obj in self.get_inflater(): - if raw_obj.type_name != b'tree': + if raw_obj.type_name != b"tree": continue if raw_obj.sha().digest() not in missing_dirs: continue yield converters.dulwich_tree_to_directory(raw_obj, log=self.log) def has_revisions(self) -> bool: - return bool(self.type_to_ids[b'commit']) + return bool(self.type_to_ids[b"commit"]) def get_revision_ids(self) -> Iterable[Sha1Git]: """Get the revision identifiers from the git repository""" - return (hashutil.hash_to_bytes(id.decode()) - for id in self.type_to_ids[b'commit']) + return ( + hashutil.hash_to_bytes(id.decode()) for id in self.type_to_ids[b"commit"] + ) def get_revisions(self) -> Iterable[Revision]: """Format commits as swh revisions""" - missing_revs = set(self.storage.revision_missing( - sorted(self.get_revision_ids()))) + missing_revs = set( + self.storage.revision_missing(sorted(self.get_revision_ids())) + ) for raw_obj in self.get_inflater(): - if raw_obj.type_name != b'commit': + if raw_obj.type_name != b"commit": continue if raw_obj.sha().digest() not in missing_revs: continue yield converters.dulwich_commit_to_revision(raw_obj, log=self.log) def has_releases(self) -> bool: - return bool(self.type_to_ids[b'tag']) + return bool(self.type_to_ids[b"tag"]) def get_release_ids(self) -> Iterable[Sha1Git]: """Get the release identifiers from the git repository""" - return (hashutil.hash_to_bytes(id.decode()) - for id in self.type_to_ids[b'tag']) + return (hashutil.hash_to_bytes(id.decode()) for id in self.type_to_ids[b"tag"]) def get_releases(self) -> Iterable[Release]: """Retrieve all the release objects from the git repository""" - missing_rels = set(self.storage.release_missing( - sorted(self.get_release_ids()))) + missing_rels = set(self.storage.release_missing(sorted(self.get_release_ids()))) for raw_obj in self.get_inflater(): - if raw_obj.type_name != b'tag': + if raw_obj.type_name != b"tag": continue if raw_obj.sha().digest() not in missing_rels: continue yield converters.dulwich_tag_to_release(raw_obj, log=self.log) def get_snapshot(self) -> Snapshot: branches: Dict[bytes, Optional[SnapshotBranch]] = {} for ref in self.remote_refs: ret_ref = self.local_refs[ref].copy() - if not ret_ref['target_type']: - target_type = self.id_to_type[ret_ref['target']] - ret_ref['target_type'] = \ - converters.DULWICH_TARGET_TYPES[target_type] + if not ret_ref["target_type"]: + target_type = self.id_to_type[ret_ref["target"]] + ret_ref["target_type"] = converters.DULWICH_TARGET_TYPES[target_type] - ret_ref['target'] = hashutil.bytehex_to_hash(ret_ref['target']) + ret_ref["target"] = hashutil.bytehex_to_hash(ret_ref["target"]) branches[ref] = SnapshotBranch( - target_type=ret_ref['target_type'], - target=ret_ref['target'], + target_type=ret_ref["target_type"], target=ret_ref["target"], ) for ref, target in self.symbolic_refs.items(): - branches[ref] = SnapshotBranch( - target_type=TargetType.ALIAS, - target=target, - ) + branches[ref] = SnapshotBranch(target_type=TargetType.ALIAS, target=target,) self.snapshot = Snapshot(branches=branches) return self.snapshot def get_fetch_history_result(self) -> Dict[str, int]: return { - 'contents': len(self.type_to_ids[b'blob']), - 'directories': len(self.type_to_ids[b'tree']), - 'revisions': len(self.type_to_ids[b'commit']), - 'releases': len(self.type_to_ids[b'tag']), + "contents": len(self.type_to_ids[b"blob"]), + "directories": len(self.type_to_ids[b"tree"]), + "revisions": len(self.type_to_ids[b"commit"]), + "releases": len(self.type_to_ids[b"tag"]), } def load_status(self) -> Dict[str, Any]: """The load was eventful if the current snapshot is different to the one we retrieved at the beginning of the run""" eventful = False if self.base_snapshot: eventful = self.snapshot.id != self.base_snapshot.id else: eventful = bool(self.snapshot.branches) - return {'status': ('eventful' if eventful else 'uneventful')} + return {"status": ("eventful" if eventful else "uneventful")} -if __name__ == '__main__': +if __name__ == "__main__": import click logging.basicConfig( - level=logging.DEBUG, - format='%(asctime)s %(process)d %(message)s' + level=logging.DEBUG, format="%(asctime)s %(process)d %(message)s" ) @click.command() - @click.option('--origin-url', help='Origin url', required=True) - @click.option('--base-url', default=None, help='Optional Base url') - @click.option('--ignore-history/--no-ignore-history', - help='Ignore the repository history', default=False) + @click.option("--origin-url", help="Origin url", required=True) + @click.option("--base-url", default=None, help="Optional Base url") + @click.option( + "--ignore-history/--no-ignore-history", + help="Ignore the repository history", + default=False, + ) def main(origin_url, base_url, ignore_history): loader = GitLoader( - origin_url, - base_url=base_url, - ignore_history=ignore_history, + origin_url, base_url=base_url, ignore_history=ignore_history, ) return loader.load() main() diff --git a/swh/loader/git/tasks.py b/swh/loader/git/tasks.py index 1bbaffd..455f27f 100644 --- a/swh/loader/git/tasks.py +++ b/swh/loader/git/tasks.py @@ -1,53 +1,49 @@ # Copyright (C) 2015-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import dateutil.parser from typing import Any, Dict, Optional from celery import shared_task from swh.loader.git.from_disk import GitLoaderFromDisk, GitLoaderFromArchive from swh.loader.git.loader import GitLoader -@shared_task(name=__name__ + '.UpdateGitRepository') +@shared_task(name=__name__ + ".UpdateGitRepository") def load_git(*, url: str, base_url: Optional[str] = None) -> Dict[str, Any]: """Import a git repository from a remote location """ loader = GitLoader(url, base_url=base_url) return loader.load() -@shared_task(name=__name__ + '.LoadDiskGitRepository') -def load_git_from_dir( - *, url: str, directory: str, date: str) -> Dict[str, Any]: +@shared_task(name=__name__ + ".LoadDiskGitRepository") +def load_git_from_dir(*, url: str, directory: str, date: str) -> Dict[str, Any]: """Import a git repository from a local repository Import a git repository, cloned in `directory` from `origin_url` at `date`. """ visit_date = dateutil.parser.parse(date) - loader = GitLoaderFromDisk( - url, directory=directory, visit_date=visit_date) + loader = GitLoaderFromDisk(url, directory=directory, visit_date=visit_date) return loader.load() -@shared_task(name=__name__ + '.UncompressAndLoadDiskGitRepository') -def load_git_from_zip( - *, url: str, archive_path: str, date: str) -> Dict[str, Any]: +@shared_task(name=__name__ + ".UncompressAndLoadDiskGitRepository") +def load_git_from_zip(*, url: str, archive_path: str, date: str) -> Dict[str, Any]: """Import a git repository from a zip archive 1. Uncompress an archive repository in a local and temporary folder 2. Load it through the git disk loader 3. Clean up the temporary folder """ visit_date = dateutil.parser.parse(date) - loader = GitLoaderFromArchive( - url, archive_path=archive_path, visit_date=visit_date) + loader = GitLoaderFromArchive(url, archive_path=archive_path, visit_date=visit_date) return loader.load() diff --git a/swh/loader/git/tests/__init__.py b/swh/loader/git/tests/__init__.py index fc649d4..c30ab7c 100644 --- a/swh/loader/git/tests/__init__.py +++ b/swh/loader/git/tests/__init__.py @@ -1,31 +1,27 @@ # Copyright (C) 2018-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information TEST_LOADER_CONFIG = { - 'storage': { - 'cls': 'pipeline', - 'steps': [ + "storage": { + "cls": "pipeline", + "steps": [ + {"cls": "filter"}, { - 'cls': 'filter' - }, - { - 'cls': 'buffer', - 'min_batch_size': { - 'content': 10, - 'content_bytes': 100 * 1024 * 1024, - 'directory': 10, - 'revision': 10, - 'release': 10, + "cls": "buffer", + "min_batch_size": { + "content": 10, + "content_bytes": 100 * 1024 * 1024, + "directory": 10, + "revision": 10, + "release": 10, }, }, - { - 'cls': 'memory' - }, - ] + {"cls": "memory"}, + ], }, - 'max_content_size': 100 * 1024 * 1024, - 'pack_size_bytes': 4 * 1024 * 1024 * 1024, - 'save_data': False, + "max_content_size": 100 * 1024 * 1024, + "pack_size_bytes": 4 * 1024 * 1024 * 1024, + "save_data": False, } diff --git a/swh/loader/git/tests/conftest.py b/swh/loader/git/tests/conftest.py index 961ddd7..ca28974 100644 --- a/swh/loader/git/tests/conftest.py +++ b/swh/loader/git/tests/conftest.py @@ -1,15 +1,15 @@ # Copyright (C) 2018-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import pytest from swh.scheduler.tests.conftest import * # noqa -@pytest.fixture(scope='session') # type: ignore # expected redefinition +@pytest.fixture(scope="session") # type: ignore # expected redefinition def celery_includes(): return [ - 'swh.loader.git.tasks', + "swh.loader.git.tasks", ] diff --git a/swh/loader/git/tests/test_converters.py b/swh/loader/git/tests/test_converters.py index 147a1ac..8b71a80 100644 --- a/swh/loader/git/tests/test_converters.py +++ b/swh/loader/git/tests/test_converters.py @@ -1,325 +1,319 @@ # Copyright (C) 2015-2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import os import pytest import shutil import subprocess import tempfile import unittest import dulwich.repo from swh.model.hashutil import bytehex_to_hash, hash_to_bytes from swh.model.model import ( - Content, Person, Release, Revision, RevisionType, ObjectType, - Timestamp, TimestampWithTimezone, + Content, + Person, + Release, + Revision, + RevisionType, + ObjectType, + Timestamp, + TimestampWithTimezone, ) import swh.loader.git.converters as converters -TEST_DATA = os.path.join(os.path.dirname(__file__), 'data') +TEST_DATA = os.path.join(os.path.dirname(__file__), "data") class SWHObjectType: """Dulwich lookalike ObjectType class """ + def __init__(self, type_name): self.type_name = type_name class SWHTag: """Dulwich lookalike tag class """ - def __init__(self, name, type_name, target, target_type, tagger, tag_time, - tag_timezone, message): + + def __init__( + self, + name, + type_name, + target, + target_type, + tagger, + tag_time, + tag_timezone, + message, + ): self.name = name self.type_name = type_name self.object = SWHObjectType(target_type), target self.tagger = tagger self._message = message self.tag_time = tag_time self.tag_timezone = tag_timezone self._tag_timezone_neg_utc = False def sha(self): from hashlib import sha1 + return sha1() @pytest.mark.fs class TestConverters(unittest.TestCase): @classmethod def setUpClass(cls): super().setUpClass() cls.repo_path = tempfile.mkdtemp() cls.repo = dulwich.repo.Repo.init_bare(cls.repo_path) fast_export = os.path.join( - TEST_DATA, 'git-repos', 'example-submodule.fast-export.xz') + TEST_DATA, "git-repos", "example-submodule.fast-export.xz" + ) xz = subprocess.Popen( - ['xzcat'], - stdin=open(fast_export, 'rb'), - stdout=subprocess.PIPE, + ["xzcat"], stdin=open(fast_export, "rb"), stdout=subprocess.PIPE, ) git = subprocess.Popen( - ['git', 'fast-import', '--quiet'], - stdin=xz.stdout, - cwd=cls.repo_path, + ["git", "fast-import", "--quiet"], stdin=xz.stdout, cwd=cls.repo_path, ) # flush stdout of xz xz.stdout.close() git.communicate() @classmethod def tearDownClass(cls): super().tearDownClass() shutil.rmtree(cls.repo_path) def test_blob_to_content(self): - content_id = b'28c6f4023d65f74e3b59a2dea3c4277ed9ee07b0' + content_id = b"28c6f4023d65f74e3b59a2dea3c4277ed9ee07b0" content = converters.dulwich_blob_to_content(self.repo[content_id]) expected_content = Content( sha1_git=bytehex_to_hash(content_id), - sha1=hash_to_bytes('4850a3420a2262ff061cb296fb915430fa92301c'), - sha256=hash_to_bytes('fee7c8a485a10321ad94b64135073cb5' - '5f22cb9f57fa2417d2adfb09d310adef'), - blake2s256=hash_to_bytes('5d71873f42a137f6d89286e43677721e574' - '1fa05ce4cd5e3c7ea7c44d4c2d10b'), - data=(b'[submodule "example-dependency"]\n' - b'\tpath = example-dependency\n' - b'\turl = https://github.com/githubtraining/' - b'example-dependency.git\n'), + sha1=hash_to_bytes("4850a3420a2262ff061cb296fb915430fa92301c"), + sha256=hash_to_bytes( + "fee7c8a485a10321ad94b64135073cb5" "5f22cb9f57fa2417d2adfb09d310adef" + ), + blake2s256=hash_to_bytes( + "5d71873f42a137f6d89286e43677721e574" "1fa05ce4cd5e3c7ea7c44d4c2d10b" + ), + data=( + b'[submodule "example-dependency"]\n' + b"\tpath = example-dependency\n" + b"\turl = https://github.com/githubtraining/" + b"example-dependency.git\n" + ), length=124, - status='visible', + status="visible", ) self.assertEqual(content, expected_content) def test_convertion_wrong_input(self): class Something: - type_name = b'something-not-the-right-type' + type_name = b"something-not-the-right-type" m = { - 'blob': converters.dulwich_blob_to_content, - 'blob2': converters.dulwich_blob_to_content_id, - 'tree': converters.dulwich_tree_to_directory, - 'commit': converters.dulwich_tree_to_directory, - 'tag': converters.dulwich_tag_to_release, + "blob": converters.dulwich_blob_to_content, + "blob2": converters.dulwich_blob_to_content_id, + "tree": converters.dulwich_tree_to_directory, + "commit": converters.dulwich_tree_to_directory, + "tag": converters.dulwich_tag_to_release, } for _callable in m.values(): with self.assertRaises(ValueError): _callable(Something()) def test_commit_to_revision(self): - sha1 = b'9768d0b576dbaaecd80abedad6dfd0d72f1476da' + sha1 = b"9768d0b576dbaaecd80abedad6dfd0d72f1476da" revision = converters.dulwich_commit_to_revision(self.repo[sha1]) expected_revision = Revision( - id=hash_to_bytes('9768d0b576dbaaecd80abedad6dfd0d72f1476da'), - directory=b'\xf0i\\./\xa7\xce\x9dW@#\xc3A7a\xa4s\xe5\x00\xca', + id=hash_to_bytes("9768d0b576dbaaecd80abedad6dfd0d72f1476da"), + directory=b"\xf0i\\./\xa7\xce\x9dW@#\xc3A7a\xa4s\xe5\x00\xca", type=RevisionType.GIT, committer=Person( - name=b'Stefano Zacchiroli', - fullname=b'Stefano Zacchiroli ', - email=b'zack@upsilon.cc', + name=b"Stefano Zacchiroli", + fullname=b"Stefano Zacchiroli ", + email=b"zack@upsilon.cc", ), author=Person( - name=b'Stefano Zacchiroli', - fullname=b'Stefano Zacchiroli ', - email=b'zack@upsilon.cc', + name=b"Stefano Zacchiroli", + fullname=b"Stefano Zacchiroli ", + email=b"zack@upsilon.cc", ), committer_date=TimestampWithTimezone( - timestamp=Timestamp( - seconds=1443083765, - microseconds=0, - ), + timestamp=Timestamp(seconds=1443083765, microseconds=0,), negative_utc=False, offset=120, ), - message=b'add submodule dependency\n', + message=b"add submodule dependency\n", metadata=None, date=TimestampWithTimezone( - timestamp=Timestamp( - seconds=1443083765, - microseconds=0, - ), + timestamp=Timestamp(seconds=1443083765, microseconds=0,), negative_utc=False, offset=120, ), - parents=[ - b'\xc3\xc5\x88q23`\x9f[\xbb\xb2\xd9\xe7\xf3\xfbJf\x0f?r' - ], + parents=[b"\xc3\xc5\x88q23`\x9f[\xbb\xb2\xd9\xe7\xf3\xfbJf\x0f?r"], synthetic=False, ) self.assertEqual(revision, expected_revision) def test_author_line_to_author(self): # edge case out of the way with self.assertRaises(TypeError): converters.parse_author(None) tests = { - b'a ': Person( - name=b'a', - email=b'b@c.com', - fullname=b'a ', - ), - b'': Person( - name=None, - email=b'foo@bar.com', - fullname=b'', - ), - b'malformed ": Person( + name=b"a", email=b"b@c.com", fullname=b"a ", ), - b'trailing ': Person( - name=b'trailing', - email=b'sp@c.e', - fullname=b'trailing ', + b"": Person( + name=None, email=b"foo@bar.com", fullname=b"", ), - b'no': Person( - name=b'no', - email=b'sp@c.e', - fullname=b'no', + b"malformed ': Person( - name=None, - email=None, - fullname=b' <>', + b"trailing ": Person( + name=b"trailing", email=b"sp@c.e", fullname=b"trailing ", ), - b'something': Person( - name=b'something', - email=None, - fullname=b'something' - ) + b"no": Person(name=b"no", email=b"sp@c.e", fullname=b"no",), + b" <>": Person(name=None, email=None, fullname=b" <>",), + b"something": Person(name=b"something", email=None, fullname=b"something"), } for author in sorted(tests): parsed_author = tests[author] - self.assertEqual(parsed_author, - converters.parse_author(author)) + self.assertEqual(parsed_author, converters.parse_author(author)) def test_dulwich_tag_to_release_no_author_no_date(self): - target = b'641fb6e08ddb2e4fd096dcf18e80b894bf' - message = b'some release message' - tag = SWHTag(name=b'blah', - type_name=b'tag', - target=target, - target_type=b'commit', - message=message, - tagger=None, - tag_time=None, tag_timezone=None) + target = b"641fb6e08ddb2e4fd096dcf18e80b894bf" + message = b"some release message" + tag = SWHTag( + name=b"blah", + type_name=b"tag", + target=target, + target_type=b"commit", + message=message, + tagger=None, + tag_time=None, + tag_timezone=None, + ) # when actual_release = converters.dulwich_tag_to_release(tag) # then expected_release = Release( author=None, date=None, - id=b'\xda9\xa3\xee^kK\r2U\xbf\xef\x95`\x18\x90\xaf\xd8\x07\t', + id=b"\xda9\xa3\xee^kK\r2U\xbf\xef\x95`\x18\x90\xaf\xd8\x07\t", message=message, metadata=None, - name=b'blah', + name=b"blah", synthetic=False, target=hash_to_bytes(target.decode()), target_type=ObjectType.REVISION, ) self.assertEqual(actual_release, expected_release) def test_dulwich_tag_to_release_author_and_date(self): - tagger = b'hey dude ' - target = b'641fb6e08ddb2e4fd096dcf18e80b894bf' - message = b'some release message' + tagger = b"hey dude " + target = b"641fb6e08ddb2e4fd096dcf18e80b894bf" + message = b"some release message" import datetime - date = datetime.datetime( - 2007, 12, 5, tzinfo=datetime.timezone.utc - ).timestamp() - - tag = SWHTag(name=b'blah', - type_name=b'tag', - target=target, - target_type=b'commit', - message=message, - tagger=tagger, - tag_time=date, - tag_timezone=0) + + date = datetime.datetime(2007, 12, 5, tzinfo=datetime.timezone.utc).timestamp() + + tag = SWHTag( + name=b"blah", + type_name=b"tag", + target=target, + target_type=b"commit", + message=message, + tagger=tagger, + tag_time=date, + tag_timezone=0, + ) # when actual_release = converters.dulwich_tag_to_release(tag) # then expected_release = Release( author=Person( - email=b'hello@mail.org', - fullname=b'hey dude ', - name=b'hey dude' + email=b"hello@mail.org", + fullname=b"hey dude ", + name=b"hey dude", ), date=TimestampWithTimezone( negative_utc=False, offset=0, - timestamp=Timestamp( - seconds=1196812800, - microseconds=0, - ) + timestamp=Timestamp(seconds=1196812800, microseconds=0,), ), - id=b'\xda9\xa3\xee^kK\r2U\xbf\xef\x95`\x18\x90\xaf\xd8\x07\t', + id=b"\xda9\xa3\xee^kK\r2U\xbf\xef\x95`\x18\x90\xaf\xd8\x07\t", message=message, metadata=None, - name=b'blah', + name=b"blah", synthetic=False, target=hash_to_bytes(target.decode()), target_type=ObjectType.REVISION, ) self.assertEqual(actual_release, expected_release) def test_dulwich_tag_to_release_author_no_date(self): # to reproduce bug T815 (fixed) - tagger = b'hey dude ' - target = b'641fb6e08ddb2e4fd096dcf18e80b894bf' - message = b'some release message' - tag = SWHTag(name=b'blah', - type_name=b'tag', - target=target, - target_type=b'commit', - message=message, - tagger=tagger, - tag_time=None, tag_timezone=None) + tagger = b"hey dude " + target = b"641fb6e08ddb2e4fd096dcf18e80b894bf" + message = b"some release message" + tag = SWHTag( + name=b"blah", + type_name=b"tag", + target=target, + target_type=b"commit", + message=message, + tagger=tagger, + tag_time=None, + tag_timezone=None, + ) # when actual_release = converters.dulwich_tag_to_release(tag) # then expected_release = Release( author=Person( - email=b'hello@mail.org', - fullname=b'hey dude ', - name=b'hey dude' + email=b"hello@mail.org", + fullname=b"hey dude ", + name=b"hey dude", ), date=None, - id=b'\xda9\xa3\xee^kK\r2U\xbf\xef\x95`\x18\x90\xaf\xd8\x07\t', + id=b"\xda9\xa3\xee^kK\r2U\xbf\xef\x95`\x18\x90\xaf\xd8\x07\t", message=message, metadata=None, - name=b'blah', + name=b"blah", synthetic=False, target=hash_to_bytes(target.decode()), target_type=ObjectType.REVISION, ) self.assertEqual(actual_release, expected_release) diff --git a/swh/loader/git/tests/test_from_disk.py b/swh/loader/git/tests/test_from_disk.py index ae14b7d..00a7ecf 100644 --- a/swh/loader/git/tests/test_from_disk.py +++ b/swh/loader/git/tests/test_from_disk.py @@ -1,289 +1,298 @@ # Copyright (C) 2018-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import os.path import subprocess -from swh.loader.git.from_disk import GitLoaderFromDisk \ - as OrigGitLoaderFromDisk -from swh.loader.git.from_disk import GitLoaderFromArchive \ - as OrigGitLoaderFromArchive +from swh.loader.git.from_disk import GitLoaderFromDisk as OrigGitLoaderFromDisk +from swh.loader.git.from_disk import GitLoaderFromArchive as OrigGitLoaderFromArchive from swh.loader.core.tests import BaseLoaderTest from swh.model.hashutil import hash_to_bytes from . import TEST_LOADER_CONFIG class GitLoaderFromArchive(OrigGitLoaderFromArchive): def project_name_from_archive(self, archive_path): # We don't want the project name to be 'resources'. - return 'testrepo' + return "testrepo" def parse_config_file(self, *args, **kwargs): return TEST_LOADER_CONFIG CONTENT1 = { - '33ab5639bfd8e7b95eb1d8d0b87781d4ffea4d5d', # README v1 - '349c4ff7d21f1ec0eda26f3d9284c293e3425417', # README v2 - '799c11e348d39f1704022b8354502e2f81f3c037', # file1.txt - '4bdb40dfd6ec75cb730e678b5d7786e30170c5fb', # file2.txt + "33ab5639bfd8e7b95eb1d8d0b87781d4ffea4d5d", # README v1 + "349c4ff7d21f1ec0eda26f3d9284c293e3425417", # README v2 + "799c11e348d39f1704022b8354502e2f81f3c037", # file1.txt + "4bdb40dfd6ec75cb730e678b5d7786e30170c5fb", # file2.txt } -SNAPSHOT_ID = 'a23699280a82a043f8c0994cf1631b568f716f95' +SNAPSHOT_ID = "a23699280a82a043f8c0994cf1631b568f716f95" SNAPSHOT1 = { - 'id': SNAPSHOT_ID, - 'branches': { - 'HEAD': { - 'target': 'refs/heads/master', - 'target_type': 'alias', + "id": SNAPSHOT_ID, + "branches": { + "HEAD": {"target": "refs/heads/master", "target_type": "alias",}, + "refs/heads/master": { + "target": "2f01f5ca7e391a2f08905990277faf81e709a649", + "target_type": "revision", }, - 'refs/heads/master': { - 'target': '2f01f5ca7e391a2f08905990277faf81e709a649', - 'target_type': 'revision', + "refs/heads/branch1": { + "target": "b0a77609903f767a2fd3d769904ef9ef68468b87", + "target_type": "revision", }, - 'refs/heads/branch1': { - 'target': 'b0a77609903f767a2fd3d769904ef9ef68468b87', - 'target_type': 'revision', + "refs/heads/branch2": { + "target": "bd746cd1913721b269b395a56a97baf6755151c2", + "target_type": "revision", }, - 'refs/heads/branch2': { - 'target': 'bd746cd1913721b269b395a56a97baf6755151c2', - 'target_type': 'revision', + "refs/tags/branch2-after-delete": { + "target": "bd746cd1913721b269b395a56a97baf6755151c2", + "target_type": "revision", }, - 'refs/tags/branch2-after-delete': { - 'target': 'bd746cd1913721b269b395a56a97baf6755151c2', - 'target_type': 'revision', - }, - 'refs/tags/branch2-before-delete': { - 'target': '1135e94ccf73b5f9bd6ef07b3fa2c5cc60bba69b', - 'target_type': 'revision', + "refs/tags/branch2-before-delete": { + "target": "1135e94ccf73b5f9bd6ef07b3fa2c5cc60bba69b", + "target_type": "revision", }, }, } # directory hashes obtained with: # gco b6f40292c4e94a8f7e7b4aff50e6c7429ab98e2a # swh-hashtree --ignore '.git' --path . # gco 2f01f5ca7e391a2f08905990277faf81e709a649 # swh-hashtree --ignore '.git' --path . # gco bcdc5ebfde1a3cd6c96e0c2ea4eed19c13208777 # swh-hashtree --ignore '.git' --path . # gco 1135e94ccf73b5f9bd6ef07b3fa2c5cc60bba69b # swh-hashtree --ignore '.git' --path . # gco 79f65ac75f79dda6ff03d66e1242702ab67fb51c # swh-hashtree --ignore '.git' --path . # gco b0a77609903f767a2fd3d769904ef9ef68468b87 # swh-hashtree --ignore '.git' --path . # gco bd746cd1913721b269b395a56a97baf6755151c2 # swh-hashtree --ignore '.git' --path . REVISIONS1 = { - 'b6f40292c4e94a8f7e7b4aff50e6c7429ab98e2a': - '40dbdf55dfd4065422462cc74a949254aefa972e', - '2f01f5ca7e391a2f08905990277faf81e709a649': - 'e1d0d894835f91a0f887a4bc8b16f81feefdfbd5', - 'bcdc5ebfde1a3cd6c96e0c2ea4eed19c13208777': - 'b43724545b4759244bb54be053c690649161411c', - '1135e94ccf73b5f9bd6ef07b3fa2c5cc60bba69b': - 'fbf70528223d263661b5ad4b80f26caf3860eb8e', - '79f65ac75f79dda6ff03d66e1242702ab67fb51c': - '5df34ec74d6f69072d9a0a6677d8efbed9b12e60', - 'b0a77609903f767a2fd3d769904ef9ef68468b87': - '9ca0c7d6ffa3f9f0de59fd7912e08f11308a1338', - 'bd746cd1913721b269b395a56a97baf6755151c2': - 'e1d0d894835f91a0f887a4bc8b16f81feefdfbd5', + "b6f40292c4e94a8f7e7b4aff50e6c7429ab98e2a": ( + "40dbdf55dfd4065422462cc74a949254aefa972e" + ), + "2f01f5ca7e391a2f08905990277faf81e709a649": ( + "e1d0d894835f91a0f887a4bc8b16f81feefdfbd5" + ), + "bcdc5ebfde1a3cd6c96e0c2ea4eed19c13208777": ( + "b43724545b4759244bb54be053c690649161411c" + ), + "1135e94ccf73b5f9bd6ef07b3fa2c5cc60bba69b": ( + "fbf70528223d263661b5ad4b80f26caf3860eb8e" + ), + "79f65ac75f79dda6ff03d66e1242702ab67fb51c": ( + "5df34ec74d6f69072d9a0a6677d8efbed9b12e60" + ), + "b0a77609903f767a2fd3d769904ef9ef68468b87": ( + "9ca0c7d6ffa3f9f0de59fd7912e08f11308a1338" + ), + "bd746cd1913721b269b395a56a97baf6755151c2": ( + "e1d0d894835f91a0f887a4bc8b16f81feefdfbd5" + ), } class BaseGitLoaderFromDiskTest(BaseLoaderTest): - def setUp(self, archive_name, uncompress_archive, filename='testrepo'): - super().setUp(archive_name=archive_name, filename=filename, - prefix_tmp_folder_name='swh.loader.git.', - start_path=os.path.dirname(__file__), - uncompress_archive=uncompress_archive) + def setUp(self, archive_name, uncompress_archive, filename="testrepo"): + super().setUp( + archive_name=archive_name, + filename=filename, + prefix_tmp_folder_name="swh.loader.git.", + start_path=os.path.dirname(__file__), + uncompress_archive=uncompress_archive, + ) class GitLoaderFromDiskTest(OrigGitLoaderFromDisk): def parse_config_file(self, *args, **kwargs): return TEST_LOADER_CONFIG class BaseDirGitLoaderFromDiskTest(BaseGitLoaderFromDiskTest): """Mixin base loader test to prepare the git repository to uncompress, load and test the results. This sets up """ + def setUp(self): - super().setUp('testrepo.tgz', uncompress_archive=True) + super().setUp("testrepo.tgz", uncompress_archive=True) self.loader = GitLoaderFromDiskTest( url=self.repo_url, - visit_date='2016-05-03 15:16:32+00', - directory=self.destination_path + visit_date="2016-05-03 15:16:32+00", + directory=self.destination_path, ) self.storage = self.loader.storage def load(self): return self.loader.load() class BaseGitLoaderFromArchiveTest(BaseGitLoaderFromDiskTest): """Mixin base loader test to prepare the git repository to uncompress, load and test the results. This sets up """ + def setUp(self): - super().setUp('testrepo.tgz', uncompress_archive=False) + super().setUp("testrepo.tgz", uncompress_archive=False) self.loader = GitLoaderFromArchive( url=self.repo_url, - visit_date='2016-05-03 15:16:32+00', + visit_date="2016-05-03 15:16:32+00", archive_path=self.destination_path, ) self.storage = self.loader.storage def load(self): return self.loader.load() class GitLoaderFromDiskTests: """Common tests for all git loaders.""" + def test_load(self): """Loads a simple repository (made available by `setUp()`), and checks everything was added in the storage.""" res = self.load() - self.assertEqual(res['status'], 'eventful', res) + self.assertEqual(res["status"], "eventful", res) self.assertContentsContain(CONTENT1) self.assertCountDirectories(7) self.assertCountReleases(0) # FIXME: why not 2? self.assertCountRevisions(7) self.assertCountSnapshots(1) self.assertRevisionsContain(REVISIONS1) self.assertSnapshotEqual(SNAPSHOT1) - self.assertEqual(self.loader.load_status(), {'status': 'eventful'}) - self.assertEqual(self.loader.visit_status(), 'full') + self.assertEqual(self.loader.load_status(), {"status": "eventful"}) + self.assertEqual(self.loader.visit_status(), "full") visit = self.storage.origin_visit_get_latest(self.repo_url) - self.assertEqual(visit['snapshot'], hash_to_bytes(SNAPSHOT1['id'])) - self.assertEqual(visit['status'], 'full') + self.assertEqual(visit["snapshot"], hash_to_bytes(SNAPSHOT1["id"])) + self.assertEqual(visit["status"], "full") def test_load_unchanged(self): """Checks loading a repository a second time does not add any extra data.""" res = self.load() - self.assertEqual(res['status'], 'eventful') + self.assertEqual(res["status"], "eventful") visit = self.storage.origin_visit_get_latest(self.repo_url) - self.assertEqual(visit['snapshot'], hash_to_bytes(SNAPSHOT1['id'])) - self.assertEqual(visit['status'], 'full') + self.assertEqual(visit["snapshot"], hash_to_bytes(SNAPSHOT1["id"])) + self.assertEqual(visit["status"], "full") res = self.load() - self.assertEqual(res['status'], 'uneventful') + self.assertEqual(res["status"], "uneventful") self.assertCountSnapshots(1) visit = self.storage.origin_visit_get_latest(self.repo_url) - self.assertEqual(visit['snapshot'], hash_to_bytes(SNAPSHOT1['id'])) - self.assertEqual(visit['status'], 'full') + self.assertEqual(visit["snapshot"], hash_to_bytes(SNAPSHOT1["id"])) + self.assertEqual(visit["status"], "full") class DirGitLoaderTest(BaseDirGitLoaderFromDiskTest, GitLoaderFromDiskTests): """Tests for the GitLoaderFromDisk. Includes the common ones, and add others that only work with a local dir.""" def _git(self, *cmd): """Small wrapper around subprocess to call Git.""" try: return subprocess.check_output( - ['git', '-C', self.destination_path] + list(cmd)) + ["git", "-C", self.destination_path] + list(cmd) + ) except subprocess.CalledProcessError as e: print(e.output) print(e.stderr) raise def test_load_changed(self): """Loads a repository, makes some changes by adding files, commits, and merges, load it again, and check the storage contains everything it should.""" # Initial load res = self.load() - self.assertEqual(res['status'], 'eventful', res) + self.assertEqual(res["status"], "eventful", res) - self._git('config', '--local', 'user.email', 'you@example.com') - self._git('config', '--local', 'user.name', 'Your Name') + self._git("config", "--local", "user.email", "you@example.com") + self._git("config", "--local", "user.name", "Your Name") # Load with a new file + revision - with open(os.path.join(self.destination_path, 'hello.py'), 'a') as fd: + with open(os.path.join(self.destination_path, "hello.py"), "a") as fd: fd.write("print('Hello world')\n") - self._git('add', 'hello.py') - self._git('commit', '-m', 'Hello world') - new_revision = self._git('rev-parse', 'master').decode().strip() + self._git("add", "hello.py") + self._git("commit", "-m", "Hello world") + new_revision = self._git("rev-parse", "master").decode().strip() revisions = REVISIONS1.copy() assert new_revision not in revisions - revisions[new_revision] = '85dae072a5aa9923ffa7a7568f819ff21bf49858' + revisions[new_revision] = "85dae072a5aa9923ffa7a7568f819ff21bf49858" res = self.load() - self.assertEqual(res['status'], 'eventful') + self.assertEqual(res["status"], "eventful") self.assertCountContents(4 + 1) self.assertCountDirectories(7 + 1) self.assertCountReleases(0) # FIXME: why not 2? self.assertCountRevisions(7 + 1) self.assertCountSnapshots(1 + 1) self.assertRevisionsContain(revisions) # TODO: how to check the snapshot id? # self.assertSnapshotEqual(SNAPSHOT1) - self.assertEqual(self.loader.load_status(), {'status': 'eventful'}) - self.assertEqual(self.loader.visit_status(), 'full') + self.assertEqual(self.loader.load_status(), {"status": "eventful"}) + self.assertEqual(self.loader.visit_status(), "full") visit = self.storage.origin_visit_get_latest(self.repo_url) - self.assertIsNotNone(visit['snapshot']) - self.assertEqual(visit['status'], 'full') + self.assertIsNotNone(visit["snapshot"]) + self.assertEqual(visit["status"], "full") # Load with a new merge - self._git('merge', 'branch1', '-m', 'merge') - new_revision = self._git('rev-parse', 'master').decode().strip() + self._git("merge", "branch1", "-m", "merge") + new_revision = self._git("rev-parse", "master").decode().strip() assert new_revision not in revisions - revisions[new_revision] = 'dab8a37df8db8666d4e277bef9a546f585b5bedd' + revisions[new_revision] = "dab8a37df8db8666d4e277bef9a546f585b5bedd" res = self.load() - self.assertEqual(res['status'], 'eventful') + self.assertEqual(res["status"], "eventful") self.assertCountContents(4 + 1) self.assertCountDirectories(7 + 2) self.assertCountReleases(0) # FIXME: why not 2? self.assertCountRevisions(7 + 2) self.assertCountSnapshots(1 + 1 + 1) self.assertRevisionsContain(revisions) # TODO: how to check the snapshot id? # self.assertSnapshotEqual(SNAPSHOT1) - self.assertEqual(self.loader.load_status(), {'status': 'eventful'}) - self.assertEqual(self.loader.visit_status(), 'full') + self.assertEqual(self.loader.load_status(), {"status": "eventful"}) + self.assertEqual(self.loader.visit_status(), "full") visit = self.storage.origin_visit_get_latest(self.repo_url) - self.assertIsNotNone(visit['snapshot']) - self.assertEqual(visit['status'], 'full') + self.assertIsNotNone(visit["snapshot"]) + self.assertEqual(visit["status"], "full") -class GitLoaderFromArchiveTest(BaseGitLoaderFromArchiveTest, - GitLoaderFromDiskTests): +class GitLoaderFromArchiveTest(BaseGitLoaderFromArchiveTest, GitLoaderFromDiskTests): """Tests for GitLoaderFromArchive. Imports the common ones from GitLoaderTests.""" + pass diff --git a/swh/loader/git/tests/test_loader.py b/swh/loader/git/tests/test_loader.py index 9a70735..53ed6d7 100644 --- a/swh/loader/git/tests/test_loader.py +++ b/swh/loader/git/tests/test_loader.py @@ -1,28 +1,26 @@ # Copyright (C) 2018-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from swh.loader.git.loader import GitLoader from swh.loader.git.tests.test_from_disk import DirGitLoaderTest from . import TEST_LOADER_CONFIG class GitLoaderTest(GitLoader): def parse_config_file(self, *args, **kwargs): - return { - **super().parse_config_file(*args, **kwargs), - **TEST_LOADER_CONFIG - } + return {**super().parse_config_file(*args, **kwargs), **TEST_LOADER_CONFIG} class TestGitLoader(DirGitLoaderTest): """Same tests as for the GitLoaderFromDisk, but running on GitLoader.""" + def setUp(self): super().setUp() self.loader = GitLoaderTest(self.repo_url) self.storage = self.loader.storage def load(self): return self.loader.load() diff --git a/swh/loader/git/tests/test_tasks.py b/swh/loader/git/tests/test_tasks.py index 8019393..a1b4ffe 100644 --- a/swh/loader/git/tests/test_tasks.py +++ b/swh/loader/git/tests/test_tasks.py @@ -1,62 +1,60 @@ # Copyright (C) 2018-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information def test_git_loader(mocker, swh_app, celery_session_worker): - mock_loader = mocker.patch('swh.loader.git.loader.GitLoader.load') - mock_loader.return_value = {'status': 'eventful'} + mock_loader = mocker.patch("swh.loader.git.loader.GitLoader.load") + mock_loader.return_value = {"status": "eventful"} res = swh_app.send_task( - 'swh.loader.git.tasks.UpdateGitRepository', - kwargs={ - 'url': 'origin_url', - }) + "swh.loader.git.tasks.UpdateGitRepository", kwargs={"url": "origin_url",} + ) assert res res.wait() assert res.successful() - assert res.result == {'status': 'eventful'} + assert res.result == {"status": "eventful"} mock_loader.assert_called_once_with() def test_git_loader_from_disk(mocker, swh_app, celery_session_worker): - mock_loader = mocker.patch( - 'swh.loader.git.from_disk.GitLoaderFromDisk.load') - mock_loader.return_value = {'status': 'uneventful'} + mock_loader = mocker.patch("swh.loader.git.from_disk.GitLoaderFromDisk.load") + mock_loader.return_value = {"status": "uneventful"} res = swh_app.send_task( - 'swh.loader.git.tasks.LoadDiskGitRepository', + "swh.loader.git.tasks.LoadDiskGitRepository", kwargs={ - 'url': 'origin_url2', - 'directory': '/some/repo', - 'date': '2018-12-10 00:00', - }) + "url": "origin_url2", + "directory": "/some/repo", + "date": "2018-12-10 00:00", + }, + ) assert res res.wait() assert res.successful() - assert res.result == {'status': 'uneventful'} + assert res.result == {"status": "uneventful"} mock_loader.assert_called_once_with() def test_git_loader_from_archive(mocker, swh_app, celery_session_worker): - mock_loader = mocker.patch( - 'swh.loader.git.from_disk.GitLoaderFromArchive.load') + mock_loader = mocker.patch("swh.loader.git.from_disk.GitLoaderFromArchive.load") - mock_loader.return_value = {'status': 'failed'} + mock_loader.return_value = {"status": "failed"} res = swh_app.send_task( - 'swh.loader.git.tasks.UncompressAndLoadDiskGitRepository', + "swh.loader.git.tasks.UncompressAndLoadDiskGitRepository", kwargs={ - 'url': 'origin_url3', - 'archive_path': '/some/repo', - 'date': '2017-01-10 00:00', - }) + "url": "origin_url3", + "archive_path": "/some/repo", + "date": "2017-01-10 00:00", + }, + ) assert res res.wait() assert res.successful() - assert res.result == {'status': 'failed'} + assert res.result == {"status": "failed"} mock_loader.assert_called_once_with() diff --git a/swh/loader/git/tests/test_utils.py b/swh/loader/git/tests/test_utils.py index 90f3005..a994eb6 100644 --- a/swh/loader/git/tests/test_utils.py +++ b/swh/loader/git/tests/test_utils.py @@ -1,30 +1,30 @@ # Copyright (C) 2015-2017 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import unittest from swh.loader.git import utils class TestUtils(unittest.TestCase): def test_check_date_time(self): """A long as datetime is fine, date time check does not raise """ for e in range(32, 37): - ts = 2**e + ts = 2 ** e utils.check_date_time(ts) def test_check_date_time_empty_value(self): self.assertIsNone(utils.check_date_time(None)) def test_check_date_time_raises(self): """From a give threshold, check will no longer works. """ exp = 38 - timestamp = 2**exp - with self.assertRaisesRegex(ValueError, 'is out of range'): + timestamp = 2 ** exp + with self.assertRaisesRegex(ValueError, "is out of range"): utils.check_date_time(timestamp) diff --git a/swh/loader/git/utils.py b/swh/loader/git/utils.py index 0a72bec..77fe274 100644 --- a/swh/loader/git/utils.py +++ b/swh/loader/git/utils.py @@ -1,75 +1,75 @@ # Copyright (C) 2017 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information """Utilities helper functions""" import datetime import os import shutil import tempfile from swh.core import tarball -def init_git_repo_from_archive(project_name, archive_path, - root_temp_dir='/tmp'): +def init_git_repo_from_archive(project_name, archive_path, root_temp_dir="/tmp"): """Given a path to an archive containing a git repository. Uncompress that archive to a temporary location and returns the path. If any problem whatsoever is raised, clean up the temporary location. Args: project_name (str): Project's name archive_path (str): Full path to the archive root_temp_dir (str): Optional temporary directory mount point (default to /tmp) Returns A tuple: - temporary folder: containing the mounted repository - repo_path, path to the mounted repository inside the temporary folder Raises ValueError in case of failure to run the command to uncompress """ temp_dir = tempfile.mkdtemp( - suffix='.swh.loader.git', prefix='tmp.', dir=root_temp_dir) + suffix=".swh.loader.git", prefix="tmp.", dir=root_temp_dir + ) try: # create the repository that will be loaded with the dump tarball.uncompress(archive_path, temp_dir) repo_path = os.path.join(temp_dir, project_name) # tarball content may not be as expected (e.g. no top level directory # or a top level directory with a name different from project_name), # so try to make it loadable anyway if not os.path.exists(repo_path): os.mkdir(repo_path) for root, dirs, files in os.walk(temp_dir): - if '.git' in dirs: - shutil.copytree(os.path.join(root, '.git'), - os.path.join(repo_path, '.git')) + if ".git" in dirs: + shutil.copytree( + os.path.join(root, ".git"), os.path.join(repo_path, ".git") + ) break return temp_dir, repo_path except Exception as e: shutil.rmtree(temp_dir) raise e def check_date_time(timestamp): """Check date time for overflow errors. Args: timestamp (timestamp): Timestamp in seconds Raise: Any error raised by datetime fromtimestamp conversion error. """ if not timestamp: return None - datetime.datetime.fromtimestamp(timestamp, - datetime.timezone.utc) + datetime.datetime.fromtimestamp(timestamp, datetime.timezone.utc) diff --git a/tox.ini b/tox.ini index bc6e6cb..0fac710 100644 --- a/tox.ini +++ b/tox.ini @@ -1,31 +1,38 @@ [tox] -envlist=flake8,mypy,py3 +envlist=black,flake8,mypy,py3 [testenv] extras = testing deps = # the dependency below is needed for now as a workaround for # https://github.com/pypa/pip/issues/6239 # TODO: remove when this issue is fixed swh.core[http] >= 0.0.61 pytest-cov commands = pytest --cov={envsitepackagesdir}/swh/loader/git \ {envsitepackagesdir}/swh/loader/git \ --cov-branch {posargs} +[testenv:black] +skip_install = true +deps = + black +commands = + {envpython} -m black --check swh + [testenv:flake8] skip_install = true deps = flake8 commands = {envpython} -m flake8 [testenv:mypy] extras = testing deps = mypy commands = mypy swh