diff --git a/swh/loader/git/converters.py b/swh/loader/git/converters.py index c9e3761..0b7e8fa 100644 --- a/swh/loader/git/converters.py +++ b/swh/loader/git/converters.py @@ -1,310 +1,327 @@ # Copyright (C) 2015-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information """Convert dulwich objects to dictionaries suitable for swh.storage""" import logging import re from typing import Any, Dict, Optional, cast import attr from dulwich.objects import Blob, Commit, ShaFile, Tag, Tree, _parse_message from swh.model.hashutil import ( DEFAULT_ALGORITHMS, MultiHash, git_object_header, hash_to_bytes, hash_to_hex, ) from swh.model.model import ( BaseContent, Content, Directory, DirectoryEntry, HashableObject, ObjectType, Person, Release, Revision, RevisionType, SkippedContent, TargetType, Timestamp, TimestampWithTimezone, ) COMMIT_MODE_MASK = 0o160000 """Mode/perms of tree entries that point to a commit. They are normally equal to this mask, but may have more bits set to 1.""" TREE_MODE_MASK = 0o040000 """Mode/perms of tree entries that point to a tree. They are normally equal to this mask, but may have more bits set to 1.""" AUTHORSHIP_LINE_RE = re.compile(rb"^.*> (?P\S+) (?P\S+)$") logger = logging.getLogger(__name__) class HashMismatch(Exception): pass def check_id(obj: HashableObject) -> None: real_id = obj.compute_hash() if obj.id != real_id: raise HashMismatch( f"Expected {type(obj).__name__} hash to be {obj.id.hex()}, " f"got {real_id.hex()}" ) def dulwich_blob_to_content_id(obj: ShaFile) -> Dict[str, Any]: """Convert a dulwich blob to a Software Heritage content id""" if obj.type_name != b"blob": raise ValueError("Argument is not a blob.") blob = cast(Blob, obj) size = blob.raw_length() data = blob.as_raw_string() hashes = MultiHash.from_data(data, DEFAULT_ALGORITHMS).digest() if hashes["sha1_git"] != blob.sha().digest(): raise HashMismatch( f"Expected Content hash to be {blob.sha().digest().hex()}, " f"got {hashes['sha1_git'].hex()}" ) hashes["length"] = size return hashes def dulwich_blob_to_content(obj: ShaFile, max_content_size=None) -> BaseContent: - """Convert a dulwich blob to a Software Heritage content - - """ + """Convert a dulwich blob to a Software Heritage content""" if obj.type_name != b"blob": raise ValueError("Argument is not a blob.") blob = cast(Blob, obj) hashes = dulwich_blob_to_content_id(blob) if max_content_size is not None and hashes["length"] >= max_content_size: - return SkippedContent(status="absent", reason="Content too large", **hashes,) + return SkippedContent( + status="absent", + reason="Content too large", + **hashes, + ) else: - return Content(data=blob.as_raw_string(), status="visible", **hashes,) + return Content( + data=blob.as_raw_string(), + status="visible", + **hashes, + ) def dulwich_tree_to_directory(obj: ShaFile) -> Directory: """Format a tree as a directory""" if obj.type_name != b"tree": raise ValueError("Argument is not a tree.") tree = cast(Tree, obj) entries = [] for entry in tree.iteritems(): if entry.mode & COMMIT_MODE_MASK == COMMIT_MODE_MASK: type_ = "rev" elif entry.mode & TREE_MODE_MASK == TREE_MODE_MASK: type_ = "dir" else: type_ = "file" entries.append( DirectoryEntry( type=type_, perms=entry.mode, name=entry.path, target=hash_to_bytes(entry.sha.decode("ascii")), ) ) - dir_ = Directory(id=tree.sha().digest(), entries=tuple(entries),) + dir_ = Directory( + id=tree.sha().digest(), + entries=tuple(entries), + ) if dir_.compute_hash() != dir_.id: expected_id = dir_.id actual_id = dir_.compute_hash() logger.warning( "Expected directory to have id %s, but got %s. Recording raw_manifest.", hash_to_hex(expected_id), hash_to_hex(actual_id), ) raw_string = tree.as_raw_string() dir_ = attr.evolve( dir_, raw_manifest=git_object_header("tree", len(raw_string)) + raw_string ) check_id(dir_) return dir_ def parse_author(name_email: bytes) -> Person: """Parse an author line""" return Person.from_fullname(name_email) def dulwich_tsinfo_to_timestamp( - timestamp, timezone: int, timezone_neg_utc: bool, timezone_bytes: Optional[bytes], + timestamp, + timezone: int, + timezone_neg_utc: bool, + timezone_bytes: Optional[bytes], ) -> TimestampWithTimezone: """Convert the dulwich timestamp information to a structure compatible with Software Heritage.""" - ts = Timestamp(seconds=int(timestamp), microseconds=0,) + ts = Timestamp( + seconds=int(timestamp), + microseconds=0, + ) if timezone_bytes is None: # Failed to parse from the raw manifest, fallback to what Dulwich managed to # parse. return TimestampWithTimezone.from_numeric_offset( - timestamp=ts, offset=timezone // 60, negative_utc=timezone_neg_utc, + timestamp=ts, + offset=timezone // 60, + negative_utc=timezone_neg_utc, ) else: return TimestampWithTimezone(timestamp=ts, offset_bytes=timezone_bytes) def dulwich_commit_to_revision(obj: ShaFile) -> Revision: if obj.type_name != b"commit": raise ValueError("Argument is not a commit.") commit = cast(Commit, obj) author_timezone = None committer_timezone = None for (field, value) in _parse_message(commit._chunked_text): if field == b"author": m = AUTHORSHIP_LINE_RE.match(value) if m: author_timezone = m.group("timezone") elif field == b"committer": m = AUTHORSHIP_LINE_RE.match(value) if m: committer_timezone = m.group("timezone") extra_headers = [] if commit.encoding is not None: extra_headers.append((b"encoding", commit.encoding)) if commit.mergetag: for mergetag in commit.mergetag: raw_string = mergetag.as_raw_string() assert raw_string.endswith(b"\n") extra_headers.append((b"mergetag", raw_string[:-1])) if commit.extra: extra_headers.extend((k, v) for k, v in commit.extra) if commit.gpgsig: extra_headers.append((b"gpgsig", commit.gpgsig)) rev = Revision( id=commit.sha().digest(), author=parse_author(commit.author), date=dulwich_tsinfo_to_timestamp( commit.author_time, commit.author_timezone, commit._author_timezone_neg_utc, author_timezone, ), committer=parse_author(commit.committer), committer_date=dulwich_tsinfo_to_timestamp( commit.commit_time, commit.commit_timezone, commit._commit_timezone_neg_utc, committer_timezone, ), type=RevisionType.GIT, directory=bytes.fromhex(commit.tree.decode()), message=commit.message, metadata=None, extra_headers=tuple(extra_headers), synthetic=False, parents=tuple(bytes.fromhex(p.decode()) for p in commit.parents), ) if rev.compute_hash() != rev.id: expected_id = rev.id actual_id = rev.compute_hash() logger.warning( "Expected revision to have id %s, but got %s. Recording raw_manifest.", hash_to_hex(expected_id), hash_to_hex(actual_id), ) raw_string = commit.as_raw_string() rev = attr.evolve( rev, raw_manifest=git_object_header("commit", len(raw_string)) + raw_string ) check_id(rev) return rev DULWICH_TARGET_TYPES = { b"blob": TargetType.CONTENT, b"tree": TargetType.DIRECTORY, b"commit": TargetType.REVISION, b"tag": TargetType.RELEASE, } DULWICH_OBJECT_TYPES = { b"blob": ObjectType.CONTENT, b"tree": ObjectType.DIRECTORY, b"commit": ObjectType.REVISION, b"tag": ObjectType.RELEASE, } def dulwich_tag_to_release(obj: ShaFile) -> Release: if obj.type_name != b"tag": raise ValueError("Argument is not a tag.") tag = cast(Tag, obj) tagger_timezone = None # FIXME: _parse_message is a private function from Dulwich. for (field, value) in _parse_message(tag.as_raw_chunks()): if field == b"tagger": m = AUTHORSHIP_LINE_RE.match(value) if m: tagger_timezone = m.group("timezone") target_type, target = tag.object if tag.tagger: author: Optional[Person] = parse_author(tag.tagger) if tag.tag_time is None: date = None else: date = dulwich_tsinfo_to_timestamp( tag.tag_time, tag.tag_timezone, tag._tag_timezone_neg_utc, tagger_timezone, ) else: author = date = None message = tag.message if tag.signature: message += tag.signature rel = Release( id=tag.sha().digest(), author=author, date=date, name=tag.name, target=bytes.fromhex(target.decode()), target_type=DULWICH_OBJECT_TYPES[target_type.type_name], message=message, metadata=None, synthetic=False, ) if rel.compute_hash() != rel.id: expected_id = rel.id actual_id = rel.compute_hash() logger.warning( "Expected release to have id %s, but got %s. Recording raw_manifest.", hash_to_hex(expected_id), hash_to_hex(actual_id), ) raw_string = tag.as_raw_string() rel = attr.evolve( rel, raw_manifest=git_object_header("tag", len(raw_string)) + raw_string ) check_id(rel) return rel diff --git a/swh/loader/git/dumb.py b/swh/loader/git/dumb.py index e642ac0..c34c19b 100644 --- a/swh/loader/git/dumb.py +++ b/swh/loader/git/dumb.py @@ -1,201 +1,204 @@ # Copyright (C) 2021 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from __future__ import annotations from collections import defaultdict import logging import stat import struct from tempfile import SpooledTemporaryFile from typing import TYPE_CHECKING, Callable, Dict, Iterable, List, Set, cast import urllib.parse from dulwich.errors import NotGitRepository from dulwich.objects import S_IFGITLINK, Commit, ShaFile, Tree from dulwich.pack import Pack, PackData, PackIndex, load_pack_index_file import requests from swh.loader.git.utils import HexBytes if TYPE_CHECKING: from .loader import RepoRepresentation logger = logging.getLogger(__name__) HEADERS = {"User-Agent": "Software Heritage dumb Git loader"} def check_protocol(repo_url: str) -> bool: """Checks if a git repository can be cloned using the dumb protocol. Args: repo_url: Base URL of a git repository Returns: Whether the dumb protocol is supported. """ if not repo_url.startswith("http"): return False url = urllib.parse.urljoin( repo_url.rstrip("/") + "/", "info/refs?service=git-upload-pack/" ) logger.debug("Fetching %s", url) response = requests.get(url, headers=HEADERS) content_type = response.headers.get("Content-Type") return ( - response.status_code in (200, 304,) + response.status_code + in ( + 200, + 304, + ) # header is not mandatory in protocol specification and (content_type is None or not content_type.startswith("application/x-git-")) ) class GitObjectsFetcher: """Git objects fetcher using dumb HTTP protocol. Fetches a set of git objects for a repository according to its archival state by Software Heritage and provides iterators on them. Args: repo_url: Base URL of a git repository base_repo: State of repository archived by Software Heritage """ def __init__(self, repo_url: str, base_repo: RepoRepresentation): self._session = requests.Session() self.repo_url = repo_url self.base_repo = base_repo self.objects: Dict[bytes, Set[bytes]] = defaultdict(set) self.refs = self._get_refs() self.head = self._get_head() if self.refs else {} self.packs = self._get_packs() def fetch_object_ids(self) -> None: - """Fetches identifiers of git objects to load into the archive. - """ + """Fetches identifiers of git objects to load into the archive.""" wants = self.base_repo.determine_wants(self.refs) # process refs commit_objects = [] for ref in wants: ref_object = self._get_git_object(ref) if ref_object.get_type() == Commit.type_num: commit_objects.append(cast(Commit, ref_object)) self.objects[b"commit"].add(ref) else: self.objects[b"tag"].add(ref) # perform DFS on commits graph while commit_objects: commit = commit_objects.pop() # fetch tree and blob ids recursively self._fetch_tree_objects(commit.tree) for parent in commit.parents: if ( # commit not already seen in the current load parent not in self.objects[b"commit"] # commit not already archived by a previous load and parent not in self.base_repo.heads ): commit_objects.append(cast(Commit, self._get_git_object(parent))) self.objects[b"commit"].add(parent) def iter_objects(self, object_type: bytes) -> Iterable[ShaFile]: """Returns a generator on fetched git objects per type. Args: object_type: Git object type, either b"blob", b"commit", b"tag" or b"tree" Returns: A generator fetching git objects on the fly. """ return map(self._get_git_object, self.objects[object_type]) def _http_get(self, path: str) -> SpooledTemporaryFile: url = urllib.parse.urljoin(self.repo_url.rstrip("/") + "/", path) logger.debug("Fetching %s", url) response = self._session.get(url, headers=HEADERS) buffer = SpooledTemporaryFile(max_size=100 * 1024 * 1024) for chunk in response.iter_content(chunk_size=10 * 1024 * 1024): buffer.write(chunk) buffer.flush() buffer.seek(0) return buffer def _get_refs(self) -> Dict[bytes, HexBytes]: refs = {} refs_resp_bytes = self._http_get("info/refs") for ref_line in refs_resp_bytes.readlines(): ref_target, ref_name = ref_line.replace(b"\n", b"").split(b"\t") refs[ref_name] = ref_target return refs def _get_head(self) -> Dict[bytes, HexBytes]: head_resp_bytes = self._http_get("HEAD") _, head_target = head_resp_bytes.readline().replace(b"\n", b"").split(b" ") return {b"HEAD": head_target} def _get_pack_data(self, pack_name: str) -> Callable[[], PackData]: def _pack_data() -> PackData: pack_data_bytes = self._http_get(f"objects/pack/{pack_name}") return PackData(pack_name, file=pack_data_bytes) return _pack_data def _get_pack_idx(self, pack_idx_name: str) -> Callable[[], PackIndex]: def _pack_idx() -> PackIndex: pack_idx_bytes = self._http_get(f"objects/pack/{pack_idx_name}") return load_pack_index_file(pack_idx_name, pack_idx_bytes) return _pack_idx def _get_packs(self) -> List[Pack]: packs = [] packs_info_bytes = self._http_get("objects/info/packs") packs_info = packs_info_bytes.read().decode() for pack_info in packs_info.split("\n"): if pack_info: pack_name = pack_info.split(" ")[1] pack_idx_name = pack_name.replace(".pack", ".idx") # pack index and data file will be lazily fetched when required packs.append( Pack.from_lazy_objects( self._get_pack_data(pack_name), self._get_pack_idx(pack_idx_name), ) ) return packs def _get_git_object(self, sha: bytes) -> ShaFile: # try to get the object from a pack file first to avoid flooding # git server with numerous HTTP requests for pack in list(self.packs): try: if sha in pack: return pack[sha] except (NotGitRepository, struct.error): # missing (dulwich http client raises NotGitRepository on 404) # or invalid pack index/content, remove it from global packs list logger.debug("A pack file is missing or its content is invalid") self.packs.remove(pack) # fetch it from objects/ directory otherwise sha_hex = sha.decode() object_path = f"objects/{sha_hex[:2]}/{sha_hex[2:]}" return ShaFile.from_file(self._http_get(object_path)) def _fetch_tree_objects(self, sha: bytes) -> None: if sha not in self.objects[b"tree"]: tree = cast(Tree, self._get_git_object(sha)) self.objects[b"tree"].add(sha) for item in tree.items(): if item.mode == S_IFGITLINK: # skip submodules as objects are not stored in repository continue if item.mode & stat.S_IFDIR: self._fetch_tree_objects(item.sha) else: self.objects[b"blob"].add(item.sha) diff --git a/swh/loader/git/from_disk.py b/swh/loader/git/from_disk.py index 41fed95..af8c7eb 100644 --- a/swh/loader/git/from_disk.py +++ b/swh/loader/git/from_disk.py @@ -1,452 +1,447 @@ # Copyright (C) 2015-2021 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from collections import defaultdict from datetime import datetime import logging import os import shutil from typing import Dict, Optional from dulwich.errors import ObjectFormatException try: from dulwich.errors import EmptyFileException # type: ignore except ImportError: # dulwich >= 0.20 from dulwich.objects import EmptyFileException from deprecated import deprecated import dulwich.objects import dulwich.repo from swh.loader.core.loader import DVCSLoader from swh.model import hashutil from swh.model.model import Origin, Snapshot, SnapshotBranch, TargetType from swh.storage.algos.origin import origin_get_latest_visit_status from swh.storage.interface import StorageInterface from . import converters, utils logger = logging.getLogger(__name__) def _check_tag(tag): """Copy-paste of dulwich.objects.Tag, minus the tagger and time checks, which are too strict and error on old tags.""" # Copyright (C) 2007 James Westby # Copyright (C) 2008-2013 Jelmer Vernooij # # Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU # General Public License as public by the Free Software Foundation; version 2.0 # or (at your option) any later version. You can redistribute it and/or # modify it under the terms of either of these two licenses. # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # You should have received a copy of the licenses; if not, see # for a copy of the GNU General Public License # and for a copy of the Apache # License, Version 2.0. dulwich.objects.ShaFile.check(tag) tag._check_has_member("_object_sha", "missing object sha") tag._check_has_member("_object_class", "missing object type") tag._check_has_member("_name", "missing tag name") if not tag._name: raise ObjectFormatException("empty tag name") dulwich.objects.check_hexsha(tag._object_sha, "invalid object sha") if tag._tag_time is not None: dulwich.objects.check_time(tag._tag_time) from dulwich.objects import ( _OBJECT_HEADER, _TAG_HEADER, _TAGGER_HEADER, _TYPE_HEADER, ) last = None for field, _ in dulwich.objects._parse_message(tag._chunked_text): if field == _OBJECT_HEADER and last is not None: raise ObjectFormatException("unexpected object") elif field == _TYPE_HEADER and last != _OBJECT_HEADER: raise ObjectFormatException("unexpected type") elif field == _TAG_HEADER and last != _TYPE_HEADER: raise ObjectFormatException("unexpected tag name") elif field == _TAGGER_HEADER and last != _TAG_HEADER: raise ObjectFormatException("unexpected tagger") last = field @deprecated(version="1.1", reason="Use `swh.loader.git.loader.GitLoader` instead") class GitLoaderFromDisk(DVCSLoader): - """Load a git repository from a directory. - - """ + """Load a git repository from a directory.""" visit_type = "git" def __init__( self, storage: StorageInterface, url: str, visit_date: Optional[datetime] = None, directory: Optional[str] = None, save_data_path: Optional[str] = None, max_content_size: Optional[int] = None, ): super().__init__( storage=storage, save_data_path=save_data_path, max_content_size=max_content_size, ) self.origin_url = url self.visit_date = visit_date self.directory = directory def prepare_origin_visit(self): self.origin = Origin(url=self.origin_url) def prepare(self): self.repo = dulwich.repo.Repo(self.directory) def iter_objects(self): object_store = self.repo.object_store for pack in object_store.packs: objs = list(pack.index.iterentries()) objs.sort(key=lambda x: x[1]) for sha, offset, crc32 in objs: yield hashutil.hash_to_bytehex(sha) yield from object_store._iter_loose_objects() yield from object_store._iter_alternate_objects() def _check(self, obj): """Check the object's repository representation. If any errors in check exists, an ObjectFormatException is raised. Args: obj (object): Dulwich object read from the repository. """ if isinstance(obj, dulwich.objects.Tag): _check_tag(obj) else: obj.check() try: # For additional checks on dulwich objects with date # for now, only checks on *time if isinstance(obj, dulwich.objects.Commit): commit_time = obj._commit_time utils.check_date_time(commit_time) author_time = obj._author_time utils.check_date_time(author_time) elif isinstance(obj, dulwich.objects.Tag): tag_time = obj._tag_time if tag_time: utils.check_date_time(tag_time) except Exception as e: raise ObjectFormatException(e) def get_object(self, oid): """Given an object id, return the object if it is found and not malformed in some way. Args: oid (bytes): the object's identifier Returns: The object if found without malformation """ try: # some errors are raised when reading the object obj = self.repo[oid] # some we need to check ourselves self._check(obj) except KeyError: _id = oid.decode("utf-8") logger.warn( "object %s not found, skipping" % _id, extra={ "swh_type": "swh_loader_git_missing_object", "swh_object_id": _id, "origin_url": self.origin.url, }, ) return None except ObjectFormatException as e: id_ = oid.decode("utf-8") logger.warn( "object %s malformed (%s), skipping", id_, e.args[0], extra={ "swh_type": "swh_loader_git_missing_object", "swh_object_id": id_, "origin_url": self.origin.url, }, ) return None except EmptyFileException: id_ = oid.decode("utf-8") logger.warn( "object %s corrupted (empty file), skipping", id_, extra={ "swh_type": "swh_loader_git_missing_object", "swh_object_id": id_, "origin_url": self.origin.url, }, ) else: return obj def fetch_data(self): """Fetch the data from the data source""" visit_status = origin_get_latest_visit_status( self.storage, self.origin_url, require_snapshot=True ) self.previous_snapshot_id = ( None if visit_status is None else visit_status.snapshot ) type_to_ids = defaultdict(list) for oid in self.iter_objects(): obj = self.get_object(oid) if obj is None: continue type_name = obj.type_name type_to_ids[type_name].append(oid) self.type_to_ids = type_to_ids def has_contents(self): """Checks whether we need to load contents""" return bool(self.type_to_ids[b"blob"]) def get_content_ids(self): """Get the content identifiers from the git repository""" for oid in self.type_to_ids[b"blob"]: yield converters.dulwich_blob_to_content_id(self.repo[oid]) def get_contents(self): """Get the contents that need to be loaded""" missing_contents = set( self.storage.content_missing(self.get_content_ids(), "sha1_git") ) for oid in missing_contents: yield converters.dulwich_blob_to_content( self.repo[hashutil.hash_to_bytehex(oid)] ) def has_directories(self): """Checks whether we need to load directories""" return bool(self.type_to_ids[b"tree"]) def get_directory_ids(self): """Get the directory identifiers from the git repository""" return (hashutil.hash_to_bytes(id.decode()) for id in self.type_to_ids[b"tree"]) def get_directories(self): """Get the directories that need to be loaded""" missing_dirs = set( self.storage.directory_missing(sorted(self.get_directory_ids())) ) for oid in missing_dirs: yield converters.dulwich_tree_to_directory( self.repo[hashutil.hash_to_bytehex(oid)], ) def has_revisions(self): """Checks whether we need to load revisions""" return bool(self.type_to_ids[b"commit"]) def get_revision_ids(self): """Get the revision identifiers from the git repository""" return ( hashutil.hash_to_bytes(id.decode()) for id in self.type_to_ids[b"commit"] ) def get_revisions(self): """Get the revisions that need to be loaded""" missing_revs = set( self.storage.revision_missing(sorted(self.get_revision_ids())) ) for oid in missing_revs: yield converters.dulwich_commit_to_revision( self.repo[hashutil.hash_to_bytehex(oid)], ) def has_releases(self): """Checks whether we need to load releases""" return bool(self.type_to_ids[b"tag"]) def get_release_ids(self): """Get the release identifiers from the git repository""" return (hashutil.hash_to_bytes(id.decode()) for id in self.type_to_ids[b"tag"]) def get_releases(self): """Get the releases that need to be loaded""" missing_rels = set(self.storage.release_missing(sorted(self.get_release_ids()))) for oid in missing_rels: yield converters.dulwich_tag_to_release( self.repo[hashutil.hash_to_bytehex(oid)], ) def get_snapshot(self): """Turn the list of branches into a snapshot to load""" branches: Dict[bytes, Optional[SnapshotBranch]] = {} for ref, target in self.repo.refs.as_dict().items(): if utils.ignore_branch_name(ref): continue obj = self.get_object(target) if obj: target_type = converters.DULWICH_TARGET_TYPES[obj.type_name] branches[ref] = SnapshotBranch( - target=hashutil.bytehex_to_hash(target), target_type=target_type, + target=hashutil.bytehex_to_hash(target), + target_type=target_type, ) else: branches[ref] = None dangling_branches = {} for ref, target in self.repo.refs.get_symrefs().items(): if utils.ignore_branch_name(ref): continue branches[ref] = SnapshotBranch(target=target, target_type=TargetType.ALIAS) if target not in branches: # This handles the case where the pointer is "dangling". # There's a chance that a further symbolic reference will # override this default value, which is totally fine. dangling_branches[target] = ref branches[target] = None utils.warn_dangling_branches( branches, dangling_branches, logger, self.origin_url ) self.snapshot = Snapshot(branches=branches) return self.snapshot def save_data(self): """We already have the data locally, no need to save it""" pass def load_status(self): """The load was eventful if the current occurrences are different to - the ones we retrieved at the beginning of the run""" + the ones we retrieved at the beginning of the run""" eventful = False if self.previous_snapshot_id: eventful = self.snapshot.id != self.previous_snapshot_id else: eventful = bool(self.snapshot.branches) return {"status": ("eventful" if eventful else "uneventful")} class GitLoaderFromArchive(GitLoaderFromDisk): """Load a git repository from an archive. This loader ingests a git repository compressed into an archive. The supported archive formats are ``.zip`` and ``.tar.gz``. From an input tarball named ``my-git-repo.zip``, the following layout is expected in it:: my-git-repo/ ├── .git │ ├── branches │ ├── COMMIT_EDITMSG │ ├── config │ ├── description │ ├── HEAD ... Nevertheless, the loader is able to ingest tarballs with the following layouts too:: . ├── .git │ ├── branches │ ├── COMMIT_EDITMSG │ ├── config │ ├── description │ ├── HEAD ... or:: other-repo-name/ ├── .git │ ├── branches │ ├── COMMIT_EDITMSG │ ├── config │ ├── description │ ├── HEAD ... """ def __init__(self, *args, archive_path, **kwargs): super().__init__(*args, **kwargs) self.temp_dir = self.repo_path = None self.archive_path = archive_path def project_name_from_archive(self, archive_path): - """Compute the project name from the archive's path. - - """ + """Compute the project name from the archive's path.""" archive_name = os.path.basename(archive_path) for ext in (".zip", ".tar.gz", ".tgz"): if archive_name.lower().endswith(ext): archive_name = archive_name[: -len(ext)] break return archive_name def prepare(self): """1. Uncompress the archive in temporary location. - 2. Prepare as the GitLoaderFromDisk does - 3. Load as GitLoaderFromDisk does + 2. Prepare as the GitLoaderFromDisk does + 3. Load as GitLoaderFromDisk does """ project_name = self.project_name_from_archive(self.archive_path) self.temp_dir, self.repo_path = utils.init_git_repo_from_archive( project_name, self.archive_path ) logger.info( "Project %s - Uncompressing archive %s at %s", self.origin_url, os.path.basename(self.archive_path), self.repo_path, ) self.directory = self.repo_path super().prepare() def cleanup(self): - """Cleanup the temporary location (if it exists). - - """ + """Cleanup the temporary location (if it exists).""" if self.temp_dir and os.path.exists(self.temp_dir): shutil.rmtree(self.temp_dir) logger.info( "Project %s - Done injecting %s" % (self.origin_url, self.repo_path) ) diff --git a/swh/loader/git/loader.py b/swh/loader/git/loader.py index 69cf605..d33255d 100644 --- a/swh/loader/git/loader.py +++ b/swh/loader/git/loader.py @@ -1,531 +1,535 @@ # Copyright (C) 2016-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from dataclasses import dataclass import datetime import logging import os import pickle import sys from tempfile import SpooledTemporaryFile from typing import Any, Callable, Dict, Iterable, Iterator, List, Optional, Set, Type import dulwich.client from dulwich.errors import GitProtocolError, NotGitRepository from dulwich.object_store import ObjectStoreGraphWalker from dulwich.objects import ShaFile from dulwich.pack import PackData, PackInflater from swh.loader.core.loader import DVCSLoader from swh.loader.exception import NotFound from swh.model import hashutil from swh.model.model import ( BaseContent, Directory, Origin, Release, Revision, Snapshot, SnapshotBranch, TargetType, ) from swh.storage.algos.snapshot import snapshot_get_latest from swh.storage.interface import StorageInterface from . import converters, dumb, utils from .utils import HexBytes logger = logging.getLogger(__name__) class RepoRepresentation: """Repository representation for a Software Heritage origin.""" def __init__( self, storage, base_snapshot: Optional[Snapshot] = None, incremental: bool = True, ): self.storage = storage self.incremental = incremental if base_snapshot and incremental: self.base_snapshot: Snapshot = base_snapshot else: self.base_snapshot = Snapshot(branches={}) self.heads: Set[HexBytes] = set() def get_parents(self, commit: bytes) -> List[bytes]: """This method should return the list of known parents""" return [] def graph_walker(self) -> ObjectStoreGraphWalker: return ObjectStoreGraphWalker(self.heads, self.get_parents) def determine_wants(self, refs: Dict[bytes, HexBytes]) -> List[HexBytes]: """Get the list of bytehex sha1s that the git loader should fetch. This compares the remote refs sent by the server with the base snapshot provided by the loader. """ if not refs: return [] # Cache existing heads local_heads: Set[HexBytes] = set() for branch_name, branch in self.base_snapshot.branches.items(): if not branch or branch.target_type == TargetType.ALIAS: continue local_heads.add(hashutil.hash_to_hex(branch.target).encode()) self.heads = local_heads # Get the remote heads that we want to fetch remote_heads: Set[HexBytes] = set() for ref_name, ref_target in refs.items(): if utils.ignore_branch_name(ref_name): continue remote_heads.add(ref_target) logger.debug("local_heads_count=%s", len(local_heads)) logger.debug("remote_heads_count=%s", len(remote_heads)) wanted_refs = list(remote_heads - local_heads) logger.debug("wanted_refs_count=%s", len(wanted_refs)) return wanted_refs @dataclass class FetchPackReturn: remote_refs: Dict[bytes, HexBytes] symbolic_refs: Dict[bytes, HexBytes] pack_buffer: SpooledTemporaryFile pack_size: int class GitLoader(DVCSLoader): """A bulk loader for a git repository""" visit_type = "git" def __init__( self, storage: StorageInterface, url: str, base_url: Optional[str] = None, incremental: bool = True, repo_representation: Type[RepoRepresentation] = RepoRepresentation, pack_size_bytes: int = 4 * 1024 * 1024 * 1024, temp_file_cutoff: int = 100 * 1024 * 1024, save_data_path: Optional[str] = None, max_content_size: Optional[int] = None, ): """Initialize the bulk updater. Args: repo_representation: swh's repository representation which is in charge of filtering between known and remote data. ... incremental: If True, the default, this starts from the last known snapshot (if any) references. Otherwise, this loads the full repository. """ super().__init__( storage=storage, save_data_path=save_data_path, max_content_size=max_content_size, ) self.origin_url = url self.base_url = base_url self.incremental = incremental self.repo_representation = repo_representation self.pack_size_bytes = pack_size_bytes self.temp_file_cutoff = temp_file_cutoff # state initialized in fetch_data self.remote_refs: Dict[bytes, HexBytes] = {} self.symbolic_refs: Dict[bytes, HexBytes] = {} self.ref_object_types: Dict[bytes, Optional[TargetType]] = {} def fetch_pack_from_origin( self, origin_url: str, base_repo: RepoRepresentation, do_activity: Callable[[bytes], None], ) -> FetchPackReturn: """Fetch a pack from the origin""" pack_buffer = SpooledTemporaryFile(max_size=self.temp_file_cutoff) # Hardcode the use of the tcp transport (for GitHub origins) # Even if the Dulwich API lets us process the packfile in chunks as it's # received, the HTTP transport implementation needs to entirely allocate # the packfile in memory *twice*, once in the HTTP library, and once in # a BytesIO managed by Dulwich, before passing chunks to the `do_pack` # method Overall this triples the memory usage before we can even try to # interrupt the loader before it overruns its memory limit. # In contrast, the Dulwich TCP transport just gives us the read handle # on the underlying socket, doing no processing or copying of the bytes. # We can interrupt it as soon as we've received too many bytes. transport_url = origin_url # Temporarily Disabled due to GitHub removing support for the tcp transport. See # https://forge.softwareheritage.org/T3544 # # if transport_url.startswith("https://github.com/"): # transport_url = "git" + transport_url[5:] logger.debug("Transport url to communicate with server: %s", transport_url) client, path = dulwich.client.get_transport_and_path( transport_url, thin_packs=False ) logger.debug("Client %s to fetch pack at %s", client, path) size_limit = self.pack_size_bytes def do_pack(data: bytes) -> None: cur_size = pack_buffer.tell() would_write = len(data) if cur_size + would_write > size_limit: raise IOError( f"Pack file too big for repository {origin_url}, " f"limit is {size_limit} bytes, current size is {cur_size}, " f"would write {would_write}" ) pack_buffer.write(data) pack_result = client.fetch_pack( path, base_repo.determine_wants, base_repo.graph_walker(), do_pack, progress=do_activity, ) remote_refs = pack_result.refs or {} symbolic_refs = pack_result.symrefs or {} pack_buffer.flush() pack_size = pack_buffer.tell() pack_buffer.seek(0) logger.debug("fetched_pack_size=%s", pack_size) # check if repository only supports git dumb transfer protocol, # fetched pack file will be empty in that case as dulwich do # not support it and do not fetch any refs self.dumb = transport_url.startswith("http") and client.dumb return FetchPackReturn( remote_refs=utils.filter_refs(remote_refs), symbolic_refs=utils.filter_refs(symbolic_refs), pack_buffer=pack_buffer, pack_size=pack_size, ) def prepare_origin_visit(self) -> None: self.visit_date = datetime.datetime.now(tz=datetime.timezone.utc) self.origin = Origin(url=self.origin_url) def get_full_snapshot(self, origin_url) -> Optional[Snapshot]: return snapshot_get_latest(self.storage, origin_url) def prepare(self) -> None: assert self.origin is not None prev_snapshot: Optional[Snapshot] = None if self.incremental: prev_snapshot = self.get_full_snapshot(self.origin.url) if self.base_url and prev_snapshot is None: base_origin = list(self.storage.origin_get([self.base_url]))[0] if base_origin: prev_snapshot = self.get_full_snapshot(base_origin.url) if prev_snapshot is not None: self.base_snapshot = prev_snapshot else: self.base_snapshot = Snapshot(branches={}) def fetch_data(self) -> bool: assert self.origin is not None base_repo = self.repo_representation( storage=self.storage, base_snapshot=self.base_snapshot, incremental=self.incremental, ) def do_progress(msg: bytes) -> None: sys.stderr.buffer.write(msg) sys.stderr.flush() try: fetch_info = self.fetch_pack_from_origin( self.origin.url, base_repo, do_progress ) except (dulwich.client.HTTPUnauthorized, NotGitRepository) as e: raise NotFound(e) except GitProtocolError as e: # unfortunately, that kind of error is not specific to a not found # scenario... It depends on the value of message within the exception. for msg in [ "Repository unavailable", # e.g DMCA takedown "Repository not found", "unexpected http resp 401", ]: if msg in e.args[0]: raise NotFound(e) # otherwise transmit the error raise except (AttributeError, NotImplementedError, ValueError): # with old dulwich versions, those exceptions types can be raised # by the fetch_pack operation when encountering a repository with # dumb transfer protocol so we check if the repository supports it # here to continue the loading if it is the case self.dumb = dumb.check_protocol(self.origin_url) if not self.dumb: raise logger.debug( "Protocol used for communication: %s", "dumb" if self.dumb else "smart" ) if self.dumb: self.dumb_fetcher = dumb.GitObjectsFetcher(self.origin_url, base_repo) self.dumb_fetcher.fetch_object_ids() self.remote_refs = utils.filter_refs(self.dumb_fetcher.refs) # type: ignore self.symbolic_refs = self.dumb_fetcher.head else: self.pack_buffer = fetch_info.pack_buffer self.pack_size = fetch_info.pack_size self.remote_refs = fetch_info.remote_refs self.symbolic_refs = fetch_info.symbolic_refs self.ref_object_types = {sha1: None for sha1 in self.remote_refs.values()} logger.info( "Listed %d refs for repo %s", len(self.remote_refs), self.origin.url, extra={ "swh_type": "git_repo_list_refs", "swh_repo": self.origin.url, "swh_num_refs": len(self.remote_refs), }, ) # No more data to fetch return False def save_data(self) -> None: """Store a pack for archival""" assert isinstance(self.visit_date, datetime.datetime) write_size = 8192 pack_dir = self.get_save_data_path() pack_name = "%s.pack" % self.visit_date.isoformat() refs_name = "%s.refs" % self.visit_date.isoformat() with open(os.path.join(pack_dir, pack_name), "xb") as f: self.pack_buffer.seek(0) while True: r = self.pack_buffer.read(write_size) if not r: break f.write(r) self.pack_buffer.seek(0) with open(os.path.join(pack_dir, refs_name), "xb") as f: pickle.dump(self.remote_refs, f) def iter_objects(self, object_type: bytes) -> Iterator[ShaFile]: """Read all the objects of type `object_type` from the packfile""" if self.dumb: yield from self.dumb_fetcher.iter_objects(object_type) else: self.pack_buffer.seek(0) count = 0 for obj in PackInflater.for_pack_data( PackData.from_file(self.pack_buffer, self.pack_size) ): if obj.type_name != object_type: continue yield obj count += 1 logger.debug("packfile_read_count_%s=%s", object_type.decode(), count) def get_contents(self) -> Iterable[BaseContent]: """Format the blobs from the git repository as swh contents""" for raw_obj in self.iter_objects(b"blob"): if raw_obj.id in self.ref_object_types: self.ref_object_types[raw_obj.id] = TargetType.CONTENT yield converters.dulwich_blob_to_content( raw_obj, max_content_size=self.max_content_size ) def get_directories(self) -> Iterable[Directory]: """Format the trees as swh directories""" for raw_obj in self.iter_objects(b"tree"): if raw_obj.id in self.ref_object_types: self.ref_object_types[raw_obj.id] = TargetType.DIRECTORY yield converters.dulwich_tree_to_directory(raw_obj) def get_revisions(self) -> Iterable[Revision]: """Format commits as swh revisions""" for raw_obj in self.iter_objects(b"commit"): if raw_obj.id in self.ref_object_types: self.ref_object_types[raw_obj.id] = TargetType.REVISION yield converters.dulwich_commit_to_revision(raw_obj) def get_releases(self) -> Iterable[Release]: """Retrieve all the release objects from the git repository""" for raw_obj in self.iter_objects(b"tag"): if raw_obj.id in self.ref_object_types: self.ref_object_types[raw_obj.id] = TargetType.RELEASE yield converters.dulwich_tag_to_release(raw_obj) def get_snapshot(self) -> Snapshot: """Get the snapshot for the current visit. The main complexity of this function is mapping target objects to their types, as the `refs` dictionaries returned by the git server only give us the identifiers for the target objects, and not their types. The loader itself only knows the types of the objects that it has fetched from the server (as it has parsed them while loading them to the archive). As we only fetched an increment between the previous snapshot and the current state of the server, we are missing the type information for the objects that would already have been referenced by the previous snapshot, and that the git server didn't send us. We infer the type of these objects from the previous snapshot. """ branches: Dict[bytes, Optional[SnapshotBranch]] = {} unfetched_refs: Dict[bytes, bytes] = {} # Retrieve types from the objects loaded by the current loader for ref_name, ref_object in self.remote_refs.items(): if ref_name in self.symbolic_refs: continue target = hashutil.hash_to_bytes(ref_object.decode()) target_type = self.ref_object_types.get(ref_object) if target_type: branches[ref_name] = SnapshotBranch( target=target, target_type=target_type ) else: # The object pointed at by this ref was not fetched, supposedly # because it existed in the base snapshot. We record it here, # and we can get it from the base snapshot later. unfetched_refs[ref_name] = target dangling_branches = {} # Handle symbolic references as alias branches for ref_name, target in self.symbolic_refs.items(): branches[ref_name] = SnapshotBranch( - target_type=TargetType.ALIAS, target=target, + target_type=TargetType.ALIAS, + target=target, ) if target not in branches and target not in unfetched_refs: # This handles the case where the pointer is "dangling". # There's a chance that a further symbolic reference # override this default value, which is totally fine. dangling_branches[target] = ref_name branches[target] = None if unfetched_refs: # Handle inference of object types from the contents of the # previous snapshot unknown_objects = {} base_snapshot_reverse_branches = { branch.target: branch for branch in self.base_snapshot.branches.values() if branch and branch.target_type != TargetType.ALIAS } for ref_name, target in unfetched_refs.items(): branch = base_snapshot_reverse_branches.get(target) branches[ref_name] = branch if not branch: unknown_objects[ref_name] = target if unknown_objects: # This object was referenced by the server; We did not fetch # it, and we do not know it from the previous snapshot. This is # likely a bug in the loader. raise RuntimeError( "Unknown objects referenced by remote refs: %s" % ( ", ".join( f"{name.decode()}: {hashutil.hash_to_hex(obj)}" for name, obj in unknown_objects.items() ) ) ) utils.warn_dangling_branches( branches, dangling_branches, logger, self.origin_url ) self.snapshot = Snapshot(branches=branches) return self.snapshot def load_status(self) -> Dict[str, Any]: """The load was eventful if the current snapshot is different to - the one we retrieved at the beginning of the run""" + the one we retrieved at the beginning of the run""" eventful = False if self.base_snapshot and self.snapshot: eventful = self.snapshot.id != self.base_snapshot.id elif self.snapshot: eventful = bool(self.snapshot.branches) return {"status": ("eventful" if eventful else "uneventful")} if __name__ == "__main__": import click logging.basicConfig( level=logging.DEBUG, format="%(asctime)s %(process)d %(message)s" ) from deprecated import deprecated @deprecated(version="1.1", reason="Use `swh loader run git --help` instead") @click.command() @click.option("--origin-url", help="Origin url", required=True) @click.option("--base-url", default=None, help="Optional Base url") @click.option( "--ignore-history/--no-ignore-history", help="Ignore the repository history", default=False, ) def main(origin_url: str, base_url: str, incremental: bool) -> Dict[str, Any]: from swh.storage import get_storage storage = get_storage(cls="memory") loader = GitLoader( - storage, origin_url, base_url=base_url, incremental=incremental, + storage, + origin_url, + base_url=base_url, + incremental=incremental, ) return loader.load() main() diff --git a/swh/loader/git/tasks.py b/swh/loader/git/tasks.py index fd039e5..51fe046 100644 --- a/swh/loader/git/tasks.py +++ b/swh/loader/git/tasks.py @@ -1,52 +1,50 @@ # Copyright (C) 2015-2021 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from typing import Any, Dict, Optional from celery import shared_task import dateutil.parser from swh.loader.git.from_disk import GitLoaderFromArchive, GitLoaderFromDisk from swh.loader.git.loader import GitLoader @shared_task(name=__name__ + ".UpdateGitRepository") def load_git(*, url: str, base_url: Optional[str] = None) -> Dict[str, Any]: - """Import a git repository from a remote location - - """ + """Import a git repository from a remote location""" loader = GitLoader.from_configfile(url=url, base_url=base_url) return loader.load() @shared_task(name=__name__ + ".LoadDiskGitRepository") def load_git_from_dir(*, url: str, directory: str, date: str) -> Dict[str, Any]: """Import a git repository from a local repository - Import a git repository, cloned in `directory` from `origin_url` at - `date`. + Import a git repository, cloned in `directory` from `origin_url` at + `date`. """ visit_date = dateutil.parser.parse(date) loader = GitLoaderFromDisk.from_configfile( url=url, directory=directory, visit_date=visit_date ) return loader.load() @shared_task(name=__name__ + ".UncompressAndLoadDiskGitRepository") def load_git_from_zip(*, url: str, archive_path: str, date: str) -> Dict[str, Any]: """Import a git repository from a zip archive 1. Uncompress an archive repository in a local and temporary folder 2. Load it through the git disk loader 3. Clean up the temporary folder """ visit_date = dateutil.parser.parse(date) loader = GitLoaderFromArchive.from_configfile( url=url, archive_path=archive_path, visit_date=visit_date ) return loader.load() diff --git a/swh/loader/git/tests/test_converters.py b/swh/loader/git/tests/test_converters.py index 929a7ab..9add9ba 100644 --- a/swh/loader/git/tests/test_converters.py +++ b/swh/loader/git/tests/test_converters.py @@ -1,813 +1,870 @@ # Copyright (C) 2015-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import copy import datetime import os import shutil import subprocess import tempfile import dulwich.objects import dulwich.repo import pytest import swh.loader.git.converters as converters from swh.model.hashutil import bytehex_to_hash, hash_to_bytehex, hash_to_bytes from swh.model.model import ( Content, Directory, DirectoryEntry, ObjectType, Person, Release, Revision, RevisionType, Timestamp, TimestampWithTimezone, ) TEST_DATA = os.path.join(os.path.dirname(__file__), "data") GPGSIG = ( b"-----BEGIN PGP SIGNATURE-----\n" b"\n" b"iQJLBAABCAA1FiEEAOWDevQbOk/9ITMF6ImSleOlnUcFAl8EnS4XHGRhdmlkLmRv\n" b"dWFyZEBzZGZhMy5vcmcACgkQ6ImSleOlnUdrqQ/8C5RO4NZ5Qr/dwAy2cPA7ktkY\n" b"1oUjKtspQoPbC1X3MXVa1aWo9B3KuOMR2URw44RhMNFwjccLOhfss06E8p7CZr2H\n" b"uR3CzdDw7i52jHLCL2M2ZMaPAEbQuHjXWiUWIUXz9So8YwpTyd2XQneyOC2RDDEI\n" b"I2NVbmiMeDz33jJYPrQO0QayW+ErW+xgBF7N/qS9jFWsdV1ZNfn9NxkTH8UdGuAX\n" b"583P+0tVC2DjXc6vORVhyFzyfn1A9wHosbtWI2Mpa+zezPjoPSkcyQAJu2GyOkMC\n" b"YzSjJdQVqyovo+INkIf6PuUNdp41886BG/06xwT8fl4sVsyO51lNIfgH0DMwfTTB\n" b"ZgThYnvvO7SrXDm3QzBTXkvAiHiFFl3iNyGkCyxvgVmaTntuFT+cP+HD/pCiGaC+\n" b"jHzRwfUrmuLd/lLPyq3JXBibyjnfd3SVS+7q1NZHJ4WUmCboZ0+pfrEl65mEQ/Hz\n" b"J1qCwQ/3SsTB77ANf6lLzGSowjjrtHcBTkTbFxR4ACUhiBbosyDKpHTM7fzGFGjo\n" b"EIjohzrEnqR3bbyxJkK+nxoOByhIRdowgyeJ02I4neMyLJqcaup8NMWCddxqjaPt\n" b"YobghnjaDqEd+suL/v83hbZUAZHNO3i1OZYGMqzp1WHikDPoTwGP76baqBoXi56T\n" b"4WSpxCAJRDODHLk1HgU=\n" b"=73wF" b"\n" b"-----END PGP SIGNATURE-----" ) MERGETAG = ( b"object 9768d0b576dbaaecd80abedad6dfd0d72f1476da\n" b"type commit\n" b"tag v0.0.1\n" b"tagger David Douard 1594138133 +0200\n" b"\n" b"v0.0.1\n" b"-----BEGIN PGP SIGNATURE-----\n" b"\n" b"iQJLBAABCAA1FiEEAOWDevQbOk/9ITMF6ImSleOlnUcFAl8EnhkXHGRhdmlkLmRv\n" b"dWFyZEBzZGZhMy5vcmcACgkQ6ImSleOlnUcdzg//ZW9y2xU5JFQuUsBe/LfKrs+m\n" b"0ohVInPKXwAfpB3+gn/XtTSLe+Nnr8+QEZyVRCUz2gpGZ2tNqRjhYLIX4x5KKlaV\n" b"rfl/6Cy7zibsxxuzA1h7HylCs3IPsueQpznVHUwD9jQ5baGJSc2Lt1LufXTueHZJ\n" b"Oc0oLiP5xCZcPqeX8R/4zUUImJZ1QrPeKmQ/3F+Iq62iWp7nWDp8PtwpykSiYlNf\n" b"KrJM8omGvrlrWLtfPNUaQFClXwnwK1/HyNY2kYan6K5NtsIl2UX0LZ42GkRjJIrb\n" b"q4TFIZWZ6xndtEhHEX6B8Q5TZV6sqPgNnfGpbhj8BDoZgjD0Y43fzfDiZ0Bl2tph\n" b"tXaLg3SX/UUjFVzC1zkoQ2MR7+j8NVKauAsBINpKF4pMGsrsVRk8764pgO49iQ+S\n" b"8JVCVV76dNNm1gd7BbhFAdIAiegBtsEF69niJBoHKYLlrT8E8hDkF/gk4IkimPqf\n" b"UHtw/fPhVW3B4G2skd013NJGcnRj5oKtaM99d2Roxc3vhSRiTsoaM8BM9NDvLmJg\n" b"35rWEOnet39iJIMCHk3AYaJl8QmUhllDdr6vygaBVeVEf27m2c3NzONmIKpWqa2J\n" b"kTpF4cmzHYro34G7WuJ1bYvmLb6qWNQt9wd8RW+J1kVm5I8dkjPzLUougBpOd0YL\n" b"Bl5UTQILbV4Tv8ZlmJM=\n" b"=s1lv\n" b"-----END PGP SIGNATURE-----" ) class SWHObjectType: - """Dulwich lookalike ObjectType class - - """ + """Dulwich lookalike ObjectType class""" def __init__(self, type_name): self.type_name = type_name @pytest.mark.fs class TestConverters: @classmethod def setup_class(cls): cls.repo_path = tempfile.mkdtemp() bundle = os.path.join(TEST_DATA, "git-repos", "example-submodule.bundle") git = subprocess.Popen( ["git", "clone", "--quiet", "--bare", "--mirror", bundle, cls.repo_path], cwd=TEST_DATA, ) # flush stdout of xz git.communicate() cls.repo = dulwich.repo.Repo(cls.repo_path) @classmethod def tearDownClass(cls): super().tearDownClass() shutil.rmtree(cls.repo_path) def test_blob_to_content(self): content_id = b"28c6f4023d65f74e3b59a2dea3c4277ed9ee07b0" content = converters.dulwich_blob_to_content(self.repo[content_id]) expected_content = Content( sha1_git=bytehex_to_hash(content_id), sha1=hash_to_bytes("4850a3420a2262ff061cb296fb915430fa92301c"), sha256=hash_to_bytes( "fee7c8a485a10321ad94b64135073cb5" "5f22cb9f57fa2417d2adfb09d310adef" ), blake2s256=hash_to_bytes( "5d71873f42a137f6d89286e43677721e574" "1fa05ce4cd5e3c7ea7c44d4c2d10b" ), data=( b'[submodule "example-dependency"]\n' b"\tpath = example-dependency\n" b"\turl = https://github.com/githubtraining/" b"example-dependency.git\n" ), length=124, status="visible", ) assert content == expected_content def test_corrupt_blob(self, mocker): # has a signature sha1 = hash_to_bytes("28c6f4023d65f74e3b59a2dea3c4277ed9ee07b0") blob = copy.deepcopy(self.repo[hash_to_bytehex(sha1)]) class hasher: def digest(): return sha1 blob._sha = hasher converters.dulwich_blob_to_content(blob) converters.dulwich_blob_to_content_id(blob) sha1 = hash_to_bytes("1234" * 10) with pytest.raises(converters.HashMismatch): converters.dulwich_blob_to_content(blob) with pytest.raises(converters.HashMismatch): converters.dulwich_blob_to_content_id(blob) def test_convertion_wrong_input(self): class Something: type_name = b"something-not-the-right-type" m = { "blob": converters.dulwich_blob_to_content, "tree": converters.dulwich_tree_to_directory, "commit": converters.dulwich_tree_to_directory, "tag": converters.dulwich_tag_to_release, } for _callable in m.values(): with pytest.raises(ValueError): _callable(Something()) def test_corrupt_tree(self): sha1 = b"a9b41fc6347d778f16c4380b598d8083e9b4c1fb" target = b"641fb6e08ddb2e4fd096dcf18e80b894bf7e25ce" tree = dulwich.objects.Tree() tree.add(b"file1", 0o644, target) assert tree.sha().hexdigest() == sha1.decode() converters.dulwich_tree_to_directory(tree) original_sha = tree.sha() tree.add(b"file2", 0o644, target) tree.sha() # reset tree._needs_serialization tree._sha = original_sha # force the wrong hash assert tree.sha().hexdigest() == sha1.decode() with pytest.raises(converters.HashMismatch): converters.dulwich_tree_to_directory(tree) def test_weird_tree(self): """Tests a tree with entries the wrong order""" raw_manifest = ( b"0644 file2\x00" b"d\x1f\xb6\xe0\x8d\xdb.O\xd0\x96\xdc\xf1\x8e\x80\xb8\x94\xbf~%\xce" b"0644 file1\x00" b"d\x1f\xb6\xe0\x8d\xdb.O\xd0\x96\xdc\xf1\x8e\x80\xb8\x94\xbf~%\xce" ) tree = dulwich.objects.Tree.from_raw_string(b"tree", raw_manifest) assert converters.dulwich_tree_to_directory(tree) == Directory( entries=( # in alphabetical order, as it should be DirectoryEntry( name=b"file1", type="file", target=hash_to_bytes("641fb6e08ddb2e4fd096dcf18e80b894bf7e25ce"), perms=0o644, ), DirectoryEntry( name=b"file2", type="file", target=hash_to_bytes("641fb6e08ddb2e4fd096dcf18e80b894bf7e25ce"), perms=0o644, ), ), raw_manifest=b"tree 62\x00" + raw_manifest, ) def test_tree_perms(self): entries = [ (b"blob_100644", 0o100644, "file"), (b"blob_100664", 0o100664, "file"), (b"blob_100666", 0o100666, "file"), (b"blob_120000", 0o120000, "file"), (b"commit_160644", 0o160644, "rev"), (b"commit_160664", 0o160664, "rev"), (b"commit_160666", 0o160666, "rev"), (b"commit_normal", 0o160000, "rev"), (b"tree_040644", 0o040644, "dir"), (b"tree_040664", 0o040664, "dir"), (b"tree_040666", 0o040666, "dir"), (b"tree_normal", 0o040000, "dir"), ] tree = dulwich.objects.Tree() for (name, mode, _) in entries: tree.add(name, mode, b"00" * 20) assert converters.dulwich_tree_to_directory(tree) == Directory( entries=tuple( DirectoryEntry(type=type, perms=perms, name=name, target=b"\x00" * 20) for (name, perms, type) in entries ) ) def test_commit_to_revision(self): sha1 = b"9768d0b576dbaaecd80abedad6dfd0d72f1476da" revision = converters.dulwich_commit_to_revision(self.repo[sha1]) expected_revision = Revision( id=hash_to_bytes("9768d0b576dbaaecd80abedad6dfd0d72f1476da"), directory=b"\xf0i\\./\xa7\xce\x9dW@#\xc3A7a\xa4s\xe5\x00\xca", type=RevisionType.GIT, committer=Person( name=b"Stefano Zacchiroli", fullname=b"Stefano Zacchiroli ", email=b"zack@upsilon.cc", ), author=Person( name=b"Stefano Zacchiroli", fullname=b"Stefano Zacchiroli ", email=b"zack@upsilon.cc", ), committer_date=TimestampWithTimezone( - timestamp=Timestamp(seconds=1443083765, microseconds=0,), + timestamp=Timestamp( + seconds=1443083765, + microseconds=0, + ), offset_bytes=b"+0200", ), message=b"add submodule dependency\n", metadata=None, extra_headers=(), date=TimestampWithTimezone( - timestamp=Timestamp(seconds=1443083765, microseconds=0,), + timestamp=Timestamp( + seconds=1443083765, + microseconds=0, + ), offset_bytes=b"+0200", ), parents=(b"\xc3\xc5\x88q23`\x9f[\xbb\xb2\xd9\xe7\xf3\xfbJf\x0f?r",), synthetic=False, ) assert revision == expected_revision def test_commit_to_revision_with_extra_headers(self): sha1 = b"322f5bc915e50fc25e85226b5a182bded0e98e4b" revision = converters.dulwich_commit_to_revision(self.repo[sha1]) expected_revision = Revision( id=hash_to_bytes(sha1.decode()), directory=bytes.fromhex("f8ec06e4ed7b9fff4918a0241a48023143f30000"), type=RevisionType.GIT, committer=Person( name=b"David Douard", fullname=b"David Douard ", email=b"david.douard@sdfa3.org", ), author=Person( name=b"David Douard", fullname=b"David Douard ", email=b"david.douard@sdfa3.org", ), committer_date=TimestampWithTimezone( - timestamp=Timestamp(seconds=1594137902, microseconds=0,), + timestamp=Timestamp( + seconds=1594137902, + microseconds=0, + ), offset_bytes=b"+0200", ), message=b"Am\xe9lioration du fichier READM\xa4\n", metadata=None, extra_headers=((b"encoding", b"ISO-8859-15"), (b"gpgsig", GPGSIG)), date=TimestampWithTimezone( - timestamp=Timestamp(seconds=1594136900, microseconds=0,), + timestamp=Timestamp( + seconds=1594136900, + microseconds=0, + ), offset_bytes=b"+0200", ), parents=(bytes.fromhex("c730509025c6e81947102b2d77bc4dc1cade9489"),), synthetic=False, ) assert revision == expected_revision def test_commit_without_manifest(self): """Tests a Release can still be produced when the manifest is not understood by the custom parser in dulwich_commit_to_revision.""" target = b"641fb6e08ddb2e4fd096dcf18e80b894bf7e25ce" message = b"some commit message" author = Person( fullname=b"Foo ", name=b"Foo", email=b"foo@example.org" ) commit = dulwich.objects.Commit() commit.tree = target commit.message = message commit.author = commit.committer = b"Foo " commit.author_time = commit.commit_time = 1641980946 commit.author_timezone = commit.commit_timezone = 3600 assert converters.dulwich_commit_to_revision(commit) == Revision( message=b"some commit message", author=author, committer=author, date=TimestampWithTimezone( timestamp=Timestamp(seconds=1641980946, microseconds=0), offset_bytes=b"+0100", ), committer_date=TimestampWithTimezone( timestamp=Timestamp(seconds=1641980946, microseconds=0), offset_bytes=b"+0100", ), type=RevisionType.GIT, directory=hash_to_bytes(target.decode()), synthetic=False, metadata=None, parents=(), ) @pytest.mark.parametrize("attribute", ["message", "encoding", "author", "gpgsig"]) def test_corrupt_commit(self, attribute): sha = hash_to_bytes("3f0ac5a6d15d89cf928209a57334e3b77c5651b9") target = b"641fb6e08ddb2e4fd096dcf18e80b894bf7e25ce" message = b"some commit message" commit = dulwich.objects.Commit() commit.tree = target commit.message = message commit.gpgsig = GPGSIG commit.author = commit.committer = b"Foo " commit.author_time = commit.commit_time = 1641980946 commit.author_timezone = commit.commit_timezone = 3600 converters.dulwich_commit_to_revision(commit) assert commit.sha().digest() == sha original_sha = commit.sha() setattr(commit, attribute, b"abcde") commit.sha() # reset tag._needs_serialization commit._sha = original_sha # force the wrong hash with pytest.raises(converters.HashMismatch): converters.dulwich_commit_to_revision(commit) if attribute == "_gpgsig": setattr(commit, attribute, None) commit.sha() # reset tag._needs_serialization commit._sha = original_sha # force the wrong hash with pytest.raises(converters.HashMismatch): converters.dulwich_commit_to_revision(commit) def test_commit_to_revision_with_extra_headers_mergetag(self): sha1 = b"3ab3da4bf0f81407be16969df09cd1c8af9ac703" revision = converters.dulwich_commit_to_revision(self.repo[sha1]) expected_revision = Revision( id=hash_to_bytes(sha1.decode()), directory=bytes.fromhex("faa4b64a841ca3e3f07d6501caebda2e3e8e544e"), type=RevisionType.GIT, committer=Person( name=b"David Douard", fullname=b"David Douard ", email=b"david.douard@sdfa3.org", ), author=Person( name=b"David Douard", fullname=b"David Douard ", email=b"david.douard@sdfa3.org", ), committer_date=TimestampWithTimezone( - timestamp=Timestamp(seconds=1594138183, microseconds=0,), + timestamp=Timestamp( + seconds=1594138183, + microseconds=0, + ), offset_bytes=b"+0200", ), message=b"Merge tag 'v0.0.1' into readme\n\nv0.0.1\n", metadata=None, extra_headers=((b"encoding", b"ISO-8859-15"), (b"mergetag", MERGETAG)), date=TimestampWithTimezone( - timestamp=Timestamp(seconds=1594138183, microseconds=0,), + timestamp=Timestamp( + seconds=1594138183, + microseconds=0, + ), offset_bytes=b"+0200", ), parents=( bytes.fromhex("322f5bc915e50fc25e85226b5a182bded0e98e4b"), bytes.fromhex("9768d0b576dbaaecd80abedad6dfd0d72f1476da"), ), synthetic=False, ) assert revision == expected_revision def test_weird_commit(self): """Checks raw_manifest is set when the commit cannot fit the data model""" # Well-formed manifest raw_manifest = ( b"tree 641fb6e08ddb2e4fd096dcf18e80b894bf7e25ce\n" b"author Foo 1640191028 +0200\n" b"committer Foo 1640191028 +0200\n\n" b"some commit message" ) commit = dulwich.objects.Commit.from_raw_string(b"commit", raw_manifest) date = TimestampWithTimezone( timestamp=Timestamp(seconds=1640191028, microseconds=0), offset_bytes=b"+0200", ) assert converters.dulwich_commit_to_revision(commit) == Revision( message=b"some commit message", directory=hash_to_bytes("641fb6e08ddb2e4fd096dcf18e80b894bf7e25ce"), synthetic=False, - author=Person.from_fullname(b"Foo ",), - committer=Person.from_fullname(b"Foo ",), + author=Person.from_fullname( + b"Foo ", + ), + committer=Person.from_fullname( + b"Foo ", + ), date=date, committer_date=date, type=RevisionType.GIT, raw_manifest=None, ) # Mess with the offset raw_manifest2 = raw_manifest.replace(b"+0200", b"+200") commit = dulwich.objects.Commit.from_raw_string(b"commit", raw_manifest2) date = TimestampWithTimezone( timestamp=Timestamp(seconds=1640191028, microseconds=0), offset_bytes=b"+200", ) assert converters.dulwich_commit_to_revision(commit) == Revision( message=b"some commit message", directory=hash_to_bytes("641fb6e08ddb2e4fd096dcf18e80b894bf7e25ce"), synthetic=False, - author=Person.from_fullname(b"Foo ",), - committer=Person.from_fullname(b"Foo ",), + author=Person.from_fullname( + b"Foo ", + ), + committer=Person.from_fullname( + b"Foo ", + ), date=date, committer_date=date, type=RevisionType.GIT, raw_manifest=None, ) # Mess with the rest of the manifest raw_manifest2 = raw_manifest.replace( b"641fb6e08ddb2e4fd096dcf18e80b894bf7e25ce", b"641FB6E08DDB2E4FD096DCF18E80B894BF7E25CE", ) commit = dulwich.objects.Commit.from_raw_string(b"commit", raw_manifest2) date = TimestampWithTimezone( timestamp=Timestamp(seconds=1640191028, microseconds=0), offset_bytes=b"+0200", ) assert converters.dulwich_commit_to_revision(commit) == Revision( message=b"some commit message", directory=hash_to_bytes("641fb6e08ddb2e4fd096dcf18e80b894bf7e25ce"), synthetic=False, - author=Person.from_fullname(b"Foo ",), - committer=Person.from_fullname(b"Foo ",), + author=Person.from_fullname( + b"Foo ", + ), + committer=Person.from_fullname( + b"Foo ", + ), date=date, committer_date=date, type=RevisionType.GIT, raw_manifest=b"commit 161\x00" + raw_manifest2, ) def test_author_line_to_author(self): # edge case out of the way with pytest.raises(TypeError): converters.parse_author(None) tests = { b"a ": Person( - name=b"a", email=b"b@c.com", fullname=b"a ", + name=b"a", + email=b"b@c.com", + fullname=b"a ", ), b"": Person( - name=None, email=b"foo@bar.com", fullname=b"", + name=None, + email=b"foo@bar.com", + fullname=b"", ), b"malformed ": Person( - name=b"trailing", email=b"sp@c.e", fullname=b"trailing ", + name=b"trailing", + email=b"sp@c.e", + fullname=b"trailing ", + ), + b"no": Person( + name=b"no", + email=b"sp@c.e", + fullname=b"no", + ), + b" <>": Person( + name=None, + email=None, + fullname=b" <>", ), - b"no": Person(name=b"no", email=b"sp@c.e", fullname=b"no",), - b" <>": Person(name=None, email=None, fullname=b" <>",), b"something": Person(name=b"something", email=None, fullname=b"something"), } for author in sorted(tests): parsed_author = tests[author] assert parsed_author == converters.parse_author(author) def test_dulwich_tag_to_release_no_author_no_date(self): sha = hash_to_bytes("f6e367357b446bd1315276de5e88ba3d0d99e136") target = b"641fb6e08ddb2e4fd096dcf18e80b894bf7e25ce" message = b"some release message" tag = dulwich.objects.Tag() tag.name = b"blah" tag.object = (dulwich.objects.Commit, target) tag.message = message tag.signature = None tag.tagger = None tag.tag_time = None tag.tag_timezone = None assert tag.sha().digest() == sha # when actual_release = converters.dulwich_tag_to_release(tag) # then expected_release = Release( author=None, date=None, id=sha, message=message, metadata=None, name=b"blah", synthetic=False, target=hash_to_bytes(target.decode()), target_type=ObjectType.REVISION, ) assert actual_release == expected_release def test_dulwich_tag_to_release_author_and_date(self): sha = hash_to_bytes("fc1e6a4f1e37e93e28e78560e73efd0b12f616ef") tagger = b"hey dude " target = b"641fb6e08ddb2e4fd096dcf18e80b894bf7e25ce" message = b"some release message" date = int( datetime.datetime(2007, 12, 5, tzinfo=datetime.timezone.utc).timestamp() ) tag = dulwich.objects.Tag() tag.name = b"blah" tag.object = (dulwich.objects.Commit, target) tag.message = message tag.signature = None tag.tagger = tagger tag.tag_time = date tag.tag_timezone = 0 assert tag.sha().digest() == sha # when actual_release = converters.dulwich_tag_to_release(tag) # then expected_release = Release( author=Person( email=b"hello@mail.org", fullname=b"hey dude ", name=b"hey dude", ), date=TimestampWithTimezone( - timestamp=Timestamp(seconds=1196812800, microseconds=0,), + timestamp=Timestamp( + seconds=1196812800, + microseconds=0, + ), offset_bytes=b"+0000", ), id=sha, message=message, metadata=None, name=b"blah", synthetic=False, target=hash_to_bytes(target.decode()), target_type=ObjectType.REVISION, ) assert actual_release == expected_release def test_dulwich_tag_to_release_author_no_date(self): # to reproduce bug T815 (fixed) sha = hash_to_bytes("41076e970975122dc6b2a878aa9797960bc4781d") tagger = b"hey dude " target = b"641fb6e08ddb2e4fd096dcf18e80b894bf7e25ce" message = b"some release message" tag = dulwich.objects.Tag() tag.name = b"blah" tag.object = (dulwich.objects.Commit, target) tag.message = message tag.signature = None tag.tagger = tagger tag.tag_time = None tag.tag_timezone = None assert tag.sha().digest() == sha # when actual_release = converters.dulwich_tag_to_release(tag) # then expected_release = Release( author=Person( email=b"hello@mail.org", fullname=b"hey dude ", name=b"hey dude", ), date=None, id=sha, message=message, metadata=None, name=b"blah", synthetic=False, target=hash_to_bytes(target.decode()), target_type=ObjectType.REVISION, ) assert actual_release == expected_release def test_dulwich_tag_to_release_author_zero_date(self): # to reproduce bug T815 (fixed) sha = hash_to_bytes("6cc1deff5cdcd853428bb63b937f43dd2566c36f") tagger = b"hey dude " target = b"641fb6e08ddb2e4fd096dcf18e80b894bf7e25ce" message = b"some release message" date = int( datetime.datetime(1970, 1, 1, tzinfo=datetime.timezone.utc).timestamp() ) tag = dulwich.objects.Tag() tag.name = b"blah" tag.object = (dulwich.objects.Commit, target) tag.message = message tag.signature = None tag.tagger = tagger tag.tag_time = date tag.tag_timezone = 0 assert tag.sha().digest() == sha # when actual_release = converters.dulwich_tag_to_release(tag) # then expected_release = Release( author=Person( email=b"hello@mail.org", fullname=b"hey dude ", name=b"hey dude", ), date=TimestampWithTimezone( - timestamp=Timestamp(seconds=0, microseconds=0,), offset_bytes=b"+0000" + timestamp=Timestamp( + seconds=0, + microseconds=0, + ), + offset_bytes=b"+0000", ), id=sha, message=message, metadata=None, name=b"blah", synthetic=False, target=hash_to_bytes(target.decode()), target_type=ObjectType.REVISION, ) assert actual_release == expected_release def test_dulwich_tag_to_release_signature(self): target = b"641fb6e08ddb2e4fd096dcf18e80b894bf7e25ce" message = b"some release message" sha = hash_to_bytes("46fff489610ed733d2cc904e363070dadee05c71") tag = dulwich.objects.Tag() tag.name = b"blah" tag.object = (dulwich.objects.Commit, target) tag.message = message tag.signature = GPGSIG tag.tagger = None tag.tag_time = None tag.tag_timezone = None assert tag.sha().digest() == sha # when actual_release = converters.dulwich_tag_to_release(tag) # then expected_release = Release( author=None, date=None, id=sha, message=message + GPGSIG, metadata=None, name=b"blah", synthetic=False, target=hash_to_bytes(target.decode()), target_type=ObjectType.REVISION, ) assert actual_release == expected_release @pytest.mark.parametrize("attribute", ["name", "message", "signature"]) def test_corrupt_tag(self, attribute): sha = hash_to_bytes("46fff489610ed733d2cc904e363070dadee05c71") target = b"641fb6e08ddb2e4fd096dcf18e80b894bf7e25ce" message = b"some release message" tag = dulwich.objects.Tag() tag.name = b"blah" tag.object = (dulwich.objects.Commit, target) tag.message = message tag.signature = GPGSIG tag.tagger = None tag.tag_time = None tag.tag_timezone = None assert tag.sha().digest() == sha converters.dulwich_tag_to_release(tag) original_sha = tag.sha() setattr(tag, attribute, b"abcde") tag.sha() # reset tag._needs_serialization tag._sha = original_sha # force the wrong hash with pytest.raises(converters.HashMismatch): converters.dulwich_tag_to_release(tag) if attribute == "signature": setattr(tag, attribute, None) tag.sha() # reset tag._needs_serialization tag._sha = original_sha # force the wrong hash with pytest.raises(converters.HashMismatch): converters.dulwich_tag_to_release(tag) def test_weird_tag(self): """Checks raw_manifest is set when the tag cannot fit the data model""" # Well-formed manifest raw_manifest = ( b"object 641fb6e08ddb2e4fd096dcf18e80b894bf7e25ce\n" b"type commit\n" b"tag blah\n" b"tagger Foo 1640191027 +0200\n\n" b"some release message" ) tag = dulwich.objects.Tag.from_raw_string(b"tag", raw_manifest) assert converters.dulwich_tag_to_release(tag) == Release( name=b"blah", message=b"some release message", target=hash_to_bytes("641fb6e08ddb2e4fd096dcf18e80b894bf7e25ce"), target_type=ObjectType.REVISION, synthetic=False, - author=Person.from_fullname(b"Foo ",), + author=Person.from_fullname( + b"Foo ", + ), date=TimestampWithTimezone( timestamp=Timestamp(seconds=1640191027, microseconds=0), offset_bytes=b"+0200", ), raw_manifest=None, ) # Mess with the offset (negative UTC) raw_manifest2 = raw_manifest.replace(b"+0200", b"-0000") tag = dulwich.objects.Tag.from_raw_string(b"tag", raw_manifest2) assert converters.dulwich_tag_to_release(tag) == Release( name=b"blah", message=b"some release message", target=hash_to_bytes("641fb6e08ddb2e4fd096dcf18e80b894bf7e25ce"), target_type=ObjectType.REVISION, synthetic=False, - author=Person.from_fullname(b"Foo ",), + author=Person.from_fullname( + b"Foo ", + ), date=TimestampWithTimezone( timestamp=Timestamp(seconds=1640191027, microseconds=0), offset_bytes=b"-0000", ), ) # Mess with the offset (other) raw_manifest2 = raw_manifest.replace(b"+0200", b"+200") tag = dulwich.objects.Tag.from_raw_string(b"tag", raw_manifest2) assert converters.dulwich_tag_to_release(tag) == Release( name=b"blah", message=b"some release message", target=hash_to_bytes("641fb6e08ddb2e4fd096dcf18e80b894bf7e25ce"), target_type=ObjectType.REVISION, synthetic=False, - author=Person.from_fullname(b"Foo ",), + author=Person.from_fullname( + b"Foo ", + ), date=TimestampWithTimezone( timestamp=Timestamp(seconds=1640191027, microseconds=0), offset_bytes=b"+200", ), ) # Mess with the rest of the manifest raw_manifest2 = raw_manifest.replace( b"641fb6e08ddb2e4fd096dcf18e80b894bf7e25ce", b"641FB6E08DDB2E4FD096DCF18E80B894BF7E25CE", ) tag = dulwich.objects.Tag.from_raw_string(b"tag", raw_manifest2) assert converters.dulwich_tag_to_release(tag) == Release( name=b"blah", message=b"some release message", target=hash_to_bytes("641fb6e08ddb2e4fd096dcf18e80b894bf7e25ce"), target_type=ObjectType.REVISION, synthetic=False, - author=Person.from_fullname(b"Foo ",), + author=Person.from_fullname( + b"Foo ", + ), date=TimestampWithTimezone( timestamp=Timestamp(seconds=1640191027, microseconds=0), offset_bytes=b"+0200", ), raw_manifest=b"tag 136\x00" + raw_manifest2, ) diff --git a/swh/loader/git/tests/test_from_disk.py b/swh/loader/git/tests/test_from_disk.py index 096ac90..ccb294f 100644 --- a/swh/loader/git/tests/test_from_disk.py +++ b/swh/loader/git/tests/test_from_disk.py @@ -1,548 +1,553 @@ # Copyright (C) 2018-2021 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import copy import datetime import os.path import dulwich.objects import dulwich.porcelain import dulwich.repo import pytest from swh.loader.git.from_disk import GitLoaderFromArchive, GitLoaderFromDisk from swh.loader.tests import ( assert_last_visit_matches, check_snapshot, get_stats, prepare_repository_from_archive, ) from swh.model.hashutil import bytehex_to_hash, hash_to_bytes from swh.model.model import ObjectType, Release, Snapshot, SnapshotBranch, TargetType from swh.storage.algos.snapshot import snapshot_get_all_branches SNAPSHOT1 = Snapshot( id=hash_to_bytes("a23699280a82a043f8c0994cf1631b568f716f95"), branches={ b"HEAD": SnapshotBranch( - target=b"refs/heads/master", target_type=TargetType.ALIAS, + target=b"refs/heads/master", + target_type=TargetType.ALIAS, ), b"refs/heads/master": SnapshotBranch( target=hash_to_bytes("2f01f5ca7e391a2f08905990277faf81e709a649"), target_type=TargetType.REVISION, ), b"refs/heads/branch1": SnapshotBranch( target=hash_to_bytes("b0a77609903f767a2fd3d769904ef9ef68468b87"), target_type=TargetType.REVISION, ), b"refs/heads/branch2": SnapshotBranch( target=hash_to_bytes("bd746cd1913721b269b395a56a97baf6755151c2"), target_type=TargetType.REVISION, ), b"refs/tags/branch2-after-delete": SnapshotBranch( target=hash_to_bytes("bd746cd1913721b269b395a56a97baf6755151c2"), target_type=TargetType.REVISION, ), b"refs/tags/branch2-before-delete": SnapshotBranch( target=hash_to_bytes("1135e94ccf73b5f9bd6ef07b3fa2c5cc60bba69b"), target_type=TargetType.REVISION, ), }, ) # directory hashes obtained with: # gco b6f40292c4e94a8f7e7b4aff50e6c7429ab98e2a # swh-hashtree --ignore '.git' --path . # gco 2f01f5ca7e391a2f08905990277faf81e709a649 # swh-hashtree --ignore '.git' --path . # gco bcdc5ebfde1a3cd6c96e0c2ea4eed19c13208777 # swh-hashtree --ignore '.git' --path . # gco 1135e94ccf73b5f9bd6ef07b3fa2c5cc60bba69b # swh-hashtree --ignore '.git' --path . # gco 79f65ac75f79dda6ff03d66e1242702ab67fb51c # swh-hashtree --ignore '.git' --path . # gco b0a77609903f767a2fd3d769904ef9ef68468b87 # swh-hashtree --ignore '.git' --path . # gco bd746cd1913721b269b395a56a97baf6755151c2 # swh-hashtree --ignore '.git' --path . REVISIONS1 = { "b6f40292c4e94a8f7e7b4aff50e6c7429ab98e2a": ( "40dbdf55dfd4065422462cc74a949254aefa972e" ), "2f01f5ca7e391a2f08905990277faf81e709a649": ( "e1d0d894835f91a0f887a4bc8b16f81feefdfbd5" ), "bcdc5ebfde1a3cd6c96e0c2ea4eed19c13208777": ( "b43724545b4759244bb54be053c690649161411c" ), "1135e94ccf73b5f9bd6ef07b3fa2c5cc60bba69b": ( "fbf70528223d263661b5ad4b80f26caf3860eb8e" ), "79f65ac75f79dda6ff03d66e1242702ab67fb51c": ( "5df34ec74d6f69072d9a0a6677d8efbed9b12e60" ), "b0a77609903f767a2fd3d769904ef9ef68468b87": ( "9ca0c7d6ffa3f9f0de59fd7912e08f11308a1338" ), "bd746cd1913721b269b395a56a97baf6755151c2": ( "e1d0d894835f91a0f887a4bc8b16f81feefdfbd5" ), } class CommonGitLoaderTests: """Common tests for all git loaders.""" def test_load(self): """Loads a simple repository (made available by `setUp()`), and checks everything was added in the storage.""" res = self.loader.load() assert res == {"status": "eventful"} assert_last_visit_matches( self.loader.storage, self.repo_url, status="full", type="git", snapshot=SNAPSHOT1.id, ) stats = get_stats(self.loader.storage) assert stats == { "content": 4, "directory": 7, "origin": 1, "origin_visit": 1, "release": 0, "revision": 7, "skipped_content": 0, "snapshot": 1, } check_snapshot(SNAPSHOT1, self.loader.storage) def test_load_unchanged(self): """Checks loading a repository a second time does not add any extra data.""" res = self.loader.load() assert res == {"status": "eventful"} assert_last_visit_matches( self.loader.storage, self.repo_url, status="full", type="git", snapshot=SNAPSHOT1.id, ) stats0 = get_stats(self.loader.storage) assert stats0 == { "content": 4, "directory": 7, "origin": 1, "origin_visit": 1, "release": 0, "revision": 7, "skipped_content": 0, "snapshot": 1, } res = self.loader.load() assert res == {"status": "uneventful"} stats1 = get_stats(self.loader.storage) expected_stats = copy.deepcopy(stats0) expected_stats["origin_visit"] += 1 assert stats1 == expected_stats check_snapshot(SNAPSHOT1, self.loader.storage) assert_last_visit_matches( self.loader.storage, self.repo_url, status="full", type="git", snapshot=SNAPSHOT1.id, ) def test_load_visit_without_snapshot_so_status_failed(self): # unfortunately, monkey-patch the hard way, self.loader is already instantiated # (patching won't work self.loader is already instantiated) # Make get_contents fail for some reason self.loader.get_contents = None res = self.loader.load() assert res == {"status": "failed"} assert_last_visit_matches( self.loader.storage, self.repo_url, status="failed", type="git", snapshot=None, ) def test_load_visit_with_snapshot_so_status_partial(self): # unfortunately, monkey-patch the hard way, self.loader is already instantiated # (patching won't work self.loader is already instantiated) # fake store_metadata raising for some reason, so we could have a snapshot id # at this point in time self.loader.store_metadata = None # fake having a snapshot so the visit status is partial self.loader.loaded_snapshot_id = hash_to_bytes( "a23699280a82a043f8c0994cf1631b568f716f95" ) res = self.loader.load() assert res == {"status": "failed"} assert_last_visit_matches( self.loader.storage, self.repo_url, status="partial", type="git", snapshot=None, ) class FullGitLoaderTests(CommonGitLoaderTests): """Tests for GitLoader (from disk or not). Includes the common ones, and - add others that only work with a local dir. + add others that only work with a local dir. """ def test_load_changed(self): """Loads a repository, makes some changes by adding files, commits, and merges, load it again, and check the storage contains everything it should.""" # Initial load res = self.loader.load() assert res == {"status": "eventful"} stats0 = get_stats(self.loader.storage) assert stats0 == { "content": 4, "directory": 7, "origin": 1, "origin_visit": 1, "release": 0, "revision": 7, "skipped_content": 0, "snapshot": 1, } # Load with a new file + revision with open(os.path.join(self.destination_path, "hello.py"), "a") as fd: fd.write("print('Hello world')\n") self.repo.stage([b"hello.py"]) new_revision = self.repo.do_commit(b"Hello world\n").decode() new_dir = "85dae072a5aa9923ffa7a7568f819ff21bf49858" assert self.repo[new_revision.encode()].tree == new_dir.encode() revisions = REVISIONS1.copy() assert new_revision not in revisions revisions[new_revision] = new_dir res = self.loader.load() assert res == {"status": "eventful"} stats1 = get_stats(self.loader.storage) expected_stats = copy.deepcopy(stats0) # did one new visit expected_stats["origin_visit"] += 1 # with one more of the following objects expected_stats["content"] += 1 expected_stats["directory"] += 1 expected_stats["revision"] += 1 # concluding into 1 new snapshot expected_stats["snapshot"] += 1 assert stats1 == expected_stats visit_status = assert_last_visit_matches( self.loader.storage, self.repo_url, status="full", type="git" ) assert visit_status.snapshot is not None snapshot_id = visit_status.snapshot snapshot = snapshot_get_all_branches(self.loader.storage, snapshot_id) branches = snapshot.branches assert branches[b"HEAD"] == SnapshotBranch( - target=b"refs/heads/master", target_type=TargetType.ALIAS, + target=b"refs/heads/master", + target_type=TargetType.ALIAS, ) assert branches[b"refs/heads/master"] == SnapshotBranch( - target=hash_to_bytes(new_revision), target_type=TargetType.REVISION, + target=hash_to_bytes(new_revision), + target_type=TargetType.REVISION, ) # Merge branch1 into HEAD. current = self.repo[b"HEAD"] branch1 = self.repo[b"refs/heads/branch1"] merged_tree = dulwich.objects.Tree() for item in self.repo[current.tree].items(): merged_tree.add(*item) for item in self.repo[branch1.tree].items(): merged_tree.add(*item) merged_dir_id = "dab8a37df8db8666d4e277bef9a546f585b5bedd" assert merged_tree.id.decode() == merged_dir_id self.repo.object_store.add_object(merged_tree) merge_commit = self.repo.do_commit( b"merge.\n", tree=merged_tree.id, merge_heads=[branch1.id] ) assert merge_commit.decode() not in revisions revisions[merge_commit.decode()] = merged_tree.id.decode() res = self.loader.load() assert res == {"status": "eventful"} stats2 = get_stats(self.loader.storage) expected_stats = copy.deepcopy(stats1) # one more visit expected_stats["origin_visit"] += 1 # with 1 new directory and revision expected_stats["directory"] += 1 expected_stats["revision"] += 1 # concluding into 1 new snapshot expected_stats["snapshot"] += 1 assert stats2 == expected_stats visit_status = assert_last_visit_matches( self.loader.storage, self.repo_url, status="full", type="git" ) assert visit_status.snapshot is not None merge_snapshot_id = visit_status.snapshot assert merge_snapshot_id != snapshot_id merge_snapshot = snapshot_get_all_branches( self.loader.storage, merge_snapshot_id ) merge_branches = merge_snapshot.branches assert merge_branches[b"HEAD"] == SnapshotBranch( - target=b"refs/heads/master", target_type=TargetType.ALIAS, + target=b"refs/heads/master", + target_type=TargetType.ALIAS, ) assert merge_branches[b"refs/heads/master"] == SnapshotBranch( target=hash_to_bytes(merge_commit.decode()), target_type=TargetType.REVISION, ) def test_load_filter_branches(self): filtered_branches = {b"refs/pull/42/merge"} unfiltered_branches = {b"refs/pull/42/head"} # Add branches to the repository on disk; some should be filtered by # the loader, some should not. for branch_name in filtered_branches | unfiltered_branches: self.repo[branch_name] = self.repo[b"refs/heads/master"] # Generate the expected snapshot from SNAPSHOT1 (which is the original # state of the git repo)... branches = dict(SNAPSHOT1.branches) # ... and the unfiltered_branches, which are all pointing to the same # commit as "refs/heads/master". for branch_name in unfiltered_branches: branches[branch_name] = branches[b"refs/heads/master"] expected_snapshot = Snapshot(branches=branches) # Load the modified repository res = self.loader.load() assert res == {"status": "eventful"} check_snapshot(expected_snapshot, self.loader.storage) assert_last_visit_matches( self.loader.storage, self.repo_url, status="full", type="git", snapshot=expected_snapshot.id, ) def test_load_dangling_symref(self): with open(os.path.join(self.destination_path, ".git/HEAD"), "wb") as f: f.write(b"ref: refs/heads/dangling-branch\n") res = self.loader.load() assert res == {"status": "eventful"} visit_status = assert_last_visit_matches( self.loader.storage, self.repo_url, status="full", type="git" ) snapshot_id = visit_status.snapshot assert snapshot_id is not None snapshot = snapshot_get_all_branches(self.loader.storage, snapshot_id) branches = snapshot.branches assert branches[b"HEAD"] == SnapshotBranch( - target=b"refs/heads/dangling-branch", target_type=TargetType.ALIAS, + target=b"refs/heads/dangling-branch", + target_type=TargetType.ALIAS, ) assert branches[b"refs/heads/dangling-branch"] is None stats = get_stats(self.loader.storage) assert stats == { "content": 4, "directory": 7, "origin": 1, "origin_visit": 1, "release": 0, "revision": 7, "skipped_content": 0, "snapshot": 1, } def test_load_empty_tree(self): empty_dir_id = "4b825dc642cb6eb9a060e54bf8d69288fbee4904" # Check the empty tree does not already exist for some reason # (it would make this test pointless) assert list( self.loader.storage.directory_missing([hash_to_bytes(empty_dir_id)]) ) == [hash_to_bytes(empty_dir_id)] empty_tree = dulwich.objects.Tree() assert empty_tree.id.decode() == empty_dir_id self.repo.object_store.add_object(empty_tree) self.repo.do_commit(b"remove all bugs\n", tree=empty_tree.id) res = self.loader.load() assert res == {"status": "eventful"} assert ( list(self.loader.storage.directory_missing([hash_to_bytes(empty_dir_id)])) == [] ) results = self.loader.storage.directory_get_entries(hash_to_bytes(empty_dir_id)) assert results.next_page_token is None assert results.results == [] def test_load_tag(self): with open(os.path.join(self.destination_path, "hello.py"), "a") as fd: fd.write("print('Hello world')\n") self.repo.stage([b"hello.py"]) new_revision = self.repo.do_commit(b"Hello world\n") # Newer Dulwich versions always add a \n to tag messages. if dulwich.__version__ >= (0, 20, 22): message = b"First release!" else: message = b"First release!\n" dulwich.porcelain.tag_create( self.repo, b"v1.0.0", message=message, annotated=True, objectish=new_revision, ) res = self.loader.load() assert res == {"status": "eventful"} branches = self.loader.storage.snapshot_get_branches(self.loader.snapshot.id) branch = branches["branches"][b"refs/tags/v1.0.0"] assert branch.target_type == TargetType.RELEASE release = self.loader.storage.release_get([branch.target])[0] assert release.date is not None assert release.author is not None assert release == Release( name=b"v1.0.0", message=b"First release!\n", target_type=ObjectType.REVISION, target=bytehex_to_hash(new_revision), author=release.author, date=release.date, synthetic=False, ) def test_load_tag_minimal(self): with open(os.path.join(self.destination_path, "hello.py"), "a") as fd: fd.write("print('Hello world')\n") self.repo.stage([b"hello.py"]) new_revision = self.repo.do_commit(b"Hello world\n") # dulwich.porcelain.tag_create doesn't allow creating tags without # a tagger or a date, so we have to create it "manually" tag = dulwich.objects.Tag() tag.message = b"First release!\n" tag.name = b"v1.0.0" tag.object = (dulwich.objects.Commit, new_revision) self.repo.object_store.add_object(tag) self.repo[b"refs/tags/v1.0.0"] = tag.id res = self.loader.load() assert res == {"status": "eventful"} branches = self.loader.storage.snapshot_get_branches(self.loader.snapshot.id) branch = branches["branches"][b"refs/tags/v1.0.0"] assert branch.target_type == TargetType.RELEASE release = self.loader.storage.release_get([branch.target])[0] assert release == Release( id=bytehex_to_hash(tag.id), name=b"v1.0.0", message=b"First release!\n", target_type=ObjectType.REVISION, target=bytehex_to_hash(new_revision), synthetic=False, ) class TestGitLoaderFromDisk(FullGitLoaderTests): """Prepare a git directory repository to be loaded through a GitLoaderFromDisk. This tests all git loader scenario. """ @pytest.fixture(autouse=True) def init(self, swh_storage, datadir, tmp_path): archive_name = "testrepo" archive_path = os.path.join(datadir, f"{archive_name}.tgz") tmp_path = str(tmp_path) self.repo_url = prepare_repository_from_archive( archive_path, archive_name, tmp_path=tmp_path ) self.destination_path = os.path.join(tmp_path, archive_name) self.loader = GitLoaderFromDisk( swh_storage, url=self.repo_url, visit_date=datetime.datetime( 2016, 5, 3, 15, 16, 32, tzinfo=datetime.timezone.utc ), directory=self.destination_path, ) self.repo = dulwich.repo.Repo(self.destination_path) class TestGitLoaderFromArchive(CommonGitLoaderTests): """Tests for GitLoaderFromArchive. Only tests common scenario.""" @pytest.fixture(autouse=True) def init(self, swh_storage, datadir, tmp_path): archive_name = "testrepo" archive_path = os.path.join(datadir, f"{archive_name}.tgz") self.repo_url = archive_path self.loader = GitLoaderFromArchive( swh_storage, url=self.repo_url, archive_path=archive_path, visit_date=datetime.datetime( 2016, 5, 3, 15, 16, 32, tzinfo=datetime.timezone.utc ), ) diff --git a/swh/loader/git/tests/test_loader.py b/swh/loader/git/tests/test_loader.py index 83a3059..0b0baaf 100644 --- a/swh/loader/git/tests/test_loader.py +++ b/swh/loader/git/tests/test_loader.py @@ -1,327 +1,328 @@ # Copyright (C) 2018-2021 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from functools import partial from http.server import HTTPServer, SimpleHTTPRequestHandler import os import subprocess from tempfile import SpooledTemporaryFile from threading import Thread from dulwich.errors import GitProtocolError, NotGitRepository, ObjectFormatException from dulwich.porcelain import push import dulwich.repo import pytest from swh.loader.git import dumb from swh.loader.git.loader import GitLoader from swh.loader.git.tests.test_from_disk import FullGitLoaderTests from swh.loader.tests import ( assert_last_visit_matches, get_stats, prepare_repository_from_archive, ) class CommonGitLoaderNotFound: @pytest.fixture(autouse=True) def __inject_fixtures(self, mocker): - """Inject required fixtures in unittest.TestCase class - - """ + """Inject required fixtures in unittest.TestCase class""" self.mocker = mocker @pytest.mark.parametrize( "failure_exception", [ GitProtocolError("Repository unavailable"), # e.g DMCA takedown GitProtocolError("Repository not found"), GitProtocolError("unexpected http resp 401"), NotGitRepository("not a git repo"), ], ) def test_load_visit_not_found(self, failure_exception): - """Ingesting an unknown url result in a visit with not_found status - - """ + """Ingesting an unknown url result in a visit with not_found status""" # simulate an initial communication error (e.g no repository found, ...) mock = self.mocker.patch( "swh.loader.git.loader.GitLoader.fetch_pack_from_origin" ) mock.side_effect = failure_exception res = self.loader.load() assert res == {"status": "uneventful"} assert_last_visit_matches( self.loader.storage, self.repo_url, status="not_found", type="git", snapshot=None, ) @pytest.mark.parametrize( "failure_exception", - [IOError, ObjectFormatException, OSError, ValueError, GitProtocolError,], + [ + IOError, + ObjectFormatException, + OSError, + ValueError, + GitProtocolError, + ], ) def test_load_visit_failure(self, failure_exception): - """Failing during the fetch pack step result in failing visit - - """ + """Failing during the fetch pack step result in failing visit""" # simulate a fetch communication error after the initial connection # server error (e.g IOError, ObjectFormatException, ...) mock = self.mocker.patch( "swh.loader.git.loader.GitLoader.fetch_pack_from_origin" ) mock.side_effect = failure_exception("failure") res = self.loader.load() assert res == {"status": "failed"} assert_last_visit_matches( self.loader.storage, self.repo_url, status="failed", type="git", snapshot=None, ) class TestGitLoader(FullGitLoaderTests, CommonGitLoaderNotFound): """Prepare a git directory repository to be loaded through a GitLoader. This tests all git loader scenario. """ @pytest.fixture(autouse=True) def init(self, swh_storage, datadir, tmp_path): archive_name = "testrepo" archive_path = os.path.join(datadir, f"{archive_name}.tgz") tmp_path = str(tmp_path) self.repo_url = prepare_repository_from_archive( archive_path, archive_name, tmp_path=tmp_path ) self.destination_path = os.path.join(tmp_path, archive_name) self.loader = GitLoader(swh_storage, self.repo_url) self.repo = dulwich.repo.Repo(self.destination_path) class TestGitLoader2(FullGitLoaderTests, CommonGitLoaderNotFound): """Mostly the same loading scenario but with a base-url different than the repo-url. To walk slightly different paths, the end result should stay the same. """ @pytest.fixture(autouse=True) def init(self, swh_storage, datadir, tmp_path): archive_name = "testrepo" archive_path = os.path.join(datadir, f"{archive_name}.tgz") tmp_path = str(tmp_path) self.repo_url = prepare_repository_from_archive( archive_path, archive_name, tmp_path=tmp_path ) self.destination_path = os.path.join(tmp_path, archive_name) base_url = f"base://{self.repo_url}" self.loader = GitLoader(swh_storage, self.repo_url, base_url=base_url) self.repo = dulwich.repo.Repo(self.destination_path) class DumbGitLoaderTestBase(FullGitLoaderTests): - """Prepare a git repository to be loaded using the HTTP dumb transfer protocol. - """ + """Prepare a git repository to be loaded using the HTTP dumb transfer protocol.""" @pytest.fixture(autouse=True) def init(self, swh_storage, datadir, tmp_path): # remove any proxy settings in order to successfully spawn a local HTTP server http_proxy = os.environ.get("http_proxy") https_proxy = os.environ.get("https_proxy") if http_proxy: del os.environ["http_proxy"] if http_proxy: del os.environ["https_proxy"] # prepare test base repository using smart transfer protocol archive_name = "testrepo" archive_path = os.path.join(datadir, f"{archive_name}.tgz") tmp_path = str(tmp_path) base_repo_url = prepare_repository_from_archive( archive_path, archive_name, tmp_path=tmp_path ) destination_path = os.path.join(tmp_path, archive_name) self.destination_path = destination_path with_pack_files = self.with_pack_files if with_pack_files: # create a bare clone of that repository in another folder, # all objects will be contained in one or two pack files in that case http_root_dir = tmp_path repo_name = archive_name + "_bare" bare_repo_path = os.path.join(http_root_dir, repo_name) subprocess.run( - ["git", "clone", "--bare", base_repo_url, bare_repo_path], check=True, + ["git", "clone", "--bare", base_repo_url, bare_repo_path], + check=True, ) else: # otherwise serve objects from the bare repository located in # the .git folder of the base repository http_root_dir = destination_path repo_name = ".git" bare_repo_path = os.path.join(http_root_dir, repo_name) # spawn local HTTP server that will serve the bare repository files hostname = "localhost" handler = partial(SimpleHTTPRequestHandler, directory=http_root_dir) httpd = HTTPServer((hostname, 0), handler, bind_and_activate=True) def serve_forever(httpd): with httpd: httpd.serve_forever() thread = Thread(target=serve_forever, args=(httpd,)) thread.start() repo = dulwich.repo.Repo(self.destination_path) class DumbGitLoaderTest(GitLoader): def load(self): """ Override load method to ensure the bare repository will be synchronized with the base one as tests can modify its content. """ if with_pack_files: # ensure HEAD ref will be the same for both repositories with open(os.path.join(bare_repo_path, "HEAD"), "wb") as fw: with open( os.path.join(destination_path, ".git/HEAD"), "rb" ) as fr: head_ref = fr.read() fw.write(head_ref) # push possibly modified refs in the base repository to the bare one for ref in repo.refs.allkeys(): if ref != b"HEAD" or head_ref in repo.refs: push( repo, remote_location=f"file://{bare_repo_path}", refspecs=ref, ) # generate or update the info/refs file used in dumb protocol subprocess.run( - ["git", "-C", bare_repo_path, "update-server-info"], check=True, + ["git", "-C", bare_repo_path, "update-server-info"], + check=True, ) return super().load() # bare repository with dumb protocol only URL self.repo_url = f"http://{httpd.server_name}:{httpd.server_port}/{repo_name}" self.loader = DumbGitLoaderTest(swh_storage, self.repo_url) self.repo = repo yield # shutdown HTTP server httpd.shutdown() thread.join() # restore HTTP proxy settings if any if http_proxy: os.environ["http_proxy"] = http_proxy if https_proxy: os.environ["https_proxy"] = https_proxy @pytest.mark.parametrize( "failure_exception", [AttributeError, NotImplementedError, ValueError] ) def test_load_despite_dulwich_exception(self, mocker, failure_exception): """Checks repository can still be loaded when dulwich raises exception when encountering a repository with dumb transfer protocol. """ fetch_pack_from_origin = mocker.patch( "swh.loader.git.loader.GitLoader.fetch_pack_from_origin" ) fetch_pack_from_origin.side_effect = failure_exception("failure") res = self.loader.load() assert res == {"status": "eventful"} stats = get_stats(self.loader.storage) assert stats == { "content": 4, "directory": 7, "origin": 1, "origin_visit": 1, "release": 0, "revision": 7, "skipped_content": 0, "snapshot": 1, } def test_load_empty_repository(self, mocker): class GitObjectsFetcherNoRefs(dumb.GitObjectsFetcher): def _get_refs(self): return {} mocker.patch.object(dumb, "GitObjectsFetcher", GitObjectsFetcherNoRefs) res = self.loader.load() assert res == {"status": "uneventful"} stats = get_stats(self.loader.storage) assert stats == { "content": 0, "directory": 0, "origin": 1, "origin_visit": 1, "release": 0, "revision": 0, "skipped_content": 0, "snapshot": 1, } class TestDumbGitLoaderWithPack(DumbGitLoaderTestBase): @classmethod def setup_class(cls): cls.with_pack_files = True def test_load_with_missing_pack(self, mocker): """Some dumb git servers might reference a no longer existing pack file while it is possible to load a repository without it. """ class GitObjectsFetcherMissingPack(dumb.GitObjectsFetcher): def _http_get(self, path: str) -> SpooledTemporaryFile: buffer = super()._http_get(path) if path == "objects/info/packs": # prepend a non existing pack to the returned packs list packs = buffer.read().decode("utf-8") buffer.seek(0) buffer.write( ( "P pack-a70762ba1a901af3a0e76de02fc3a99226842745.pack\n" + packs ).encode() ) buffer.flush() buffer.seek(0) return buffer mocker.patch.object(dumb, "GitObjectsFetcher", GitObjectsFetcherMissingPack) res = self.loader.load() assert res == {"status": "eventful"} class TestDumbGitLoaderWithoutPack(DumbGitLoaderTestBase): @classmethod def setup_class(cls): cls.with_pack_files = False diff --git a/swh/loader/git/tests/test_tasks.py b/swh/loader/git/tests/test_tasks.py index 39c7b3f..0ec2596 100644 --- a/swh/loader/git/tests/test_tasks.py +++ b/swh/loader/git/tests/test_tasks.py @@ -1,66 +1,69 @@ # Copyright (C) 2018-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information def test_git_loader( mocker, swh_config, swh_scheduler_celery_app, swh_scheduler_celery_worker ): mock_loader = mocker.patch("swh.loader.git.loader.GitLoader.load") mock_loader.return_value = {"status": "eventful"} res = swh_scheduler_celery_app.send_task( - "swh.loader.git.tasks.UpdateGitRepository", kwargs={"url": "origin_url",} + "swh.loader.git.tasks.UpdateGitRepository", + kwargs={ + "url": "origin_url", + }, ) assert res res.wait() assert res.successful() assert res.result == {"status": "eventful"} mock_loader.assert_called_once_with() def test_git_loader_from_disk( mocker, swh_config, swh_scheduler_celery_app, swh_scheduler_celery_worker ): mock_loader = mocker.patch("swh.loader.git.from_disk.GitLoaderFromDisk.load") mock_loader.return_value = {"status": "uneventful"} res = swh_scheduler_celery_app.send_task( "swh.loader.git.tasks.LoadDiskGitRepository", kwargs={ "url": "origin_url2", "directory": "/some/repo", "date": "2018-12-10 00:00", }, ) assert res res.wait() assert res.successful() assert res.result == {"status": "uneventful"} mock_loader.assert_called_once_with() def test_git_loader_from_archive( mocker, swh_config, swh_scheduler_celery_app, swh_scheduler_celery_worker ): mock_loader = mocker.patch("swh.loader.git.from_disk.GitLoaderFromArchive.load") mock_loader.return_value = {"status": "failed"} res = swh_scheduler_celery_app.send_task( "swh.loader.git.tasks.UncompressAndLoadDiskGitRepository", kwargs={ "url": "origin_url3", "archive_path": "/some/repo", "date": "2017-01-10 00:00", }, ) assert res res.wait() assert res.successful() assert res.result == {"status": "failed"} mock_loader.assert_called_once_with() diff --git a/swh/loader/git/tests/test_utils.py b/swh/loader/git/tests/test_utils.py index 0f50bf5..97459c3 100644 --- a/swh/loader/git/tests/test_utils.py +++ b/swh/loader/git/tests/test_utils.py @@ -1,54 +1,50 @@ # Copyright (C) 2015-2021 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import pytest from swh.loader.git import utils class TestUtils: def test_check_date_time(self): - """A long as datetime is fine, date time check does not raise - - """ + """A long as datetime is fine, date time check does not raise""" for e in range(32, 37): - ts = 2 ** e + ts = 2**e utils.check_date_time(ts) def test_check_date_time_empty_value(self): assert utils.check_date_time(None) is None def test_check_date_time_raises(self): - """From a give threshold, check will no longer works. - - """ + """From a give threshold, check will no longer works.""" exp = 38 - timestamp = 2 ** exp + timestamp = 2**exp with pytest.raises(ValueError, match=".*is out of range.*"): utils.check_date_time(timestamp) def test_ignore_branch_name(): branches = { b"HEAD", b"refs/heads/master", b"refs/{}", b"refs/pull/10/head", b"refs/pull/100/head", b"refs/pull/xyz/merge", # auto-merged GitHub pull requests filtered out b"refs/^{}", # Peeled refs filtered out } actual_branches = {b for b in branches if not utils.ignore_branch_name(b)} assert actual_branches == set( [ b"HEAD", b"refs/heads/master", b"refs/{}", b"refs/pull/10/head", b"refs/pull/100/head", ] )