diff --git a/swh/loader/git/loader.py b/swh/loader/git/loader.py index fa7c9ec..bdb302b 100644 --- a/swh/loader/git/loader.py +++ b/swh/loader/git/loader.py @@ -1,317 +1,360 @@ # Copyright (C) 2015-2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import datetime import dulwich.repo import os import shutil from dulwich.errors import ObjectFormatException, EmptyFileException from collections import defaultdict from swh.model import hashutil from swh.loader.core.loader import UnbufferedLoader from . import converters, utils class GitLoader(UnbufferedLoader): """Load a git repository from a directory. """ CONFIG_BASE_FILENAME = 'loader/git-loader' def __init__(self, config=None): super().__init__(logging_class='swh.loader.git.Loader', config=config) def _prepare_origin_visit(self, origin_url, visit_date): self.origin_url = origin_url self.origin = converters.origin_url_to_origin(self.origin_url) self.visit_date = visit_date def prepare_origin_visit(self, origin_url, directory, visit_date): self._prepare_origin_visit(origin_url, visit_date) def prepare(self, origin_url, directory, visit_date): self.repo = dulwich.repo.Repo(directory) def iter_objects(self): object_store = self.repo.object_store for pack in object_store.packs: objs = list(pack.index.iterentries()) objs.sort(key=lambda x: x[1]) for sha, offset, crc32 in objs: yield hashutil.hash_to_bytehex(sha) yield from object_store._iter_loose_objects() yield from object_store._iter_alternate_objects() def _check(self, obj): """Check the object's repository representation. If any errors in check exists, an ObjectFormatException is raised. Args: obj (object): Dulwich object read from the repository. """ obj.check() from dulwich.objects import Commit, Tag try: # For additional checks on dulwich objects with date # for now, only checks on *time if isinstance(obj, Commit): commit_time = obj._commit_time utils.check_date_time(commit_time) author_time = obj._author_time utils.check_date_time(author_time) elif isinstance(obj, Tag): tag_time = obj._tag_time utils.check_date_time(tag_time) except Exception as e: raise ObjectFormatException(e) def get_object(self, oid): """Given an object id, return the object if it is found and not malformed in some way. Args: oid (bytes): the object's identifier Returns: The object if found without malformation """ try: # some errors are raised when reading the object obj = self.repo[oid] # some we need to check ourselves self._check(obj) except KeyError: _id = oid.decode('utf-8') self.log.warn('object %s not found, skipping' % _id, extra={ 'swh_type': 'swh_loader_git_missing_object', 'swh_object_id': _id, 'origin_id': self.origin_id, }) return None except ObjectFormatException: _id = oid.decode('utf-8') self.log.warn('object %s malformed, skipping' % _id, extra={ 'swh_type': 'swh_loader_git_missing_object', 'swh_object_id': _id, 'origin_id': self.origin_id, }) return None except EmptyFileException: _id = oid.decode('utf-8') self.log.warn('object %s corrupted (empty file), skipping' % _id, extra={ 'swh_type': 'swh_loader_git_missing_object', 'swh_object_id': _id, 'origin_id': self.origin_id, }) else: return obj def fetch_data(self): """Fetch the data from the data source""" self.previous_snapshot = self.storage.snapshot_get_latest( self.origin_id ) type_to_ids = defaultdict(list) for oid in self.iter_objects(): obj = self.get_object(oid) if not obj: continue type_name = obj.type_name type_to_ids[type_name].append(oid) self.type_to_ids = type_to_ids def has_contents(self): """Checks whether we need to load contents""" return bool(self.type_to_ids[b'blob']) def get_content_ids(self): """Get the content identifiers from the git repository""" for oid in self.type_to_ids[b'blob']: yield converters.dulwich_blob_to_content_id(self.repo[oid]) def get_contents(self): """Get the contents that need to be loaded""" max_content_size = self.config['content_size_limit'] missing_contents = set(self.storage.content_missing( self.get_content_ids(), 'sha1_git')) for oid in missing_contents: yield converters.dulwich_blob_to_content( self.repo[hashutil.hash_to_bytehex(oid)], log=self.log, max_content_size=max_content_size, origin_id=self.origin_id) def has_directories(self): """Checks whether we need to load directories""" return bool(self.type_to_ids[b'tree']) def get_directory_ids(self): """Get the directory identifiers from the git repository""" return (hashutil.hash_to_bytes(id.decode()) for id in self.type_to_ids[b'tree']) def get_directories(self): """Get the directories that need to be loaded""" missing_dirs = set(self.storage.directory_missing( sorted(self.get_directory_ids()))) for oid in missing_dirs: yield converters.dulwich_tree_to_directory( self.repo[hashutil.hash_to_bytehex(oid)], log=self.log) def has_revisions(self): """Checks whether we need to load revisions""" return bool(self.type_to_ids[b'commit']) def get_revision_ids(self): """Get the revision identifiers from the git repository""" return (hashutil.hash_to_bytes(id.decode()) for id in self.type_to_ids[b'commit']) def get_revisions(self): """Get the revisions that need to be loaded""" missing_revs = set(self.storage.revision_missing( sorted(self.get_revision_ids()))) for oid in missing_revs: yield converters.dulwich_commit_to_revision( self.repo[hashutil.hash_to_bytehex(oid)], log=self.log) def has_releases(self): """Checks whether we need to load releases""" return bool(self.type_to_ids[b'tag']) def get_release_ids(self): """Get the release identifiers from the git repository""" return (hashutil.hash_to_bytes(id.decode()) for id in self.type_to_ids[b'tag']) def get_releases(self): """Get the releases that need to be loaded""" missing_rels = set(self.storage.release_missing( sorted(self.get_release_ids()))) for oid in missing_rels: yield converters.dulwich_tag_to_release( self.repo[hashutil.hash_to_bytehex(oid)], log=self.log) def get_snapshot(self): """Turn the list of branches into a snapshot to load""" branches = {} for ref, target in self.repo.refs.as_dict().items(): obj = self.get_object(target) if obj: branches[ref] = { 'target': hashutil.bytehex_to_hash(target), 'target_type': converters.DULWICH_TYPES[obj.type_name], } else: branches[ref] = None self.snapshot = converters.branches_to_snapshot(branches) return self.snapshot def get_fetch_history_result(self): """Return the data to store in fetch_history for the current loader""" return { 'contents': len(self.type_to_ids[b'blob']), 'directories': len(self.type_to_ids[b'tree']), 'revisions': len(self.type_to_ids[b'commit']), 'releases': len(self.type_to_ids[b'tag']), } def save_data(self): """We already have the data locally, no need to save it""" pass def load_status(self): """The load was eventful if the current occurrences are different to the ones we retrieved at the beginning of the run""" eventful = False if self.previous_snapshot: eventful = self.snapshot['id'] != self.previous_snapshot['id'] else: eventful = bool(self.snapshot['branches']) return {'status': ('eventful' if eventful else 'uneventful')} class GitLoaderFromArchive(GitLoader): """Load a git repository from an archive. + This loader ingests a git repository compressed into an archive. + The supported archive formats are ``.zip`` and ``.tar.gz``. + + From an input tarball named ``my-git-repo.zip``, the following layout is + expected in it:: + + my-git-repo/ + ├── .git + │ ├── branches + │ ├── COMMIT_EDITMSG + │ ├── config + │ ├── description + │ ├── HEAD + ... + + Nevertheless, the loader is able to ingest tarballs with the following + layouts too:: + + . + ├── .git + │ ├── branches + │ ├── COMMIT_EDITMSG + │ ├── config + │ ├── description + │ ├── HEAD + ... + + or:: + + other-repo-name/ + ├── .git + │ ├── branches + │ ├── COMMIT_EDITMSG + │ ├── config + │ ├── description + │ ├── HEAD + ... + """ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.temp_dir = self.repo_path = None def project_name_from_archive(self, archive_path): """Compute the project name from the archive's path. """ - return os.path.basename(os.path.dirname(archive_path)) + archive_name = os.path.basename(archive_path) + for ext in ('.zip', '.tar.gz', '.tgz'): + if archive_name.lower().endswith(ext): + archive_name = archive_name[:-len(ext)] + break + return archive_name def prepare_origin_visit(self, origin_url, archive_path, visit_date): self._prepare_origin_visit(origin_url, visit_date) def prepare(self, origin_url, archive_path, visit_date): """1. Uncompress the archive in temporary location. 2. Prepare as the GitLoader does 3. Load as GitLoader does """ project_name = self.project_name_from_archive(archive_path) self.temp_dir, self.repo_path = utils.init_git_repo_from_archive( project_name, archive_path) self.log.info('Project %s - Uncompressing archive %s at %s' % ( origin_url, os.path.basename(archive_path), self.repo_path)) super().prepare(origin_url, self.repo_path, visit_date) def cleanup(self): """Cleanup the temporary location (if it exists). """ if self.temp_dir and os.path.exists(self.temp_dir): shutil.rmtree(self.temp_dir) self.log.info('Project %s - Done injecting %s' % ( self.origin_url, self.repo_path)) if __name__ == '__main__': import click import logging logging.basicConfig( level=logging.DEBUG, format='%(asctime)s %(process)d %(message)s' ) @click.command() @click.option('--origin-url', help='origin url') @click.option('--git-directory', help='Path to git repository to load') @click.option('--visit-date', default=None, help='Visit date') def main(origin_url, git_directory, visit_date): if not visit_date: visit_date = datetime.datetime.now(tz=datetime.timezone.utc) return GitLoader().load(origin_url, git_directory, visit_date) main() diff --git a/swh/loader/git/tests/test_loader.py b/swh/loader/git/tests/test_loader.py index f3a7c28..f1c5f48 100644 --- a/swh/loader/git/tests/test_loader.py +++ b/swh/loader/git/tests/test_loader.py @@ -1,287 +1,261 @@ # Copyright (C) 2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import os.path -import zipfile -import tempfile import subprocess from swh.loader.git.loader import GitLoader, GitLoaderFromArchive from swh.loader.core.tests import BaseLoaderTest from . import TEST_LOADER_CONFIG class GitLoaderFromArchive(GitLoaderFromArchive): def project_name_from_archive(self, archive_path): # We don't want the project name to be 'resources'. return 'testrepo' + def parse_config_file(self, *args, **kwargs): + return TEST_LOADER_CONFIG + CONTENT1 = { '33ab5639bfd8e7b95eb1d8d0b87781d4ffea4d5d', # README v1 '349c4ff7d21f1ec0eda26f3d9284c293e3425417', # README v2 '799c11e348d39f1704022b8354502e2f81f3c037', # file1.txt '4bdb40dfd6ec75cb730e678b5d7786e30170c5fb', # file2.txt - } +} SNAPSHOT_ID = 'bdf3b06d6017e0d9ad6447a73da6ff1ae9efb8f0' SNAPSHOT1 = { 'id': SNAPSHOT_ID, 'branches': { 'HEAD': { 'target': '2f01f5ca7e391a2f08905990277faf81e709a649', 'target_type': 'revision', }, 'refs/heads/master': { 'target': '2f01f5ca7e391a2f08905990277faf81e709a649', 'target_type': 'revision', }, 'refs/heads/branch1': { 'target': 'b0a77609903f767a2fd3d769904ef9ef68468b87', 'target_type': 'revision', }, 'refs/heads/branch2': { 'target': 'bd746cd1913721b269b395a56a97baf6755151c2', 'target_type': 'revision', }, 'refs/tags/branch2-after-delete': { 'target': 'bd746cd1913721b269b395a56a97baf6755151c2', 'target_type': 'revision', }, 'refs/tags/branch2-before-delete': { 'target': '1135e94ccf73b5f9bd6ef07b3fa2c5cc60bba69b', 'target_type': 'revision', }, }, } # directory hashes obtained with: # gco b6f40292c4e94a8f7e7b4aff50e6c7429ab98e2a # swh-hashtree --ignore '.git' --path . # gco 2f01f5ca7e391a2f08905990277faf81e709a649 # swh-hashtree --ignore '.git' --path . # gco bcdc5ebfde1a3cd6c96e0c2ea4eed19c13208777 # swh-hashtree --ignore '.git' --path . # gco 1135e94ccf73b5f9bd6ef07b3fa2c5cc60bba69b # swh-hashtree --ignore '.git' --path . # gco 79f65ac75f79dda6ff03d66e1242702ab67fb51c # swh-hashtree --ignore '.git' --path . # gco b0a77609903f767a2fd3d769904ef9ef68468b87 # swh-hashtree --ignore '.git' --path . # gco bd746cd1913721b269b395a56a97baf6755151c2 # swh-hashtree --ignore '.git' --path . REVISIONS1 = { 'b6f40292c4e94a8f7e7b4aff50e6c7429ab98e2a': '40dbdf55dfd4065422462cc74a949254aefa972e', '2f01f5ca7e391a2f08905990277faf81e709a649': 'e1d0d894835f91a0f887a4bc8b16f81feefdfbd5', 'bcdc5ebfde1a3cd6c96e0c2ea4eed19c13208777': 'b43724545b4759244bb54be053c690649161411c', '1135e94ccf73b5f9bd6ef07b3fa2c5cc60bba69b': 'fbf70528223d263661b5ad4b80f26caf3860eb8e', '79f65ac75f79dda6ff03d66e1242702ab67fb51c': '5df34ec74d6f69072d9a0a6677d8efbed9b12e60', 'b0a77609903f767a2fd3d769904ef9ef68468b87': '9ca0c7d6ffa3f9f0de59fd7912e08f11308a1338', 'bd746cd1913721b269b395a56a97baf6755151c2': 'e1d0d894835f91a0f887a4bc8b16f81feefdfbd5', - } +} class BaseGitLoaderTest(BaseLoaderTest): def setUp(self, archive_name, uncompress_archive, filename='testrepo'): super().setUp(archive_name=archive_name, filename=filename, prefix_tmp_folder_name='swh.loader.git.', start_path=os.path.dirname(__file__), uncompress_archive=uncompress_archive) class GitLoaderTest(GitLoader): def parse_config_file(self, *args, **kwargs): return TEST_LOADER_CONFIG class BaseDirGitLoaderTest(BaseGitLoaderTest): """Mixin base loader test to prepare the git repository to uncompress, load and test the results. This sets up """ def setUp(self): - super().setUp('testrepo.tgz', True) + super().setUp('testrepo.tgz', uncompress_archive=True) self.loader = GitLoaderTest() self.storage = self.loader.storage def load(self): return self.loader.load( origin_url=self.repo_url, visit_date='2016-05-03 15:16:32+00', directory=self.destination_path) -class GitLoaderFromArchiveTest(GitLoaderFromArchive): - def parse_config_file(self, *args, **kwargs): - return TEST_LOADER_CONFIG - - -class BaseZipGitLoaderTest(BaseGitLoaderTest): +class BaseGitLoaderFromArchiveTest(BaseGitLoaderTest): """Mixin base loader test to prepare the git repository to uncompress, load and test the results. This sets up """ def setUp(self): - super().setUp('testrepo.tgz', True) - self._setup_zip() - self.loader = GitLoaderFromArchiveTest() + super().setUp('testrepo.tgz', uncompress_archive=False) + self.loader = GitLoaderFromArchive() self.storage = self.loader.storage - def _setup_zip(self): - self._zip_file = tempfile.NamedTemporaryFile('ab', suffix='.zip') - dest_dir = os.path.normpath(self.destination_path) + '/' - with zipfile.ZipFile(self._zip_file, 'a') as zip_writer: - for root, dirs, files in os.walk(dest_dir): - assert root.startswith(dest_dir) - relative_root = os.path.join( - 'testrepo', - root[len(dest_dir):]) - for file_ in files: - zip_writer.write( - filename=os.path.join(root, file_), - arcname=os.path.join(relative_root, file_)) - self.destination_path = self._zip_file.name - self.tmp_root_path = None - self.repo_url = 'file://' + self.destination_path - - def tearDown(self): - self._zip_file.close() - super().tearDown() - def load(self): return self.loader.load( origin_url=self.repo_url, visit_date='2016-05-03 15:16:32+00', archive_path=self.destination_path) class GitLoaderTests: """Common tests for all git loaders.""" def test_load(self): """Loads a simple repository (made available by `setUp()`), and checks everything was added in the storage.""" res = self.load() self.assertEqual(res['status'], 'eventful', res) self.assertContentsContain(CONTENT1) self.assertCountDirectories(7) self.assertCountReleases(0) # FIXME: why not 2? self.assertCountRevisions(7) self.assertCountSnapshots(1) self.assertRevisionsContain(REVISIONS1) self.assertSnapshotEqual(SNAPSHOT1) self.assertEqual(self.loader.load_status(), {'status': 'eventful'}) self.assertEqual(self.loader.visit_status(), 'full') def test_load_unchanged(self): """Checks loading a repository a second time does not add any extra data.""" res = self.load() self.assertEqual(res['status'], 'eventful') res = self.load() self.assertEqual(res['status'], 'uneventful') self.assertCountSnapshots(1) class DirGitLoaderTest(BaseDirGitLoaderTest, GitLoaderTests): """Tests for the GitLoader. Includes the common ones, and add others that only work with a local dir.""" def _git(self, *cmd): """Small wrapper around subprocess to call Git.""" try: return subprocess.check_output( ['git', '-C', self.destination_path] + list(cmd)) except subprocess.CalledProcessError as e: print(e.output) print(e.stderr) raise def test_load_changed(self): """Loads a repository, makes some changes by adding files, commits, and merges, load it again, and check the storage contains everything it should.""" # Initial load res = self.load() self.assertEqual(res['status'], 'eventful', res) self._git('config', '--local', 'user.email', 'you@example.com') self._git('config', '--local', 'user.name', 'Your Name') # Load with a new file + revision with open(os.path.join(self.destination_path, 'hello.py'), 'a') as fd: fd.write("print('Hello world')\n") self._git('add', 'hello.py') self._git('commit', '-m', 'Hello world') new_revision = self._git('rev-parse', 'master').decode().strip() revisions = REVISIONS1.copy() assert new_revision not in revisions revisions[new_revision] = '85dae072a5aa9923ffa7a7568f819ff21bf49858' res = self.load() self.assertEqual(res['status'], 'eventful') self.assertCountContents(4 + 1) self.assertCountDirectories(7 + 1) self.assertCountReleases(0) # FIXME: why not 2? self.assertCountRevisions(7 + 1) self.assertCountSnapshots(1 + 1) self.assertRevisionsContain(revisions) # TODO: how to check the snapshot id? # self.assertSnapshotEqual(SNAPSHOT1) self.assertEqual(self.loader.load_status(), {'status': 'eventful'}) self.assertEqual(self.loader.visit_status(), 'full') # Load with a new merge self._git('merge', 'branch1', '-m', 'merge') new_revision = self._git('rev-parse', 'master').decode().strip() assert new_revision not in revisions revisions[new_revision] = 'dab8a37df8db8666d4e277bef9a546f585b5bedd' res = self.load() self.assertEqual(res['status'], 'eventful') self.assertCountContents(4 + 1) self.assertCountDirectories(7 + 2) self.assertCountReleases(0) # FIXME: why not 2? self.assertCountRevisions(7 + 2) self.assertCountSnapshots(1 + 1 + 1) self.assertRevisionsContain(revisions) # TODO: how to check the snapshot id? # self.assertSnapshotEqual(SNAPSHOT1) self.assertEqual(self.loader.load_status(), {'status': 'eventful'}) self.assertEqual(self.loader.visit_status(), 'full') -class ZipGitLoaderTest(BaseZipGitLoaderTest, GitLoaderTests): +class GitLoaderFromArchiveTest(BaseGitLoaderFromArchiveTest, GitLoaderTests): """Tests for GitLoaderFromArchive. Imports the common ones from GitLoaderTests.""" pass diff --git a/swh/loader/git/utils.py b/swh/loader/git/utils.py index 3b46f68..0a72bec 100644 --- a/swh/loader/git/utils.py +++ b/swh/loader/git/utils.py @@ -1,68 +1,75 @@ # Copyright (C) 2017 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information """Utilities helper functions""" import datetime import os import shutil import tempfile -from subprocess import call +from swh.core import tarball def init_git_repo_from_archive(project_name, archive_path, root_temp_dir='/tmp'): """Given a path to an archive containing a git repository. Uncompress that archive to a temporary location and returns the path. If any problem whatsoever is raised, clean up the temporary location. Args: project_name (str): Project's name archive_path (str): Full path to the archive root_temp_dir (str): Optional temporary directory mount point (default to /tmp) Returns A tuple: - temporary folder: containing the mounted repository - repo_path, path to the mounted repository inside the temporary folder Raises ValueError in case of failure to run the command to uncompress """ temp_dir = tempfile.mkdtemp( suffix='.swh.loader.git', prefix='tmp.', dir=root_temp_dir) try: # create the repository that will be loaded with the dump - r = call(['unzip', '-q', '-o', archive_path, '-d', temp_dir]) - if r != 0: - raise ValueError('Failed to uncompress archive %s' % archive_path) - + tarball.uncompress(archive_path, temp_dir) repo_path = os.path.join(temp_dir, project_name) + # tarball content may not be as expected (e.g. no top level directory + # or a top level directory with a name different from project_name), + # so try to make it loadable anyway + if not os.path.exists(repo_path): + os.mkdir(repo_path) + for root, dirs, files in os.walk(temp_dir): + if '.git' in dirs: + shutil.copytree(os.path.join(root, '.git'), + os.path.join(repo_path, '.git')) + break return temp_dir, repo_path except Exception as e: shutil.rmtree(temp_dir) raise e def check_date_time(timestamp): """Check date time for overflow errors. Args: timestamp (timestamp): Timestamp in seconds Raise: Any error raised by datetime fromtimestamp conversion error. """ if not timestamp: return None datetime.datetime.fromtimestamp(timestamp, datetime.timezone.utc)