diff --git a/README.md b/README.md index 0ff3b7d..6c3139b 100644 --- a/README.md +++ b/README.md @@ -1,44 +1,82 @@ swh-loader-mercurial ========================= # Configuration file In usual location for a loader, *{/etc/softwareheritage/ | ~/.swh/ | ~/.config/swh/}loader/hg.yml*: ``` YAML storage: cls: remote args: url: http://localhost:5002/ send_contents: True send_directories: True send_revisions: True send_releases: True send_occurrences: True content_packet_size: 1000 content_packet_size_bytes: 1073741824 directory_packet_size: 2500 revision_packet_size: 1000 release_packet_size: 1000 occurrence_packet_size: 1000 ``` # Basic use From python3's toplevel: +## local directory (failure) + +Only origin and contents are filled. + +Remaining objects are empty (directory, revision, release, +occurrence). + ``` Python project = '756015-ipv6' directory = '/home/storage/hg/repo/%s' % project origin_url = 'https://%s.googlecode.com' % project import logging logging.basicConfig(level=logging.DEBUG) +from swh.loader.mercurial.tasks import SlowLoadMercurialTsk + +t = SlowLoadMercurialTsk() +t.run(origin_url=origin_url, directory=directory, visit_date='2016-05-03T15:16:32+00:00') +``` + +## local archive + +``` Python +project = '756015-ipv6-source-archive.zip' +archive_path = '/home/storage/hg/repo/%s' % project +origin_url = 'https://%s-archive.googlecode.com' % project + +import logging +logging.basicConfig(level=logging.DEBUG) + +from swh.loader.mercurial.tasks import SlowLoadMercurialArchiveTsk + +t = SlowLoadMercurialArchiveTsk() +t.run(origin_url=origin_url, archive_path=archive_path, visit_date='2016-05-03T15:16:32+00:00') +``` + +## Remote (failure) + +``` Python +origin_url = 'https://code.launchpad.net/~bzr-svn/bzr-svn/1.2' +directory = '/tmp/' + +import logging +logging.basicConfig(level=logging.DEBUG) + from swh.loader.mercurial.tasks import LoadMercurialTsk t = LoadMercurialTsk() t.run(origin_url=origin_url, directory=directory, visit_date='2016-05-03T15:16:32+00:00') ``` diff --git a/swh/loader/mercurial/tasks.py b/swh/loader/mercurial/tasks.py index 7bff11c..1143992 100644 --- a/swh/loader/mercurial/tasks.py +++ b/swh/loader/mercurial/tasks.py @@ -1,27 +1,66 @@ # Copyright (C) 2017 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from swh.scheduler.task import Task from .bundle20_loader import HgBundle20Loader +from .slow_loader import HgLoader, HgLoaderFromArchive class LoadMercurialTsk(Task): """Mercurial repository loading """ task_queue = 'swh_loader_mercurial' def run_task(self, *, origin_url, directory, visit_date): """Import a mercurial tarball into swh. Args: see :func:`DepositLoader.load`. """ loader = HgBundle20Loader() loader.log = self.log loader.load(origin_url=origin_url, directory=directory, visit_date=visit_date) + + +class SlowLoadMercurialTsk(Task): + """Mercurial repository loading + + """ + task_queue = 'swh_loader_mercurial_slow' + + def run_task(self, *, origin_url, directory, visit_date): + """Import a mercurial tarball into swh. + + Args: see :func:`DepositLoader.load`. + + """ + loader = HgLoader() + loader.log = self.log + loader.load(origin_url=origin_url, + directory=directory, + visit_date=visit_date) + + +class SlowLoadMercurialArchiveTsk(Task): + """Mercurial repository loading + + """ + task_queue = 'swh_loader_mercurial_slow_archive' + + def run_task(self, *, origin_url, archive_path, visit_date): + """Import a mercurial tarball into swh. + + Args: see :func:`DepositLoader.load`. + + """ + loader = HgLoaderFromArchive() + loader.log = self.log + loader.load(origin_url=origin_url, + archive_path=archive_path, + visit_date=visit_date)