diff --git a/debian/control b/debian/control index b14b7c8..51c321c 100644 --- a/debian/control +++ b/debian/control @@ -1,29 +1,29 @@ Source: swh-loader-svn Maintainer: Software Heritage developers Section: python Priority: optional Build-Depends: debhelper (>= 9), dh-python, python3-all, python3-nose, python3-setuptools, python3-swh.core (>= 0.0.19~), python3-swh.storage (>= 0.0.76~), python3-swh.model (>= 0.0.11~), - python3-swh.scheduler (>= 0.0.7~), + python3-swh.scheduler (>= 0.0.11~), python3-swh.loader.core (>= 0.0.12~), python3-subvertpy (>= 0.9.4~), python3-dateutil, python3-retrying, python3-click, python3-vcversioner Standards-Version: 3.9.6 Homepage: https://forge.softwareheritage.org/diffusion/DLDSVN/ Package: python3-swh.loader.svn Architecture: all Depends: python3-swh.core (>= 0.0.19~), python3-swh.storage (>= 0.0.76~), python3-swh.model (>= 0.0.11~), - python3-swh.scheduler (>= 0.0.7~), python3-swh.loader.core (>= 0.0.12~), + python3-swh.scheduler (>= 0.0.11~), python3-swh.loader.core (>= 0.0.12~), subversion, pigz, ${misc:Depends}, ${python3:Depends} Description: Software Heritage Loader Svn Module in charge of loading svn repositories into swh storage. diff --git a/requirements-swh.txt b/requirements-swh.txt index 05eb342..51a9298 100644 --- a/requirements-swh.txt +++ b/requirements-swh.txt @@ -1,5 +1,5 @@ swh.core >= 0.0.19 swh.storage >= 0.0.76 swh.model >= 0.0.11 -swh.scheduler >= 0.0.7 +swh.scheduler >= 0.0.11 swh.loader.core >= 0.0.12 diff --git a/swh/loader/svn/producer.py b/swh/loader/svn/producer.py index 0c9ccea..aa15a8c 100644 --- a/swh/loader/svn/producer.py +++ b/swh/loader/svn/producer.py @@ -1,115 +1,106 @@ -# Copyright (C) 2015-2016 The Software Heritage developers +# Copyright (C) 2015-2017 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import click import sys - -def get_task(task_name): - """Retrieve task object in the application by its fully qualified name. - - """ - from swh.scheduler.celery_backend.config import app - for module in app.conf.CELERY_IMPORTS: - __import__(module) - - return app.tasks[task_name] +from swh.scheduler.utils import get_task def _produce_svn_to_load( svn_url, origin_url, destination_path=None, visit_date=None, synchroneous=False, task_name='swh.loader.svn.tasks.LoadSWHSvnRepositoryTsk'): """Produce svn urls on the message queue. Those urls can either be read from stdin or directly passed as argument. """ task = get_task(task_name) if not synchroneous and svn_url: task.delay(svn_url=svn_url, origin_url=origin_url, visit_date=visit_date, destination_path=destination_path) elif synchroneous and svn_url: # for debug purpose task(svn_url=svn_url, origin_url=origin_url, visit_date=visit_date, destination_path=destination_path) else: # input from stdin, so we ignore most of the function's input for line in sys.stdin: line = line.rstrip() data = line.split(' ') svn_url = data[0] if len(data) > 1: origin_url = data[1] else: origin_url = None if svn_url: print(svn_url, origin_url) task.delay(svn_url=svn_url, origin_url=origin_url, destination_path=destination_path) def _produce_archive_to_mount_and_load( archive_path, visit_date, task_name='swh.loader.svn.tasks.MountAndLoadSvnRepositoryTsk'): task = get_task(task_name) if archive_path: task.delay(archive_path) else: for line in sys.stdin: line = line.rstrip() data = line.split(' ') archive_path = data[0] if len(data) > 1: origin_url = data[1] else: origin_url = None if archive_path: print(archive_path, origin_url) task.delay(archive_path, origin_url, visit_date=visit_date) @click.group() def cli(): pass @cli.command('svn', help='Default svn urls producer') @click.option('--url', help="svn repository's mirror url.") @click.option('--origin-url', default=None, help='svn repository\'s original remote url ' '(if different than --svn-url).') @click.option('--destination-path', help="(optional) svn checkout destination.") @click.option('--visit-date', help="(optional) visit date to override") @click.option('--synchroneous', is_flag=True, help="To execute directly the svn loading.") def produce_svn_to_load(url, origin_url, destination_path, visit_date, synchroneous): _produce_svn_to_load(svn_url=url, origin_url=origin_url, destination_path=destination_path, synchroneous=synchroneous) @cli.command('svn-archive', help='Default svndump archive producer') @click.option('--visit-date', help="(optional) visit date to override") @click.option('--path', help="Archive's Path to load and mount") def produce_archive_to_mount_and_load(path, visit_date): _produce_archive_to_mount_and_load(path, visit_date) if __name__ == '__main__': cli()