Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/svn/tasks.py
| # Copyright (C) 2015-2019 The Software Heritage developers | # Copyright (C) 2015-2021 The Software Heritage developers | ||||
| # See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
| # License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
| # See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
| from typing import Optional | |||||
| from celery import shared_task | from celery import shared_task | ||||
| from .loader import SvnLoader, SvnLoaderFromDumpArchive, SvnLoaderFromRemoteDump | from .loader import SvnLoader, SvnLoaderFromDumpArchive, SvnLoaderFromRemoteDump | ||||
| @shared_task(name=__name__ + ".LoadSvnRepository") | @shared_task(name=__name__ + ".LoadSvnRepository") | ||||
| def load_svn( | def load_svn( | ||||
| *, | *, | ||||
| url=None, | url: Optional[str] = None, | ||||
| origin_url=None, | origin_url: Optional[str] = None, | ||||
| destination_path=None, | destination_path: Optional[str] = None, | ||||
| swh_revision=None, | swh_revision: Optional[str] = None, | ||||
| visit_date=None, | visit_date: Optional[str] = None, | ||||
| start_from_scratch=False, | start_from_scratch: Optional[bool] = False, | ||||
| ): | ): | ||||
| """Import a svn repository | """Import a svn repository | ||||
| Args: | Args: | ||||
| args: ordered arguments (expected None) | - url: (mandatory) svn's repository url to ingest data from | ||||
| kwargs: Dictionary with the following expected keys: | - origin_url: Optional original url override to use as origin reference | ||||
| in the archive. If not provided, "url" is used as origin. | |||||
| - url (str): (mandatory) svn's repository url | - destination_path: (optional) root directory to | ||||
| - origin_url (str): Optional original url override | |||||
| - destination_path (str): (optional) root directory to | |||||
| locally retrieve svn's data | locally retrieve svn's data | ||||
| - swh_revision (dict): (optional) extra revision hex to | - swh_revision: (optional) extra revision hex to | ||||
| start from. see swh.loader.svn.SvnLoader.process | start from. See swh.loader.svn.SvnLoader.process | ||||
| docstring | docstring | ||||
| - visit_date: Optional date to override the visit date | |||||
| - start_from_scratch: Flag to allow starting back the svn repository from the | |||||
| start | |||||
| """ | """ | ||||
| loader = SvnLoader( | loader = SvnLoader.from_configfile( | ||||
| url, | url=url, | ||||
| origin_url=origin_url, | origin_url=origin_url, | ||||
| destination_path=destination_path, | destination_path=destination_path, | ||||
| swh_revision=swh_revision, | swh_revision=swh_revision, | ||||
| visit_date=visit_date, | visit_date=visit_date, | ||||
| start_from_scratch=start_from_scratch, | start_from_scratch=start_from_scratch, | ||||
| ) | ) | ||||
| return loader.load() | return loader.load() | ||||
| @shared_task(name=__name__ + ".MountAndLoadSvnRepository") | @shared_task(name=__name__ + ".MountAndLoadSvnRepository") | ||||
| def load_svn_from_archive( | def load_svn_from_archive( | ||||
| *, url=None, archive_path=None, visit_date=None, start_from_scratch=False | *, | ||||
| url: Optional[str] = None, | |||||
| archive_path: Optional[str] = None, | |||||
| visit_date: Optional[str] = None, | |||||
| start_from_scratch: Optional[bool] = False, | |||||
| ): | ): | ||||
| """1. Mount an svn dump from archive as a local svn repository | """1. Mount an svn dump from archive as a local svn repository | ||||
| 2. Load it through the svn loader | 2. Load it through the svn loader | ||||
| 3. Clean up mounted svn repository archive | 3. Clean up mounted svn repository archive | ||||
| Args: | |||||
| - url: origin url | |||||
| - archive_path: Path on disk to the archive holdin the svn repository to ingest | |||||
| - visit_date: Optional date to override the visit date | |||||
| - start_from_scratch: Flag to allow starting back the svn repository from the | |||||
| start | |||||
| """ | """ | ||||
| loader = SvnLoaderFromDumpArchive( | loader = SvnLoaderFromDumpArchive.from_configfile( | ||||
| url, | url=url, | ||||
| archive_path=archive_path, | archive_path=archive_path, | ||||
| visit_date=visit_date, | visit_date=visit_date, | ||||
| start_from_scratch=start_from_scratch, | start_from_scratch=start_from_scratch, | ||||
| ) | ) | ||||
| return loader.load() | return loader.load() | ||||
| @shared_task(name=__name__ + ".DumpMountAndLoadSvnRepository") | @shared_task(name=__name__ + ".DumpMountAndLoadSvnRepository") | ||||
| def load_svn_from_remote_dump( | def load_svn_from_remote_dump( | ||||
| *, url=None, origin_url=None, visit_date=None, start_from_scratch=False | *, | ||||
| url: Optional[str] = None, | |||||
| origin_url: Optional[str] = None, | |||||
| visit_date: Optional[str] = None, | |||||
| start_from_scratch: Optional[bool] = False, | |||||
| ): | ): | ||||
| """1. Mount a remote svn dump as a local svn repository. | """1. Mount a remote svn dump as a local svn repository. | ||||
| 2. Load it through the svn loader. | 2. Load it through the svn loader. | ||||
| 3. Clean up mounted svn repository archive. | 3. Clean up mounted svn repository archive. | ||||
| Args: | |||||
| - url: (mandatory) svn's repository url to ingest data from | |||||
| - origin_url: Optional original url override to use as origin reference | |||||
| in the archive. If not provided, "url" is used as origin. | |||||
| - visit_date: Optional date to override the visit date | |||||
| - start_from_scratch: Flag to allow starting back the svn repository from the | |||||
| start | |||||
| """ | """ | ||||
| loader = SvnLoaderFromRemoteDump( | loader = SvnLoaderFromRemoteDump.from_configfile( | ||||
| url, | url=url, | ||||
| origin_url=origin_url, | origin_url=origin_url, | ||||
| visit_date=visit_date, | visit_date=visit_date, | ||||
| start_from_scratch=start_from_scratch, | start_from_scratch=start_from_scratch, | ||||
| ) | ) | ||||
| return loader.load() | return loader.load() | ||||