diff --git a/requirements-swh.txt b/requirements-swh.txt --- a/requirements-swh.txt +++ b/requirements-swh.txt @@ -1,4 +1,4 @@ -swh.storage >= 0.0.143 +swh.storage >= 0.0.163 swh.model >= 0.0.18 swh.scheduler >= 0.0.39 -swh.loader.core >= 0.0.67 +swh.loader.core >= 0.0.74 diff --git a/swh/loader/svn/loader.py b/swh/loader/svn/loader.py --- a/swh/loader/svn/loader.py +++ b/swh/loader/svn/loader.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2019 The Software Heritage developers +# Copyright (C) 2015-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -20,7 +20,8 @@ from swh.model.from_disk import Directory from swh.model.identifiers import identifier_to_bytes, revision_identifier from swh.model.identifiers import snapshot_identifier -from swh.loader.core.loader import BufferedLoader +from swh.loader.core.converters import content_for_storage +from swh.loader.core.loader import BaseLoader from swh.loader.core.utils import clean_dangling_folders from swh.storage.algos.snapshot import snapshot_get_all_branches @@ -58,7 +59,7 @@ TEMPORARY_DIR_PREFIX_PATTERN = 'swh.loader.svn.' -class SvnLoader(BufferedLoader): +class SvnLoader(BaseLoader): """Swh svn loader. The repository is either remote or local. The loader deals with @@ -540,9 +541,14 @@ This also resets the internal instance variable state. """ - self.send_contents(self._contents) - self.send_directories(self._directories) - self.send_revisions(self._revisions) + contents = [ + content_for_storage(c, max_content_size=self.max_content_size, + origin_url=self.origin['url']) + for c in self._contents + ] + self.storage.content_add(contents) + self.storage.directory_add(self._directories) + self.storage.revision_add(self._revisions) if self.done: # finish line, snapshot! self.generate_and_load_snapshot(revision=self._last_revision, @@ -572,7 +578,7 @@ else: return None self.log.debug('snapshot: %s' % snap) - self.send_snapshot(snap) + self.storage.snapshot_add([snap]) def load_status(self): return { diff --git a/swh/loader/svn/tests/test_loader.py b/swh/loader/svn/tests/test_loader.py --- a/swh/loader/svn/tests/test_loader.py +++ b/swh/loader/svn/tests/test_loader.py @@ -1,4 +1,4 @@ -# Copyright (C) 2016-2018 The Software Heritage developers +# Copyright (C) 2016-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -40,7 +40,10 @@ 'cls': 'pipeline', 'steps': [ { - 'cls': 'filter' + 'cls': 'retry', + }, + { + 'cls': 'filter', }, { 'cls': 'buffer',