diff --git a/requirements-swh.txt b/requirements-swh.txt --- a/requirements-swh.txt +++ b/requirements-swh.txt @@ -1,4 +1,4 @@ swh.storage >= 0.0.143 swh.model >= 0.0.18 swh.scheduler >= 0.0.39 -swh.loader.core >= 0.0.37 +swh.loader.core >= 0.0.66 diff --git a/swh/loader/svn/loader.py b/swh/loader/svn/loader.py --- a/swh/loader/svn/loader.py +++ b/swh/loader/svn/loader.py @@ -540,9 +540,9 @@ This also resets the internal instance variable state. """ - self.maybe_load_contents(self._contents) - self.maybe_load_directories(self._directories) - self.maybe_load_revisions(self._revisions) + self.send_contents(self._contents) + self.send_directories(self._directories) + self.send_revisions(self._revisions) if self.done: # finish line, snapshot! self.generate_and_load_snapshot(revision=self._last_revision, @@ -572,7 +572,7 @@ else: return None self.log.debug('snapshot: %s' % snap) - self.maybe_load_snapshot(snap) + self.send_snapshot(snap) def load_status(self): return { diff --git a/swh/loader/svn/tests/conftest.py b/swh/loader/svn/tests/conftest.py --- a/swh/loader/svn/tests/conftest.py +++ b/swh/loader/svn/tests/conftest.py @@ -16,27 +16,31 @@ def swh_loader_config() -> Dict[str, Any]: return { 'storage': { - 'cls': 'memory', + 'cls': 'pipeline', + 'steps': [ + { + 'cls': 'filter' + }, + { + 'cls': 'buffer', + 'min_batch_size': { + 'content': 10000, + 'content_bytes': 1073741824, + 'directory': 2500, + 'revision': 10, + 'release': 100, + }, + }, + { + 'cls': 'memory' + }, + ] }, 'check_revision': {'limit': 100, 'status': False}, - 'content_packet_block_size_bytes': 104857600, - 'content_packet_size': 10000, - 'content_packet_size_bytes': 1073741824, - 'content_size_limit': 104857600, 'debug': False, - 'directory_packet_size': 2500, 'log_db': 'dbname=softwareheritage-log', - 'occurrence_packet_size': 1000, - 'release_packet_size': 1000, - 'revision_packet_size': 10, 'save_data': False, 'save_data_path': '', - 'send_contents': True, - 'send_directories': True, - 'send_occurrences': True, - 'send_releases': True, - 'send_revisions': True, - 'send_snapshot': True, 'temp_directory': '/tmp', } diff --git a/swh/loader/svn/tests/test_loader.py b/swh/loader/svn/tests/test_loader.py --- a/swh/loader/svn/tests/test_loader.py +++ b/swh/loader/svn/tests/test_loader.py @@ -30,26 +30,33 @@ _LOADER_TEST_CONFIG = { 'check_revision': {'limit': 100, 'status': False}, - 'content_packet_block_size_bytes': 104857600, - 'content_packet_size': 10000, - 'content_packet_size_bytes': 1073741824, - 'content_size_limit': 104857600, 'debug': False, - 'directory_packet_size': 2500, 'log_db': 'dbname=softwareheritage-log', - 'occurrence_packet_size': 1000, - 'release_packet_size': 1000, - 'revision_packet_size': 10, 'save_data': False, 'save_data_path': '', - 'send_contents': True, - 'send_directories': True, - 'send_occurrences': True, - 'send_releases': True, - 'send_revisions': True, - 'send_snapshot': True, - 'storage': {'args': {}, 'cls': 'memory'}, - 'temp_directory': '/tmp' + 'temp_directory': '/tmp', + 'max_content_size': 100 * 1024 * 1024, + 'storage': { + 'cls': 'pipeline', + 'steps': [ + { + 'cls': 'filter' + }, + { + 'cls': 'buffer', + 'min_batch_size': { + 'content': 10000, + 'content_bytes': 1073741824, + 'directory': 2500, + 'revision': 10, + 'release': 100, + }, + }, + { + 'cls': 'memory' + }, + ] + }, }