Page MenuHomeSoftware Heritage
Files F9337605

D2439.id8628.diff
No OneTemporary
Actions

D2439.id8628.diff
View Options

diff --git a/requirements-swh.txt b/requirements-swh.txt
--- a/requirements-swh.txt
+++ b/requirements-swh.txt
@@ -1,5 +1,5 @@
swh.core >= 0.0.7
-swh.loader.core >= 0.0.37
+swh.loader.core >= 0.0.67
swh.model >= 0.0.27
swh.scheduler >= 0.0.39
swh.storage >= 0.0.108
diff --git a/swh/loader/git/from_disk.py b/swh/loader/git/from_disk.py
--- a/swh/loader/git/from_disk.py
+++ b/swh/loader/git/from_disk.py
@@ -147,7 +147,7 @@
def get_contents(self):
"""Get the contents that need to be loaded"""
- max_content_size = self.config['content_size_limit']
+ max_content_size = self.config['max_content_size']
missing_contents = set(self.storage.content_missing(
self.get_content_ids(), 'sha1_git'))
diff --git a/swh/loader/git/loader.py b/swh/loader/git/loader.py
--- a/swh/loader/git/loader.py
+++ b/swh/loader/git/loader.py
@@ -379,7 +379,7 @@
def get_contents(self):
"""Format the blobs from the git repository as swh contents"""
- max_content_size = self.config['content_size_limit']
+ max_content_size = self.config['max_content_size']
missing_contents = set(self.storage.content_missing(
self.get_content_ids(), 'sha1_git'))
diff --git a/swh/loader/git/tests/__init__.py b/swh/loader/git/tests/__init__.py
--- a/swh/loader/git/tests/__init__.py
+++ b/swh/loader/git/tests/__init__.py
@@ -1,21 +1,31 @@
+# Copyright (C) 2018-2019 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
TEST_LOADER_CONFIG = {
'storage': {
- 'cls': 'memory',
- 'args': {
- }
+ 'cls': 'pipeline',
+ 'steps': [
+ {
+ 'cls': 'filter'
+ },
+ {
+ 'cls': 'buffer',
+ 'min_batch_size': {
+ 'content': 10,
+ 'content_bytes': 100 * 1024 * 1024,
+ 'directory': 10,
+ 'revision': 10,
+ 'release': 10,
+ },
+ },
+ {
+ 'cls': 'memory'
+ },
+ ]
},
- 'send_contents': True,
- 'send_directories': True,
- 'send_revisions': True,
- 'send_releases': True,
- 'send_snapshot': True,
-
- 'content_size_limit': 100 * 1024 * 1024,
- 'content_packet_size': 10,
- 'content_packet_size_bytes': 100 * 1024 * 1024,
- 'directory_packet_size': 10,
- 'revision_packet_size': 10,
- 'release_packet_size': 10,
-
+ 'max_content_size': 100 * 1024 * 1024,
+ 'pack_size_bytes': 4 * 1024 * 1024 * 1024,
'save_data': False,
}
diff --git a/swh/loader/git/tests/conftest.py b/swh/loader/git/tests/conftest.py
--- a/swh/loader/git/tests/conftest.py
+++ b/swh/loader/git/tests/conftest.py
@@ -1,3 +1,8 @@
+# Copyright (C) 2018-2019 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
import pytest
from swh.scheduler.tests.conftest import * # noqa
diff --git a/swh/loader/git/tests/test_loader.py b/swh/loader/git/tests/test_loader.py
--- a/swh/loader/git/tests/test_loader.py
+++ b/swh/loader/git/tests/test_loader.py
@@ -3,16 +3,17 @@
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
-
from swh.loader.git.loader import GitLoader
from swh.loader.git.tests.test_from_disk import DirGitLoaderTest
+from . import TEST_LOADER_CONFIG
+
class GitLoaderTest(GitLoader):
def parse_config_file(self, *args, **kwargs):
return {
**super().parse_config_file(*args, **kwargs),
- 'storage': {'cls': 'memory', 'args': {}}
+ **TEST_LOADER_CONFIG
}

File Metadata

Mime Type
text/plain
Expires
Jul 3 2025, 8:10 AM (10 w, 1 d ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3223981

Event Timeline

About · Developer information · Legal