Page MenuHomeSoftware Heritage
Files F9345676

D2286.id7863.diff
No OneTemporary
Actions

D2286.id7863.diff
View Options

diff --git a/swh/indexer/storage/in_memory.py b/swh/indexer/storage/in_memory.py
--- a/swh/indexer/storage/in_memory.py
+++ b/swh/indexer/storage/in_memory.py
@@ -199,6 +199,9 @@
self._revision_intrinsic_metadata = SubStorage(self._tools)
self._origin_intrinsic_metadata = SubStorage(self._tools)
+ def check_config(self, *, check_write):
+ return True
+
def content_mimetype_missing(self, mimetypes):
"""Generate mimetypes missing from storage.
diff --git a/swh/indexer/tests/storage/conftest.py b/swh/indexer/tests/storage/conftest.py
--- a/swh/indexer/tests/storage/conftest.py
+++ b/swh/indexer/tests/storage/conftest.py
@@ -10,63 +10,13 @@
from swh.storage.tests.conftest import postgresql_fact
from swh.indexer.storage import get_indexer_storage
from swh.model.hashutil import hash_to_bytes
-from .generate_data_test import MIMETYPE_OBJECTS, FOSSOLOGY_LICENSES
+from .generate_data_test import (
+ MIMETYPE_OBJECTS, FOSSOLOGY_LICENSES, TOOLS
+)
DUMP_FILES = join(SQL_DIR, '*.sql')
-TOOLS = [
- {
- 'tool_name': 'universal-ctags',
- 'tool_version': '~git7859817b',
- 'tool_configuration': {
- "command_line": "ctags --fields=+lnz --sort=no --links=no "
- "--output-format=json <filepath>"}
- },
- {
- 'tool_name': 'swh-metadata-translator',
- 'tool_version': '0.0.1',
- 'tool_configuration': {"type": "local", "context": "NpmMapping"},
- },
- {
- 'tool_name': 'swh-metadata-detector',
- 'tool_version': '0.0.1',
- 'tool_configuration': {
- "type": "local", "context": ["NpmMapping", "CodemetaMapping"]},
- },
- {
- 'tool_name': 'swh-metadata-detector2',
- 'tool_version': '0.0.1',
- 'tool_configuration': {
- "type": "local", "context": ["NpmMapping", "CodemetaMapping"]},
- },
- {
- 'tool_name': 'file',
- 'tool_version': '5.22',
- 'tool_configuration': {"command_line": "file --mime <filepath>"},
- },
- {
- 'tool_name': 'pygments',
- 'tool_version': '2.0.1+dfsg-1.1+deb8u1',
- 'tool_configuration': {
- "type": "library", "debian-package": "python3-pygments"},
- },
- {
- 'tool_name': 'pygments2',
- 'tool_version': '2.0.1+dfsg-1.1+deb8u1',
- 'tool_configuration': {
- "type": "library",
- "debian-package": "python3-pygments",
- "max_content_size": 10240
- },
- },
- {
- 'tool_name': 'nomos',
- 'tool_version': '3.1.0rc2-31-ga2cbb8c',
- 'tool_configuration': {"command_line": "nomossa <filepath>"},
- }
-]
-
class DataObj(dict):
def __getattr__(self, key):
@@ -101,10 +51,15 @@
data.origin_url_1 = 'file:///dev/0/zero' # 44434341
data.origin_url_2 = 'file:///dev/1/one' # 44434342
data.origin_url_3 = 'file:///dev/2/two' # 54974445
- data.mimetypes = MIMETYPE_OBJECTS[:]
- swh_indexer_storage.content_mimetype_add(
- MIMETYPE_OBJECTS)
- data.fossology_licenses = FOSSOLOGY_LICENSES[:]
+ data.mimetypes = [
+ {**mimetype_obj, 'indexer_configuration_id': tools['file']['id']}
+ for mimetype_obj in MIMETYPE_OBJECTS
+ ]
+ swh_indexer_storage.content_mimetype_add(data.mimetypes)
+ data.fossology_licenses = [
+ {**fossology_obj, 'indexer_configuration_id': tools['nomos']['id']}
+ for fossology_obj in FOSSOLOGY_LICENSES
+ ]
swh_indexer_storage._test_data = data
return (swh_indexer_storage, data)
diff --git a/swh/indexer/tests/storage/generate_data_test.py b/swh/indexer/tests/storage/generate_data_test.py
--- a/swh/indexer/tests/storage/generate_data_test.py
+++ b/swh/indexer/tests/storage/generate_data_test.py
@@ -86,11 +86,64 @@
return content_mimetypes
+TOOLS = [
+ {
+ 'tool_name': 'universal-ctags',
+ 'tool_version': '~git7859817b',
+ 'tool_configuration': {
+ "command_line": "ctags --fields=+lnz --sort=no --links=no "
+ "--output-format=json <filepath>"}
+ },
+ {
+ 'tool_name': 'swh-metadata-translator',
+ 'tool_version': '0.0.1',
+ 'tool_configuration': {"type": "local", "context": "NpmMapping"},
+ },
+ {
+ 'tool_name': 'swh-metadata-detector',
+ 'tool_version': '0.0.1',
+ 'tool_configuration': {
+ "type": "local", "context": ["NpmMapping", "CodemetaMapping"]},
+ },
+ {
+ 'tool_name': 'swh-metadata-detector2',
+ 'tool_version': '0.0.1',
+ 'tool_configuration': {
+ "type": "local", "context": ["NpmMapping", "CodemetaMapping"]},
+ },
+ {
+ 'tool_name': 'file',
+ 'tool_version': '5.22',
+ 'tool_configuration': {"command_line": "file --mime <filepath>"},
+ },
+ {
+ 'tool_name': 'pygments',
+ 'tool_version': '2.0.1+dfsg-1.1+deb8u1',
+ 'tool_configuration': {
+ "type": "library", "debian-package": "python3-pygments"},
+ },
+ {
+ 'tool_name': 'pygments2',
+ 'tool_version': '2.0.1+dfsg-1.1+deb8u1',
+ 'tool_configuration': {
+ "type": "library",
+ "debian-package": "python3-pygments",
+ "max_content_size": 10240
+ },
+ },
+ {
+ 'tool_name': 'nomos',
+ 'tool_version': '3.1.0rc2-31-ga2cbb8c',
+ 'tool_configuration': {"command_line": "nomossa <filepath>"},
+ },
+]
+
+
MIMETYPE_OBJECTS = [
{'id': MultiHash.from_data(uuid1().bytes, {'sha1'}).digest()['sha1'],
- 'indexer_configuration_id': 1,
'mimetype': mt,
'encoding': enc,
+ # 'indexer_configuration_id' will be added after TOOLS get registered
}
for mt in MIMETYPES
for enc in ENCODINGS]
@@ -105,8 +158,8 @@
FOSSOLOGY_LICENSES = [
{'id': MultiHash.from_data(uuid1().bytes, {'sha1'}).digest()['sha1'],
- 'indexer_configuration_id': 1,
'licenses': [LICENSES[i % len(LICENSES)], ],
+ # 'indexer_configuration_id' will be added after TOOLS get registered
}
for i in range(10)
]
diff --git a/swh/indexer/tests/storage/test_in_memory.py b/swh/indexer/tests/storage/test_in_memory.py
--- a/swh/indexer/tests/storage/test_in_memory.py
+++ b/swh/indexer/tests/storage/test_in_memory.py
@@ -11,11 +11,10 @@
@pytest.fixture
-def swh_indexer_storage(swh_indexer_storage_postgresql):
+def swh_indexer_storage():
storage_config = {
- 'cls': 'local',
+ 'cls': 'memory',
'args': {
- 'db': swh_indexer_storage_postgresql.dsn,
},
}
return get_indexer_storage(**storage_config)
diff --git a/swh/indexer/tests/storage/test_storage.py b/swh/indexer/tests/storage/test_storage.py
--- a/swh/indexer/tests/storage/test_storage.py
+++ b/swh/indexer/tests/storage/test_storage.py
@@ -349,6 +349,8 @@
"""mimetype_get_range paginates results if limit exceeded"""
storage, data = swh_indexer_storage_with_data
+ indexer_configuration_id = data.tools['file']['id']
+
# input the list of sha1s we want from storage
content_ids = sorted(
[c['id'] for c in data.mimetypes])
@@ -360,7 +362,7 @@
# retrieve mimetypes limited to 10 results
actual_result = storage.content_mimetype_get_range(
start, end,
- indexer_configuration_id=1,
+ indexer_configuration_id=indexer_configuration_id,
limit=10)
assert actual_result
@@ -377,7 +379,8 @@
# retrieve next part
actual_result = storage.content_mimetype_get_range(
- start=end, end=end, indexer_configuration_id=1)
+ start=end, end=end,
+ indexer_configuration_id=indexer_configuration_id)
assert set(actual_result.keys()) == {'ids', 'next'}
actual_ids = actual_result['ids']
actual_next = actual_result['next']

File Metadata

Mime Type
text/plain
Expires
Thu, Jul 3, 3:28 PM (1 w, 2 d ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3230604

Event Timeline

About · Developer information · Legal