diff --git a/conftest.py b/conftest.py index 1e93ce8..8392690 100644 --- a/conftest.py +++ b/conftest.py @@ -1,27 +1,31 @@ # Copyright (C) 2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from hypothesis import settings import pytest # define tests profile. Full documentation is at: # https://hypothesis.readthedocs.io/en/latest/settings.html#settings-profiles settings.register_profile("fast", max_examples=5, deadline=5000) settings.register_profile("slow", max_examples=20, deadline=5000) # Ignore the following modules because wsgi module fails as no # configuration file is found (--doctest-modules forces the module # loading) collect_ignore = ["swh/indexer/storage/api/wsgi.py"] -# we use the swh_scheduler fixture -pytest_plugins = ["swh.scheduler.pytest_plugin"] +# we use the various swh fixtures +pytest_plugins = [ + "swh.scheduler.pytest_plugin", + "swh.storage.pytest_plugin", + "swh.core.db.pytest_plugin", +] @pytest.fixture(scope="session") def swh_scheduler_celery_includes(swh_scheduler_celery_includes): return swh_scheduler_celery_includes + [ "swh.indexer.tasks", ] diff --git a/requirements-swh.txt b/requirements-swh.txt index 40b5ff6..854fbce 100644 --- a/requirements-swh.txt +++ b/requirements-swh.txt @@ -1,6 +1,6 @@ -swh.core[db,http] >= 0.3 +swh.core[db,http] >= 0.9.1 swh.model >= 0.0.15 swh.objstorage >= 0.2.2 swh.scheduler >= 0.5.2 swh.storage >= 0.12.0 swh.journal >= 0.1.0 diff --git a/swh/indexer/storage/db.py b/swh/indexer/storage/db.py index ec2e084..6f41d44 100644 --- a/swh/indexer/storage/db.py +++ b/swh/indexer/storage/db.py @@ -1,550 +1,550 @@ # Copyright (C) 2015-2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from typing import Dict, Iterable, Iterator, List from swh.core.db import BaseDb from swh.core.db.db_utils import execute_values_generator, stored_procedure from swh.model import hashutil from .interface import Sha1 class Db(BaseDb): """Proxy to the SWH Indexer DB, with wrappers around stored procedures """ content_mimetype_hash_keys = ["id", "indexer_configuration_id"] def _missing_from_list( self, table: str, data: Iterable[Dict], hash_keys: List[str], cur=None ): """Read from table the data with hash_keys that are missing. Args: table: Table name (e.g content_mimetype, content_language, etc...) data: Dict of data to read from hash_keys: List of keys to read in the data dict. Yields: The data which is missing from the db. """ cur = self._cursor(cur) keys = ", ".join(hash_keys) equality = " AND ".join(("t.%s = c.%s" % (key, key)) for key in hash_keys) yield from execute_values_generator( cur, """ select %s from (values %%s) as t(%s) where not exists ( select 1 from %s c where %s ) """ % (keys, keys, table, equality), (tuple(m[k] for k in hash_keys) for m in data), ) def content_mimetype_missing_from_list( self, mimetypes: Iterable[Dict], cur=None ) -> Iterator[Sha1]: """List missing mimetypes. """ yield from self._missing_from_list( "content_mimetype", mimetypes, self.content_mimetype_hash_keys, cur=cur ) content_mimetype_cols = [ "id", "mimetype", "encoding", "tool_id", "tool_name", "tool_version", "tool_configuration", ] @stored_procedure("swh_mktemp_content_mimetype") def mktemp_content_mimetype(self, cur=None): pass def content_mimetype_add_from_temp(self, cur=None): cur = self._cursor(cur) cur.execute("select * from swh_content_mimetype_add()") return cur.fetchone()[0] def _convert_key(self, key, main_table="c"): """Convert keys according to specific use in the module. Args: key (str): Key expression to change according to the alias used in the query main_table (str): Alias to use for the main table. Default to c for content_{something}. Expected: Tables content_{something} being aliased as 'c' (something in {language, mimetype, ...}), table indexer_configuration being aliased as 'i'. """ if key == "id": return "%s.id" % main_table elif key == "tool_id": return "i.id as tool_id" elif key == "license": return ( """ ( select name from fossology_license where id = %s.license_id ) as licenses""" % main_table ) return key def _get_from_list(self, table, ids, cols, cur=None, id_col="id"): """Fetches entries from the `table` such that their `id` field (or whatever is given to `id_col`) is in `ids`. Returns the columns `cols`. The `cur` parameter is used to connect to the database. """ cur = self._cursor(cur) keys = map(self._convert_key, cols) query = """ select {keys} from (values %s) as t(id) inner join {table} c on c.{id_col}=t.id inner join indexer_configuration i on c.indexer_configuration_id=i.id; """.format( keys=", ".join(keys), id_col=id_col, table=table ) yield from execute_values_generator(cur, query, ((_id,) for _id in ids)) content_indexer_names = { "mimetype": "content_mimetype", "fossology_license": "content_fossology_license", } def content_get_range( self, content_type, start, end, indexer_configuration_id, limit=1000, with_textual_data=False, cur=None, ): """Retrieve contents with content_type, within range [start, end] bound by limit and associated to the given indexer configuration id. When asking to work on textual content, that filters on the mimetype table with any mimetype that is not binary. """ cur = self._cursor(cur) table = self.content_indexer_names[content_type] if with_textual_data: extra = """inner join content_mimetype cm on (t.id=cm.id and cm.mimetype like 'text/%%' and %(start)s <= cm.id and cm.id <= %(end)s) """ else: extra = "" query = f"""select t.id from {table} t {extra} where t.indexer_configuration_id=%(tool_id)s and %(start)s <= t.id and t.id <= %(end)s order by t.indexer_configuration_id, t.id limit %(limit)s""" cur.execute( query, { "start": start, "end": end, "tool_id": indexer_configuration_id, "limit": limit, }, ) yield from cur def content_mimetype_get_from_list(self, ids, cur=None): yield from self._get_from_list( "content_mimetype", ids, self.content_mimetype_cols, cur=cur ) content_language_hash_keys = ["id", "indexer_configuration_id"] def content_language_missing_from_list(self, languages, cur=None): """List missing languages. """ yield from self._missing_from_list( "content_language", languages, self.content_language_hash_keys, cur=cur ) content_language_cols = [ "id", "lang", "tool_id", "tool_name", "tool_version", "tool_configuration", ] @stored_procedure("swh_mktemp_content_language") def mktemp_content_language(self, cur=None): pass def content_language_add_from_temp(self, cur=None): cur = self._cursor(cur) cur.execute("select * from swh_content_language_add()") return cur.fetchone()[0] def content_language_get_from_list(self, ids, cur=None): yield from self._get_from_list( "content_language", ids, self.content_language_cols, cur=cur ) content_ctags_hash_keys = ["id", "indexer_configuration_id"] def content_ctags_missing_from_list(self, ctags, cur=None): """List missing ctags. """ yield from self._missing_from_list( "content_ctags", ctags, self.content_ctags_hash_keys, cur=cur ) content_ctags_cols = [ "id", "name", "kind", "line", "lang", "tool_id", "tool_name", "tool_version", "tool_configuration", ] @stored_procedure("swh_mktemp_content_ctags") def mktemp_content_ctags(self, cur=None): pass def content_ctags_add_from_temp(self, cur=None): cur = self._cursor(cur) cur.execute("select * from swh_content_ctags_add()") return cur.fetchone()[0] def content_ctags_get_from_list(self, ids, cur=None): cur = self._cursor(cur) keys = map(self._convert_key, self.content_ctags_cols) yield from execute_values_generator( cur, """ select %s from (values %%s) as t(id) inner join content_ctags c on c.id=t.id inner join indexer_configuration i on c.indexer_configuration_id=i.id order by line """ % ", ".join(keys), ((_id,) for _id in ids), ) def content_ctags_search(self, expression, last_sha1, limit, cur=None): cur = self._cursor(cur) if not last_sha1: query = """SELECT %s FROM swh_content_ctags_search(%%s, %%s)""" % ( ",".join(self.content_ctags_cols) ) cur.execute(query, (expression, limit)) else: if last_sha1 and isinstance(last_sha1, bytes): last_sha1 = "\\x%s" % hashutil.hash_to_hex(last_sha1) elif last_sha1: last_sha1 = "\\x%s" % last_sha1 query = """SELECT %s FROM swh_content_ctags_search(%%s, %%s, %%s)""" % ( ",".join(self.content_ctags_cols) ) cur.execute(query, (expression, limit, last_sha1)) yield from cur content_fossology_license_cols = [ "id", "tool_id", "tool_name", "tool_version", "tool_configuration", "license", ] @stored_procedure("swh_mktemp_content_fossology_license") def mktemp_content_fossology_license(self, cur=None): pass def content_fossology_license_add_from_temp(self, cur=None): """Add new licenses per content. """ cur = self._cursor(cur) cur.execute("select * from swh_content_fossology_license_add()") return cur.fetchone()[0] def content_fossology_license_get_from_list(self, ids, cur=None): """Retrieve licenses per id. """ cur = self._cursor(cur) keys = map(self._convert_key, self.content_fossology_license_cols) yield from execute_values_generator( cur, """ select %s from (values %%s) as t(id) inner join content_fossology_license c on t.id=c.id inner join indexer_configuration i on i.id=c.indexer_configuration_id """ % ", ".join(keys), ((_id,) for _id in ids), ) content_metadata_hash_keys = ["id", "indexer_configuration_id"] def content_metadata_missing_from_list(self, metadata, cur=None): """List missing metadata. """ yield from self._missing_from_list( "content_metadata", metadata, self.content_metadata_hash_keys, cur=cur ) content_metadata_cols = [ "id", "metadata", "tool_id", "tool_name", "tool_version", "tool_configuration", ] @stored_procedure("swh_mktemp_content_metadata") def mktemp_content_metadata(self, cur=None): pass def content_metadata_add_from_temp(self, cur=None): cur = self._cursor(cur) cur.execute("select * from swh_content_metadata_add()") return cur.fetchone()[0] def content_metadata_get_from_list(self, ids, cur=None): yield from self._get_from_list( "content_metadata", ids, self.content_metadata_cols, cur=cur ) revision_intrinsic_metadata_hash_keys = ["id", "indexer_configuration_id"] def revision_intrinsic_metadata_missing_from_list(self, metadata, cur=None): """List missing metadata. """ yield from self._missing_from_list( "revision_intrinsic_metadata", metadata, self.revision_intrinsic_metadata_hash_keys, cur=cur, ) revision_intrinsic_metadata_cols = [ "id", "metadata", "mappings", "tool_id", "tool_name", "tool_version", "tool_configuration", ] @stored_procedure("swh_mktemp_revision_intrinsic_metadata") def mktemp_revision_intrinsic_metadata(self, cur=None): pass def revision_intrinsic_metadata_add_from_temp(self, cur=None): cur = self._cursor(cur) cur.execute("select * from swh_revision_intrinsic_metadata_add()") return cur.fetchone()[0] def revision_intrinsic_metadata_get_from_list(self, ids, cur=None): yield from self._get_from_list( "revision_intrinsic_metadata", ids, self.revision_intrinsic_metadata_cols, cur=cur, ) origin_intrinsic_metadata_cols = [ "id", "metadata", "from_revision", "mappings", "tool_id", "tool_name", "tool_version", "tool_configuration", ] origin_intrinsic_metadata_regconfig = "pg_catalog.simple" """The dictionary used to normalize 'metadata' and queries. 'pg_catalog.simple' provides no stopword, so it should be suitable for proper names and non-English content. When updating this value, make sure to add a new index on origin_intrinsic_metadata.metadata.""" @stored_procedure("swh_mktemp_origin_intrinsic_metadata") def mktemp_origin_intrinsic_metadata(self, cur=None): pass def origin_intrinsic_metadata_add_from_temp(self, cur=None): cur = self._cursor(cur) cur.execute("select * from swh_origin_intrinsic_metadata_add()") return cur.fetchone()[0] def origin_intrinsic_metadata_get_from_list(self, ids, cur=None): yield from self._get_from_list( "origin_intrinsic_metadata", ids, self.origin_intrinsic_metadata_cols, cur=cur, id_col="id", ) def origin_intrinsic_metadata_search_fulltext(self, terms, *, limit, cur): regconfig = self.origin_intrinsic_metadata_regconfig tsquery_template = " && ".join( "plainto_tsquery('%s', %%s)" % regconfig for _ in terms ) tsquery_args = [(term,) for term in terms] keys = ( self._convert_key(col, "oim") for col in self.origin_intrinsic_metadata_cols ) query = ( "SELECT {keys} FROM origin_intrinsic_metadata AS oim " "INNER JOIN indexer_configuration AS i " "ON oim.indexer_configuration_id=i.id " "JOIN LATERAL (SELECT {tsquery_template}) AS s(tsq) ON true " "WHERE oim.metadata_tsvector @@ tsq " "ORDER BY ts_rank(oim.metadata_tsvector, tsq, 1) DESC " "LIMIT %s;" ).format(keys=", ".join(keys), tsquery_template=tsquery_template) cur.execute(query, tsquery_args + [limit]) yield from cur def origin_intrinsic_metadata_search_by_producer( self, last, limit, ids_only, mappings, tool_ids, cur ): if ids_only: keys = "oim.id" else: keys = ", ".join( ( self._convert_key(col, "oim") for col in self.origin_intrinsic_metadata_cols ) ) query_parts = [ "SELECT %s" % keys, "FROM origin_intrinsic_metadata AS oim", "INNER JOIN indexer_configuration AS i", "ON oim.indexer_configuration_id=i.id", ] args = [] where = [] if last: where.append("oim.id > %s") args.append(last) if mappings is not None: where.append("oim.mappings && %s") - args.append(mappings) + args.append(list(mappings)) if tool_ids is not None: where.append("oim.indexer_configuration_id = ANY(%s)") - args.append(tool_ids) + args.append(list(tool_ids)) if where: query_parts.append("WHERE") query_parts.append(" AND ".join(where)) if limit: query_parts.append("LIMIT %s") args.append(limit) cur.execute(" ".join(query_parts), args) yield from cur indexer_configuration_cols = [ "id", "tool_name", "tool_version", "tool_configuration", ] @stored_procedure("swh_mktemp_indexer_configuration") def mktemp_indexer_configuration(self, cur=None): pass def indexer_configuration_add_from_temp(self, cur=None): cur = self._cursor(cur) cur.execute( "SELECT %s from swh_indexer_configuration_add()" % (",".join(self.indexer_configuration_cols),) ) yield from cur def indexer_configuration_get( self, tool_name, tool_version, tool_configuration, cur=None ): cur = self._cursor(cur) cur.execute( """select %s from indexer_configuration where tool_name=%%s and tool_version=%%s and tool_configuration=%%s""" % (",".join(self.indexer_configuration_cols)), (tool_name, tool_version, tool_configuration), ) return cur.fetchone() def indexer_configuration_get_from_id(self, id_, cur=None): cur = self._cursor(cur) cur.execute( """select %s from indexer_configuration where id=%%s""" % (",".join(self.indexer_configuration_cols)), (id_,), ) return cur.fetchone() diff --git a/swh/indexer/tests/conftest.py b/swh/indexer/tests/conftest.py index e16588b..270812d 100644 --- a/swh/indexer/tests/conftest.py +++ b/swh/indexer/tests/conftest.py @@ -1,105 +1,127 @@ # Copyright (C) 2019-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from datetime import timedelta import os +from os import path from typing import List, Tuple from unittest.mock import patch import pytest import yaml +from swh.core.db.pytest_plugin import postgresql_fact +import swh.indexer from swh.indexer.storage import get_indexer_storage from swh.objstorage.factory import get_objstorage from swh.storage import get_storage from .utils import fill_obj_storage, fill_storage TASK_NAMES: List[Tuple[str, str]] = [ # (scheduler-task-type, task-class-test-name) ("index-revision-metadata", "revision_intrinsic_metadata"), ("index-origin-metadata", "origin_intrinsic_metadata"), ] +SQL_FILES = path.join(path.dirname(swh.indexer.__file__), "sql", "*.sql") + + +idx_storage_postgresql = postgresql_fact( + "postgresql_proc", db_name="indexer_storage", dump_files=SQL_FILES, +) + + @pytest.fixture def indexer_scheduler(swh_scheduler): # Insert the expected task types within the scheduler for task_name, task_class_name in TASK_NAMES: swh_scheduler.create_task_type( { "type": task_name, "description": f"The {task_class_name} indexer testing task", "backend_name": f"swh.indexer.tests.tasks.{task_class_name}", "default_interval": timedelta(days=1), "min_interval": timedelta(hours=6), "max_interval": timedelta(days=12), "num_retries": 3, } ) return swh_scheduler @pytest.fixture -def idx_storage(): +def idx_storage_backend_config(idx_storage_postgresql): + """Basic pg storage configuration with no journal collaborator for the indexer + storage (to avoid pulling optional dependency on clients of this fixture) + + """ + return { + "cls": "local", + "db": idx_storage_postgresql.dsn, + } + + +@pytest.fixture +def swh_indexer_config( + swh_storage_backend_config, idx_storage_backend_config, swh_scheduler_config +): + return { + "storage": swh_storage_backend_config, + "objstorage": {"cls": "memory"}, + "indexer_storage": idx_storage_backend_config, + "scheduler": {"cls": "local", **swh_scheduler_config}, + "tools": { + "name": "file", + "version": "1:5.30-1+deb9u1", + "configuration": {"type": "library", "debian-package": "python3-magic"}, + }, + "compute_checksums": ["blake2b512"], # for rehash indexer + } + + +@pytest.fixture +def idx_storage(swh_indexer_config): """An instance of in-memory indexer storage that gets injected into all indexers classes. """ - idx_storage = get_indexer_storage("memory") - with patch("swh.indexer.storage.in_memory.IndexerStorage") as idx_storage_mock: - idx_storage_mock.return_value = idx_storage - yield idx_storage + idx_storage_config = swh_indexer_config["indexer_storage"] + return get_indexer_storage(**idx_storage_config) @pytest.fixture -def storage(): +def storage(swh_indexer_config): """An instance of in-memory storage that gets injected into all indexers classes. """ - storage = get_storage(cls="memory") + storage = get_storage(**swh_indexer_config["storage"]) fill_storage(storage) - with patch("swh.storage.in_memory.InMemoryStorage") as storage_mock: - storage_mock.return_value = storage - yield storage + return storage @pytest.fixture -def obj_storage(): +def obj_storage(swh_indexer_config): """An instance of in-memory objstorage that gets injected into all indexers classes. """ - objstorage = get_objstorage("memory") + objstorage = get_objstorage(**swh_indexer_config["objstorage"]) fill_obj_storage(objstorage) with patch.dict( "swh.objstorage.factory._STORAGE_CLASSES", {"memory": lambda: objstorage} ): yield objstorage -@pytest.fixture -def swh_indexer_config(): - return { - "storage": {"cls": "memory"}, - "objstorage": {"cls": "memory"}, - "indexer_storage": {"cls": "memory"}, - "tools": { - "name": "file", - "version": "1:5.30-1+deb9u1", - "configuration": {"type": "library", "debian-package": "python3-magic"}, - }, - "compute_checksums": ["blake2b512"], # for rehash indexer - } - - @pytest.fixture def swh_config(swh_indexer_config, monkeypatch, tmp_path): conffile = os.path.join(str(tmp_path), "indexer.yml") with open(conffile, "w") as f: f.write(yaml.dump(swh_indexer_config)) monkeypatch.setenv("SWH_CONFIG_FILENAME", conffile) return conffile diff --git a/swh/indexer/tests/test_cli.py b/swh/indexer/tests/test_cli.py index 3deca15..2fe3ca2 100644 --- a/swh/indexer/tests/test_cli.py +++ b/swh/indexer/tests/test_cli.py @@ -1,397 +1,432 @@ # Copyright (C) 2019-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from functools import reduce import re -import tempfile from typing import Any, Dict, List from unittest.mock import patch from click.testing import CliRunner from confluent_kafka import Consumer, Producer import pytest from swh.indexer.cli import indexer_cli_group from swh.indexer.storage.interface import IndexerStorageInterface from swh.indexer.storage.model import ( OriginIntrinsicMetadataRow, RevisionIntrinsicMetadataRow, ) from swh.journal.serializers import value_to_kafka from swh.model.hashutil import hash_to_bytes -CLI_CONFIG = """ -scheduler: - cls: foo - args: {} -storage: - cls: memory -indexer_storage: - cls: memory -""" - def fill_idx_storage(idx_storage: IndexerStorageInterface, nb_rows: int) -> List[int]: tools: List[Dict[str, Any]] = [ {"tool_name": "tool %d" % i, "tool_version": "0.0.1", "tool_configuration": {},} for i in range(2) ] tools = idx_storage.indexer_configuration_add(tools) origin_metadata = [ OriginIntrinsicMetadataRow( id="file://dev/%04d" % origin_id, - from_revision=hash_to_bytes("abcd{:0>4}".format(origin_id)), + from_revision=hash_to_bytes("abcd{:0>36}".format(origin_id)), indexer_configuration_id=tools[origin_id % 2]["id"], metadata={"name": "origin %d" % origin_id}, mappings=["mapping%d" % (origin_id % 10)], ) for origin_id in range(nb_rows) ] revision_metadata = [ RevisionIntrinsicMetadataRow( - id=hash_to_bytes("abcd{:0>4}".format(origin_id)), + id=hash_to_bytes("abcd{:0>36}".format(origin_id)), indexer_configuration_id=tools[origin_id % 2]["id"], metadata={"name": "origin %d" % origin_id}, mappings=["mapping%d" % (origin_id % 10)], ) for origin_id in range(nb_rows) ] idx_storage.revision_intrinsic_metadata_add(revision_metadata) idx_storage.origin_intrinsic_metadata_add(origin_metadata) return [tool["id"] for tool in tools] def _origins_in_task_args(tasks): """Returns the set of origins contained in the arguments of the provided tasks (assumed to be of type index-origin-metadata).""" return reduce( set.union, (set(task["arguments"]["args"][0]) for task in tasks), set() ) def _assert_tasks_for_origins(tasks, origins): expected_kwargs = {} assert {task["type"] for task in tasks} == {"index-origin-metadata"} assert all(len(task["arguments"]["args"]) == 1 for task in tasks) for task in tasks: assert task["arguments"]["kwargs"] == expected_kwargs, task assert _origins_in_task_args(tasks) == set(["file://dev/%04d" % i for i in origins]) -def invoke(scheduler, catch_exceptions, args): - runner = CliRunner() - with patch( - "swh.scheduler.get_scheduler" - ) as get_scheduler_mock, tempfile.NamedTemporaryFile( - "a", suffix=".yml" - ) as config_fd: - config_fd.write(CLI_CONFIG) - config_fd.seek(0) - get_scheduler_mock.return_value = scheduler - result = runner.invoke(indexer_cli_group, ["-C" + config_fd.name] + args) - if not catch_exceptions and result.exception: - print(result.output) - raise result.exception - return result - - -def test_mapping_list(indexer_scheduler): - result = invoke(indexer_scheduler, False, ["mapping", "list",]) +@pytest.fixture +def cli_runner(): + return CliRunner() + + +def test_cli_mapping_list(cli_runner, swh_config): + result = cli_runner.invoke( + indexer_cli_group, + ["-C", swh_config, "mapping", "list"], + catch_exceptions=False, + ) expected_output = "\n".join( ["codemeta", "gemspec", "maven", "npm", "pkg-info", "",] ) assert result.exit_code == 0, result.output assert result.output == expected_output -def test_mapping_list_terms(indexer_scheduler): - result = invoke(indexer_scheduler, False, ["mapping", "list-terms",]) +def test_cli_mapping_list_terms(cli_runner, swh_config): + result = cli_runner.invoke( + indexer_cli_group, + ["-C", swh_config, "mapping", "list-terms"], + catch_exceptions=False, + ) assert result.exit_code == 0, result.output assert re.search(r"http://schema.org/url:\n.*npm", result.output) assert re.search(r"http://schema.org/url:\n.*codemeta", result.output) assert re.search( r"https://codemeta.github.io/terms/developmentStatus:\n\tcodemeta", result.output, ) -def test_mapping_list_terms_exclude(indexer_scheduler): - result = invoke( - indexer_scheduler, - False, - ["mapping", "list-terms", "--exclude-mapping", "codemeta"], +def test_cli_mapping_list_terms_exclude(cli_runner, swh_config): + result = cli_runner.invoke( + indexer_cli_group, + ["-C", swh_config, "mapping", "list-terms", "--exclude-mapping", "codemeta"], + catch_exceptions=False, ) assert result.exit_code == 0, result.output assert re.search(r"http://schema.org/url:\n.*npm", result.output) assert not re.search(r"http://schema.org/url:\n.*codemeta", result.output) assert not re.search( r"https://codemeta.github.io/terms/developmentStatus:\n\tcodemeta", result.output, ) @patch("swh.scheduler.cli.utils.TASK_BATCH_SIZE", 3) @patch("swh.scheduler.cli_utils.TASK_BATCH_SIZE", 3) -def test_origin_metadata_reindex_empty_db(indexer_scheduler, idx_storage, storage): - result = invoke(indexer_scheduler, False, ["schedule", "reindex_origin_metadata",]) +def test_cli_origin_metadata_reindex_empty_db( + cli_runner, swh_config, indexer_scheduler, idx_storage, storage +): + result = cli_runner.invoke( + indexer_cli_group, + ["-C", swh_config, "schedule", "reindex_origin_metadata",], + catch_exceptions=False, + ) expected_output = "Nothing to do (no origin metadata matched the criteria).\n" assert result.exit_code == 0, result.output assert result.output == expected_output tasks = indexer_scheduler.search_tasks() assert len(tasks) == 0 @patch("swh.scheduler.cli.utils.TASK_BATCH_SIZE", 3) @patch("swh.scheduler.cli_utils.TASK_BATCH_SIZE", 3) -def test_origin_metadata_reindex_divisor(indexer_scheduler, idx_storage, storage): +def test_cli_origin_metadata_reindex_divisor( + cli_runner, swh_config, indexer_scheduler, idx_storage, storage +): """Tests the re-indexing when origin_batch_size*task_batch_size is a divisor of nb_origins.""" fill_idx_storage(idx_storage, 90) - result = invoke(indexer_scheduler, False, ["schedule", "reindex_origin_metadata",]) + result = cli_runner.invoke( + indexer_cli_group, + ["-C", swh_config, "schedule", "reindex_origin_metadata",], + catch_exceptions=False, + ) # Check the output expected_output = ( "Scheduled 3 tasks (30 origins).\n" "Scheduled 6 tasks (60 origins).\n" "Scheduled 9 tasks (90 origins).\n" "Done.\n" ) assert result.exit_code == 0, result.output assert result.output == expected_output # Check scheduled tasks tasks = indexer_scheduler.search_tasks() assert len(tasks) == 9 _assert_tasks_for_origins(tasks, range(90)) @patch("swh.scheduler.cli.utils.TASK_BATCH_SIZE", 3) @patch("swh.scheduler.cli_utils.TASK_BATCH_SIZE", 3) -def test_origin_metadata_reindex_dry_run(indexer_scheduler, idx_storage, storage): +def test_cli_origin_metadata_reindex_dry_run( + cli_runner, swh_config, indexer_scheduler, idx_storage, storage +): """Tests the re-indexing when origin_batch_size*task_batch_size is a divisor of nb_origins.""" fill_idx_storage(idx_storage, 90) - result = invoke( - indexer_scheduler, False, ["schedule", "--dry-run", "reindex_origin_metadata",] + result = cli_runner.invoke( + indexer_cli_group, + ["-C", swh_config, "schedule", "--dry-run", "reindex_origin_metadata",], + catch_exceptions=False, ) # Check the output expected_output = ( "Scheduled 3 tasks (30 origins).\n" "Scheduled 6 tasks (60 origins).\n" "Scheduled 9 tasks (90 origins).\n" "Done.\n" ) assert result.exit_code == 0, result.output assert result.output == expected_output # Check scheduled tasks tasks = indexer_scheduler.search_tasks() assert len(tasks) == 0 @patch("swh.scheduler.cli.utils.TASK_BATCH_SIZE", 3) @patch("swh.scheduler.cli_utils.TASK_BATCH_SIZE", 3) -def test_origin_metadata_reindex_nondivisor(indexer_scheduler, idx_storage, storage): +def test_cli_origin_metadata_reindex_nondivisor( + cli_runner, swh_config, indexer_scheduler, idx_storage, storage +): """Tests the re-indexing when neither origin_batch_size or task_batch_size is a divisor of nb_origins.""" fill_idx_storage(idx_storage, 70) - result = invoke( - indexer_scheduler, - False, - ["schedule", "reindex_origin_metadata", "--batch-size", "20",], + result = cli_runner.invoke( + indexer_cli_group, + [ + "-C", + swh_config, + "schedule", + "reindex_origin_metadata", + "--batch-size", + "20", + ], + catch_exceptions=False, ) # Check the output expected_output = ( "Scheduled 3 tasks (60 origins).\n" "Scheduled 4 tasks (70 origins).\n" "Done.\n" ) assert result.exit_code == 0, result.output assert result.output == expected_output # Check scheduled tasks tasks = indexer_scheduler.search_tasks() assert len(tasks) == 4 _assert_tasks_for_origins(tasks, range(70)) @patch("swh.scheduler.cli.utils.TASK_BATCH_SIZE", 3) @patch("swh.scheduler.cli_utils.TASK_BATCH_SIZE", 3) -def test_origin_metadata_reindex_filter_one_mapping( - indexer_scheduler, idx_storage, storage +def test_cli_origin_metadata_reindex_filter_one_mapping( + cli_runner, swh_config, indexer_scheduler, idx_storage, storage ): """Tests the re-indexing when origin_batch_size*task_batch_size is a divisor of nb_origins.""" fill_idx_storage(idx_storage, 110) - result = invoke( - indexer_scheduler, - False, - ["schedule", "reindex_origin_metadata", "--mapping", "mapping1",], + result = cli_runner.invoke( + indexer_cli_group, + [ + "-C", + swh_config, + "schedule", + "reindex_origin_metadata", + "--mapping", + "mapping1", + ], + catch_exceptions=False, ) # Check the output expected_output = "Scheduled 2 tasks (11 origins).\nDone.\n" assert result.exit_code == 0, result.output assert result.output == expected_output # Check scheduled tasks tasks = indexer_scheduler.search_tasks() assert len(tasks) == 2 _assert_tasks_for_origins(tasks, [1, 11, 21, 31, 41, 51, 61, 71, 81, 91, 101]) @patch("swh.scheduler.cli.utils.TASK_BATCH_SIZE", 3) @patch("swh.scheduler.cli_utils.TASK_BATCH_SIZE", 3) -def test_origin_metadata_reindex_filter_two_mappings( - indexer_scheduler, idx_storage, storage +def test_cli_origin_metadata_reindex_filter_two_mappings( + cli_runner, swh_config, indexer_scheduler, idx_storage, storage ): """Tests the re-indexing when origin_batch_size*task_batch_size is a divisor of nb_origins.""" fill_idx_storage(idx_storage, 110) - result = invoke( - indexer_scheduler, - False, + result = cli_runner.invoke( + indexer_cli_group, [ + "--config-file", + swh_config, "schedule", "reindex_origin_metadata", "--mapping", "mapping1", "--mapping", "mapping2", ], + catch_exceptions=False, ) # Check the output expected_output = "Scheduled 3 tasks (22 origins).\nDone.\n" assert result.exit_code == 0, result.output assert result.output == expected_output # Check scheduled tasks tasks = indexer_scheduler.search_tasks() assert len(tasks) == 3 _assert_tasks_for_origins( tasks, [ 1, 11, 21, 31, 41, 51, 61, 71, 81, 91, 101, 2, 12, 22, 32, 42, 52, 62, 72, 82, 92, 102, ], ) @patch("swh.scheduler.cli.utils.TASK_BATCH_SIZE", 3) @patch("swh.scheduler.cli_utils.TASK_BATCH_SIZE", 3) -def test_origin_metadata_reindex_filter_one_tool( - indexer_scheduler, idx_storage, storage +def test_cli_origin_metadata_reindex_filter_one_tool( + cli_runner, swh_config, indexer_scheduler, idx_storage, storage ): """Tests the re-indexing when origin_batch_size*task_batch_size is a divisor of nb_origins.""" tool_ids = fill_idx_storage(idx_storage, 110) - result = invoke( - indexer_scheduler, - False, - ["schedule", "reindex_origin_metadata", "--tool-id", str(tool_ids[0]),], + result = cli_runner.invoke( + indexer_cli_group, + [ + "-C", + swh_config, + "schedule", + "reindex_origin_metadata", + "--tool-id", + str(tool_ids[0]), + ], + catch_exceptions=False, ) # Check the output expected_output = ( "Scheduled 3 tasks (30 origins).\n" "Scheduled 6 tasks (55 origins).\n" "Done.\n" ) assert result.exit_code == 0, result.output assert result.output == expected_output # Check scheduled tasks tasks = indexer_scheduler.search_tasks() assert len(tasks) == 6 _assert_tasks_for_origins(tasks, [x * 2 for x in range(55)]) -def test_journal_client( - storage, indexer_scheduler, kafka_prefix: str, kafka_server, consumer: Consumer +def test_cli_journal_client( + cli_runner, + swh_config, + indexer_scheduler, + kafka_prefix: str, + kafka_server, + consumer: Consumer, ): """Test the 'swh indexer journal-client' cli tool.""" producer = Producer( { "bootstrap.servers": kafka_server, "client.id": "test producer", "acks": "all", } ) STATUS = {"status": "full", "origin": {"url": "file://dev/0000",}} producer.produce( topic=f"{kafka_prefix}.origin_visit_status", key=b"bogus", value=value_to_kafka(STATUS), ) - result = invoke( - indexer_scheduler, - False, + result = cli_runner.invoke( + indexer_cli_group, [ + "-C", + swh_config, "journal-client", "--stop-after-objects", "1", "--broker", kafka_server, "--prefix", kafka_prefix, "--group-id", "test-consumer", ], + catch_exceptions=False, ) # Check the output expected_output = "Done.\n" assert result.exit_code == 0, result.output assert result.output == expected_output # Check scheduled tasks tasks = indexer_scheduler.search_tasks() assert len(tasks) == 1 _assert_tasks_for_origins(tasks, [0]) -def test_journal_client_without_brokers( - storage, indexer_scheduler, kafka_prefix: str, kafka_server, consumer: Consumer +def test_cli_journal_client_without_brokers( + cli_runner, swh_config, kafka_prefix: str, kafka_server, consumer: Consumer ): """Without brokers configuration, the cli fails.""" with pytest.raises(ValueError, match="brokers"): - invoke( - indexer_scheduler, False, ["journal-client",], + cli_runner.invoke( + indexer_cli_group, + ["-C", swh_config, "journal-client",], + catch_exceptions=False, ) diff --git a/swh/indexer/tests/test_origin_head.py b/swh/indexer/tests/test_origin_head.py index 3ad457c..67b5b05 100644 --- a/swh/indexer/tests/test_origin_head.py +++ b/swh/indexer/tests/test_origin_head.py @@ -1,190 +1,176 @@ # Copyright (C) 2017-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import copy from datetime import datetime, timezone import unittest import pytest from swh.indexer.origin_head import OriginHeadIndexer from swh.indexer.tests.utils import fill_storage from swh.model.model import ( Origin, OriginVisit, OriginVisitStatus, Snapshot, SnapshotBranch, TargetType, ) from swh.storage.utils import now @pytest.fixture def swh_indexer_config(swh_indexer_config): config = copy.deepcopy(swh_indexer_config) config.update( { "tools": { "name": "origin-metadata", "version": "0.0.1", "configuration": {}, }, "tasks": { "revision_intrinsic_metadata": None, "origin_intrinsic_metadata": None, }, } ) return config class OriginHeadTestIndexer(OriginHeadIndexer): """Specific indexer whose configuration is enough to satisfy the indexing tests. """ def persist_index_computations(self, results): self.results = results +SAMPLE_SNAPSHOT = Snapshot( + branches={ + b"foo": None, + b"HEAD": SnapshotBranch(target_type=TargetType.ALIAS, target=b"foo",), + }, +) + + class OriginHead(unittest.TestCase): @pytest.fixture(autouse=True) def init(self, swh_config): super().setUp() self.indexer = OriginHeadTestIndexer() self.indexer.catch_exceptions = False fill_storage(self.indexer.storage) def test_git(self): origin_url = "https://github.com/SoftwareHeritage/swh-storage" self.indexer.run([origin_url]) rev_id = b"8K\x12\x00d\x03\xcc\xe4]bS\xe3\x8f{\xd7}\xac\xefrm" self.assertEqual( self.indexer.results, [{"revision_id": rev_id, "origin_url": origin_url,}], ) def test_git_partial_snapshot(self): """Checks partial snapshots are ignored.""" origin_url = "https://github.com/SoftwareHeritage/swh-core" self.indexer.storage.origin_add([Origin(url=origin_url)]) visit = self.indexer.storage.origin_visit_add( [ OriginVisit( origin=origin_url, date=datetime(2019, 2, 27, tzinfo=timezone.utc), type="git", ) ] )[0] - self.indexer.storage.snapshot_add( - [ - Snapshot( - branches={ - b"foo": None, - b"HEAD": SnapshotBranch( - target_type=TargetType.ALIAS, target=b"foo", - ), - }, - ), - ] - ) + self.indexer.storage.snapshot_add([SAMPLE_SNAPSHOT]) visit_status = OriginVisitStatus( origin=origin_url, visit=visit.visit, date=now(), status="partial", - snapshot=b"foo", + snapshot=SAMPLE_SNAPSHOT.id, ) self.indexer.storage.origin_visit_status_add([visit_status]) self.indexer.run([origin_url]) self.assertEqual(self.indexer.results, []) def test_vcs_missing_snapshot(self): origin_url = "https://github.com/SoftwareHeritage/swh-indexer" self.indexer.storage.origin_add([Origin(url=origin_url)]) self.indexer.run([origin_url]) self.assertEqual(self.indexer.results, []) def test_pypi_missing_branch(self): origin_url = "https://pypi.org/project/abcdef/" self.indexer.storage.origin_add([Origin(url=origin_url,)]) visit = self.indexer.storage.origin_visit_add( [ OriginVisit( origin=origin_url, date=datetime(2019, 2, 27, tzinfo=timezone.utc), type="pypi", ) ] )[0] - self.indexer.storage.snapshot_add( - [ - Snapshot( - branches={ - b"foo": None, - b"HEAD": SnapshotBranch( - target_type=TargetType.ALIAS, target=b"foo", - ), - }, - ) - ] - ) + self.indexer.storage.snapshot_add([SAMPLE_SNAPSHOT]) visit_status = OriginVisitStatus( origin=origin_url, visit=visit.visit, date=now(), status="full", - snapshot=b"foo", + snapshot=SAMPLE_SNAPSHOT.id, ) self.indexer.storage.origin_visit_status_add([visit_status]) self.indexer.run(["https://pypi.org/project/abcdef/"]) self.assertEqual(self.indexer.results, []) def test_ftp(self): origin_url = "rsync://ftp.gnu.org/gnu/3dldf" self.indexer.run([origin_url]) rev_id = b"\x8e\xa9\x8e/\xea}\x9feF\xf4\x9f\xfd\xee\xcc\x1a\xb4`\x8c\x8by" self.assertEqual( self.indexer.results, [{"revision_id": rev_id, "origin_url": origin_url,}], ) def test_ftp_missing_snapshot(self): origin_url = "rsync://ftp.gnu.org/gnu/foobar" self.indexer.storage.origin_add([Origin(url=origin_url)]) self.indexer.run([origin_url]) self.assertEqual(self.indexer.results, []) def test_deposit(self): origin_url = "https://forge.softwareheritage.org/source/jesuisgpl/" self.indexer.storage.origin_add([Origin(url=origin_url)]) self.indexer.run([origin_url]) rev_id = b"\xe7n\xa4\x9c\x9f\xfb\xb7\xf76\x11\x08{\xa6\xe9\x99\xb1\x9e]q\xeb" self.assertEqual( self.indexer.results, [{"revision_id": rev_id, "origin_url": origin_url,}], ) def test_deposit_missing_snapshot(self): origin_url = "https://forge.softwareheritage.org/source/foobar" self.indexer.storage.origin_add([Origin(url=origin_url,)]) self.indexer.run([origin_url]) self.assertEqual(self.indexer.results, []) def test_pypi(self): origin_url = "https://pypi.org/project/limnoria/" self.indexer.run([origin_url]) rev_id = b"\x83\xb9\xb6\xc7\x05\xb1%\xd0\xfem\xd8kA\x10\x9d\xc5\xfa2\xf8t" self.assertEqual( self.indexer.results, [{"revision_id": rev_id, "origin_url": origin_url}], ) def test_svn(self): origin_url = "http://0-512-md.googlecode.com/svn/" self.indexer.run([origin_url]) rev_id = b"\xe4?r\xe1,\x88\xab\xec\xe7\x9a\x87\xb8\xc9\xad#.\x1bw=\x18" self.assertEqual( self.indexer.results, [{"revision_id": rev_id, "origin_url": origin_url,}], ) diff --git a/swh/indexer/tests/test_origin_metadata.py b/swh/indexer/tests/test_origin_metadata.py index 63ce2aa..a555e9c 100644 --- a/swh/indexer/tests/test_origin_metadata.py +++ b/swh/indexer/tests/test_origin_metadata.py @@ -1,220 +1,255 @@ # Copyright (C) 2018-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +import copy from unittest.mock import patch +import pytest + from swh.indexer.metadata import OriginMetadataIndexer from swh.indexer.storage.interface import IndexerStorageInterface from swh.indexer.storage.model import ( OriginIntrinsicMetadataRow, RevisionIntrinsicMetadataRow, ) from swh.model.model import Origin from swh.storage.interface import StorageInterface -from .test_metadata import REVISION_METADATA_CONFIG +from .test_metadata import TRANSLATOR_TOOL from .utils import REVISION, YARN_PARSER_METADATA +@pytest.fixture +def swh_indexer_config(swh_indexer_config): + """Override the default configuration to override the tools entry + + """ + cfg = copy.deepcopy(swh_indexer_config) + cfg["tools"] = TRANSLATOR_TOOL + return cfg + + def test_origin_metadata_indexer( - idx_storage: IndexerStorageInterface, storage: StorageInterface, obj_storage + swh_indexer_config, + idx_storage: IndexerStorageInterface, + storage: StorageInterface, + obj_storage, ) -> None: - - indexer = OriginMetadataIndexer(config=REVISION_METADATA_CONFIG) + indexer = OriginMetadataIndexer(config=swh_indexer_config) origin = "https://github.com/librariesio/yarn-parser" indexer.run([origin]) - tool = { - "name": "swh-metadata-translator", - "version": "0.0.2", - "configuration": {"context": "NpmMapping", "type": "local"}, - } + tool = swh_indexer_config["tools"] rev_id = REVISION.id rev_metadata = RevisionIntrinsicMetadataRow( id=rev_id, tool=tool, metadata=YARN_PARSER_METADATA, mappings=["npm"], ) origin_metadata = OriginIntrinsicMetadataRow( id=origin, tool=tool, from_revision=rev_id, metadata=YARN_PARSER_METADATA, mappings=["npm"], ) - rev_results = list(indexer.idx_storage.revision_intrinsic_metadata_get([rev_id])) + rev_results = list(idx_storage.revision_intrinsic_metadata_get([rev_id])) for rev_result in rev_results: assert rev_result.tool del rev_result.tool["id"] assert rev_results == [rev_metadata] - orig_results = list(indexer.idx_storage.origin_intrinsic_metadata_get([origin])) + orig_results = list(idx_storage.origin_intrinsic_metadata_get([origin])) for orig_result in orig_results: assert orig_result.tool del orig_result.tool["id"] assert orig_results == [origin_metadata] def test_origin_metadata_indexer_duplicate_origin( - idx_storage: IndexerStorageInterface, storage: StorageInterface, obj_storage + swh_indexer_config, + idx_storage: IndexerStorageInterface, + storage: StorageInterface, + obj_storage, ) -> None: - indexer = OriginMetadataIndexer(config=REVISION_METADATA_CONFIG) + indexer = OriginMetadataIndexer(config=swh_indexer_config) indexer.storage = storage indexer.idx_storage = idx_storage indexer.run(["https://github.com/librariesio/yarn-parser"]) indexer.run(["https://github.com/librariesio/yarn-parser"] * 2) origin = "https://github.com/librariesio/yarn-parser" rev_id = REVISION.id rev_results = list(indexer.idx_storage.revision_intrinsic_metadata_get([rev_id])) assert len(rev_results) == 1 orig_results = list(indexer.idx_storage.origin_intrinsic_metadata_get([origin])) assert len(orig_results) == 1 def test_origin_metadata_indexer_missing_head( - idx_storage: IndexerStorageInterface, storage: StorageInterface, obj_storage + swh_indexer_config, + idx_storage: IndexerStorageInterface, + storage: StorageInterface, + obj_storage, ) -> None: storage.origin_add([Origin(url="https://example.com")]) - indexer = OriginMetadataIndexer(config=REVISION_METADATA_CONFIG) + indexer = OriginMetadataIndexer(config=swh_indexer_config) indexer.run(["https://example.com"]) origin = "https://example.com" results = list(indexer.idx_storage.origin_intrinsic_metadata_get([origin])) assert results == [] def test_origin_metadata_indexer_partial_missing_head( - idx_storage: IndexerStorageInterface, storage: StorageInterface, obj_storage + swh_indexer_config, + idx_storage: IndexerStorageInterface, + storage: StorageInterface, + obj_storage, ) -> None: origin1 = "https://example.com" origin2 = "https://github.com/librariesio/yarn-parser" storage.origin_add([Origin(url=origin1)]) - indexer = OriginMetadataIndexer(config=REVISION_METADATA_CONFIG) + indexer = OriginMetadataIndexer(config=swh_indexer_config) indexer.run([origin1, origin2]) rev_id = REVISION.id rev_results = list(indexer.idx_storage.revision_intrinsic_metadata_get([rev_id])) assert rev_results == [ RevisionIntrinsicMetadataRow( id=rev_id, metadata=YARN_PARSER_METADATA, mappings=["npm"], tool=rev_results[0].tool, ) ] orig_results = list( indexer.idx_storage.origin_intrinsic_metadata_get([origin1, origin2]) ) for orig_result in orig_results: assert orig_results == [ OriginIntrinsicMetadataRow( id=origin2, from_revision=rev_id, metadata=YARN_PARSER_METADATA, mappings=["npm"], tool=orig_results[0].tool, ) ] def test_origin_metadata_indexer_duplicate_revision( - idx_storage: IndexerStorageInterface, storage: StorageInterface, obj_storage + swh_indexer_config, + idx_storage: IndexerStorageInterface, + storage: StorageInterface, + obj_storage, ) -> None: - indexer = OriginMetadataIndexer(config=REVISION_METADATA_CONFIG) + indexer = OriginMetadataIndexer(config=swh_indexer_config) indexer.storage = storage indexer.idx_storage = idx_storage indexer.catch_exceptions = False origin1 = "https://github.com/librariesio/yarn-parser" origin2 = "https://github.com/librariesio/yarn-parser.git" indexer.run([origin1, origin2]) rev_id = REVISION.id rev_results = list(indexer.idx_storage.revision_intrinsic_metadata_get([rev_id])) assert len(rev_results) == 1 orig_results = list( indexer.idx_storage.origin_intrinsic_metadata_get([origin1, origin2]) ) assert len(orig_results) == 2 def test_origin_metadata_indexer_no_metadata_file( - idx_storage: IndexerStorageInterface, storage: StorageInterface, obj_storage + swh_indexer_config, + idx_storage: IndexerStorageInterface, + storage: StorageInterface, + obj_storage, ) -> None: - indexer = OriginMetadataIndexer(config=REVISION_METADATA_CONFIG) + indexer = OriginMetadataIndexer(config=swh_indexer_config) origin = "https://github.com/librariesio/yarn-parser" with patch("swh.indexer.metadata_dictionary.npm.NpmMapping.filename", b"foo.json"): indexer.run([origin]) rev_id = REVISION.id rev_results = list(indexer.idx_storage.revision_intrinsic_metadata_get([rev_id])) assert rev_results == [] orig_results = list(indexer.idx_storage.origin_intrinsic_metadata_get([origin])) assert orig_results == [] def test_origin_metadata_indexer_no_metadata( - idx_storage: IndexerStorageInterface, storage: StorageInterface, obj_storage + swh_indexer_config, + idx_storage: IndexerStorageInterface, + storage: StorageInterface, + obj_storage, ) -> None: - indexer = OriginMetadataIndexer(config=REVISION_METADATA_CONFIG) + indexer = OriginMetadataIndexer(config=swh_indexer_config) origin = "https://github.com/librariesio/yarn-parser" with patch( "swh.indexer.metadata.RevisionMetadataIndexer" ".translate_revision_intrinsic_metadata", return_value=(["npm"], {"@context": "foo"}), ): indexer.run([origin]) rev_id = REVISION.id rev_results = list(indexer.idx_storage.revision_intrinsic_metadata_get([rev_id])) assert rev_results == [] orig_results = list(indexer.idx_storage.origin_intrinsic_metadata_get([origin])) assert orig_results == [] def test_origin_metadata_indexer_error( - idx_storage: IndexerStorageInterface, storage: StorageInterface, obj_storage + swh_indexer_config, + idx_storage: IndexerStorageInterface, + storage: StorageInterface, + obj_storage, ) -> None: - indexer = OriginMetadataIndexer(config=REVISION_METADATA_CONFIG) + indexer = OriginMetadataIndexer(config=swh_indexer_config) origin = "https://github.com/librariesio/yarn-parser" with patch( "swh.indexer.metadata.RevisionMetadataIndexer" ".translate_revision_intrinsic_metadata", return_value=None, ): indexer.run([origin]) rev_id = REVISION.id rev_results = list(indexer.idx_storage.revision_intrinsic_metadata_get([rev_id])) assert rev_results == [] orig_results = list(indexer.idx_storage.origin_intrinsic_metadata_get([origin])) assert orig_results == [] def test_origin_metadata_indexer_unknown_origin( - idx_storage: IndexerStorageInterface, storage: StorageInterface, obj_storage + swh_indexer_config, + idx_storage: IndexerStorageInterface, + storage: StorageInterface, + obj_storage, ) -> None: - indexer = OriginMetadataIndexer(config=REVISION_METADATA_CONFIG) + indexer = OriginMetadataIndexer(config=swh_indexer_config) result = indexer.index_list(["https://unknown.org/foo"]) assert not result