Changeset View
Changeset View
Standalone View
Standalone View
swh/indexer/tests/test_cli.py
| Show All 10 Lines | |||||
| from click.testing import CliRunner | from click.testing import CliRunner | ||||
| from confluent_kafka import Consumer | from confluent_kafka import Consumer | ||||
| import pytest | import pytest | ||||
| from swh.indexer.cli import indexer_cli_group | from swh.indexer.cli import indexer_cli_group | ||||
| from swh.indexer.storage.interface import IndexerStorageInterface | from swh.indexer.storage.interface import IndexerStorageInterface | ||||
| from swh.indexer.storage.model import ( | from swh.indexer.storage.model import ( | ||||
| DirectoryIntrinsicMetadataRow, | |||||
| OriginIntrinsicMetadataRow, | OriginIntrinsicMetadataRow, | ||||
| RevisionIntrinsicMetadataRow, | |||||
| ) | ) | ||||
| from swh.journal.writer import get_journal_writer | from swh.journal.writer import get_journal_writer | ||||
| from swh.model.hashutil import hash_to_bytes | from swh.model.hashutil import hash_to_bytes | ||||
| from swh.model.model import OriginVisitStatus | from swh.model.model import OriginVisitStatus | ||||
| from .utils import REVISION | from .utils import DIRECTORY2, REVISION | ||||
| def fill_idx_storage(idx_storage: IndexerStorageInterface, nb_rows: int) -> List[int]: | def fill_idx_storage(idx_storage: IndexerStorageInterface, nb_rows: int) -> List[int]: | ||||
| tools: List[Dict[str, Any]] = [ | tools: List[Dict[str, Any]] = [ | ||||
| { | { | ||||
| "tool_name": "tool %d" % i, | "tool_name": "tool %d" % i, | ||||
| "tool_version": "0.0.1", | "tool_version": "0.0.1", | ||||
| "tool_configuration": {}, | "tool_configuration": {}, | ||||
| } | } | ||||
| for i in range(2) | for i in range(2) | ||||
| ] | ] | ||||
| tools = idx_storage.indexer_configuration_add(tools) | tools = idx_storage.indexer_configuration_add(tools) | ||||
| origin_metadata = [ | origin_metadata = [ | ||||
| OriginIntrinsicMetadataRow( | OriginIntrinsicMetadataRow( | ||||
| id="file://dev/%04d" % origin_id, | id="file://dev/%04d" % origin_id, | ||||
| from_revision=hash_to_bytes("abcd{:0>36}".format(origin_id)), | from_directory=hash_to_bytes("abcd{:0>36}".format(origin_id)), | ||||
| indexer_configuration_id=tools[origin_id % 2]["id"], | indexer_configuration_id=tools[origin_id % 2]["id"], | ||||
| metadata={"name": "origin %d" % origin_id}, | metadata={"name": "origin %d" % origin_id}, | ||||
| mappings=["mapping%d" % (origin_id % 10)], | mappings=["mapping%d" % (origin_id % 10)], | ||||
| ) | ) | ||||
| for origin_id in range(nb_rows) | for origin_id in range(nb_rows) | ||||
| ] | ] | ||||
| revision_metadata = [ | directory_metadata = [ | ||||
| RevisionIntrinsicMetadataRow( | DirectoryIntrinsicMetadataRow( | ||||
| id=hash_to_bytes("abcd{:0>36}".format(origin_id)), | id=hash_to_bytes("abcd{:0>36}".format(origin_id)), | ||||
| indexer_configuration_id=tools[origin_id % 2]["id"], | indexer_configuration_id=tools[origin_id % 2]["id"], | ||||
| metadata={"name": "origin %d" % origin_id}, | metadata={"name": "origin %d" % origin_id}, | ||||
| mappings=["mapping%d" % (origin_id % 10)], | mappings=["mapping%d" % (origin_id % 10)], | ||||
| ) | ) | ||||
| for origin_id in range(nb_rows) | for origin_id in range(nb_rows) | ||||
| ] | ] | ||||
| idx_storage.revision_intrinsic_metadata_add(revision_metadata) | idx_storage.directory_intrinsic_metadata_add(directory_metadata) | ||||
| idx_storage.origin_intrinsic_metadata_add(origin_metadata) | idx_storage.origin_intrinsic_metadata_add(origin_metadata) | ||||
| return [tool["id"] for tool in tools] | return [tool["id"] for tool in tools] | ||||
| def _origins_in_task_args(tasks): | def _origins_in_task_args(tasks): | ||||
| """Returns the set of origins contained in the arguments of the | """Returns the set of origins contained in the arguments of the | ||||
| provided tasks (assumed to be of type index-origin-metadata).""" | provided tasks (assumed to be of type index-origin-metadata).""" | ||||
| ▲ Show 20 Lines • Show All 531 Lines • ▼ Show 20 Lines | ): | ||||
| storage.revision_add([REVISION]) | storage.revision_add([REVISION]) | ||||
| mocker.patch( | mocker.patch( | ||||
| "swh.indexer.metadata.get_head_swhid", | "swh.indexer.metadata.get_head_swhid", | ||||
| return_value=REVISION.swhid(), | return_value=REVISION.swhid(), | ||||
| ) | ) | ||||
| mocker.patch( | mocker.patch( | ||||
| "swh.indexer.metadata.RevisionMetadataIndexer.index", | "swh.indexer.metadata.DirectoryMetadataIndexer.index", | ||||
| return_value=[ | return_value=[ | ||||
| RevisionIntrinsicMetadataRow( | DirectoryIntrinsicMetadataRow( | ||||
| id=REVISION.id, | id=DIRECTORY2.id, | ||||
| indexer_configuration_id=1, | indexer_configuration_id=1, | ||||
| mappings=["cff"], | mappings=["cff"], | ||||
| metadata={"foo": "bar"}, | metadata={"foo": "bar"}, | ||||
| ) | ) | ||||
| ], | ], | ||||
| ) | ) | ||||
| result = cli_runner.invoke( | result = cli_runner.invoke( | ||||
| indexer_cli_group, | indexer_cli_group, | ||||
| Show All 20 Lines | ): | ||||
| assert result.output == expected_output | assert result.output == expected_output | ||||
| results = idx_storage.origin_intrinsic_metadata_get( | results = idx_storage.origin_intrinsic_metadata_get( | ||||
| [status.origin for status in visit_statuses] | [status.origin for status in visit_statuses] | ||||
| ) | ) | ||||
| expected_results = [ | expected_results = [ | ||||
| OriginIntrinsicMetadataRow( | OriginIntrinsicMetadataRow( | ||||
| id=status.origin, | id=status.origin, | ||||
| from_revision=REVISION.id, | from_directory=DIRECTORY2.id, | ||||
| tool={"id": 1, **swh_indexer_config["tools"]}, | tool={"id": 1, **swh_indexer_config["tools"]}, | ||||
| mappings=["cff"], | mappings=["cff"], | ||||
| metadata={"foo": "bar"}, | metadata={"foo": "bar"}, | ||||
| ) | ) | ||||
| for status in sorted(visit_statuses_full, key=lambda r: r.origin) | for status in sorted(visit_statuses_full, key=lambda r: r.origin) | ||||
| ] | ] | ||||
| assert sorted(results, key=lambda r: r.id) == expected_results | assert sorted(results, key=lambda r: r.id) == expected_results | ||||