diff --git a/swh/indexer/cli.py b/swh/indexer/cli.py index f037dba..27d2a27 100644 --- a/swh/indexer/cli.py +++ b/swh/indexer/cli.py @@ -1,300 +1,301 @@ # Copyright (C) 2019-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +from typing import Iterator # WARNING: do not import unnecessary things here to keep cli startup time under # control import click from swh.core.cli import CONTEXT_SETTINGS, AliasedGroup from swh.core.cli import swh as swh_cli_group @swh_cli_group.group( name="indexer", context_settings=CONTEXT_SETTINGS, cls=AliasedGroup ) @click.option( "--config-file", "-C", default=None, type=click.Path(exists=True, dir_okay=False,), help="Configuration file.", ) @click.pass_context def indexer_cli_group(ctx, config_file): """Software Heritage Indexer tools. The Indexer is used to mine the content of the archive and extract derived information from archive source code artifacts. """ from swh.core import config ctx.ensure_object(dict) conf = config.read(config_file) ctx.obj["config"] = conf def _get_api(getter, config, config_key, url): if url: config[config_key] = {"cls": "remote", "args": {"url": url}} elif config_key not in config: raise click.ClickException("Missing configuration for {}".format(config_key)) return getter(**config[config_key]) @indexer_cli_group.group("mapping") def mapping(): """Manage Software Heritage Indexer mappings.""" pass @mapping.command("list") def mapping_list(): """Prints the list of known mappings.""" from swh.indexer import metadata_dictionary mapping_names = [mapping.name for mapping in metadata_dictionary.MAPPINGS.values()] mapping_names.sort() for mapping_name in mapping_names: click.echo(mapping_name) @mapping.command("list-terms") @click.option( "--exclude-mapping", multiple=True, help="Exclude the given mapping from the output" ) @click.option( "--concise", is_flag=True, default=False, help="Don't print the list of mappings supporting each term.", ) def mapping_list_terms(concise, exclude_mapping): """Prints the list of known CodeMeta terms, and which mappings support them.""" from swh.indexer import metadata_dictionary properties = metadata_dictionary.list_terms() for (property_name, supported_mappings) in sorted(properties.items()): supported_mappings = {m.name for m in supported_mappings} supported_mappings -= set(exclude_mapping) if supported_mappings: if concise: click.echo(property_name) else: click.echo("{}:".format(property_name)) click.echo("\t" + ", ".join(sorted(supported_mappings))) @mapping.command("translate") @click.argument("mapping-name") @click.argument("file", type=click.File("rb")) def mapping_translate(mapping_name, file): """Prints the list of known mappings.""" import json from swh.indexer import metadata_dictionary mapping_cls = [ cls for cls in metadata_dictionary.MAPPINGS.values() if cls.name == mapping_name ] if not mapping_cls: raise click.ClickException("Unknown mapping {}".format(mapping_name)) assert len(mapping_cls) == 1 mapping_cls = mapping_cls[0] mapping = mapping_cls() codemeta_doc = mapping.translate(file.read()) click.echo(json.dumps(codemeta_doc, indent=4)) @indexer_cli_group.group("schedule") @click.option("--scheduler-url", "-s", default=None, help="URL of the scheduler API") @click.option( "--indexer-storage-url", "-i", default=None, help="URL of the indexer storage API" ) @click.option( "--storage-url", "-g", default=None, help="URL of the (graph) storage API" ) @click.option( "--dry-run/--no-dry-run", is_flag=True, default=False, help="List only what would be scheduled.", ) @click.pass_context def schedule(ctx, scheduler_url, storage_url, indexer_storage_url, dry_run): """Manipulate Software Heritage Indexer tasks. Via SWH Scheduler's API.""" from swh.indexer.storage import get_indexer_storage from swh.scheduler import get_scheduler from swh.storage import get_storage ctx.obj["indexer_storage"] = _get_api( get_indexer_storage, ctx.obj["config"], "indexer_storage", indexer_storage_url ) ctx.obj["storage"] = _get_api( get_storage, ctx.obj["config"], "storage", storage_url ) ctx.obj["scheduler"] = _get_api( get_scheduler, ctx.obj["config"], "scheduler", scheduler_url ) if dry_run: ctx.obj["scheduler"] = None -def list_origins_by_producer(idx_storage, mappings, tool_ids): +def list_origins_by_producer(idx_storage, mappings, tool_ids) -> Iterator[str]: next_page_token = "" limit = 10000 while next_page_token is not None: result = idx_storage.origin_intrinsic_metadata_search_by_producer( page_token=next_page_token, limit=limit, ids_only=True, mappings=mappings or None, tool_ids=tool_ids or None, ) - next_page_token = result.get("next_page_token") - yield from result["origins"] + next_page_token = result.next_page_token + yield from result.results @schedule.command("reindex_origin_metadata") @click.option( "--batch-size", "-b", "origin_batch_size", default=10, show_default=True, type=int, help="Number of origins per task", ) @click.option( "--tool-id", "-t", "tool_ids", type=int, multiple=True, help="Restrict search of old metadata to this/these tool ids.", ) @click.option( "--mapping", "-m", "mappings", multiple=True, help="Mapping(s) that should be re-scheduled (eg. 'npm', 'gemspec', 'maven')", ) @click.option( "--task-type", default="index-origin-metadata", show_default=True, help="Name of the task type to schedule.", ) @click.pass_context def schedule_origin_metadata_reindex( ctx, origin_batch_size, tool_ids, mappings, task_type ): """Schedules indexing tasks for origins that were already indexed.""" from swh.scheduler.cli_utils import schedule_origin_batches idx_storage = ctx.obj["indexer_storage"] scheduler = ctx.obj["scheduler"] origins = list_origins_by_producer(idx_storage, mappings, tool_ids) kwargs = {"policy_update": "update-dups"} schedule_origin_batches(scheduler, task_type, origins, origin_batch_size, kwargs) @indexer_cli_group.command("journal-client") @click.option("--scheduler-url", "-s", default=None, help="URL of the scheduler API") @click.option( "--origin-metadata-task-type", default="index-origin-metadata", help="Name of the task running the origin metadata indexer.", ) @click.option( "--broker", "brokers", type=str, multiple=True, help="Kafka broker to connect to." ) @click.option( "--prefix", type=str, default=None, help="Prefix of Kafka topic names to read from." ) @click.option("--group-id", type=str, help="Consumer/group id for reading from Kafka.") @click.option( "--stop-after-objects", "-m", default=None, type=int, help="Maximum number of objects to replay. Default is to run forever.", ) @click.pass_context def journal_client( ctx, scheduler_url, origin_metadata_task_type, brokers, prefix, group_id, stop_after_objects, ): """Listens for new objects from the SWH Journal, and schedules tasks to run relevant indexers (currently, only origin-intrinsic-metadata) on these new objects.""" import functools from swh.indexer.journal_client import process_journal_objects from swh.journal.client import get_journal_client from swh.scheduler import get_scheduler scheduler = _get_api(get_scheduler, ctx.obj["config"], "scheduler", scheduler_url) client = get_journal_client( cls="kafka", brokers=brokers, prefix=prefix, group_id=group_id, object_types=["origin_visit"], stop_after_objects=stop_after_objects, ) worker_fn = functools.partial( process_journal_objects, scheduler=scheduler, task_names={"origin_metadata": origin_metadata_task_type,}, ) try: client.process(worker_fn) except KeyboardInterrupt: ctx.exit(0) else: print("Done.") finally: client.close() @indexer_cli_group.command("rpc-serve") @click.argument("config-path", required=True) @click.option("--host", default="0.0.0.0", help="Host to run the server") @click.option("--port", default=5007, type=click.INT, help="Binding port of the server") @click.option( "--debug/--nodebug", default=True, help="Indicates if the server should run in debug mode", ) def rpc_server(config_path, host, port, debug): """Starts a Software Heritage Indexer RPC HTTP server.""" from swh.indexer.storage.api.server import app, load_and_check_config api_cfg = load_and_check_config(config_path, type="any") app.config.update(api_cfg) app.run(host, port=int(port), debug=bool(debug)) def main(): return indexer_cli_group(auto_envvar_prefix="SWH_INDEXER") if __name__ == "__main__": main() diff --git a/swh/indexer/indexer.py b/swh/indexer/indexer.py index ebb7dae..3abb378 100644 --- a/swh/indexer/indexer.py +++ b/swh/indexer/indexer.py @@ -1,617 +1,614 @@ # Copyright (C) 2016-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import abc from contextlib import contextmanager import logging import os import shutil import tempfile from typing import Any, Dict, Generic, Iterator, List, Optional, Set, TypeVar, Union from swh.core import utils from swh.core.config import load_from_envvar, merge_configs from swh.indexer.storage import INDEXER_CFG_KEY, PagedResult, Sha1, get_indexer_storage from swh.indexer.storage.interface import IndexerStorageInterface from swh.model import hashutil -from swh.model.model import Revision from swh.objstorage.exc import ObjNotFoundError from swh.objstorage.factory import get_objstorage from swh.scheduler import CONFIG as SWH_CONFIG from swh.storage import get_storage from swh.storage.interface import StorageInterface @contextmanager def write_to_temp(filename: str, data: bytes, working_directory: str) -> Iterator[str]: """Write the sha1's content in a temporary file. Args: filename: one of sha1's many filenames data: the sha1's content to write in temporary file working_directory: the directory into which the file is written Returns: The path to the temporary file created. That file is filled in with the raw content's data. """ os.makedirs(working_directory, exist_ok=True) temp_dir = tempfile.mkdtemp(dir=working_directory) content_path = os.path.join(temp_dir, filename) with open(content_path, "wb") as f: f.write(data) yield content_path shutil.rmtree(temp_dir) DEFAULT_CONFIG = { INDEXER_CFG_KEY: {"cls": "memory"}, "storage": {"cls": "memory"}, "objstorage": {"cls": "memory"}, } # TODO: should be bound=Optional[BaseRow] when all endpoints move away from dicts TResult = TypeVar("TResult") class BaseIndexer(Generic[TResult], metaclass=abc.ABCMeta): """Base class for indexers to inherit from. The main entry point is the :func:`run` function which is in charge of triggering the computations on the batch dict/ids received. Indexers can: - filter out ids whose data has already been indexed. - retrieve ids data from storage or objstorage - index this data depending on the object and store the result in storage. To implement a new object type indexer, inherit from the BaseIndexer and implement indexing: :meth:`~BaseIndexer.run`: object_ids are different depending on object. For example: sha1 for content, sha1_git for revision, directory, release, and id for origin To implement a new concrete indexer, inherit from the object level classes: :class:`ContentIndexer`, :class:`RevisionIndexer`, :class:`OriginIndexer`. Then you need to implement the following functions: :meth:`~BaseIndexer.filter`: filter out data already indexed (in storage). :meth:`~BaseIndexer.index_object`: compute index on id with data (retrieved from the storage or the objstorage by the id key) and return the resulting index computation. :meth:`~BaseIndexer.persist_index_computations`: persist the results of multiple index computations in the storage. The new indexer implementation can also override the following functions: :meth:`~BaseIndexer.prepare`: Configuration preparation for the indexer. When overriding, this must call the `super().prepare()` instruction. :meth:`~BaseIndexer.check`: Configuration check for the indexer. When overriding, this must call the `super().check()` instruction. :meth:`~BaseIndexer.register_tools`: This should return a dict of the tool(s) to use when indexing or filtering. """ results: List[TResult] USE_TOOLS = True catch_exceptions = True """Prevents exceptions in `index()` from raising too high. Set to False in tests to properly catch all exceptions.""" scheduler: Any storage: StorageInterface objstorage: Any idx_storage: IndexerStorageInterface def __init__(self, config=None, **kw) -> None: """Prepare and check that the indexer is ready to run. """ super().__init__() if config is not None: self.config = config elif SWH_CONFIG: self.config = SWH_CONFIG.copy() else: self.config = load_from_envvar() self.config = merge_configs(DEFAULT_CONFIG, self.config) self.prepare() self.check() self.log.debug("%s: config=%s", self, self.config) def prepare(self) -> None: """Prepare the indexer's needed runtime configuration. Without this step, the indexer cannot possibly run. """ config_storage = self.config.get("storage") if config_storage: self.storage = get_storage(**config_storage) objstorage = self.config["objstorage"] self.objstorage = get_objstorage(objstorage["cls"], objstorage["args"]) idx_storage = self.config[INDEXER_CFG_KEY] self.idx_storage = get_indexer_storage(**idx_storage) _log = logging.getLogger("requests.packages.urllib3.connectionpool") _log.setLevel(logging.WARN) self.log = logging.getLogger("swh.indexer") if self.USE_TOOLS: self.tools = list(self.register_tools(self.config.get("tools", []))) self.results = [] @property def tool(self) -> Dict: return self.tools[0] def check(self) -> None: """Check the indexer's configuration is ok before proceeding. If ok, does nothing. If not raise error. """ if self.USE_TOOLS and not self.tools: raise ValueError("Tools %s is unknown, cannot continue" % self.tools) def _prepare_tool(self, tool: Dict[str, Any]) -> Dict[str, Any]: """Prepare the tool dict to be compliant with the storage api. """ return {"tool_%s" % key: value for key, value in tool.items()} def register_tools( self, tools: Union[Dict[str, Any], List[Dict[str, Any]]] ) -> List[Dict[str, Any]]: """Permit to register tools to the storage. Add a sensible default which can be overridden if not sufficient. (For now, all indexers use only one tool) Expects the self.config['tools'] property to be set with one or more tools. Args: tools: Either a dict or a list of dict. Returns: list: List of dicts with additional id key. Raises: ValueError: if not a list nor a dict. """ if isinstance(tools, list): tools = list(map(self._prepare_tool, tools)) elif isinstance(tools, dict): tools = [self._prepare_tool(tools)] else: raise ValueError("Configuration tool(s) must be a dict or list!") if tools: return self.idx_storage.indexer_configuration_add(tools) else: return [] - def index( - self, id: Union[bytes, Dict, Revision], data: Optional[bytes] = None, **kwargs - ) -> List[TResult]: + def index(self, id, data: Optional[bytes] = None, **kwargs) -> List[TResult]: """Index computation for the id and associated raw data. Args: id: identifier or Dict object data: id's data from storage or objstorage depending on object type Returns: dict: a dict that makes sense for the :meth:`.persist_index_computations` method. """ raise NotImplementedError() def filter(self, ids: List[bytes]) -> Iterator[bytes]: """Filter missing ids for that particular indexer. Args: ids: list of ids Yields: iterator of missing ids """ yield from ids @abc.abstractmethod def persist_index_computations( self, results: List[TResult], policy_update: str ) -> Dict[str, int]: """Persist the computation resulting from the index. Args: results: List of results. One result is the result of the index function. policy_update: either 'update-dups' or 'ignore-dups' to respectively update duplicates or ignore them Returns: a summary dict of what has been inserted in the storage """ return {} class ContentIndexer(BaseIndexer[TResult], Generic[TResult]): """A content indexer working on a list of ids directly. To work on indexer partition, use the :class:`ContentPartitionIndexer` instead. Note: :class:`ContentIndexer` is not an instantiable object. To use it, one should inherit from this class and override the methods mentioned in the :class:`BaseIndexer` class. """ def run( self, ids: Union[List[bytes], bytes, str], policy_update: str, **kwargs ) -> Dict: """Given a list of ids: - retrieve the content from the storage - execute the indexing computations - store the results (according to policy_update) Args: ids (Iterable[Union[bytes, str]]): sha1's identifier list policy_update (str): either 'update-dups' or 'ignore-dups' to respectively update duplicates or ignore them **kwargs: passed to the `index` method Returns: A summary Dict of the task's status """ sha1s = [ hashutil.hash_to_bytes(id_) if isinstance(id_, str) else id_ for id_ in ids ] results = [] summary: Dict = {"status": "uneventful"} try: for sha1 in sha1s: try: raw_content = self.objstorage.get(sha1) except ObjNotFoundError: self.log.warning( "Content %s not found in objstorage" % hashutil.hash_to_hex(sha1) ) continue res = self.index(sha1, raw_content, **kwargs) if res: # If no results, skip it results.extend(res) summary["status"] = "eventful" summary = self.persist_index_computations(results, policy_update) self.results = results except Exception: if not self.catch_exceptions: raise self.log.exception("Problem when reading contents metadata.") summary["status"] = "failed" return summary class ContentPartitionIndexer(BaseIndexer[TResult], Generic[TResult]): """A content partition indexer. This expects as input a partition_id and a nb_partitions. This will then index the contents within that partition. To work on a list of ids, use the :class:`ContentIndexer` instead. Note: :class:`ContentPartitionIndexer` is not an instantiable object. To use it, one should inherit from this class and override the methods mentioned in the :class:`BaseIndexer` class. """ @abc.abstractmethod def indexed_contents_in_partition( self, partition_id: int, nb_partitions: int, page_token: Optional[str] = None ) -> PagedResult[Sha1]: """Retrieve indexed contents within range [start, end]. Args: partition_id: Index of the partition to fetch nb_partitions: Total number of partitions to split into page_token: opaque token used for pagination Returns: PagedResult of Sha1. If next_page_token is None, there is no more data to fetch """ pass def _list_contents_to_index( self, partition_id: int, nb_partitions: int, indexed: Set[Sha1] ) -> Iterator[Sha1]: """Compute from storage the new contents to index in the partition_id . The already indexed contents are skipped. Args: partition_id: Index of the partition to fetch data from nb_partitions: Total number of partition indexed: Set of content already indexed. Yields: Sha1 id (bytes) of contents to index """ if not isinstance(partition_id, int) or not isinstance(nb_partitions, int): raise TypeError( f"identifiers must be int, not {partition_id!r} and {nb_partitions!r}." ) next_page_token = None while True: result = self.storage.content_get_partition( partition_id, nb_partitions, page_token=next_page_token ) contents = result.results for c in contents: _id = hashutil.hash_to_bytes(c.sha1) if _id in indexed: continue yield _id next_page_token = result.next_page_token if next_page_token is None: break def _index_contents( self, partition_id: int, nb_partitions: int, indexed: Set[Sha1], **kwargs: Any ) -> Iterator[TResult]: """Index the contents within the partition_id. Args: start: Starting bound from range identifier end: End range identifier indexed: Set of content already indexed. Yields: indexing result as dict to persist in the indexer backend """ for sha1 in self._list_contents_to_index(partition_id, nb_partitions, indexed): try: raw_content = self.objstorage.get(sha1) except ObjNotFoundError: self.log.warning(f"Content {sha1.hex()} not found in objstorage") continue results = self.index(sha1, raw_content, **kwargs) for res in results: # TODO: remove this check when all endpoints moved away from dicts. if isinstance(res, dict) and not isinstance(res["id"], bytes): raise TypeError( "%r.index should return ids as bytes, not %r" % (self.__class__.__name__, res["id"]) ) yield res def _index_with_skipping_already_done( self, partition_id: int, nb_partitions: int ) -> Iterator[TResult]: """Index not already indexed contents within the partition partition_id Args: partition_id: Index of the partition to fetch nb_partitions: Total number of partitions to split into Yields: indexing result as dict to persist in the indexer backend """ next_page_token = None contents = set() while True: indexed_page = self.indexed_contents_in_partition( partition_id, nb_partitions, page_token=next_page_token ) for sha1 in indexed_page.results: contents.add(sha1) yield from self._index_contents(partition_id, nb_partitions, contents) next_page_token = indexed_page.next_page_token if next_page_token is None: break def run( self, partition_id: int, nb_partitions: int, skip_existing: bool = True, **kwargs, ) -> Dict: """Given a partition of content ids, index the contents within. Either the indexer is incremental (filter out existing computed data) or it computes everything from scratch. Args: partition_id: Index of the partition to fetch nb_partitions: Total number of partitions to split into skip_existing: Skip existing indexed data (default) or not **kwargs: passed to the `index` method Returns: dict with the indexing task status """ summary: Dict[str, Any] = {"status": "uneventful"} count = 0 try: if skip_existing: gen = self._index_with_skipping_already_done( partition_id, nb_partitions ) else: gen = self._index_contents(partition_id, nb_partitions, indexed=set([])) count_object_added_key: Optional[str] = None for contents in utils.grouper(gen, n=self.config["write_batch_size"]): res = self.persist_index_computations( list(contents), policy_update="update-dups" ) if not count_object_added_key: count_object_added_key = list(res.keys())[0] count += res[count_object_added_key] if count > 0: summary["status"] = "eventful" except Exception: if not self.catch_exceptions: raise self.log.exception("Problem when computing metadata.") summary["status"] = "failed" if count > 0 and count_object_added_key: summary[count_object_added_key] = count return summary class OriginIndexer(BaseIndexer[TResult], Generic[TResult]): """An object type indexer, inherits from the :class:`BaseIndexer` and implements Origin indexing using the run method Note: the :class:`OriginIndexer` is not an instantiable object. To use it in another context one should inherit from this class and override the methods mentioned in the :class:`BaseIndexer` class. """ def run( self, origin_urls: List[str], policy_update: str = "update-dups", **kwargs ) -> Dict: """Given a list of origin urls: - retrieve origins from storage - execute the indexing computations - store the results (according to policy_update) Args: origin_urls: list of origin urls. policy_update: either 'update-dups' or 'ignore-dups' to respectively update duplicates (default) or ignore them **kwargs: passed to the `index` method """ summary: Dict[str, Any] = {"status": "uneventful"} try: results = self.index_list(origin_urls, **kwargs) except Exception: if not self.catch_exceptions: raise summary["status"] = "failed" return summary summary_persist = self.persist_index_computations(results, policy_update) self.results = results if summary_persist: for value in summary_persist.values(): if value > 0: summary["status"] = "eventful" summary.update(summary_persist) return summary - def index_list(self, origins: List[Any], **kwargs: Any) -> List[TResult]: + def index_list(self, origin_urls: List[str], **kwargs) -> List[TResult]: results = [] - for origin in origins: + for origin_url in origin_urls: try: - results.extend(self.index(origin, **kwargs)) + results.extend(self.index(origin_url, **kwargs)) except Exception: - self.log.exception("Problem when processing origin %s", origin) + self.log.exception("Problem when processing origin %s", origin_url) raise return results class RevisionIndexer(BaseIndexer[TResult], Generic[TResult]): """An object type indexer, inherits from the :class:`BaseIndexer` and implements Revision indexing using the run method Note: the :class:`RevisionIndexer` is not an instantiable object. To use it in another context one should inherit from this class and override the methods mentioned in the :class:`BaseIndexer` class. """ def run(self, ids: Union[str, bytes], policy_update: str) -> Dict: """Given a list of sha1_gits: - retrieve revisions from storage - execute the indexing computations - store the results (according to policy_update) Args: ids: sha1_git's identifier list policy_update: either 'update-dups' or 'ignore-dups' to respectively update duplicates or ignore them """ summary: Dict[str, Any] = {"status": "uneventful"} results = [] revision_ids = [ hashutil.hash_to_bytes(id_) if isinstance(id_, str) else id_ for id_ in ids ] for rev in self.storage.revision_get(revision_ids): if not rev: self.log.warning( "Revisions %s not found in storage" % list(map(hashutil.hash_to_hex, ids)) ) continue try: results.extend(self.index(rev)) except Exception: if not self.catch_exceptions: raise self.log.exception("Problem when processing revision") summary["status"] = "failed" return summary summary_persist = self.persist_index_computations(results, policy_update) if summary_persist: for value in summary_persist.values(): if value > 0: summary["status"] = "eventful" summary.update(summary_persist) self.results = results return summary diff --git a/swh/indexer/metadata.py b/swh/indexer/metadata.py index ea6adc6..228137a 100644 --- a/swh/indexer/metadata.py +++ b/swh/indexer/metadata.py @@ -1,399 +1,426 @@ # Copyright (C) 2017-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from copy import deepcopy -from typing import Any, Callable, Dict, Iterator, List, Optional, Tuple +from typing import ( + Any, + Callable, + Dict, + Iterable, + Iterator, + List, + Optional, + Tuple, + TypeVar, +) from swh.core.config import merge_configs from swh.core.utils import grouper from swh.indexer.codemeta import merge_documents from swh.indexer.indexer import ContentIndexer, OriginIndexer, RevisionIndexer from swh.indexer.metadata_detector import detect_metadata from swh.indexer.metadata_dictionary import MAPPINGS from swh.indexer.origin_head import OriginHeadIndexer from swh.indexer.storage import INDEXER_CFG_KEY -from swh.indexer.storage.model import ContentMetadataRow, RevisionIntrinsicMetadataRow +from swh.indexer.storage.model import ( + ContentMetadataRow, + OriginIntrinsicMetadataRow, + RevisionIntrinsicMetadataRow, +) from swh.model import hashutil from swh.model.model import Revision REVISION_GET_BATCH_SIZE = 10 ORIGIN_GET_BATCH_SIZE = 10 +T1 = TypeVar("T1") +T2 = TypeVar("T2") + + def call_with_batches( - f: Callable[[List[Dict[str, Any]]], Dict["str", Any]], - args: List[Dict[str, str]], - batch_size: int, -) -> Iterator[str]: + f: Callable[[List[T1]], Iterable[T2]], args: List[T1], batch_size: int, +) -> Iterator[T2]: """Calls a function with batches of args, and concatenates the results. """ groups = grouper(args, batch_size) for group in groups: yield from f(list(group)) class ContentMetadataIndexer(ContentIndexer[ContentMetadataRow]): """Content-level indexer This indexer is in charge of: - filtering out content already indexed in content_metadata - reading content from objstorage with the content's id sha1 - computing metadata by given context - using the metadata_dictionary as the 'swh-metadata-translator' tool - store result in content_metadata table """ def filter(self, ids): """Filter out known sha1s and return only missing ones. """ yield from self.idx_storage.content_metadata_missing( ({"id": sha1, "indexer_configuration_id": self.tool["id"],} for sha1 in ids) ) def index( self, id, data: Optional[bytes] = None, log_suffix="unknown revision", **kwargs ) -> List[ContentMetadataRow]: """Index sha1s' content and store result. Args: id (bytes): content's identifier data (bytes): raw content in bytes Returns: dict: dictionary representing a content_metadata. If the translation wasn't successful the metadata keys will be returned as None """ assert isinstance(id, bytes) assert data is not None try: mapping_name = self.tool["tool_configuration"]["context"] log_suffix += ", content_id=%s" % hashutil.hash_to_hex(id) metadata = MAPPINGS[mapping_name](log_suffix).translate(data) except Exception: self.log.exception( "Problem during metadata translation " "for content %s" % hashutil.hash_to_hex(id) ) if metadata is None: return [] return [ ContentMetadataRow( id=id, indexer_configuration_id=self.tool["id"], metadata=metadata, ) ] def persist_index_computations( self, results: List[ContentMetadataRow], policy_update: str ) -> Dict[str, int]: """Persist the results in storage. Args: results: list of content_metadata, dict with the following keys: - id (bytes): content's identifier (sha1) - metadata (jsonb): detected metadata policy_update: either 'update-dups' or 'ignore-dups' to respectively update duplicates or ignore them """ return self.idx_storage.content_metadata_add( results, conflict_update=(policy_update == "update-dups") ) DEFAULT_CONFIG: Dict[str, Any] = { "tools": { "name": "swh-metadata-detector", "version": "0.0.2", "configuration": {}, }, } class RevisionMetadataIndexer(RevisionIndexer[RevisionIntrinsicMetadataRow]): """Revision-level indexer This indexer is in charge of: - filtering revisions already indexed in revision_intrinsic_metadata table with defined computation tool - retrieve all entry_files in root directory - use metadata_detector for file_names containing metadata - compute metadata translation if necessary and possible (depends on tool) - send sha1s to content indexing if possible - store the results for revision """ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.config = merge_configs(DEFAULT_CONFIG, self.config) def filter(self, sha1_gits): """Filter out known sha1s and return only missing ones. """ yield from self.idx_storage.revision_intrinsic_metadata_missing( ( {"id": sha1_git, "indexer_configuration_id": self.tool["id"],} for sha1_git in sha1_gits ) ) def index(self, id, data=None, **kwargs) -> List[RevisionIntrinsicMetadataRow]: """Index rev by processing it and organizing result. use metadata_detector to iterate on filenames - if one filename detected -> sends file to content indexer - if multiple file detected -> translation needed at revision level Args: rev: revision model object from storage Returns: dict: dictionary representing a revision_intrinsic_metadata, with keys: - id (str): rev's identifier (sha1_git) - indexer_configuration_id (bytes): tool used - metadata: dict of retrieved metadata """ rev = id assert isinstance(rev, Revision) assert data is None try: root_dir = rev.directory dir_ls = list(self.storage.directory_ls(root_dir, recursive=False)) if [entry["type"] for entry in dir_ls] == ["dir"]: # If the root is just a single directory, recurse into it # eg. PyPI packages, GNU tarballs subdir = dir_ls[0]["target"] dir_ls = list(self.storage.directory_ls(subdir, recursive=False)) files = [entry for entry in dir_ls if entry["type"] == "file"] detected_files = detect_metadata(files) (mappings, metadata) = self.translate_revision_intrinsic_metadata( detected_files, log_suffix="revision=%s" % hashutil.hash_to_hex(rev.id), ) except Exception as e: self.log.exception("Problem when indexing rev: %r", e) return [ RevisionIntrinsicMetadataRow( id=rev.id, indexer_configuration_id=self.tool["id"], mappings=mappings, metadata=metadata, ) ] def persist_index_computations( self, results: List[RevisionIntrinsicMetadataRow], policy_update: str ) -> Dict[str, int]: """Persist the results in storage. Args: results: list of content_mimetype, dict with the following keys: - id (bytes): content's identifier (sha1) - mimetype (bytes): mimetype in bytes - encoding (bytes): encoding in bytes policy_update: either 'update-dups' or 'ignore-dups' to respectively update duplicates or ignore them """ # TODO: add functions in storage to keep data in # revision_intrinsic_metadata return self.idx_storage.revision_intrinsic_metadata_add( results, conflict_update=(policy_update == "update-dups") ) def translate_revision_intrinsic_metadata( self, detected_files: Dict[str, List[Any]], log_suffix: str ) -> Tuple[List[Any], Any]: """ Determine plan of action to translate metadata when containing one or multiple detected files: Args: detected_files: dictionary mapping context names (e.g., "npm", "authors") to list of sha1 Returns: (List[str], dict): list of mappings used and dict with translated metadata according to the CodeMeta vocabulary """ used_mappings = [MAPPINGS[context].name for context in detected_files] metadata = [] tool = { "name": "swh-metadata-translator", "version": "0.0.2", "configuration": {}, } # TODO: iterate on each context, on each file # -> get raw_contents # -> translate each content config = {k: self.config[k] for k in [INDEXER_CFG_KEY, "objstorage", "storage"]} config["tools"] = [tool] for context in detected_files.keys(): cfg = deepcopy(config) cfg["tools"][0]["configuration"]["context"] = context c_metadata_indexer = ContentMetadataIndexer(config=cfg) # sha1s that are in content_metadata table sha1s_in_storage = [] metadata_generator = self.idx_storage.content_metadata_get( detected_files[context] ) for c in metadata_generator: # extracting metadata sha1 = c.id sha1s_in_storage.append(sha1) local_metadata = c.metadata # local metadata is aggregated if local_metadata: metadata.append(local_metadata) sha1s_filtered = [ item for item in detected_files[context] if item not in sha1s_in_storage ] if sha1s_filtered: # content indexing try: c_metadata_indexer.run( sha1s_filtered, policy_update="ignore-dups", log_suffix=log_suffix, ) # on the fly possibility: for result in c_metadata_indexer.results: local_metadata = result.metadata metadata.append(local_metadata) except Exception: self.log.exception("Exception while indexing metadata on contents") metadata = merge_documents(metadata) return (used_mappings, metadata) -class OriginMetadataIndexer(OriginIndexer[Tuple[Dict, RevisionIntrinsicMetadataRow]]): +class OriginMetadataIndexer( + OriginIndexer[Tuple[OriginIntrinsicMetadataRow, RevisionIntrinsicMetadataRow]] +): USE_TOOLS = False def __init__(self, config=None, **kwargs) -> None: super().__init__(config=config, **kwargs) self.origin_head_indexer = OriginHeadIndexer(config=config) self.revision_metadata_indexer = RevisionMetadataIndexer(config=config) - def index_list(self, origin_urls, **kwargs): + def index_list( + self, origin_urls: List[str], **kwargs + ) -> List[Tuple[OriginIntrinsicMetadataRow, RevisionIntrinsicMetadataRow]]: head_rev_ids = [] origins_with_head = [] origins = list( call_with_batches( self.storage.origin_get, origin_urls, ORIGIN_GET_BATCH_SIZE, ) ) for origin in origins: if origin is None: continue head_results = self.origin_head_indexer.index(origin.url) if head_results: (head_result,) = head_results origins_with_head.append(origin) head_rev_ids.append(head_result["revision_id"]) head_revs = list( call_with_batches( self.storage.revision_get, head_rev_ids, REVISION_GET_BATCH_SIZE ) ) assert len(head_revs) == len(head_rev_ids) results = [] for (origin, rev) in zip(origins_with_head, head_revs): if not rev: self.log.warning("Missing head revision of origin %r", origin.url) continue for rev_metadata in self.revision_metadata_indexer.index(rev): # There is at most one rev_metadata - orig_metadata = { - "from_revision": rev_metadata.id, - "id": origin.url, - "metadata": rev_metadata.metadata, - "mappings": rev_metadata.mappings, - "indexer_configuration_id": rev_metadata.indexer_configuration_id, - } + orig_metadata = OriginIntrinsicMetadataRow( + from_revision=rev_metadata.id, + id=origin.url, + metadata=rev_metadata.metadata, + mappings=rev_metadata.mappings, + indexer_configuration_id=rev_metadata.indexer_configuration_id, + ) results.append((orig_metadata, rev_metadata)) return results def persist_index_computations( self, - results: List[Tuple[Dict, RevisionIntrinsicMetadataRow]], + results: List[Tuple[OriginIntrinsicMetadataRow, RevisionIntrinsicMetadataRow]], policy_update: str, ) -> Dict[str, int]: conflict_update = policy_update == "update-dups" # Deduplicate revisions rev_metadata: List[RevisionIntrinsicMetadataRow] = [] - orig_metadata: List[Dict] = [] + orig_metadata: List[OriginIntrinsicMetadataRow] = [] revs_to_delete: List[Dict] = [] origs_to_delete: List[Dict] = [] summary: Dict = {} for (orig_item, rev_item) in results: - assert rev_item.metadata == orig_item["metadata"] + assert rev_item.metadata == orig_item.metadata if not rev_item.metadata or rev_item.metadata.keys() <= {"@context"}: # If we didn't find any metadata, don't store a DB record # (and delete existing ones, if any) if rev_item not in revs_to_delete: revs_to_delete.append( { "id": rev_item.id, "indexer_configuration_id": ( rev_item.indexer_configuration_id ), } ) if orig_item not in origs_to_delete: - origs_to_delete.append(orig_item) + origs_to_delete.append( + { + "id": orig_item.id, + "indexer_configuration_id": ( + orig_item.indexer_configuration_id + ), + } + ) else: if rev_item not in rev_metadata: rev_metadata.append(rev_item) if orig_item not in orig_metadata: orig_metadata.append(orig_item) if rev_metadata: summary_rev = self.idx_storage.revision_intrinsic_metadata_add( rev_metadata, conflict_update=conflict_update ) summary.update(summary_rev) if orig_metadata: summary_ori = self.idx_storage.origin_intrinsic_metadata_add( orig_metadata, conflict_update=conflict_update ) summary.update(summary_ori) # revs_to_delete should always be empty unless we changed a mapping # to detect less files or less content. # However, origs_to_delete may be empty whenever an upstream deletes # a metadata file. if origs_to_delete: summary_ori = self.idx_storage.origin_intrinsic_metadata_delete( origs_to_delete ) summary.update(summary_ori) if revs_to_delete: summary_rev = self.idx_storage.revision_intrinsic_metadata_delete( revs_to_delete ) summary.update(summary_rev) return summary diff --git a/swh/indexer/storage/__init__.py b/swh/indexer/storage/__init__.py index 8fe7d52..d0a37dc 100644 --- a/swh/indexer/storage/__init__.py +++ b/swh/indexer/storage/__init__.py @@ -1,706 +1,723 @@ # Copyright (C) 2015-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from collections import Counter import json -from typing import Dict, Iterable, List, Optional, Tuple +from typing import Dict, Iterable, List, Optional, Tuple, Union import psycopg2 import psycopg2.pool from swh.core.db.common import db_transaction from swh.model.hashutil import hash_to_bytes, hash_to_hex from swh.model.model import SHA1_SIZE from swh.storage.exc import StorageDBError from swh.storage.utils import get_partition_bounds_bytes from . import converters from .db import Db from .exc import DuplicateId, IndexerStorageArgumentException from .interface import PagedResult, Sha1 from .metrics import process_metrics, send_metric, timed from .model import ( ContentCtagsRow, ContentLanguageRow, ContentLicenseRow, ContentMetadataRow, ContentMimetypeRow, OriginIntrinsicMetadataRow, RevisionIntrinsicMetadataRow, ) INDEXER_CFG_KEY = "indexer_storage" MAPPING_NAMES = ["codemeta", "gemspec", "maven", "npm", "pkg-info"] def get_indexer_storage(cls, args): """Get an indexer storage object of class `storage_class` with arguments `storage_args`. Args: cls (str): storage's class, either 'local' or 'remote' args (dict): dictionary of arguments passed to the storage class constructor Returns: an instance of swh.indexer's storage (either local or remote) Raises: ValueError if passed an unknown storage class. """ if cls == "remote": from .api.client import RemoteStorage as IndexerStorage elif cls == "local": from . import IndexerStorage elif cls == "memory": from .in_memory import IndexerStorage else: raise ValueError("Unknown indexer storage class `%s`" % cls) return IndexerStorage(**args) def check_id_duplicates(data): """ If any two row models in `data` have the same unique key, raises a `ValueError`. Values associated to the key must be hashable. Args: data (List[dict]): List of dictionaries to be inserted >>> check_id_duplicates([ ... ContentLanguageRow(id=b'foo', indexer_configuration_id=42, lang="python"), ... ContentLanguageRow(id=b'foo', indexer_configuration_id=32, lang="python"), ... ]) >>> check_id_duplicates([ ... ContentLanguageRow(id=b'foo', indexer_configuration_id=42, lang="python"), ... ContentLanguageRow(id=b'foo', indexer_configuration_id=42, lang="python"), ... ]) Traceback (most recent call last): ... swh.indexer.storage.exc.DuplicateId: [{'id': b'foo', 'indexer_configuration_id': 42}] """ # noqa counter = Counter(tuple(sorted(item.unique_key().items())) for item in data) duplicates = [id_ for (id_, count) in counter.items() if count >= 2] if duplicates: raise DuplicateId(list(map(dict, duplicates))) class IndexerStorage: """SWH Indexer Storage """ def __init__(self, db, min_pool_conns=1, max_pool_conns=10): """ Args: db_conn: either a libpq connection string, or a psycopg2 connection """ try: if isinstance(db, psycopg2.extensions.connection): self._pool = None self._db = Db(db) else: self._pool = psycopg2.pool.ThreadedConnectionPool( min_pool_conns, max_pool_conns, db ) self._db = None except psycopg2.OperationalError as e: raise StorageDBError(e) def get_db(self): if self._db: return self._db return Db.from_pool(self._pool) def put_db(self, db): if db is not self._db: db.put_conn() @timed @db_transaction() def check_config(self, *, check_write, db=None, cur=None): # Check permissions on one of the tables if check_write: check = "INSERT" else: check = "SELECT" cur.execute( "select has_table_privilege(current_user, 'content_mimetype', %s)", # noqa (check,), ) return cur.fetchone()[0] @timed @db_transaction() def content_mimetype_missing( self, mimetypes: Iterable[Dict], db=None, cur=None ) -> List[Tuple[Sha1, int]]: return [obj[0] for obj in db.content_mimetype_missing_from_list(mimetypes, cur)] @timed @db_transaction() def get_partition( self, indexer_type: str, indexer_configuration_id: int, partition_id: int, nb_partitions: int, page_token: Optional[str] = None, limit: int = 1000, with_textual_data=False, db=None, cur=None, ) -> PagedResult[Sha1]: """Retrieve ids of content with `indexer_type` within within partition partition_id bound by limit. Args: **indexer_type**: Type of data content to index (mimetype, language, etc...) **indexer_configuration_id**: The tool used to index data **partition_id**: index of the partition to fetch **nb_partitions**: total number of partitions to split into **page_token**: opaque token used for pagination **limit**: Limit result (default to 1000) **with_textual_data** (bool): Deal with only textual content (True) or all content (all contents by defaults, False) Raises: IndexerStorageArgumentException for; - limit to None - wrong indexer_type provided Returns: PagedResult of Sha1. If next_page_token is None, there is no more data to fetch """ if limit is None: raise IndexerStorageArgumentException("limit should not be None") if indexer_type not in db.content_indexer_names: err = f"Wrong type. Should be one of [{','.join(db.content_indexer_names)}]" raise IndexerStorageArgumentException(err) start, end = get_partition_bounds_bytes(partition_id, nb_partitions, SHA1_SIZE) if page_token is not None: start = hash_to_bytes(page_token) if end is None: end = b"\xff" * SHA1_SIZE next_page_token: Optional[str] = None ids = [ row[0] for row in db.content_get_range( indexer_type, start, end, indexer_configuration_id, limit=limit + 1, with_textual_data=with_textual_data, cur=cur, ) ] if len(ids) >= limit: next_page_token = hash_to_hex(ids[-1]) ids = ids[:limit] assert len(ids) <= limit return PagedResult(results=ids, next_page_token=next_page_token) @timed @db_transaction() def content_mimetype_get_partition( self, indexer_configuration_id: int, partition_id: int, nb_partitions: int, page_token: Optional[str] = None, limit: int = 1000, db=None, cur=None, ) -> PagedResult[Sha1]: return self.get_partition( "mimetype", indexer_configuration_id, partition_id, nb_partitions, page_token=page_token, limit=limit, db=db, cur=cur, ) @timed @process_metrics @db_transaction() def content_mimetype_add( self, mimetypes: List[ContentMimetypeRow], conflict_update: bool = False, db=None, cur=None, ) -> Dict[str, int]: check_id_duplicates(mimetypes) mimetypes.sort(key=lambda m: m.id) db.mktemp_content_mimetype(cur) db.copy_to( [m.to_dict() for m in mimetypes], "tmp_content_mimetype", ["id", "mimetype", "encoding", "indexer_configuration_id"], cur, ) count = db.content_mimetype_add_from_temp(conflict_update, cur) return {"content_mimetype:add": count} @timed @db_transaction() def content_mimetype_get( self, ids: Iterable[Sha1], db=None, cur=None ) -> List[ContentMimetypeRow]: return [ ContentMimetypeRow.from_dict( converters.db_to_mimetype(dict(zip(db.content_mimetype_cols, c))) ) for c in db.content_mimetype_get_from_list(ids, cur) ] @timed @db_transaction() def content_language_missing( self, languages: Iterable[Dict], db=None, cur=None ) -> List[Tuple[Sha1, int]]: return [obj[0] for obj in db.content_language_missing_from_list(languages, cur)] @timed @db_transaction() def content_language_get( self, ids: Iterable[Sha1], db=None, cur=None ) -> List[ContentLanguageRow]: return [ ContentLanguageRow.from_dict( converters.db_to_language(dict(zip(db.content_language_cols, c))) ) for c in db.content_language_get_from_list(ids, cur) ] @timed @process_metrics @db_transaction() def content_language_add( self, languages: List[ContentLanguageRow], conflict_update: bool = False, db=None, cur=None, ) -> Dict[str, int]: check_id_duplicates(languages) languages.sort(key=lambda m: m.id) db.mktemp_content_language(cur) # empty language is mapped to 'unknown' db.copy_to( ( { "id": lang.id, "lang": lang.lang or "unknown", "indexer_configuration_id": lang.indexer_configuration_id, } for lang in languages ), "tmp_content_language", ["id", "lang", "indexer_configuration_id"], cur, ) count = db.content_language_add_from_temp(conflict_update, cur) return {"content_language:add": count} @timed @db_transaction() def content_ctags_missing( self, ctags: Iterable[Dict], db=None, cur=None ) -> List[Tuple[Sha1, int]]: return [obj[0] for obj in db.content_ctags_missing_from_list(ctags, cur)] @timed @db_transaction() def content_ctags_get( self, ids: Iterable[Sha1], db=None, cur=None ) -> List[ContentCtagsRow]: return [ ContentCtagsRow.from_dict( converters.db_to_ctags(dict(zip(db.content_ctags_cols, c))) ) for c in db.content_ctags_get_from_list(ids, cur) ] @timed @process_metrics @db_transaction() def content_ctags_add( self, ctags: List[ContentCtagsRow], conflict_update: bool = False, db=None, cur=None, ) -> Dict[str, int]: check_id_duplicates(ctags) ctags.sort(key=lambda m: m.id) db.mktemp_content_ctags(cur) db.copy_to( [ctag.to_dict() for ctag in ctags], tblname="tmp_content_ctags", columns=["id", "name", "kind", "line", "lang", "indexer_configuration_id"], cur=cur, ) count = db.content_ctags_add_from_temp(conflict_update, cur) return {"content_ctags:add": count} @timed @db_transaction() def content_ctags_search( self, expression: str, limit: int = 10, last_sha1: Optional[Sha1] = None, db=None, cur=None, ) -> List[ContentCtagsRow]: return [ ContentCtagsRow.from_dict( converters.db_to_ctags(dict(zip(db.content_ctags_cols, obj))) ) for obj in db.content_ctags_search(expression, last_sha1, limit, cur=cur) ] @timed @db_transaction() def content_fossology_license_get( self, ids: Iterable[Sha1], db=None, cur=None ) -> List[ContentLicenseRow]: return [ ContentLicenseRow.from_dict( converters.db_to_fossology_license( dict(zip(db.content_fossology_license_cols, c)) ) ) for c in db.content_fossology_license_get_from_list(ids, cur) ] @timed @process_metrics @db_transaction() def content_fossology_license_add( self, licenses: List[ContentLicenseRow], conflict_update: bool = False, db=None, cur=None, ) -> Dict[str, int]: check_id_duplicates(licenses) licenses.sort(key=lambda m: m.id) db.mktemp_content_fossology_license(cur) db.copy_to( [license.to_dict() for license in licenses], tblname="tmp_content_fossology_license", columns=["id", "license", "indexer_configuration_id"], cur=cur, ) count = db.content_fossology_license_add_from_temp(conflict_update, cur) return {"content_fossology_license:add": count} @timed @db_transaction() def content_fossology_license_get_partition( self, indexer_configuration_id: int, partition_id: int, nb_partitions: int, page_token: Optional[str] = None, limit: int = 1000, db=None, cur=None, ) -> PagedResult[Sha1]: return self.get_partition( "fossology_license", indexer_configuration_id, partition_id, nb_partitions, page_token=page_token, limit=limit, with_textual_data=True, db=db, cur=cur, ) @timed @db_transaction() def content_metadata_missing( self, metadata: Iterable[Dict], db=None, cur=None ) -> List[Tuple[Sha1, int]]: return [obj[0] for obj in db.content_metadata_missing_from_list(metadata, cur)] @timed @db_transaction() def content_metadata_get( self, ids: Iterable[Sha1], db=None, cur=None ) -> List[ContentMetadataRow]: return [ ContentMetadataRow.from_dict( converters.db_to_metadata(dict(zip(db.content_metadata_cols, c))) ) for c in db.content_metadata_get_from_list(ids, cur) ] @timed @process_metrics @db_transaction() def content_metadata_add( self, metadata: List[ContentMetadataRow], conflict_update: bool = False, db=None, cur=None, ) -> Dict[str, int]: check_id_duplicates(metadata) metadata.sort(key=lambda m: m.id) db.mktemp_content_metadata(cur) db.copy_to( [m.to_dict() for m in metadata], "tmp_content_metadata", ["id", "metadata", "indexer_configuration_id"], cur, ) count = db.content_metadata_add_from_temp(conflict_update, cur) return { "content_metadata:add": count, } @timed @db_transaction() def revision_intrinsic_metadata_missing( self, metadata: Iterable[Dict], db=None, cur=None ) -> List[Tuple[Sha1, int]]: return [ obj[0] for obj in db.revision_intrinsic_metadata_missing_from_list(metadata, cur) ] @timed @db_transaction() def revision_intrinsic_metadata_get( self, ids: Iterable[Sha1], db=None, cur=None ) -> List[RevisionIntrinsicMetadataRow]: return [ RevisionIntrinsicMetadataRow.from_dict( converters.db_to_metadata( dict(zip(db.revision_intrinsic_metadata_cols, c)) ) ) for c in db.revision_intrinsic_metadata_get_from_list(ids, cur) ] @timed @process_metrics @db_transaction() def revision_intrinsic_metadata_add( self, metadata: List[RevisionIntrinsicMetadataRow], conflict_update: bool = False, db=None, cur=None, ) -> Dict[str, int]: check_id_duplicates(metadata) metadata.sort(key=lambda m: m.id) db.mktemp_revision_intrinsic_metadata(cur) db.copy_to( [m.to_dict() for m in metadata], "tmp_revision_intrinsic_metadata", ["id", "metadata", "mappings", "indexer_configuration_id"], cur, ) count = db.revision_intrinsic_metadata_add_from_temp(conflict_update, cur) return { "revision_intrinsic_metadata:add": count, } @timed @process_metrics @db_transaction() def revision_intrinsic_metadata_delete( self, entries: List[Dict], db=None, cur=None ) -> Dict: count = db.revision_intrinsic_metadata_delete(entries, cur) return {"revision_intrinsic_metadata:del": count} @timed @db_transaction() - def origin_intrinsic_metadata_get(self, ids, db=None, cur=None): + def origin_intrinsic_metadata_get( + self, urls: Iterable[str], db=None, cur=None + ) -> List[OriginIntrinsicMetadataRow]: return [ - converters.db_to_metadata(dict(zip(db.origin_intrinsic_metadata_cols, c))) - for c in db.origin_intrinsic_metadata_get_from_list(ids, cur) + OriginIntrinsicMetadataRow.from_dict( + converters.db_to_metadata( + dict(zip(db.origin_intrinsic_metadata_cols, c)) + ) + ) + for c in db.origin_intrinsic_metadata_get_from_list(urls, cur) ] @timed @process_metrics @db_transaction() def origin_intrinsic_metadata_add( - self, metadata: List[Dict], conflict_update: bool = False, db=None, cur=None + self, + metadata: List[OriginIntrinsicMetadataRow], + conflict_update: bool = False, + db=None, + cur=None, ) -> Dict[str, int]: - check_id_duplicates(map(OriginIntrinsicMetadataRow.from_dict, metadata)) - metadata.sort(key=lambda m: m["id"]) + check_id_duplicates(metadata) + metadata.sort(key=lambda m: m.id) db.mktemp_origin_intrinsic_metadata(cur) db.copy_to( - metadata, + [m.to_dict() for m in metadata], "tmp_origin_intrinsic_metadata", ["id", "metadata", "indexer_configuration_id", "from_revision", "mappings"], cur, ) count = db.origin_intrinsic_metadata_add_from_temp(conflict_update, cur) return { "origin_intrinsic_metadata:add": count, } @timed @process_metrics @db_transaction() def origin_intrinsic_metadata_delete( self, entries: List[Dict], db=None, cur=None ) -> Dict: count = db.origin_intrinsic_metadata_delete(entries, cur) return { "origin_intrinsic_metadata:del": count, } @timed @db_transaction() def origin_intrinsic_metadata_search_fulltext( - self, conjunction, limit=100, db=None, cur=None - ): + self, conjunction: List[str], limit: int = 100, db=None, cur=None + ) -> List[OriginIntrinsicMetadataRow]: return [ - converters.db_to_metadata(dict(zip(db.origin_intrinsic_metadata_cols, c))) + OriginIntrinsicMetadataRow.from_dict( + converters.db_to_metadata( + dict(zip(db.origin_intrinsic_metadata_cols, c)) + ) + ) for c in db.origin_intrinsic_metadata_search_fulltext( conjunction, limit=limit, cur=cur ) ] @timed @db_transaction() def origin_intrinsic_metadata_search_by_producer( self, - page_token="", - limit=100, - ids_only=False, - mappings=None, - tool_ids=None, + page_token: str = "", + limit: int = 100, + ids_only: bool = False, + mappings: Optional[List[str]] = None, + tool_ids: Optional[List[int]] = None, db=None, cur=None, - ): + ) -> PagedResult[Union[str, OriginIntrinsicMetadataRow]]: assert isinstance(page_token, str) # we go to limit+1 to check whether we should add next_page_token in # the response - res = db.origin_intrinsic_metadata_search_by_producer( + rows = db.origin_intrinsic_metadata_search_by_producer( page_token, limit + 1, ids_only, mappings, tool_ids, cur ) - result = {} + next_page_token = None if ids_only: - result["origins"] = [origin for (origin,) in res] - if len(result["origins"]) > limit: - result["origins"][limit:] = [] - result["next_page_token"] = result["origins"][-1] + results = [origin for (origin,) in rows] + if len(results) > limit: + results[limit:] = [] + next_page_token = results[-1] else: - result["origins"] = [ - converters.db_to_metadata( - dict(zip(db.origin_intrinsic_metadata_cols, c)) + results = [ + OriginIntrinsicMetadataRow.from_dict( + converters.db_to_metadata( + dict(zip(db.origin_intrinsic_metadata_cols, row)) + ) ) - for c in res + for row in rows ] - if len(result["origins"]) > limit: - result["origins"][limit:] = [] - result["next_page_token"] = result["origins"][-1]["id"] - return result + if len(results) > limit: + results[limit:] = [] + next_page_token = results[-1].id + + return PagedResult(results=results, next_page_token=next_page_token,) @timed @db_transaction() def origin_intrinsic_metadata_stats(self, db=None, cur=None): mapping_names = [m for m in MAPPING_NAMES] select_parts = [] # Count rows for each mapping for mapping_name in mapping_names: select_parts.append( ( "sum(case when (mappings @> ARRAY['%s']) " " then 1 else 0 end)" ) % mapping_name ) # Total select_parts.append("sum(1)") # Rows whose metadata has at least one key that is not '@context' select_parts.append( "sum(case when ('{}'::jsonb @> (metadata - '@context')) " " then 0 else 1 end)" ) cur.execute( "select " + ", ".join(select_parts) + " from origin_intrinsic_metadata" ) results = dict(zip(mapping_names + ["total", "non_empty"], cur.fetchone())) return { "total": results.pop("total"), "non_empty": results.pop("non_empty"), "per_mapping": results, } @timed @db_transaction() def indexer_configuration_add(self, tools, db=None, cur=None): db.mktemp_indexer_configuration(cur) db.copy_to( tools, "tmp_indexer_configuration", ["tool_name", "tool_version", "tool_configuration"], cur, ) tools = db.indexer_configuration_add_from_temp(cur) results = [dict(zip(db.indexer_configuration_cols, line)) for line in tools] send_metric( "indexer_configuration:add", len(results), method_name="indexer_configuration_add", ) return results @timed @db_transaction() def indexer_configuration_get(self, tool, db=None, cur=None): tool_conf = tool["tool_configuration"] if isinstance(tool_conf, dict): tool_conf = json.dumps(tool_conf) idx = db.indexer_configuration_get( tool["tool_name"], tool["tool_version"], tool_conf ) if not idx: return None return dict(zip(db.indexer_configuration_cols, idx)) diff --git a/swh/indexer/storage/in_memory.py b/swh/indexer/storage/in_memory.py index d04dd19..5f8c517 100644 --- a/swh/indexer/storage/in_memory.py +++ b/swh/indexer/storage/in_memory.py @@ -1,512 +1,520 @@ # Copyright (C) 2018-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from collections import Counter, defaultdict import itertools import json import math import operator import re from typing import ( Any, Dict, Generic, Iterable, List, Optional, Set, Tuple, Type, TypeVar, + Union, ) from swh.core.collections import SortedList from swh.model.hashutil import hash_to_bytes, hash_to_hex from swh.model.model import SHA1_SIZE, Sha1Git from swh.storage.utils import get_partition_bounds_bytes from . import MAPPING_NAMES, check_id_duplicates from .exc import IndexerStorageArgumentException from .interface import PagedResult, Sha1 from .model import ( BaseRow, ContentCtagsRow, ContentLanguageRow, ContentLicenseRow, ContentMetadataRow, ContentMimetypeRow, OriginIntrinsicMetadataRow, RevisionIntrinsicMetadataRow, ) SHA1_DIGEST_SIZE = 160 def _transform_tool(tool): return { "id": tool["id"], "name": tool["tool_name"], "version": tool["tool_version"], "configuration": tool["tool_configuration"], } def check_id_types(data: List[Dict[str, Any]]): """Checks all elements of the list have an 'id' whose type is 'bytes'.""" if not all(isinstance(item.get("id"), bytes) for item in data): raise IndexerStorageArgumentException("identifiers must be bytes.") def _key_from_dict(d): return tuple(sorted(d.items())) ToolId = int TValue = TypeVar("TValue", bound=BaseRow) class SubStorage(Generic[TValue]): """Implements common missing/get/add logic for each indexer type.""" _data: Dict[Sha1, Dict[Tuple, Dict[str, Any]]] _tools_per_id: Dict[Sha1, Set[ToolId]] def __init__(self, row_class: Type[TValue], tools): self.row_class = row_class self._tools = tools self._sorted_ids = SortedList[bytes, Sha1]() self._data = defaultdict(dict) self._tools_per_id = defaultdict(set) def _key_from_dict(self, d) -> Tuple: """Like the global _key_from_dict, but filters out dict keys that don't belong in the unique key.""" return _key_from_dict({k: d[k] for k in self.row_class.UNIQUE_KEY_FIELDS}) def missing(self, keys: Iterable[Dict]) -> List[Sha1]: """List data missing from storage. Args: data (iterable): dictionaries with keys: - **id** (bytes): sha1 identifier - **indexer_configuration_id** (int): tool used to compute the results Yields: missing sha1s """ results = [] for key in keys: tool_id = key["indexer_configuration_id"] id_ = key["id"] if tool_id not in self._tools_per_id.get(id_, set()): results.append(id_) return results def get(self, ids: Iterable[Sha1]) -> List[TValue]: """Retrieve data per id. Args: ids (iterable): sha1 checksums Yields: dict: dictionaries with the following keys: - **id** (bytes) - **tool** (dict): tool used to compute metadata - arbitrary data (as provided to `add`) """ results = [] for id_ in ids: for entry in self._data[id_].values(): entry = entry.copy() tool_id = entry.pop("indexer_configuration_id") results.append( self.row_class( id=id_, tool=_transform_tool(self._tools[tool_id]), **entry, ) ) return results def get_all(self) -> List[TValue]: return self.get(self._sorted_ids) def get_partition( self, indexer_configuration_id: int, partition_id: int, nb_partitions: int, page_token: Optional[str] = None, limit: int = 1000, ) -> PagedResult[Sha1]: """Retrieve ids of content with `indexer_type` within partition partition_id bound by limit. Args: **indexer_type**: Type of data content to index (mimetype, language, etc...) **indexer_configuration_id**: The tool used to index data **partition_id**: index of the partition to fetch **nb_partitions**: total number of partitions to split into **page_token**: opaque token used for pagination **limit**: Limit result (default to 1000) **with_textual_data** (bool): Deal with only textual content (True) or all content (all contents by defaults, False) Raises: IndexerStorageArgumentException for; - limit to None - wrong indexer_type provided Returns: PagedResult of Sha1. If next_page_token is None, there is no more data to fetch """ if limit is None: raise IndexerStorageArgumentException("limit should not be None") (start, end) = get_partition_bounds_bytes( partition_id, nb_partitions, SHA1_SIZE ) if page_token: start = hash_to_bytes(page_token) if end is None: end = b"\xff" * SHA1_SIZE next_page_token: Optional[str] = None ids: List[Sha1] = [] sha1s = (sha1 for sha1 in self._sorted_ids.iter_from(start)) for counter, sha1 in enumerate(sha1s): if sha1 > end: break if counter >= limit: next_page_token = hash_to_hex(sha1) break ids.append(sha1) assert len(ids) <= limit return PagedResult(results=ids, next_page_token=next_page_token) def add(self, data: Iterable[TValue], conflict_update: bool) -> int: """Add data not present in storage. Args: data (iterable): dictionaries with keys: - **id**: sha1 - **indexer_configuration_id**: tool used to compute the results - arbitrary data conflict_update (bool): Flag to determine if we want to overwrite (true) or skip duplicates (false) """ data = list(data) check_id_duplicates(data) count = 0 for obj in data: item = obj.to_dict() id_ = item.pop("id") tool_id = item["indexer_configuration_id"] key = _key_from_dict(obj.unique_key()) if not conflict_update and key in self._data[id_]: # Duplicate, should not be updated continue self._data[id_][key] = item self._tools_per_id[id_].add(tool_id) count += 1 if id_ not in self._sorted_ids: self._sorted_ids.add(id_) return count def delete(self, entries: List[Dict]) -> int: """Delete entries and return the number of entries deleted. """ deleted = 0 for entry in entries: (id_, tool_id) = (entry["id"], entry["indexer_configuration_id"]) if tool_id in self._tools_per_id[id_]: self._tools_per_id[id_].remove(tool_id) if id_ in self._data: key = self._key_from_dict(entry) if key in self._data[id_]: deleted += 1 del self._data[id_][key] return deleted class IndexerStorage: """In-memory SWH indexer storage.""" def __init__(self): self._tools = {} self._mimetypes = SubStorage(ContentMimetypeRow, self._tools) self._languages = SubStorage(ContentLanguageRow, self._tools) self._content_ctags = SubStorage(ContentCtagsRow, self._tools) self._licenses = SubStorage(ContentLicenseRow, self._tools) self._content_metadata = SubStorage(ContentMetadataRow, self._tools) self._revision_intrinsic_metadata = SubStorage( RevisionIntrinsicMetadataRow, self._tools ) self._origin_intrinsic_metadata = SubStorage( OriginIntrinsicMetadataRow, self._tools ) def check_config(self, *, check_write): return True def content_mimetype_missing( self, mimetypes: Iterable[Dict] ) -> List[Tuple[Sha1, int]]: return self._mimetypes.missing(mimetypes) def content_mimetype_get_partition( self, indexer_configuration_id: int, partition_id: int, nb_partitions: int, page_token: Optional[str] = None, limit: int = 1000, ) -> PagedResult[Sha1]: return self._mimetypes.get_partition( indexer_configuration_id, partition_id, nb_partitions, page_token, limit ) def content_mimetype_add( self, mimetypes: List[ContentMimetypeRow], conflict_update: bool = False ) -> Dict[str, int]: added = self._mimetypes.add(mimetypes, conflict_update) return {"content_mimetype:add": added} def content_mimetype_get(self, ids: Iterable[Sha1]) -> List[ContentMimetypeRow]: return self._mimetypes.get(ids) def content_language_missing( self, languages: Iterable[Dict] ) -> List[Tuple[Sha1, int]]: return self._languages.missing(languages) def content_language_get(self, ids: Iterable[Sha1]) -> List[ContentLanguageRow]: return self._languages.get(ids) def content_language_add( self, languages: List[ContentLanguageRow], conflict_update: bool = False ) -> Dict[str, int]: added = self._languages.add(languages, conflict_update) return {"content_language:add": added} def content_ctags_missing(self, ctags: Iterable[Dict]) -> List[Tuple[Sha1, int]]: return self._content_ctags.missing(ctags) def content_ctags_get(self, ids: Iterable[Sha1]) -> List[ContentCtagsRow]: return self._content_ctags.get(ids) def content_ctags_add( self, ctags: List[ContentCtagsRow], conflict_update: bool = False ) -> Dict[str, int]: added = self._content_ctags.add(ctags, conflict_update,) return {"content_ctags:add": added} def content_ctags_search( self, expression: str, limit: int = 10, last_sha1: Optional[Sha1] = None ) -> List[ContentCtagsRow]: nb_matches = 0 items_per_id: Dict[Tuple[Sha1Git, ToolId], List[ContentCtagsRow]] = {} for item in sorted(self._content_ctags.get_all()): if item.id <= (last_sha1 or bytes(0 for _ in range(SHA1_DIGEST_SIZE))): continue items_per_id.setdefault( (item.id, item.indexer_configuration_id), [] ).append(item) results = [] for items in items_per_id.values(): for item in items: if item.name != expression: continue nb_matches += 1 if nb_matches > limit: break results.append(item) return results def content_fossology_license_get( self, ids: Iterable[Sha1] ) -> List[ContentLicenseRow]: return self._licenses.get(ids) def content_fossology_license_add( self, licenses: List[ContentLicenseRow], conflict_update: bool = False ) -> Dict[str, int]: added = self._licenses.add(licenses, conflict_update) return {"content_fossology_license:add": added} def content_fossology_license_get_partition( self, indexer_configuration_id: int, partition_id: int, nb_partitions: int, page_token: Optional[str] = None, limit: int = 1000, ) -> PagedResult[Sha1]: return self._licenses.get_partition( indexer_configuration_id, partition_id, nb_partitions, page_token, limit ) def content_metadata_missing( self, metadata: Iterable[Dict] ) -> List[Tuple[Sha1, int]]: return self._content_metadata.missing(metadata) def content_metadata_get(self, ids: Iterable[Sha1]) -> List[ContentMetadataRow]: return self._content_metadata.get(ids) def content_metadata_add( self, metadata: List[ContentMetadataRow], conflict_update: bool = False ) -> Dict[str, int]: added = self._content_metadata.add(metadata, conflict_update) return {"content_metadata:add": added} def revision_intrinsic_metadata_missing( self, metadata: Iterable[Dict] ) -> List[Tuple[Sha1, int]]: return self._revision_intrinsic_metadata.missing(metadata) def revision_intrinsic_metadata_get( self, ids: Iterable[Sha1] ) -> List[RevisionIntrinsicMetadataRow]: return self._revision_intrinsic_metadata.get(ids) def revision_intrinsic_metadata_add( self, metadata: List[RevisionIntrinsicMetadataRow], conflict_update: bool = False, ) -> Dict[str, int]: added = self._revision_intrinsic_metadata.add(metadata, conflict_update) return {"revision_intrinsic_metadata:add": added} def revision_intrinsic_metadata_delete(self, entries: List[Dict]) -> Dict: deleted = self._revision_intrinsic_metadata.delete(entries) return {"revision_intrinsic_metadata:del": deleted} - def origin_intrinsic_metadata_get(self, ids): - return [obj.to_dict() for obj in self._origin_intrinsic_metadata.get(ids)] + def origin_intrinsic_metadata_get( + self, urls: Iterable[str] + ) -> List[OriginIntrinsicMetadataRow]: + return self._origin_intrinsic_metadata.get(urls) def origin_intrinsic_metadata_add( - self, metadata: List[Dict], conflict_update: bool = False + self, metadata: List[OriginIntrinsicMetadataRow], conflict_update: bool = False ) -> Dict[str, int]: - added = self._origin_intrinsic_metadata.add( - map(OriginIntrinsicMetadataRow.from_dict, metadata), conflict_update - ) + added = self._origin_intrinsic_metadata.add(metadata, conflict_update) return {"origin_intrinsic_metadata:add": added} def origin_intrinsic_metadata_delete(self, entries: List[Dict]) -> Dict: deleted = self._origin_intrinsic_metadata.delete(entries) return {"origin_intrinsic_metadata:del": deleted} - def origin_intrinsic_metadata_search_fulltext(self, conjunction, limit=100): + def origin_intrinsic_metadata_search_fulltext( + self, conjunction: List[str], limit: int = 100 + ) -> List[OriginIntrinsicMetadataRow]: # A very crude fulltext search implementation, but that's enough # to work on English metadata tokens_re = re.compile("[a-zA-Z0-9]+") search_tokens = list(itertools.chain(*map(tokens_re.findall, conjunction))) def rank(data): # Tokenize the metadata text = json.dumps(data.metadata) text_tokens = tokens_re.findall(text) text_token_occurences = Counter(text_tokens) # Count the number of occurrences of search tokens in the text score = 0 for search_token in search_tokens: if text_token_occurences[search_token] == 0: # Search token is not in the text. return 0 score += text_token_occurences[search_token] # Normalize according to the text's length return score / math.log(len(text_tokens)) results = [ (rank(data), data) for data in self._origin_intrinsic_metadata.get_all() ] results = [(rank_, data) for (rank_, data) in results if rank_ > 0] results.sort( key=operator.itemgetter(0), reverse=True # Don't try to order 'data' ) - return [result.to_dict() for (rank_, result) in results[:limit]] + return [result for (rank_, result) in results[:limit]] def origin_intrinsic_metadata_search_by_producer( - self, page_token="", limit=100, ids_only=False, mappings=None, tool_ids=None - ): + self, + page_token: str = "", + limit: int = 100, + ids_only: bool = False, + mappings: Optional[List[str]] = None, + tool_ids: Optional[List[int]] = None, + ) -> PagedResult[Union[str, OriginIntrinsicMetadataRow]]: assert isinstance(page_token, str) nb_results = 0 if mappings is not None: - mappings = frozenset(mappings) + mapping_set = frozenset(mappings) if tool_ids is not None: - tool_ids = frozenset(tool_ids) - origins = [] + tool_id_set = frozenset(tool_ids) + rows = [] # we go to limit+1 to check whether we should add next_page_token in # the response for entry in self._origin_intrinsic_metadata.get_all(): if entry.id <= page_token: continue if nb_results >= (limit + 1): break - if mappings is not None and mappings.isdisjoint(entry.mappings): + if mappings and mapping_set.isdisjoint(entry.mappings): continue - if tool_ids is not None and entry.tool["id"] not in tool_ids: + if tool_ids and entry.tool["id"] not in tool_id_set: continue - origins.append(entry.to_dict()) + rows.append(entry) nb_results += 1 - result = {} - if len(origins) > limit: - origins = origins[:limit] - result["next_page_token"] = origins[-1]["id"] + if len(rows) > limit: + rows = rows[:limit] + next_page_token = rows[-1].id + else: + next_page_token = None if ids_only: - origins = [origin["id"] for origin in origins] - result["origins"] = origins - return result + rows = [row.id for row in rows] + return PagedResult(results=rows, next_page_token=next_page_token,) def origin_intrinsic_metadata_stats(self): mapping_count = {m: 0 for m in MAPPING_NAMES} total = non_empty = 0 for data in self._origin_intrinsic_metadata.get_all(): total += 1 if set(data.metadata) - {"@context"}: non_empty += 1 for mapping in data.mappings: mapping_count[mapping] += 1 return {"per_mapping": mapping_count, "total": total, "non_empty": non_empty} def indexer_configuration_add(self, tools): inserted = [] for tool in tools: tool = tool.copy() id_ = self._tool_key(tool) tool["id"] = id_ self._tools[id_] = tool inserted.append(tool) return inserted def indexer_configuration_get(self, tool): return self._tools.get(self._tool_key(tool)) def _tool_key(self, tool): return hash( ( tool["tool_name"], tool["tool_version"], json.dumps(tool["tool_configuration"], sort_keys=True), ) ) diff --git a/swh/indexer/storage/interface.py b/swh/indexer/storage/interface.py index 468933a..731062f 100644 --- a/swh/indexer/storage/interface.py +++ b/swh/indexer/storage/interface.py @@ -1,598 +1,569 @@ # Copyright (C) 2015-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -from typing import Dict, Iterable, List, Optional, Tuple, TypeVar +from typing import Dict, Iterable, List, Optional, Tuple, TypeVar, Union from swh.core.api import remote_api_endpoint from swh.core.api.classes import PagedResult as CorePagedResult from swh.indexer.storage.model import ( ContentCtagsRow, ContentLanguageRow, ContentLicenseRow, ContentMetadataRow, ContentMimetypeRow, + OriginIntrinsicMetadataRow, RevisionIntrinsicMetadataRow, ) TResult = TypeVar("TResult") PagedResult = CorePagedResult[TResult, str] Sha1 = bytes class IndexerStorageInterface: @remote_api_endpoint("check_config") def check_config(self, *, check_write): """Check that the storage is configured and ready to go.""" ... @remote_api_endpoint("content_mimetype/missing") def content_mimetype_missing( self, mimetypes: Iterable[Dict] ) -> List[Tuple[Sha1, int]]: """Generate mimetypes missing from storage. Args: mimetypes (iterable): iterable of dict with keys: - **id** (bytes): sha1 identifier - **indexer_configuration_id** (int): tool used to compute the results Returns: list of tuple (id, indexer_configuration_id) missing """ ... @remote_api_endpoint("content_mimetype/range") def content_mimetype_get_partition( self, indexer_configuration_id: int, partition_id: int, nb_partitions: int, page_token: Optional[str] = None, limit: int = 1000, ) -> PagedResult[Sha1]: """Retrieve mimetypes within partition partition_id bound by limit. Args: **indexer_configuration_id**: The tool used to index data **partition_id**: index of the partition to fetch **nb_partitions**: total number of partitions to split into **page_token**: opaque token used for pagination **limit**: Limit result (default to 1000) Raises: IndexerStorageArgumentException for; - limit to None - wrong indexer_type provided Returns: PagedResult of Sha1. If next_page_token is None, there is no more data to fetch """ ... @remote_api_endpoint("content_mimetype/add") def content_mimetype_add( self, mimetypes: List[ContentMimetypeRow], conflict_update: bool = False ) -> Dict[str, int]: """Add mimetypes not present in storage. Args: mimetypes: mimetype rows to be added, with their `tool` attribute set to None. conflict_update: Flag to determine if we want to overwrite (``True``) or skip duplicates (``False``, the default) Returns: Dict summary of number of rows added """ ... @remote_api_endpoint("content_mimetype") def content_mimetype_get(self, ids: Iterable[Sha1]) -> List[ContentMimetypeRow]: """Retrieve full content mimetype per ids. Args: ids: sha1 identifiers Returns: mimetype row objects """ ... @remote_api_endpoint("content_language/missing") def content_language_missing( self, languages: Iterable[Dict] ) -> List[Tuple[Sha1, int]]: """List languages missing from storage. Args: languages (iterable): dictionaries with keys: - **id** (bytes): sha1 identifier - **indexer_configuration_id** (int): tool used to compute the results Returns: list of tuple (id, indexer_configuration_id) missing """ ... @remote_api_endpoint("content_language") def content_language_get(self, ids: Iterable[Sha1]) -> List[ContentLanguageRow]: """Retrieve full content language per ids. Args: ids (iterable): sha1 identifier Returns: language row objects """ ... @remote_api_endpoint("content_language/add") def content_language_add( self, languages: List[ContentLanguageRow], conflict_update: bool = False ) -> Dict[str, int]: """Add languages not present in storage. Args: languages: language row objects conflict_update (bool): Flag to determine if we want to overwrite (true) or skip duplicates (false, the default) Returns: Dict summary of number of rows added """ ... @remote_api_endpoint("content/ctags/missing") def content_ctags_missing(self, ctags: Iterable[Dict]) -> List[Tuple[Sha1, int]]: """List ctags missing from storage. Args: ctags (iterable): dicts with keys: - **id** (bytes): sha1 identifier - **indexer_configuration_id** (int): tool used to compute the results Returns: list of missing id for the tuple (id, indexer_configuration_id) """ ... @remote_api_endpoint("content/ctags") def content_ctags_get(self, ids: Iterable[Sha1]) -> List[ContentCtagsRow]: """Retrieve ctags per id. Args: ids (iterable): sha1 checksums Returns: list of language rows """ ... @remote_api_endpoint("content/ctags/add") def content_ctags_add( self, ctags: List[ContentCtagsRow], conflict_update: bool = False ) -> Dict[str, int]: """Add ctags not present in storage Args: ctags (iterable): dictionaries with keys: - **id** (bytes): sha1 - **ctags** ([list): List of dictionary with keys: name, kind, line, lang Returns: Dict summary of number of rows added """ ... @remote_api_endpoint("content/ctags/search") def content_ctags_search( self, expression: str, limit: int = 10, last_sha1: Optional[Sha1] = None ) -> List[ContentCtagsRow]: """Search through content's raw ctags symbols. Args: expression (str): Expression to search for limit (int): Number of rows to return (default to 10). last_sha1 (str): Offset from which retrieving data (default to ''). Returns: rows of ctags including id, name, lang, kind, line, etc... """ ... @remote_api_endpoint("content/fossology_license") def content_fossology_license_get( self, ids: Iterable[Sha1] ) -> List[ContentLicenseRow]: """Retrieve licenses per id. Args: ids: sha1 identifiers Yields: license rows; possibly more than one per (sha1, tool_id) if there are multiple licenses. """ ... @remote_api_endpoint("content/fossology_license/add") def content_fossology_license_add( self, licenses: List[ContentLicenseRow], conflict_update: bool = False ) -> Dict[str, int]: """Add licenses not present in storage. Args: license: license rows to be added, with their `tool` attribute set to None. conflict_update: Flag to determine if we want to overwrite (true) or skip duplicates (false, the default) Returns: Dict summary of number of rows added """ ... @remote_api_endpoint("content/fossology_license/range") def content_fossology_license_get_partition( self, indexer_configuration_id: int, partition_id: int, nb_partitions: int, page_token: Optional[str] = None, limit: int = 1000, ) -> PagedResult[Sha1]: """Retrieve licenses within the partition partition_id bound by limit. Args: **indexer_configuration_id**: The tool used to index data **partition_id**: index of the partition to fetch **nb_partitions**: total number of partitions to split into **page_token**: opaque token used for pagination **limit**: Limit result (default to 1000) Raises: IndexerStorageArgumentException for; - limit to None - wrong indexer_type provided Returns: PagedResult of Sha1. If next_page_token is None, there is no more data to fetch """ ... @remote_api_endpoint("content_metadata/missing") def content_metadata_missing( self, metadata: Iterable[Dict] ) -> List[Tuple[Sha1, int]]: """List metadata missing from storage. Args: metadata (iterable): dictionaries with keys: - **id** (bytes): sha1 identifier - **indexer_configuration_id** (int): tool used to compute the results Yields: missing sha1s """ ... @remote_api_endpoint("content_metadata") def content_metadata_get(self, ids: Iterable[Sha1]) -> List[ContentMetadataRow]: """Retrieve metadata per id. Args: ids (iterable): sha1 checksums Yields: dictionaries with the following keys: id (bytes) metadata (str): associated metadata tool (dict): tool used to compute metadata """ ... @remote_api_endpoint("content_metadata/add") def content_metadata_add( self, metadata: List[ContentMetadataRow], conflict_update: bool = False ) -> Dict[str, int]: """Add metadata not present in storage. Args: metadata (iterable): dictionaries with keys: - **id**: sha1 - **metadata**: arbitrary dict conflict_update: Flag to determine if we want to overwrite (true) or skip duplicates (false, the default) Returns: Dict summary of number of rows added """ ... @remote_api_endpoint("revision_intrinsic_metadata/missing") def revision_intrinsic_metadata_missing( self, metadata: Iterable[Dict] ) -> List[Tuple[Sha1, int]]: """List metadata missing from storage. Args: metadata (iterable): dictionaries with keys: - **id** (bytes): sha1_git revision identifier - **indexer_configuration_id** (int): tool used to compute the results Returns: missing ids """ ... @remote_api_endpoint("revision_intrinsic_metadata") def revision_intrinsic_metadata_get( self, ids: Iterable[Sha1] ) -> List[RevisionIntrinsicMetadataRow]: """Retrieve revision metadata per id. Args: ids (iterable): sha1 checksums Returns: ContentMetadataRow objects """ ... @remote_api_endpoint("revision_intrinsic_metadata/add") def revision_intrinsic_metadata_add( self, metadata: List[RevisionIntrinsicMetadataRow], conflict_update: bool = False, ) -> Dict[str, int]: """Add metadata not present in storage. Args: metadata: ContentMetadataRow objects conflict_update: Flag to determine if we want to overwrite (true) or skip duplicates (false, the default) Returns: Dict summary of number of rows added """ ... @remote_api_endpoint("revision_intrinsic_metadata/delete") def revision_intrinsic_metadata_delete(self, entries: List[Dict]) -> Dict: """Remove revision metadata from the storage. Args: entries (dict): dictionaries with the following keys: - **id** (bytes): revision identifier - **indexer_configuration_id** (int): tool used to compute metadata Returns: Summary of number of rows deleted """ ... @remote_api_endpoint("origin_intrinsic_metadata") - def origin_intrinsic_metadata_get(self, ids): + def origin_intrinsic_metadata_get( + self, urls: Iterable[str] + ) -> List[OriginIntrinsicMetadataRow]: """Retrieve origin metadata per id. Args: - ids (iterable): origin identifiers - - Yields: - list: dictionaries with the following keys: - - - **id** (str): origin url - - **from_revision** (bytes): which revision this metadata - was extracted from - - **metadata** (str): associated metadata - - **tool** (dict): tool used to compute metadata - - **mappings** (List[str]): list of mappings used to translate - these metadata + urls (iterable): origin URLs + Returns: list of OriginIntrinsicMetadataRow """ ... @remote_api_endpoint("origin_intrinsic_metadata/add") def origin_intrinsic_metadata_add( - self, metadata: List[Dict], conflict_update: bool = False + self, metadata: List[OriginIntrinsicMetadataRow], conflict_update: bool = False ) -> Dict[str, int]: """Add origin metadata not present in storage. Args: - metadata (iterable): dictionaries with keys: - - - **id**: origin urls - - **from_revision**: sha1 id of the revision used to generate - these metadata. - - **metadata**: arbitrary dict - - **indexer_configuration_id**: tool used to compute metadata - - **mappings** (List[str]): list of mappings used to translate - these metadata + metadata: list of OriginIntrinsicMetadataRow objects conflict_update: Flag to determine if we want to overwrite (true) or skip duplicates (false, the default) Returns: Dict summary of number of rows added """ ... @remote_api_endpoint("origin_intrinsic_metadata/delete") def origin_intrinsic_metadata_delete(self, entries: List[Dict]) -> Dict: """Remove origin metadata from the storage. Args: entries (dict): dictionaries with the following keys: - **id** (str): origin urls - **indexer_configuration_id** (int): tool used to compute metadata Returns: Summary of number of rows deleted """ ... @remote_api_endpoint("origin_intrinsic_metadata/search/fulltext") - def origin_intrinsic_metadata_search_fulltext(self, conjunction, limit=100): + def origin_intrinsic_metadata_search_fulltext( + self, conjunction: List[str], limit: int = 100 + ) -> List[OriginIntrinsicMetadataRow]: """Returns the list of origins whose metadata contain all the terms. Args: - conjunction (List[str]): List of terms to be searched for. - limit (int): The maximum number of results to return + conjunction: List of terms to be searched for. + limit: The maximum number of results to return - Yields: - list: dictionaries with the following keys: - - - **id** (str): origin urls - - **from_revision**: sha1 id of the revision used to generate - these metadata. - - **metadata** (str): associated metadata - - **tool** (dict): tool used to compute metadata - - **mappings** (List[str]): list of mappings used to translate - these metadata + Returns: + list of OriginIntrinsicMetadataRow """ ... @remote_api_endpoint("origin_intrinsic_metadata/search/by_producer") def origin_intrinsic_metadata_search_by_producer( - self, page_token="", limit=100, ids_only=False, mappings=None, tool_ids=None - ): + self, + page_token: str = "", + limit: int = 100, + ids_only: bool = False, + mappings: Optional[List[str]] = None, + tool_ids: Optional[List[int]] = None, + ) -> PagedResult[Union[str, OriginIntrinsicMetadataRow]]: """Returns the list of origins whose metadata contain all the terms. Args: page_token (str): Opaque token used for pagination. limit (int): The maximum number of results to return ids_only (bool): Determines whether only origin urls are returned or the content as well mappings (List[str]): Returns origins whose intrinsic metadata were generated using at least one of these mappings. Returns: - dict: dict with the following keys: - - **next_page_token** (str, optional): opaque token to be used as - `page_token` for retrieving the next page. If absent, there is - no more pages to gather. - - **origins** (list): list of origin url (str) if `ids_only=True` - else dictionaries with the following keys: - - - **id** (str): origin urls - - **from_revision**: sha1 id of the revision used to generate - these metadata. - - **metadata** (str): associated metadata - - **tool** (dict): tool used to compute metadata - - **mappings** (List[str]): list of mappings used to translate - these metadata + OriginIntrinsicMetadataRow objects """ ... @remote_api_endpoint("origin_intrinsic_metadata/stats") def origin_intrinsic_metadata_stats(self): """Returns counts of indexed metadata per origins, broken down into metadata types. Returns: dict: dictionary with keys: - total (int): total number of origins that were indexed (possibly yielding an empty metadata dictionary) - non_empty (int): total number of origins that we extracted a non-empty metadata dictionary from - per_mapping (dict): a dictionary with mapping names as keys and number of origins whose indexing used this mapping. Note that indexing a given origin may use 0, 1, or many mappings. """ ... @remote_api_endpoint("indexer_configuration/add") def indexer_configuration_add(self, tools): """Add new tools to the storage. Args: tools ([dict]): List of dictionary representing tool to insert in the db. Dictionary with the following keys: - **tool_name** (str): tool's name - **tool_version** (str): tool's version - **tool_configuration** (dict): tool's configuration (free form dict) Returns: List of dict inserted in the db (holding the id key as well). The order of the list is not guaranteed to match the order of the initial list. """ ... @remote_api_endpoint("indexer_configuration/data") def indexer_configuration_get(self, tool): """Retrieve tool information. Args: tool (dict): Dictionary representing a tool with the following keys: - **tool_name** (str): tool's name - **tool_version** (str): tool's version - **tool_configuration** (dict): tool's configuration (free form dict) Returns: The same dictionary with an `id` key, None otherwise. """ ... diff --git a/swh/indexer/storage/model.py b/swh/indexer/storage/model.py index c91f1b7..cc9c954 100644 --- a/swh/indexer/storage/model.py +++ b/swh/indexer/storage/model.py @@ -1,115 +1,122 @@ # Copyright (C) 2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information """Classes used internally by the in-memory idx-storage, and will be used for the interface of the idx-storage in the near future.""" from __future__ import annotations from typing import Any, Dict, List, Optional, Tuple, Type, TypeVar import attr from swh.model.model import Sha1Git, dictify TSelf = TypeVar("TSelf") @attr.s class BaseRow: UNIQUE_KEY_FIELDS: Tuple = ("id", "indexer_configuration_id") - id = attr.ib(type=Sha1Git) + id = attr.ib(type=Any) indexer_configuration_id = attr.ib(type=Optional[int], default=None, kw_only=True) tool = attr.ib(type=Optional[Dict], default=None, kw_only=True) def __attrs_post_init__(self): if self.indexer_configuration_id is None and self.tool is None: raise TypeError("Either indexer_configuration_id or tool must be not None.") if self.indexer_configuration_id is not None and self.tool is not None: raise TypeError( "indexer_configuration_id and tool are mutually exclusive; " "only one may be not None." ) def anonymize(self: TSelf) -> Optional[TSelf]: # Needed to implement swh.journal.writer.ValueProtocol return None def to_dict(self) -> Dict[str, Any]: """Wrapper of `attr.asdict` that can be overridden by subclasses that have special handling of some of the fields.""" d = dictify(attr.asdict(self, recurse=False)) if d["indexer_configuration_id"] is None: del d["indexer_configuration_id"] if d["tool"] is None: del d["tool"] return d @classmethod def from_dict(cls: Type[TSelf], d) -> TSelf: return cls(**d) # type: ignore def unique_key(self) -> Dict: if self.indexer_configuration_id is None: raise ValueError( "Can only call unique_key() on objects without " "indexer_configuration_id." ) return {key: getattr(self, key) for key in self.UNIQUE_KEY_FIELDS} @attr.s class ContentMimetypeRow(BaseRow): + id = attr.ib(type=Sha1Git) mimetype = attr.ib(type=str) encoding = attr.ib(type=str) @attr.s class ContentLanguageRow(BaseRow): + id = attr.ib(type=Sha1Git) lang = attr.ib(type=str) @attr.s class ContentCtagsRow(BaseRow): UNIQUE_KEY_FIELDS = ( "id", "indexer_configuration_id", "name", "kind", "line", "lang", ) + id = attr.ib(type=Sha1Git) name = attr.ib(type=str) kind = attr.ib(type=str) line = attr.ib(type=int) lang = attr.ib(type=str) @attr.s class ContentLicenseRow(BaseRow): UNIQUE_KEY_FIELDS = ("id", "indexer_configuration_id", "license") + id = attr.ib(type=Sha1Git) license = attr.ib(type=str) @attr.s class ContentMetadataRow(BaseRow): + id = attr.ib(type=Sha1Git) metadata = attr.ib(type=Dict[str, Any]) @attr.s class RevisionIntrinsicMetadataRow(BaseRow): + id = attr.ib(type=Sha1Git) metadata = attr.ib(type=Dict[str, Any]) mappings = attr.ib(type=List[str]) @attr.s class OriginIntrinsicMetadataRow(BaseRow): + id = attr.ib(type=str) metadata = attr.ib(type=Dict[str, Any]) from_revision = attr.ib(type=Sha1Git) mappings = attr.ib(type=List[str]) diff --git a/swh/indexer/tests/storage/test_storage.py b/swh/indexer/tests/storage/test_storage.py index 99d96ef..38d3ff4 100644 --- a/swh/indexer/tests/storage/test_storage.py +++ b/swh/indexer/tests/storage/test_storage.py @@ -1,1939 +1,1937 @@ # Copyright (C) 2015-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import inspect import math import threading -from typing import Any, Dict, List, Tuple, Union +from typing import Any, Dict, List, Tuple, Union, cast import attr import pytest from swh.indexer.storage.exc import DuplicateId, IndexerStorageArgumentException -from swh.indexer.storage.interface import IndexerStorageInterface +from swh.indexer.storage.interface import IndexerStorageInterface, PagedResult from swh.indexer.storage.model import ( BaseRow, ContentCtagsRow, ContentLanguageRow, ContentLicenseRow, ContentMetadataRow, ContentMimetypeRow, + OriginIntrinsicMetadataRow, RevisionIntrinsicMetadataRow, ) from swh.model.hashutil import hash_to_bytes def prepare_mimetypes_from_licenses( fossology_licenses: List[ContentLicenseRow], ) -> List[ContentMimetypeRow]: """Fossology license needs some consistent data in db to run. """ mimetypes = [] for c in fossology_licenses: mimetypes.append( ContentMimetypeRow( id=c.id, mimetype="text/plain", # for filtering on textual data to work encoding="utf-8", indexer_configuration_id=c.indexer_configuration_id, ) ) return mimetypes def endpoint_name(etype: str, ename: str) -> str: """Compute the storage's endpoint's name >>> endpoint_name('content_mimetype', 'add') 'content_mimetype_add' >>> endpoint_name('content_fosso_license', 'delete') 'content_fosso_license_delete' """ return f"{etype}_{ename}" def endpoint(storage, etype: str, ename: str): return getattr(storage, endpoint_name(etype, ename)) def expected_summary(count: int, etype: str, ename: str = "add") -> Dict[str, int]: """Compute the expected summary The key is determine according to etype and ename >>> expected_summary(10, 'content_mimetype', 'add') {'content_mimetype:add': 10} >>> expected_summary(9, 'origin_intrinsic_metadata', 'delete') {'origin_intrinsic_metadata:del': 9} """ pattern = ename[0:3] key = endpoint_name(etype, ename).replace(f"_{ename}", f":{pattern}") return {key: count} def test_check_config(swh_indexer_storage) -> None: assert swh_indexer_storage.check_config(check_write=True) assert swh_indexer_storage.check_config(check_write=False) def test_types(swh_indexer_storage) -> None: """Checks all methods of StorageInterface are implemented by this backend, and that they have the same signature.""" # Create an instance of the protocol (which cannot be instantiated # directly, so this creates a subclass, then instantiates it) interface = type("_", (IndexerStorageInterface,), {})() assert "content_mimetype_add" in dir(interface) missing_methods = [] for meth_name in dir(interface): if meth_name.startswith("_"): continue interface_meth = getattr(interface, meth_name) try: concrete_meth = getattr(swh_indexer_storage, meth_name) except AttributeError: missing_methods.append(meth_name) continue expected_signature = inspect.signature(interface_meth) actual_signature = inspect.signature(concrete_meth) assert expected_signature == actual_signature, meth_name assert missing_methods == [] class StorageETypeTester: """Base class for testing a series of common behaviour between a bunch of endpoint types supported by an IndexerStorage. This is supposed to be inherited with the following class attributes: - endpoint_type - tool_name - example_data See below for example usage. """ endpoint_type: str tool_name: str example_data: List[Dict] # For old endpoints (which use dicts), this is just the identity function. # For new endpoints, it returns a BaseRow instance. @staticmethod def row_from_dict(d: Dict) -> Union[Dict, BaseRow]: return d # Inverse function of row_from_dict # TODO: remove this once all endpoints are migrated to rows @staticmethod def dict_from_row(r: Union[Dict, BaseRow]) -> Dict: assert isinstance(r, Dict) return r def test_missing( self, swh_indexer_storage_with_data: Tuple[IndexerStorageInterface, Any] ) -> None: storage, data = swh_indexer_storage_with_data etype = self.endpoint_type tool_id = data.tools[self.tool_name]["id"] # given 2 (hopefully) unknown objects query = [ {"id": data.sha1_1, "indexer_configuration_id": tool_id,}, {"id": data.sha1_2, "indexer_configuration_id": tool_id,}, ] # we expect these are both returned by the xxx_missing endpoint actual_missing = endpoint(storage, etype, "missing")(query) assert list(actual_missing) == [ data.sha1_1, data.sha1_2, ] # now, when we add one of them summary = endpoint(storage, etype, "add")( [ self.row_from_dict( { "id": data.sha1_2, **self.example_data[0], "indexer_configuration_id": tool_id, } ) ] ) assert summary == expected_summary(1, etype) # we expect only the other one returned actual_missing = endpoint(storage, etype, "missing")(query) assert list(actual_missing) == [data.sha1_1] def test_add__drop_duplicate( self, swh_indexer_storage_with_data: Tuple[IndexerStorageInterface, Any] ) -> None: storage, data = swh_indexer_storage_with_data etype = self.endpoint_type tool_id = data.tools[self.tool_name]["id"] # add the first object data_v1 = { "id": data.sha1_2, **self.example_data[0], "indexer_configuration_id": tool_id, } summary = endpoint(storage, etype, "add")([self.row_from_dict(data_v1)]) assert summary == expected_summary(1, etype) # should be able to retrieve it actual_data = list(endpoint(storage, etype, "get")([data.sha1_2])) expected_data_v1 = [ self.row_from_dict( { "id": data.sha1_2, **self.example_data[0], "tool": data.tools[self.tool_name], } ) ] assert actual_data == expected_data_v1 # now if we add a modified version of the same object (same id) data_v2 = data_v1.copy() data_v2.update(self.example_data[1]) summary2 = endpoint(storage, etype, "add")([self.row_from_dict(data_v2)]) assert summary2 == expected_summary(0, etype) # not added # we expect to retrieve the original data, not the modified one actual_data = list(endpoint(storage, etype, "get")([data.sha1_2])) assert actual_data == expected_data_v1 def test_add__update_in_place_duplicate( self, swh_indexer_storage_with_data: Tuple[IndexerStorageInterface, Any] ) -> None: storage, data = swh_indexer_storage_with_data etype = self.endpoint_type tool = data.tools[self.tool_name] data_v1 = { "id": data.sha1_2, **self.example_data[0], "indexer_configuration_id": tool["id"], } # given summary = endpoint(storage, etype, "add")([self.row_from_dict(data_v1)]) assert summary == expected_summary(1, etype) # not added # when actual_data = list(endpoint(storage, etype, "get")([data.sha1_2])) expected_data_v1 = [ self.row_from_dict( {"id": data.sha1_2, **self.example_data[0], "tool": tool} ) ] # then assert actual_data == expected_data_v1 # given data_v2 = data_v1.copy() data_v2.update(self.example_data[1]) endpoint(storage, etype, "add")( [self.row_from_dict(data_v2)], conflict_update=True ) assert summary == expected_summary(1, etype) # modified so counted actual_data = list(endpoint(storage, etype, "get")([data.sha1_2])) expected_data_v2 = [ self.row_from_dict( {"id": data.sha1_2, **self.example_data[1], "tool": tool,} ) ] # data did change as the v2 was used to overwrite v1 assert actual_data == expected_data_v2 def test_add__update_in_place_deadlock( self, swh_indexer_storage_with_data: Tuple[IndexerStorageInterface, Any] ) -> None: storage, data = swh_indexer_storage_with_data etype = self.endpoint_type tool = data.tools[self.tool_name] hashes = [ hash_to_bytes("34973274ccef6ab4dfaaf86599792fa9c3fe4{:03d}".format(i)) for i in range(1000) ] data_v1 = [ self.row_from_dict( { "id": hash_, **self.example_data[0], "indexer_configuration_id": tool["id"], } ) for hash_ in hashes ] data_v2 = [ self.row_from_dict( { "id": hash_, **self.example_data[1], "indexer_configuration_id": tool["id"], } ) for hash_ in hashes ] # Remove one item from each, so that both queries have to succeed for # all items to be in the DB. data_v2a = data_v2[1:] data_v2b = list(reversed(data_v2[0:-1])) # given endpoint(storage, etype, "add")(data_v1) # when actual_data = list(endpoint(storage, etype, "get")(hashes)) expected_data_v1 = [ self.row_from_dict({"id": hash_, **self.example_data[0], "tool": tool}) for hash_ in hashes ] # then assert actual_data == expected_data_v1 # given def f1() -> None: endpoint(storage, etype, "add")(data_v2a, conflict_update=True) def f2() -> None: endpoint(storage, etype, "add")(data_v2b, conflict_update=True) t1 = threading.Thread(target=f1) t2 = threading.Thread(target=f2) t2.start() t1.start() t1.join() t2.join() actual_data = sorted( map(self.dict_from_row, endpoint(storage, etype, "get")(hashes)), key=lambda x: x["id"], ) expected_data_v2 = [ {"id": hash_, **self.example_data[1], "tool": tool} for hash_ in hashes ] assert actual_data == expected_data_v2 def test_add__duplicate_twice( self, swh_indexer_storage_with_data: Tuple[IndexerStorageInterface, Any] ) -> None: storage, data = swh_indexer_storage_with_data etype = self.endpoint_type tool = data.tools[self.tool_name] data_rev1 = self.row_from_dict( { "id": data.revision_id_2, **self.example_data[0], "indexer_configuration_id": tool["id"], } ) data_rev2 = self.row_from_dict( { "id": data.revision_id_2, **self.example_data[1], "indexer_configuration_id": tool["id"], } ) # when summary = endpoint(storage, etype, "add")([data_rev1]) assert summary == expected_summary(1, etype) with pytest.raises(DuplicateId): endpoint(storage, etype, "add")( [data_rev2, data_rev2], conflict_update=True ) # then actual_data = list( endpoint(storage, etype, "get")([data.revision_id_2, data.revision_id_1]) ) expected_data = [ self.row_from_dict( {"id": data.revision_id_2, **self.example_data[0], "tool": tool} ) ] assert actual_data == expected_data def test_get( self, swh_indexer_storage_with_data: Tuple[IndexerStorageInterface, Any] ) -> None: storage, data = swh_indexer_storage_with_data etype = self.endpoint_type tool = data.tools[self.tool_name] query = [data.sha1_2, data.sha1_1] data1 = self.row_from_dict( { "id": data.sha1_2, **self.example_data[0], "indexer_configuration_id": tool["id"], } ) # when summary = endpoint(storage, etype, "add")([data1]) assert summary == expected_summary(1, etype) # then actual_data = list(endpoint(storage, etype, "get")(query)) # then expected_data = [ self.row_from_dict( {"id": data.sha1_2, **self.example_data[0], "tool": tool} ) ] assert actual_data == expected_data class TestIndexerStorageContentMimetypes(StorageETypeTester): """Test Indexer Storage content_mimetype related methods """ endpoint_type = "content_mimetype" tool_name = "file" example_data = [ {"mimetype": "text/plain", "encoding": "utf-8",}, {"mimetype": "text/html", "encoding": "us-ascii",}, ] row_from_dict = ContentMimetypeRow.from_dict dict_from_row = staticmethod(lambda x: x.to_dict()) # type: ignore def test_generate_content_mimetype_get_partition_failure( self, swh_indexer_storage: IndexerStorageInterface ) -> None: """get_partition call with wrong limit input should fail""" storage = swh_indexer_storage indexer_configuration_id = 42 with pytest.raises( IndexerStorageArgumentException, match="limit should not be None" ): storage.content_mimetype_get_partition( indexer_configuration_id, 0, 3, limit=None # type: ignore ) def test_generate_content_mimetype_get_partition_no_limit( self, swh_indexer_storage_with_data: Tuple[IndexerStorageInterface, Any] ) -> None: """get_partition should return result""" storage, data = swh_indexer_storage_with_data mimetypes = data.mimetypes expected_ids = set([c.id for c in mimetypes]) indexer_configuration_id = mimetypes[0].indexer_configuration_id assert len(mimetypes) == 16 nb_partitions = 16 actual_ids = [] for partition_id in range(nb_partitions): actual_result = storage.content_mimetype_get_partition( indexer_configuration_id, partition_id, nb_partitions ) assert actual_result.next_page_token is None actual_ids.extend(actual_result.results) assert len(actual_ids) == len(expected_ids) for actual_id in actual_ids: assert actual_id in expected_ids def test_generate_content_mimetype_get_partition_full( self, swh_indexer_storage_with_data: Tuple[IndexerStorageInterface, Any] ) -> None: """get_partition for a single partition should return available ids """ storage, data = swh_indexer_storage_with_data mimetypes = data.mimetypes expected_ids = set([c.id for c in mimetypes]) indexer_configuration_id = mimetypes[0].indexer_configuration_id actual_result = storage.content_mimetype_get_partition( indexer_configuration_id, 0, 1 ) assert actual_result.next_page_token is None actual_ids = actual_result.results assert len(actual_ids) == len(expected_ids) for actual_id in actual_ids: assert actual_id in expected_ids def test_generate_content_mimetype_get_partition_empty( self, swh_indexer_storage_with_data: Tuple[IndexerStorageInterface, Any] ) -> None: """get_partition when at least one of the partitions is empty""" storage, data = swh_indexer_storage_with_data mimetypes = data.mimetypes expected_ids = set([c.id for c in mimetypes]) indexer_configuration_id = mimetypes[0].indexer_configuration_id # nb_partitions = smallest power of 2 such that at least one of # the partitions is empty nb_mimetypes = len(mimetypes) nb_partitions = 1 << math.floor(math.log2(nb_mimetypes) + 1) seen_ids = [] for partition_id in range(nb_partitions): actual_result = storage.content_mimetype_get_partition( indexer_configuration_id, partition_id, nb_partitions, limit=nb_mimetypes + 1, ) for actual_id in actual_result.results: seen_ids.append(actual_id) # Limit is higher than the max number of results assert actual_result.next_page_token is None assert set(seen_ids) == expected_ids def test_generate_content_mimetype_get_partition_with_pagination( self, swh_indexer_storage_with_data: Tuple[IndexerStorageInterface, Any] ) -> None: """get_partition should return ids provided with pagination """ storage, data = swh_indexer_storage_with_data mimetypes = data.mimetypes expected_ids = set([c.id for c in mimetypes]) indexer_configuration_id = mimetypes[0].indexer_configuration_id nb_partitions = 4 actual_ids = [] for partition_id in range(nb_partitions): next_page_token = None while True: actual_result = storage.content_mimetype_get_partition( indexer_configuration_id, partition_id, nb_partitions, limit=2, page_token=next_page_token, ) actual_ids.extend(actual_result.results) next_page_token = actual_result.next_page_token if next_page_token is None: break assert len(set(actual_ids)) == len(set(expected_ids)) for actual_id in actual_ids: assert actual_id in expected_ids class TestIndexerStorageContentLanguage(StorageETypeTester): """Test Indexer Storage content_language related methods """ endpoint_type = "content_language" tool_name = "pygments" example_data = [ {"lang": "haskell",}, {"lang": "common-lisp",}, ] row_from_dict = ContentLanguageRow.from_dict dict_from_row = staticmethod(lambda x: x.to_dict()) # type: ignore class TestIndexerStorageContentCTags(StorageETypeTester): """Test Indexer Storage content_ctags related methods """ endpoint_type = "content_ctags" tool_name = "universal-ctags" example_data = [ {"name": "done", "kind": "variable", "line": 119, "lang": "OCaml",}, {"name": "done", "kind": "variable", "line": 100, "lang": "Python",}, {"name": "main", "kind": "function", "line": 119, "lang": "Python",}, ] row_from_dict = ContentCtagsRow.from_dict dict_from_row = staticmethod(lambda x: x.to_dict()) # type: ignore # the following tests are disabled because CTAGS behaves differently @pytest.mark.skip def test_add__drop_duplicate(self): pass @pytest.mark.skip def test_add__update_in_place_duplicate(self): pass @pytest.mark.skip def test_add__update_in_place_deadlock(self): pass @pytest.mark.skip def test_add__duplicate_twice(self): pass @pytest.mark.skip def test_get(self): pass def test_content_ctags_search( self, swh_indexer_storage_with_data: Tuple[IndexerStorageInterface, Any] ) -> None: storage, data = swh_indexer_storage_with_data # 1. given tool = data.tools["universal-ctags"] tool_id = tool["id"] ctags1 = [ ContentCtagsRow( id=data.sha1_1, indexer_configuration_id=tool_id, **kwargs, # type: ignore ) for kwargs in [ {"name": "hello", "kind": "function", "line": 133, "lang": "Python",}, {"name": "counter", "kind": "variable", "line": 119, "lang": "Python",}, {"name": "hello", "kind": "variable", "line": 210, "lang": "Python",}, ] ] ctags1_with_tool = [ attr.evolve(ctag, indexer_configuration_id=None, tool=tool) for ctag in ctags1 ] ctags2 = [ ContentCtagsRow( id=data.sha1_2, indexer_configuration_id=tool_id, **kwargs, # type: ignore ) for kwargs in [ {"name": "hello", "kind": "variable", "line": 100, "lang": "C",}, {"name": "result", "kind": "variable", "line": 120, "lang": "C",}, ] ] ctags2_with_tool = [ attr.evolve(ctag, indexer_configuration_id=None, tool=tool) for ctag in ctags2 ] storage.content_ctags_add(ctags1 + ctags2) # 1. when actual_ctags = list(storage.content_ctags_search("hello", limit=1)) # 1. then assert actual_ctags == [ctags1_with_tool[0]] # 2. when actual_ctags = list( storage.content_ctags_search("hello", limit=1, last_sha1=data.sha1_1) ) # 2. then assert actual_ctags == [ctags2_with_tool[0]] # 3. when actual_ctags = list(storage.content_ctags_search("hello")) # 3. then assert actual_ctags == [ ctags1_with_tool[0], ctags1_with_tool[2], ctags2_with_tool[0], ] # 4. when actual_ctags = list(storage.content_ctags_search("counter")) # then assert actual_ctags == [ctags1_with_tool[1]] # 5. when actual_ctags = list(storage.content_ctags_search("result", limit=1)) # then assert actual_ctags == [ctags2_with_tool[1]] def test_content_ctags_search_no_result( self, swh_indexer_storage: IndexerStorageInterface ) -> None: storage = swh_indexer_storage actual_ctags = list(storage.content_ctags_search("counter")) assert not actual_ctags def test_content_ctags_add__add_new_ctags_added( self, swh_indexer_storage_with_data: Tuple[IndexerStorageInterface, Any] ) -> None: storage, data = swh_indexer_storage_with_data # given tool = data.tools["universal-ctags"] tool_id = tool["id"] ctag1 = ContentCtagsRow( id=data.sha1_2, indexer_configuration_id=tool_id, name="done", kind="variable", line=100, lang="Scheme", ) ctag1_with_tool = attr.evolve(ctag1, indexer_configuration_id=None, tool=tool) # given storage.content_ctags_add([ctag1]) storage.content_ctags_add([ctag1]) # conflict does nothing # when actual_ctags = list(storage.content_ctags_get([data.sha1_2])) # then assert actual_ctags == [ctag1_with_tool] # given ctag2 = ContentCtagsRow( id=data.sha1_2, indexer_configuration_id=tool_id, name="defn", kind="function", line=120, lang="Scheme", ) ctag2_with_tool = attr.evolve(ctag2, indexer_configuration_id=None, tool=tool) storage.content_ctags_add([ctag2]) actual_ctags = list(storage.content_ctags_get([data.sha1_2])) assert actual_ctags == [ctag1_with_tool, ctag2_with_tool] def test_content_ctags_add__update_in_place( self, swh_indexer_storage_with_data: Tuple[IndexerStorageInterface, Any] ) -> None: storage, data = swh_indexer_storage_with_data # given tool = data.tools["universal-ctags"] tool_id = tool["id"] ctag1 = ContentCtagsRow( id=data.sha1_2, indexer_configuration_id=tool_id, name="done", kind="variable", line=100, lang="Scheme", ) ctag1_with_tool = attr.evolve(ctag1, indexer_configuration_id=None, tool=tool) # given storage.content_ctags_add([ctag1]) # when actual_ctags = list(storage.content_ctags_get([data.sha1_2])) # then assert actual_ctags == [ctag1_with_tool] # given ctag2 = ContentCtagsRow( id=data.sha1_2, indexer_configuration_id=tool_id, name="defn", kind="function", line=120, lang="Scheme", ) ctag2_with_tool = attr.evolve(ctag2, indexer_configuration_id=None, tool=tool) storage.content_ctags_add([ctag1, ctag2], conflict_update=True) actual_ctags = list(storage.content_ctags_get([data.sha1_2])) assert actual_ctags == [ctag1_with_tool, ctag2_with_tool] def test_add_empty( self, swh_indexer_storage_with_data: Tuple[IndexerStorageInterface, Any] ) -> None: (storage, data) = swh_indexer_storage_with_data etype = self.endpoint_type summary = endpoint(storage, etype, "add")([]) assert summary == {"content_ctags:add": 0} actual_ctags = list(endpoint(storage, etype, "get")([data.sha1_2])) assert actual_ctags == [] def test_get_unknown( self, swh_indexer_storage_with_data: Tuple[IndexerStorageInterface, Any] ) -> None: (storage, data) = swh_indexer_storage_with_data etype = self.endpoint_type actual_ctags = list(endpoint(storage, etype, "get")([data.sha1_2])) assert actual_ctags == [] class TestIndexerStorageContentMetadata(StorageETypeTester): """Test Indexer Storage content_metadata related methods """ tool_name = "swh-metadata-detector" endpoint_type = "content_metadata" example_data = [ { "metadata": { "other": {}, "codeRepository": { "type": "git", "url": "https://github.com/moranegg/metadata_test", }, "description": "Simple package.json test for indexer", "name": "test_metadata", "version": "0.0.1", }, }, {"metadata": {"other": {}, "name": "test_metadata", "version": "0.0.1"},}, ] row_from_dict = ContentMetadataRow.from_dict dict_from_row = staticmethod(lambda x: x.to_dict()) # type: ignore class TestIndexerStorageRevisionIntrinsicMetadata(StorageETypeTester): """Test Indexer Storage revision_intrinsic_metadata related methods """ tool_name = "swh-metadata-detector" endpoint_type = "revision_intrinsic_metadata" example_data = [ { "metadata": { "other": {}, "codeRepository": { "type": "git", "url": "https://github.com/moranegg/metadata_test", }, "description": "Simple package.json test for indexer", "name": "test_metadata", "version": "0.0.1", }, "mappings": ["mapping1"], }, { "metadata": {"other": {}, "name": "test_metadata", "version": "0.0.1"}, "mappings": ["mapping2"], }, ] row_from_dict = RevisionIntrinsicMetadataRow.from_dict dict_from_row = staticmethod(lambda x: x.to_dict()) # type: ignore def test_revision_intrinsic_metadata_delete( self, swh_indexer_storage_with_data: Tuple[IndexerStorageInterface, Any] ) -> None: storage, data = swh_indexer_storage_with_data etype = self.endpoint_type tool = data.tools[self.tool_name] query = [data.sha1_2, data.sha1_1] data1 = RevisionIntrinsicMetadataRow( id=data.sha1_2, indexer_configuration_id=tool["id"], **self.example_data[0], # type: ignore ) # when summary = endpoint(storage, etype, "add")([data1]) assert summary == expected_summary(1, etype) summary2 = endpoint(storage, etype, "delete")( [{"id": data.sha1_2, "indexer_configuration_id": tool["id"],}] ) assert summary2 == expected_summary(1, etype, "del") # then actual_data = list(endpoint(storage, etype, "get")(query)) # then assert not actual_data def test_revision_intrinsic_metadata_delete_nonexisting( self, swh_indexer_storage_with_data: Tuple[IndexerStorageInterface, Any] ) -> None: storage, data = swh_indexer_storage_with_data etype = self.endpoint_type tool = data.tools[self.tool_name] endpoint(storage, etype, "delete")( [{"id": data.sha1_2, "indexer_configuration_id": tool["id"],}] ) class TestIndexerStorageContentFossologyLicense: endpoint_type = "content_fossology_license" tool_name = "nomos" row_from_dict = ContentLicenseRow.from_dict dict_from_row = staticmethod(lambda x: x.to_dict()) def test_content_fossology_license_add__new_license_added( self, swh_indexer_storage_with_data: Tuple[IndexerStorageInterface, Any] ) -> None: storage, data = swh_indexer_storage_with_data # given tool = data.tools["nomos"] tool_id = tool["id"] license1 = ContentLicenseRow( id=data.sha1_1, license="Apache-2.0", indexer_configuration_id=tool_id, ) # given storage.content_fossology_license_add([license1]) # conflict does nothing storage.content_fossology_license_add([license1]) # when actual_licenses = list(storage.content_fossology_license_get([data.sha1_1])) # then expected_licenses = [ ContentLicenseRow(id=data.sha1_1, license="Apache-2.0", tool=tool,) ] assert actual_licenses == expected_licenses # given license2 = ContentLicenseRow( id=data.sha1_1, license="BSD-2-Clause", indexer_configuration_id=tool_id, ) storage.content_fossology_license_add([license2]) actual_licenses = list(storage.content_fossology_license_get([data.sha1_1])) expected_licenses.append( ContentLicenseRow(id=data.sha1_1, license="BSD-2-Clause", tool=tool,) ) # first license was not removed when the second one was added assert sorted(actual_licenses) == sorted(expected_licenses) def test_generate_content_fossology_license_get_partition_failure( self, swh_indexer_storage_with_data: Tuple[IndexerStorageInterface, Any] ) -> None: """get_partition call with wrong limit input should fail""" storage, data = swh_indexer_storage_with_data indexer_configuration_id = 42 with pytest.raises( IndexerStorageArgumentException, match="limit should not be None" ): storage.content_fossology_license_get_partition( indexer_configuration_id, 0, 3, limit=None, # type: ignore ) def test_generate_content_fossology_license_get_partition_no_limit( self, swh_indexer_storage_with_data: Tuple[IndexerStorageInterface, Any] ) -> None: """get_partition should return results""" storage, data = swh_indexer_storage_with_data # craft some consistent mimetypes fossology_licenses = data.fossology_licenses mimetypes = prepare_mimetypes_from_licenses(fossology_licenses) indexer_configuration_id = fossology_licenses[0].indexer_configuration_id storage.content_mimetype_add(mimetypes, conflict_update=True) # add fossology_licenses to storage storage.content_fossology_license_add(fossology_licenses) # All ids from the db expected_ids = set([c.id for c in fossology_licenses]) assert len(fossology_licenses) == 10 assert len(mimetypes) == 10 nb_partitions = 4 actual_ids = [] for partition_id in range(nb_partitions): actual_result = storage.content_fossology_license_get_partition( indexer_configuration_id, partition_id, nb_partitions ) assert actual_result.next_page_token is None actual_ids.extend(actual_result.results) assert len(set(actual_ids)) == len(expected_ids) for actual_id in actual_ids: assert actual_id in expected_ids def test_generate_content_fossology_license_get_partition_full( self, swh_indexer_storage_with_data: Tuple[IndexerStorageInterface, Any] ) -> None: """get_partition for a single partition should return available ids """ storage, data = swh_indexer_storage_with_data # craft some consistent mimetypes fossology_licenses = data.fossology_licenses mimetypes = prepare_mimetypes_from_licenses(fossology_licenses) indexer_configuration_id = fossology_licenses[0].indexer_configuration_id storage.content_mimetype_add(mimetypes, conflict_update=True) # add fossology_licenses to storage storage.content_fossology_license_add(fossology_licenses) # All ids from the db expected_ids = set([c.id for c in fossology_licenses]) actual_result = storage.content_fossology_license_get_partition( indexer_configuration_id, 0, 1 ) assert actual_result.next_page_token is None actual_ids = actual_result.results assert len(set(actual_ids)) == len(expected_ids) for actual_id in actual_ids: assert actual_id in expected_ids def test_generate_content_fossology_license_get_partition_empty( self, swh_indexer_storage_with_data: Tuple[IndexerStorageInterface, Any] ) -> None: """get_partition when at least one of the partitions is empty""" storage, data = swh_indexer_storage_with_data # craft some consistent mimetypes fossology_licenses = data.fossology_licenses mimetypes = prepare_mimetypes_from_licenses(fossology_licenses) indexer_configuration_id = fossology_licenses[0].indexer_configuration_id storage.content_mimetype_add(mimetypes, conflict_update=True) # add fossology_licenses to storage storage.content_fossology_license_add(fossology_licenses) # All ids from the db expected_ids = set([c.id for c in fossology_licenses]) # nb_partitions = smallest power of 2 such that at least one of # the partitions is empty nb_licenses = len(fossology_licenses) nb_partitions = 1 << math.floor(math.log2(nb_licenses) + 1) seen_ids = [] for partition_id in range(nb_partitions): actual_result = storage.content_fossology_license_get_partition( indexer_configuration_id, partition_id, nb_partitions, limit=nb_licenses + 1, ) for actual_id in actual_result.results: seen_ids.append(actual_id) # Limit is higher than the max number of results assert actual_result.next_page_token is None assert set(seen_ids) == expected_ids def test_generate_content_fossology_license_get_partition_with_pagination( self, swh_indexer_storage_with_data: Tuple[IndexerStorageInterface, Any] ) -> None: """get_partition should return ids provided with paginationv """ storage, data = swh_indexer_storage_with_data # craft some consistent mimetypes fossology_licenses = data.fossology_licenses mimetypes = prepare_mimetypes_from_licenses(fossology_licenses) indexer_configuration_id = fossology_licenses[0].indexer_configuration_id storage.content_mimetype_add(mimetypes, conflict_update=True) # add fossology_licenses to storage storage.content_fossology_license_add(fossology_licenses) # All ids from the db expected_ids = [c.id for c in fossology_licenses] nb_partitions = 4 actual_ids = [] for partition_id in range(nb_partitions): next_page_token = None while True: actual_result = storage.content_fossology_license_get_partition( indexer_configuration_id, partition_id, nb_partitions, limit=2, page_token=next_page_token, ) actual_ids.extend(actual_result.results) next_page_token = actual_result.next_page_token if next_page_token is None: break assert len(set(actual_ids)) == len(set(expected_ids)) for actual_id in actual_ids: assert actual_id in expected_ids def test_add_empty( self, swh_indexer_storage_with_data: Tuple[IndexerStorageInterface, Any] ) -> None: (storage, data) = swh_indexer_storage_with_data etype = self.endpoint_type summary = endpoint(storage, etype, "add")([]) assert summary == {"content_fossology_license:add": 0} actual_license = list(endpoint(storage, etype, "get")([data.sha1_2])) assert actual_license == [] def test_get_unknown( self, swh_indexer_storage_with_data: Tuple[IndexerStorageInterface, Any] ) -> None: (storage, data) = swh_indexer_storage_with_data etype = self.endpoint_type actual_license = list(endpoint(storage, etype, "get")([data.sha1_2])) assert actual_license == [] class TestIndexerStorageOriginIntrinsicMetadata: def test_origin_intrinsic_metadata_get( self, swh_indexer_storage_with_data: Tuple[IndexerStorageInterface, Any] ) -> None: storage, data = swh_indexer_storage_with_data # given tool_id = data.tools["swh-metadata-detector"]["id"] metadata = { "version": None, "name": None, } metadata_rev = RevisionIntrinsicMetadataRow( id=data.revision_id_2, metadata=metadata, mappings=["mapping1"], indexer_configuration_id=tool_id, ) - metadata_origin = { - "id": data.origin_url_1, - "metadata": metadata, - "indexer_configuration_id": tool_id, - "mappings": ["mapping1"], - "from_revision": data.revision_id_2, - } + metadata_origin = OriginIntrinsicMetadataRow( + id=data.origin_url_1, + metadata=metadata, + indexer_configuration_id=tool_id, + mappings=["mapping1"], + from_revision=data.revision_id_2, + ) # when storage.revision_intrinsic_metadata_add([metadata_rev]) storage.origin_intrinsic_metadata_add([metadata_origin]) # then actual_metadata = list( storage.origin_intrinsic_metadata_get([data.origin_url_1, "no://where"]) ) expected_metadata = [ - { - "id": data.origin_url_1, - "metadata": metadata, - "tool": data.tools["swh-metadata-detector"], - "from_revision": data.revision_id_2, - "mappings": ["mapping1"], - } + OriginIntrinsicMetadataRow( + id=data.origin_url_1, + metadata=metadata, + tool=data.tools["swh-metadata-detector"], + from_revision=data.revision_id_2, + mappings=["mapping1"], + ) ] assert actual_metadata == expected_metadata def test_origin_intrinsic_metadata_delete( self, swh_indexer_storage_with_data: Tuple[IndexerStorageInterface, Any] ) -> None: storage, data = swh_indexer_storage_with_data # given tool_id = data.tools["swh-metadata-detector"]["id"] metadata = { "version": None, "name": None, } metadata_rev = RevisionIntrinsicMetadataRow( id=data.revision_id_2, indexer_configuration_id=tool_id, metadata=metadata, mappings=["mapping1"], ) - metadata_origin = { - "id": data.origin_url_1, - "metadata": metadata, - "indexer_configuration_id": tool_id, - "mappings": ["mapping1"], - "from_revision": data.revision_id_2, - } - metadata_origin2 = metadata_origin.copy() - metadata_origin2["id"] = data.origin_url_2 + metadata_origin = OriginIntrinsicMetadataRow( + id=data.origin_url_1, + metadata=metadata, + indexer_configuration_id=tool_id, + mappings=["mapping1"], + from_revision=data.revision_id_2, + ) + metadata_origin2 = attr.evolve(metadata_origin, id=data.origin_url_2) # when storage.revision_intrinsic_metadata_add([metadata_rev]) storage.origin_intrinsic_metadata_add([metadata_origin, metadata_origin2]) storage.origin_intrinsic_metadata_delete( [{"id": data.origin_url_1, "indexer_configuration_id": tool_id}] ) # then actual_metadata = list( storage.origin_intrinsic_metadata_get( [data.origin_url_1, data.origin_url_2, "no://where"] ) ) - for item in actual_metadata: - item["indexer_configuration_id"] = item.pop("tool")["id"] - assert actual_metadata == [metadata_origin2] + assert [ + attr.evolve(m, indexer_configuration_id=cast(Dict, m.tool)["id"], tool=None) + for m in actual_metadata + ] == [metadata_origin2] def test_origin_intrinsic_metadata_delete_nonexisting( self, swh_indexer_storage_with_data: Tuple[IndexerStorageInterface, Any] ) -> None: storage, data = swh_indexer_storage_with_data tool_id = data.tools["swh-metadata-detector"]["id"] storage.origin_intrinsic_metadata_delete( [{"id": data.origin_url_1, "indexer_configuration_id": tool_id}] ) def test_origin_intrinsic_metadata_add_drop_duplicate( self, swh_indexer_storage_with_data: Tuple[IndexerStorageInterface, Any] ) -> None: storage, data = swh_indexer_storage_with_data # given tool_id = data.tools["swh-metadata-detector"]["id"] metadata_v1: Dict[str, Any] = { "version": None, "name": None, } metadata_rev_v1 = RevisionIntrinsicMetadataRow( id=data.revision_id_1, metadata=metadata_v1.copy(), mappings=[], indexer_configuration_id=tool_id, ) - metadata_origin_v1 = { - "id": data.origin_url_1, - "metadata": metadata_v1.copy(), - "indexer_configuration_id": tool_id, - "mappings": [], - "from_revision": data.revision_id_1, - } + metadata_origin_v1 = OriginIntrinsicMetadataRow( + id=data.origin_url_1, + metadata=metadata_v1.copy(), + indexer_configuration_id=tool_id, + mappings=[], + from_revision=data.revision_id_1, + ) # given storage.revision_intrinsic_metadata_add([metadata_rev_v1]) storage.origin_intrinsic_metadata_add([metadata_origin_v1]) # when actual_metadata = list( storage.origin_intrinsic_metadata_get([data.origin_url_1, "no://where"]) ) expected_metadata_v1 = [ - { - "id": data.origin_url_1, - "metadata": metadata_v1, - "tool": data.tools["swh-metadata-detector"], - "from_revision": data.revision_id_1, - "mappings": [], - } + OriginIntrinsicMetadataRow( + id=data.origin_url_1, + metadata=metadata_v1, + tool=data.tools["swh-metadata-detector"], + from_revision=data.revision_id_1, + mappings=[], + ) ] assert actual_metadata == expected_metadata_v1 # given metadata_v2 = metadata_v1.copy() metadata_v2.update( {"name": "test_metadata", "author": "MG",} ) metadata_rev_v2 = attr.evolve(metadata_rev_v1, metadata=metadata_v2) - metadata_origin_v2 = metadata_origin_v1.copy() - metadata_origin_v2["metadata"] = metadata_v2 + metadata_origin_v2 = attr.evolve(metadata_origin_v1, metadata=metadata_v2) storage.revision_intrinsic_metadata_add([metadata_rev_v2]) storage.origin_intrinsic_metadata_add([metadata_origin_v2]) # then actual_metadata = list( storage.origin_intrinsic_metadata_get([data.origin_url_1]) ) # metadata did not change as the v2 was dropped. assert actual_metadata == expected_metadata_v1 def test_origin_intrinsic_metadata_add_update_in_place_duplicate( self, swh_indexer_storage_with_data: Tuple[IndexerStorageInterface, Any] ) -> None: storage, data = swh_indexer_storage_with_data # given tool_id = data.tools["swh-metadata-detector"]["id"] metadata_v1: Dict[str, Any] = { "version": None, "name": None, } metadata_rev_v1 = RevisionIntrinsicMetadataRow( id=data.revision_id_2, metadata=metadata_v1, mappings=[], indexer_configuration_id=tool_id, ) - metadata_origin_v1 = { - "id": data.origin_url_1, - "metadata": metadata_v1.copy(), - "indexer_configuration_id": tool_id, - "mappings": [], - "from_revision": data.revision_id_2, - } + metadata_origin_v1 = OriginIntrinsicMetadataRow( + id=data.origin_url_1, + metadata=metadata_v1.copy(), + indexer_configuration_id=tool_id, + mappings=[], + from_revision=data.revision_id_2, + ) # given storage.revision_intrinsic_metadata_add([metadata_rev_v1]) storage.origin_intrinsic_metadata_add([metadata_origin_v1]) # when actual_metadata = list( storage.origin_intrinsic_metadata_get([data.origin_url_1]) ) # then expected_metadata_v1 = [ - { - "id": data.origin_url_1, - "metadata": metadata_v1, - "tool": data.tools["swh-metadata-detector"], - "from_revision": data.revision_id_2, - "mappings": [], - } + OriginIntrinsicMetadataRow( + id=data.origin_url_1, + metadata=metadata_v1, + tool=data.tools["swh-metadata-detector"], + from_revision=data.revision_id_2, + mappings=[], + ) ] assert actual_metadata == expected_metadata_v1 # given metadata_v2 = metadata_v1.copy() metadata_v2.update( {"name": "test_update_duplicated_metadata", "author": "MG",} ) metadata_rev_v2 = attr.evolve(metadata_rev_v1, metadata=metadata_v2) - metadata_origin_v2 = metadata_origin_v1.copy() - metadata_origin_v2 = { - "id": data.origin_url_1, - "metadata": metadata_v2.copy(), - "indexer_configuration_id": tool_id, - "mappings": ["npm"], - "from_revision": data.revision_id_1, - } + metadata_origin_v2 = OriginIntrinsicMetadataRow( + id=data.origin_url_1, + metadata=metadata_v2.copy(), + indexer_configuration_id=tool_id, + mappings=["npm"], + from_revision=data.revision_id_1, + ) storage.revision_intrinsic_metadata_add([metadata_rev_v2], conflict_update=True) storage.origin_intrinsic_metadata_add( [metadata_origin_v2], conflict_update=True ) actual_metadata = list( storage.origin_intrinsic_metadata_get([data.origin_url_1]) ) expected_metadata_v2 = [ - { - "id": data.origin_url_1, - "metadata": metadata_v2, - "tool": data.tools["swh-metadata-detector"], - "from_revision": data.revision_id_1, - "mappings": ["npm"], - } + OriginIntrinsicMetadataRow( + id=data.origin_url_1, + metadata=metadata_v2, + tool=data.tools["swh-metadata-detector"], + from_revision=data.revision_id_1, + mappings=["npm"], + ) ] # metadata did change as the v2 was used to overwrite v1 assert actual_metadata == expected_metadata_v2 def test_origin_intrinsic_metadata_add__update_in_place_deadlock( self, swh_indexer_storage_with_data: Tuple[IndexerStorageInterface, Any] ) -> None: storage, data = swh_indexer_storage_with_data # given tool_id = data.tools["swh-metadata-detector"]["id"] ids = list(range(10)) - example_data1 = { + example_data1: Dict[str, Any] = { "metadata": {"version": None, "name": None,}, "mappings": [], } - example_data2 = { + example_data2: Dict[str, Any] = { "metadata": {"version": "v1.1.1", "name": "foo",}, "mappings": [], } metadata_rev_v1 = RevisionIntrinsicMetadataRow( id=data.revision_id_2, metadata={"version": None, "name": None,}, mappings=[], indexer_configuration_id=tool_id, ) data_v1 = [ - { - "id": "file:///tmp/origin%d" % id_, - "from_revision": data.revision_id_2, + OriginIntrinsicMetadataRow( + id="file:///tmp/origin%d" % id_, + from_revision=data.revision_id_2, + indexer_configuration_id=tool_id, **example_data1, - "indexer_configuration_id": tool_id, - } + ) for id_ in ids ] data_v2 = [ - { - "id": "file:///tmp/origin%d" % id_, - "from_revision": data.revision_id_2, + OriginIntrinsicMetadataRow( + id="file:///tmp/origin%d" % id_, + from_revision=data.revision_id_2, + indexer_configuration_id=tool_id, **example_data2, - "indexer_configuration_id": tool_id, - } + ) for id_ in ids ] # Remove one item from each, so that both queries have to succeed for # all items to be in the DB. data_v2a = data_v2[1:] data_v2b = list(reversed(data_v2[0:-1])) # given storage.revision_intrinsic_metadata_add([metadata_rev_v1]) storage.origin_intrinsic_metadata_add(data_v1) # when origins = ["file:///tmp/origin%d" % i for i in ids] actual_data = list(storage.origin_intrinsic_metadata_get(origins)) expected_data_v1 = [ - { - "id": "file:///tmp/origin%d" % id_, - "from_revision": data.revision_id_2, + OriginIntrinsicMetadataRow( + id="file:///tmp/origin%d" % id_, + from_revision=data.revision_id_2, + tool=data.tools["swh-metadata-detector"], **example_data1, - "tool": data.tools["swh-metadata-detector"], - } + ) for id_ in ids ] # then assert actual_data == expected_data_v1 # given def f1() -> None: storage.origin_intrinsic_metadata_add(data_v2a, conflict_update=True) def f2() -> None: storage.origin_intrinsic_metadata_add(data_v2b, conflict_update=True) t1 = threading.Thread(target=f1) t2 = threading.Thread(target=f2) t2.start() t1.start() t1.join() t2.join() actual_data = list(storage.origin_intrinsic_metadata_get(origins)) expected_data_v2 = [ - { - "id": "file:///tmp/origin%d" % id_, - "from_revision": data.revision_id_2, + OriginIntrinsicMetadataRow( + id="file:///tmp/origin%d" % id_, + from_revision=data.revision_id_2, + tool=data.tools["swh-metadata-detector"], **example_data2, - "tool": data.tools["swh-metadata-detector"], - } + ) for id_ in ids ] assert len(actual_data) == len(expected_data_v2) - assert sorted(actual_data, key=lambda x: x["id"]) == expected_data_v2 + assert sorted(actual_data, key=lambda x: x.id) == expected_data_v2 def test_origin_intrinsic_metadata_add__duplicate_twice( self, swh_indexer_storage_with_data: Tuple[IndexerStorageInterface, Any] ) -> None: storage, data = swh_indexer_storage_with_data # given tool_id = data.tools["swh-metadata-detector"]["id"] metadata = { "developmentStatus": None, "name": None, } metadata_rev = RevisionIntrinsicMetadataRow( id=data.revision_id_2, metadata=metadata, mappings=["mapping1"], indexer_configuration_id=tool_id, ) - metadata_origin = { - "id": data.origin_url_1, - "metadata": metadata, - "indexer_configuration_id": tool_id, - "mappings": ["mapping1"], - "from_revision": data.revision_id_2, - } + metadata_origin = OriginIntrinsicMetadataRow( + id=data.origin_url_1, + metadata=metadata, + indexer_configuration_id=tool_id, + mappings=["mapping1"], + from_revision=data.revision_id_2, + ) # when storage.revision_intrinsic_metadata_add([metadata_rev]) with pytest.raises(DuplicateId): storage.origin_intrinsic_metadata_add([metadata_origin, metadata_origin]) def test_origin_intrinsic_metadata_search_fulltext( self, swh_indexer_storage_with_data: Tuple[IndexerStorageInterface, Any] ) -> None: storage, data = swh_indexer_storage_with_data # given tool_id = data.tools["swh-metadata-detector"]["id"] metadata1 = { "author": "John Doe", } metadata1_rev = RevisionIntrinsicMetadataRow( id=data.revision_id_1, metadata=metadata1, mappings=[], indexer_configuration_id=tool_id, ) - metadata1_origin = { - "id": data.origin_url_1, - "metadata": metadata1, - "mappings": [], - "indexer_configuration_id": tool_id, - "from_revision": data.revision_id_1, - } + metadata1_origin = OriginIntrinsicMetadataRow( + id=data.origin_url_1, + metadata=metadata1, + mappings=[], + indexer_configuration_id=tool_id, + from_revision=data.revision_id_1, + ) metadata2 = { "author": "Jane Doe", } metadata2_rev = RevisionIntrinsicMetadataRow( id=data.revision_id_2, metadata=metadata2, mappings=[], indexer_configuration_id=tool_id, ) - metadata2_origin = { - "id": data.origin_url_2, - "metadata": metadata2, - "mappings": [], - "indexer_configuration_id": tool_id, - "from_revision": data.revision_id_2, - } + metadata2_origin = OriginIntrinsicMetadataRow( + id=data.origin_url_2, + metadata=metadata2, + mappings=[], + indexer_configuration_id=tool_id, + from_revision=data.revision_id_2, + ) # when storage.revision_intrinsic_metadata_add([metadata1_rev]) storage.origin_intrinsic_metadata_add([metadata1_origin]) storage.revision_intrinsic_metadata_add([metadata2_rev]) storage.origin_intrinsic_metadata_add([metadata2_origin]) # then search = storage.origin_intrinsic_metadata_search_fulltext - assert set([res["id"] for res in search(["Doe"])]) == set( + assert set([res.id for res in search(["Doe"])]) == set( [data.origin_url_1, data.origin_url_2] ) - assert [res["id"] for res in search(["John", "Doe"])] == [data.origin_url_1] - assert [res["id"] for res in search(["John"])] == [data.origin_url_1] + assert [res.id for res in search(["John", "Doe"])] == [data.origin_url_1] + assert [res.id for res in search(["John"])] == [data.origin_url_1] assert not list(search(["John", "Jane"])) def test_origin_intrinsic_metadata_search_fulltext_rank( self, swh_indexer_storage_with_data: Tuple[IndexerStorageInterface, Any] ) -> None: storage, data = swh_indexer_storage_with_data # given tool_id = data.tools["swh-metadata-detector"]["id"] # The following authors have "Random Person" to add some more content # to the JSON data, to work around normalization quirks when there # are few words (rank/(1+ln(nb_words)) is very sensitive to nb_words # for small values of nb_words). metadata1 = {"author": ["Random Person", "John Doe", "Jane Doe",]} metadata1_rev = RevisionIntrinsicMetadataRow( id=data.revision_id_1, metadata=metadata1, mappings=[], indexer_configuration_id=tool_id, ) - metadata1_origin = { - "id": data.origin_url_1, - "metadata": metadata1, - "mappings": [], - "indexer_configuration_id": tool_id, - "from_revision": data.revision_id_1, - } + metadata1_origin = OriginIntrinsicMetadataRow( + id=data.origin_url_1, + metadata=metadata1, + mappings=[], + indexer_configuration_id=tool_id, + from_revision=data.revision_id_1, + ) metadata2 = {"author": ["Random Person", "Jane Doe",]} metadata2_rev = RevisionIntrinsicMetadataRow( id=data.revision_id_2, metadata=metadata2, mappings=[], indexer_configuration_id=tool_id, ) - metadata2_origin = { - "id": data.origin_url_2, - "metadata": metadata2, - "mappings": [], - "indexer_configuration_id": tool_id, - "from_revision": data.revision_id_2, - } + metadata2_origin = OriginIntrinsicMetadataRow( + id=data.origin_url_2, + metadata=metadata2, + mappings=[], + indexer_configuration_id=tool_id, + from_revision=data.revision_id_2, + ) # when storage.revision_intrinsic_metadata_add([metadata1_rev]) storage.origin_intrinsic_metadata_add([metadata1_origin]) storage.revision_intrinsic_metadata_add([metadata2_rev]) storage.origin_intrinsic_metadata_add([metadata2_origin]) # then search = storage.origin_intrinsic_metadata_search_fulltext - assert [res["id"] for res in search(["Doe"])] == [ + assert [res.id for res in search(["Doe"])] == [ data.origin_url_1, data.origin_url_2, ] - assert [res["id"] for res in search(["Doe"], limit=1)] == [data.origin_url_1] - assert [res["id"] for res in search(["John"])] == [data.origin_url_1] - assert [res["id"] for res in search(["Jane"])] == [ + assert [res.id for res in search(["Doe"], limit=1)] == [data.origin_url_1] + assert [res.id for res in search(["John"])] == [data.origin_url_1] + assert [res.id for res in search(["Jane"])] == [ data.origin_url_2, data.origin_url_1, ] - assert [res["id"] for res in search(["John", "Jane"])] == [data.origin_url_1] + assert [res.id for res in search(["John", "Jane"])] == [data.origin_url_1] def _fill_origin_intrinsic_metadata( self, swh_indexer_storage_with_data: Tuple[IndexerStorageInterface, Any] ) -> None: storage, data = swh_indexer_storage_with_data tool1_id = data.tools["swh-metadata-detector"]["id"] tool2_id = data.tools["swh-metadata-detector2"]["id"] metadata1 = { "@context": "foo", "author": "John Doe", } metadata1_rev = RevisionIntrinsicMetadataRow( id=data.revision_id_1, metadata=metadata1, mappings=["npm"], indexer_configuration_id=tool1_id, ) - metadata1_origin = { - "id": data.origin_url_1, - "metadata": metadata1, - "mappings": ["npm"], - "indexer_configuration_id": tool1_id, - "from_revision": data.revision_id_1, - } + metadata1_origin = OriginIntrinsicMetadataRow( + id=data.origin_url_1, + metadata=metadata1, + mappings=["npm"], + indexer_configuration_id=tool1_id, + from_revision=data.revision_id_1, + ) metadata2 = { "@context": "foo", "author": "Jane Doe", } metadata2_rev = RevisionIntrinsicMetadataRow( id=data.revision_id_2, metadata=metadata2, mappings=["npm", "gemspec"], indexer_configuration_id=tool2_id, ) - metadata2_origin = { - "id": data.origin_url_2, - "metadata": metadata2, - "mappings": ["npm", "gemspec"], - "indexer_configuration_id": tool2_id, - "from_revision": data.revision_id_2, - } + metadata2_origin = OriginIntrinsicMetadataRow( + id=data.origin_url_2, + metadata=metadata2, + mappings=["npm", "gemspec"], + indexer_configuration_id=tool2_id, + from_revision=data.revision_id_2, + ) metadata3 = { "@context": "foo", } metadata3_rev = RevisionIntrinsicMetadataRow( id=data.revision_id_3, metadata=metadata3, mappings=["npm", "gemspec"], indexer_configuration_id=tool2_id, ) - metadata3_origin = { - "id": data.origin_url_3, - "metadata": metadata3, - "mappings": ["pkg-info"], - "indexer_configuration_id": tool2_id, - "from_revision": data.revision_id_3, - } + metadata3_origin = OriginIntrinsicMetadataRow( + id=data.origin_url_3, + metadata=metadata3, + mappings=["pkg-info"], + indexer_configuration_id=tool2_id, + from_revision=data.revision_id_3, + ) storage.revision_intrinsic_metadata_add([metadata1_rev]) storage.origin_intrinsic_metadata_add([metadata1_origin]) storage.revision_intrinsic_metadata_add([metadata2_rev]) storage.origin_intrinsic_metadata_add([metadata2_origin]) storage.revision_intrinsic_metadata_add([metadata3_rev]) storage.origin_intrinsic_metadata_add([metadata3_origin]) def test_origin_intrinsic_metadata_search_by_producer( self, swh_indexer_storage_with_data: Tuple[IndexerStorageInterface, Any] ) -> None: storage, data = swh_indexer_storage_with_data self._fill_origin_intrinsic_metadata(swh_indexer_storage_with_data) tool1 = data.tools["swh-metadata-detector"] tool2 = data.tools["swh-metadata-detector2"] endpoint = storage.origin_intrinsic_metadata_search_by_producer # test pagination # no 'page_token' param, return all origins result = endpoint(ids_only=True) - assert result["origins"] == [ - data.origin_url_1, - data.origin_url_2, - data.origin_url_3, - ] - assert "next_page_token" not in result + assert result == PagedResult( + results=[data.origin_url_1, data.origin_url_2, data.origin_url_3,], + next_page_token=None, + ) # 'page_token' is < than origin_1, return everything result = endpoint(page_token=data.origin_url_1[:-1], ids_only=True) - assert result["origins"] == [ - data.origin_url_1, - data.origin_url_2, - data.origin_url_3, - ] - assert "next_page_token" not in result + assert result == PagedResult( + results=[data.origin_url_1, data.origin_url_2, data.origin_url_3,], + next_page_token=None, + ) # 'page_token' is origin_3, return nothing result = endpoint(page_token=data.origin_url_3, ids_only=True) - assert not result["origins"] - assert "next_page_token" not in result + assert result == PagedResult(results=[], next_page_token=None) # test limit argument result = endpoint(page_token=data.origin_url_1[:-1], limit=2, ids_only=True) - assert result["origins"] == [data.origin_url_1, data.origin_url_2] - assert result["next_page_token"] == result["origins"][-1] + assert result == PagedResult( + results=[data.origin_url_1, data.origin_url_2], + next_page_token=data.origin_url_2, + ) result = endpoint(page_token=data.origin_url_1, limit=2, ids_only=True) - assert result["origins"] == [data.origin_url_2, data.origin_url_3] - assert "next_page_token" not in result + assert result == PagedResult( + results=[data.origin_url_2, data.origin_url_3], next_page_token=None, + ) result = endpoint(page_token=data.origin_url_2, limit=2, ids_only=True) - assert result["origins"] == [data.origin_url_3] - assert "next_page_token" not in result + assert result == PagedResult(results=[data.origin_url_3], next_page_token=None,) # test mappings filtering result = endpoint(mappings=["npm"], ids_only=True) - assert result["origins"] == [data.origin_url_1, data.origin_url_2] - assert "next_page_token" not in result + assert result == PagedResult( + results=[data.origin_url_1, data.origin_url_2], next_page_token=None, + ) result = endpoint(mappings=["npm", "gemspec"], ids_only=True) - assert result["origins"] == [data.origin_url_1, data.origin_url_2] - assert "next_page_token" not in result + assert result == PagedResult( + results=[data.origin_url_1, data.origin_url_2], next_page_token=None, + ) result = endpoint(mappings=["gemspec"], ids_only=True) - assert result["origins"] == [data.origin_url_2] - assert "next_page_token" not in result + assert result == PagedResult(results=[data.origin_url_2], next_page_token=None,) result = endpoint(mappings=["pkg-info"], ids_only=True) - assert result["origins"] == [data.origin_url_3] - assert "next_page_token" not in result + assert result == PagedResult(results=[data.origin_url_3], next_page_token=None,) result = endpoint(mappings=["foobar"], ids_only=True) - assert not result["origins"] - assert "next_page_token" not in result + assert result == PagedResult(results=[], next_page_token=None,) # test pagination + mappings result = endpoint(mappings=["npm"], limit=1, ids_only=True) - assert result["origins"] == [data.origin_url_1] - assert result["next_page_token"] == result["origins"][-1] + assert result == PagedResult( + results=[data.origin_url_1], next_page_token=data.origin_url_1, + ) # test tool filtering result = endpoint(tool_ids=[tool1["id"]], ids_only=True) - assert result["origins"] == [data.origin_url_1] - assert "next_page_token" not in result + assert result == PagedResult(results=[data.origin_url_1], next_page_token=None,) result = endpoint(tool_ids=[tool2["id"]], ids_only=True) - assert sorted(result["origins"]) == [data.origin_url_2, data.origin_url_3] - assert "next_page_token" not in result + assert sorted(result.results) == [data.origin_url_2, data.origin_url_3] + assert result.next_page_token is None result = endpoint(tool_ids=[tool1["id"], tool2["id"]], ids_only=True) - assert sorted(result["origins"]) == [ + assert sorted(result.results) == [ data.origin_url_1, data.origin_url_2, data.origin_url_3, ] - assert "next_page_token" not in result + assert result.next_page_token is None # test ids_only=False - assert endpoint(mappings=["gemspec"])["origins"] == [ - { - "id": data.origin_url_2, - "metadata": {"@context": "foo", "author": "Jane Doe",}, - "mappings": ["npm", "gemspec"], - "tool": tool2, - "from_revision": data.revision_id_2, - } - ] + assert endpoint(mappings=["gemspec"]) == PagedResult( + results=[ + OriginIntrinsicMetadataRow( + id=data.origin_url_2, + metadata={"@context": "foo", "author": "Jane Doe",}, + mappings=["npm", "gemspec"], + tool=tool2, + from_revision=data.revision_id_2, + ) + ], + next_page_token=None, + ) def test_origin_intrinsic_metadata_stats( self, swh_indexer_storage_with_data: Tuple[IndexerStorageInterface, Any] ) -> None: storage, data = swh_indexer_storage_with_data self._fill_origin_intrinsic_metadata(swh_indexer_storage_with_data) result = storage.origin_intrinsic_metadata_stats() assert result == { "per_mapping": { "gemspec": 1, "npm": 2, "pkg-info": 1, "codemeta": 0, "maven": 0, }, "total": 3, "non_empty": 2, } class TestIndexerStorageIndexerConfiguration: def test_indexer_configuration_add( self, swh_indexer_storage_with_data: Tuple[IndexerStorageInterface, Any] ) -> None: storage, data = swh_indexer_storage_with_data tool = { "tool_name": "some-unknown-tool", "tool_version": "some-version", "tool_configuration": {"debian-package": "some-package"}, } actual_tool = storage.indexer_configuration_get(tool) assert actual_tool is None # does not exist # add it actual_tools = list(storage.indexer_configuration_add([tool])) assert len(actual_tools) == 1 actual_tool = actual_tools[0] assert actual_tool is not None # now it exists new_id = actual_tool.pop("id") assert actual_tool == tool actual_tools2 = list(storage.indexer_configuration_add([tool])) actual_tool2 = actual_tools2[0] assert actual_tool2 is not None # now it exists new_id2 = actual_tool2.pop("id") assert new_id == new_id2 assert actual_tool == actual_tool2 def test_indexer_configuration_add_multiple( self, swh_indexer_storage_with_data: Tuple[IndexerStorageInterface, Any] ) -> None: storage, data = swh_indexer_storage_with_data tool = { "tool_name": "some-unknown-tool", "tool_version": "some-version", "tool_configuration": {"debian-package": "some-package"}, } actual_tools = list(storage.indexer_configuration_add([tool])) assert len(actual_tools) == 1 new_tools = [ tool, { "tool_name": "yet-another-tool", "tool_version": "version", "tool_configuration": {}, }, ] actual_tools = list(storage.indexer_configuration_add(new_tools)) assert len(actual_tools) == 2 # order not guaranteed, so we iterate over results to check for tool in actual_tools: _id = tool.pop("id") assert _id is not None assert tool in new_tools def test_indexer_configuration_get_missing( self, swh_indexer_storage_with_data: Tuple[IndexerStorageInterface, Any] ) -> None: storage, data = swh_indexer_storage_with_data tool = { "tool_name": "unknown-tool", "tool_version": "3.1.0rc2-31-ga2cbb8c", "tool_configuration": {"command_line": "nomossa "}, } actual_tool = storage.indexer_configuration_get(tool) assert actual_tool is None def test_indexer_configuration_get( self, swh_indexer_storage_with_data: Tuple[IndexerStorageInterface, Any] ) -> None: storage, data = swh_indexer_storage_with_data tool = { "tool_name": "nomos", "tool_version": "3.1.0rc2-31-ga2cbb8c", "tool_configuration": {"command_line": "nomossa "}, } actual_tool = storage.indexer_configuration_get(tool) assert actual_tool expected_tool = tool.copy() del actual_tool["id"] assert expected_tool == actual_tool def test_indexer_configuration_metadata_get_missing_context( self, swh_indexer_storage_with_data: Tuple[IndexerStorageInterface, Any] ) -> None: storage, data = swh_indexer_storage_with_data tool = { "tool_name": "swh-metadata-translator", "tool_version": "0.0.1", "tool_configuration": {"context": "unknown-context"}, } actual_tool = storage.indexer_configuration_get(tool) assert actual_tool is None def test_indexer_configuration_metadata_get( self, swh_indexer_storage_with_data: Tuple[IndexerStorageInterface, Any] ) -> None: storage, data = swh_indexer_storage_with_data tool = { "tool_name": "swh-metadata-translator", "tool_version": "0.0.1", "tool_configuration": {"type": "local", "context": "NpmMapping"}, } storage.indexer_configuration_add([tool]) actual_tool = storage.indexer_configuration_get(tool) assert actual_tool expected_tool = tool.copy() expected_tool["id"] = actual_tool["id"] assert expected_tool == actual_tool diff --git a/swh/indexer/tests/test_cli.py b/swh/indexer/tests/test_cli.py index 68484b3..6da0e7b 100644 --- a/swh/indexer/tests/test_cli.py +++ b/swh/indexer/tests/test_cli.py @@ -1,383 +1,386 @@ # Copyright (C) 2019-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from functools import reduce import re import tempfile from typing import Any, Dict, List from unittest.mock import patch from click.testing import CliRunner from confluent_kafka import Consumer, Producer from swh.indexer.cli import indexer_cli_group from swh.indexer.storage.interface import IndexerStorageInterface -from swh.indexer.storage.model import RevisionIntrinsicMetadataRow +from swh.indexer.storage.model import ( + OriginIntrinsicMetadataRow, + RevisionIntrinsicMetadataRow, +) from swh.journal.serializers import value_to_kafka from swh.model.hashutil import hash_to_bytes CLI_CONFIG = """ scheduler: cls: foo args: {} storage: cls: memory indexer_storage: cls: memory args: {} """ def fill_idx_storage(idx_storage: IndexerStorageInterface, nb_rows: int) -> List[int]: tools: List[Dict[str, Any]] = [ {"tool_name": "tool %d" % i, "tool_version": "0.0.1", "tool_configuration": {},} for i in range(2) ] tools = idx_storage.indexer_configuration_add(tools) origin_metadata = [ - { - "id": "file://dev/%04d" % origin_id, - "from_revision": hash_to_bytes("abcd{:0>4}".format(origin_id)), - "indexer_configuration_id": tools[origin_id % 2]["id"], - "metadata": {"name": "origin %d" % origin_id}, - "mappings": ["mapping%d" % (origin_id % 10)], - } + OriginIntrinsicMetadataRow( + id="file://dev/%04d" % origin_id, + from_revision=hash_to_bytes("abcd{:0>4}".format(origin_id)), + indexer_configuration_id=tools[origin_id % 2]["id"], + metadata={"name": "origin %d" % origin_id}, + mappings=["mapping%d" % (origin_id % 10)], + ) for origin_id in range(nb_rows) ] revision_metadata = [ RevisionIntrinsicMetadataRow( id=hash_to_bytes("abcd{:0>4}".format(origin_id)), indexer_configuration_id=tools[origin_id % 2]["id"], metadata={"name": "origin %d" % origin_id}, mappings=["mapping%d" % (origin_id % 10)], ) for origin_id in range(nb_rows) ] idx_storage.revision_intrinsic_metadata_add(revision_metadata) idx_storage.origin_intrinsic_metadata_add(origin_metadata) return [tool["id"] for tool in tools] def _origins_in_task_args(tasks): """Returns the set of origins contained in the arguments of the provided tasks (assumed to be of type index-origin-metadata).""" return reduce( set.union, (set(task["arguments"]["args"][0]) for task in tasks), set() ) def _assert_tasks_for_origins(tasks, origins): expected_kwargs = {"policy_update": "update-dups"} assert {task["type"] for task in tasks} == {"index-origin-metadata"} assert all(len(task["arguments"]["args"]) == 1 for task in tasks) for task in tasks: assert task["arguments"]["kwargs"] == expected_kwargs, task assert _origins_in_task_args(tasks) == set(["file://dev/%04d" % i for i in origins]) def invoke(scheduler, catch_exceptions, args): runner = CliRunner() with patch( "swh.scheduler.get_scheduler" ) as get_scheduler_mock, tempfile.NamedTemporaryFile( "a", suffix=".yml" ) as config_fd: config_fd.write(CLI_CONFIG) config_fd.seek(0) get_scheduler_mock.return_value = scheduler result = runner.invoke(indexer_cli_group, ["-C" + config_fd.name] + args) if not catch_exceptions and result.exception: print(result.output) raise result.exception return result def test_mapping_list(indexer_scheduler): result = invoke(indexer_scheduler, False, ["mapping", "list",]) expected_output = "\n".join( ["codemeta", "gemspec", "maven", "npm", "pkg-info", "",] ) assert result.exit_code == 0, result.output assert result.output == expected_output def test_mapping_list_terms(indexer_scheduler): result = invoke(indexer_scheduler, False, ["mapping", "list-terms",]) assert result.exit_code == 0, result.output assert re.search(r"http://schema.org/url:\n.*npm", result.output) assert re.search(r"http://schema.org/url:\n.*codemeta", result.output) assert re.search( r"https://codemeta.github.io/terms/developmentStatus:\n\tcodemeta", result.output, ) def test_mapping_list_terms_exclude(indexer_scheduler): result = invoke( indexer_scheduler, False, ["mapping", "list-terms", "--exclude-mapping", "codemeta"], ) assert result.exit_code == 0, result.output assert re.search(r"http://schema.org/url:\n.*npm", result.output) assert not re.search(r"http://schema.org/url:\n.*codemeta", result.output) assert not re.search( r"https://codemeta.github.io/terms/developmentStatus:\n\tcodemeta", result.output, ) @patch("swh.scheduler.cli.utils.TASK_BATCH_SIZE", 3) @patch("swh.scheduler.cli_utils.TASK_BATCH_SIZE", 3) def test_origin_metadata_reindex_empty_db(indexer_scheduler, idx_storage, storage): result = invoke(indexer_scheduler, False, ["schedule", "reindex_origin_metadata",]) expected_output = "Nothing to do (no origin metadata matched the criteria).\n" assert result.exit_code == 0, result.output assert result.output == expected_output tasks = indexer_scheduler.search_tasks() assert len(tasks) == 0 @patch("swh.scheduler.cli.utils.TASK_BATCH_SIZE", 3) @patch("swh.scheduler.cli_utils.TASK_BATCH_SIZE", 3) def test_origin_metadata_reindex_divisor(indexer_scheduler, idx_storage, storage): """Tests the re-indexing when origin_batch_size*task_batch_size is a divisor of nb_origins.""" fill_idx_storage(idx_storage, 90) result = invoke(indexer_scheduler, False, ["schedule", "reindex_origin_metadata",]) # Check the output expected_output = ( "Scheduled 3 tasks (30 origins).\n" "Scheduled 6 tasks (60 origins).\n" "Scheduled 9 tasks (90 origins).\n" "Done.\n" ) assert result.exit_code == 0, result.output assert result.output == expected_output # Check scheduled tasks tasks = indexer_scheduler.search_tasks() assert len(tasks) == 9 _assert_tasks_for_origins(tasks, range(90)) @patch("swh.scheduler.cli.utils.TASK_BATCH_SIZE", 3) @patch("swh.scheduler.cli_utils.TASK_BATCH_SIZE", 3) def test_origin_metadata_reindex_dry_run(indexer_scheduler, idx_storage, storage): """Tests the re-indexing when origin_batch_size*task_batch_size is a divisor of nb_origins.""" fill_idx_storage(idx_storage, 90) result = invoke( indexer_scheduler, False, ["schedule", "--dry-run", "reindex_origin_metadata",] ) # Check the output expected_output = ( "Scheduled 3 tasks (30 origins).\n" "Scheduled 6 tasks (60 origins).\n" "Scheduled 9 tasks (90 origins).\n" "Done.\n" ) assert result.exit_code == 0, result.output assert result.output == expected_output # Check scheduled tasks tasks = indexer_scheduler.search_tasks() assert len(tasks) == 0 @patch("swh.scheduler.cli.utils.TASK_BATCH_SIZE", 3) @patch("swh.scheduler.cli_utils.TASK_BATCH_SIZE", 3) def test_origin_metadata_reindex_nondivisor(indexer_scheduler, idx_storage, storage): """Tests the re-indexing when neither origin_batch_size or task_batch_size is a divisor of nb_origins.""" fill_idx_storage(idx_storage, 70) result = invoke( indexer_scheduler, False, ["schedule", "reindex_origin_metadata", "--batch-size", "20",], ) # Check the output expected_output = ( "Scheduled 3 tasks (60 origins).\n" "Scheduled 4 tasks (70 origins).\n" "Done.\n" ) assert result.exit_code == 0, result.output assert result.output == expected_output # Check scheduled tasks tasks = indexer_scheduler.search_tasks() assert len(tasks) == 4 _assert_tasks_for_origins(tasks, range(70)) @patch("swh.scheduler.cli.utils.TASK_BATCH_SIZE", 3) @patch("swh.scheduler.cli_utils.TASK_BATCH_SIZE", 3) def test_origin_metadata_reindex_filter_one_mapping( indexer_scheduler, idx_storage, storage ): """Tests the re-indexing when origin_batch_size*task_batch_size is a divisor of nb_origins.""" fill_idx_storage(idx_storage, 110) result = invoke( indexer_scheduler, False, ["schedule", "reindex_origin_metadata", "--mapping", "mapping1",], ) # Check the output expected_output = "Scheduled 2 tasks (11 origins).\nDone.\n" assert result.exit_code == 0, result.output assert result.output == expected_output # Check scheduled tasks tasks = indexer_scheduler.search_tasks() assert len(tasks) == 2 _assert_tasks_for_origins(tasks, [1, 11, 21, 31, 41, 51, 61, 71, 81, 91, 101]) @patch("swh.scheduler.cli.utils.TASK_BATCH_SIZE", 3) @patch("swh.scheduler.cli_utils.TASK_BATCH_SIZE", 3) def test_origin_metadata_reindex_filter_two_mappings( indexer_scheduler, idx_storage, storage ): """Tests the re-indexing when origin_batch_size*task_batch_size is a divisor of nb_origins.""" fill_idx_storage(idx_storage, 110) result = invoke( indexer_scheduler, False, [ "schedule", "reindex_origin_metadata", "--mapping", "mapping1", "--mapping", "mapping2", ], ) # Check the output expected_output = "Scheduled 3 tasks (22 origins).\nDone.\n" assert result.exit_code == 0, result.output assert result.output == expected_output # Check scheduled tasks tasks = indexer_scheduler.search_tasks() assert len(tasks) == 3 _assert_tasks_for_origins( tasks, [ 1, 11, 21, 31, 41, 51, 61, 71, 81, 91, 101, 2, 12, 22, 32, 42, 52, 62, 72, 82, 92, 102, ], ) @patch("swh.scheduler.cli.utils.TASK_BATCH_SIZE", 3) @patch("swh.scheduler.cli_utils.TASK_BATCH_SIZE", 3) def test_origin_metadata_reindex_filter_one_tool( indexer_scheduler, idx_storage, storage ): """Tests the re-indexing when origin_batch_size*task_batch_size is a divisor of nb_origins.""" tool_ids = fill_idx_storage(idx_storage, 110) result = invoke( indexer_scheduler, False, ["schedule", "reindex_origin_metadata", "--tool-id", str(tool_ids[0]),], ) # Check the output expected_output = ( "Scheduled 3 tasks (30 origins).\n" "Scheduled 6 tasks (55 origins).\n" "Done.\n" ) assert result.exit_code == 0, result.output assert result.output == expected_output # Check scheduled tasks tasks = indexer_scheduler.search_tasks() assert len(tasks) == 6 _assert_tasks_for_origins(tasks, [x * 2 for x in range(55)]) def test_journal_client( storage, indexer_scheduler, kafka_prefix: str, kafka_server, consumer: Consumer ): """Test the 'swh indexer journal-client' cli tool.""" producer = Producer( { "bootstrap.servers": kafka_server, "client.id": "test producer", "acks": "all", } ) STATUS = {"status": "full", "origin": {"url": "file://dev/0000",}} producer.produce( topic=kafka_prefix + ".origin_visit", key=b"bogus", value=value_to_kafka(STATUS), ) result = invoke( indexer_scheduler, False, [ "journal-client", "--stop-after-objects", "1", "--broker", kafka_server, "--prefix", kafka_prefix, "--group-id", "test-consumer", ], ) # Check the output expected_output = "Done.\n" assert result.exit_code == 0, result.output assert result.output == expected_output # Check scheduled tasks tasks = indexer_scheduler.search_tasks() assert len(tasks) == 1 _assert_tasks_for_origins(tasks, [0]) diff --git a/swh/indexer/tests/test_origin_metadata.py b/swh/indexer/tests/test_origin_metadata.py index 6cc9276..65e8af2 100644 --- a/swh/indexer/tests/test_origin_metadata.py +++ b/swh/indexer/tests/test_origin_metadata.py @@ -1,242 +1,246 @@ # Copyright (C) 2018-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from unittest.mock import patch from swh.indexer.metadata import OriginMetadataIndexer from swh.indexer.storage.interface import IndexerStorageInterface -from swh.indexer.storage.model import RevisionIntrinsicMetadataRow +from swh.indexer.storage.model import ( + OriginIntrinsicMetadataRow, + RevisionIntrinsicMetadataRow, +) from swh.model.model import Origin from swh.storage.interface import StorageInterface from .test_metadata import REVISION_METADATA_CONFIG from .utils import REVISION, YARN_PARSER_METADATA def test_origin_metadata_indexer( idx_storage: IndexerStorageInterface, storage: StorageInterface, obj_storage ) -> None: indexer = OriginMetadataIndexer(config=REVISION_METADATA_CONFIG) origin = "https://github.com/librariesio/yarn-parser" indexer.run([origin]) tool = { "name": "swh-metadata-translator", "version": "0.0.2", "configuration": {"context": "NpmMapping", "type": "local"}, } rev_id = REVISION.id rev_metadata = RevisionIntrinsicMetadataRow( id=rev_id, tool=tool, metadata=YARN_PARSER_METADATA, mappings=["npm"], ) - origin_metadata = { - "id": origin, - "tool": tool, - "from_revision": rev_id, - "metadata": YARN_PARSER_METADATA, - "mappings": ["npm"], - } + origin_metadata = OriginIntrinsicMetadataRow( + id=origin, + tool=tool, + from_revision=rev_id, + metadata=YARN_PARSER_METADATA, + mappings=["npm"], + ) rev_results = list(indexer.idx_storage.revision_intrinsic_metadata_get([rev_id])) for rev_result in rev_results: assert rev_result.tool del rev_result.tool["id"] assert rev_results == [rev_metadata] orig_results = list(indexer.idx_storage.origin_intrinsic_metadata_get([origin])) for orig_result in orig_results: - del orig_result["tool"]["id"] + assert orig_result.tool + del orig_result.tool["id"] assert orig_results == [origin_metadata] def test_origin_metadata_indexer_duplicate_origin( idx_storage: IndexerStorageInterface, storage: StorageInterface, obj_storage ) -> None: indexer = OriginMetadataIndexer(config=REVISION_METADATA_CONFIG) indexer.storage = storage indexer.idx_storage = idx_storage indexer.run(["https://github.com/librariesio/yarn-parser"]) indexer.run(["https://github.com/librariesio/yarn-parser"] * 2) origin = "https://github.com/librariesio/yarn-parser" rev_id = REVISION.id - results = list(indexer.idx_storage.revision_intrinsic_metadata_get([rev_id])) - assert len(results) == 1 + rev_results = list(indexer.idx_storage.revision_intrinsic_metadata_get([rev_id])) + assert len(rev_results) == 1 - results = list(indexer.idx_storage.origin_intrinsic_metadata_get([origin])) - assert len(results) == 1 + orig_results = list(indexer.idx_storage.origin_intrinsic_metadata_get([origin])) + assert len(orig_results) == 1 def test_origin_metadata_indexer_missing_head( idx_storage: IndexerStorageInterface, storage: StorageInterface, obj_storage ) -> None: storage.origin_add([Origin(url="https://example.com")]) indexer = OriginMetadataIndexer(config=REVISION_METADATA_CONFIG) indexer.run(["https://example.com"]) origin = "https://example.com" results = list(indexer.idx_storage.origin_intrinsic_metadata_get([origin])) assert results == [] def test_origin_metadata_indexer_partial_missing_head( idx_storage: IndexerStorageInterface, storage: StorageInterface, obj_storage ) -> None: origin1 = "https://example.com" origin2 = "https://github.com/librariesio/yarn-parser" storage.origin_add([Origin(url=origin1)]) indexer = OriginMetadataIndexer(config=REVISION_METADATA_CONFIG) indexer.run([origin1, origin2]) rev_id = REVISION.id rev_results = list(indexer.idx_storage.revision_intrinsic_metadata_get([rev_id])) assert rev_results == [ RevisionIntrinsicMetadataRow( id=rev_id, metadata=YARN_PARSER_METADATA, mappings=["npm"], tool=rev_results[0].tool, ) ] orig_results = list( indexer.idx_storage.origin_intrinsic_metadata_get([origin1, origin2]) ) for orig_result in orig_results: - del orig_result["tool"] assert orig_results == [ - { - "id": origin2, - "from_revision": rev_id, - "metadata": YARN_PARSER_METADATA, - "mappings": ["npm"], - } + OriginIntrinsicMetadataRow( + id=origin2, + from_revision=rev_id, + metadata=YARN_PARSER_METADATA, + mappings=["npm"], + tool=orig_results[0].tool, + ) ] def test_origin_metadata_indexer_duplicate_revision( idx_storage: IndexerStorageInterface, storage: StorageInterface, obj_storage ) -> None: indexer = OriginMetadataIndexer(config=REVISION_METADATA_CONFIG) indexer.storage = storage indexer.idx_storage = idx_storage indexer.catch_exceptions = False origin1 = "https://github.com/librariesio/yarn-parser" origin2 = "https://github.com/librariesio/yarn-parser.git" indexer.run([origin1, origin2]) rev_id = REVISION.id - results = list(indexer.idx_storage.revision_intrinsic_metadata_get([rev_id])) - assert len(results) == 1 + rev_results = list(indexer.idx_storage.revision_intrinsic_metadata_get([rev_id])) + assert len(rev_results) == 1 - results = list( + orig_results = list( indexer.idx_storage.origin_intrinsic_metadata_get([origin1, origin2]) ) - assert len(results) == 2 + assert len(orig_results) == 2 def test_origin_metadata_indexer_no_metadata_file( idx_storage: IndexerStorageInterface, storage: StorageInterface, obj_storage ) -> None: indexer = OriginMetadataIndexer(config=REVISION_METADATA_CONFIG) origin = "https://github.com/librariesio/yarn-parser" with patch("swh.indexer.metadata_dictionary.npm.NpmMapping.filename", b"foo.json"): indexer.run([origin]) rev_id = REVISION.id - results = list(indexer.idx_storage.revision_intrinsic_metadata_get([rev_id])) - assert results == [] + rev_results = list(indexer.idx_storage.revision_intrinsic_metadata_get([rev_id])) + assert rev_results == [] - results = list(indexer.idx_storage.origin_intrinsic_metadata_get([origin])) - assert results == [] + orig_results = list(indexer.idx_storage.origin_intrinsic_metadata_get([origin])) + assert orig_results == [] def test_origin_metadata_indexer_no_metadata( idx_storage: IndexerStorageInterface, storage: StorageInterface, obj_storage ) -> None: indexer = OriginMetadataIndexer(config=REVISION_METADATA_CONFIG) origin = "https://github.com/librariesio/yarn-parser" with patch( "swh.indexer.metadata.RevisionMetadataIndexer" ".translate_revision_intrinsic_metadata", return_value=(["npm"], {"@context": "foo"}), ): indexer.run([origin]) rev_id = REVISION.id - results = list(indexer.idx_storage.revision_intrinsic_metadata_get([rev_id])) - assert results == [] + rev_results = list(indexer.idx_storage.revision_intrinsic_metadata_get([rev_id])) + assert rev_results == [] - results = list(indexer.idx_storage.origin_intrinsic_metadata_get([origin])) - assert results == [] + orig_results = list(indexer.idx_storage.origin_intrinsic_metadata_get([origin])) + assert orig_results == [] def test_origin_metadata_indexer_error( idx_storage: IndexerStorageInterface, storage: StorageInterface, obj_storage ) -> None: indexer = OriginMetadataIndexer(config=REVISION_METADATA_CONFIG) origin = "https://github.com/librariesio/yarn-parser" with patch( "swh.indexer.metadata.RevisionMetadataIndexer" ".translate_revision_intrinsic_metadata", return_value=None, ): indexer.run([origin]) rev_id = REVISION.id - results = list(indexer.idx_storage.revision_intrinsic_metadata_get([rev_id])) - assert results == [] + rev_results = list(indexer.idx_storage.revision_intrinsic_metadata_get([rev_id])) + assert rev_results == [] - results = list(indexer.idx_storage.origin_intrinsic_metadata_get([origin])) - assert results == [] + orig_results = list(indexer.idx_storage.origin_intrinsic_metadata_get([origin])) + assert orig_results == [] def test_origin_metadata_indexer_delete_metadata( idx_storage: IndexerStorageInterface, storage: StorageInterface, obj_storage ) -> None: indexer = OriginMetadataIndexer(config=REVISION_METADATA_CONFIG) origin = "https://github.com/librariesio/yarn-parser" indexer.run([origin]) rev_id = REVISION.id - results = list(indexer.idx_storage.revision_intrinsic_metadata_get([rev_id])) - assert results != [] + rev_results = list(indexer.idx_storage.revision_intrinsic_metadata_get([rev_id])) + assert rev_results != [] - results = list(indexer.idx_storage.origin_intrinsic_metadata_get([origin])) - assert results != [] + orig_results = list(indexer.idx_storage.origin_intrinsic_metadata_get([origin])) + assert orig_results != [] with patch("swh.indexer.metadata_dictionary.npm.NpmMapping.filename", b"foo.json"): indexer.run([origin]) - results = list(indexer.idx_storage.revision_intrinsic_metadata_get([rev_id])) - assert results == [] + rev_results = list(indexer.idx_storage.revision_intrinsic_metadata_get([rev_id])) + assert rev_results == [] - results = list(indexer.idx_storage.origin_intrinsic_metadata_get([origin])) - assert results == [] + orig_results = list(indexer.idx_storage.origin_intrinsic_metadata_get([origin])) + assert orig_results == [] def test_origin_metadata_indexer_unknown_origin( idx_storage: IndexerStorageInterface, storage: StorageInterface, obj_storage ) -> None: indexer = OriginMetadataIndexer(config=REVISION_METADATA_CONFIG) result = indexer.index_list(["https://unknown.org/foo"]) assert not result