diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2218a39..d1f84e3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,46 +1,40 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v2.4.0 hooks: - id: trailing-whitespace - id: flake8 - id: check-json - id: check-yaml - repo: https://github.com/codespell-project/codespell rev: v1.16.0 hooks: - id: codespell exclude: ^(swh/indexer/data/codemeta/crosswalk.csv)$ - repo: local hooks: - id: mypy name: mypy entry: mypy args: [swh] pass_filenames: false language: system types: [python] +- repo: https://github.com/python/black + rev: 19.10b0 + hooks: + - id: black + # unfortunately, we are far from being able to enable this... # - repo: https://github.com/PyCQA/pydocstyle.git # rev: 4.0.0 # hooks: # - id: pydocstyle # name: pydocstyle # description: pydocstyle is a static analysis tool for checking compliance with Python docstring conventions. # entry: pydocstyle --convention=google # language: python # types: [python] -# black requires py3.6+ -#- repo: https://github.com/python/black -# rev: 19.3b0 -# hooks: -# - id: black -# language_version: python3 -#- repo: https://github.com/asottile/blacken-docs -# rev: v1.0.0-1 -# hooks: -# - id: blacken-docs -# additional_dependencies: [black==19.3b0] diff --git a/conftest.py b/conftest.py index 1c893de..57b3d14 100644 --- a/conftest.py +++ b/conftest.py @@ -1,11 +1,11 @@ from hypothesis import settings # define tests profile. Full documentation is at: # https://hypothesis.readthedocs.io/en/latest/settings.html#settings-profiles settings.register_profile("fast", max_examples=5, deadline=5000) settings.register_profile("slow", max_examples=20, deadline=5000) # Ignore the following modules because wsgi module fails as no # configuration file is found (--doctest-modules forces the module # loading) -collect_ignore = ['swh/indexer/storage/api/wsgi.py'] +collect_ignore = ["swh/indexer/storage/api/wsgi.py"] diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..8d79b7e --- /dev/null +++ b/setup.cfg @@ -0,0 +1,6 @@ +[flake8] +# E203: whitespaces before ':' +# E231: missing whitespace after ',' +# W503: line break before binary operator +ignore = E203,E231,W503 +max-line-length = 88 diff --git a/setup.py b/setup.py index 299b040..9fbbebb 100755 --- a/setup.py +++ b/setup.py @@ -1,71 +1,71 @@ #!/usr/bin/env python3 # Copyright (C) 2015-2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from setuptools import setup, find_packages from os import path from io import open here = path.abspath(path.dirname(__file__)) # Get the long description from the README file -with open(path.join(here, 'README.md'), encoding='utf-8') as f: +with open(path.join(here, "README.md"), encoding="utf-8") as f: long_description = f.read() def parse_requirements(name=None): if name: - reqf = 'requirements-%s.txt' % name + reqf = "requirements-%s.txt" % name else: - reqf = 'requirements.txt' + reqf = "requirements.txt" requirements = [] if not path.exists(reqf): return requirements with open(reqf) as f: for line in f.readlines(): line = line.strip() - if not line or line.startswith('#'): + if not line or line.startswith("#"): continue requirements.append(line) return requirements setup( - name='swh.indexer', - description='Software Heritage Content Indexer', + name="swh.indexer", + description="Software Heritage Content Indexer", long_description=long_description, - long_description_content_type='text/markdown', - author='Software Heritage developers', - author_email='swh-devel@inria.fr', - url='https://forge.softwareheritage.org/diffusion/78/', + long_description_content_type="text/markdown", + author="Software Heritage developers", + author_email="swh-devel@inria.fr", + url="https://forge.softwareheritage.org/diffusion/78/", packages=find_packages(), scripts=[], - install_requires=parse_requirements() + parse_requirements('swh'), - setup_requires=['vcversioner'], - extras_require={'testing': parse_requirements('test')}, + install_requires=parse_requirements() + parse_requirements("swh"), + setup_requires=["vcversioner"], + extras_require={"testing": parse_requirements("test")}, vcversioner={}, include_package_data=True, - entry_points=''' + entry_points=""" [console_scripts] swh-indexer=swh.indexer.cli:main [swh.cli.subcommands] indexer=swh.indexer.cli:cli - ''', + """, classifiers=[ "Programming Language :: Python :: 3", "Intended Audience :: Developers", "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", "Operating System :: OS Independent", "Development Status :: 5 - Production/Stable", ], project_urls={ - 'Bug Reports': 'https://forge.softwareheritage.org/maniphest', - 'Funding': 'https://www.softwareheritage.org/donate', - 'Source': 'https://forge.softwareheritage.org/source/swh-indexer', + "Bug Reports": "https://forge.softwareheritage.org/maniphest", + "Funding": "https://www.softwareheritage.org/donate", + "Source": "https://forge.softwareheritage.org/source/swh-indexer", }, ) diff --git a/swh/indexer/cli.py b/swh/indexer/cli.py index 7f66ff5..d4ef976 100644 --- a/swh/indexer/cli.py +++ b/swh/indexer/cli.py @@ -1,253 +1,284 @@ # Copyright (C) 2019-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import functools import json import click from swh.core import config from swh.core.cli import CONTEXT_SETTINGS, AliasedGroup from swh.journal.cli import get_journal_client from swh.scheduler import get_scheduler from swh.scheduler.cli_utils import schedule_origin_batches from swh.storage import get_storage from swh.indexer import metadata_dictionary from swh.indexer.journal_client import process_journal_objects from swh.indexer.storage import get_indexer_storage from swh.indexer.storage.api.server import load_and_check_config, app -@click.group(name='indexer', context_settings=CONTEXT_SETTINGS, - cls=AliasedGroup) -@click.option('--config-file', '-C', default=None, - type=click.Path(exists=True, dir_okay=False,), - help="Configuration file.") +@click.group(name="indexer", context_settings=CONTEXT_SETTINGS, cls=AliasedGroup) +@click.option( + "--config-file", + "-C", + default=None, + type=click.Path(exists=True, dir_okay=False,), + help="Configuration file.", +) @click.pass_context def cli(ctx, config_file): """Software Heritage Indexer tools. The Indexer is used to mine the content of the archive and extract derived information from archive source code artifacts. """ ctx.ensure_object(dict) conf = config.read(config_file) - ctx.obj['config'] = conf + ctx.obj["config"] = conf def _get_api(getter, config, config_key, url): if url: - config[config_key] = { - 'cls': 'remote', - 'args': {'url': url} - } + config[config_key] = {"cls": "remote", "args": {"url": url}} elif config_key not in config: - raise click.ClickException( - 'Missing configuration for {}'.format(config_key)) + raise click.ClickException("Missing configuration for {}".format(config_key)) return getter(**config[config_key]) -@cli.group('mapping') +@cli.group("mapping") def mapping(): - '''Manage Software Heritage Indexer mappings.''' + """Manage Software Heritage Indexer mappings.""" pass -@mapping.command('list') +@mapping.command("list") def mapping_list(): """Prints the list of known mappings.""" - mapping_names = [mapping.name - for mapping in metadata_dictionary.MAPPINGS.values()] + mapping_names = [mapping.name for mapping in metadata_dictionary.MAPPINGS.values()] mapping_names.sort() for mapping_name in mapping_names: click.echo(mapping_name) -@mapping.command('list-terms') -@click.option('--exclude-mapping', multiple=True, - help='Exclude the given mapping from the output') -@click.option('--concise', is_flag=True, - default=False, - help='Don\'t print the list of mappings supporting each term.') +@mapping.command("list-terms") +@click.option( + "--exclude-mapping", multiple=True, help="Exclude the given mapping from the output" +) +@click.option( + "--concise", + is_flag=True, + default=False, + help="Don't print the list of mappings supporting each term.", +) def mapping_list_terms(concise, exclude_mapping): """Prints the list of known CodeMeta terms, and which mappings support them.""" properties = metadata_dictionary.list_terms() for (property_name, supported_mappings) in sorted(properties.items()): supported_mappings = {m.name for m in supported_mappings} supported_mappings -= set(exclude_mapping) if supported_mappings: if concise: click.echo(property_name) else: - click.echo('{}:'.format(property_name)) - click.echo('\t' + ', '.join(sorted(supported_mappings))) + click.echo("{}:".format(property_name)) + click.echo("\t" + ", ".join(sorted(supported_mappings))) -@mapping.command('translate') -@click.argument('mapping-name') -@click.argument('file', type=click.File('rb')) +@mapping.command("translate") +@click.argument("mapping-name") +@click.argument("file", type=click.File("rb")) def mapping_translate(mapping_name, file): """Prints the list of known mappings.""" - mapping_cls = [cls for cls in metadata_dictionary.MAPPINGS.values() - if cls.name == mapping_name] + mapping_cls = [ + cls for cls in metadata_dictionary.MAPPINGS.values() if cls.name == mapping_name + ] if not mapping_cls: - raise click.ClickException('Unknown mapping {}'.format(mapping_name)) + raise click.ClickException("Unknown mapping {}".format(mapping_name)) assert len(mapping_cls) == 1 mapping_cls = mapping_cls[0] mapping = mapping_cls() codemeta_doc = mapping.translate(file.read()) click.echo(json.dumps(codemeta_doc, indent=4)) -@cli.group('schedule') -@click.option('--scheduler-url', '-s', default=None, - help="URL of the scheduler API") -@click.option('--indexer-storage-url', '-i', default=None, - help="URL of the indexer storage API") -@click.option('--storage-url', '-g', default=None, - help="URL of the (graph) storage API") -@click.option('--dry-run/--no-dry-run', is_flag=True, - default=False, - help='List only what would be scheduled.') +@cli.group("schedule") +@click.option("--scheduler-url", "-s", default=None, help="URL of the scheduler API") +@click.option( + "--indexer-storage-url", "-i", default=None, help="URL of the indexer storage API" +) +@click.option( + "--storage-url", "-g", default=None, help="URL of the (graph) storage API" +) +@click.option( + "--dry-run/--no-dry-run", + is_flag=True, + default=False, + help="List only what would be scheduled.", +) @click.pass_context -def schedule(ctx, scheduler_url, storage_url, indexer_storage_url, - dry_run): +def schedule(ctx, scheduler_url, storage_url, indexer_storage_url, dry_run): """Manipulate Software Heritage Indexer tasks. Via SWH Scheduler's API.""" - ctx.obj['indexer_storage'] = _get_api( - get_indexer_storage, - ctx.obj['config'], - 'indexer_storage', - indexer_storage_url + ctx.obj["indexer_storage"] = _get_api( + get_indexer_storage, ctx.obj["config"], "indexer_storage", indexer_storage_url ) - ctx.obj['storage'] = _get_api( - get_storage, - ctx.obj['config'], - 'storage', - storage_url + ctx.obj["storage"] = _get_api( + get_storage, ctx.obj["config"], "storage", storage_url ) - ctx.obj['scheduler'] = _get_api( - get_scheduler, - ctx.obj['config'], - 'scheduler', - scheduler_url + ctx.obj["scheduler"] = _get_api( + get_scheduler, ctx.obj["config"], "scheduler", scheduler_url ) if dry_run: - ctx.obj['scheduler'] = None + ctx.obj["scheduler"] = None def list_origins_by_producer(idx_storage, mappings, tool_ids): - next_page_token = '' + next_page_token = "" limit = 10000 while next_page_token is not None: result = idx_storage.origin_intrinsic_metadata_search_by_producer( - page_token=next_page_token, limit=limit, ids_only=True, - mappings=mappings or None, tool_ids=tool_ids or None) - next_page_token = result.get('next_page_token') - yield from result['origins'] - - -@schedule.command('reindex_origin_metadata') -@click.option('--batch-size', '-b', 'origin_batch_size', - default=10, show_default=True, type=int, - help="Number of origins per task") -@click.option('--tool-id', '-t', 'tool_ids', type=int, multiple=True, - help="Restrict search of old metadata to this/these tool ids.") -@click.option('--mapping', '-m', 'mappings', multiple=True, - help="Mapping(s) that should be re-scheduled (eg. 'npm', " - "'gemspec', 'maven')") -@click.option('--task-type', - default='index-origin-metadata', show_default=True, - help="Name of the task type to schedule.") + page_token=next_page_token, + limit=limit, + ids_only=True, + mappings=mappings or None, + tool_ids=tool_ids or None, + ) + next_page_token = result.get("next_page_token") + yield from result["origins"] + + +@schedule.command("reindex_origin_metadata") +@click.option( + "--batch-size", + "-b", + "origin_batch_size", + default=10, + show_default=True, + type=int, + help="Number of origins per task", +) +@click.option( + "--tool-id", + "-t", + "tool_ids", + type=int, + multiple=True, + help="Restrict search of old metadata to this/these tool ids.", +) +@click.option( + "--mapping", + "-m", + "mappings", + multiple=True, + help="Mapping(s) that should be re-scheduled (eg. 'npm', " "'gemspec', 'maven')", +) +@click.option( + "--task-type", + default="index-origin-metadata", + show_default=True, + help="Name of the task type to schedule.", +) @click.pass_context def schedule_origin_metadata_reindex( - ctx, origin_batch_size, tool_ids, mappings, task_type): + ctx, origin_batch_size, tool_ids, mappings, task_type +): """Schedules indexing tasks for origins that were already indexed.""" - idx_storage = ctx.obj['indexer_storage'] - scheduler = ctx.obj['scheduler'] + idx_storage = ctx.obj["indexer_storage"] + scheduler = ctx.obj["scheduler"] origins = list_origins_by_producer(idx_storage, mappings, tool_ids) kwargs = {"policy_update": "update-dups"} - schedule_origin_batches( - scheduler, task_type, origins, origin_batch_size, kwargs) - - -@cli.command('journal-client') -@click.option('--scheduler-url', '-s', default=None, - help="URL of the scheduler API") -@click.option('--origin-metadata-task-type', - default='index-origin-metadata', - help='Name of the task running the origin metadata indexer.') -@click.option('--broker', 'brokers', type=str, multiple=True, - help='Kafka broker to connect to.') -@click.option('--prefix', type=str, default=None, - help='Prefix of Kafka topic names to read from.') -@click.option('--group-id', type=str, - help='Consumer/group id for reading from Kafka.') -@click.option('--stop-after-objects', '-m', default=None, type=int, - help='Maximum number of objects to replay. Default is to ' - 'run forever.') + schedule_origin_batches(scheduler, task_type, origins, origin_batch_size, kwargs) + + +@cli.command("journal-client") +@click.option("--scheduler-url", "-s", default=None, help="URL of the scheduler API") +@click.option( + "--origin-metadata-task-type", + default="index-origin-metadata", + help="Name of the task running the origin metadata indexer.", +) +@click.option( + "--broker", "brokers", type=str, multiple=True, help="Kafka broker to connect to." +) +@click.option( + "--prefix", type=str, default=None, help="Prefix of Kafka topic names to read from." +) +@click.option("--group-id", type=str, help="Consumer/group id for reading from Kafka.") +@click.option( + "--stop-after-objects", + "-m", + default=None, + type=int, + help="Maximum number of objects to replay. Default is to " "run forever.", +) @click.pass_context -def journal_client(ctx, scheduler_url, origin_metadata_task_type, - brokers, prefix, group_id, stop_after_objects): +def journal_client( + ctx, + scheduler_url, + origin_metadata_task_type, + brokers, + prefix, + group_id, + stop_after_objects, +): """Listens for new objects from the SWH Journal, and schedules tasks to run relevant indexers (currently, only origin-intrinsic-metadata) on these new objects.""" - scheduler = _get_api( - get_scheduler, - ctx.obj['config'], - 'scheduler', - scheduler_url - ) + scheduler = _get_api(get_scheduler, ctx.obj["config"], "scheduler", scheduler_url) client = get_journal_client( - ctx, brokers=brokers, prefix=prefix, group_id=group_id, - object_types=['origin_visit'], + ctx, + brokers=brokers, + prefix=prefix, + group_id=group_id, + object_types=["origin_visit"], stop_after_objects=stop_after_objects, ) worker_fn = functools.partial( process_journal_objects, scheduler=scheduler, - task_names={ - 'origin_metadata': origin_metadata_task_type, - } + task_names={"origin_metadata": origin_metadata_task_type,}, ) try: client.process(worker_fn) except KeyboardInterrupt: ctx.exit(0) else: - print('Done.') + print("Done.") finally: client.close() -@cli.command('rpc-serve') -@click.argument('config-path', required=True) -@click.option('--host', default='0.0.0.0', help="Host to run the server") -@click.option('--port', default=5007, type=click.INT, - help="Binding port of the server") -@click.option('--debug/--nodebug', default=True, - help="Indicates if the server should run in debug mode") +@cli.command("rpc-serve") +@click.argument("config-path", required=True) +@click.option("--host", default="0.0.0.0", help="Host to run the server") +@click.option("--port", default=5007, type=click.INT, help="Binding port of the server") +@click.option( + "--debug/--nodebug", + default=True, + help="Indicates if the server should run in debug mode", +) def rpc_server(config_path, host, port, debug): """Starts a Software Heritage Indexer RPC HTTP server.""" - api_cfg = load_and_check_config(config_path, type='any') + api_cfg = load_and_check_config(config_path, type="any") app.config.update(api_cfg) app.run(host, port=int(port), debug=bool(debug)) def main(): - return cli(auto_envvar_prefix='SWH_INDEXER') + return cli(auto_envvar_prefix="SWH_INDEXER") -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/swh/indexer/codemeta.py b/swh/indexer/codemeta.py index fe66a16..3978ca6 100644 --- a/swh/indexer/codemeta.py +++ b/swh/indexer/codemeta.py @@ -1,205 +1,203 @@ # Copyright (C) 2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import collections import csv import itertools import json import os.path import re import swh.indexer from pyld import jsonld -_DATA_DIR = os.path.join(os.path.dirname(swh.indexer.__file__), 'data') +_DATA_DIR = os.path.join(os.path.dirname(swh.indexer.__file__), "data") -CROSSWALK_TABLE_PATH = os.path.join(_DATA_DIR, 'codemeta', 'crosswalk.csv') +CROSSWALK_TABLE_PATH = os.path.join(_DATA_DIR, "codemeta", "crosswalk.csv") -CODEMETA_CONTEXT_PATH = os.path.join(_DATA_DIR, 'codemeta', 'codemeta.jsonld') +CODEMETA_CONTEXT_PATH = os.path.join(_DATA_DIR, "codemeta", "codemeta.jsonld") with open(CODEMETA_CONTEXT_PATH) as fd: CODEMETA_CONTEXT = json.load(fd) -CODEMETA_CONTEXT_URL = 'https://doi.org/10.5063/schema/codemeta-2.0' +CODEMETA_CONTEXT_URL = "https://doi.org/10.5063/schema/codemeta-2.0" CODEMETA_ALTERNATE_CONTEXT_URLS = { - ('https://raw.githubusercontent.com/codemeta/codemeta/' - 'master/codemeta.jsonld') + ("https://raw.githubusercontent.com/codemeta/codemeta/" "master/codemeta.jsonld") } -CODEMETA_URI = 'https://codemeta.github.io/terms/' -SCHEMA_URI = 'http://schema.org/' +CODEMETA_URI = "https://codemeta.github.io/terms/" +SCHEMA_URI = "http://schema.org/" PROPERTY_BLACKLIST = { # CodeMeta properties that we cannot properly represent. - SCHEMA_URI + 'softwareRequirements', - CODEMETA_URI + 'softwareSuggestions', - + SCHEMA_URI + "softwareRequirements", + CODEMETA_URI + "softwareSuggestions", # Duplicate of 'author' - SCHEMA_URI + 'creator', - } + SCHEMA_URI + "creator", +} -_codemeta_field_separator = re.compile(r'\s*[,/]\s*') +_codemeta_field_separator = re.compile(r"\s*[,/]\s*") def make_absolute_uri(local_name): - definition = CODEMETA_CONTEXT['@context'][local_name] + definition = CODEMETA_CONTEXT["@context"][local_name] if isinstance(definition, str): return definition elif isinstance(definition, dict): - prefixed_name = definition['@id'] - (prefix, local_name) = prefixed_name.split(':') - if prefix == 'schema': + prefixed_name = definition["@id"] + (prefix, local_name) = prefixed_name.split(":") + if prefix == "schema": canonical_name = SCHEMA_URI + local_name - elif prefix == 'codemeta': + elif prefix == "codemeta": canonical_name = CODEMETA_URI + local_name else: assert False, prefix return canonical_name else: assert False, definition def _read_crosstable(fd): reader = csv.reader(fd) try: header = next(reader) except StopIteration: - raise ValueError('empty file') + raise ValueError("empty file") - data_sources = set(header) - {'Parent Type', 'Property', - 'Type', 'Description'} - assert 'codemeta-V1' in data_sources + data_sources = set(header) - {"Parent Type", "Property", "Type", "Description"} + assert "codemeta-V1" in data_sources codemeta_translation = {data_source: {} for data_source in data_sources} terms = set() for line in reader: # For each canonical name - local_name = dict(zip(header, line))['Property'] + local_name = dict(zip(header, line))["Property"] if not local_name: continue canonical_name = make_absolute_uri(local_name) if canonical_name in PROPERTY_BLACKLIST: continue terms.add(canonical_name) for (col, value) in zip(header, line): # For each cell in the row if col in data_sources: # If that's not the parentType/property/type/description for local_name in _codemeta_field_separator.split(value): # For each of the data source's properties that maps # to this canonical name if local_name.strip(): - codemeta_translation[col][local_name.strip()] = \ - canonical_name + codemeta_translation[col][local_name.strip()] = canonical_name return (terms, codemeta_translation) with open(CROSSWALK_TABLE_PATH) as fd: (CODEMETA_TERMS, CROSSWALK_TABLE) = _read_crosstable(fd) def _document_loader(url): """Document loader for pyld. Reads the local codemeta.jsonld file instead of fetching it from the Internet every single time.""" if url == CODEMETA_CONTEXT_URL or url in CODEMETA_ALTERNATE_CONTEXT_URLS: return { - 'contextUrl': None, - 'documentUrl': url, - 'document': CODEMETA_CONTEXT, - } + "contextUrl": None, + "documentUrl": url, + "document": CODEMETA_CONTEXT, + } elif url == CODEMETA_URI: - raise Exception('{} is CodeMeta\'s URI, use {} as context url'.format( - CODEMETA_URI, CODEMETA_CONTEXT_URL)) + raise Exception( + "{} is CodeMeta's URI, use {} as context url".format( + CODEMETA_URI, CODEMETA_CONTEXT_URL + ) + ) else: raise Exception(url) def compact(doc): """Same as `pyld.jsonld.compact`, but in the context of CodeMeta.""" - return jsonld.compact(doc, CODEMETA_CONTEXT_URL, - options={'documentLoader': _document_loader}) + return jsonld.compact( + doc, CODEMETA_CONTEXT_URL, options={"documentLoader": _document_loader} + ) def expand(doc): """Same as `pyld.jsonld.expand`, but in the context of CodeMeta.""" - return jsonld.expand(doc, - options={'documentLoader': _document_loader}) + return jsonld.expand(doc, options={"documentLoader": _document_loader}) def merge_values(v1, v2): """If v1 and v2 are of the form `{"@list": l1}` and `{"@list": l2}`, returns `{"@list": l1 + l2}`. Otherwise, make them lists (if they are not already) and concatenate them. >>> merge_values('a', 'b') ['a', 'b'] >>> merge_values(['a', 'b'], 'c') ['a', 'b', 'c'] >>> merge_values({'@list': ['a', 'b']}, {'@list': ['c']}) {'@list': ['a', 'b', 'c']} """ if v1 is None: return v2 elif v2 is None: return v1 - elif isinstance(v1, dict) and set(v1) == {'@list'}: - assert isinstance(v1['@list'], list) - if isinstance(v2, dict) and set(v2) == {'@list'}: - assert isinstance(v2['@list'], list) - return {'@list': v1['@list'] + v2['@list']} + elif isinstance(v1, dict) and set(v1) == {"@list"}: + assert isinstance(v1["@list"], list) + if isinstance(v2, dict) and set(v2) == {"@list"}: + assert isinstance(v2["@list"], list) + return {"@list": v1["@list"] + v2["@list"]} else: - raise ValueError('Cannot merge %r and %r' % (v1, v2)) + raise ValueError("Cannot merge %r and %r" % (v1, v2)) else: - if isinstance(v2, dict) and '@list' in v2: - raise ValueError('Cannot merge %r and %r' % (v1, v2)) + if isinstance(v2, dict) and "@list" in v2: + raise ValueError("Cannot merge %r and %r" % (v1, v2)) if not isinstance(v1, list): v1 = [v1] if not isinstance(v2, list): v2 = [v2] return v1 + v2 def merge_documents(documents): """Takes a list of metadata dicts, each generated from a different metadata file, and merges them. Removes duplicates, if any.""" documents = list(itertools.chain.from_iterable(map(expand, documents))) merged_document = collections.defaultdict(list) for document in documents: for (key, values) in document.items(): - if key == '@id': + if key == "@id": # @id does not get expanded to a list value = values # Only one @id is allowed, move it to sameAs - if '@id' not in merged_document: - merged_document['@id'] = value - elif value != merged_document['@id']: - if value not in merged_document[SCHEMA_URI + 'sameAs']: - merged_document[SCHEMA_URI + 'sameAs'].append(value) + if "@id" not in merged_document: + merged_document["@id"] = value + elif value != merged_document["@id"]: + if value not in merged_document[SCHEMA_URI + "sameAs"]: + merged_document[SCHEMA_URI + "sameAs"].append(value) else: for value in values: - if isinstance(value, dict) and set(value) == {'@list'}: + if isinstance(value, dict) and set(value) == {"@list"}: # Value is of the form {'@list': [item1, item2]} # instead of the usual [item1, item2]. # We need to merge the inner lists (and mostly # preserve order). - merged_value = merged_document.setdefault( - key, {'@list': []}) - for subvalue in value['@list']: + merged_value = merged_document.setdefault(key, {"@list": []}) + for subvalue in value["@list"]: # merged_value must be of the form # {'@list': [item1, item2]}; as it is the same # type as value, which is an @list. - if subvalue not in merged_value['@list']: - merged_value['@list'].append(subvalue) + if subvalue not in merged_value["@list"]: + merged_value["@list"].append(subvalue) elif value not in merged_document[key]: merged_document[key].append(value) return compact(merged_document) diff --git a/swh/indexer/ctags.py b/swh/indexer/ctags.py index 0ea73a9..27ae18c 100644 --- a/swh/indexer/ctags.py +++ b/swh/indexer/ctags.py @@ -1,154 +1,159 @@ # Copyright (C) 2015-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import subprocess import json from typing import Dict, List from swh.model import hashutil from .indexer import ContentIndexer, write_to_temp # Options used to compute tags __FLAGS = [ - '--fields=+lnz', # +l: language - # +n: line number of tag definition - # +z: include the symbol's kind (function, variable, ...) - '--sort=no', # sort output on tag name - '--links=no', # do not follow symlinks - '--output-format=json', # outputs in json + "--fields=+lnz", # +l: language + # +n: line number of tag definition + # +z: include the symbol's kind (function, variable, ...) + "--sort=no", # sort output on tag name + "--links=no", # do not follow symlinks + "--output-format=json", # outputs in json ] def compute_language(content, log=None): raise NotImplementedError( - 'Language detection was unreliable, so it is currently disabled. ' - 'See https://forge.softwareheritage.org/D1455') + "Language detection was unreliable, so it is currently disabled. " + "See https://forge.softwareheritage.org/D1455" + ) -def run_ctags(path, lang=None, ctags_command='ctags'): +def run_ctags(path, lang=None, ctags_command="ctags"): """Run ctags on file path with optional language. Args: path: path to the file lang: language for that path (optional) Yields: dict: ctags' output """ optional = [] if lang: - optional = ['--language-force=%s' % lang] + optional = ["--language-force=%s" % lang] cmd = [ctags_command] + __FLAGS + optional + [path] output = subprocess.check_output(cmd, universal_newlines=True) - for symbol in output.split('\n'): + for symbol in output.split("\n"): if not symbol: continue js_symbol = json.loads(symbol) yield { - 'name': js_symbol['name'], - 'kind': js_symbol['kind'], - 'line': js_symbol['line'], - 'lang': js_symbol['language'], + "name": js_symbol["name"], + "kind": js_symbol["kind"], + "line": js_symbol["line"], + "lang": js_symbol["language"], } class CtagsIndexer(ContentIndexer): - CONFIG_BASE_FILENAME = 'indexer/ctags' + CONFIG_BASE_FILENAME = "indexer/ctags" ADDITIONAL_CONFIG = { - 'workdir': ('str', '/tmp/swh/indexer.ctags'), - 'tools': ('dict', { - 'name': 'universal-ctags', - 'version': '~git7859817b', - 'configuration': { - 'command_line': '''ctags --fields=+lnz --sort=no --links=no ''' - '''--output-format=json ''' + "workdir": ("str", "/tmp/swh/indexer.ctags"), + "tools": ( + "dict", + { + "name": "universal-ctags", + "version": "~git7859817b", + "configuration": { + "command_line": """ctags --fields=+lnz --sort=no --links=no """ + """--output-format=json """ + }, + }, + ), + "languages": ( + "dict", + { + "ada": "Ada", + "adl": None, + "agda": None, + # ... }, - }), - 'languages': ('dict', { - 'ada': 'Ada', - 'adl': None, - 'agda': None, - # ... - }) + ), } def prepare(self): super().prepare() - self.working_directory = self.config['workdir'] - self.language_map = self.config['languages'] + self.working_directory = self.config["workdir"] + self.language_map = self.config["languages"] def filter(self, ids): """Filter out known sha1s and return only missing ones. """ - yield from self.idx_storage.content_ctags_missing(( - { - 'id': sha1, - 'indexer_configuration_id': self.tool['id'], - } for sha1 in ids - )) + yield from self.idx_storage.content_ctags_missing( + ({"id": sha1, "indexer_configuration_id": self.tool["id"],} for sha1 in ids) + ) def index(self, id, data): """Index sha1s' content and store result. Args: id (bytes): content's identifier data (bytes): raw content in bytes Returns: dict: a dict representing a content_mimetype with keys: - **id** (bytes): content's identifier (sha1) - **ctags** ([dict]): ctags list of symbols """ - lang = compute_language(data, log=self.log)['lang'] + lang = compute_language(data, log=self.log)["lang"] if not lang: return None ctags_lang = self.language_map.get(lang) if not ctags_lang: return None ctags = { - 'id': id, + "id": id, } filename = hashutil.hash_to_hex(id) with write_to_temp( - filename=filename, data=data, - working_directory=self.working_directory) as content_path: + filename=filename, data=data, working_directory=self.working_directory + ) as content_path: result = run_ctags(content_path, lang=ctags_lang) - ctags.update({ - 'ctags': list(result), - 'indexer_configuration_id': self.tool['id'], - }) + ctags.update( + {"ctags": list(result), "indexer_configuration_id": self.tool["id"],} + ) return ctags def persist_index_computations( - self, results: List[Dict], policy_update: str) -> Dict[str, int]: + self, results: List[Dict], policy_update: str + ) -> Dict[str, int]: """Persist the results in storage. Args: results: list of content_mimetype, dict with the following keys: - id (bytes): content's identifier (sha1) - ctags ([dict]): ctags list of symbols policy_update: either 'update-dups' or 'ignore-dups' to respectively update duplicates or ignore them """ return self.idx_storage.content_ctags_add( - results, conflict_update=(policy_update == 'update-dups')) + results, conflict_update=(policy_update == "update-dups") + ) diff --git a/swh/indexer/fossology_license.py b/swh/indexer/fossology_license.py index 5a2c820..1b4cffa 100644 --- a/swh/indexer/fossology_license.py +++ b/swh/indexer/fossology_license.py @@ -1,181 +1,185 @@ # Copyright (C) 2016-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import logging import subprocess from typing import Any, Dict, List, Optional from swh.model import hashutil from .indexer import ContentIndexer, ContentRangeIndexer, write_to_temp logger = logging.getLogger(__name__) def compute_license(path): """Determine license from file at path. Args: path: filepath to determine the license Returns: dict: A dict with the following keys: - licenses ([str]): associated detected licenses to path - path (bytes): content filepath """ try: - properties = subprocess.check_output(['nomossa', path], - universal_newlines=True) + properties = subprocess.check_output(["nomossa", path], universal_newlines=True) if properties: - res = properties.rstrip().split(' contains license(s) ') - licenses = res[1].split(',') + res = properties.rstrip().split(" contains license(s) ") + licenses = res[1].split(",") else: licenses = [] return { - 'licenses': licenses, - 'path': path, + "licenses": licenses, + "path": path, } except subprocess.CalledProcessError: from os import path as __path - logger.exception('Problem during license detection for sha1 %s' % - __path.basename(path)) + + logger.exception( + "Problem during license detection for sha1 %s" % __path.basename(path) + ) return { - 'licenses': [], - 'path': path, + "licenses": [], + "path": path, } class MixinFossologyLicenseIndexer: """Mixin fossology license indexer. See :class:`FossologyLicenseIndexer` and :class:`FossologyLicenseRangeIndexer` """ + ADDITIONAL_CONFIG = { - 'workdir': ('str', '/tmp/swh/indexer.fossology.license'), - 'tools': ('dict', { - 'name': 'nomos', - 'version': '3.1.0rc2-31-ga2cbb8c', - 'configuration': { - 'command_line': 'nomossa ', + "workdir": ("str", "/tmp/swh/indexer.fossology.license"), + "tools": ( + "dict", + { + "name": "nomos", + "version": "3.1.0rc2-31-ga2cbb8c", + "configuration": {"command_line": "nomossa ",}, }, - }), - 'write_batch_size': ('int', 1000), + ), + "write_batch_size": ("int", 1000), } - CONFIG_BASE_FILENAME = 'indexer/fossology_license' # type: Optional[str] + CONFIG_BASE_FILENAME = "indexer/fossology_license" # type: Optional[str] tool: Any idx_storage: Any def prepare(self): super().prepare() - self.working_directory = self.config['workdir'] + self.working_directory = self.config["workdir"] - def index(self, id: bytes, data: Optional[bytes] = None, - **kwargs) -> Dict[str, Any]: + def index( + self, id: bytes, data: Optional[bytes] = None, **kwargs + ) -> Dict[str, Any]: """Index sha1s' content and store result. Args: id (bytes): content's identifier raw_content (bytes): associated raw content to content id Returns: dict: A dict, representing a content_license, with keys: - id (bytes): content's identifier (sha1) - license (bytes): license in bytes - path (bytes): path - indexer_configuration_id (int): tool used to compute the output """ assert isinstance(id, bytes) assert data is not None with write_to_temp( - filename=hashutil.hash_to_hex(id), # use the id as pathname - data=data, - working_directory=self.working_directory) as content_path: + filename=hashutil.hash_to_hex(id), # use the id as pathname + data=data, + working_directory=self.working_directory, + ) as content_path: properties = compute_license(path=content_path) - properties.update({ - 'id': id, - 'indexer_configuration_id': self.tool['id'], - }) + properties.update( + {"id": id, "indexer_configuration_id": self.tool["id"],} + ) return properties def persist_index_computations( - self, results: List[Dict], policy_update: str) -> Dict[str, int]: + self, results: List[Dict], policy_update: str + ) -> Dict[str, int]: """Persist the results in storage. Args: results: list of content_license dict with the following keys: - id (bytes): content's identifier (sha1) - license (bytes): license in bytes - path (bytes): path policy_update: either 'update-dups' or 'ignore-dups' to respectively update duplicates or ignore them """ return self.idx_storage.content_fossology_license_add( - results, conflict_update=(policy_update == 'update-dups')) + results, conflict_update=(policy_update == "update-dups") + ) -class FossologyLicenseIndexer( - MixinFossologyLicenseIndexer, ContentIndexer): +class FossologyLicenseIndexer(MixinFossologyLicenseIndexer, ContentIndexer): """Indexer in charge of: - filtering out content already indexed - reading content from objstorage per the content's id (sha1) - computing {license, encoding} from that content - store result in storage """ + def filter(self, ids): """Filter out known sha1s and return only missing ones. """ - yield from self.idx_storage.content_fossology_license_missing(( - { - 'id': sha1, - 'indexer_configuration_id': self.tool['id'], - } for sha1 in ids - )) + yield from self.idx_storage.content_fossology_license_missing( + ({"id": sha1, "indexer_configuration_id": self.tool["id"],} for sha1 in ids) + ) -class FossologyLicenseRangeIndexer( - MixinFossologyLicenseIndexer, ContentRangeIndexer): +class FossologyLicenseRangeIndexer(MixinFossologyLicenseIndexer, ContentRangeIndexer): """FossologyLicense Range Indexer working on range of content identifiers. - filters out the non textual content - (optionally) filters out content already indexed (cf :meth:`.indexed_contents_in_range`) - reads content from objstorage per the content's id (sha1) - computes {mimetype, encoding} from that content - stores result in storage """ + def indexed_contents_in_range(self, start, end): """Retrieve indexed content id within range [start, end]. Args: start (bytes): Starting bound from range identifier end (bytes): End range identifier Returns: dict: a dict with keys: - **ids** [bytes]: iterable of content ids within the range. - **next** (Optional[bytes]): The next range of sha1 starts at this sha1 if any """ return self.idx_storage.content_fossology_license_get_range( - start, end, self.tool['id']) + start, end, self.tool["id"] + ) diff --git a/swh/indexer/indexer.py b/swh/indexer/indexer.py index 3f48743..7e332a7 100644 --- a/swh/indexer/indexer.py +++ b/swh/indexer/indexer.py @@ -1,621 +1,621 @@ # Copyright (C) 2016-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import abc import os import logging import shutil import tempfile from contextlib import contextmanager -from typing import ( - Any, Dict, Iterator, List, Optional, Set, Tuple, Union -) +from typing import Any, Dict, Iterator, List, Optional, Set, Tuple, Union from swh.scheduler import CONFIG as SWH_CONFIG from swh.storage import get_storage from swh.core.config import SWHConfig from swh.objstorage import get_objstorage from swh.objstorage.exc import ObjNotFoundError from swh.indexer.storage import get_indexer_storage, INDEXER_CFG_KEY from swh.model import hashutil from swh.core import utils @contextmanager -def write_to_temp( - filename: str, data: bytes, working_directory: str) -> Iterator[str]: +def write_to_temp(filename: str, data: bytes, working_directory: str) -> Iterator[str]: """Write the sha1's content in a temporary file. Args: filename: one of sha1's many filenames data: the sha1's content to write in temporary file working_directory: the directory into which the file is written Returns: The path to the temporary file created. That file is filled in with the raw content's data. """ os.makedirs(working_directory, exist_ok=True) temp_dir = tempfile.mkdtemp(dir=working_directory) content_path = os.path.join(temp_dir, filename) - with open(content_path, 'wb') as f: + with open(content_path, "wb") as f: f.write(data) yield content_path shutil.rmtree(temp_dir) class BaseIndexer(SWHConfig, metaclass=abc.ABCMeta): """Base class for indexers to inherit from. The main entry point is the :func:`run` function which is in charge of triggering the computations on the batch dict/ids received. Indexers can: - filter out ids whose data has already been indexed. - retrieve ids data from storage or objstorage - index this data depending on the object and store the result in storage. To implement a new object type indexer, inherit from the BaseIndexer and implement indexing: :meth:`~BaseIndexer.run`: object_ids are different depending on object. For example: sha1 for content, sha1_git for revision, directory, release, and id for origin To implement a new concrete indexer, inherit from the object level classes: :class:`ContentIndexer`, :class:`RevisionIndexer`, :class:`OriginIndexer`. Then you need to implement the following functions: :meth:`~BaseIndexer.filter`: filter out data already indexed (in storage). :meth:`~BaseIndexer.index_object`: compute index on id with data (retrieved from the storage or the objstorage by the id key) and return the resulting index computation. :meth:`~BaseIndexer.persist_index_computations`: persist the results of multiple index computations in the storage. The new indexer implementation can also override the following functions: :meth:`~BaseIndexer.prepare`: Configuration preparation for the indexer. When overriding, this must call the `super().prepare()` instruction. :meth:`~BaseIndexer.check`: Configuration check for the indexer. When overriding, this must call the `super().check()` instruction. :meth:`~BaseIndexer.register_tools`: This should return a dict of the tool(s) to use when indexing or filtering. """ + results: List[Dict] - CONFIG = 'indexer/base' + CONFIG = "indexer/base" DEFAULT_CONFIG = { - INDEXER_CFG_KEY: ('dict', { - 'cls': 'remote', - 'args': { - 'url': 'http://localhost:5007/' - } - }), - 'storage': ('dict', { - 'cls': 'remote', - 'args': { - 'url': 'http://localhost:5002/', - } - }), - 'objstorage': ('dict', { - 'cls': 'remote', - 'args': { - 'url': 'http://localhost:5003/', - } - }) + INDEXER_CFG_KEY: ( + "dict", + {"cls": "remote", "args": {"url": "http://localhost:5007/"}}, + ), + "storage": ( + "dict", + {"cls": "remote", "args": {"url": "http://localhost:5002/",}}, + ), + "objstorage": ( + "dict", + {"cls": "remote", "args": {"url": "http://localhost:5003/",}}, + ), } ADDITIONAL_CONFIG = {} # type: Dict[str, Tuple[str, Any]] USE_TOOLS = True catch_exceptions = True """Prevents exceptions in `index()` from raising too high. Set to False in tests to properly catch all exceptions.""" scheduler: Any def __init__(self, config=None, **kw) -> None: """Prepare and check that the indexer is ready to run. """ super().__init__() if config is not None: self.config = config elif SWH_CONFIG: self.config = SWH_CONFIG.copy() else: - config_keys = ('base_filename', 'config_filename', - 'additional_configs', 'global_config') + config_keys = ( + "base_filename", + "config_filename", + "additional_configs", + "global_config", + ) config_args = {k: v for k, v in kw.items() if k in config_keys} if self.ADDITIONAL_CONFIG: - config_args.setdefault('additional_configs', []).append( - self.ADDITIONAL_CONFIG) + config_args.setdefault("additional_configs", []).append( + self.ADDITIONAL_CONFIG + ) self.config = self.parse_config_file(**config_args) self.prepare() self.check() - self.log.debug('%s: config=%s', self, self.config) + self.log.debug("%s: config=%s", self, self.config) def prepare(self) -> None: """Prepare the indexer's needed runtime configuration. Without this step, the indexer cannot possibly run. """ - config_storage = self.config.get('storage') + config_storage = self.config.get("storage") if config_storage: self.storage = get_storage(**config_storage) - objstorage = self.config['objstorage'] - self.objstorage = get_objstorage(objstorage['cls'], - objstorage['args']) + objstorage = self.config["objstorage"] + self.objstorage = get_objstorage(objstorage["cls"], objstorage["args"]) idx_storage = self.config[INDEXER_CFG_KEY] self.idx_storage = get_indexer_storage(**idx_storage) - _log = logging.getLogger('requests.packages.urllib3.connectionpool') + _log = logging.getLogger("requests.packages.urllib3.connectionpool") _log.setLevel(logging.WARN) - self.log = logging.getLogger('swh.indexer') + self.log = logging.getLogger("swh.indexer") if self.USE_TOOLS: - self.tools = list(self.register_tools( - self.config.get('tools', []))) + self.tools = list(self.register_tools(self.config.get("tools", []))) self.results = [] @property def tool(self) -> Dict: return self.tools[0] def check(self) -> None: """Check the indexer's configuration is ok before proceeding. If ok, does nothing. If not raise error. """ if self.USE_TOOLS and not self.tools: - raise ValueError('Tools %s is unknown, cannot continue' % - self.tools) + raise ValueError("Tools %s is unknown, cannot continue" % self.tools) def _prepare_tool(self, tool: Dict[str, Any]) -> Dict[str, Any]: """Prepare the tool dict to be compliant with the storage api. """ - return {'tool_%s' % key: value for key, value in tool.items()} + return {"tool_%s" % key: value for key, value in tool.items()} def register_tools( self, tools: Union[Dict[str, Any], List[Dict[str, Any]]] ) -> List[Dict[str, Any]]: """Permit to register tools to the storage. Add a sensible default which can be overridden if not sufficient. (For now, all indexers use only one tool) Expects the self.config['tools'] property to be set with one or more tools. Args: tools: Either a dict or a list of dict. Returns: list: List of dicts with additional id key. Raises: ValueError: if not a list nor a dict. """ if isinstance(tools, list): tools = list(map(self._prepare_tool, tools)) elif isinstance(tools, dict): tools = [self._prepare_tool(tools)] else: - raise ValueError('Configuration tool(s) must be a dict or list!') + raise ValueError("Configuration tool(s) must be a dict or list!") if tools: return self.idx_storage.indexer_configuration_add(tools) else: return [] - def index(self, id: bytes, data: Optional[bytes] = None, - **kwargs) -> Dict[str, Any]: + def index( + self, id: bytes, data: Optional[bytes] = None, **kwargs + ) -> Dict[str, Any]: """Index computation for the id and associated raw data. Args: id: identifier data: id's data from storage or objstorage depending on object type Returns: dict: a dict that makes sense for the :meth:`.persist_index_computations` method. """ raise NotImplementedError() def filter(self, ids: List[bytes]) -> Iterator[bytes]: """Filter missing ids for that particular indexer. Args: ids: list of ids Yields: iterator of missing ids """ yield from ids @abc.abstractmethod - def persist_index_computations( - self, results, policy_update) -> Dict[str, int]: + def persist_index_computations(self, results, policy_update) -> Dict[str, int]: """Persist the computation resulting from the index. Args: results ([result]): List of results. One result is the result of the index function. policy_update ([str]): either 'update-dups' or 'ignore-dups' to respectively update duplicates or ignore them Returns: a summary dict of what has been inserted in the storage """ return {} class ContentIndexer(BaseIndexer): """A content indexer working on a list of ids directly. To work on indexer range, use the :class:`ContentRangeIndexer` instead. Note: :class:`ContentIndexer` is not an instantiable object. To use it, one should inherit from this class and override the methods mentioned in the :class:`BaseIndexer` class. """ - def run(self, ids: Union[List[bytes], bytes, str], policy_update: str, - **kwargs) -> Dict: + + def run( + self, ids: Union[List[bytes], bytes, str], policy_update: str, **kwargs + ) -> Dict: """Given a list of ids: - retrieve the content from the storage - execute the indexing computations - store the results (according to policy_update) Args: ids (Iterable[Union[bytes, str]]): sha1's identifier list policy_update (str): either 'update-dups' or 'ignore-dups' to respectively update duplicates or ignore them **kwargs: passed to the `index` method Returns: A summary Dict of the task's status """ - status = 'uneventful' - sha1s = [hashutil.hash_to_bytes(id_) if isinstance(id_, str) else id_ - for id_ in ids] + status = "uneventful" + sha1s = [ + hashutil.hash_to_bytes(id_) if isinstance(id_, str) else id_ for id_ in ids + ] results = [] summary: Dict = {} try: for sha1 in sha1s: try: raw_content = self.objstorage.get(sha1) except ObjNotFoundError: - self.log.warning('Content %s not found in objstorage' % - hashutil.hash_to_hex(sha1)) + self.log.warning( + "Content %s not found in objstorage" + % hashutil.hash_to_hex(sha1) + ) continue res = self.index(sha1, raw_content, **kwargs) if res: # If no results, skip it results.append(res) - status = 'eventful' + status = "eventful" summary = self.persist_index_computations(results, policy_update) self.results = results except Exception: if not self.catch_exceptions: raise - self.log.exception( - 'Problem when reading contents metadata.') - status = 'failed' + self.log.exception("Problem when reading contents metadata.") + status = "failed" finally: - summary['status'] = status + summary["status"] = status return summary class ContentRangeIndexer(BaseIndexer): """A content range indexer. This expects as input a range of ids to index. To work on a list of ids, use the :class:`ContentIndexer` instead. Note: :class:`ContentRangeIndexer` is not an instantiable object. To use it, one should inherit from this class and override the methods mentioned in the :class:`BaseIndexer` class. """ + @abc.abstractmethod - def indexed_contents_in_range( - self, start: bytes, end: bytes - ) -> Any: + def indexed_contents_in_range(self, start: bytes, end: bytes) -> Any: """Retrieve indexed contents within range [start, end]. Args: start: Starting bound from range identifier end: End range identifier Yields: bytes: Content identifier present in the range ``[start, end]`` """ pass def _list_contents_to_index( self, start: bytes, end: bytes, indexed: Set[bytes] ) -> Iterator[bytes]: """Compute from storage the new contents to index in the range [start, end]. The already indexed contents are skipped. Args: start: Starting bound from range identifier end: End range identifier indexed: Set of content already indexed. Yields: bytes: Identifier of contents to index. """ if not isinstance(start, bytes) or not isinstance(end, bytes): - raise TypeError('identifiers must be bytes, not %r and %r.' % - (start, end)) + raise TypeError("identifiers must be bytes, not %r and %r." % (start, end)) while start: result = self.storage.content_get_range(start, end) - contents = result['contents'] + contents = result["contents"] for c in contents: - _id = hashutil.hash_to_bytes(c['sha1']) + _id = hashutil.hash_to_bytes(c["sha1"]) if _id in indexed: continue yield _id - start = result['next'] + start = result["next"] def _index_contents( self, start: bytes, end: bytes, indexed: Set[bytes], **kwargs: Any ) -> Iterator[Dict]: """Index the contents from within range [start, end] Args: start: Starting bound from range identifier end: End range identifier indexed: Set of content already indexed. Yields: dict: Data indexed to persist using the indexer storage """ for sha1 in self._list_contents_to_index(start, end, indexed): try: raw_content = self.objstorage.get(sha1) except ObjNotFoundError: - self.log.warning('Content %s not found in objstorage' % - hashutil.hash_to_hex(sha1)) + self.log.warning( + "Content %s not found in objstorage" % hashutil.hash_to_hex(sha1) + ) continue res = self.index(sha1, raw_content, **kwargs) if res: - if not isinstance(res['id'], bytes): + if not isinstance(res["id"], bytes): raise TypeError( - '%r.index should return ids as bytes, not %r' % - (self.__class__.__name__, res['id'])) + "%r.index should return ids as bytes, not %r" + % (self.__class__.__name__, res["id"]) + ) yield res def _index_with_skipping_already_done( - self, start: bytes, end: bytes) -> Iterator[Dict]: + self, start: bytes, end: bytes + ) -> Iterator[Dict]: """Index not already indexed contents in range [start, end]. Args: start: Starting range identifier end: Ending range identifier Yields: dict: Content identifier present in the range ``[start, end]`` which are not already indexed. """ while start: indexed_page = self.indexed_contents_in_range(start, end) - contents = indexed_page['ids'] + contents = indexed_page["ids"] _end = contents[-1] if contents else end - yield from self._index_contents( - start, _end, contents) - start = indexed_page['next'] - - def run(self, start: Union[bytes, str], end: Union[bytes, str], - skip_existing: bool = True, **kwargs) -> Dict: + yield from self._index_contents(start, _end, contents) + start = indexed_page["next"] + + def run( + self, + start: Union[bytes, str], + end: Union[bytes, str], + skip_existing: bool = True, + **kwargs + ) -> Dict: """Given a range of content ids, compute the indexing computations on the contents within. Either the indexer is incremental (filter out existing computed data) or not (compute everything from scratch). Args: start: Starting range identifier end: Ending range identifier skip_existing: Skip existing indexed data (default) or not **kwargs: passed to the `index` method Returns: A dict with the task's status """ - status = 'uneventful' + status = "uneventful" summary: Dict[str, Any] = {} count = 0 try: - range_start = hashutil.hash_to_bytes(start) \ - if isinstance(start, str) else start - range_end = hashutil.hash_to_bytes(end) \ - if isinstance(end, str) else end + range_start = ( + hashutil.hash_to_bytes(start) if isinstance(start, str) else start + ) + range_end = hashutil.hash_to_bytes(end) if isinstance(end, str) else end if skip_existing: - gen = self._index_with_skipping_already_done( - range_start, range_end) + gen = self._index_with_skipping_already_done(range_start, range_end) else: - gen = self._index_contents( - range_start, range_end, indexed=set([])) + gen = self._index_contents(range_start, range_end, indexed=set([])) count_object_added_key: Optional[str] = None - for contents in utils.grouper( - gen, n=self.config['write_batch_size']): + for contents in utils.grouper(gen, n=self.config["write_batch_size"]): res = self.persist_index_computations( - contents, policy_update='update-dups') + contents, policy_update="update-dups" + ) if not count_object_added_key: count_object_added_key = list(res.keys())[0] count += res[count_object_added_key] if count > 0: - status = 'eventful' + status = "eventful" except Exception: if not self.catch_exceptions: raise - self.log.exception( - 'Problem when computing metadata.') - status = 'failed' + self.log.exception("Problem when computing metadata.") + status = "failed" finally: - summary['status'] = status + summary["status"] = status if count > 0 and count_object_added_key: summary[count_object_added_key] = count return summary class OriginIndexer(BaseIndexer): """An object type indexer, inherits from the :class:`BaseIndexer` and implements Origin indexing using the run method Note: the :class:`OriginIndexer` is not an instantiable object. To use it in another context one should inherit from this class and override the methods mentioned in the :class:`BaseIndexer` class. """ - def run(self, origin_urls: List[str], - policy_update: str = 'update-dups', **kwargs) -> Dict: + + def run( + self, origin_urls: List[str], policy_update: str = "update-dups", **kwargs + ) -> Dict: """Given a list of origin urls: - retrieve origins from storage - execute the indexing computations - store the results (according to policy_update) Args: origin_urls: list of origin urls. policy_update: either 'update-dups' or 'ignore-dups' to respectively update duplicates (default) or ignore them **kwargs: passed to the `index` method """ summary: Dict[str, Any] = {} - status = 'uneventful' + status = "uneventful" results = self.index_list(origin_urls, **kwargs) - summary_persist = self.persist_index_computations( - results, policy_update) + summary_persist = self.persist_index_computations(results, policy_update) self.results = results if summary_persist: for value in summary_persist.values(): if value > 0: - status = 'eventful' + status = "eventful" summary.update(summary_persist) - summary['status'] = status + summary["status"] = status return summary def index_list(self, origins: List[Any], **kwargs: Any) -> List[Dict]: results = [] for origin in origins: try: res = self.index(origin, **kwargs) if res: # If no results, skip it results.append(res) except Exception: if not self.catch_exceptions: raise - self.log.exception( - 'Problem when processing origin %s', - origin['id']) + self.log.exception("Problem when processing origin %s", origin["id"]) return results class RevisionIndexer(BaseIndexer): """An object type indexer, inherits from the :class:`BaseIndexer` and implements Revision indexing using the run method Note: the :class:`RevisionIndexer` is not an instantiable object. To use it in another context one should inherit from this class and override the methods mentioned in the :class:`BaseIndexer` class. """ + def run(self, ids: Union[str, bytes], policy_update: str) -> Dict: """Given a list of sha1_gits: - retrieve revisions from storage - execute the indexing computations - store the results (according to policy_update) Args: ids: sha1_git's identifier list policy_update: either 'update-dups' or 'ignore-dups' to respectively update duplicates or ignore them """ summary: Dict[str, Any] = {} - status = 'uneventful' + status = "uneventful" results = [] revs = self.storage.revision_get( - hashutil.hash_to_bytes(id_) if isinstance(id_, str) else id_ - for id_ in ids) + hashutil.hash_to_bytes(id_) if isinstance(id_, str) else id_ for id_ in ids + ) for rev in revs: if not rev: - self.log.warning('Revisions %s not found in storage' % - list(map(hashutil.hash_to_hex, ids))) + self.log.warning( + "Revisions %s not found in storage" + % list(map(hashutil.hash_to_hex, ids)) + ) continue try: res = self.index(rev) if res: # If no results, skip it results.append(res) except Exception: if not self.catch_exceptions: raise - self.log.exception( - 'Problem when processing revision') - status = 'failed' - summary_persist = self.persist_index_computations( - results, policy_update) + self.log.exception("Problem when processing revision") + status = "failed" + summary_persist = self.persist_index_computations(results, policy_update) if summary_persist: for value in summary_persist.values(): if value > 0: - status = 'eventful' + status = "eventful" summary.update(summary_persist) self.results = results - summary['status'] = status + summary["status"] = status return summary diff --git a/swh/indexer/journal_client.py b/swh/indexer/journal_client.py index 642ee1c..9c4a75c 100644 --- a/swh/indexer/journal_client.py +++ b/swh/indexer/journal_client.py @@ -1,43 +1,45 @@ # Copyright (C) 2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import logging from swh.core.utils import grouper from swh.scheduler.utils import create_task_dict MAX_ORIGINS_PER_TASK = 100 def process_journal_objects(messages, *, scheduler, task_names): """Worker function for `JournalClient.process(worker_fn)`, after currification of `scheduler` and `task_names`.""" - assert set(messages) == {'origin_visit'}, set(messages) - process_origin_visits(messages['origin_visit'], scheduler, task_names) + assert set(messages) == {"origin_visit"}, set(messages) + process_origin_visits(messages["origin_visit"], scheduler, task_names) def process_origin_visits(visits, scheduler, task_names): task_dicts = [] - logging.debug('processing origin visits %r', visits) - if task_names.get('origin_metadata'): - visits = [visit for visit in visits if visit['status'] == 'full'] + logging.debug("processing origin visits %r", visits) + if task_names.get("origin_metadata"): + visits = [visit for visit in visits if visit["status"] == "full"] visit_batches = grouper(visits, MAX_ORIGINS_PER_TASK) for visit_batch in visit_batches: visit_urls = [] for visit in visit_batch: - if isinstance(visit['origin'], str): - visit_urls.append(visit['origin']) + if isinstance(visit["origin"], str): + visit_urls.append(visit["origin"]) else: - visit_urls.append(visit['origin']['url']) - task_dicts.append(create_task_dict( - task_names['origin_metadata'], - 'oneshot', - visit_urls, - policy_update='update-dups', - )) + visit_urls.append(visit["origin"]["url"]) + task_dicts.append( + create_task_dict( + task_names["origin_metadata"], + "oneshot", + visit_urls, + policy_update="update-dups", + ) + ) if task_dicts: scheduler.create_tasks(task_dicts) diff --git a/swh/indexer/metadata.py b/swh/indexer/metadata.py index 339e3cf..0fdb0db 100644 --- a/swh/indexer/metadata.py +++ b/swh/indexer/metadata.py @@ -1,377 +1,383 @@ # Copyright (C) 2017-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from copy import deepcopy from typing import Any, Callable, Dict, Iterator, List, Tuple from swh.core.utils import grouper from swh.indexer.codemeta import merge_documents from swh.indexer.indexer import ContentIndexer, RevisionIndexer, OriginIndexer from swh.indexer.origin_head import OriginHeadIndexer from swh.indexer.metadata_dictionary import MAPPINGS from swh.indexer.metadata_detector import detect_metadata from swh.indexer.storage import INDEXER_CFG_KEY from swh.model import hashutil REVISION_GET_BATCH_SIZE = 10 ORIGIN_GET_BATCH_SIZE = 10 def call_with_batches( - f: Callable[[List[Dict[str, Any]]], Dict['str', Any]], - args: List[Dict[str, str]], batch_size: int + f: Callable[[List[Dict[str, Any]]], Dict["str", Any]], + args: List[Dict[str, str]], + batch_size: int, ) -> Iterator[str]: """Calls a function with batches of args, and concatenates the results. """ groups = grouper(args, batch_size) for group in groups: yield from f(list(group)) class ContentMetadataIndexer(ContentIndexer): """Content-level indexer This indexer is in charge of: - filtering out content already indexed in content_metadata - reading content from objstorage with the content's id sha1 - computing metadata by given context - using the metadata_dictionary as the 'swh-metadata-translator' tool - store result in content_metadata table """ + def filter(self, ids): """Filter out known sha1s and return only missing ones. """ - yield from self.idx_storage.content_metadata_missing(( - { - 'id': sha1, - 'indexer_configuration_id': self.tool['id'], - } for sha1 in ids - )) - - def index(self, id, data, log_suffix='unknown revision'): + yield from self.idx_storage.content_metadata_missing( + ({"id": sha1, "indexer_configuration_id": self.tool["id"],} for sha1 in ids) + ) + + def index(self, id, data, log_suffix="unknown revision"): """Index sha1s' content and store result. Args: id (bytes): content's identifier data (bytes): raw content in bytes Returns: dict: dictionary representing a content_metadata. If the translation wasn't successful the metadata keys will be returned as None """ result = { - 'id': id, - 'indexer_configuration_id': self.tool['id'], - 'metadata': None + "id": id, + "indexer_configuration_id": self.tool["id"], + "metadata": None, } try: - mapping_name = self.tool['tool_configuration']['context'] - log_suffix += ', content_id=%s' % hashutil.hash_to_hex(id) - result['metadata'] = \ - MAPPINGS[mapping_name](log_suffix).translate(data) + mapping_name = self.tool["tool_configuration"]["context"] + log_suffix += ", content_id=%s" % hashutil.hash_to_hex(id) + result["metadata"] = MAPPINGS[mapping_name](log_suffix).translate(data) except Exception: self.log.exception( "Problem during metadata translation " - "for content %s" % hashutil.hash_to_hex(id)) - if result['metadata'] is None: + "for content %s" % hashutil.hash_to_hex(id) + ) + if result["metadata"] is None: return None return result def persist_index_computations( self, results: List[Dict], policy_update: str ) -> Dict[str, int]: """Persist the results in storage. Args: results: list of content_metadata, dict with the following keys: - id (bytes): content's identifier (sha1) - metadata (jsonb): detected metadata policy_update: either 'update-dups' or 'ignore-dups' to respectively update duplicates or ignore them """ return self.idx_storage.content_metadata_add( - results, conflict_update=(policy_update == 'update-dups')) + results, conflict_update=(policy_update == "update-dups") + ) class RevisionMetadataIndexer(RevisionIndexer): """Revision-level indexer This indexer is in charge of: - filtering revisions already indexed in revision_intrinsic_metadata table with defined computation tool - retrieve all entry_files in root directory - use metadata_detector for file_names containing metadata - compute metadata translation if necessary and possible (depends on tool) - send sha1s to content indexing if possible - store the results for revision """ + ADDITIONAL_CONFIG = { - 'tools': ('dict', { - 'name': 'swh-metadata-detector', - 'version': '0.0.2', - 'configuration': { - }, - }), + "tools": ( + "dict", + {"name": "swh-metadata-detector", "version": "0.0.2", "configuration": {},}, + ), } def filter(self, sha1_gits): """Filter out known sha1s and return only missing ones. """ - yield from self.idx_storage.revision_intrinsic_metadata_missing(( - { - 'id': sha1_git, - 'indexer_configuration_id': self.tool['id'], - } for sha1_git in sha1_gits - )) + yield from self.idx_storage.revision_intrinsic_metadata_missing( + ( + {"id": sha1_git, "indexer_configuration_id": self.tool["id"],} + for sha1_git in sha1_gits + ) + ) def index(self, rev): """Index rev by processing it and organizing result. use metadata_detector to iterate on filenames - if one filename detected -> sends file to content indexer - if multiple file detected -> translation needed at revision level Args: rev (dict): revision artifact from storage Returns: dict: dictionary representing a revision_intrinsic_metadata, with keys: - id (str): rev's identifier (sha1_git) - indexer_configuration_id (bytes): tool used - metadata: dict of retrieved metadata """ result = { - 'id': rev['id'], - 'indexer_configuration_id': self.tool['id'], - 'mappings': None, - 'metadata': None + "id": rev["id"], + "indexer_configuration_id": self.tool["id"], + "mappings": None, + "metadata": None, } try: - root_dir = rev['directory'] + root_dir = rev["directory"] dir_ls = list(self.storage.directory_ls(root_dir, recursive=False)) - if [entry['type'] for entry in dir_ls] == ['dir']: + if [entry["type"] for entry in dir_ls] == ["dir"]: # If the root is just a single directory, recurse into it # eg. PyPI packages, GNU tarballs - subdir = dir_ls[0]['target'] + subdir = dir_ls[0]["target"] dir_ls = self.storage.directory_ls(subdir, recursive=False) - files = [entry for entry in dir_ls if entry['type'] == 'file'] + files = [entry for entry in dir_ls if entry["type"] == "file"] detected_files = detect_metadata(files) (mappings, metadata) = self.translate_revision_intrinsic_metadata( detected_files, - log_suffix='revision=%s' % hashutil.hash_to_hex(rev['id'])) - result['mappings'] = mappings - result['metadata'] = metadata + log_suffix="revision=%s" % hashutil.hash_to_hex(rev["id"]), + ) + result["mappings"] = mappings + result["metadata"] = metadata except Exception as e: - self.log.exception( - 'Problem when indexing rev: %r', e) + self.log.exception("Problem when indexing rev: %r", e) return result def persist_index_computations( self, results: List[Dict], policy_update: str ) -> Dict[str, int]: """Persist the results in storage. Args: results: list of content_mimetype, dict with the following keys: - id (bytes): content's identifier (sha1) - mimetype (bytes): mimetype in bytes - encoding (bytes): encoding in bytes policy_update: either 'update-dups' or 'ignore-dups' to respectively update duplicates or ignore them """ # TODO: add functions in storage to keep data in # revision_intrinsic_metadata return self.idx_storage.revision_intrinsic_metadata_add( - results, conflict_update=(policy_update == 'update-dups')) + results, conflict_update=(policy_update == "update-dups") + ) def translate_revision_intrinsic_metadata( self, detected_files: Dict[str, List[Any]], log_suffix: str ) -> Tuple[List[Any], List[Any]]: """ Determine plan of action to translate metadata when containing one or multiple detected files: Args: detected_files: dictionary mapping context names (e.g., "npm", "authors") to list of sha1 Returns: (List[str], dict): list of mappings used and dict with translated metadata according to the CodeMeta vocabulary """ used_mappings = [MAPPINGS[context].name for context in detected_files] metadata = [] tool = { - 'name': 'swh-metadata-translator', - 'version': '0.0.2', - 'configuration': { - }, - } + "name": "swh-metadata-translator", + "version": "0.0.2", + "configuration": {}, + } # TODO: iterate on each context, on each file # -> get raw_contents # -> translate each content - config = { - k: self.config[k] - for k in [INDEXER_CFG_KEY, 'objstorage', 'storage'] - } - config['tools'] = [tool] + config = {k: self.config[k] for k in [INDEXER_CFG_KEY, "objstorage", "storage"]} + config["tools"] = [tool] for context in detected_files.keys(): cfg = deepcopy(config) - cfg['tools'][0]['configuration']['context'] = context + cfg["tools"][0]["configuration"]["context"] = context c_metadata_indexer = ContentMetadataIndexer(config=cfg) # sha1s that are in content_metadata table sha1s_in_storage = [] metadata_generator = self.idx_storage.content_metadata_get( - detected_files[context]) + detected_files[context] + ) for c in metadata_generator: # extracting metadata - sha1 = c['id'] + sha1 = c["id"] sha1s_in_storage.append(sha1) - local_metadata = c['metadata'] + local_metadata = c["metadata"] # local metadata is aggregated if local_metadata: metadata.append(local_metadata) - sha1s_filtered = [item for item in detected_files[context] - if item not in sha1s_in_storage] + sha1s_filtered = [ + item for item in detected_files[context] if item not in sha1s_in_storage + ] if sha1s_filtered: # content indexing try: - c_metadata_indexer.run(sha1s_filtered, - policy_update='ignore-dups', - log_suffix=log_suffix) + c_metadata_indexer.run( + sha1s_filtered, + policy_update="ignore-dups", + log_suffix=log_suffix, + ) # on the fly possibility: for result in c_metadata_indexer.results: - local_metadata = result['metadata'] + local_metadata = result["metadata"] metadata.append(local_metadata) except Exception: - self.log.exception( - "Exception while indexing metadata on contents") + self.log.exception("Exception while indexing metadata on contents") metadata = merge_documents(metadata) return (used_mappings, metadata) class OriginMetadataIndexer(OriginIndexer): ADDITIONAL_CONFIG = RevisionMetadataIndexer.ADDITIONAL_CONFIG USE_TOOLS = False def __init__(self, config=None, **kwargs) -> None: super().__init__(config=config, **kwargs) self.origin_head_indexer = OriginHeadIndexer(config=config) self.revision_metadata_indexer = RevisionMetadataIndexer(config=config) def index_list(self, origin_urls, **kwargs): head_rev_ids = [] origins_with_head = [] - origins = list(call_with_batches( - self.storage.origin_get, - [{'url': url} for url in origin_urls], ORIGIN_GET_BATCH_SIZE)) + origins = list( + call_with_batches( + self.storage.origin_get, + [{"url": url} for url in origin_urls], + ORIGIN_GET_BATCH_SIZE, + ) + ) for origin in origins: if origin is None: continue - head_result = self.origin_head_indexer.index(origin['url']) + head_result = self.origin_head_indexer.index(origin["url"]) if head_result: origins_with_head.append(origin) - head_rev_ids.append(head_result['revision_id']) + head_rev_ids.append(head_result["revision_id"]) - head_revs = list(call_with_batches( - self.storage.revision_get, - head_rev_ids, REVISION_GET_BATCH_SIZE)) + head_revs = list( + call_with_batches( + self.storage.revision_get, head_rev_ids, REVISION_GET_BATCH_SIZE + ) + ) assert len(head_revs) == len(head_rev_ids) results = [] for (origin, rev) in zip(origins_with_head, head_revs): if not rev: - self.log.warning('Missing head revision of origin %r', - origin['url']) + self.log.warning("Missing head revision of origin %r", origin["url"]) continue rev_metadata = self.revision_metadata_indexer.index(rev) orig_metadata = { - 'from_revision': rev_metadata['id'], - 'id': origin['url'], - 'metadata': rev_metadata['metadata'], - 'mappings': rev_metadata['mappings'], - 'indexer_configuration_id': - rev_metadata['indexer_configuration_id'], + "from_revision": rev_metadata["id"], + "id": origin["url"], + "metadata": rev_metadata["metadata"], + "mappings": rev_metadata["mappings"], + "indexer_configuration_id": rev_metadata["indexer_configuration_id"], } results.append((orig_metadata, rev_metadata)) return results def persist_index_computations( self, results: List[Dict], policy_update: str ) -> Dict[str, int]: - conflict_update = (policy_update == 'update-dups') + conflict_update = policy_update == "update-dups" # Deduplicate revisions rev_metadata: List[Any] = [] orig_metadata: List[Any] = [] revs_to_delete: List[Any] = [] origs_to_delete: List[Any] = [] summary: Dict = {} for (orig_item, rev_item) in results: - assert rev_item['metadata'] == orig_item['metadata'] - if not rev_item['metadata'] or \ - rev_item['metadata'].keys() <= {'@context'}: + assert rev_item["metadata"] == orig_item["metadata"] + if not rev_item["metadata"] or rev_item["metadata"].keys() <= {"@context"}: # If we didn't find any metadata, don't store a DB record # (and delete existing ones, if any) if rev_item not in revs_to_delete: revs_to_delete.append(rev_item) if orig_item not in origs_to_delete: origs_to_delete.append(orig_item) else: if rev_item not in rev_metadata: rev_metadata.append(rev_item) if orig_item not in orig_metadata: orig_metadata.append(orig_item) if rev_metadata: summary_rev = self.idx_storage.revision_intrinsic_metadata_add( - rev_metadata, conflict_update=conflict_update) + rev_metadata, conflict_update=conflict_update + ) summary.update(summary_rev) if orig_metadata: summary_ori = self.idx_storage.origin_intrinsic_metadata_add( - orig_metadata, conflict_update=conflict_update) + orig_metadata, conflict_update=conflict_update + ) summary.update(summary_ori) # revs_to_delete should always be empty unless we changed a mapping # to detect less files or less content. # However, origs_to_delete may be empty whenever an upstream deletes # a metadata file. if origs_to_delete: summary_ori = self.idx_storage.origin_intrinsic_metadata_delete( - origs_to_delete) + origs_to_delete + ) summary.update(summary_ori) if revs_to_delete: summary_rev = self.idx_storage.revision_intrinsic_metadata_delete( - revs_to_delete) + revs_to_delete + ) summary.update(summary_rev) return summary diff --git a/swh/indexer/metadata_dictionary/__init__.py b/swh/indexer/metadata_dictionary/__init__.py index 107a8b3..9be3507 100644 --- a/swh/indexer/metadata_dictionary/__init__.py +++ b/swh/indexer/metadata_dictionary/__init__.py @@ -1,38 +1,39 @@ import collections import click from . import maven, npm, codemeta, python, ruby MAPPINGS = { - 'CodemetaMapping': codemeta.CodemetaMapping, - 'MavenMapping': maven.MavenMapping, - 'NpmMapping': npm.NpmMapping, - 'PythonPkginfoMapping': python.PythonPkginfoMapping, - 'GemspecMapping': ruby.GemspecMapping, + "CodemetaMapping": codemeta.CodemetaMapping, + "MavenMapping": maven.MavenMapping, + "NpmMapping": npm.NpmMapping, + "PythonPkginfoMapping": python.PythonPkginfoMapping, + "GemspecMapping": ruby.GemspecMapping, } def list_terms(): """Returns a dictionary with all supported CodeMeta terms as keys, and the mappings that support each of them as values.""" d = collections.defaultdict(set) for mapping in MAPPINGS.values(): for term in mapping.supported_terms(): d[term].add(mapping) return d @click.command() -@click.argument('mapping_name') -@click.argument('file_name') +@click.argument("mapping_name") +@click.argument("file_name") def main(mapping_name, file_name): from pprint import pprint - with open(file_name, 'rb') as fd: + + with open(file_name, "rb") as fd: file_content = fd.read() res = MAPPINGS[mapping_name]().translate(file_content) pprint(res) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/swh/indexer/metadata_dictionary/base.py b/swh/indexer/metadata_dictionary/base.py index e01276e..9ca9d89 100644 --- a/swh/indexer/metadata_dictionary/base.py +++ b/swh/indexer/metadata_dictionary/base.py @@ -1,179 +1,185 @@ # Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import abc import json import logging from typing import List from swh.indexer.codemeta import SCHEMA_URI from swh.indexer.codemeta import compact, merge_values class BaseMapping(metaclass=abc.ABCMeta): """Base class for mappings to inherit from To implement a new mapping: - inherit this class - override translate function """ - def __init__(self, log_suffix=''): + + def __init__(self, log_suffix=""): self.log_suffix = log_suffix - self.log = logging.getLogger('%s.%s' % ( - self.__class__.__module__, - self.__class__.__name__)) + self.log = logging.getLogger( + "%s.%s" % (self.__class__.__module__, self.__class__.__name__) + ) @property @abc.abstractmethod def name(self): """A name of this mapping, used as an identifier in the indexer storage.""" pass @classmethod @abc.abstractmethod def detect_metadata_files(cls, files): """ Detects files potentially containing metadata Args: file_entries (list): list of files Returns: list: list of sha1 (possibly empty) """ pass @abc.abstractmethod def translate(self, file_content): pass def normalize_translation(self, metadata): return compact(metadata) class SingleFileMapping(BaseMapping): """Base class for all mappings that use a single file as input.""" @property @abc.abstractmethod def filename(self): """The .json file to extract metadata from.""" pass @classmethod def detect_metadata_files(cls, file_entries): for entry in file_entries: - if entry['name'] == cls.filename: - return [entry['sha1']] + if entry["name"] == cls.filename: + return [entry["sha1"]] return [] class DictMapping(BaseMapping): """Base class for mappings that take as input a file that is mostly a key-value store (eg. a shallow JSON dict).""" string_fields = [] # type: List[str] - '''List of fields that are simple strings, and don't need any - normalization.''' + """List of fields that are simple strings, and don't need any + normalization.""" @property @abc.abstractmethod def mapping(self): """A translation dict to map dict keys into a canonical name.""" pass @staticmethod def _normalize_method_name(name): - return name.replace('-', '_') + return name.replace("-", "_") @classmethod def supported_terms(cls): return { - term for (key, term) in cls.mapping.items() + term + for (key, term) in cls.mapping.items() if key in cls.string_fields - or hasattr(cls, 'translate_' + cls._normalize_method_name(key)) - or hasattr(cls, 'normalize_' + cls._normalize_method_name(key))} + or hasattr(cls, "translate_" + cls._normalize_method_name(key)) + or hasattr(cls, "normalize_" + cls._normalize_method_name(key)) + } def _translate_dict(self, content_dict, *, normalize=True): """ Translates content by parsing content from a dict object and translating with the appropriate mapping Args: content_dict (dict): content dict to translate Returns: dict: translated metadata in json-friendly form needed for the indexer """ - translated_metadata = {'@type': SCHEMA_URI + 'SoftwareSourceCode'} + translated_metadata = {"@type": SCHEMA_URI + "SoftwareSourceCode"} for k, v in content_dict.items(): # First, check if there is a specific translation # method for this key translation_method = getattr( - self, 'translate_' + self._normalize_method_name(k), None) + self, "translate_" + self._normalize_method_name(k), None + ) if translation_method: translation_method(translated_metadata, v) elif k in self.mapping: # if there is no method, but the key is known from the # crosswalk table codemeta_key = self.mapping[k] # if there is a normalization method, use it on the value normalization_method = getattr( - self, 'normalize_' + self._normalize_method_name(k), None) + self, "normalize_" + self._normalize_method_name(k), None + ) if normalization_method: v = normalization_method(v) elif k in self.string_fields and isinstance(v, str): pass elif k in self.string_fields and isinstance(v, list): v = [x for x in v if isinstance(x, str)] else: continue # set the translation metadata with the normalized value if codemeta_key in translated_metadata: translated_metadata[codemeta_key] = merge_values( - translated_metadata[codemeta_key], v) + translated_metadata[codemeta_key], v + ) else: translated_metadata[codemeta_key] = v if normalize: return self.normalize_translation(translated_metadata) else: return translated_metadata class JsonMapping(DictMapping, SingleFileMapping): """Base class for all mappings that use a JSON file as input.""" def translate(self, raw_content): """ Translates content by parsing content from a bytestring containing json data and translating with the appropriate mapping Args: raw_content (bytes): raw content to translate Returns: dict: translated metadata in json-friendly form needed for the indexer """ try: raw_content = raw_content.decode() except UnicodeDecodeError: - self.log.warning('Error unidecoding from %s', self.log_suffix) + self.log.warning("Error unidecoding from %s", self.log_suffix) return try: content_dict = json.loads(raw_content) except json.JSONDecodeError: - self.log.warning('Error unjsoning from %s', self.log_suffix) + self.log.warning("Error unjsoning from %s", self.log_suffix) return if isinstance(content_dict, dict): return self._translate_dict(content_dict) diff --git a/swh/indexer/metadata_dictionary/codemeta.py b/swh/indexer/metadata_dictionary/codemeta.py index bfb336c..d04c0e3 100644 --- a/swh/indexer/metadata_dictionary/codemeta.py +++ b/swh/indexer/metadata_dictionary/codemeta.py @@ -1,30 +1,30 @@ # Copyright (C) 2018-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import json from swh.indexer.codemeta import CODEMETA_TERMS from swh.indexer.codemeta import expand from .base import SingleFileMapping class CodemetaMapping(SingleFileMapping): """ dedicated class for CodeMeta (codemeta.json) mapping and translation """ - name = 'codemeta' - filename = b'codemeta.json' + + name = "codemeta" + filename = b"codemeta.json" string_fields = None @classmethod def supported_terms(cls): - return [term for term in CODEMETA_TERMS if not term.startswith('@')] + return [term for term in CODEMETA_TERMS if not term.startswith("@")] def translate(self, content): try: - return self.normalize_translation(expand( - json.loads(content.decode()))) + return self.normalize_translation(expand(json.loads(content.decode()))) except Exception: return None diff --git a/swh/indexer/metadata_dictionary/maven.py b/swh/indexer/metadata_dictionary/maven.py index 38592ba..40c236a 100644 --- a/swh/indexer/metadata_dictionary/maven.py +++ b/swh/indexer/metadata_dictionary/maven.py @@ -1,154 +1,157 @@ # Copyright (C) 2018-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import os import xml.parsers.expat import xmltodict from swh.indexer.codemeta import CROSSWALK_TABLE, SCHEMA_URI from .base import DictMapping, SingleFileMapping class MavenMapping(DictMapping, SingleFileMapping): """ dedicated class for Maven (pom.xml) mapping and translation """ - name = 'maven' - filename = b'pom.xml' - mapping = CROSSWALK_TABLE['Java (Maven)'] - string_fields = ['name', 'version', 'description', 'email'] + + name = "maven" + filename = b"pom.xml" + mapping = CROSSWALK_TABLE["Java (Maven)"] + string_fields = ["name", "version", "description", "email"] def translate(self, content): try: - d = xmltodict.parse(content).get('project') or {} + d = xmltodict.parse(content).get("project") or {} except xml.parsers.expat.ExpatError: - self.log.warning('Error parsing XML from %s', self.log_suffix) + self.log.warning("Error parsing XML from %s", self.log_suffix) return None except UnicodeDecodeError: - self.log.warning('Error unidecoding XML from %s', self.log_suffix) + self.log.warning("Error unidecoding XML from %s", self.log_suffix) return None except (LookupError, ValueError): # unknown encoding or multi-byte encoding - self.log.warning('Error detecting XML encoding from %s', - self.log_suffix) + self.log.warning("Error detecting XML encoding from %s", self.log_suffix) return None metadata = self._translate_dict(d, normalize=False) - metadata[SCHEMA_URI+'codeRepository'] = self.parse_repositories(d) - metadata[SCHEMA_URI+'license'] = self.parse_licenses(d) + metadata[SCHEMA_URI + "codeRepository"] = self.parse_repositories(d) + metadata[SCHEMA_URI + "license"] = self.parse_licenses(d) return self.normalize_translation(metadata) - _default_repository = {'url': 'https://repo.maven.apache.org/maven2/'} + _default_repository = {"url": "https://repo.maven.apache.org/maven2/"} def parse_repositories(self, d): """https://maven.apache.org/pom.html#Repositories >>> import xmltodict >>> from pprint import pprint >>> d = xmltodict.parse(''' ... ... ... codehausSnapshots ... Codehaus Snapshots ... http://snapshots.maven.codehaus.org/maven2 ... default ... ... ... ''') >>> MavenMapping().parse_repositories(d) """ - repositories = d.get('repositories') + repositories = d.get("repositories") if not repositories: results = [self.parse_repository(d, self._default_repository)] elif isinstance(repositories, dict): - repositories = repositories.get('repository') or [] + repositories = repositories.get("repository") or [] if not isinstance(repositories, list): repositories = [repositories] - results = [self.parse_repository(d, repo) - for repo in repositories] + results = [self.parse_repository(d, repo) for repo in repositories] else: results = [] return [res for res in results if res] or None def parse_repository(self, d, repo): if not isinstance(repo, dict): return - if repo.get('layout', 'default') != 'default': + if repo.get("layout", "default") != "default": return # TODO ? - url = repo.get('url') - group_id = d.get('groupId') - artifact_id = d.get('artifactId') - if (isinstance(url, str) and isinstance(group_id, str) - and isinstance(artifact_id, str)): - repo = os.path.join(url, *group_id.split('.'), artifact_id) + url = repo.get("url") + group_id = d.get("groupId") + artifact_id = d.get("artifactId") + if ( + isinstance(url, str) + and isinstance(group_id, str) + and isinstance(artifact_id, str) + ): + repo = os.path.join(url, *group_id.split("."), artifact_id) return {"@id": repo} def normalize_groupId(self, id_): """https://maven.apache.org/pom.html#Maven_Coordinates >>> MavenMapping().normalize_groupId('org.example') {'@id': 'org.example'} """ if isinstance(id_, str): return {"@id": id_} def parse_licenses(self, d): """https://maven.apache.org/pom.html#Licenses >>> import xmltodict >>> import json >>> d = xmltodict.parse(''' ... ... ... Apache License, Version 2.0 ... https://www.apache.org/licenses/LICENSE-2.0.txt ... ... ... ''') >>> print(json.dumps(d, indent=4)) { "licenses": { "license": { "name": "Apache License, Version 2.0", "url": "https://www.apache.org/licenses/LICENSE-2.0.txt" } } } >>> MavenMapping().parse_licenses(d) [{'@id': 'https://www.apache.org/licenses/LICENSE-2.0.txt'}] or, if there are more than one license: >>> import xmltodict >>> from pprint import pprint >>> d = xmltodict.parse(''' ... ... ... Apache License, Version 2.0 ... https://www.apache.org/licenses/LICENSE-2.0.txt ... ... ... MIT License ... https://opensource.org/licenses/MIT ... ... ... ''') >>> pprint(MavenMapping().parse_licenses(d)) [{'@id': 'https://www.apache.org/licenses/LICENSE-2.0.txt'}, {'@id': 'https://opensource.org/licenses/MIT'}] """ - licenses = d.get('licenses') + licenses = d.get("licenses") if not isinstance(licenses, dict): return - licenses = licenses.get('license') + licenses = licenses.get("license") if isinstance(licenses, dict): licenses = [licenses] elif not isinstance(licenses, list): return - return [{"@id": license['url']} - for license in licenses - if isinstance(license, dict) - and isinstance(license.get('url'), str)] or None + return [ + {"@id": license["url"]} + for license in licenses + if isinstance(license, dict) and isinstance(license.get("url"), str) + ] or None diff --git a/swh/indexer/metadata_dictionary/npm.py b/swh/indexer/metadata_dictionary/npm.py index 9ac73eb..5876e58 100644 --- a/swh/indexer/metadata_dictionary/npm.py +++ b/swh/indexer/metadata_dictionary/npm.py @@ -1,158 +1,160 @@ # Copyright (C) 2018-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import re from swh.indexer.codemeta import CROSSWALK_TABLE, SCHEMA_URI from .base import JsonMapping class NpmMapping(JsonMapping): """ dedicated class for NPM (package.json) mapping and translation """ - name = 'npm' - mapping = CROSSWALK_TABLE['NodeJS'] - filename = b'package.json' - string_fields = ['name', 'version', 'homepage', 'description', 'email'] + + name = "npm" + mapping = CROSSWALK_TABLE["NodeJS"] + filename = b"package.json" + string_fields = ["name", "version", "homepage", "description", "email"] _schema_shortcuts = { - 'github': 'git+https://github.com/%s.git', - 'gist': 'git+https://gist.github.com/%s.git', - 'gitlab': 'git+https://gitlab.com/%s.git', - # Bitbucket supports both hg and git, and the shortcut does not - # tell which one to use. - # 'bitbucket': 'https://bitbucket.org/', - } + "github": "git+https://github.com/%s.git", + "gist": "git+https://gist.github.com/%s.git", + "gitlab": "git+https://gitlab.com/%s.git", + # Bitbucket supports both hg and git, and the shortcut does not + # tell which one to use. + # 'bitbucket': 'https://bitbucket.org/', + } def normalize_repository(self, d): """https://docs.npmjs.com/files/package.json#repository >>> NpmMapping().normalize_repository({ ... 'type': 'git', ... 'url': 'https://example.org/foo.git' ... }) {'@id': 'git+https://example.org/foo.git'} >>> NpmMapping().normalize_repository( ... 'gitlab:foo/bar') {'@id': 'git+https://gitlab.com/foo/bar.git'} >>> NpmMapping().normalize_repository( ... 'foo/bar') {'@id': 'git+https://github.com/foo/bar.git'} """ - if isinstance(d, dict) and isinstance(d.get('type'), str) \ - and isinstance(d.get('url'), str): - url = '{type}+{url}'.format(**d) + if ( + isinstance(d, dict) + and isinstance(d.get("type"), str) + and isinstance(d.get("url"), str) + ): + url = "{type}+{url}".format(**d) elif isinstance(d, str): - if '://' in d: + if "://" in d: url = d - elif ':' in d: - (schema, rest) = d.split(':', 1) + elif ":" in d: + (schema, rest) = d.split(":", 1) if schema in self._schema_shortcuts: url = self._schema_shortcuts[schema] % rest else: return None else: - url = self._schema_shortcuts['github'] % d + url = self._schema_shortcuts["github"] % d else: return None - return {'@id': url} + return {"@id": url} def normalize_bugs(self, d): """https://docs.npmjs.com/files/package.json#bugs >>> NpmMapping().normalize_bugs({ ... 'url': 'https://example.org/bugs/', ... 'email': 'bugs@example.org' ... }) {'@id': 'https://example.org/bugs/'} >>> NpmMapping().normalize_bugs( ... 'https://example.org/bugs/') {'@id': 'https://example.org/bugs/'} """ - if isinstance(d, dict) and isinstance(d.get('url'), str): - return {'@id': d['url']} + if isinstance(d, dict) and isinstance(d.get("url"), str): + return {"@id": d["url"]} elif isinstance(d, str): - return {'@id': d} + return {"@id": d} else: return None - _parse_author = re.compile(r'^ *' - r'(?P.*?)' - r'( +<(?P.*)>)?' - r'( +\((?P.*)\))?' - r' *$') + _parse_author = re.compile( + r"^ *" r"(?P.*?)" r"( +<(?P.*)>)?" r"( +\((?P.*)\))?" r" *$" + ) def normalize_author(self, d): """https://docs.npmjs.com/files/package.json#people-fields-author-contributors' >>> from pprint import pprint >>> pprint(NpmMapping().normalize_author({ ... 'name': 'John Doe', ... 'email': 'john.doe@example.org', ... 'url': 'https://example.org/~john.doe', ... })) {'@list': [{'@type': 'http://schema.org/Person', 'http://schema.org/email': 'john.doe@example.org', 'http://schema.org/name': 'John Doe', 'http://schema.org/url': {'@id': 'https://example.org/~john.doe'}}]} >>> pprint(NpmMapping().normalize_author( ... 'John Doe (https://example.org/~john.doe)' ... )) {'@list': [{'@type': 'http://schema.org/Person', 'http://schema.org/email': 'john.doe@example.org', 'http://schema.org/name': 'John Doe', 'http://schema.org/url': {'@id': 'https://example.org/~john.doe'}}]} - """ # noqa - author = {'@type': SCHEMA_URI+'Person'} + """ # noqa + author = {"@type": SCHEMA_URI + "Person"} if isinstance(d, dict): - name = d.get('name', None) - email = d.get('email', None) - url = d.get('url', None) + name = d.get("name", None) + email = d.get("email", None) + url = d.get("url", None) elif isinstance(d, str): match = self._parse_author.match(d) if not match: return None - name = match.group('name') - email = match.group('email') - url = match.group('url') + name = match.group("name") + email = match.group("email") + url = match.group("url") else: return None if name and isinstance(name, str): - author[SCHEMA_URI+'name'] = name + author[SCHEMA_URI + "name"] = name if email and isinstance(email, str): - author[SCHEMA_URI+'email'] = email + author[SCHEMA_URI + "email"] = email if url and isinstance(url, str): - author[SCHEMA_URI+'url'] = {'@id': url} + author[SCHEMA_URI + "url"] = {"@id": url} return {"@list": [author]} def normalize_license(self, s): """https://docs.npmjs.com/files/package.json#license >>> NpmMapping().normalize_license('MIT') {'@id': 'https://spdx.org/licenses/MIT'} """ if isinstance(s, str): return {"@id": "https://spdx.org/licenses/" + s} def normalize_homepage(self, s): """https://docs.npmjs.com/files/package.json#homepage >>> NpmMapping().normalize_homepage('https://example.org/~john.doe') {'@id': 'https://example.org/~john.doe'} """ if isinstance(s, str): return {"@id": s} def normalize_keywords(self, l): """https://docs.npmjs.com/files/package.json#homepage >>> NpmMapping().normalize_keywords(['foo', 'bar']) ['foo', 'bar'] """ if isinstance(l, list): return [x for x in l if isinstance(x, str)] diff --git a/swh/indexer/metadata_dictionary/python.py b/swh/indexer/metadata_dictionary/python.py index 4bcb81b..510cf57 100644 --- a/swh/indexer/metadata_dictionary/python.py +++ b/swh/indexer/metadata_dictionary/python.py @@ -1,67 +1,76 @@ # Copyright (C) 2018-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import email.parser import email.policy import itertools from swh.indexer.codemeta import CROSSWALK_TABLE, SCHEMA_URI from .base import DictMapping, SingleFileMapping _normalize_pkginfo_key = str.lower class LinebreakPreservingEmailPolicy(email.policy.EmailPolicy): def header_fetch_parse(self, name, value): - if hasattr(value, 'name'): + if hasattr(value, "name"): return value - value = value.replace('\n ', '\n') + value = value.replace("\n ", "\n") return self.header_factory(name, value) class PythonPkginfoMapping(DictMapping, SingleFileMapping): """Dedicated class for Python's PKG-INFO mapping and translation. https://www.python.org/dev/peps/pep-0314/""" - name = 'pkg-info' - filename = b'PKG-INFO' - mapping = {_normalize_pkginfo_key(k): v - for (k, v) in CROSSWALK_TABLE['Python PKG-INFO'].items()} - string_fields = ['name', 'version', 'description', 'summary', - 'author', 'author-email'] - _parser = email.parser.BytesHeaderParser( - policy=LinebreakPreservingEmailPolicy()) + name = "pkg-info" + filename = b"PKG-INFO" + mapping = { + _normalize_pkginfo_key(k): v + for (k, v) in CROSSWALK_TABLE["Python PKG-INFO"].items() + } + string_fields = [ + "name", + "version", + "description", + "summary", + "author", + "author-email", + ] + + _parser = email.parser.BytesHeaderParser(policy=LinebreakPreservingEmailPolicy()) def translate(self, content): msg = self._parser.parsebytes(content) d = {} for (key, value) in msg.items(): key = _normalize_pkginfo_key(key) - if value != 'UNKNOWN': + if value != "UNKNOWN": d.setdefault(key, []).append(value) metadata = self._translate_dict(d, normalize=False) - if SCHEMA_URI+'author' in metadata or SCHEMA_URI+'email' in metadata: - metadata[SCHEMA_URI+'author'] = { - '@list': [{ - '@type': SCHEMA_URI+'Person', - SCHEMA_URI+'name': - metadata.pop(SCHEMA_URI+'author', [None])[0], - SCHEMA_URI+'email': - metadata.pop(SCHEMA_URI+'email', [None])[0], - }] + if SCHEMA_URI + "author" in metadata or SCHEMA_URI + "email" in metadata: + metadata[SCHEMA_URI + "author"] = { + "@list": [ + { + "@type": SCHEMA_URI + "Person", + SCHEMA_URI + + "name": metadata.pop(SCHEMA_URI + "author", [None])[0], + SCHEMA_URI + + "email": metadata.pop(SCHEMA_URI + "email", [None])[0], + } + ] } return self.normalize_translation(metadata) def normalize_home_page(self, urls): - return [{'@id': url} for url in urls] + return [{"@id": url} for url in urls] def normalize_keywords(self, keywords): - return list(itertools.chain.from_iterable( - s.split(' ') for s in keywords)) + return list(itertools.chain.from_iterable(s.split(" ") for s in keywords)) def normalize_license(self, licenses): - return [{'@id': license} for license in licenses] + return [{"@id": license} for license in licenses] diff --git a/swh/indexer/metadata_dictionary/ruby.py b/swh/indexer/metadata_dictionary/ruby.py index ddf0151..62e5216 100644 --- a/swh/indexer/metadata_dictionary/ruby.py +++ b/swh/indexer/metadata_dictionary/ruby.py @@ -1,124 +1,131 @@ # Copyright (C) 2018-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import ast import itertools import re from swh.indexer.codemeta import CROSSWALK_TABLE, SCHEMA_URI from .base import DictMapping def name_to_person(name): return { - '@type': SCHEMA_URI + 'Person', - SCHEMA_URI + 'name': name, + "@type": SCHEMA_URI + "Person", + SCHEMA_URI + "name": name, } class GemspecMapping(DictMapping): - name = 'gemspec' - mapping = CROSSWALK_TABLE['Ruby Gem'] - string_fields = ['name', 'version', 'description', 'summary', 'email'] + name = "gemspec" + mapping = CROSSWALK_TABLE["Ruby Gem"] + string_fields = ["name", "version", "description", "summary", "email"] - _re_spec_new = re.compile(r'.*Gem::Specification.new +(do|\{) +\|.*\|.*') - _re_spec_entry = re.compile(r'\s*\w+\.(?P\w+)\s*=\s*(?P.*)') + _re_spec_new = re.compile(r".*Gem::Specification.new +(do|\{) +\|.*\|.*") + _re_spec_entry = re.compile(r"\s*\w+\.(?P\w+)\s*=\s*(?P.*)") @classmethod def detect_metadata_files(cls, file_entries): for entry in file_entries: - if entry['name'].endswith(b'.gemspec'): - return [entry['sha1']] + if entry["name"].endswith(b".gemspec"): + return [entry["sha1"]] return [] def translate(self, raw_content): try: raw_content = raw_content.decode() except UnicodeDecodeError: - self.log.warning('Error unidecoding from %s', self.log_suffix) + self.log.warning("Error unidecoding from %s", self.log_suffix) return # Skip lines before 'Gem::Specification.new' lines = itertools.dropwhile( - lambda x: not self._re_spec_new.match(x), - raw_content.split('\n')) + lambda x: not self._re_spec_new.match(x), raw_content.split("\n") + ) try: next(lines) # Consume 'Gem::Specification.new' except StopIteration: - self.log.warning('Could not find Gem::Specification in %s', - self.log_suffix) + self.log.warning("Could not find Gem::Specification in %s", self.log_suffix) return content_dict = {} for line in lines: match = self._re_spec_entry.match(line) if match: - value = self.eval_ruby_expression(match.group('expr')) + value = self.eval_ruby_expression(match.group("expr")) if value: - content_dict[match.group('key')] = value + content_dict[match.group("key")] = value return self._translate_dict(content_dict) def eval_ruby_expression(self, expr): """Very simple evaluator of Ruby expressions. >>> GemspecMapping().eval_ruby_expression('"Foo bar"') 'Foo bar' >>> GemspecMapping().eval_ruby_expression("'Foo bar'") 'Foo bar' >>> GemspecMapping().eval_ruby_expression("['Foo', 'bar']") ['Foo', 'bar'] >>> GemspecMapping().eval_ruby_expression("'Foo bar'.freeze") 'Foo bar' >>> GemspecMapping().eval_ruby_expression( \ "['Foo'.freeze, 'bar'.freeze]") ['Foo', 'bar'] """ + def evaluator(node): if isinstance(node, ast.Str): return node.s elif isinstance(node, ast.List): res = [] for element in node.elts: val = evaluator(element) if not val: return res.append(val) return res - expr = expr.replace('.freeze', '') + expr = expr.replace(".freeze", "") try: # We're parsing Ruby expressions here, but Python's # ast.parse works for very simple Ruby expressions # (mainly strings delimited with " or ', and lists # of such strings). - tree = ast.parse(expr, mode='eval') + tree = ast.parse(expr, mode="eval") except (SyntaxError, ValueError): return if isinstance(tree, ast.Expression): return evaluator(tree.body) def normalize_homepage(self, s): if isinstance(s, str): return {"@id": s} def normalize_license(self, s): if isinstance(s, str): return [{"@id": "https://spdx.org/licenses/" + s}] def normalize_licenses(self, licenses): if isinstance(licenses, list): - return [{"@id": "https://spdx.org/licenses/" + license} - for license in licenses - if isinstance(license, str)] + return [ + {"@id": "https://spdx.org/licenses/" + license} + for license in licenses + if isinstance(license, str) + ] def normalize_author(self, author): if isinstance(author, str): return {"@list": [name_to_person(author)]} def normalize_authors(self, authors): if isinstance(authors, list): - return {"@list": [name_to_person(author) for author in authors - if isinstance(author, str)]} + return { + "@list": [ + name_to_person(author) + for author in authors + if isinstance(author, str) + ] + } diff --git a/swh/indexer/mimetype.py b/swh/indexer/mimetype.py index 152e20e..384e5bc 100644 --- a/swh/indexer/mimetype.py +++ b/swh/indexer/mimetype.py @@ -1,157 +1,157 @@ # Copyright (C) 2016-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from typing import Optional, Dict, Any, List import magic from .indexer import ContentIndexer, ContentRangeIndexer -if not hasattr(magic.Magic, 'from_buffer'): +if not hasattr(magic.Magic, "from_buffer"): raise ImportError( 'Expected "import magic" to import python-magic, but file_magic ' - 'was imported instead.') + "was imported instead." + ) def compute_mimetype_encoding(raw_content: bytes) -> Dict[str, bytes]: """Determine mimetype and encoding from the raw content. Args: raw_content: content's raw data Returns: dict: mimetype and encoding key and corresponding values. """ m = magic.Magic(mime=True, mime_encoding=True) res = m.from_buffer(raw_content) try: - mimetype, encoding = res.split('; charset=') + mimetype, encoding = res.split("; charset=") except ValueError: - mimetype, encoding = res, '' + mimetype, encoding = res, "" return { - 'mimetype': mimetype, - 'encoding': encoding, + "mimetype": mimetype, + "encoding": encoding, } class MixinMimetypeIndexer: """Mixin mimetype indexer. See :class:`MimetypeIndexer` and :class:`MimetypeRangeIndexer` """ + tool: Any idx_storage: Any ADDITIONAL_CONFIG = { - 'tools': ('dict', { - 'name': 'file', - 'version': '1:5.30-1+deb9u1', - 'configuration': { - "type": "library", - "debian-package": "python3-magic" + "tools": ( + "dict", + { + "name": "file", + "version": "1:5.30-1+deb9u1", + "configuration": {"type": "library", "debian-package": "python3-magic"}, }, - }), - 'write_batch_size': ('int', 1000), + ), + "write_batch_size": ("int", 1000), } - CONFIG_BASE_FILENAME = 'indexer/mimetype' # type: Optional[str] + CONFIG_BASE_FILENAME = "indexer/mimetype" # type: Optional[str] - def index(self, id: bytes, data: Optional[bytes] = None, - **kwargs) -> Dict[str, Any]: + def index( + self, id: bytes, data: Optional[bytes] = None, **kwargs + ) -> Dict[str, Any]: """Index sha1s' content and store result. Args: id: content's identifier data: raw content in bytes Returns: dict: content's mimetype; dict keys being - id: content's identifier (sha1) - mimetype: mimetype in bytes - encoding: encoding in bytes """ assert data is not None properties = compute_mimetype_encoding(data) - properties.update({ - 'id': id, - 'indexer_configuration_id': self.tool['id'], - }) + properties.update( + {"id": id, "indexer_configuration_id": self.tool["id"],} + ) return properties def persist_index_computations( self, results: List[Dict], policy_update: str ) -> Dict[str, int]: """Persist the results in storage. Args: results: list of content's mimetype dicts (see :meth:`.index`) policy_update: either 'update-dups' or 'ignore-dups' to respectively update duplicates or ignore them """ return self.idx_storage.content_mimetype_add( - results, conflict_update=(policy_update == 'update-dups')) + results, conflict_update=(policy_update == "update-dups") + ) class MimetypeIndexer(MixinMimetypeIndexer, ContentIndexer): """Mimetype Indexer working on list of content identifiers. It: - (optionally) filters out content already indexed (cf. :meth:`.filter`) - reads content from objstorage per the content's id (sha1) - computes {mimetype, encoding} from that content - stores result in storage """ + def filter(self, ids): """Filter out known sha1s and return only missing ones. """ - yield from self.idx_storage.content_mimetype_missing(( - { - 'id': sha1, - 'indexer_configuration_id': self.tool['id'], - } for sha1 in ids - )) + yield from self.idx_storage.content_mimetype_missing( + ({"id": sha1, "indexer_configuration_id": self.tool["id"],} for sha1 in ids) + ) class MimetypeRangeIndexer(MixinMimetypeIndexer, ContentRangeIndexer): """Mimetype Range Indexer working on range of content identifiers. It: - (optionally) filters out content already indexed (cf :meth:`.indexed_contents_in_range`) - reads content from objstorage per the content's id (sha1) - computes {mimetype, encoding} from that content - stores result in storage """ def indexed_contents_in_range( self, start: bytes, end: bytes ) -> Dict[str, Optional[bytes]]: """Retrieve indexed content id within range [start, end]. Args: start: Starting bound from range identifier end: End range identifier Returns: dict: a dict with keys: - ids: iterable of content ids within the range. - next: The next range of sha1 starts at this sha1 if any """ - return self.idx_storage.content_mimetype_get_range( - start, end, self.tool['id']) + return self.idx_storage.content_mimetype_get_range(start, end, self.tool["id"]) diff --git a/swh/indexer/origin_head.py b/swh/indexer/origin_head.py index d25b92d..a0ce14d 100644 --- a/swh/indexer/origin_head.py +++ b/swh/indexer/origin_head.py @@ -1,158 +1,155 @@ # Copyright (C) 2018-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from typing import List, Tuple, Any, Dict, Union import re import click import logging from swh.indexer.indexer import OriginIndexer class OriginHeadIndexer(OriginIndexer): """Origin-level indexer. This indexer is in charge of looking up the revision that acts as the "head" of an origin. In git, this is usually the commit pointed to by the 'master' branch.""" USE_TOOLS = False def persist_index_computations( self, results: Any, policy_update: str ) -> Dict[str, int]: """Do nothing. The indexer's results are not persistent, they should only be piped to another indexer.""" return {} # Dispatch def index(self, origin_url): latest_visit = self.storage.origin_visit_get_latest( - origin_url, allowed_statuses=['full'], require_snapshot=True) + origin_url, allowed_statuses=["full"], require_snapshot=True + ) if latest_visit is None: return None - latest_snapshot = self.storage.snapshot_get(latest_visit['snapshot']) + latest_snapshot = self.storage.snapshot_get(latest_visit["snapshot"]) method = getattr( - self, '_try_get_%s_head' % latest_visit['type'], - self._try_get_head_generic) + self, "_try_get_%s_head" % latest_visit["type"], self._try_get_head_generic + ) rev_id = method(latest_snapshot) if rev_id is not None: return { - 'origin_url': origin_url, - 'revision_id': rev_id, - } + "origin_url": origin_url, + "revision_id": rev_id, + } # could not find a head revision return None # Tarballs _archive_filename_re = re.compile( - rb'^' - rb'(?P.*)[-_]' - rb'(?P[0-9]+(\.[0-9])*)' - rb'(?P[-+][a-zA-Z0-9.~]+?)?' - rb'(?P(\.[a-zA-Z0-9]+)+)' - rb'$') + rb"^" + rb"(?P.*)[-_]" + rb"(?P[0-9]+(\.[0-9])*)" + rb"(?P[-+][a-zA-Z0-9.~]+?)?" + rb"(?P(\.[a-zA-Z0-9]+)+)" + rb"$" + ) @classmethod - def _parse_version( - cls: Any, filename: str - ) -> Tuple[Union[float, int], ...]: + def _parse_version(cls: Any, filename: str) -> Tuple[Union[float, int], ...]: """Extracts the release version from an archive filename, to get an ordering whose maximum is likely to be the last version of the software >>> OriginHeadIndexer._parse_version(b'foo') (-inf,) >>> OriginHeadIndexer._parse_version(b'foo.tar.gz') (-inf,) >>> OriginHeadIndexer._parse_version(b'gnu-hello-0.0.1.tar.gz') (0, 0, 1, 0) >>> OriginHeadIndexer._parse_version(b'gnu-hello-0.0.1-beta2.tar.gz') (0, 0, 1, -1, 'beta2') >>> OriginHeadIndexer._parse_version(b'gnu-hello-0.0.1+foobar.tar.gz') (0, 0, 1, 1, 'foobar') """ res = cls._archive_filename_re.match(filename) if res is None: - return (float('-infinity'),) - version = [int(n) for n in res.group('version').decode().split('.')] - if res.group('preversion') is None: + return (float("-infinity"),) + version = [int(n) for n in res.group("version").decode().split(".")] + if res.group("preversion") is None: version.append(0) else: - preversion = res.group('preversion').decode() - if preversion.startswith('-'): + preversion = res.group("preversion").decode() + if preversion.startswith("-"): version.append(-1) version.append(preversion[1:]) - elif preversion.startswith('+'): + elif preversion.startswith("+"): version.append(1) version.append(preversion[1:]) else: - assert False, res.group('preversion') + assert False, res.group("preversion") return tuple(version) def _try_get_ftp_head(self, snapshot: Dict[str, Any]) -> Any: - archive_names = list(snapshot['branches']) + archive_names = list(snapshot["branches"]) max_archive_name = max(archive_names, key=self._parse_version) - r = self._try_resolve_target(snapshot['branches'], max_archive_name) + r = self._try_resolve_target(snapshot["branches"], max_archive_name) return r # Generic - def _try_get_head_generic( - self, snapshot: Dict[str, Any] - ) -> Any: + def _try_get_head_generic(self, snapshot: Dict[str, Any]) -> Any: # Works on 'deposit', 'pypi', and VCSs. try: - branches = snapshot['branches'] + branches = snapshot["branches"] except KeyError: return None else: - return ( - self._try_resolve_target(branches, b'HEAD') or - self._try_resolve_target(branches, b'master') - ) + return self._try_resolve_target( + branches, b"HEAD" + ) or self._try_resolve_target(branches, b"master") def _try_resolve_target(self, branches: Dict, target_name: bytes) -> Any: try: target = branches[target_name] if target is None: return None - while target['target_type'] == 'alias': - target = branches[target['target']] + while target["target_type"] == "alias": + target = branches[target["target"]] if target is None: return None - if target['target_type'] == 'revision': - return target['target'] - elif target['target_type'] == 'content': + if target["target_type"] == "revision": + return target["target"] + elif target["target_type"] == "content": return None # TODO - elif target['target_type'] == 'directory': + elif target["target_type"] == "directory": return None # TODO - elif target['target_type'] == 'release': + elif target["target_type"] == "release": return None # TODO else: assert False except KeyError: return None @click.command() -@click.option('--origins', '-i', - help='Origins to lookup, in the "type+url" format', - multiple=True) +@click.option( + "--origins", "-i", help='Origins to lookup, in the "type+url" format', multiple=True +) def main(origins: List[str]) -> None: rev_metadata_indexer = OriginHeadIndexer() rev_metadata_indexer.run(origins) -if __name__ == '__main__': +if __name__ == "__main__": logging.basicConfig(level=logging.INFO) main() diff --git a/swh/indexer/rehash.py b/swh/indexer/rehash.py index bec65ec..2593d67 100644 --- a/swh/indexer/rehash.py +++ b/swh/indexer/rehash.py @@ -1,190 +1,189 @@ # Copyright (C) 2017-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import logging import itertools from collections import defaultdict from typing import Dict, Any, Tuple, List, Generator from swh.core import utils from swh.core.config import SWHConfig from swh.model import hashutil from swh.objstorage import get_objstorage from swh.objstorage.exc import ObjNotFoundError from swh.storage import get_storage class RecomputeChecksums(SWHConfig): """Class in charge of (re)computing content's hashes. Hashes to compute are defined across 2 configuration options: compute_checksums ([str]) list of hash algorithms that py:func:`swh.model.hashutil.MultiHash.from_data` function should be able to deal with. For variable-length checksums, a desired checksum length should also be provided. Their format is : e.g: blake2:512 recompute_checksums (bool) a boolean to notify that we also want to recompute potential existing hashes specified in compute_checksums. Default to False. """ + DEFAULT_CONFIG = { # The storage to read from or update metadata to - 'storage': ('dict', { - 'cls': 'remote', - 'args': { - 'url': 'http://localhost:5002/' - }, - }), + "storage": ( + "dict", + {"cls": "remote", "args": {"url": "http://localhost:5002/"},}, + ), # The objstorage to read contents' data from - 'objstorage': ('dict', { - 'cls': 'pathslicing', - 'args': { - 'root': '/srv/softwareheritage/objects', - 'slicing': '0:2/2:4/4:6', + "objstorage": ( + "dict", + { + "cls": "pathslicing", + "args": { + "root": "/srv/softwareheritage/objects", + "slicing": "0:2/2:4/4:6", + }, }, - }), + ), # the set of checksums that should be computed. # Examples: 'sha1_git', 'blake2b512', 'blake2s256' - 'compute_checksums': ( - 'list[str]', []), + "compute_checksums": ("list[str]", []), # whether checksums that already exist in the DB should be # recomputed/updated or left untouched - 'recompute_checksums': ('bool', False), + "recompute_checksums": ("bool", False), # Number of contents to retrieve blobs at the same time - 'batch_size_retrieve_content': ('int', 10), + "batch_size_retrieve_content": ("int", 10), # Number of contents to update at the same time - 'batch_size_update': ('int', 100), + "batch_size_update": ("int", 100), } - CONFIG_BASE_FILENAME = 'indexer/rehash' + CONFIG_BASE_FILENAME = "indexer/rehash" def __init__(self) -> None: self.config = self.parse_config_file() - self.storage = get_storage(**self.config['storage']) - self.objstorage = get_objstorage(**self.config['objstorage']) - self.compute_checksums = self.config['compute_checksums'] - self.recompute_checksums = self.config[ - 'recompute_checksums'] - self.batch_size_retrieve_content = self.config[ - 'batch_size_retrieve_content'] - self.batch_size_update = self.config[ - 'batch_size_update'] - self.log = logging.getLogger('swh.indexer.rehash') + self.storage = get_storage(**self.config["storage"]) + self.objstorage = get_objstorage(**self.config["objstorage"]) + self.compute_checksums = self.config["compute_checksums"] + self.recompute_checksums = self.config["recompute_checksums"] + self.batch_size_retrieve_content = self.config["batch_size_retrieve_content"] + self.batch_size_update = self.config["batch_size_update"] + self.log = logging.getLogger("swh.indexer.rehash") if not self.compute_checksums: - raise ValueError('Checksums list should not be empty.') + raise ValueError("Checksums list should not be empty.") def _read_content_ids( self, contents: List[Dict[str, Any]] ) -> Generator[bytes, Any, None]: """Read the content identifiers from the contents. """ for c in contents: - h = c['sha1'] + h = c["sha1"] if isinstance(h, str): h = hashutil.hash_to_bytes(h) yield h def get_new_contents_metadata( self, all_contents: List[Dict[str, Any]] ) -> Generator[Tuple[Dict[str, Any], List[Any]], Any, None]: """Retrieve raw contents and compute new checksums on the contents. Unknown or corrupted contents are skipped. Args: all_contents: List of contents as dictionary with the necessary primary keys Yields: tuple: tuple of (content to update, list of checksums computed) """ content_ids = self._read_content_ids(all_contents) - for contents in utils.grouper(content_ids, - self.batch_size_retrieve_content): + for contents in utils.grouper(content_ids, self.batch_size_retrieve_content): contents_iter = itertools.tee(contents, 2) try: content_metadata = self.storage.content_get_metadata( - [s for s in contents_iter[0]]) + [s for s in contents_iter[0]] + ) except Exception: - self.log.exception( - 'Problem when reading contents metadata.') + self.log.exception("Problem when reading contents metadata.") continue for content in content_metadata: # Recompute checksums provided in compute_checksums options if self.recompute_checksums: checksums_to_compute = list(self.compute_checksums) else: # Compute checksums provided in compute_checksums # options not already defined for that content - checksums_to_compute = [h for h in self.compute_checksums - if not content.get(h)] + checksums_to_compute = [ + h for h in self.compute_checksums if not content.get(h) + ] if not checksums_to_compute: # Nothing to recompute continue try: - raw_content = self.objstorage.get(content['sha1']) + raw_content = self.objstorage.get(content["sha1"]) except ObjNotFoundError: - self.log.warning('Content %s not found in objstorage!' % - content['sha1']) + self.log.warning( + "Content %s not found in objstorage!" % content["sha1"] + ) continue content_hashes = hashutil.MultiHash.from_data( - raw_content, hash_names=checksums_to_compute).digest() + raw_content, hash_names=checksums_to_compute + ).digest() content.update(content_hashes) yield content, checksums_to_compute def run(self, contents: List[Dict[str, Any]]) -> Dict: """Given a list of content: - (re)compute a given set of checksums on contents available in our object storage - update those contents with the new metadata Args: contents: contents as dictionary with necessary keys. key present in such dictionary should be the ones defined in the 'primary_key' option. Returns: A summary dict with key 'status', task' status and 'count' the number of updated contents. """ - status = 'uneventful' + status = "uneventful" count = 0 for data in utils.grouper( - self.get_new_contents_metadata(contents), - self.batch_size_update): + self.get_new_contents_metadata(contents), self.batch_size_update + ): groups: Dict[str, List[Any]] = defaultdict(list) for content, keys_to_update in data: - keys = ','.join(keys_to_update) + keys = ",".join(keys_to_update) groups[keys].append(content) for keys_to_update, contents in groups.items(): - keys = keys_to_update.split(',') + keys = keys_to_update.split(",") try: - self.storage.content_update(contents, - keys=keys) + self.storage.content_update(contents, keys=keys) count += len(contents) - status = 'eventful' + status = "eventful" except Exception: - self.log.exception('Problem during update.') + self.log.exception("Problem during update.") continue return { - 'status': status, - 'count': count, + "status": status, + "count": count, } diff --git a/swh/indexer/storage/__init__.py b/swh/indexer/storage/__init__.py index 7578e52..7542b28 100644 --- a/swh/indexer/storage/__init__.py +++ b/swh/indexer/storage/__init__.py @@ -1,544 +1,599 @@ # Copyright (C) 2015-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import json import psycopg2 import psycopg2.pool from collections import defaultdict, Counter from typing import Dict, List from swh.storage.common import db_transaction_generator, db_transaction from swh.storage.exc import StorageDBError from . import converters from .db import Db from .exc import IndexerStorageArgumentException, DuplicateId from .metrics import process_metrics, send_metric, timed -INDEXER_CFG_KEY = 'indexer_storage' +INDEXER_CFG_KEY = "indexer_storage" -MAPPING_NAMES = ['codemeta', 'gemspec', 'maven', 'npm', 'pkg-info'] +MAPPING_NAMES = ["codemeta", "gemspec", "maven", "npm", "pkg-info"] def get_indexer_storage(cls, args): """Get an indexer storage object of class `storage_class` with arguments `storage_args`. Args: cls (str): storage's class, either 'local' or 'remote' args (dict): dictionary of arguments passed to the storage class constructor Returns: an instance of swh.indexer's storage (either local or remote) Raises: ValueError if passed an unknown storage class. """ - if cls == 'remote': + if cls == "remote": from .api.client import RemoteStorage as IndexerStorage - elif cls == 'local': + elif cls == "local": from . import IndexerStorage - elif cls == 'memory': + elif cls == "memory": from .in_memory import IndexerStorage else: - raise ValueError('Unknown indexer storage class `%s`' % cls) + raise ValueError("Unknown indexer storage class `%s`" % cls) return IndexerStorage(**args) def check_id_duplicates(data): """ If any two dictionaries in `data` have the same id, raises a `ValueError`. Values associated to the key must be hashable. Args: data (List[dict]): List of dictionaries to be inserted >>> check_id_duplicates([ ... {'id': 'foo', 'data': 'spam'}, ... {'id': 'bar', 'data': 'egg'}, ... ]) >>> check_id_duplicates([ ... {'id': 'foo', 'data': 'spam'}, ... {'id': 'foo', 'data': 'egg'}, ... ]) Traceback (most recent call last): ... swh.indexer.storage.exc.DuplicateId: ['foo'] """ - counter = Counter(item['id'] for item in data) + counter = Counter(item["id"] for item in data) duplicates = [id_ for (id_, count) in counter.items() if count >= 2] if duplicates: raise DuplicateId(duplicates) class IndexerStorage: """SWH Indexer Storage """ + def __init__(self, db, min_pool_conns=1, max_pool_conns=10): """ Args: db_conn: either a libpq connection string, or a psycopg2 connection """ try: if isinstance(db, psycopg2.extensions.connection): self._pool = None self._db = Db(db) else: self._pool = psycopg2.pool.ThreadedConnectionPool( min_pool_conns, max_pool_conns, db ) self._db = None except psycopg2.OperationalError as e: raise StorageDBError(e) def get_db(self): if self._db: return self._db return Db.from_pool(self._pool) def put_db(self, db): if db is not self._db: db.put_conn() @timed @db_transaction() def check_config(self, *, check_write, db=None, cur=None): # Check permissions on one of the tables if check_write: - check = 'INSERT' + check = "INSERT" else: - check = 'SELECT' + check = "SELECT" cur.execute( "select has_table_privilege(current_user, 'content_mimetype', %s)", # noqa - (check,) + (check,), ) return cur.fetchone()[0] @timed @db_transaction_generator() def content_mimetype_missing(self, mimetypes, db=None, cur=None): for obj in db.content_mimetype_missing_from_list(mimetypes, cur): yield obj[0] - def _content_get_range(self, content_type, start, end, - indexer_configuration_id, limit=1000, - with_textual_data=False, - db=None, cur=None): + def _content_get_range( + self, + content_type, + start, + end, + indexer_configuration_id, + limit=1000, + with_textual_data=False, + db=None, + cur=None, + ): if limit is None: - raise IndexerStorageArgumentException('limit should not be None') + raise IndexerStorageArgumentException("limit should not be None") if content_type not in db.content_indexer_names: - err = 'Wrong type. Should be one of [%s]' % ( - ','.join(db.content_indexer_names)) + err = "Wrong type. Should be one of [%s]" % ( + ",".join(db.content_indexer_names) + ) raise IndexerStorageArgumentException(err) ids = [] next_id = None - for counter, obj in enumerate(db.content_get_range( - content_type, start, end, indexer_configuration_id, - limit=limit+1, with_textual_data=with_textual_data, cur=cur)): + for counter, obj in enumerate( + db.content_get_range( + content_type, + start, + end, + indexer_configuration_id, + limit=limit + 1, + with_textual_data=with_textual_data, + cur=cur, + ) + ): _id = obj[0] if counter >= limit: next_id = _id break ids.append(_id) - return { - 'ids': ids, - 'next': next_id - } + return {"ids": ids, "next": next_id} @timed @db_transaction() - def content_mimetype_get_range(self, start, end, indexer_configuration_id, - limit=1000, db=None, cur=None): - return self._content_get_range('mimetype', start, end, - indexer_configuration_id, limit=limit, - db=db, cur=cur) + def content_mimetype_get_range( + self, start, end, indexer_configuration_id, limit=1000, db=None, cur=None + ): + return self._content_get_range( + "mimetype", + start, + end, + indexer_configuration_id, + limit=limit, + db=db, + cur=cur, + ) @timed @process_metrics @db_transaction() def content_mimetype_add( - self, mimetypes: List[Dict], conflict_update: bool = False, - db=None, cur=None) -> Dict[str, int]: + self, mimetypes: List[Dict], conflict_update: bool = False, db=None, cur=None + ) -> Dict[str, int]: """Add mimetypes to the storage (if conflict_update is True, this will override existing data if any). Returns: A dict with the number of new elements added to the storage. """ check_id_duplicates(mimetypes) - mimetypes.sort(key=lambda m: m['id']) + mimetypes.sort(key=lambda m: m["id"]) db.mktemp_content_mimetype(cur) - db.copy_to(mimetypes, 'tmp_content_mimetype', - ['id', 'mimetype', 'encoding', 'indexer_configuration_id'], - cur) + db.copy_to( + mimetypes, + "tmp_content_mimetype", + ["id", "mimetype", "encoding", "indexer_configuration_id"], + cur, + ) count = db.content_mimetype_add_from_temp(conflict_update, cur) - return { - 'content_mimetype:add': count - } + return {"content_mimetype:add": count} @timed @db_transaction_generator() def content_mimetype_get(self, ids, db=None, cur=None): for c in db.content_mimetype_get_from_list(ids, cur): - yield converters.db_to_mimetype( - dict(zip(db.content_mimetype_cols, c))) + yield converters.db_to_mimetype(dict(zip(db.content_mimetype_cols, c))) @timed @db_transaction_generator() def content_language_missing(self, languages, db=None, cur=None): for obj in db.content_language_missing_from_list(languages, cur): yield obj[0] @timed @db_transaction_generator() def content_language_get(self, ids, db=None, cur=None): for c in db.content_language_get_from_list(ids, cur): - yield converters.db_to_language( - dict(zip(db.content_language_cols, c))) + yield converters.db_to_language(dict(zip(db.content_language_cols, c))) @timed @process_metrics @db_transaction() def content_language_add( - self, languages: List[Dict], conflict_update: bool = False, - db=None, cur=None) -> Dict[str, int]: + self, languages: List[Dict], conflict_update: bool = False, db=None, cur=None + ) -> Dict[str, int]: check_id_duplicates(languages) - languages.sort(key=lambda m: m['id']) + languages.sort(key=lambda m: m["id"]) db.mktemp_content_language(cur) # empty language is mapped to 'unknown' db.copy_to( - ({ - 'id': l['id'], - 'lang': 'unknown' if not l['lang'] else l['lang'], - 'indexer_configuration_id': l['indexer_configuration_id'], - } for l in languages), - 'tmp_content_language', - ['id', 'lang', 'indexer_configuration_id'], cur) + ( + { + "id": l["id"], + "lang": "unknown" if not l["lang"] else l["lang"], + "indexer_configuration_id": l["indexer_configuration_id"], + } + for l in languages + ), + "tmp_content_language", + ["id", "lang", "indexer_configuration_id"], + cur, + ) count = db.content_language_add_from_temp(conflict_update, cur) - return { - 'content_language:add': count - } + return {"content_language:add": count} @timed @db_transaction_generator() def content_ctags_missing(self, ctags, db=None, cur=None): for obj in db.content_ctags_missing_from_list(ctags, cur): yield obj[0] @timed @db_transaction_generator() def content_ctags_get(self, ids, db=None, cur=None): for c in db.content_ctags_get_from_list(ids, cur): yield converters.db_to_ctags(dict(zip(db.content_ctags_cols, c))) @timed @process_metrics @db_transaction() def content_ctags_add( - self, ctags: List[Dict], conflict_update: bool = False, - db=None, cur=None) -> Dict[str, int]: + self, ctags: List[Dict], conflict_update: bool = False, db=None, cur=None + ) -> Dict[str, int]: check_id_duplicates(ctags) - ctags.sort(key=lambda m: m['id']) + ctags.sort(key=lambda m: m["id"]) def _convert_ctags(__ctags): """Convert ctags dict to list of ctags. """ for ctags in __ctags: yield from converters.ctags_to_db(ctags) db.mktemp_content_ctags(cur) - db.copy_to(list(_convert_ctags(ctags)), - tblname='tmp_content_ctags', - columns=['id', 'name', 'kind', 'line', - 'lang', 'indexer_configuration_id'], - cur=cur) + db.copy_to( + list(_convert_ctags(ctags)), + tblname="tmp_content_ctags", + columns=["id", "name", "kind", "line", "lang", "indexer_configuration_id"], + cur=cur, + ) count = db.content_ctags_add_from_temp(conflict_update, cur) - return { - 'content_ctags:add': count - } + return {"content_ctags:add": count} @timed @db_transaction_generator() - def content_ctags_search(self, expression, - limit=10, last_sha1=None, db=None, cur=None): - for obj in db.content_ctags_search(expression, last_sha1, limit, - cur=cur): + def content_ctags_search( + self, expression, limit=10, last_sha1=None, db=None, cur=None + ): + for obj in db.content_ctags_search(expression, last_sha1, limit, cur=cur): yield converters.db_to_ctags(dict(zip(db.content_ctags_cols, obj))) @timed @db_transaction_generator() def content_fossology_license_get(self, ids, db=None, cur=None): d = defaultdict(list) for c in db.content_fossology_license_get_from_list(ids, cur): license = dict(zip(db.content_fossology_license_cols, c)) - id_ = license['id'] + id_ = license["id"] d[id_].append(converters.db_to_fossology_license(license)) for id_, facts in d.items(): yield {id_: facts} @timed @process_metrics @db_transaction() def content_fossology_license_add( - self, licenses: List[Dict], conflict_update: bool = False, - db=None, cur=None) -> Dict[str, int]: + self, licenses: List[Dict], conflict_update: bool = False, db=None, cur=None + ) -> Dict[str, int]: check_id_duplicates(licenses) - licenses.sort(key=lambda m: m['id']) + licenses.sort(key=lambda m: m["id"]) db.mktemp_content_fossology_license(cur) db.copy_to( - ({ - 'id': sha1['id'], - 'indexer_configuration_id': sha1['indexer_configuration_id'], - 'license': license, - } for sha1 in licenses - for license in sha1['licenses']), - tblname='tmp_content_fossology_license', - columns=['id', 'license', 'indexer_configuration_id'], - cur=cur) - count = db.content_fossology_license_add_from_temp( - conflict_update, cur) - return { - 'content_fossology_license:add': count - } + ( + { + "id": sha1["id"], + "indexer_configuration_id": sha1["indexer_configuration_id"], + "license": license, + } + for sha1 in licenses + for license in sha1["licenses"] + ), + tblname="tmp_content_fossology_license", + columns=["id", "license", "indexer_configuration_id"], + cur=cur, + ) + count = db.content_fossology_license_add_from_temp(conflict_update, cur) + return {"content_fossology_license:add": count} @timed @db_transaction() def content_fossology_license_get_range( - self, start, end, indexer_configuration_id, - limit=1000, db=None, cur=None): - return self._content_get_range('fossology_license', start, end, - indexer_configuration_id, limit=limit, - with_textual_data=True, db=db, cur=cur) + self, start, end, indexer_configuration_id, limit=1000, db=None, cur=None + ): + return self._content_get_range( + "fossology_license", + start, + end, + indexer_configuration_id, + limit=limit, + with_textual_data=True, + db=db, + cur=cur, + ) @timed @db_transaction_generator() def content_metadata_missing(self, metadata, db=None, cur=None): for obj in db.content_metadata_missing_from_list(metadata, cur): yield obj[0] @timed @db_transaction_generator() def content_metadata_get(self, ids, db=None, cur=None): for c in db.content_metadata_get_from_list(ids, cur): - yield converters.db_to_metadata( - dict(zip(db.content_metadata_cols, c))) + yield converters.db_to_metadata(dict(zip(db.content_metadata_cols, c))) @timed @process_metrics @db_transaction() def content_metadata_add( - self, metadata: List[Dict], conflict_update: bool = False, - db=None, cur=None) -> Dict[str, int]: + self, metadata: List[Dict], conflict_update: bool = False, db=None, cur=None + ) -> Dict[str, int]: check_id_duplicates(metadata) - metadata.sort(key=lambda m: m['id']) + metadata.sort(key=lambda m: m["id"]) db.mktemp_content_metadata(cur) - db.copy_to(metadata, 'tmp_content_metadata', - ['id', 'metadata', 'indexer_configuration_id'], - cur) + db.copy_to( + metadata, + "tmp_content_metadata", + ["id", "metadata", "indexer_configuration_id"], + cur, + ) count = db.content_metadata_add_from_temp(conflict_update, cur) return { - 'content_metadata:add': count, + "content_metadata:add": count, } @timed @db_transaction_generator() def revision_intrinsic_metadata_missing(self, metadata, db=None, cur=None): - for obj in db.revision_intrinsic_metadata_missing_from_list( - metadata, cur): + for obj in db.revision_intrinsic_metadata_missing_from_list(metadata, cur): yield obj[0] @timed @db_transaction_generator() def revision_intrinsic_metadata_get(self, ids, db=None, cur=None): for c in db.revision_intrinsic_metadata_get_from_list(ids, cur): yield converters.db_to_metadata( - dict(zip(db.revision_intrinsic_metadata_cols, c))) + dict(zip(db.revision_intrinsic_metadata_cols, c)) + ) @timed @process_metrics @db_transaction() def revision_intrinsic_metadata_add( - self, metadata: List[Dict], conflict_update: bool = False, - db=None, cur=None) -> Dict[str, int]: + self, metadata: List[Dict], conflict_update: bool = False, db=None, cur=None + ) -> Dict[str, int]: check_id_duplicates(metadata) - metadata.sort(key=lambda m: m['id']) + metadata.sort(key=lambda m: m["id"]) db.mktemp_revision_intrinsic_metadata(cur) - db.copy_to(metadata, 'tmp_revision_intrinsic_metadata', - ['id', 'metadata', 'mappings', - 'indexer_configuration_id'], - cur) - count = db.revision_intrinsic_metadata_add_from_temp( - conflict_update, cur) + db.copy_to( + metadata, + "tmp_revision_intrinsic_metadata", + ["id", "metadata", "mappings", "indexer_configuration_id"], + cur, + ) + count = db.revision_intrinsic_metadata_add_from_temp(conflict_update, cur) return { - 'revision_intrinsic_metadata:add': count, + "revision_intrinsic_metadata:add": count, } @timed @process_metrics @db_transaction() def revision_intrinsic_metadata_delete( - self, entries: List[Dict], db=None, cur=None) -> Dict: + self, entries: List[Dict], db=None, cur=None + ) -> Dict: count = db.revision_intrinsic_metadata_delete(entries, cur) - return { - 'revision_intrinsic_metadata:del': count - } + return {"revision_intrinsic_metadata:del": count} @timed @db_transaction_generator() def origin_intrinsic_metadata_get(self, ids, db=None, cur=None): for c in db.origin_intrinsic_metadata_get_from_list(ids, cur): yield converters.db_to_metadata( - dict(zip(db.origin_intrinsic_metadata_cols, c))) + dict(zip(db.origin_intrinsic_metadata_cols, c)) + ) @timed @process_metrics @db_transaction() def origin_intrinsic_metadata_add( - self, metadata: List[Dict], conflict_update: bool = False, - db=None, cur=None) -> Dict[str, int]: + self, metadata: List[Dict], conflict_update: bool = False, db=None, cur=None + ) -> Dict[str, int]: check_id_duplicates(metadata) - metadata.sort(key=lambda m: m['id']) + metadata.sort(key=lambda m: m["id"]) db.mktemp_origin_intrinsic_metadata(cur) - db.copy_to(metadata, 'tmp_origin_intrinsic_metadata', - ['id', 'metadata', - 'indexer_configuration_id', - 'from_revision', 'mappings'], - cur) - count = db.origin_intrinsic_metadata_add_from_temp( - conflict_update, cur) + db.copy_to( + metadata, + "tmp_origin_intrinsic_metadata", + ["id", "metadata", "indexer_configuration_id", "from_revision", "mappings"], + cur, + ) + count = db.origin_intrinsic_metadata_add_from_temp(conflict_update, cur) return { - 'origin_intrinsic_metadata:add': count, + "origin_intrinsic_metadata:add": count, } @timed @process_metrics @db_transaction() def origin_intrinsic_metadata_delete( - self, entries: List[Dict], db=None, cur=None) -> Dict: + self, entries: List[Dict], db=None, cur=None + ) -> Dict: count = db.origin_intrinsic_metadata_delete(entries, cur) return { - 'origin_intrinsic_metadata:del': count, + "origin_intrinsic_metadata:del": count, } @timed @db_transaction_generator() def origin_intrinsic_metadata_search_fulltext( - self, conjunction, limit=100, db=None, cur=None): + self, conjunction, limit=100, db=None, cur=None + ): for c in db.origin_intrinsic_metadata_search_fulltext( - conjunction, limit=limit, cur=cur): + conjunction, limit=limit, cur=cur + ): yield converters.db_to_metadata( - dict(zip(db.origin_intrinsic_metadata_cols, c))) + dict(zip(db.origin_intrinsic_metadata_cols, c)) + ) @timed @db_transaction() def origin_intrinsic_metadata_search_by_producer( - self, page_token='', limit=100, ids_only=False, - mappings=None, tool_ids=None, - db=None, cur=None): + self, + page_token="", + limit=100, + ids_only=False, + mappings=None, + tool_ids=None, + db=None, + cur=None, + ): assert isinstance(page_token, str) # we go to limit+1 to check whether we should add next_page_token in # the response res = db.origin_intrinsic_metadata_search_by_producer( - page_token, limit + 1, ids_only, mappings, tool_ids, cur) + page_token, limit + 1, ids_only, mappings, tool_ids, cur + ) result = {} if ids_only: - result['origins'] = [origin for (origin,) in res] - if len(result['origins']) > limit: - result['origins'][limit:] = [] - result['next_page_token'] = result['origins'][-1] + result["origins"] = [origin for (origin,) in res] + if len(result["origins"]) > limit: + result["origins"][limit:] = [] + result["next_page_token"] = result["origins"][-1] else: - result['origins'] = [converters.db_to_metadata( - dict(zip(db.origin_intrinsic_metadata_cols, c)))for c in res] - if len(result['origins']) > limit: - result['origins'][limit:] = [] - result['next_page_token'] = result['origins'][-1]['id'] + result["origins"] = [ + converters.db_to_metadata( + dict(zip(db.origin_intrinsic_metadata_cols, c)) + ) + for c in res + ] + if len(result["origins"]) > limit: + result["origins"][limit:] = [] + result["next_page_token"] = result["origins"][-1]["id"] return result @timed @db_transaction() - def origin_intrinsic_metadata_stats( - self, db=None, cur=None): + def origin_intrinsic_metadata_stats(self, db=None, cur=None): mapping_names = [m for m in MAPPING_NAMES] select_parts = [] # Count rows for each mapping for mapping_name in mapping_names: - select_parts.append(( - "sum(case when (mappings @> ARRAY['%s']) " - " then 1 else 0 end)" - ) % mapping_name) + select_parts.append( + ( + "sum(case when (mappings @> ARRAY['%s']) " + " then 1 else 0 end)" + ) + % mapping_name + ) # Total select_parts.append("sum(1)") # Rows whose metadata has at least one key that is not '@context' select_parts.append( "sum(case when ('{}'::jsonb @> (metadata - '@context')) " - " then 0 else 1 end)") - cur.execute('select ' + ', '.join(select_parts) - + ' from origin_intrinsic_metadata') - results = dict(zip(mapping_names + ['total', 'non_empty'], - cur.fetchone())) + " then 0 else 1 end)" + ) + cur.execute( + "select " + ", ".join(select_parts) + " from origin_intrinsic_metadata" + ) + results = dict(zip(mapping_names + ["total", "non_empty"], cur.fetchone())) return { - 'total': results.pop('total'), - 'non_empty': results.pop('non_empty'), - 'per_mapping': results, + "total": results.pop("total"), + "non_empty": results.pop("non_empty"), + "per_mapping": results, } @timed @db_transaction_generator() def indexer_configuration_add(self, tools, db=None, cur=None): db.mktemp_indexer_configuration(cur) - db.copy_to(tools, 'tmp_indexer_configuration', - ['tool_name', 'tool_version', 'tool_configuration'], - cur) + db.copy_to( + tools, + "tmp_indexer_configuration", + ["tool_name", "tool_version", "tool_configuration"], + cur, + ) tools = db.indexer_configuration_add_from_temp(cur) count = 0 for line in tools: yield dict(zip(db.indexer_configuration_cols, line)) count += 1 - send_metric('indexer_configuration:add', count, - method_name='indexer_configuration_add') + send_metric( + "indexer_configuration:add", count, method_name="indexer_configuration_add" + ) @timed @db_transaction() def indexer_configuration_get(self, tool, db=None, cur=None): - tool_conf = tool['tool_configuration'] + tool_conf = tool["tool_configuration"] if isinstance(tool_conf, dict): tool_conf = json.dumps(tool_conf) - idx = db.indexer_configuration_get(tool['tool_name'], - tool['tool_version'], - tool_conf) + idx = db.indexer_configuration_get( + tool["tool_name"], tool["tool_version"], tool_conf + ) if not idx: return None return dict(zip(db.indexer_configuration_cols, idx)) diff --git a/swh/indexer/storage/api/client.py b/swh/indexer/storage/api/client.py index f1768f9..f6b6a73 100644 --- a/swh/indexer/storage/api/client.py +++ b/swh/indexer/storage/api/client.py @@ -1,23 +1,22 @@ # Copyright (C) 2015-2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from swh.core.api import RPCClient from swh.indexer.storage.exc import ( - IndexerStorageAPIError, IndexerStorageArgumentException, + IndexerStorageAPIError, + IndexerStorageArgumentException, DuplicateId, ) from ..interface import IndexerStorageInterface class RemoteStorage(RPCClient): """Proxy to a remote storage API""" backend_class = IndexerStorageInterface api_exception = IndexerStorageAPIError - reraise_exceptions = [ - IndexerStorageArgumentException, DuplicateId - ] + reraise_exceptions = [IndexerStorageArgumentException, DuplicateId] diff --git a/swh/indexer/storage/api/server.py b/swh/indexer/storage/api/server.py index 0266a55..88e35ba 100644 --- a/swh/indexer/storage/api/server.py +++ b/swh/indexer/storage/api/server.py @@ -1,113 +1,109 @@ # Copyright (C) 2015-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import os import logging from swh.core import config -from swh.core.api import (RPCServerApp, error_handler, - encode_data_server as encode_data) -from swh.indexer.storage import ( - get_indexer_storage, INDEXER_CFG_KEY -) +from swh.core.api import RPCServerApp, error_handler, encode_data_server as encode_data +from swh.indexer.storage import get_indexer_storage, INDEXER_CFG_KEY from swh.indexer.storage.exc import IndexerStorageArgumentException from swh.indexer.storage.interface import IndexerStorageInterface def get_storage(): global storage if not storage: storage = get_indexer_storage(**app.config[INDEXER_CFG_KEY]) return storage -app = RPCServerApp(__name__, - backend_class=IndexerStorageInterface, - backend_factory=get_storage) +app = RPCServerApp( + __name__, backend_class=IndexerStorageInterface, backend_factory=get_storage +) storage = None @app.errorhandler(Exception) def my_error_handler(exception): return error_handler(exception, encode_data) @app.errorhandler(IndexerStorageArgumentException) def argument_error_handler(exception): return error_handler(exception, encode_data, status_code=400) -@app.route('/') +@app.route("/") def index(): - return 'SWH Indexer Storage API server' + return "SWH Indexer Storage API server" api_cfg = None -def load_and_check_config(config_file, type='local'): +def load_and_check_config(config_file, type="local"): """Check the minimal configuration is set to run the api or raise an error explanation. Args: config_file (str): Path to the configuration file to load type (str): configuration type. For 'local' type, more checks are done. Raises: Error if the setup is not as expected Returns: configuration as a dict """ if not config_file: - raise EnvironmentError('Configuration file must be defined') + raise EnvironmentError("Configuration file must be defined") if not os.path.exists(config_file): - raise FileNotFoundError('Configuration file %s does not exist' % ( - config_file, )) + raise FileNotFoundError("Configuration file %s does not exist" % (config_file,)) cfg = config.read(config_file) - if 'indexer_storage' not in cfg: + if "indexer_storage" not in cfg: raise KeyError("Missing '%indexer_storage' configuration") - if type == 'local': - vcfg = cfg['indexer_storage'] - cls = vcfg.get('cls') - if cls != 'local': + if type == "local": + vcfg = cfg["indexer_storage"] + cls = vcfg.get("cls") + if cls != "local": raise ValueError( "The indexer_storage backend can only be started with a " - "'local' configuration") + "'local' configuration" + ) - args = vcfg['args'] - if not args.get('db'): - raise ValueError( - "Invalid configuration; missing 'db' config entry") + args = vcfg["args"] + if not args.get("db"): + raise ValueError("Invalid configuration; missing 'db' config entry") return cfg def make_app_from_configfile(): """Run the WSGI app from the webserver, loading the configuration from a configuration file. SWH_CONFIG_FILENAME environment variable defines the configuration path to load. """ global api_cfg if not api_cfg: - config_file = os.environ.get('SWH_CONFIG_FILENAME') + config_file = os.environ.get("SWH_CONFIG_FILENAME") api_cfg = load_and_check_config(config_file) app.config.update(api_cfg) handler = logging.StreamHandler() app.logger.addHandler(handler) return app -if __name__ == '__main__': - print('Deprecated. Use swh-indexer') +if __name__ == "__main__": + print("Deprecated. Use swh-indexer") diff --git a/swh/indexer/storage/converters.py b/swh/indexer/storage/converters.py index 177dd53..7cee1c9 100644 --- a/swh/indexer/storage/converters.py +++ b/swh/indexer/storage/converters.py @@ -1,140 +1,140 @@ # Copyright (C) 2015-2017 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information def ctags_to_db(ctags): """Convert a ctags entry into a ready ctags entry. Args: ctags (dict): ctags entry with the following keys: - id (bytes): content's identifier - tool_id (int): tool id used to compute ctags - ctags ([dict]): List of dictionary with the following keys: - name (str): symbol's name - kind (str): symbol's kind - line (int): symbol's line in the content - language (str): language Returns: list: list of ctags entries as dicts with the following keys: - id (bytes): content's identifier - name (str): symbol's name - kind (str): symbol's kind - language (str): language for that content - tool_id (int): tool id used to compute ctags """ - id = ctags['id'] - tool_id = ctags['indexer_configuration_id'] - for ctag in ctags['ctags']: + id = ctags["id"] + tool_id = ctags["indexer_configuration_id"] + for ctag in ctags["ctags"]: yield { - 'id': id, - 'name': ctag['name'], - 'kind': ctag['kind'], - 'line': ctag['line'], - 'lang': ctag['lang'], - 'indexer_configuration_id': tool_id, + "id": id, + "name": ctag["name"], + "kind": ctag["kind"], + "line": ctag["line"], + "lang": ctag["lang"], + "indexer_configuration_id": tool_id, } def db_to_ctags(ctag): """Convert a ctags entry into a ready ctags entry. Args: ctags (dict): ctags entry with the following keys: - id (bytes): content's identifier - ctags ([dict]): List of dictionary with the following keys: - name (str): symbol's name - kind (str): symbol's kind - line (int): symbol's line in the content - language (str): language Returns: list: list of ctags ready entry (dict with the following keys): - id (bytes): content's identifier - name (str): symbol's name - kind (str): symbol's kind - language (str): language for that content - tool (dict): tool used to compute the ctags """ return { - 'id': ctag['id'], - 'name': ctag['name'], - 'kind': ctag['kind'], - 'line': ctag['line'], - 'lang': ctag['lang'], - 'tool': { - 'id': ctag['tool_id'], - 'name': ctag['tool_name'], - 'version': ctag['tool_version'], - 'configuration': ctag['tool_configuration'] - } + "id": ctag["id"], + "name": ctag["name"], + "kind": ctag["kind"], + "line": ctag["line"], + "lang": ctag["lang"], + "tool": { + "id": ctag["tool_id"], + "name": ctag["tool_name"], + "version": ctag["tool_version"], + "configuration": ctag["tool_configuration"], + }, } def db_to_mimetype(mimetype): """Convert a ctags entry into a ready ctags output. """ return { - 'id': mimetype['id'], - 'encoding': mimetype['encoding'], - 'mimetype': mimetype['mimetype'], - 'tool': { - 'id': mimetype['tool_id'], - 'name': mimetype['tool_name'], - 'version': mimetype['tool_version'], - 'configuration': mimetype['tool_configuration'] - } + "id": mimetype["id"], + "encoding": mimetype["encoding"], + "mimetype": mimetype["mimetype"], + "tool": { + "id": mimetype["tool_id"], + "name": mimetype["tool_name"], + "version": mimetype["tool_version"], + "configuration": mimetype["tool_configuration"], + }, } def db_to_language(language): """Convert a language entry into a ready language output. """ return { - 'id': language['id'], - 'lang': language['lang'], - 'tool': { - 'id': language['tool_id'], - 'name': language['tool_name'], - 'version': language['tool_version'], - 'configuration': language['tool_configuration'] - } + "id": language["id"], + "lang": language["lang"], + "tool": { + "id": language["tool_id"], + "name": language["tool_name"], + "version": language["tool_version"], + "configuration": language["tool_configuration"], + }, } def db_to_metadata(metadata): """Convert a metadata entry into a ready metadata output. """ - metadata['tool'] = { - 'id': metadata['tool_id'], - 'name': metadata['tool_name'], - 'version': metadata['tool_version'], - 'configuration': metadata['tool_configuration'] + metadata["tool"] = { + "id": metadata["tool_id"], + "name": metadata["tool_name"], + "version": metadata["tool_version"], + "configuration": metadata["tool_configuration"], } - del metadata['tool_id'], metadata['tool_configuration'] - del metadata['tool_version'], metadata['tool_name'] + del metadata["tool_id"], metadata["tool_configuration"] + del metadata["tool_version"], metadata["tool_name"] return metadata def db_to_fossology_license(license): return { - 'licenses': license['licenses'], - 'tool': { - 'id': license['tool_id'], - 'name': license['tool_name'], - 'version': license['tool_version'], - 'configuration': license['tool_configuration'], - } + "licenses": license["licenses"], + "tool": { + "id": license["tool_id"], + "name": license["tool_name"], + "version": license["tool_version"], + "configuration": license["tool_configuration"], + }, } diff --git a/swh/indexer/storage/db.py b/swh/indexer/storage/db.py index 80e14ef..0fae4ea 100644 --- a/swh/indexer/storage/db.py +++ b/swh/indexer/storage/db.py @@ -1,479 +1,559 @@ # Copyright (C) 2015-2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from swh.model import hashutil from swh.core.db import BaseDb from swh.core.db.db_utils import execute_values_generator, stored_procedure class Db(BaseDb): """Proxy to the SWH Indexer DB, with wrappers around stored procedures """ - content_mimetype_hash_keys = ['id', 'indexer_configuration_id'] + + content_mimetype_hash_keys = ["id", "indexer_configuration_id"] def _missing_from_list(self, table, data, hash_keys, cur=None): """Read from table the data with hash_keys that are missing. Args: table (str): Table name (e.g content_mimetype, content_language, etc...) data (dict): Dict of data to read from hash_keys ([str]): List of keys to read in the data dict. Yields: The data which is missing from the db. """ cur = self._cursor(cur) - keys = ', '.join(hash_keys) - equality = ' AND '.join( - ('t.%s = c.%s' % (key, key)) for key in hash_keys - ) + keys = ", ".join(hash_keys) + equality = " AND ".join(("t.%s = c.%s" % (key, key)) for key in hash_keys) yield from execute_values_generator( - cur, """ + cur, + """ select %s from (values %%s) as t(%s) where not exists ( select 1 from %s c where %s ) - """ % (keys, keys, table, equality), - (tuple(m[k] for k in hash_keys) for m in data) + """ + % (keys, keys, table, equality), + (tuple(m[k] for k in hash_keys) for m in data), ) def content_mimetype_missing_from_list(self, mimetypes, cur=None): """List missing mimetypes. """ yield from self._missing_from_list( - 'content_mimetype', mimetypes, self.content_mimetype_hash_keys, - cur=cur) + "content_mimetype", mimetypes, self.content_mimetype_hash_keys, cur=cur + ) content_mimetype_cols = [ - 'id', 'mimetype', 'encoding', - 'tool_id', 'tool_name', 'tool_version', 'tool_configuration'] - - @stored_procedure('swh_mktemp_content_mimetype') - def mktemp_content_mimetype(self, cur=None): pass + "id", + "mimetype", + "encoding", + "tool_id", + "tool_name", + "tool_version", + "tool_configuration", + ] + + @stored_procedure("swh_mktemp_content_mimetype") + def mktemp_content_mimetype(self, cur=None): + pass def content_mimetype_add_from_temp(self, conflict_update, cur=None): cur = self._cursor(cur) - cur.execute('select * from swh_content_mimetype_add(%s)', - (conflict_update, )) + cur.execute("select * from swh_content_mimetype_add(%s)", (conflict_update,)) return cur.fetchone()[0] - def _convert_key(self, key, main_table='c'): + def _convert_key(self, key, main_table="c"): """Convert keys according to specific use in the module. Args: key (str): Key expression to change according to the alias used in the query main_table (str): Alias to use for the main table. Default to c for content_{something}. Expected: Tables content_{something} being aliased as 'c' (something in {language, mimetype, ...}), table indexer_configuration being aliased as 'i'. """ - if key == 'id': - return '%s.id' % main_table - elif key == 'tool_id': - return 'i.id as tool_id' - elif key == 'licenses': - return ''' + if key == "id": + return "%s.id" % main_table + elif key == "tool_id": + return "i.id as tool_id" + elif key == "licenses": + return ( + """ array(select name from fossology_license where id = ANY( - array_agg(%s.license_id))) as licenses''' % main_table + array_agg(%s.license_id))) as licenses""" + % main_table + ) return key - def _get_from_list(self, table, ids, cols, cur=None, id_col='id'): + def _get_from_list(self, table, ids, cols, cur=None, id_col="id"): """Fetches entries from the `table` such that their `id` field (or whatever is given to `id_col`) is in `ids`. Returns the columns `cols`. The `cur` parameter is used to connect to the database. """ cur = self._cursor(cur) keys = map(self._convert_key, cols) query = """ select {keys} from (values %s) as t(id) inner join {table} c on c.{id_col}=t.id inner join indexer_configuration i on c.indexer_configuration_id=i.id; """.format( - keys=', '.join(keys), - id_col=id_col, - table=table) - yield from execute_values_generator( - cur, query, - ((_id,) for _id in ids) + keys=", ".join(keys), id_col=id_col, table=table ) + yield from execute_values_generator(cur, query, ((_id,) for _id in ids)) content_indexer_names = { - 'mimetype': 'content_mimetype', - 'fossology_license': 'content_fossology_license', + "mimetype": "content_mimetype", + "fossology_license": "content_fossology_license", } - def content_get_range(self, content_type, start, end, - indexer_configuration_id, limit=1000, - with_textual_data=False, cur=None): + def content_get_range( + self, + content_type, + start, + end, + indexer_configuration_id, + limit=1000, + with_textual_data=False, + cur=None, + ): """Retrieve contents with content_type, within range [start, end] bound by limit and associated to the given indexer configuration id. When asking to work on textual content, that filters on the mimetype table with any mimetype that is not binary. """ cur = self._cursor(cur) table = self.content_indexer_names[content_type] if with_textual_data: extra = """inner join content_mimetype cm on (t.id=cm.id and cm.mimetype like 'text/%%' and %(start)s <= cm.id and cm.id <= %(end)s) """ else: extra = "" query = f"""select t.id from {table} t {extra} where t.indexer_configuration_id=%(tool_id)s and %(start)s <= t.id and t.id <= %(end)s order by t.indexer_configuration_id, t.id limit %(limit)s""" - cur.execute(query, { - 'start': start, - 'end': end, - 'tool_id': indexer_configuration_id, - 'limit': limit, - }) + cur.execute( + query, + { + "start": start, + "end": end, + "tool_id": indexer_configuration_id, + "limit": limit, + }, + ) yield from cur def content_mimetype_get_from_list(self, ids, cur=None): yield from self._get_from_list( - 'content_mimetype', ids, self.content_mimetype_cols, cur=cur) + "content_mimetype", ids, self.content_mimetype_cols, cur=cur + ) - content_language_hash_keys = ['id', 'indexer_configuration_id'] + content_language_hash_keys = ["id", "indexer_configuration_id"] def content_language_missing_from_list(self, languages, cur=None): """List missing languages. """ yield from self._missing_from_list( - 'content_language', languages, self.content_language_hash_keys, - cur=cur) + "content_language", languages, self.content_language_hash_keys, cur=cur + ) content_language_cols = [ - 'id', 'lang', - 'tool_id', 'tool_name', 'tool_version', 'tool_configuration'] - - @stored_procedure('swh_mktemp_content_language') - def mktemp_content_language(self, cur=None): pass + "id", + "lang", + "tool_id", + "tool_name", + "tool_version", + "tool_configuration", + ] + + @stored_procedure("swh_mktemp_content_language") + def mktemp_content_language(self, cur=None): + pass def content_language_add_from_temp(self, conflict_update, cur=None): cur = self._cursor(cur) - cur.execute('select * from swh_content_language_add(%s)', - (conflict_update, )) + cur.execute("select * from swh_content_language_add(%s)", (conflict_update,)) return cur.fetchone()[0] def content_language_get_from_list(self, ids, cur=None): yield from self._get_from_list( - 'content_language', ids, self.content_language_cols, cur=cur) + "content_language", ids, self.content_language_cols, cur=cur + ) - content_ctags_hash_keys = ['id', 'indexer_configuration_id'] + content_ctags_hash_keys = ["id", "indexer_configuration_id"] def content_ctags_missing_from_list(self, ctags, cur=None): """List missing ctags. """ yield from self._missing_from_list( - 'content_ctags', ctags, self.content_ctags_hash_keys, - cur=cur) + "content_ctags", ctags, self.content_ctags_hash_keys, cur=cur + ) content_ctags_cols = [ - 'id', 'name', 'kind', 'line', 'lang', - 'tool_id', 'tool_name', 'tool_version', 'tool_configuration'] - - @stored_procedure('swh_mktemp_content_ctags') - def mktemp_content_ctags(self, cur=None): pass + "id", + "name", + "kind", + "line", + "lang", + "tool_id", + "tool_name", + "tool_version", + "tool_configuration", + ] + + @stored_procedure("swh_mktemp_content_ctags") + def mktemp_content_ctags(self, cur=None): + pass def content_ctags_add_from_temp(self, conflict_update, cur=None): cur = self._cursor(cur) - cur.execute( - 'select * from swh_content_ctags_add(%s)', - (conflict_update, )) + cur.execute("select * from swh_content_ctags_add(%s)", (conflict_update,)) return cur.fetchone()[0] def content_ctags_get_from_list(self, ids, cur=None): cur = self._cursor(cur) keys = map(self._convert_key, self.content_ctags_cols) yield from execute_values_generator( - cur, """ + cur, + """ select %s from (values %%s) as t(id) inner join content_ctags c on c.id=t.id inner join indexer_configuration i on c.indexer_configuration_id=i.id order by line - """ % ', '.join(keys), - ((_id,) for _id in ids) - ) + """ + % ", ".join(keys), + ((_id,) for _id in ids), + ) def content_ctags_search(self, expression, last_sha1, limit, cur=None): cur = self._cursor(cur) if not last_sha1: query = """SELECT %s FROM swh_content_ctags_search(%%s, %%s)""" % ( - ','.join(self.content_ctags_cols)) + ",".join(self.content_ctags_cols) + ) cur.execute(query, (expression, limit)) else: if last_sha1 and isinstance(last_sha1, bytes): - last_sha1 = '\\x%s' % hashutil.hash_to_hex(last_sha1) + last_sha1 = "\\x%s" % hashutil.hash_to_hex(last_sha1) elif last_sha1: - last_sha1 = '\\x%s' % last_sha1 + last_sha1 = "\\x%s" % last_sha1 query = """SELECT %s FROM swh_content_ctags_search(%%s, %%s, %%s)""" % ( - ','.join(self.content_ctags_cols)) + ",".join(self.content_ctags_cols) + ) cur.execute(query, (expression, limit, last_sha1)) yield from cur content_fossology_license_cols = [ - 'id', 'tool_id', 'tool_name', 'tool_version', 'tool_configuration', - 'licenses'] - - @stored_procedure('swh_mktemp_content_fossology_license') - def mktemp_content_fossology_license(self, cur=None): pass + "id", + "tool_id", + "tool_name", + "tool_version", + "tool_configuration", + "licenses", + ] + + @stored_procedure("swh_mktemp_content_fossology_license") + def mktemp_content_fossology_license(self, cur=None): + pass - def content_fossology_license_add_from_temp(self, conflict_update, - cur=None): + def content_fossology_license_add_from_temp(self, conflict_update, cur=None): """Add new licenses per content. """ cur = self._cursor(cur) cur.execute( - 'select * from swh_content_fossology_license_add(%s)', - (conflict_update, )) + "select * from swh_content_fossology_license_add(%s)", (conflict_update,) + ) return cur.fetchone()[0] def content_fossology_license_get_from_list(self, ids, cur=None): """Retrieve licenses per id. """ cur = self._cursor(cur) keys = map(self._convert_key, self.content_fossology_license_cols) yield from execute_values_generator( - cur, """ + cur, + """ select %s from (values %%s) as t(id) inner join content_fossology_license c on t.id=c.id inner join indexer_configuration i on i.id=c.indexer_configuration_id group by c.id, i.id, i.tool_name, i.tool_version, i.tool_configuration; - """ % ', '.join(keys), - ((_id,) for _id in ids) + """ + % ", ".join(keys), + ((_id,) for _id in ids), ) - content_metadata_hash_keys = ['id', 'indexer_configuration_id'] + content_metadata_hash_keys = ["id", "indexer_configuration_id"] def content_metadata_missing_from_list(self, metadata, cur=None): """List missing metadata. """ yield from self._missing_from_list( - 'content_metadata', metadata, self.content_metadata_hash_keys, - cur=cur) + "content_metadata", metadata, self.content_metadata_hash_keys, cur=cur + ) content_metadata_cols = [ - 'id', 'metadata', - 'tool_id', 'tool_name', 'tool_version', 'tool_configuration'] - - @stored_procedure('swh_mktemp_content_metadata') - def mktemp_content_metadata(self, cur=None): pass + "id", + "metadata", + "tool_id", + "tool_name", + "tool_version", + "tool_configuration", + ] + + @stored_procedure("swh_mktemp_content_metadata") + def mktemp_content_metadata(self, cur=None): + pass def content_metadata_add_from_temp(self, conflict_update, cur=None): cur = self._cursor(cur) - cur.execute( - 'select * from swh_content_metadata_add(%s)', - (conflict_update, )) + cur.execute("select * from swh_content_metadata_add(%s)", (conflict_update,)) return cur.fetchone()[0] def content_metadata_get_from_list(self, ids, cur=None): yield from self._get_from_list( - 'content_metadata', ids, self.content_metadata_cols, cur=cur) + "content_metadata", ids, self.content_metadata_cols, cur=cur + ) - revision_intrinsic_metadata_hash_keys = [ - 'id', 'indexer_configuration_id'] + revision_intrinsic_metadata_hash_keys = ["id", "indexer_configuration_id"] - def revision_intrinsic_metadata_missing_from_list( - self, metadata, cur=None): + def revision_intrinsic_metadata_missing_from_list(self, metadata, cur=None): """List missing metadata. """ yield from self._missing_from_list( - 'revision_intrinsic_metadata', metadata, - self.revision_intrinsic_metadata_hash_keys, cur=cur) + "revision_intrinsic_metadata", + metadata, + self.revision_intrinsic_metadata_hash_keys, + cur=cur, + ) revision_intrinsic_metadata_cols = [ - 'id', 'metadata', 'mappings', - 'tool_id', 'tool_name', 'tool_version', 'tool_configuration'] - - @stored_procedure('swh_mktemp_revision_intrinsic_metadata') - def mktemp_revision_intrinsic_metadata(self, cur=None): pass + "id", + "metadata", + "mappings", + "tool_id", + "tool_name", + "tool_version", + "tool_configuration", + ] + + @stored_procedure("swh_mktemp_revision_intrinsic_metadata") + def mktemp_revision_intrinsic_metadata(self, cur=None): + pass - def revision_intrinsic_metadata_add_from_temp( - self, conflict_update, cur=None): + def revision_intrinsic_metadata_add_from_temp(self, conflict_update, cur=None): cur = self._cursor(cur) cur.execute( - 'select * from swh_revision_intrinsic_metadata_add(%s)', - (conflict_update, )) + "select * from swh_revision_intrinsic_metadata_add(%s)", (conflict_update,) + ) return cur.fetchone()[0] - def revision_intrinsic_metadata_delete( - self, entries, cur=None): + def revision_intrinsic_metadata_delete(self, entries, cur=None): cur = self._cursor(cur) cur.execute( - "DELETE from revision_intrinsic_metadata " - "WHERE (id, indexer_configuration_id) IN " - " (VALUES %s) " - "RETURNING id" % (', '.join('%s' for _ in entries)), - tuple((e['id'], e['indexer_configuration_id']) - for e in entries),) + "DELETE from revision_intrinsic_metadata " + "WHERE (id, indexer_configuration_id) IN " + " (VALUES %s) " + "RETURNING id" % (", ".join("%s" for _ in entries)), + tuple((e["id"], e["indexer_configuration_id"]) for e in entries), + ) return len(cur.fetchall()) def revision_intrinsic_metadata_get_from_list(self, ids, cur=None): yield from self._get_from_list( - 'revision_intrinsic_metadata', ids, - self.revision_intrinsic_metadata_cols, cur=cur) + "revision_intrinsic_metadata", + ids, + self.revision_intrinsic_metadata_cols, + cur=cur, + ) origin_intrinsic_metadata_cols = [ - 'id', 'metadata', 'from_revision', 'mappings', - 'tool_id', 'tool_name', 'tool_version', 'tool_configuration'] - - origin_intrinsic_metadata_regconfig = 'pg_catalog.simple' + "id", + "metadata", + "from_revision", + "mappings", + "tool_id", + "tool_name", + "tool_version", + "tool_configuration", + ] + + origin_intrinsic_metadata_regconfig = "pg_catalog.simple" """The dictionary used to normalize 'metadata' and queries. 'pg_catalog.simple' provides no stopword, so it should be suitable for proper names and non-English content. When updating this value, make sure to add a new index on origin_intrinsic_metadata.metadata.""" - @stored_procedure('swh_mktemp_origin_intrinsic_metadata') - def mktemp_origin_intrinsic_metadata(self, cur=None): pass + @stored_procedure("swh_mktemp_origin_intrinsic_metadata") + def mktemp_origin_intrinsic_metadata(self, cur=None): + pass - def origin_intrinsic_metadata_add_from_temp( - self, conflict_update, cur=None): + def origin_intrinsic_metadata_add_from_temp(self, conflict_update, cur=None): cur = self._cursor(cur) cur.execute( - 'select * from swh_origin_intrinsic_metadata_add(%s)', - (conflict_update, )) + "select * from swh_origin_intrinsic_metadata_add(%s)", (conflict_update,) + ) return cur.fetchone()[0] - def origin_intrinsic_metadata_delete( - self, entries, cur=None): + def origin_intrinsic_metadata_delete(self, entries, cur=None): cur = self._cursor(cur) cur.execute( - "DELETE from origin_intrinsic_metadata " - "WHERE (id, indexer_configuration_id) IN" - " (VALUES %s) " - "RETURNING id" % (', '.join('%s' for _ in entries)), - tuple((e['id'], e['indexer_configuration_id']) - for e in entries),) + "DELETE from origin_intrinsic_metadata " + "WHERE (id, indexer_configuration_id) IN" + " (VALUES %s) " + "RETURNING id" % (", ".join("%s" for _ in entries)), + tuple((e["id"], e["indexer_configuration_id"]) for e in entries), + ) return len(cur.fetchall()) def origin_intrinsic_metadata_get_from_list(self, ids, cur=None): yield from self._get_from_list( - 'origin_intrinsic_metadata', ids, - self.origin_intrinsic_metadata_cols, cur=cur, - id_col='id') + "origin_intrinsic_metadata", + ids, + self.origin_intrinsic_metadata_cols, + cur=cur, + id_col="id", + ) def origin_intrinsic_metadata_search_fulltext(self, terms, *, limit, cur): regconfig = self.origin_intrinsic_metadata_regconfig - tsquery_template = ' && '.join("plainto_tsquery('%s', %%s)" % regconfig - for _ in terms) + tsquery_template = " && ".join( + "plainto_tsquery('%s', %%s)" % regconfig for _ in terms + ) tsquery_args = [(term,) for term in terms] - keys = (self._convert_key(col, 'oim') for col in - self.origin_intrinsic_metadata_cols) - - query = ("SELECT {keys} FROM origin_intrinsic_metadata AS oim " - "INNER JOIN indexer_configuration AS i " - "ON oim.indexer_configuration_id=i.id " - "JOIN LATERAL (SELECT {tsquery_template}) AS s(tsq) ON true " - "WHERE oim.metadata_tsvector @@ tsq " - "ORDER BY ts_rank(oim.metadata_tsvector, tsq, 1) DESC " - "LIMIT %s;" - ).format(keys=', '.join(keys), - tsquery_template=tsquery_template) + keys = ( + self._convert_key(col, "oim") for col in self.origin_intrinsic_metadata_cols + ) + + query = ( + "SELECT {keys} FROM origin_intrinsic_metadata AS oim " + "INNER JOIN indexer_configuration AS i " + "ON oim.indexer_configuration_id=i.id " + "JOIN LATERAL (SELECT {tsquery_template}) AS s(tsq) ON true " + "WHERE oim.metadata_tsvector @@ tsq " + "ORDER BY ts_rank(oim.metadata_tsvector, tsq, 1) DESC " + "LIMIT %s;" + ).format(keys=", ".join(keys), tsquery_template=tsquery_template) cur.execute(query, tsquery_args + [limit]) yield from cur def origin_intrinsic_metadata_search_by_producer( - self, last, limit, ids_only, mappings, tool_ids, cur): + self, last, limit, ids_only, mappings, tool_ids, cur + ): if ids_only: - keys = 'oim.id' + keys = "oim.id" else: - keys = ', '.join((self._convert_key(col, 'oim') for col in - self.origin_intrinsic_metadata_cols)) + keys = ", ".join( + ( + self._convert_key(col, "oim") + for col in self.origin_intrinsic_metadata_cols + ) + ) query_parts = [ "SELECT %s" % keys, "FROM origin_intrinsic_metadata AS oim", "INNER JOIN indexer_configuration AS i", "ON oim.indexer_configuration_id=i.id", ] args = [] where = [] if last: - where.append('oim.id > %s') + where.append("oim.id > %s") args.append(last) if mappings is not None: - where.append('oim.mappings && %s') + where.append("oim.mappings && %s") args.append(mappings) if tool_ids is not None: - where.append('oim.indexer_configuration_id = ANY(%s)') + where.append("oim.indexer_configuration_id = ANY(%s)") args.append(tool_ids) if where: - query_parts.append('WHERE') - query_parts.append(' AND '.join(where)) + query_parts.append("WHERE") + query_parts.append(" AND ".join(where)) if limit: - query_parts.append('LIMIT %s') + query_parts.append("LIMIT %s") args.append(limit) - cur.execute(' '.join(query_parts), args) + cur.execute(" ".join(query_parts), args) yield from cur - indexer_configuration_cols = ['id', 'tool_name', 'tool_version', - 'tool_configuration'] + indexer_configuration_cols = [ + "id", + "tool_name", + "tool_version", + "tool_configuration", + ] - @stored_procedure('swh_mktemp_indexer_configuration') + @stored_procedure("swh_mktemp_indexer_configuration") def mktemp_indexer_configuration(self, cur=None): pass def indexer_configuration_add_from_temp(self, cur=None): cur = self._cursor(cur) - cur.execute("SELECT %s from swh_indexer_configuration_add()" % ( - ','.join(self.indexer_configuration_cols), )) + cur.execute( + "SELECT %s from swh_indexer_configuration_add()" + % (",".join(self.indexer_configuration_cols),) + ) yield from cur - def indexer_configuration_get(self, tool_name, - tool_version, tool_configuration, cur=None): + def indexer_configuration_get( + self, tool_name, tool_version, tool_configuration, cur=None + ): cur = self._cursor(cur) - cur.execute('''select %s + cur.execute( + """select %s from indexer_configuration where tool_name=%%s and tool_version=%%s and - tool_configuration=%%s''' % ( - ','.join(self.indexer_configuration_cols)), - (tool_name, tool_version, tool_configuration)) + tool_configuration=%%s""" + % (",".join(self.indexer_configuration_cols)), + (tool_name, tool_version, tool_configuration), + ) return cur.fetchone() diff --git a/swh/indexer/storage/exc.py b/swh/indexer/storage/exc.py index 841ee0c..3b583c3 100644 --- a/swh/indexer/storage/exc.py +++ b/swh/indexer/storage/exc.py @@ -1,19 +1,22 @@ # Copyright (C) 2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information class IndexerStorageAPIError(Exception): """Generic error of the indexer storage.""" + pass class IndexerStorageArgumentException(Exception): """Argument passed to an IndexerStorage endpoint is invalid.""" + pass class DuplicateId(IndexerStorageArgumentException): """The same identifier is present more than once.""" + pass diff --git a/swh/indexer/storage/in_memory.py b/swh/indexer/storage/in_memory.py index 43db6d3..1bcc46d 100644 --- a/swh/indexer/storage/in_memory.py +++ b/swh/indexer/storage/in_memory.py @@ -1,457 +1,455 @@ # Copyright (C) 2018-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import bisect from collections import defaultdict, Counter import itertools import json import operator import math import re from typing import Any, Dict, List from . import MAPPING_NAMES, check_id_duplicates from .exc import IndexerStorageArgumentException SHA1_DIGEST_SIZE = 160 def _transform_tool(tool): return { - 'id': tool['id'], - 'name': tool['tool_name'], - 'version': tool['tool_version'], - 'configuration': tool['tool_configuration'], + "id": tool["id"], + "name": tool["tool_name"], + "version": tool["tool_version"], + "configuration": tool["tool_configuration"], } def check_id_types(data: List[Dict[str, Any]]): """Checks all elements of the list have an 'id' whose type is 'bytes'.""" - if not all(isinstance(item.get('id'), bytes) for item in data): - raise IndexerStorageArgumentException('identifiers must be bytes.') + if not all(isinstance(item.get("id"), bytes) for item in data): + raise IndexerStorageArgumentException("identifiers must be bytes.") class SubStorage: """Implements common missing/get/add logic for each indexer type.""" + def __init__(self, tools): self._tools = tools self._sorted_ids = [] self._data = {} # map (id_, tool_id) -> metadata_dict self._tools_per_id = defaultdict(set) # map id_ -> Set[tool_id] def missing(self, ids): """List data missing from storage. Args: data (iterable): dictionaries with keys: - **id** (bytes): sha1 identifier - **indexer_configuration_id** (int): tool used to compute the results Yields: missing sha1s """ for id_ in ids: - tool_id = id_['indexer_configuration_id'] - id_ = id_['id'] + tool_id = id_["indexer_configuration_id"] + id_ = id_["id"] if tool_id not in self._tools_per_id.get(id_, set()): yield id_ def get(self, ids): """Retrieve data per id. Args: ids (iterable): sha1 checksums Yields: dict: dictionaries with the following keys: - **id** (bytes) - **tool** (dict): tool used to compute metadata - arbitrary data (as provided to `add`) """ for id_ in ids: for tool_id in self._tools_per_id.get(id_, set()): key = (id_, tool_id) yield { - 'id': id_, - 'tool': _transform_tool(self._tools[tool_id]), + "id": id_, + "tool": _transform_tool(self._tools[tool_id]), **self._data[key], } def get_all(self): yield from self.get(self._sorted_ids) def get_range(self, start, end, indexer_configuration_id, limit): """Retrieve data within range [start, end] bound by limit. Args: **start** (bytes): Starting identifier range (expected smaller than end) **end** (bytes): Ending identifier range (expected larger than start) **indexer_configuration_id** (int): The tool used to index data **limit** (int): Limit result Raises: IndexerStorageArgumentException for limit to None Returns: a dict with keys: - **ids** [bytes]: iterable of content ids within the range. - **next** (Optional[bytes]): The next range of sha1 starts at this sha1 if any """ if limit is None: - raise IndexerStorageArgumentException('limit should not be None') + raise IndexerStorageArgumentException("limit should not be None") from_index = bisect.bisect_left(self._sorted_ids, start) to_index = bisect.bisect_right(self._sorted_ids, end, lo=from_index) if to_index - from_index >= limit: return { - 'ids': self._sorted_ids[from_index:from_index+limit], - 'next': self._sorted_ids[from_index+limit], + "ids": self._sorted_ids[from_index : from_index + limit], + "next": self._sorted_ids[from_index + limit], } else: return { - 'ids': self._sorted_ids[from_index:to_index], - 'next': None, - } + "ids": self._sorted_ids[from_index:to_index], + "next": None, + } def add(self, data: List[Dict], conflict_update: bool) -> int: """Add data not present in storage. Args: data (iterable): dictionaries with keys: - **id**: sha1 - **indexer_configuration_id**: tool used to compute the results - arbitrary data conflict_update (bool): Flag to determine if we want to overwrite (true) or skip duplicates (false) """ data = list(data) check_id_duplicates(data) count = 0 for item in data: item = item.copy() - tool_id = item.pop('indexer_configuration_id') - id_ = item.pop('id') + tool_id = item.pop("indexer_configuration_id") + id_ = item.pop("id") data_item = item - if not conflict_update and \ - tool_id in self._tools_per_id.get(id_, set()): + if not conflict_update and tool_id in self._tools_per_id.get(id_, set()): # Duplicate, should not be updated continue key = (id_, tool_id) self._data[key] = data_item self._tools_per_id[id_].add(tool_id) count += 1 if id_ not in self._sorted_ids: bisect.insort(self._sorted_ids, id_) return count - def add_merge(self, new_data: List[Dict], conflict_update: bool, - merged_key: str) -> int: + def add_merge( + self, new_data: List[Dict], conflict_update: bool, merged_key: str + ) -> int: added = 0 all_subitems: List for new_item in new_data: - id_ = new_item['id'] - tool_id = new_item['indexer_configuration_id'] + id_ = new_item["id"] + tool_id = new_item["indexer_configuration_id"] if conflict_update: all_subitems = [] else: existing = list(self.get([id_])) all_subitems = [ old_subitem for existing_item in existing - if existing_item['tool']['id'] == tool_id + if existing_item["tool"]["id"] == tool_id for old_subitem in existing_item[merged_key] ] for new_subitem in new_item[merged_key]: if new_subitem not in all_subitems: all_subitems.append(new_subitem) - added += self.add([ - { - 'id': id_, - 'indexer_configuration_id': tool_id, - merged_key: all_subitems, - } - ], conflict_update=True) + added += self.add( + [ + { + "id": id_, + "indexer_configuration_id": tool_id, + merged_key: all_subitems, + } + ], + conflict_update=True, + ) if id_ not in self._sorted_ids: bisect.insort(self._sorted_ids, id_) return added def delete(self, entries: List[Dict]) -> int: """Delete entries and return the number of entries deleted. """ deleted = 0 for entry in entries: - (id_, tool_id) = (entry['id'], entry['indexer_configuration_id']) + (id_, tool_id) = (entry["id"], entry["indexer_configuration_id"]) key = (id_, tool_id) if tool_id in self._tools_per_id[id_]: self._tools_per_id[id_].remove(tool_id) if key in self._data: deleted += 1 del self._data[key] return deleted class IndexerStorage: """In-memory SWH indexer storage.""" def __init__(self): self._tools = {} self._mimetypes = SubStorage(self._tools) self._languages = SubStorage(self._tools) self._content_ctags = SubStorage(self._tools) self._licenses = SubStorage(self._tools) self._content_metadata = SubStorage(self._tools) self._revision_intrinsic_metadata = SubStorage(self._tools) self._origin_intrinsic_metadata = SubStorage(self._tools) def check_config(self, *, check_write): return True def content_mimetype_missing(self, mimetypes): yield from self._mimetypes.missing(mimetypes) def content_mimetype_get_range( - self, start, end, indexer_configuration_id, limit=1000): - return self._mimetypes.get_range( - start, end, indexer_configuration_id, limit) + self, start, end, indexer_configuration_id, limit=1000 + ): + return self._mimetypes.get_range(start, end, indexer_configuration_id, limit) def content_mimetype_add( - self, mimetypes: List[Dict], - conflict_update: bool = False) -> Dict[str, int]: + self, mimetypes: List[Dict], conflict_update: bool = False + ) -> Dict[str, int]: check_id_types(mimetypes) added = self._mimetypes.add(mimetypes, conflict_update) - return {'content_mimetype:add': added} + return {"content_mimetype:add": added} def content_mimetype_get(self, ids): yield from self._mimetypes.get(ids) def content_language_missing(self, languages): yield from self._languages.missing(languages) def content_language_get(self, ids): yield from self._languages.get(ids) def content_language_add( - self, languages: List[Dict], - conflict_update: bool = False) -> Dict[str, int]: + self, languages: List[Dict], conflict_update: bool = False + ) -> Dict[str, int]: check_id_types(languages) added = self._languages.add(languages, conflict_update) - return {'content_language:add': added} + return {"content_language:add": added} def content_ctags_missing(self, ctags): yield from self._content_ctags.missing(ctags) def content_ctags_get(self, ids): for item in self._content_ctags.get(ids): - for item_ctags_item in item['ctags']: - yield { - 'id': item['id'], - 'tool': item['tool'], - **item_ctags_item - } + for item_ctags_item in item["ctags"]: + yield {"id": item["id"], "tool": item["tool"], **item_ctags_item} def content_ctags_add( - self, ctags: List[Dict], - conflict_update: bool = False) -> Dict[str, int]: + self, ctags: List[Dict], conflict_update: bool = False + ) -> Dict[str, int]: check_id_types(ctags) - added = self._content_ctags.add_merge(ctags, conflict_update, 'ctags') - return {'content_ctags:add': added} + added = self._content_ctags.add_merge(ctags, conflict_update, "ctags") + return {"content_ctags:add": added} - def content_ctags_search(self, expression, - limit=10, last_sha1=None): + def content_ctags_search(self, expression, limit=10, last_sha1=None): nb_matches = 0 - for ((id_, tool_id), item) in \ - sorted(self._content_ctags._data.items()): + for ((id_, tool_id), item) in sorted(self._content_ctags._data.items()): if id_ <= (last_sha1 or bytes(0 for _ in range(SHA1_DIGEST_SIZE))): continue - for ctags_item in item['ctags']: - if ctags_item['name'] != expression: + for ctags_item in item["ctags"]: + if ctags_item["name"] != expression: continue nb_matches += 1 yield { - 'id': id_, - 'tool': _transform_tool(self._tools[tool_id]), - **ctags_item + "id": id_, + "tool": _transform_tool(self._tools[tool_id]), + **ctags_item, } if nb_matches >= limit: return def content_fossology_license_get(self, ids): # Rewrites the output of SubStorage.get from the old format to # the new one. SubStorage.get should be updated once all other # *_get methods use the new format. # See: https://forge.softwareheritage.org/T1433 res = {} for d in self._licenses.get(ids): - res.setdefault(d.pop('id'), []).append(d) + res.setdefault(d.pop("id"), []).append(d) for (id_, facts) in res.items(): yield {id_: facts} def content_fossology_license_add( - self, licenses: List[Dict], - conflict_update: bool = False) -> Dict[str, int]: + self, licenses: List[Dict], conflict_update: bool = False + ) -> Dict[str, int]: check_id_types(licenses) - added = self._licenses.add_merge(licenses, conflict_update, 'licenses') - return {'fossology_license_add:add': added} + added = self._licenses.add_merge(licenses, conflict_update, "licenses") + return {"fossology_license_add:add": added} def content_fossology_license_get_range( - self, start, end, indexer_configuration_id, limit=1000): - return self._licenses.get_range( - start, end, indexer_configuration_id, limit) + self, start, end, indexer_configuration_id, limit=1000 + ): + return self._licenses.get_range(start, end, indexer_configuration_id, limit) def content_metadata_missing(self, metadata): yield from self._content_metadata.missing(metadata) def content_metadata_get(self, ids): yield from self._content_metadata.get(ids) def content_metadata_add( - self, metadata: List[Dict], - conflict_update: bool = False) -> Dict[str, int]: + self, metadata: List[Dict], conflict_update: bool = False + ) -> Dict[str, int]: check_id_types(metadata) added = self._content_metadata.add(metadata, conflict_update) - return {'content_metadata:add': added} + return {"content_metadata:add": added} def revision_intrinsic_metadata_missing(self, metadata): yield from self._revision_intrinsic_metadata.missing(metadata) def revision_intrinsic_metadata_get(self, ids): yield from self._revision_intrinsic_metadata.get(ids) def revision_intrinsic_metadata_add( - self, metadata: List[Dict], - conflict_update: bool = False) -> Dict[str, int]: + self, metadata: List[Dict], conflict_update: bool = False + ) -> Dict[str, int]: check_id_types(metadata) - added = self._revision_intrinsic_metadata.add( - metadata, conflict_update) - return {'revision_intrinsic_metadata:add': added} + added = self._revision_intrinsic_metadata.add(metadata, conflict_update) + return {"revision_intrinsic_metadata:add": added} def revision_intrinsic_metadata_delete(self, entries: List[Dict]) -> Dict: deleted = self._revision_intrinsic_metadata.delete(entries) - return {'revision_intrinsic_metadata:del': deleted} + return {"revision_intrinsic_metadata:del": deleted} def origin_intrinsic_metadata_get(self, ids): yield from self._origin_intrinsic_metadata.get(ids) def origin_intrinsic_metadata_add( - self, metadata: List[Dict], - conflict_update: bool = False) -> Dict[str, int]: + self, metadata: List[Dict], conflict_update: bool = False + ) -> Dict[str, int]: added = self._origin_intrinsic_metadata.add(metadata, conflict_update) - return {'origin_intrinsic_metadata:add': added} + return {"origin_intrinsic_metadata:add": added} def origin_intrinsic_metadata_delete(self, entries: List[Dict]) -> Dict: deleted = self._origin_intrinsic_metadata.delete(entries) - return {'origin_intrinsic_metadata:del': deleted} + return {"origin_intrinsic_metadata:del": deleted} - def origin_intrinsic_metadata_search_fulltext( - self, conjunction, limit=100): + def origin_intrinsic_metadata_search_fulltext(self, conjunction, limit=100): # A very crude fulltext search implementation, but that's enough # to work on English metadata - tokens_re = re.compile('[a-zA-Z0-9]+') - search_tokens = list(itertools.chain( - *map(tokens_re.findall, conjunction))) + tokens_re = re.compile("[a-zA-Z0-9]+") + search_tokens = list(itertools.chain(*map(tokens_re.findall, conjunction))) def rank(data): # Tokenize the metadata - text = json.dumps(data['metadata']) + text = json.dumps(data["metadata"]) text_tokens = tokens_re.findall(text) text_token_occurences = Counter(text_tokens) # Count the number of occurrences of search tokens in the text score = 0 for search_token in search_tokens: if text_token_occurences[search_token] == 0: # Search token is not in the text. return 0 score += text_token_occurences[search_token] # Normalize according to the text's length return score / math.log(len(text_tokens)) - results = [(rank(data), data) - for data in self._origin_intrinsic_metadata.get_all()] + results = [ + (rank(data), data) for data in self._origin_intrinsic_metadata.get_all() + ] results = [(rank_, data) for (rank_, data) in results if rank_ > 0] - results.sort(key=operator.itemgetter(0), # Don't try to order 'data' - reverse=True) + results.sort( + key=operator.itemgetter(0), reverse=True # Don't try to order 'data' + ) for (rank_, result) in results[:limit]: yield result def origin_intrinsic_metadata_search_by_producer( - self, page_token='', limit=100, ids_only=False, - mappings=None, tool_ids=None): + self, page_token="", limit=100, ids_only=False, mappings=None, tool_ids=None + ): assert isinstance(page_token, str) nb_results = 0 if mappings is not None: mappings = frozenset(mappings) if tool_ids is not None: tool_ids = frozenset(tool_ids) origins = [] # we go to limit+1 to check whether we should add next_page_token in # the response for entry in self._origin_intrinsic_metadata.get_all(): - if entry['id'] <= page_token: + if entry["id"] <= page_token: continue if nb_results >= (limit + 1): break - if mappings is not None and mappings.isdisjoint(entry['mappings']): + if mappings is not None and mappings.isdisjoint(entry["mappings"]): continue - if tool_ids is not None and entry['tool']['id'] not in tool_ids: + if tool_ids is not None and entry["tool"]["id"] not in tool_ids: continue origins.append(entry) nb_results += 1 result = {} if len(origins) > limit: origins = origins[:limit] - result['next_page_token'] = origins[-1]['id'] + result["next_page_token"] = origins[-1]["id"] if ids_only: - origins = [origin['id'] for origin in origins] - result['origins'] = origins + origins = [origin["id"] for origin in origins] + result["origins"] = origins return result def origin_intrinsic_metadata_stats(self): mapping_count = {m: 0 for m in MAPPING_NAMES} total = non_empty = 0 for data in self._origin_intrinsic_metadata.get_all(): total += 1 - if set(data['metadata']) - {'@context'}: + if set(data["metadata"]) - {"@context"}: non_empty += 1 - for mapping in data['mappings']: + for mapping in data["mappings"]: mapping_count[mapping] += 1 - return { - 'per_mapping': mapping_count, - 'total': total, - 'non_empty': non_empty - } + return {"per_mapping": mapping_count, "total": total, "non_empty": non_empty} def indexer_configuration_add(self, tools): inserted = [] for tool in tools: tool = tool.copy() id_ = self._tool_key(tool) - tool['id'] = id_ + tool["id"] = id_ self._tools[id_] = tool inserted.append(tool) return inserted def indexer_configuration_get(self, tool): return self._tools.get(self._tool_key(tool)) def _tool_key(self, tool): - return hash((tool['tool_name'], tool['tool_version'], - json.dumps(tool['tool_configuration'], sort_keys=True))) + return hash( + ( + tool["tool_name"], + tool["tool_version"], + json.dumps(tool["tool_configuration"], sort_keys=True), + ) + ) diff --git a/swh/indexer/storage/interface.py b/swh/indexer/storage/interface.py index c64925f..d059cf5 100644 --- a/swh/indexer/storage/interface.py +++ b/swh/indexer/storage/interface.py @@ -1,636 +1,641 @@ # Copyright (C) 2015-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from typing import Dict, List from swh.core.api import remote_api_endpoint class IndexerStorageInterface: - @remote_api_endpoint('check_config') + @remote_api_endpoint("check_config") def check_config(self, *, check_write): """Check that the storage is configured and ready to go.""" ... - @remote_api_endpoint('content_mimetype/missing') + @remote_api_endpoint("content_mimetype/missing") def content_mimetype_missing(self, mimetypes): """Generate mimetypes missing from storage. Args: mimetypes (iterable): iterable of dict with keys: - **id** (bytes): sha1 identifier - **indexer_configuration_id** (int): tool used to compute the results Yields: tuple (id, indexer_configuration_id): missing id """ ... - def _content_get_range(self, content_type, start, end, - indexer_configuration_id, limit=1000, - with_textual_data=False): + def _content_get_range( + self, + content_type, + start, + end, + indexer_configuration_id, + limit=1000, + with_textual_data=False, + ): """Retrieve ids of type content_type within range [start, end] bound by limit. Args: **content_type** (str): content's type (mimetype, language, etc...) **start** (bytes): Starting identifier range (expected smaller than end) **end** (bytes): Ending identifier range (expected larger than start) **indexer_configuration_id** (int): The tool used to index data **limit** (int): Limit result (default to 1000) **with_textual_data** (bool): Deal with only textual content (True) or all content (all contents by defaults, False) Raises: ValueError for; - limit to None - wrong content_type provided Returns: a dict with keys: - **ids** [bytes]: iterable of content ids within the range. - **next** (Optional[bytes]): The next range of sha1 starts at this sha1 if any """ ... - @remote_api_endpoint('content_mimetype/range') - def content_mimetype_get_range(self, start, end, indexer_configuration_id, - limit=1000): + @remote_api_endpoint("content_mimetype/range") + def content_mimetype_get_range( + self, start, end, indexer_configuration_id, limit=1000 + ): """Retrieve mimetypes within range [start, end] bound by limit. Args: **start** (bytes): Starting identifier range (expected smaller than end) **end** (bytes): Ending identifier range (expected larger than start) **indexer_configuration_id** (int): The tool used to index data **limit** (int): Limit result (default to 1000) Raises: ValueError for limit to None Returns: a dict with keys: - **ids** [bytes]: iterable of content ids within the range. - **next** (Optional[bytes]): The next range of sha1 starts at this sha1 if any """ ... - @remote_api_endpoint('content_mimetype/add') - def content_mimetype_add(self, mimetypes: List[Dict], - conflict_update: bool = False) -> Dict[str, int]: + @remote_api_endpoint("content_mimetype/add") + def content_mimetype_add( + self, mimetypes: List[Dict], conflict_update: bool = False + ) -> Dict[str, int]: """Add mimetypes not present in storage. Args: mimetypes (iterable): dictionaries with keys: - **id** (bytes): sha1 identifier - **mimetype** (bytes): raw content's mimetype - **encoding** (bytes): raw content's encoding - **indexer_configuration_id** (int): tool's id used to compute the results - **conflict_update** (bool): Flag to determine if we want to overwrite (``True``) or skip duplicates (``False``, the default) Returns: Dict summary of number of rows added """ ... - @remote_api_endpoint('content_mimetype') + @remote_api_endpoint("content_mimetype") def content_mimetype_get(self, ids): """Retrieve full content mimetype per ids. Args: ids (iterable): sha1 identifier Yields: mimetypes (iterable): dictionaries with keys: - **id** (bytes): sha1 identifier - **mimetype** (bytes): raw content's mimetype - **encoding** (bytes): raw content's encoding - **tool** (dict): Tool used to compute the language """ ... - @remote_api_endpoint('content_language/missing') + @remote_api_endpoint("content_language/missing") def content_language_missing(self, languages): """List languages missing from storage. Args: languages (iterable): dictionaries with keys: - **id** (bytes): sha1 identifier - **indexer_configuration_id** (int): tool used to compute the results Yields: an iterable of missing id for the tuple (id, indexer_configuration_id) """ ... - @remote_api_endpoint('content_language') + @remote_api_endpoint("content_language") def content_language_get(self, ids): """Retrieve full content language per ids. Args: ids (iterable): sha1 identifier Yields: languages (iterable): dictionaries with keys: - **id** (bytes): sha1 identifier - **lang** (bytes): raw content's language - **tool** (dict): Tool used to compute the language """ ... - @remote_api_endpoint('content_language/add') + @remote_api_endpoint("content_language/add") def content_language_add( - self, languages: List[Dict], - conflict_update: bool = False) -> Dict[str, int]: + self, languages: List[Dict], conflict_update: bool = False + ) -> Dict[str, int]: """Add languages not present in storage. Args: languages (iterable): dictionaries with keys: - **id** (bytes): sha1 - **lang** (bytes): language detected conflict_update (bool): Flag to determine if we want to overwrite (true) or skip duplicates (false, the default) Returns: Dict summary of number of rows added """ ... - @remote_api_endpoint('content/ctags/missing') + @remote_api_endpoint("content/ctags/missing") def content_ctags_missing(self, ctags): """List ctags missing from storage. Args: ctags (iterable): dicts with keys: - **id** (bytes): sha1 identifier - **indexer_configuration_id** (int): tool used to compute the results Yields: an iterable of missing id for the tuple (id, indexer_configuration_id) """ ... - @remote_api_endpoint('content/ctags') + @remote_api_endpoint("content/ctags") def content_ctags_get(self, ids): """Retrieve ctags per id. Args: ids (iterable): sha1 checksums Yields: Dictionaries with keys: - **id** (bytes): content's identifier - **name** (str): symbol's name - **kind** (str): symbol's kind - **lang** (str): language for that content - **tool** (dict): tool used to compute the ctags' info """ ... - @remote_api_endpoint('content/ctags/add') - def content_ctags_add(self, ctags: List[Dict], - conflict_update: bool = False) -> Dict[str, int]: + @remote_api_endpoint("content/ctags/add") + def content_ctags_add( + self, ctags: List[Dict], conflict_update: bool = False + ) -> Dict[str, int]: """Add ctags not present in storage Args: ctags (iterable): dictionaries with keys: - **id** (bytes): sha1 - **ctags** ([list): List of dictionary with keys: name, kind, line, lang Returns: Dict summary of number of rows added """ ... - @remote_api_endpoint('content/ctags/search') - def content_ctags_search(self, expression, - limit=10, last_sha1=None): + @remote_api_endpoint("content/ctags/search") + def content_ctags_search(self, expression, limit=10, last_sha1=None): """Search through content's raw ctags symbols. Args: expression (str): Expression to search for limit (int): Number of rows to return (default to 10). last_sha1 (str): Offset from which retrieving data (default to ''). Yields: rows of ctags including id, name, lang, kind, line, etc... """ ... - @remote_api_endpoint('content/fossology_license') + @remote_api_endpoint("content/fossology_license") def content_fossology_license_get(self, ids): """Retrieve licenses per id. Args: ids (iterable): sha1 checksums Yields: dict: ``{id: facts}`` where ``facts`` is a dict with the following keys: - **licenses** ([str]): associated licenses for that content - **tool** (dict): Tool used to compute the license """ ... - @remote_api_endpoint('content/fossology_license/add') + @remote_api_endpoint("content/fossology_license/add") def content_fossology_license_add( - self, licenses: List[Dict], - conflict_update: bool = False) -> Dict[str, int]: + self, licenses: List[Dict], conflict_update: bool = False + ) -> Dict[str, int]: """Add licenses not present in storage. Args: licenses (iterable): dictionaries with keys: - **id**: sha1 - **licenses** ([bytes]): List of licenses associated to sha1 - **tool** (str): nomossa conflict_update: Flag to determine if we want to overwrite (true) or skip duplicates (false, the default) Returns: Dict summary of number of rows added """ ... - @remote_api_endpoint('content/fossology_license/range') + @remote_api_endpoint("content/fossology_license/range") def content_fossology_license_get_range( - self, start, end, indexer_configuration_id, - limit=1000): + self, start, end, indexer_configuration_id, limit=1000 + ): """Retrieve licenses within range [start, end] bound by limit. Args: **start** (bytes): Starting identifier range (expected smaller than end) **end** (bytes): Ending identifier range (expected larger than start) **indexer_configuration_id** (int): The tool used to index data **limit** (int): Limit result (default to 1000) Raises: ValueError for limit to None Returns: a dict with keys: - **ids** [bytes]: iterable of content ids within the range. - **next** (Optional[bytes]): The next range of sha1 starts at this sha1 if any """ ... - @remote_api_endpoint('content_metadata/missing') + @remote_api_endpoint("content_metadata/missing") def content_metadata_missing(self, metadata): """List metadata missing from storage. Args: metadata (iterable): dictionaries with keys: - **id** (bytes): sha1 identifier - **indexer_configuration_id** (int): tool used to compute the results Yields: missing sha1s """ ... - @remote_api_endpoint('content_metadata') + @remote_api_endpoint("content_metadata") def content_metadata_get(self, ids): """Retrieve metadata per id. Args: ids (iterable): sha1 checksums Yields: dictionaries with the following keys: id (bytes) metadata (str): associated metadata tool (dict): tool used to compute metadata """ ... - @remote_api_endpoint('content_metadata/add') + @remote_api_endpoint("content_metadata/add") def content_metadata_add( - self, metadata: List[Dict], - conflict_update: bool = False) -> Dict[str, int]: + self, metadata: List[Dict], conflict_update: bool = False + ) -> Dict[str, int]: """Add metadata not present in storage. Args: metadata (iterable): dictionaries with keys: - **id**: sha1 - **metadata**: arbitrary dict conflict_update: Flag to determine if we want to overwrite (true) or skip duplicates (false, the default) Returns: Dict summary of number of rows added """ ... - @remote_api_endpoint('revision_intrinsic_metadata/missing') + @remote_api_endpoint("revision_intrinsic_metadata/missing") def revision_intrinsic_metadata_missing(self, metadata): """List metadata missing from storage. Args: metadata (iterable): dictionaries with keys: - **id** (bytes): sha1_git revision identifier - **indexer_configuration_id** (int): tool used to compute the results Yields: missing ids """ ... - @remote_api_endpoint('revision_intrinsic_metadata') + @remote_api_endpoint("revision_intrinsic_metadata") def revision_intrinsic_metadata_get(self, ids): """Retrieve revision metadata per id. Args: ids (iterable): sha1 checksums Yields: : dictionaries with the following keys: - **id** (bytes) - **metadata** (str): associated metadata - **tool** (dict): tool used to compute metadata - **mappings** (List[str]): list of mappings used to translate these metadata """ ... - @remote_api_endpoint('revision_intrinsic_metadata/add') + @remote_api_endpoint("revision_intrinsic_metadata/add") def revision_intrinsic_metadata_add( - self, metadata: List[Dict], - conflict_update: bool = False) -> Dict[str, int]: + self, metadata: List[Dict], conflict_update: bool = False + ) -> Dict[str, int]: """Add metadata not present in storage. Args: metadata (iterable): dictionaries with keys: - **id**: sha1_git of revision - **metadata**: arbitrary dict - **indexer_configuration_id**: tool used to compute metadata - **mappings** (List[str]): list of mappings used to translate these metadata conflict_update: Flag to determine if we want to overwrite (true) or skip duplicates (false, the default) Returns: Dict summary of number of rows added """ ... - @remote_api_endpoint('revision_intrinsic_metadata/delete') + @remote_api_endpoint("revision_intrinsic_metadata/delete") def revision_intrinsic_metadata_delete(self, entries: List[Dict]) -> Dict: """Remove revision metadata from the storage. Args: entries (dict): dictionaries with the following keys: - **id** (bytes): revision identifier - **indexer_configuration_id** (int): tool used to compute metadata Returns: Summary of number of rows deleted """ ... - @remote_api_endpoint('origin_intrinsic_metadata') + @remote_api_endpoint("origin_intrinsic_metadata") def origin_intrinsic_metadata_get(self, ids): """Retrieve origin metadata per id. Args: ids (iterable): origin identifiers Yields: list: dictionaries with the following keys: - **id** (str): origin url - **from_revision** (bytes): which revision this metadata was extracted from - **metadata** (str): associated metadata - **tool** (dict): tool used to compute metadata - **mappings** (List[str]): list of mappings used to translate these metadata """ ... - @remote_api_endpoint('origin_intrinsic_metadata/add') + @remote_api_endpoint("origin_intrinsic_metadata/add") def origin_intrinsic_metadata_add( - self, metadata: List[Dict], - conflict_update: bool = False) -> Dict[str, int]: + self, metadata: List[Dict], conflict_update: bool = False + ) -> Dict[str, int]: """Add origin metadata not present in storage. Args: metadata (iterable): dictionaries with keys: - **id**: origin urls - **from_revision**: sha1 id of the revision used to generate these metadata. - **metadata**: arbitrary dict - **indexer_configuration_id**: tool used to compute metadata - **mappings** (List[str]): list of mappings used to translate these metadata conflict_update: Flag to determine if we want to overwrite (true) or skip duplicates (false, the default) Returns: Dict summary of number of rows added """ ... - @remote_api_endpoint('origin_intrinsic_metadata/delete') - def origin_intrinsic_metadata_delete( - self, entries: List[Dict]) -> Dict: + @remote_api_endpoint("origin_intrinsic_metadata/delete") + def origin_intrinsic_metadata_delete(self, entries: List[Dict]) -> Dict: """Remove origin metadata from the storage. Args: entries (dict): dictionaries with the following keys: - **id** (str): origin urls - **indexer_configuration_id** (int): tool used to compute metadata Returns: Summary of number of rows deleted """ ... - @remote_api_endpoint('origin_intrinsic_metadata/search/fulltext') - def origin_intrinsic_metadata_search_fulltext( - self, conjunction, limit=100): + @remote_api_endpoint("origin_intrinsic_metadata/search/fulltext") + def origin_intrinsic_metadata_search_fulltext(self, conjunction, limit=100): """Returns the list of origins whose metadata contain all the terms. Args: conjunction (List[str]): List of terms to be searched for. limit (int): The maximum number of results to return Yields: list: dictionaries with the following keys: - **id** (str): origin urls - **from_revision**: sha1 id of the revision used to generate these metadata. - **metadata** (str): associated metadata - **tool** (dict): tool used to compute metadata - **mappings** (List[str]): list of mappings used to translate these metadata """ ... - @remote_api_endpoint('origin_intrinsic_metadata/search/by_producer') + @remote_api_endpoint("origin_intrinsic_metadata/search/by_producer") def origin_intrinsic_metadata_search_by_producer( - self, page_token='', limit=100, ids_only=False, - mappings=None, tool_ids=None): + self, page_token="", limit=100, ids_only=False, mappings=None, tool_ids=None + ): """Returns the list of origins whose metadata contain all the terms. Args: page_token (str): Opaque token used for pagination. limit (int): The maximum number of results to return ids_only (bool): Determines whether only origin urls are returned or the content as well mappings (List[str]): Returns origins whose intrinsic metadata were generated using at least one of these mappings. Returns: dict: dict with the following keys: - **next_page_token** (str, optional): opaque token to be used as `page_token` for retrieving the next page. If absent, there is no more pages to gather. - **origins** (list): list of origin url (str) if `ids_only=True` else dictionaries with the following keys: - **id** (str): origin urls - **from_revision**: sha1 id of the revision used to generate these metadata. - **metadata** (str): associated metadata - **tool** (dict): tool used to compute metadata - **mappings** (List[str]): list of mappings used to translate these metadata """ ... - @remote_api_endpoint('origin_intrinsic_metadata/stats') - def origin_intrinsic_metadata_stats( - self): + @remote_api_endpoint("origin_intrinsic_metadata/stats") + def origin_intrinsic_metadata_stats(self): """Returns counts of indexed metadata per origins, broken down into metadata types. Returns: dict: dictionary with keys: - total (int): total number of origins that were indexed (possibly yielding an empty metadata dictionary) - non_empty (int): total number of origins that we extracted a non-empty metadata dictionary from - per_mapping (dict): a dictionary with mapping names as keys and number of origins whose indexing used this mapping. Note that indexing a given origin may use 0, 1, or many mappings. """ ... - @remote_api_endpoint('indexer_configuration/add') + @remote_api_endpoint("indexer_configuration/add") def indexer_configuration_add(self, tools): """Add new tools to the storage. Args: tools ([dict]): List of dictionary representing tool to insert in the db. Dictionary with the following keys: - **tool_name** (str): tool's name - **tool_version** (str): tool's version - **tool_configuration** (dict): tool's configuration (free form dict) Returns: List of dict inserted in the db (holding the id key as well). The order of the list is not guaranteed to match the order of the initial list. """ ... - @remote_api_endpoint('indexer_configuration/data') + @remote_api_endpoint("indexer_configuration/data") def indexer_configuration_get(self, tool): """Retrieve tool information. Args: tool (dict): Dictionary representing a tool with the following keys: - **tool_name** (str): tool's name - **tool_version** (str): tool's version - **tool_configuration** (dict): tool's configuration (free form dict) Returns: The same dictionary with an `id` key, None otherwise. """ ... diff --git a/swh/indexer/storage/metrics.py b/swh/indexer/storage/metrics.py index 61aafa3..55634d2 100644 --- a/swh/indexer/storage/metrics.py +++ b/swh/indexer/storage/metrics.py @@ -1,79 +1,83 @@ # Copyright (C) 2019-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from functools import wraps import logging from swh.core.statsd import statsd -OPERATIONS_METRIC = 'swh_indexer_storage_operations_total' +OPERATIONS_METRIC = "swh_indexer_storage_operations_total" OPERATIONS_UNIT_METRIC = "swh_indexer_storage_operations_{unit}_total" DURATION_METRIC = "swh_indexer_storage_request_duration_seconds" def timed(f): """Time that function! """ + @wraps(f) def d(*a, **kw): - with statsd.timed(DURATION_METRIC, tags={'endpoint': f.__name__}): + with statsd.timed(DURATION_METRIC, tags={"endpoint": f.__name__}): return f(*a, **kw) return d def send_metric(metric, count, method_name): """Send statsd metric with count for method `method_name` If count is 0, the metric is discarded. If the metric is not parseable, the metric is discarded with a log message. Args: metric (str): Metric's name (e.g content:add, content:add:bytes) count (int): Associated value for the metric method_name (str): Method's name Returns: Bool to explicit if metric has been set or not """ if count == 0: return False - metric_type = metric.split(':') + metric_type = metric.split(":") _length = len(metric_type) if _length == 2: object_type, operation = metric_type metric_name = OPERATIONS_METRIC elif _length == 3: object_type, operation, unit = metric_type metric_name = OPERATIONS_UNIT_METRIC.format(unit=unit) else: - logging.warning('Skipping unknown metric {%s: %s}' % ( - metric, count)) + logging.warning("Skipping unknown metric {%s: %s}" % (metric, count)) return False statsd.increment( - metric_name, count, tags={ - 'endpoint': method_name, - 'object_type': object_type, - 'operation': operation, - }) + metric_name, + count, + tags={ + "endpoint": method_name, + "object_type": object_type, + "operation": operation, + }, + ) return True def process_metrics(f): """Increment object counters for the decorated function. """ + @wraps(f) def d(*a, **kw): r = f(*a, **kw) for metric, count in r.items(): send_metric(metric=metric, count=count, method_name=f.__name__) return r return d diff --git a/swh/indexer/tasks.py b/swh/indexer/tasks.py index 0e4331f..2ca6cdd 100644 --- a/swh/indexer/tasks.py +++ b/swh/indexer/tasks.py @@ -1,50 +1,48 @@ # Copyright (C) 2016-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from celery import current_app as app from .mimetype import MimetypeIndexer, MimetypeRangeIndexer from .ctags import CtagsIndexer -from .fossology_license import ( - FossologyLicenseIndexer, FossologyLicenseRangeIndexer -) +from .fossology_license import FossologyLicenseIndexer, FossologyLicenseRangeIndexer from .rehash import RecomputeChecksums from .metadata import OriginMetadataIndexer -@app.task(name=__name__ + '.OriginMetadata') +@app.task(name=__name__ + ".OriginMetadata") def origin_metadata(*args, **kwargs): return OriginMetadataIndexer().run(*args, **kwargs) -@app.task(name=__name__ + '.Ctags') +@app.task(name=__name__ + ".Ctags") def ctags(*args, **kwargs): return CtagsIndexer().run(*args, **kwargs) -@app.task(name=__name__ + '.ContentFossologyLicense') +@app.task(name=__name__ + ".ContentFossologyLicense") def fossology_license(*args, **kwargs): return FossologyLicenseIndexer().run(*args, **kwargs) -@app.task(name=__name__ + '.RecomputeChecksums') +@app.task(name=__name__ + ".RecomputeChecksums") def recompute_checksums(*args, **kwargs): return RecomputeChecksums().run(*args, **kwargs) -@app.task(name=__name__ + '.ContentMimetype') +@app.task(name=__name__ + ".ContentMimetype") def mimetype(*args, **kwargs): return MimetypeIndexer().run(*args, **kwargs) -@app.task(name=__name__ + '.ContentRangeMimetype') +@app.task(name=__name__ + ".ContentRangeMimetype") def range_mimetype(*args, **kwargs): return MimetypeRangeIndexer().run(*args, **kwargs) -@app.task(name=__name__ + '.ContentRangeFossologyLicense') +@app.task(name=__name__ + ".ContentRangeFossologyLicense") def range_license(*args, **kwargs): return FossologyLicenseRangeIndexer().run(*args, **kwargs) diff --git a/swh/indexer/tests/__init__.py b/swh/indexer/tests/__init__.py index 35c2fa8..4661c01 100644 --- a/swh/indexer/tests/__init__.py +++ b/swh/indexer/tests/__init__.py @@ -1,15 +1,15 @@ import swh.indexer from os import path from celery.contrib.testing.worker import start_worker import celery.contrib.testing.tasks # noqa from swh.scheduler.celery_backend.config import app -__all__ = ['start_worker_thread'] +__all__ = ["start_worker_thread"] -SQL_DIR = path.join(path.dirname(swh.indexer.__file__), 'sql') +SQL_DIR = path.join(path.dirname(swh.indexer.__file__), "sql") def start_worker_thread(): return start_worker(app) diff --git a/swh/indexer/tests/conftest.py b/swh/indexer/tests/conftest.py index a2ea6ec..3891b36 100644 --- a/swh/indexer/tests/conftest.py +++ b/swh/indexer/tests/conftest.py @@ -1,90 +1,84 @@ # Copyright (C) 2019-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from datetime import timedelta from unittest.mock import patch import pytest from swh.objstorage import get_objstorage from swh.scheduler.tests.conftest import * # noqa from swh.storage import get_storage from swh.indexer.storage import get_indexer_storage from .utils import fill_storage, fill_obj_storage -TASK_NAMES = ['revision_intrinsic_metadata', 'origin_intrinsic_metadata'] +TASK_NAMES = ["revision_intrinsic_metadata", "origin_intrinsic_metadata"] -storage_config = { - 'cls': 'pipeline', - 'steps': [ - {'cls': 'validate'}, - {'cls': 'memory'}, - ] -} +storage_config = {"cls": "pipeline", "steps": [{"cls": "validate"}, {"cls": "memory"},]} @pytest.fixture def indexer_scheduler(swh_scheduler): for taskname in TASK_NAMES: - swh_scheduler.create_task_type({ - 'type': taskname, - 'description': 'The {} indexer testing task'.format(taskname), - 'backend_name': 'swh.indexer.tests.tasks.{}'.format(taskname), - 'default_interval': timedelta(days=1), - 'min_interval': timedelta(hours=6), - 'max_interval': timedelta(days=12), - 'num_retries': 3, - }) + swh_scheduler.create_task_type( + { + "type": taskname, + "description": "The {} indexer testing task".format(taskname), + "backend_name": "swh.indexer.tests.tasks.{}".format(taskname), + "default_interval": timedelta(days=1), + "min_interval": timedelta(hours=6), + "max_interval": timedelta(days=12), + "num_retries": 3, + } + ) return swh_scheduler @pytest.fixture def idx_storage(): """An instance of in-memory indexer storage that gets injected into all indexers classes. """ - idx_storage = get_indexer_storage('memory', {}) - with patch('swh.indexer.storage.in_memory.IndexerStorage') \ - as idx_storage_mock: + idx_storage = get_indexer_storage("memory", {}) + with patch("swh.indexer.storage.in_memory.IndexerStorage") as idx_storage_mock: idx_storage_mock.return_value = idx_storage yield idx_storage @pytest.fixture def storage(): """An instance of in-memory storage that gets injected into all indexers classes. """ storage = get_storage(**storage_config) fill_storage(storage) - with patch('swh.storage.in_memory.InMemoryStorage') as storage_mock: + with patch("swh.storage.in_memory.InMemoryStorage") as storage_mock: storage_mock.return_value = storage yield storage @pytest.fixture def obj_storage(): """An instance of in-memory objstorage that gets injected into all indexers classes. """ - objstorage = get_objstorage('memory', {}) + objstorage = get_objstorage("memory", {}) fill_obj_storage(objstorage) - with patch.dict('swh.objstorage._STORAGE_CLASSES', - {'memory': lambda: objstorage}): + with patch.dict("swh.objstorage._STORAGE_CLASSES", {"memory": lambda: objstorage}): yield objstorage -@pytest.fixture(scope='session') # type: ignore # expected redefinition +@pytest.fixture(scope="session") # type: ignore # expected redefinition def celery_includes(): return [ - 'swh.indexer.tests.tasks', - 'swh.indexer.tasks', + "swh.indexer.tests.tasks", + "swh.indexer.tasks", ] diff --git a/swh/indexer/tests/storage/__init__.py b/swh/indexer/tests/storage/__init__.py index afb1c71..ba29787 100644 --- a/swh/indexer/tests/storage/__init__.py +++ b/swh/indexer/tests/storage/__init__.py @@ -1,9 +1,9 @@ # Copyright (C) 2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from os import path import swh.storage -SQL_DIR = path.join(path.dirname(swh.indexer.__file__), 'sql') +SQL_DIR = path.join(path.dirname(swh.indexer.__file__), "sql") diff --git a/swh/indexer/tests/storage/conftest.py b/swh/indexer/tests/storage/conftest.py index cc6b500..e2df26c 100644 --- a/swh/indexer/tests/storage/conftest.py +++ b/swh/indexer/tests/storage/conftest.py @@ -1,80 +1,73 @@ # Copyright (C) 2015-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from os.path import join import pytest from . import SQL_DIR from swh.storage.tests.conftest import postgresql_fact from swh.indexer.storage import get_indexer_storage from swh.model.hashutil import hash_to_bytes -from .generate_data_test import ( - MIMETYPE_OBJECTS, FOSSOLOGY_LICENSES, TOOLS -) +from .generate_data_test import MIMETYPE_OBJECTS, FOSSOLOGY_LICENSES, TOOLS -DUMP_FILES = join(SQL_DIR, '*.sql') +DUMP_FILES = join(SQL_DIR, "*.sql") class DataObj(dict): def __getattr__(self, key): return self.__getitem__(key) def __setattr__(self, key, value): return self.__setitem__(key, value) @pytest.fixture def swh_indexer_storage_with_data(swh_indexer_storage): data = DataObj() tools = { - tool['tool_name']: { - 'id': tool['id'], - 'name': tool['tool_name'], - 'version': tool['tool_version'], - 'configuration': tool['tool_configuration'], - } - for tool in swh_indexer_storage.indexer_configuration_add(TOOLS)} + tool["tool_name"]: { + "id": tool["id"], + "name": tool["tool_name"], + "version": tool["tool_version"], + "configuration": tool["tool_configuration"], + } + for tool in swh_indexer_storage.indexer_configuration_add(TOOLS) + } data.tools = tools - data.sha1_1 = hash_to_bytes( - '34973274ccef6ab4dfaaf86599792fa9c3fe4689') - data.sha1_2 = hash_to_bytes( - '61c2b3a30496d329e21af70dd2d7e097046d07b7') - data.revision_id_1 = hash_to_bytes( - '7026b7c1a2af56521e951c01ed20f255fa054238') - data.revision_id_2 = hash_to_bytes( - '7026b7c1a2af56521e9587659012345678904321') - data.revision_id_3 = hash_to_bytes( - '7026b7c1a2af56521e9587659012345678904320') - data.origin_url_1 = 'file:///dev/0/zero' # 44434341 - data.origin_url_2 = 'file:///dev/1/one' # 44434342 - data.origin_url_3 = 'file:///dev/2/two' # 54974445 + data.sha1_1 = hash_to_bytes("34973274ccef6ab4dfaaf86599792fa9c3fe4689") + data.sha1_2 = hash_to_bytes("61c2b3a30496d329e21af70dd2d7e097046d07b7") + data.revision_id_1 = hash_to_bytes("7026b7c1a2af56521e951c01ed20f255fa054238") + data.revision_id_2 = hash_to_bytes("7026b7c1a2af56521e9587659012345678904321") + data.revision_id_3 = hash_to_bytes("7026b7c1a2af56521e9587659012345678904320") + data.origin_url_1 = "file:///dev/0/zero" # 44434341 + data.origin_url_2 = "file:///dev/1/one" # 44434342 + data.origin_url_3 = "file:///dev/2/two" # 54974445 data.mimetypes = [ - {**mimetype_obj, 'indexer_configuration_id': tools['file']['id']} + {**mimetype_obj, "indexer_configuration_id": tools["file"]["id"]} for mimetype_obj in MIMETYPE_OBJECTS ] swh_indexer_storage.content_mimetype_add(data.mimetypes) data.fossology_licenses = [ - {**fossology_obj, 'indexer_configuration_id': tools['nomos']['id']} + {**fossology_obj, "indexer_configuration_id": tools["nomos"]["id"]} for fossology_obj in FOSSOLOGY_LICENSES ] swh_indexer_storage._test_data = data return (swh_indexer_storage, data) swh_indexer_storage_postgresql = postgresql_fact( - 'postgresql_proc', dump_files=DUMP_FILES) + "postgresql_proc", dump_files=DUMP_FILES +) @pytest.fixture def swh_indexer_storage(swh_indexer_storage_postgresql): storage_config = { - 'cls': 'local', - 'args': { - 'db': swh_indexer_storage_postgresql.dsn, - }, + "cls": "local", + "args": {"db": swh_indexer_storage_postgresql.dsn,}, } return get_indexer_storage(**storage_config) diff --git a/swh/indexer/tests/storage/generate_data_test.py b/swh/indexer/tests/storage/generate_data_test.py index 5a6798e..5c42ef9 100644 --- a/swh/indexer/tests/storage/generate_data_test.py +++ b/swh/indexer/tests/storage/generate_data_test.py @@ -1,206 +1,201 @@ # Copyright (C) 2018-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from uuid import uuid1 from swh.model.hashutil import MultiHash -from hypothesis.strategies import (composite, sets, one_of, uuids, - tuples, sampled_from) +from hypothesis.strategies import composite, sets, one_of, uuids, tuples, sampled_from + MIMETYPES = [ - b'application/json', - b'application/octet-stream', - b'application/xml', - b'text/plain', + b"application/json", + b"application/octet-stream", + b"application/xml", + b"text/plain", ] ENCODINGS = [ - b'iso8859-1', - b'iso8859-15', - b'latin1', - b'utf-8', + b"iso8859-1", + b"iso8859-15", + b"latin1", + b"utf-8", ] def gen_mimetype(): """Generate one mimetype strategy. """ return one_of(sampled_from(MIMETYPES)) def gen_encoding(): """Generate one encoding strategy. """ return one_of(sampled_from(ENCODINGS)) def _init_content(uuid): """Given a uuid, initialize a content """ return { - 'id': MultiHash.from_data(uuid.bytes, {'sha1'}).digest()['sha1'], - 'indexer_configuration_id': 1, + "id": MultiHash.from_data(uuid.bytes, {"sha1"}).digest()["sha1"], + "indexer_configuration_id": 1, } @composite def gen_content_mimetypes(draw, *, min_size=0, max_size=100): """Generate valid and consistent content_mimetypes. Context: Test purposes Args: **draw** (callable): Used by hypothesis to generate data **min_size** (int): Minimal number of elements to generate (default: 0) **max_size** (int): Maximal number of elements to generate (default: 100) Returns: List of content_mimetypes as expected by the content_mimetype_add api endpoint. """ _ids = draw( sets( - tuples( - uuids(), - gen_mimetype(), - gen_encoding() - ), - min_size=min_size, max_size=max_size + tuples(uuids(), gen_mimetype(), gen_encoding()), + min_size=min_size, + max_size=max_size, ) ) content_mimetypes = [] for uuid, mimetype, encoding in _ids: - content_mimetypes.append({ - **_init_content(uuid), - 'mimetype': mimetype, - 'encoding': encoding, - }) + content_mimetypes.append( + {**_init_content(uuid), "mimetype": mimetype, "encoding": encoding,} + ) return content_mimetypes TOOLS = [ { - 'tool_name': 'universal-ctags', - 'tool_version': '~git7859817b', - 'tool_configuration': { + "tool_name": "universal-ctags", + "tool_version": "~git7859817b", + "tool_configuration": { "command_line": "ctags --fields=+lnz --sort=no --links=no " - "--output-format=json "} + "--output-format=json " + }, }, { - 'tool_name': 'swh-metadata-translator', - 'tool_version': '0.0.1', - 'tool_configuration': {"type": "local", "context": "NpmMapping"}, + "tool_name": "swh-metadata-translator", + "tool_version": "0.0.1", + "tool_configuration": {"type": "local", "context": "NpmMapping"}, }, { - 'tool_name': 'swh-metadata-detector', - 'tool_version': '0.0.1', - 'tool_configuration': { - "type": "local", "context": ["NpmMapping", "CodemetaMapping"]}, + "tool_name": "swh-metadata-detector", + "tool_version": "0.0.1", + "tool_configuration": { + "type": "local", + "context": ["NpmMapping", "CodemetaMapping"], + }, }, { - 'tool_name': 'swh-metadata-detector2', - 'tool_version': '0.0.1', - 'tool_configuration': { - "type": "local", "context": ["NpmMapping", "CodemetaMapping"]}, + "tool_name": "swh-metadata-detector2", + "tool_version": "0.0.1", + "tool_configuration": { + "type": "local", + "context": ["NpmMapping", "CodemetaMapping"], + }, }, { - 'tool_name': 'file', - 'tool_version': '5.22', - 'tool_configuration': {"command_line": "file --mime "}, + "tool_name": "file", + "tool_version": "5.22", + "tool_configuration": {"command_line": "file --mime "}, }, { - 'tool_name': 'pygments', - 'tool_version': '2.0.1+dfsg-1.1+deb8u1', - 'tool_configuration': { - "type": "library", "debian-package": "python3-pygments"}, + "tool_name": "pygments", + "tool_version": "2.0.1+dfsg-1.1+deb8u1", + "tool_configuration": {"type": "library", "debian-package": "python3-pygments"}, }, { - 'tool_name': 'pygments2', - 'tool_version': '2.0.1+dfsg-1.1+deb8u1', - 'tool_configuration': { + "tool_name": "pygments2", + "tool_version": "2.0.1+dfsg-1.1+deb8u1", + "tool_configuration": { "type": "library", "debian-package": "python3-pygments", - "max_content_size": 10240 + "max_content_size": 10240, }, }, { - 'tool_name': 'nomos', - 'tool_version': '3.1.0rc2-31-ga2cbb8c', - 'tool_configuration': {"command_line": "nomossa "}, + "tool_name": "nomos", + "tool_version": "3.1.0rc2-31-ga2cbb8c", + "tool_configuration": {"command_line": "nomossa "}, }, ] MIMETYPE_OBJECTS = [ - {'id': MultiHash.from_data(uuid1().bytes, {'sha1'}).digest()['sha1'], - 'mimetype': mt, - 'encoding': enc, - # 'indexer_configuration_id' will be added after TOOLS get registered - } + { + "id": MultiHash.from_data(uuid1().bytes, {"sha1"}).digest()["sha1"], + "mimetype": mt, + "encoding": enc, + # 'indexer_configuration_id' will be added after TOOLS get registered + } for mt in MIMETYPES - for enc in ENCODINGS] + for enc in ENCODINGS +] LICENSES = [ - b'3DFX', - b'BSD', - b'GPL', - b'Apache2', - b'MIT', + b"3DFX", + b"BSD", + b"GPL", + b"Apache2", + b"MIT", ] FOSSOLOGY_LICENSES = [ - {'id': MultiHash.from_data(uuid1().bytes, {'sha1'}).digest()['sha1'], - 'licenses': [LICENSES[i % len(LICENSES)], ], - # 'indexer_configuration_id' will be added after TOOLS get registered - } + { + "id": MultiHash.from_data(uuid1().bytes, {"sha1"}).digest()["sha1"], + "licenses": [LICENSES[i % len(LICENSES)],], + # 'indexer_configuration_id' will be added after TOOLS get registered + } for i in range(10) - ] +] def gen_license(): return one_of(sampled_from(LICENSES)) @composite def gen_content_fossology_licenses(draw, *, min_size=0, max_size=100): """Generate valid and consistent content_fossology_licenses. Context: Test purposes Args: **draw** (callable): Used by hypothesis to generate data **min_size** (int): Minimal number of elements to generate (default: 0) **max_size** (int): Maximal number of elements to generate (default: 100) Returns: List of content_fossology_licenses as expected by the content_fossology_license_add api endpoint. """ _ids = draw( - sets( - tuples( - uuids(), - gen_license(), - ), - min_size=min_size, max_size=max_size - ) + sets(tuples(uuids(), gen_license(),), min_size=min_size, max_size=max_size) ) content_licenses = [] for uuid, license in _ids: - content_licenses.append({ - **_init_content(uuid), - 'licenses': [license], - }) + content_licenses.append( + {**_init_content(uuid), "licenses": [license],} + ) return content_licenses diff --git a/swh/indexer/tests/storage/test_api_client.py b/swh/indexer/tests/storage/test_api_client.py index 4fe100c..dc52b59 100644 --- a/swh/indexer/tests/storage/test_api_client.py +++ b/swh/indexer/tests/storage/test_api_client.py @@ -1,42 +1,40 @@ # Copyright (C) 2015-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import pytest from swh.indexer.storage.api.client import RemoteStorage import swh.indexer.storage.api.server as server from swh.indexer.storage import get_indexer_storage from .test_storage import * # noqa @pytest.fixture def app(swh_indexer_storage_postgresql): storage_config = { - 'cls': 'local', - 'args': { - 'db': swh_indexer_storage_postgresql.dsn, - }, + "cls": "local", + "args": {"db": swh_indexer_storage_postgresql.dsn,}, } server.storage = get_indexer_storage(**storage_config) return server.app @pytest.fixture def swh_rpc_client_class(): # these are needed for the swh_indexer_storage_with_data fixture - assert hasattr(RemoteStorage, 'indexer_configuration_add') - assert hasattr(RemoteStorage, 'content_mimetype_add') + assert hasattr(RemoteStorage, "indexer_configuration_add") + assert hasattr(RemoteStorage, "content_mimetype_add") return RemoteStorage @pytest.fixture def swh_indexer_storage(swh_rpc_client, app): # This version of the swh_storage fixture uses the swh_rpc_client fixture # to instantiate a RemoteStorage (see swh_rpc_client_class above) that # proxies, via the swh.core RPC mechanism, the local (in memory) storage # configured in the app fixture above. return swh_rpc_client diff --git a/swh/indexer/tests/storage/test_converters.py b/swh/indexer/tests/storage/test_converters.py index 0aa5d6a..32f7c94 100644 --- a/swh/indexer/tests/storage/test_converters.py +++ b/swh/indexer/tests/storage/test_converters.py @@ -1,187 +1,175 @@ # Copyright (C) 2015-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from swh.indexer.storage import converters def test_ctags_to_db(): input_ctag = { - 'id': b'some-id', - 'indexer_configuration_id': 100, - 'ctags': [ - { - 'name': 'some-name', - 'kind': 'some-kind', - 'line': 10, - 'lang': 'Yaml', - }, { - 'name': 'main', - 'kind': 'function', - 'line': 12, - 'lang': 'Yaml', - }, - ] + "id": b"some-id", + "indexer_configuration_id": 100, + "ctags": [ + {"name": "some-name", "kind": "some-kind", "line": 10, "lang": "Yaml",}, + {"name": "main", "kind": "function", "line": 12, "lang": "Yaml",}, + ], } expected_ctags = [ { - 'id': b'some-id', - 'name': 'some-name', - 'kind': 'some-kind', - 'line': 10, - 'lang': 'Yaml', - 'indexer_configuration_id': 100, - }, { - 'id': b'some-id', - 'name': 'main', - 'kind': 'function', - 'line': 12, - 'lang': 'Yaml', - 'indexer_configuration_id': 100, - }] + "id": b"some-id", + "name": "some-name", + "kind": "some-kind", + "line": 10, + "lang": "Yaml", + "indexer_configuration_id": 100, + }, + { + "id": b"some-id", + "name": "main", + "kind": "function", + "line": 12, + "lang": "Yaml", + "indexer_configuration_id": 100, + }, + ] # when actual_ctags = list(converters.ctags_to_db(input_ctag)) # then assert actual_ctags == expected_ctags def test_db_to_ctags(): input_ctags = { - 'id': b'some-id', - 'name': 'some-name', - 'kind': 'some-kind', - 'line': 10, - 'lang': 'Yaml', - 'tool_id': 200, - 'tool_name': 'some-toolname', - 'tool_version': 'some-toolversion', - 'tool_configuration': {} + "id": b"some-id", + "name": "some-name", + "kind": "some-kind", + "line": 10, + "lang": "Yaml", + "tool_id": 200, + "tool_name": "some-toolname", + "tool_version": "some-toolversion", + "tool_configuration": {}, } expected_ctags = { - 'id': b'some-id', - 'name': 'some-name', - 'kind': 'some-kind', - 'line': 10, - 'lang': 'Yaml', - 'tool': { - 'id': 200, - 'name': 'some-toolname', - 'version': 'some-toolversion', - 'configuration': {}, - } + "id": b"some-id", + "name": "some-name", + "kind": "some-kind", + "line": 10, + "lang": "Yaml", + "tool": { + "id": 200, + "name": "some-toolname", + "version": "some-toolversion", + "configuration": {}, + }, } # when actual_ctags = converters.db_to_ctags(input_ctags) # then assert actual_ctags == expected_ctags def test_db_to_mimetype(): input_mimetype = { - 'id': b'some-id', - 'tool_id': 10, - 'tool_name': 'some-toolname', - 'tool_version': 'some-toolversion', - 'tool_configuration': {}, - 'encoding': b'ascii', - 'mimetype': b'text/plain', + "id": b"some-id", + "tool_id": 10, + "tool_name": "some-toolname", + "tool_version": "some-toolversion", + "tool_configuration": {}, + "encoding": b"ascii", + "mimetype": b"text/plain", } expected_mimetype = { - 'id': b'some-id', - 'encoding': b'ascii', - 'mimetype': b'text/plain', - 'tool': { - 'id': 10, - 'name': 'some-toolname', - 'version': 'some-toolversion', - 'configuration': {}, - } + "id": b"some-id", + "encoding": b"ascii", + "mimetype": b"text/plain", + "tool": { + "id": 10, + "name": "some-toolname", + "version": "some-toolversion", + "configuration": {}, + }, } actual_mimetype = converters.db_to_mimetype(input_mimetype) assert actual_mimetype == expected_mimetype def test_db_to_language(): input_language = { - 'id': b'some-id', - 'tool_id': 20, - 'tool_name': 'some-toolname', - 'tool_version': 'some-toolversion', - 'tool_configuration': {}, - 'lang': b'css', + "id": b"some-id", + "tool_id": 20, + "tool_name": "some-toolname", + "tool_version": "some-toolversion", + "tool_configuration": {}, + "lang": b"css", } expected_language = { - 'id': b'some-id', - 'lang': b'css', - 'tool': { - 'id': 20, - 'name': 'some-toolname', - 'version': 'some-toolversion', - 'configuration': {}, - } + "id": b"some-id", + "lang": b"css", + "tool": { + "id": 20, + "name": "some-toolname", + "version": "some-toolversion", + "configuration": {}, + }, } actual_language = converters.db_to_language(input_language) assert actual_language == expected_language def test_db_to_fossology_license(): input_license = { - 'id': b'some-id', - 'tool_id': 20, - 'tool_name': 'nomossa', - 'tool_version': '5.22', - 'tool_configuration': {}, - 'licenses': ['GPL2.0'], + "id": b"some-id", + "tool_id": 20, + "tool_name": "nomossa", + "tool_version": "5.22", + "tool_configuration": {}, + "licenses": ["GPL2.0"], } expected_license = { - 'licenses': ['GPL2.0'], - 'tool': { - 'id': 20, - 'name': 'nomossa', - 'version': '5.22', - 'configuration': {}, - } + "licenses": ["GPL2.0"], + "tool": {"id": 20, "name": "nomossa", "version": "5.22", "configuration": {},}, } actual_license = converters.db_to_fossology_license(input_license) assert actual_license == expected_license def test_db_to_metadata(): input_metadata = { - 'id': b'some-id', - 'tool_id': 20, - 'tool_name': 'some-toolname', - 'tool_version': 'some-toolversion', - 'tool_configuration': {}, - 'metadata': b'metadata', + "id": b"some-id", + "tool_id": 20, + "tool_name": "some-toolname", + "tool_version": "some-toolversion", + "tool_configuration": {}, + "metadata": b"metadata", } expected_metadata = { - 'id': b'some-id', - 'metadata': b'metadata', - 'tool': { - 'id': 20, - 'name': 'some-toolname', - 'version': 'some-toolversion', - 'configuration': {}, - } + "id": b"some-id", + "metadata": b"metadata", + "tool": { + "id": 20, + "name": "some-toolname", + "version": "some-toolversion", + "configuration": {}, + }, } actual_metadata = converters.db_to_metadata(input_metadata) assert actual_metadata == expected_metadata diff --git a/swh/indexer/tests/storage/test_in_memory.py b/swh/indexer/tests/storage/test_in_memory.py index 8e7b0e5..79bf365 100644 --- a/swh/indexer/tests/storage/test_in_memory.py +++ b/swh/indexer/tests/storage/test_in_memory.py @@ -1,20 +1,19 @@ # Copyright (C) 2015-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import pytest from swh.indexer.storage import get_indexer_storage from .test_storage import * # noqa @pytest.fixture def swh_indexer_storage(): storage_config = { - 'cls': 'memory', - 'args': { - }, + "cls": "memory", + "args": {}, } return get_indexer_storage(**storage_config) diff --git a/swh/indexer/tests/storage/test_metrics.py b/swh/indexer/tests/storage/test_metrics.py index 88499b3..f64f5eb 100644 --- a/swh/indexer/tests/storage/test_metrics.py +++ b/swh/indexer/tests/storage/test_metrics.py @@ -1,53 +1,58 @@ # Copyright (C) 2019-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from unittest.mock import patch from swh.indexer.storage.metrics import ( - send_metric, OPERATIONS_METRIC, OPERATIONS_UNIT_METRIC + send_metric, + OPERATIONS_METRIC, + OPERATIONS_UNIT_METRIC, ) def test_send_metric_unknown_unit(): - r = send_metric('content', count=10, method_name='content_add') + r = send_metric("content", count=10, method_name="content_add") assert r is False - r = send_metric('sthg:add:bytes:extra', count=10, method_name='sthg_add') + r = send_metric("sthg:add:bytes:extra", count=10, method_name="sthg_add") assert r is False def test_send_metric_no_value(): - r = send_metric('content_mimetype:add', count=0, - method_name='content_mimetype_add') + r = send_metric("content_mimetype:add", count=0, method_name="content_mimetype_add") assert r is False -@patch('swh.indexer.storage.metrics.statsd.increment') +@patch("swh.indexer.storage.metrics.statsd.increment") def test_send_metric_no_unit(mock_statsd): - r = send_metric('content_mimetype:add', count=10, - method_name='content_mimetype_add') + r = send_metric( + "content_mimetype:add", count=10, method_name="content_mimetype_add" + ) - mock_statsd.assert_called_with(OPERATIONS_METRIC, 10, tags={ - 'endpoint': 'content_mimetype_add', - 'object_type': 'content_mimetype', - 'operation': 'add', - }) + mock_statsd.assert_called_with( + OPERATIONS_METRIC, + 10, + tags={ + "endpoint": "content_mimetype_add", + "object_type": "content_mimetype", + "operation": "add", + }, + ) assert r -@patch('swh.indexer.storage.metrics.statsd.increment') +@patch("swh.indexer.storage.metrics.statsd.increment") def test_send_metric_unit(mock_statsd): - unit_ = 'bytes' - r = send_metric('c:add:%s' % unit_, count=100, method_name='c_add') + unit_ = "bytes" + r = send_metric("c:add:%s" % unit_, count=100, method_name="c_add") expected_metric = OPERATIONS_UNIT_METRIC.format(unit=unit_) mock_statsd.assert_called_with( - expected_metric, 100, tags={ - 'endpoint': 'c_add', - 'object_type': 'c', - 'operation': 'add', - }) + expected_metric, + 100, + tags={"endpoint": "c_add", "object_type": "c", "operation": "add",}, + ) assert r diff --git a/swh/indexer/tests/storage/test_server.py b/swh/indexer/tests/storage/test_server.py index 9130e1f..5cb14c2 100644 --- a/swh/indexer/tests/storage/test_server.py +++ b/swh/indexer/tests/storage/test_server.py @@ -1,123 +1,96 @@ # Copyright (C) 2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import pytest import yaml from swh.indexer.storage.api.server import load_and_check_config -def prepare_config_file(tmpdir, content, name='config.yml'): +def prepare_config_file(tmpdir, content, name="config.yml"): """Prepare configuration file in `$tmpdir/name` with content `content`. Args: tmpdir (LocalPath): root directory content (str/dict): Content of the file either as string or as a dict. If a dict, converts the dict into a yaml string. name (str): configuration filename Returns path (str) of the configuration file prepared. """ config_path = tmpdir / name if isinstance(content, dict): # convert if needed content = yaml.dump(content) - config_path.write_text(content, encoding='utf-8') + config_path.write_text(content, encoding="utf-8") # pytest on python3.5 does not support LocalPath manipulation, so # convert path to string return str(config_path) def test_load_and_check_config_no_configuration(): """Inexistent configuration files raises""" with pytest.raises(EnvironmentError) as e: load_and_check_config(None) - assert e.value.args[0] == 'Configuration file must be defined' + assert e.value.args[0] == "Configuration file must be defined" - config_path = '/indexer/inexistent/config.yml' + config_path = "/indexer/inexistent/config.yml" with pytest.raises(FileNotFoundError) as e: load_and_check_config(config_path) - assert e.value.args[0] == 'Configuration file %s does not exist' % ( - config_path, ) + assert e.value.args[0] == "Configuration file %s does not exist" % (config_path,) def test_load_and_check_config_wrong_configuration(tmpdir): """Wrong configuration raises""" - config_path = prepare_config_file(tmpdir, 'something: useless') + config_path = prepare_config_file(tmpdir, "something: useless") with pytest.raises(KeyError) as e: load_and_check_config(config_path) - assert e.value.args[0] == 'Missing \'%indexer_storage\' configuration' + assert e.value.args[0] == "Missing '%indexer_storage' configuration" def test_load_and_check_config_remote_config_local_type_raise(tmpdir): """'local' configuration without 'local' storage raises""" - config = { - 'indexer_storage': { - 'cls': 'remote', - 'args': {} - } - } + config = {"indexer_storage": {"cls": "remote", "args": {}}} config_path = prepare_config_file(tmpdir, config) with pytest.raises(ValueError) as e: - load_and_check_config(config_path, type='local') + load_and_check_config(config_path, type="local") assert ( - e.value.args[0] == - "The indexer_storage backend can only be started with a 'local' " + e.value.args[0] + == "The indexer_storage backend can only be started with a 'local' " "configuration" ) def test_load_and_check_config_local_incomplete_configuration(tmpdir): """Incomplete 'local' configuration should raise""" - config = { - 'indexer_storage': { - 'cls': 'local', - 'args': { - } - } - } + config = {"indexer_storage": {"cls": "local", "args": {}}} config_path = prepare_config_file(tmpdir, config) with pytest.raises(ValueError) as e: load_and_check_config(config_path) - assert ( - e.value.args[0] == - "Invalid configuration; missing 'db' config entry" - ) + assert e.value.args[0] == "Invalid configuration; missing 'db' config entry" def test_load_and_check_config_local_config_fine(tmpdir): """'Remote configuration is fine""" - config = { - 'indexer_storage': { - 'cls': 'local', - 'args': { - 'db': 'db', - } - } - } + config = {"indexer_storage": {"cls": "local", "args": {"db": "db",}}} config_path = prepare_config_file(tmpdir, config) - cfg = load_and_check_config(config_path, type='local') + cfg = load_and_check_config(config_path, type="local") assert cfg == config def test_load_and_check_config_remote_config_fine(tmpdir): """'Remote configuration is fine""" - config = { - 'indexer_storage': { - 'cls': 'remote', - 'args': {} - } - } + config = {"indexer_storage": {"cls": "remote", "args": {}}} config_path = prepare_config_file(tmpdir, config) - cfg = load_and_check_config(config_path, type='any') + cfg = load_and_check_config(config_path, type="any") assert cfg == config diff --git a/swh/indexer/tests/storage/test_storage.py b/swh/indexer/tests/storage/test_storage.py index ab11855..9c5b892 100644 --- a/swh/indexer/tests/storage/test_storage.py +++ b/swh/indexer/tests/storage/test_storage.py @@ -1,1929 +1,1831 @@ # Copyright (C) 2015-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import inspect import threading from typing import Dict import pytest from swh.model.hashutil import hash_to_bytes from swh.indexer.storage.exc import ( - IndexerStorageArgumentException, DuplicateId, + IndexerStorageArgumentException, + DuplicateId, ) from swh.indexer.storage.interface import IndexerStorageInterface def prepare_mimetypes_from(fossology_licenses): """Fossology license needs some consistent data in db to run. """ mimetypes = [] for c in fossology_licenses: - mimetypes.append({ - 'id': c['id'], - 'mimetype': 'text/plain', - 'encoding': 'utf-8', - 'indexer_configuration_id': c['indexer_configuration_id'], - }) + mimetypes.append( + { + "id": c["id"], + "mimetype": "text/plain", + "encoding": "utf-8", + "indexer_configuration_id": c["indexer_configuration_id"], + } + ) return mimetypes def endpoint_name(etype: str, ename: str) -> str: """Compute the storage's endpoint's name >>> endpoint_name('content_mimetype', 'add') 'content_mimetype_add' >>> endpoint_name('content_fosso_license', 'delete') 'content_fosso_license_delete' """ - return f'{etype}_{ename}' + return f"{etype}_{ename}" def endpoint(storage, etype: str, ename: str): return getattr(storage, endpoint_name(etype, ename)) -def expected_summary( - count: int, etype: str, ename: str = 'add') -> Dict[str, int]: +def expected_summary(count: int, etype: str, ename: str = "add") -> Dict[str, int]: """Compute the expected summary The key is determine according to etype and ename >>> expected_summary(10, 'content_mimetype', 'add') {'content_mimetype:add': 10} >>> expected_summary(9, 'origin_intrinsic_metadata', 'delete') {'origin_intrinsic_metadata:del': 9} """ pattern = ename[0:3] - key = endpoint_name(etype, ename).replace(f'_{ename}', f':{pattern}') - return { - key: count - } + key = endpoint_name(etype, ename).replace(f"_{ename}", f":{pattern}") + return {key: count} def test_check_config(swh_indexer_storage): assert swh_indexer_storage.check_config(check_write=True) assert swh_indexer_storage.check_config(check_write=False) def test_types(swh_indexer_storage): """Checks all methods of StorageInterface are implemented by this backend, and that they have the same signature.""" # Create an instance of the protocol (which cannot be instantiated # directly, so this creates a subclass, then instantiates it) - interface = type('_', (IndexerStorageInterface,), {})() + interface = type("_", (IndexerStorageInterface,), {})() - assert 'content_mimetype_add' in dir(interface) + assert "content_mimetype_add" in dir(interface) missing_methods = [] for meth_name in dir(interface): - if meth_name.startswith('_'): + if meth_name.startswith("_"): continue interface_meth = getattr(interface, meth_name) try: concrete_meth = getattr(swh_indexer_storage, meth_name) except AttributeError: missing_methods.append(meth_name) continue expected_signature = inspect.signature(interface_meth) actual_signature = inspect.signature(concrete_meth) assert expected_signature == actual_signature, meth_name assert missing_methods == [] class StorageETypeTester: """Base class for testing a series of common behaviour between a bunch of endpoint types supported by an IndexerStorage. This is supposed to be inherited with the following class attributes: - endpoint_type - tool_name - example_data See below for example usage. """ def test_missing(self, swh_indexer_storage_with_data): storage, data = swh_indexer_storage_with_data etype = self.endpoint_type - tool_id = data.tools[self.tool_name]['id'] + tool_id = data.tools[self.tool_name]["id"] # given 2 (hopefully) unknown objects query = [ - { - 'id': data.sha1_1, - 'indexer_configuration_id': tool_id, - }, - { - 'id': data.sha1_2, - 'indexer_configuration_id': tool_id, - }] + {"id": data.sha1_1, "indexer_configuration_id": tool_id,}, + {"id": data.sha1_2, "indexer_configuration_id": tool_id,}, + ] # we expect these are both returned by the xxx_missing endpoint - actual_missing = endpoint(storage, etype, 'missing')(query) + actual_missing = endpoint(storage, etype, "missing")(query) assert list(actual_missing) == [ data.sha1_1, data.sha1_2, ] # now, when we add one of them - summary = endpoint(storage, etype, 'add')([{ - 'id': data.sha1_2, - **self.example_data[0], - 'indexer_configuration_id': tool_id, - }]) + summary = endpoint(storage, etype, "add")( + [ + { + "id": data.sha1_2, + **self.example_data[0], + "indexer_configuration_id": tool_id, + } + ] + ) assert summary == expected_summary(1, etype) # we expect only the other one returned - actual_missing = endpoint(storage, etype, 'missing')(query) + actual_missing = endpoint(storage, etype, "missing")(query) assert list(actual_missing) == [data.sha1_1] def test_add__drop_duplicate(self, swh_indexer_storage_with_data): storage, data = swh_indexer_storage_with_data etype = self.endpoint_type - tool_id = data.tools[self.tool_name]['id'] + tool_id = data.tools[self.tool_name]["id"] # add the first object data_v1 = { - 'id': data.sha1_2, + "id": data.sha1_2, **self.example_data[0], - 'indexer_configuration_id': tool_id, + "indexer_configuration_id": tool_id, } - summary = endpoint(storage, etype, 'add')([data_v1]) + summary = endpoint(storage, etype, "add")([data_v1]) assert summary == expected_summary(1, etype) # should be able to retrieve it - actual_data = list(endpoint(storage, etype, 'get')([data.sha1_2])) - expected_data_v1 = [{ - 'id': data.sha1_2, - **self.example_data[0], - 'tool': data.tools[self.tool_name], - }] + actual_data = list(endpoint(storage, etype, "get")([data.sha1_2])) + expected_data_v1 = [ + { + "id": data.sha1_2, + **self.example_data[0], + "tool": data.tools[self.tool_name], + } + ] assert actual_data == expected_data_v1 # now if we add a modified version of the same object (same id) data_v2 = data_v1.copy() data_v2.update(self.example_data[1]) - summary2 = endpoint(storage, etype, 'add')([data_v2]) + summary2 = endpoint(storage, etype, "add")([data_v2]) assert summary2 == expected_summary(0, etype) # not added # we expect to retrieve the original data, not the modified one - actual_data = list(endpoint(storage, etype, 'get')([data.sha1_2])) + actual_data = list(endpoint(storage, etype, "get")([data.sha1_2])) assert actual_data == expected_data_v1 - def test_add__update_in_place_duplicate( - self, swh_indexer_storage_with_data): + def test_add__update_in_place_duplicate(self, swh_indexer_storage_with_data): storage, data = swh_indexer_storage_with_data etype = self.endpoint_type tool = data.tools[self.tool_name] data_v1 = { - 'id': data.sha1_2, + "id": data.sha1_2, **self.example_data[0], - 'indexer_configuration_id': tool['id'], + "indexer_configuration_id": tool["id"], } # given - summary = endpoint(storage, etype, 'add')([data_v1]) + summary = endpoint(storage, etype, "add")([data_v1]) assert summary == expected_summary(1, etype) # not added # when - actual_data = list(endpoint(storage, etype, 'get')([data.sha1_2])) + actual_data = list(endpoint(storage, etype, "get")([data.sha1_2])) - expected_data_v1 = [{ - 'id': data.sha1_2, - **self.example_data[0], - 'tool': tool, - }] + expected_data_v1 = [{"id": data.sha1_2, **self.example_data[0], "tool": tool,}] # then assert actual_data == expected_data_v1 # given data_v2 = data_v1.copy() data_v2.update(self.example_data[1]) - endpoint(storage, etype, 'add')([data_v2], conflict_update=True) + endpoint(storage, etype, "add")([data_v2], conflict_update=True) assert summary == expected_summary(1, etype) # modified so counted - actual_data = list(endpoint(storage, etype, 'get')([data.sha1_2])) + actual_data = list(endpoint(storage, etype, "get")([data.sha1_2])) - expected_data_v2 = [{ - 'id': data.sha1_2, - **self.example_data[1], - 'tool': tool, - }] + expected_data_v2 = [{"id": data.sha1_2, **self.example_data[1], "tool": tool,}] # data did change as the v2 was used to overwrite v1 assert actual_data == expected_data_v2 - def test_add__update_in_place_deadlock( - self, swh_indexer_storage_with_data): + def test_add__update_in_place_deadlock(self, swh_indexer_storage_with_data): storage, data = swh_indexer_storage_with_data etype = self.endpoint_type tool = data.tools[self.tool_name] hashes = [ - hash_to_bytes( - '34973274ccef6ab4dfaaf86599792fa9c3fe4{:03d}'.format(i)) - for i in range(1000)] + hash_to_bytes("34973274ccef6ab4dfaaf86599792fa9c3fe4{:03d}".format(i)) + for i in range(1000) + ] data_v1 = [ { - 'id': hash_, + "id": hash_, **self.example_data[0], - 'indexer_configuration_id': tool['id'], + "indexer_configuration_id": tool["id"], } for hash_ in hashes ] data_v2 = [ { - 'id': hash_, + "id": hash_, **self.example_data[1], - 'indexer_configuration_id': tool['id'], + "indexer_configuration_id": tool["id"], } for hash_ in hashes ] # Remove one item from each, so that both queries have to succeed for # all items to be in the DB. data_v2a = data_v2[1:] data_v2b = list(reversed(data_v2[0:-1])) # given - endpoint(storage, etype, 'add')(data_v1) + endpoint(storage, etype, "add")(data_v1) # when - actual_data = list(endpoint(storage, etype, 'get')(hashes)) + actual_data = list(endpoint(storage, etype, "get")(hashes)) expected_data_v1 = [ - { - 'id': hash_, - **self.example_data[0], - 'tool': tool, - } - for hash_ in hashes + {"id": hash_, **self.example_data[0], "tool": tool,} for hash_ in hashes ] # then assert actual_data == expected_data_v1 # given def f1(): - endpoint(storage, etype, 'add')(data_v2a, conflict_update=True) + endpoint(storage, etype, "add")(data_v2a, conflict_update=True) def f2(): - endpoint(storage, etype, 'add')(data_v2b, conflict_update=True) + endpoint(storage, etype, "add")(data_v2b, conflict_update=True) t1 = threading.Thread(target=f1) t2 = threading.Thread(target=f2) t2.start() t1.start() t1.join() t2.join() - actual_data = sorted(endpoint(storage, etype, 'get')(hashes), - key=lambda x: x['id']) + actual_data = sorted( + endpoint(storage, etype, "get")(hashes), key=lambda x: x["id"] + ) expected_data_v2 = [ - { - 'id': hash_, - **self.example_data[1], - 'tool': tool, - } - for hash_ in hashes + {"id": hash_, **self.example_data[1], "tool": tool,} for hash_ in hashes ] assert actual_data == expected_data_v2 def test_add__duplicate_twice(self, swh_indexer_storage_with_data): storage, data = swh_indexer_storage_with_data etype = self.endpoint_type tool = data.tools[self.tool_name] data_rev1 = { - 'id': data.revision_id_2, + "id": data.revision_id_2, **self.example_data[0], - 'indexer_configuration_id': tool['id'] + "indexer_configuration_id": tool["id"], } data_rev2 = { - 'id': data.revision_id_2, + "id": data.revision_id_2, **self.example_data[1], - 'indexer_configuration_id': tool['id'] + "indexer_configuration_id": tool["id"], } # when - summary = endpoint(storage, etype, 'add')([data_rev1]) + summary = endpoint(storage, etype, "add")([data_rev1]) assert summary == expected_summary(1, etype) with pytest.raises(DuplicateId): - endpoint(storage, etype, 'add')( - [data_rev2, data_rev2], - conflict_update=True) + endpoint(storage, etype, "add")( + [data_rev2, data_rev2], conflict_update=True + ) # then - actual_data = list(endpoint(storage, etype, 'get')( - [data.revision_id_2, data.revision_id_1])) + actual_data = list( + endpoint(storage, etype, "get")([data.revision_id_2, data.revision_id_1]) + ) - expected_data = [{ - 'id': data.revision_id_2, - **self.example_data[0], - 'tool': tool, - }] + expected_data = [ + {"id": data.revision_id_2, **self.example_data[0], "tool": tool,} + ] assert actual_data == expected_data def test_get(self, swh_indexer_storage_with_data): storage, data = swh_indexer_storage_with_data etype = self.endpoint_type tool = data.tools[self.tool_name] query = [data.sha1_2, data.sha1_1] data1 = { - 'id': data.sha1_2, + "id": data.sha1_2, **self.example_data[0], - 'indexer_configuration_id': tool['id'], + "indexer_configuration_id": tool["id"], } # when - summary = endpoint(storage, etype, 'add')([data1]) + summary = endpoint(storage, etype, "add")([data1]) assert summary == expected_summary(1, etype) # then - actual_data = list(endpoint(storage, etype, 'get')(query)) + actual_data = list(endpoint(storage, etype, "get")(query)) # then - expected_data = [{ - 'id': data.sha1_2, - **self.example_data[0], - 'tool': tool, - }] + expected_data = [{"id": data.sha1_2, **self.example_data[0], "tool": tool,}] assert actual_data == expected_data class TestIndexerStorageContentMimetypes(StorageETypeTester): """Test Indexer Storage content_mimetype related methods """ - endpoint_type = 'content_mimetype' - tool_name = 'file' + + endpoint_type = "content_mimetype" + tool_name = "file" example_data = [ - { - 'mimetype': 'text/plain', - 'encoding': 'utf-8', - }, - { - 'mimetype': 'text/html', - 'encoding': 'us-ascii', - }, - ] + {"mimetype": "text/plain", "encoding": "utf-8",}, + {"mimetype": "text/html", "encoding": "us-ascii",}, + ] - def test_generate_content_mimetype_get_range_limit_none( - self, swh_indexer_storage): + def test_generate_content_mimetype_get_range_limit_none(self, swh_indexer_storage): """mimetype_get_range call with wrong limit input should fail""" storage = swh_indexer_storage with pytest.raises(IndexerStorageArgumentException) as e: storage.content_mimetype_get_range( - start=None, end=None, indexer_configuration_id=None, - limit=None) + start=None, end=None, indexer_configuration_id=None, limit=None + ) - assert e.value.args == ('limit should not be None',) + assert e.value.args == ("limit should not be None",) def test_generate_content_mimetype_get_range_no_limit( - self, swh_indexer_storage_with_data): + self, swh_indexer_storage_with_data + ): """mimetype_get_range returns mimetypes within range provided""" storage, data = swh_indexer_storage_with_data mimetypes = data.mimetypes # All ids from the db - content_ids = sorted([c['id'] for c in mimetypes]) + content_ids = sorted([c["id"] for c in mimetypes]) start = content_ids[0] end = content_ids[-1] # retrieve mimetypes - tool_id = mimetypes[0]['indexer_configuration_id'] + tool_id = mimetypes[0]["indexer_configuration_id"] actual_result = storage.content_mimetype_get_range( - start, end, indexer_configuration_id=tool_id) + start, end, indexer_configuration_id=tool_id + ) - actual_ids = actual_result['ids'] - actual_next = actual_result['next'] + actual_ids = actual_result["ids"] + actual_next = actual_result["next"] assert len(mimetypes) == len(actual_ids) assert actual_next is None assert content_ids == actual_ids def test_generate_content_mimetype_get_range_limit( - self, swh_indexer_storage_with_data): + self, swh_indexer_storage_with_data + ): """mimetype_get_range paginates results if limit exceeded""" storage, data = swh_indexer_storage_with_data - indexer_configuration_id = data.tools['file']['id'] + indexer_configuration_id = data.tools["file"]["id"] # input the list of sha1s we want from storage - content_ids = sorted( - [c['id'] for c in data.mimetypes]) + content_ids = sorted([c["id"] for c in data.mimetypes]) mimetypes = list(storage.content_mimetype_get(content_ids)) assert len(mimetypes) == len(data.mimetypes) start = content_ids[0] end = content_ids[-1] # retrieve mimetypes limited to 10 results actual_result = storage.content_mimetype_get_range( - start, end, - indexer_configuration_id=indexer_configuration_id, - limit=10) + start, end, indexer_configuration_id=indexer_configuration_id, limit=10 + ) assert actual_result - assert set(actual_result.keys()) == {'ids', 'next'} - actual_ids = actual_result['ids'] - actual_next = actual_result['next'] + assert set(actual_result.keys()) == {"ids", "next"} + actual_ids = actual_result["ids"] + actual_next = actual_result["next"] assert len(actual_ids) == 10 assert actual_next is not None assert actual_next == content_ids[10] expected_mimetypes = content_ids[:10] assert expected_mimetypes == actual_ids # retrieve next part actual_result = storage.content_mimetype_get_range( - start=end, end=end, - indexer_configuration_id=indexer_configuration_id) - assert set(actual_result.keys()) == {'ids', 'next'} - actual_ids = actual_result['ids'] - actual_next = actual_result['next'] + start=end, end=end, indexer_configuration_id=indexer_configuration_id + ) + assert set(actual_result.keys()) == {"ids", "next"} + actual_ids = actual_result["ids"] + actual_next = actual_result["next"] assert actual_next is None expected_mimetypes = [content_ids[-1]] assert expected_mimetypes == actual_ids class TestIndexerStorageContentLanguage(StorageETypeTester): """Test Indexer Storage content_language related methods """ - endpoint_type = 'content_language' - tool_name = 'pygments' + + endpoint_type = "content_language" + tool_name = "pygments" example_data = [ - { - 'lang': 'haskell', - }, - { - 'lang': 'common-lisp', - }, - ] + {"lang": "haskell",}, + {"lang": "common-lisp",}, + ] class TestIndexerStorageContentCTags(StorageETypeTester): """Test Indexer Storage content_ctags related methods """ - endpoint_type = 'content_ctags' - tool_name = 'universal-ctags' + + endpoint_type = "content_ctags" + tool_name = "universal-ctags" example_data = [ { - 'ctags': [{ - 'name': 'done', - 'kind': 'variable', - 'line': 119, - 'lang': 'OCaml', - }] + "ctags": [ + {"name": "done", "kind": "variable", "line": 119, "lang": "OCaml",} + ] }, { - 'ctags': [ - { - 'name': 'done', - 'kind': 'variable', - 'line': 100, - 'lang': 'Python', - }, - { - 'name': 'main', - 'kind': 'function', - 'line': 119, - 'lang': 'Python', - }] + "ctags": [ + {"name": "done", "kind": "variable", "line": 100, "lang": "Python",}, + {"name": "main", "kind": "function", "line": 119, "lang": "Python",}, + ] }, - ] + ] # the following tests are disabled because CTAGS behaves differently @pytest.mark.skip def test_add__drop_duplicate(self): pass @pytest.mark.skip def test_add__update_in_place_duplicate(self): pass @pytest.mark.skip def test_add__update_in_place_deadlock(self): pass @pytest.mark.skip def test_add__duplicate_twice(self): pass @pytest.mark.skip def test_get(self): pass def test_content_ctags_search(self, swh_indexer_storage_with_data): storage, data = swh_indexer_storage_with_data # 1. given - tool = data.tools['universal-ctags'] - tool_id = tool['id'] + tool = data.tools["universal-ctags"] + tool_id = tool["id"] ctag1 = { - 'id': data.sha1_1, - 'indexer_configuration_id': tool_id, - 'ctags': [ - { - 'name': 'hello', - 'kind': 'function', - 'line': 133, - 'lang': 'Python', - }, - { - 'name': 'counter', - 'kind': 'variable', - 'line': 119, - 'lang': 'Python', - }, - { - 'name': 'hello', - 'kind': 'variable', - 'line': 210, - 'lang': 'Python', - }, - ] + "id": data.sha1_1, + "indexer_configuration_id": tool_id, + "ctags": [ + {"name": "hello", "kind": "function", "line": 133, "lang": "Python",}, + {"name": "counter", "kind": "variable", "line": 119, "lang": "Python",}, + {"name": "hello", "kind": "variable", "line": 210, "lang": "Python",}, + ], } ctag2 = { - 'id': data.sha1_2, - 'indexer_configuration_id': tool_id, - 'ctags': [ - { - 'name': 'hello', - 'kind': 'variable', - 'line': 100, - 'lang': 'C', - }, - { - 'name': 'result', - 'kind': 'variable', - 'line': 120, - 'lang': 'C', - }, - ] + "id": data.sha1_2, + "indexer_configuration_id": tool_id, + "ctags": [ + {"name": "hello", "kind": "variable", "line": 100, "lang": "C",}, + {"name": "result", "kind": "variable", "line": 120, "lang": "C",}, + ], } storage.content_ctags_add([ctag1, ctag2]) # 1. when - actual_ctags = list(storage.content_ctags_search('hello', limit=1)) + actual_ctags = list(storage.content_ctags_search("hello", limit=1)) # 1. then assert actual_ctags == [ { - 'id': ctag1['id'], - 'tool': tool, - 'name': 'hello', - 'kind': 'function', - 'line': 133, - 'lang': 'Python', + "id": ctag1["id"], + "tool": tool, + "name": "hello", + "kind": "function", + "line": 133, + "lang": "Python", } ] # 2. when - actual_ctags = list(storage.content_ctags_search( - 'hello', - limit=1, - last_sha1=ctag1['id'])) + actual_ctags = list( + storage.content_ctags_search("hello", limit=1, last_sha1=ctag1["id"]) + ) # 2. then assert actual_ctags == [ { - 'id': ctag2['id'], - 'tool': tool, - 'name': 'hello', - 'kind': 'variable', - 'line': 100, - 'lang': 'C', + "id": ctag2["id"], + "tool": tool, + "name": "hello", + "kind": "variable", + "line": 100, + "lang": "C", } ] # 3. when - actual_ctags = list(storage.content_ctags_search('hello')) + actual_ctags = list(storage.content_ctags_search("hello")) # 3. then assert actual_ctags == [ { - 'id': ctag1['id'], - 'tool': tool, - 'name': 'hello', - 'kind': 'function', - 'line': 133, - 'lang': 'Python', + "id": ctag1["id"], + "tool": tool, + "name": "hello", + "kind": "function", + "line": 133, + "lang": "Python", }, { - 'id': ctag1['id'], - 'tool': tool, - 'name': 'hello', - 'kind': 'variable', - 'line': 210, - 'lang': 'Python', + "id": ctag1["id"], + "tool": tool, + "name": "hello", + "kind": "variable", + "line": 210, + "lang": "Python", }, { - 'id': ctag2['id'], - 'tool': tool, - 'name': 'hello', - 'kind': 'variable', - 'line': 100, - 'lang': 'C', + "id": ctag2["id"], + "tool": tool, + "name": "hello", + "kind": "variable", + "line": 100, + "lang": "C", }, ] # 4. when - actual_ctags = list(storage.content_ctags_search('counter')) + actual_ctags = list(storage.content_ctags_search("counter")) # then - assert actual_ctags == [{ - 'id': ctag1['id'], - 'tool': tool, - 'name': 'counter', - 'kind': 'variable', - 'line': 119, - 'lang': 'Python', - }] + assert actual_ctags == [ + { + "id": ctag1["id"], + "tool": tool, + "name": "counter", + "kind": "variable", + "line": 119, + "lang": "Python", + } + ] # 5. when - actual_ctags = list(storage.content_ctags_search('result', limit=1)) + actual_ctags = list(storage.content_ctags_search("result", limit=1)) # then - assert actual_ctags == [{ - 'id': ctag2['id'], - 'tool': tool, - 'name': 'result', - 'kind': 'variable', - 'line': 120, - 'lang': 'C', - }] + assert actual_ctags == [ + { + "id": ctag2["id"], + "tool": tool, + "name": "result", + "kind": "variable", + "line": 120, + "lang": "C", + } + ] def test_content_ctags_search_no_result(self, swh_indexer_storage): storage = swh_indexer_storage - actual_ctags = list(storage.content_ctags_search('counter')) + actual_ctags = list(storage.content_ctags_search("counter")) assert not actual_ctags def test_content_ctags_add__add_new_ctags_added( - self, swh_indexer_storage_with_data): + self, swh_indexer_storage_with_data + ): storage, data = swh_indexer_storage_with_data # given - tool = data.tools['universal-ctags'] - tool_id = tool['id'] + tool = data.tools["universal-ctags"] + tool_id = tool["id"] ctag_v1 = { - 'id': data.sha1_2, - 'indexer_configuration_id': tool_id, - 'ctags': [{ - 'name': 'done', - 'kind': 'variable', - 'line': 100, - 'lang': 'Scheme', - }] + "id": data.sha1_2, + "indexer_configuration_id": tool_id, + "ctags": [ + {"name": "done", "kind": "variable", "line": 100, "lang": "Scheme",} + ], } # given storage.content_ctags_add([ctag_v1]) storage.content_ctags_add([ctag_v1]) # conflict does nothing # when actual_ctags = list(storage.content_ctags_get([data.sha1_2])) # then - expected_ctags = [{ - 'id': data.sha1_2, - 'name': 'done', - 'kind': 'variable', - 'line': 100, - 'lang': 'Scheme', - 'tool': tool, - }] + expected_ctags = [ + { + "id": data.sha1_2, + "name": "done", + "kind": "variable", + "line": 100, + "lang": "Scheme", + "tool": tool, + } + ] assert actual_ctags == expected_ctags # given ctag_v2 = ctag_v1.copy() - ctag_v2.update({ - 'ctags': [ - { - 'name': 'defn', - 'kind': 'function', - 'line': 120, - 'lang': 'Scheme', - } - ] - }) + ctag_v2.update( + { + "ctags": [ + {"name": "defn", "kind": "function", "line": 120, "lang": "Scheme",} + ] + } + ) storage.content_ctags_add([ctag_v2]) expected_ctags = [ { - 'id': data.sha1_2, - 'name': 'done', - 'kind': 'variable', - 'line': 100, - 'lang': 'Scheme', - 'tool': tool, - }, { - 'id': data.sha1_2, - 'name': 'defn', - 'kind': 'function', - 'line': 120, - 'lang': 'Scheme', - 'tool': tool, - } + "id": data.sha1_2, + "name": "done", + "kind": "variable", + "line": 100, + "lang": "Scheme", + "tool": tool, + }, + { + "id": data.sha1_2, + "name": "defn", + "kind": "function", + "line": 120, + "lang": "Scheme", + "tool": tool, + }, ] - actual_ctags = list(storage.content_ctags_get( - [data.sha1_2])) + actual_ctags = list(storage.content_ctags_get([data.sha1_2])) assert actual_ctags == expected_ctags - def test_content_ctags_add__update_in_place( - self, swh_indexer_storage_with_data): + def test_content_ctags_add__update_in_place(self, swh_indexer_storage_with_data): storage, data = swh_indexer_storage_with_data # given - tool = data.tools['universal-ctags'] - tool_id = tool['id'] + tool = data.tools["universal-ctags"] + tool_id = tool["id"] ctag_v1 = { - 'id': data.sha1_2, - 'indexer_configuration_id': tool_id, - 'ctags': [{ - 'name': 'done', - 'kind': 'variable', - 'line': 100, - 'lang': 'Scheme', - }] + "id": data.sha1_2, + "indexer_configuration_id": tool_id, + "ctags": [ + {"name": "done", "kind": "variable", "line": 100, "lang": "Scheme",} + ], } # given storage.content_ctags_add([ctag_v1]) # when - actual_ctags = list(storage.content_ctags_get( - [data.sha1_2])) + actual_ctags = list(storage.content_ctags_get([data.sha1_2])) # then expected_ctags = [ { - 'id': data.sha1_2, - 'name': 'done', - 'kind': 'variable', - 'line': 100, - 'lang': 'Scheme', - 'tool': tool + "id": data.sha1_2, + "name": "done", + "kind": "variable", + "line": 100, + "lang": "Scheme", + "tool": tool, } ] assert actual_ctags == expected_ctags # given ctag_v2 = ctag_v1.copy() - ctag_v2.update({ - 'ctags': [ - { - 'name': 'done', - 'kind': 'variable', - 'line': 100, - 'lang': 'Scheme', - }, - { - 'name': 'defn', - 'kind': 'function', - 'line': 120, - 'lang': 'Scheme', - } - ] - }) + ctag_v2.update( + { + "ctags": [ + { + "name": "done", + "kind": "variable", + "line": 100, + "lang": "Scheme", + }, + { + "name": "defn", + "kind": "function", + "line": 120, + "lang": "Scheme", + }, + ] + } + ) storage.content_ctags_add([ctag_v2], conflict_update=True) - actual_ctags = list(storage.content_ctags_get( - [data.sha1_2])) + actual_ctags = list(storage.content_ctags_get([data.sha1_2])) # ctag did change as the v2 was used to overwrite v1 expected_ctags = [ { - 'id': data.sha1_2, - 'name': 'done', - 'kind': 'variable', - 'line': 100, - 'lang': 'Scheme', - 'tool': tool, + "id": data.sha1_2, + "name": "done", + "kind": "variable", + "line": 100, + "lang": "Scheme", + "tool": tool, }, { - 'id': data.sha1_2, - 'name': 'defn', - 'kind': 'function', - 'line': 120, - 'lang': 'Scheme', - 'tool': tool, - } + "id": data.sha1_2, + "name": "defn", + "kind": "function", + "line": 120, + "lang": "Scheme", + "tool": tool, + }, ] assert actual_ctags == expected_ctags class TestIndexerStorageContentMetadata(StorageETypeTester): """Test Indexer Storage content_metadata related methods """ - tool_name = 'swh-metadata-detector' - endpoint_type = 'content_metadata' + + tool_name = "swh-metadata-detector" + endpoint_type = "content_metadata" example_data = [ { - 'metadata': { - 'other': {}, - 'codeRepository': { - 'type': 'git', - 'url': 'https://github.com/moranegg/metadata_test' + "metadata": { + "other": {}, + "codeRepository": { + "type": "git", + "url": "https://github.com/moranegg/metadata_test", }, - 'description': 'Simple package.json test for indexer', - 'name': 'test_metadata', - 'version': '0.0.1' + "description": "Simple package.json test for indexer", + "name": "test_metadata", + "version": "0.0.1", }, }, - { - 'metadata': { - 'other': {}, - 'name': 'test_metadata', - 'version': '0.0.1' - }, - }, - ] + {"metadata": {"other": {}, "name": "test_metadata", "version": "0.0.1"},}, + ] class TestIndexerStorageRevisionIntrinsicMetadata(StorageETypeTester): """Test Indexer Storage revision_intrinsic_metadata related methods """ - tool_name = 'swh-metadata-detector' - endpoint_type = 'revision_intrinsic_metadata' + + tool_name = "swh-metadata-detector" + endpoint_type = "revision_intrinsic_metadata" example_data = [ { - 'metadata': { - 'other': {}, - 'codeRepository': { - 'type': 'git', - 'url': 'https://github.com/moranegg/metadata_test' + "metadata": { + "other": {}, + "codeRepository": { + "type": "git", + "url": "https://github.com/moranegg/metadata_test", }, - 'description': 'Simple package.json test for indexer', - 'name': 'test_metadata', - 'version': '0.0.1' + "description": "Simple package.json test for indexer", + "name": "test_metadata", + "version": "0.0.1", }, - 'mappings': ['mapping1'], + "mappings": ["mapping1"], }, { - 'metadata': { - 'other': {}, - 'name': 'test_metadata', - 'version': '0.0.1' - }, - 'mappings': ['mapping2'], + "metadata": {"other": {}, "name": "test_metadata", "version": "0.0.1"}, + "mappings": ["mapping2"], }, - ] + ] - def test_revision_intrinsic_metadata_delete( - self, swh_indexer_storage_with_data): + def test_revision_intrinsic_metadata_delete(self, swh_indexer_storage_with_data): storage, data = swh_indexer_storage_with_data etype = self.endpoint_type tool = data.tools[self.tool_name] query = [data.sha1_2, data.sha1_1] data1 = { - 'id': data.sha1_2, + "id": data.sha1_2, **self.example_data[0], - 'indexer_configuration_id': tool['id'], + "indexer_configuration_id": tool["id"], } # when - summary = endpoint(storage, etype, 'add')([data1]) + summary = endpoint(storage, etype, "add")([data1]) assert summary == expected_summary(1, etype) - summary2 = endpoint(storage, etype, 'delete')([ - { - 'id': data.sha1_2, - 'indexer_configuration_id': tool['id'], - } - ]) - assert summary2 == expected_summary(1, etype, 'del') + summary2 = endpoint(storage, etype, "delete")( + [{"id": data.sha1_2, "indexer_configuration_id": tool["id"],}] + ) + assert summary2 == expected_summary(1, etype, "del") # then - actual_data = list(endpoint(storage, etype, 'get')(query)) + actual_data = list(endpoint(storage, etype, "get")(query)) # then assert not actual_data def test_revision_intrinsic_metadata_delete_nonexisting( - self, swh_indexer_storage_with_data): + self, swh_indexer_storage_with_data + ): storage, data = swh_indexer_storage_with_data etype = self.endpoint_type tool = data.tools[self.tool_name] - endpoint(storage, etype, 'delete')([ - { - 'id': data.sha1_2, - 'indexer_configuration_id': tool['id'], - } - ]) + endpoint(storage, etype, "delete")( + [{"id": data.sha1_2, "indexer_configuration_id": tool["id"],}] + ) class TestIndexerStorageContentFossologyLicence: def test_content_fossology_license_add__new_license_added( - self, swh_indexer_storage_with_data): + self, swh_indexer_storage_with_data + ): storage, data = swh_indexer_storage_with_data # given - tool = data.tools['nomos'] - tool_id = tool['id'] + tool = data.tools["nomos"] + tool_id = tool["id"] license_v1 = { - 'id': data.sha1_1, - 'licenses': ['Apache-2.0'], - 'indexer_configuration_id': tool_id, + "id": data.sha1_1, + "licenses": ["Apache-2.0"], + "indexer_configuration_id": tool_id, } # given storage.content_fossology_license_add([license_v1]) # conflict does nothing storage.content_fossology_license_add([license_v1]) # when - actual_licenses = list(storage.content_fossology_license_get( - [data.sha1_1])) + actual_licenses = list(storage.content_fossology_license_get([data.sha1_1])) # then - expected_license = { - data.sha1_1: [{ - 'licenses': ['Apache-2.0'], - 'tool': tool, - }] - } + expected_license = {data.sha1_1: [{"licenses": ["Apache-2.0"], "tool": tool,}]} assert actual_licenses == [expected_license] # given license_v2 = license_v1.copy() - license_v2.update({ - 'licenses': ['BSD-2-Clause'], - }) + license_v2.update( + {"licenses": ["BSD-2-Clause"],} + ) storage.content_fossology_license_add([license_v2]) - actual_licenses = list(storage.content_fossology_license_get( - [data.sha1_1])) + actual_licenses = list(storage.content_fossology_license_get([data.sha1_1])) expected_license = { - data.sha1_1: [{ - 'licenses': ['Apache-2.0', 'BSD-2-Clause'], - 'tool': tool - }] + data.sha1_1: [{"licenses": ["Apache-2.0", "BSD-2-Clause"], "tool": tool}] } # license did not change as the v2 was dropped. assert actual_licenses == [expected_license] def test_generate_content_fossology_license_get_range_limit_none( - self, swh_indexer_storage_with_data): + self, swh_indexer_storage_with_data + ): storage, data = swh_indexer_storage_with_data """license_get_range call with wrong limit input should fail""" with pytest.raises(IndexerStorageArgumentException) as e: storage.content_fossology_license_get_range( - start=None, end=None, indexer_configuration_id=None, - limit=None) + start=None, end=None, indexer_configuration_id=None, limit=None + ) - assert e.value.args == ('limit should not be None',) + assert e.value.args == ("limit should not be None",) def test_generate_content_fossology_license_get_range_no_limit( - self, swh_indexer_storage_with_data): + self, swh_indexer_storage_with_data + ): """license_get_range returns licenses within range provided""" storage, data = swh_indexer_storage_with_data # craft some consistent mimetypes fossology_licenses = data.fossology_licenses mimetypes = prepare_mimetypes_from(fossology_licenses) storage.content_mimetype_add(mimetypes, conflict_update=True) # add fossology_licenses to storage storage.content_fossology_license_add(fossology_licenses) # All ids from the db - content_ids = sorted([c['id'] for c in fossology_licenses]) + content_ids = sorted([c["id"] for c in fossology_licenses]) start = content_ids[0] end = content_ids[-1] # retrieve fossology_licenses - tool_id = fossology_licenses[0]['indexer_configuration_id'] + tool_id = fossology_licenses[0]["indexer_configuration_id"] actual_result = storage.content_fossology_license_get_range( - start, end, indexer_configuration_id=tool_id) + start, end, indexer_configuration_id=tool_id + ) - actual_ids = actual_result['ids'] - actual_next = actual_result['next'] + actual_ids = actual_result["ids"] + actual_next = actual_result["next"] assert len(fossology_licenses) == len(actual_ids) assert actual_next is None assert content_ids == actual_ids def test_generate_content_fossology_license_get_range_no_limit_with_filter( - self, swh_indexer_storage_with_data): + self, swh_indexer_storage_with_data + ): """This filters non textual, then returns results within range""" storage, data = swh_indexer_storage_with_data fossology_licenses = data.fossology_licenses mimetypes = data.mimetypes # craft some consistent mimetypes _mimetypes = prepare_mimetypes_from(fossology_licenses) # add binary mimetypes which will get filtered out in results for m in mimetypes: - _mimetypes.append({ - 'mimetype': 'binary', - **m, - }) + _mimetypes.append( + {"mimetype": "binary", **m,} + ) storage.content_mimetype_add(_mimetypes, conflict_update=True) # add fossology_licenses to storage storage.content_fossology_license_add(fossology_licenses) # All ids from the db - content_ids = sorted([c['id'] for c in fossology_licenses]) + content_ids = sorted([c["id"] for c in fossology_licenses]) start = content_ids[0] end = content_ids[-1] # retrieve fossology_licenses - tool_id = fossology_licenses[0]['indexer_configuration_id'] + tool_id = fossology_licenses[0]["indexer_configuration_id"] actual_result = storage.content_fossology_license_get_range( - start, end, indexer_configuration_id=tool_id) + start, end, indexer_configuration_id=tool_id + ) - actual_ids = actual_result['ids'] - actual_next = actual_result['next'] + actual_ids = actual_result["ids"] + actual_next = actual_result["next"] assert len(fossology_licenses) == len(actual_ids) assert actual_next is None assert content_ids == actual_ids def test_generate_fossology_license_get_range_limit( - self, swh_indexer_storage_with_data): + self, swh_indexer_storage_with_data + ): """fossology_license_get_range paginates results if limit exceeded""" storage, data = swh_indexer_storage_with_data fossology_licenses = data.fossology_licenses # craft some consistent mimetypes mimetypes = prepare_mimetypes_from(fossology_licenses) # add fossology_licenses to storage storage.content_mimetype_add(mimetypes, conflict_update=True) storage.content_fossology_license_add(fossology_licenses) # input the list of sha1s we want from storage - content_ids = sorted([c['id'] for c in fossology_licenses]) + content_ids = sorted([c["id"] for c in fossology_licenses]) start = content_ids[0] end = content_ids[-1] # retrieve fossology_licenses limited to 3 results limited_results = len(fossology_licenses) - 1 - tool_id = fossology_licenses[0]['indexer_configuration_id'] + tool_id = fossology_licenses[0]["indexer_configuration_id"] actual_result = storage.content_fossology_license_get_range( - start, end, - indexer_configuration_id=tool_id, limit=limited_results) + start, end, indexer_configuration_id=tool_id, limit=limited_results + ) - actual_ids = actual_result['ids'] - actual_next = actual_result['next'] + actual_ids = actual_result["ids"] + actual_next = actual_result["next"] assert limited_results == len(actual_ids) assert actual_next is not None assert actual_next == content_ids[-1] expected_fossology_licenses = content_ids[:-1] assert expected_fossology_licenses == actual_ids # retrieve next part actual_results2 = storage.content_fossology_license_get_range( - start=end, end=end, indexer_configuration_id=tool_id) - actual_ids2 = actual_results2['ids'] - actual_next2 = actual_results2['next'] + start=end, end=end, indexer_configuration_id=tool_id + ) + actual_ids2 = actual_results2["ids"] + actual_next2 = actual_results2["next"] assert actual_next2 is None expected_fossology_licenses2 = [content_ids[-1]] assert expected_fossology_licenses2 == actual_ids2 class TestIndexerStorageOriginIntrinsicMetadata: - def test_origin_intrinsic_metadata_get( - self, swh_indexer_storage_with_data): + def test_origin_intrinsic_metadata_get(self, swh_indexer_storage_with_data): storage, data = swh_indexer_storage_with_data # given - tool_id = data.tools['swh-metadata-detector']['id'] + tool_id = data.tools["swh-metadata-detector"]["id"] metadata = { - 'version': None, - 'name': None, + "version": None, + "name": None, } metadata_rev = { - 'id': data.revision_id_2, - 'metadata': metadata, - 'mappings': ['mapping1'], - 'indexer_configuration_id': tool_id, + "id": data.revision_id_2, + "metadata": metadata, + "mappings": ["mapping1"], + "indexer_configuration_id": tool_id, } metadata_origin = { - 'id': data.origin_url_1, - 'metadata': metadata, - 'indexer_configuration_id': tool_id, - 'mappings': ['mapping1'], - 'from_revision': data.revision_id_2, - } + "id": data.origin_url_1, + "metadata": metadata, + "indexer_configuration_id": tool_id, + "mappings": ["mapping1"], + "from_revision": data.revision_id_2, + } # when storage.revision_intrinsic_metadata_add([metadata_rev]) storage.origin_intrinsic_metadata_add([metadata_origin]) # then - actual_metadata = list(storage.origin_intrinsic_metadata_get( - [data.origin_url_1, 'no://where'])) + actual_metadata = list( + storage.origin_intrinsic_metadata_get([data.origin_url_1, "no://where"]) + ) - expected_metadata = [{ - 'id': data.origin_url_1, - 'metadata': metadata, - 'tool': data.tools['swh-metadata-detector'], - 'from_revision': data.revision_id_2, - 'mappings': ['mapping1'], - }] + expected_metadata = [ + { + "id": data.origin_url_1, + "metadata": metadata, + "tool": data.tools["swh-metadata-detector"], + "from_revision": data.revision_id_2, + "mappings": ["mapping1"], + } + ] assert actual_metadata == expected_metadata - def test_origin_intrinsic_metadata_delete( - self, swh_indexer_storage_with_data): + def test_origin_intrinsic_metadata_delete(self, swh_indexer_storage_with_data): storage, data = swh_indexer_storage_with_data # given - tool_id = data.tools['swh-metadata-detector']['id'] + tool_id = data.tools["swh-metadata-detector"]["id"] metadata = { - 'version': None, - 'name': None, + "version": None, + "name": None, } metadata_rev = { - 'id': data.revision_id_2, - 'metadata': metadata, - 'mappings': ['mapping1'], - 'indexer_configuration_id': tool_id, + "id": data.revision_id_2, + "metadata": metadata, + "mappings": ["mapping1"], + "indexer_configuration_id": tool_id, } metadata_origin = { - 'id': data.origin_url_1, - 'metadata': metadata, - 'indexer_configuration_id': tool_id, - 'mappings': ['mapping1'], - 'from_revision': data.revision_id_2, - } + "id": data.origin_url_1, + "metadata": metadata, + "indexer_configuration_id": tool_id, + "mappings": ["mapping1"], + "from_revision": data.revision_id_2, + } metadata_origin2 = metadata_origin.copy() - metadata_origin2['id'] = data.origin_url_2 + metadata_origin2["id"] = data.origin_url_2 # when storage.revision_intrinsic_metadata_add([metadata_rev]) - storage.origin_intrinsic_metadata_add([ - metadata_origin, metadata_origin2]) + storage.origin_intrinsic_metadata_add([metadata_origin, metadata_origin2]) - storage.origin_intrinsic_metadata_delete([ - { - 'id': data.origin_url_1, - 'indexer_configuration_id': tool_id - } - ]) + storage.origin_intrinsic_metadata_delete( + [{"id": data.origin_url_1, "indexer_configuration_id": tool_id}] + ) # then - actual_metadata = list(storage.origin_intrinsic_metadata_get( - [data.origin_url_1, data.origin_url_2, 'no://where'])) + actual_metadata = list( + storage.origin_intrinsic_metadata_get( + [data.origin_url_1, data.origin_url_2, "no://where"] + ) + ) for item in actual_metadata: - item['indexer_configuration_id'] = item.pop('tool')['id'] + item["indexer_configuration_id"] = item.pop("tool")["id"] assert actual_metadata == [metadata_origin2] def test_origin_intrinsic_metadata_delete_nonexisting( - self, swh_indexer_storage_with_data): + self, swh_indexer_storage_with_data + ): storage, data = swh_indexer_storage_with_data - tool_id = data.tools['swh-metadata-detector']['id'] - storage.origin_intrinsic_metadata_delete([ - { - 'id': data.origin_url_1, - 'indexer_configuration_id': tool_id - } - ]) + tool_id = data.tools["swh-metadata-detector"]["id"] + storage.origin_intrinsic_metadata_delete( + [{"id": data.origin_url_1, "indexer_configuration_id": tool_id}] + ) def test_origin_intrinsic_metadata_add_drop_duplicate( - self, swh_indexer_storage_with_data): + self, swh_indexer_storage_with_data + ): storage, data = swh_indexer_storage_with_data # given - tool_id = data.tools['swh-metadata-detector']['id'] + tool_id = data.tools["swh-metadata-detector"]["id"] metadata_v1 = { - 'version': None, - 'name': None, + "version": None, + "name": None, } metadata_rev_v1 = { - 'id': data.revision_id_1, - 'metadata': metadata_v1.copy(), - 'mappings': [], - 'indexer_configuration_id': tool_id, + "id": data.revision_id_1, + "metadata": metadata_v1.copy(), + "mappings": [], + "indexer_configuration_id": tool_id, } metadata_origin_v1 = { - 'id': data.origin_url_1, - 'metadata': metadata_v1.copy(), - 'indexer_configuration_id': tool_id, - 'mappings': [], - 'from_revision': data.revision_id_1, + "id": data.origin_url_1, + "metadata": metadata_v1.copy(), + "indexer_configuration_id": tool_id, + "mappings": [], + "from_revision": data.revision_id_1, } # given storage.revision_intrinsic_metadata_add([metadata_rev_v1]) storage.origin_intrinsic_metadata_add([metadata_origin_v1]) # when - actual_metadata = list(storage.origin_intrinsic_metadata_get( - [data.origin_url_1, 'no://where'])) + actual_metadata = list( + storage.origin_intrinsic_metadata_get([data.origin_url_1, "no://where"]) + ) - expected_metadata_v1 = [{ - 'id': data.origin_url_1, - 'metadata': metadata_v1, - 'tool': data.tools['swh-metadata-detector'], - 'from_revision': data.revision_id_1, - 'mappings': [], - }] + expected_metadata_v1 = [ + { + "id": data.origin_url_1, + "metadata": metadata_v1, + "tool": data.tools["swh-metadata-detector"], + "from_revision": data.revision_id_1, + "mappings": [], + } + ] assert actual_metadata == expected_metadata_v1 # given metadata_v2 = metadata_v1.copy() - metadata_v2.update({ - 'name': 'test_metadata', - 'author': 'MG', - }) + metadata_v2.update( + {"name": "test_metadata", "author": "MG",} + ) metadata_rev_v2 = metadata_rev_v1.copy() metadata_origin_v2 = metadata_origin_v1.copy() - metadata_rev_v2['metadata'] = metadata_v2 - metadata_origin_v2['metadata'] = metadata_v2 + metadata_rev_v2["metadata"] = metadata_v2 + metadata_origin_v2["metadata"] = metadata_v2 storage.revision_intrinsic_metadata_add([metadata_rev_v2]) storage.origin_intrinsic_metadata_add([metadata_origin_v2]) # then - actual_metadata = list(storage.origin_intrinsic_metadata_get( - [data.origin_url_1])) + actual_metadata = list( + storage.origin_intrinsic_metadata_get([data.origin_url_1]) + ) # metadata did not change as the v2 was dropped. assert actual_metadata == expected_metadata_v1 def test_origin_intrinsic_metadata_add_update_in_place_duplicate( - self, swh_indexer_storage_with_data): + self, swh_indexer_storage_with_data + ): storage, data = swh_indexer_storage_with_data # given - tool_id = data.tools['swh-metadata-detector']['id'] + tool_id = data.tools["swh-metadata-detector"]["id"] metadata_v1 = { - 'version': None, - 'name': None, + "version": None, + "name": None, } metadata_rev_v1 = { - 'id': data.revision_id_2, - 'metadata': metadata_v1, - 'mappings': [], - 'indexer_configuration_id': tool_id, + "id": data.revision_id_2, + "metadata": metadata_v1, + "mappings": [], + "indexer_configuration_id": tool_id, } metadata_origin_v1 = { - 'id': data.origin_url_1, - 'metadata': metadata_v1.copy(), - 'indexer_configuration_id': tool_id, - 'mappings': [], - 'from_revision': data.revision_id_2, + "id": data.origin_url_1, + "metadata": metadata_v1.copy(), + "indexer_configuration_id": tool_id, + "mappings": [], + "from_revision": data.revision_id_2, } # given storage.revision_intrinsic_metadata_add([metadata_rev_v1]) storage.origin_intrinsic_metadata_add([metadata_origin_v1]) # when - actual_metadata = list(storage.origin_intrinsic_metadata_get( - [data.origin_url_1])) + actual_metadata = list( + storage.origin_intrinsic_metadata_get([data.origin_url_1]) + ) # then - expected_metadata_v1 = [{ - 'id': data.origin_url_1, - 'metadata': metadata_v1, - 'tool': data.tools['swh-metadata-detector'], - 'from_revision': data.revision_id_2, - 'mappings': [], - }] + expected_metadata_v1 = [ + { + "id": data.origin_url_1, + "metadata": metadata_v1, + "tool": data.tools["swh-metadata-detector"], + "from_revision": data.revision_id_2, + "mappings": [], + } + ] assert actual_metadata == expected_metadata_v1 # given metadata_v2 = metadata_v1.copy() - metadata_v2.update({ - 'name': 'test_update_duplicated_metadata', - 'author': 'MG', - }) + metadata_v2.update( + {"name": "test_update_duplicated_metadata", "author": "MG",} + ) metadata_rev_v2 = metadata_rev_v1.copy() metadata_origin_v2 = metadata_origin_v1.copy() - metadata_rev_v2['metadata'] = metadata_v2 + metadata_rev_v2["metadata"] = metadata_v2 metadata_origin_v2 = { - 'id': data.origin_url_1, - 'metadata': metadata_v2.copy(), - 'indexer_configuration_id': tool_id, - 'mappings': ['npm'], - 'from_revision': data.revision_id_1, + "id": data.origin_url_1, + "metadata": metadata_v2.copy(), + "indexer_configuration_id": tool_id, + "mappings": ["npm"], + "from_revision": data.revision_id_1, } - storage.revision_intrinsic_metadata_add( - [metadata_rev_v2], conflict_update=True) + storage.revision_intrinsic_metadata_add([metadata_rev_v2], conflict_update=True) storage.origin_intrinsic_metadata_add( - [metadata_origin_v2], conflict_update=True) + [metadata_origin_v2], conflict_update=True + ) - actual_metadata = list(storage.origin_intrinsic_metadata_get( - [data.origin_url_1])) + actual_metadata = list( + storage.origin_intrinsic_metadata_get([data.origin_url_1]) + ) - expected_metadata_v2 = [{ - 'id': data.origin_url_1, - 'metadata': metadata_v2, - 'tool': data.tools['swh-metadata-detector'], - 'from_revision': data.revision_id_1, - 'mappings': ['npm'], - }] + expected_metadata_v2 = [ + { + "id": data.origin_url_1, + "metadata": metadata_v2, + "tool": data.tools["swh-metadata-detector"], + "from_revision": data.revision_id_1, + "mappings": ["npm"], + } + ] # metadata did change as the v2 was used to overwrite v1 assert actual_metadata == expected_metadata_v2 def test_origin_intrinsic_metadata_add__update_in_place_deadlock( - self, swh_indexer_storage_with_data): + self, swh_indexer_storage_with_data + ): storage, data = swh_indexer_storage_with_data # given - tool_id = data.tools['swh-metadata-detector']['id'] + tool_id = data.tools["swh-metadata-detector"]["id"] ids = list(range(10)) example_data1 = { - 'metadata': { - 'version': None, - 'name': None, - }, - 'mappings': [], + "metadata": {"version": None, "name": None,}, + "mappings": [], } example_data2 = { - 'metadata': { - 'version': 'v1.1.1', - 'name': 'foo', - }, - 'mappings': [], + "metadata": {"version": "v1.1.1", "name": "foo",}, + "mappings": [], } metadata_rev_v1 = { - 'id': data.revision_id_2, - 'metadata': { - 'version': None, - 'name': None, - }, - 'mappings': [], - 'indexer_configuration_id': tool_id, + "id": data.revision_id_2, + "metadata": {"version": None, "name": None,}, + "mappings": [], + "indexer_configuration_id": tool_id, } data_v1 = [ { - 'id': 'file:///tmp/origin%d' % id_, - 'from_revision': data.revision_id_2, + "id": "file:///tmp/origin%d" % id_, + "from_revision": data.revision_id_2, **example_data1, - 'indexer_configuration_id': tool_id, + "indexer_configuration_id": tool_id, } for id_ in ids ] data_v2 = [ { - 'id': 'file:///tmp/origin%d' % id_, - 'from_revision': data.revision_id_2, + "id": "file:///tmp/origin%d" % id_, + "from_revision": data.revision_id_2, **example_data2, - 'indexer_configuration_id': tool_id, + "indexer_configuration_id": tool_id, } for id_ in ids ] # Remove one item from each, so that both queries have to succeed for # all items to be in the DB. data_v2a = data_v2[1:] data_v2b = list(reversed(data_v2[0:-1])) # given storage.revision_intrinsic_metadata_add([metadata_rev_v1]) storage.origin_intrinsic_metadata_add(data_v1) # when - origins = ['file:///tmp/origin%d' % i for i in ids] + origins = ["file:///tmp/origin%d" % i for i in ids] actual_data = list(storage.origin_intrinsic_metadata_get(origins)) expected_data_v1 = [ { - 'id': 'file:///tmp/origin%d' % id_, - 'from_revision': data.revision_id_2, + "id": "file:///tmp/origin%d" % id_, + "from_revision": data.revision_id_2, **example_data1, - 'tool': data.tools['swh-metadata-detector'], + "tool": data.tools["swh-metadata-detector"], } for id_ in ids ] # then assert actual_data == expected_data_v1 # given def f1(): - storage.origin_intrinsic_metadata_add( - data_v2a, conflict_update=True) + storage.origin_intrinsic_metadata_add(data_v2a, conflict_update=True) def f2(): - storage.origin_intrinsic_metadata_add( - data_v2b, conflict_update=True) + storage.origin_intrinsic_metadata_add(data_v2b, conflict_update=True) t1 = threading.Thread(target=f1) t2 = threading.Thread(target=f2) t2.start() t1.start() t1.join() t2.join() actual_data = list(storage.origin_intrinsic_metadata_get(origins)) expected_data_v2 = [ { - 'id': 'file:///tmp/origin%d' % id_, - 'from_revision': data.revision_id_2, + "id": "file:///tmp/origin%d" % id_, + "from_revision": data.revision_id_2, **example_data2, - 'tool': data.tools['swh-metadata-detector'], + "tool": data.tools["swh-metadata-detector"], } for id_ in ids ] assert len(actual_data) == len(expected_data_v2) - assert sorted(actual_data, key=lambda x: x['id']) == expected_data_v2 + assert sorted(actual_data, key=lambda x: x["id"]) == expected_data_v2 def test_origin_intrinsic_metadata_add__duplicate_twice( - self, swh_indexer_storage_with_data): + self, swh_indexer_storage_with_data + ): storage, data = swh_indexer_storage_with_data # given - tool_id = data.tools['swh-metadata-detector']['id'] + tool_id = data.tools["swh-metadata-detector"]["id"] metadata = { - 'developmentStatus': None, - 'name': None, + "developmentStatus": None, + "name": None, } metadata_rev = { - 'id': data.revision_id_2, - 'metadata': metadata, - 'mappings': ['mapping1'], - 'indexer_configuration_id': tool_id, + "id": data.revision_id_2, + "metadata": metadata, + "mappings": ["mapping1"], + "indexer_configuration_id": tool_id, } metadata_origin = { - 'id': data.origin_url_1, - 'metadata': metadata, - 'indexer_configuration_id': tool_id, - 'mappings': ['mapping1'], - 'from_revision': data.revision_id_2, - } + "id": data.origin_url_1, + "metadata": metadata, + "indexer_configuration_id": tool_id, + "mappings": ["mapping1"], + "from_revision": data.revision_id_2, + } # when storage.revision_intrinsic_metadata_add([metadata_rev]) with pytest.raises(DuplicateId): - storage.origin_intrinsic_metadata_add([ - metadata_origin, metadata_origin]) + storage.origin_intrinsic_metadata_add([metadata_origin, metadata_origin]) def test_origin_intrinsic_metadata_search_fulltext( - self, swh_indexer_storage_with_data): + self, swh_indexer_storage_with_data + ): storage, data = swh_indexer_storage_with_data # given - tool_id = data.tools['swh-metadata-detector']['id'] + tool_id = data.tools["swh-metadata-detector"]["id"] metadata1 = { - 'author': 'John Doe', + "author": "John Doe", } metadata1_rev = { - 'id': data.revision_id_1, - 'metadata': metadata1, - 'mappings': [], - 'indexer_configuration_id': tool_id, + "id": data.revision_id_1, + "metadata": metadata1, + "mappings": [], + "indexer_configuration_id": tool_id, } metadata1_origin = { - 'id': data.origin_url_1, - 'metadata': metadata1, - 'mappings': [], - 'indexer_configuration_id': tool_id, - 'from_revision': data.revision_id_1, + "id": data.origin_url_1, + "metadata": metadata1, + "mappings": [], + "indexer_configuration_id": tool_id, + "from_revision": data.revision_id_1, } metadata2 = { - 'author': 'Jane Doe', + "author": "Jane Doe", } metadata2_rev = { - 'id': data.revision_id_2, - 'metadata': metadata2, - 'mappings': [], - 'indexer_configuration_id': tool_id, + "id": data.revision_id_2, + "metadata": metadata2, + "mappings": [], + "indexer_configuration_id": tool_id, } metadata2_origin = { - 'id': data.origin_url_2, - 'metadata': metadata2, - 'mappings': [], - 'indexer_configuration_id': tool_id, - 'from_revision': data.revision_id_2, + "id": data.origin_url_2, + "metadata": metadata2, + "mappings": [], + "indexer_configuration_id": tool_id, + "from_revision": data.revision_id_2, } # when storage.revision_intrinsic_metadata_add([metadata1_rev]) storage.origin_intrinsic_metadata_add([metadata1_origin]) storage.revision_intrinsic_metadata_add([metadata2_rev]) storage.origin_intrinsic_metadata_add([metadata2_origin]) # then search = storage.origin_intrinsic_metadata_search_fulltext - assert set([res['id'] for res in search(['Doe'])]) \ - == set([data.origin_url_1, data.origin_url_2]) - assert [res['id'] for res in search(['John', 'Doe'])] \ - == [data.origin_url_1] - assert [res['id'] for res in search(['John'])] \ - == [data.origin_url_1] - assert not list(search(['John', 'Jane'])) + assert set([res["id"] for res in search(["Doe"])]) == set( + [data.origin_url_1, data.origin_url_2] + ) + assert [res["id"] for res in search(["John", "Doe"])] == [data.origin_url_1] + assert [res["id"] for res in search(["John"])] == [data.origin_url_1] + assert not list(search(["John", "Jane"])) def test_origin_intrinsic_metadata_search_fulltext_rank( - self, swh_indexer_storage_with_data): + self, swh_indexer_storage_with_data + ): storage, data = swh_indexer_storage_with_data # given - tool_id = data.tools['swh-metadata-detector']['id'] + tool_id = data.tools["swh-metadata-detector"]["id"] # The following authors have "Random Person" to add some more content # to the JSON data, to work around normalization quirks when there # are few words (rank/(1+ln(nb_words)) is very sensitive to nb_words # for small values of nb_words). - metadata1 = { - 'author': [ - 'Random Person', - 'John Doe', - 'Jane Doe', - ] - } + metadata1 = {"author": ["Random Person", "John Doe", "Jane Doe",]} metadata1_rev = { - 'id': data.revision_id_1, - 'metadata': metadata1, - 'mappings': [], - 'indexer_configuration_id': tool_id, + "id": data.revision_id_1, + "metadata": metadata1, + "mappings": [], + "indexer_configuration_id": tool_id, } metadata1_origin = { - 'id': data.origin_url_1, - 'metadata': metadata1, - 'mappings': [], - 'indexer_configuration_id': tool_id, - 'from_revision': data.revision_id_1, - } - metadata2 = { - 'author': [ - 'Random Person', - 'Jane Doe', - ] + "id": data.origin_url_1, + "metadata": metadata1, + "mappings": [], + "indexer_configuration_id": tool_id, + "from_revision": data.revision_id_1, } + metadata2 = {"author": ["Random Person", "Jane Doe",]} metadata2_rev = { - 'id': data.revision_id_2, - 'metadata': metadata2, - 'mappings': [], - 'indexer_configuration_id': tool_id, + "id": data.revision_id_2, + "metadata": metadata2, + "mappings": [], + "indexer_configuration_id": tool_id, } metadata2_origin = { - 'id': data.origin_url_2, - 'metadata': metadata2, - 'mappings': [], - 'indexer_configuration_id': tool_id, - 'from_revision': data.revision_id_2, + "id": data.origin_url_2, + "metadata": metadata2, + "mappings": [], + "indexer_configuration_id": tool_id, + "from_revision": data.revision_id_2, } # when storage.revision_intrinsic_metadata_add([metadata1_rev]) storage.origin_intrinsic_metadata_add([metadata1_origin]) storage.revision_intrinsic_metadata_add([metadata2_rev]) storage.origin_intrinsic_metadata_add([metadata2_origin]) # then search = storage.origin_intrinsic_metadata_search_fulltext - assert [res['id'] for res in search(['Doe'])] \ - == [data.origin_url_1, data.origin_url_2] - assert [res['id'] for res in search(['Doe'], limit=1)] \ - == [data.origin_url_1] - assert [res['id'] for res in search(['John'])] \ - == [data.origin_url_1] - assert [res['id'] for res in search(['Jane'])] \ - == [data.origin_url_2, data.origin_url_1] - assert [res['id'] for res in search(['John', 'Jane'])] \ - == [data.origin_url_1] - - def _fill_origin_intrinsic_metadata( - self, swh_indexer_storage_with_data): + assert [res["id"] for res in search(["Doe"])] == [ + data.origin_url_1, + data.origin_url_2, + ] + assert [res["id"] for res in search(["Doe"], limit=1)] == [data.origin_url_1] + assert [res["id"] for res in search(["John"])] == [data.origin_url_1] + assert [res["id"] for res in search(["Jane"])] == [ + data.origin_url_2, + data.origin_url_1, + ] + assert [res["id"] for res in search(["John", "Jane"])] == [data.origin_url_1] + + def _fill_origin_intrinsic_metadata(self, swh_indexer_storage_with_data): storage, data = swh_indexer_storage_with_data - tool1_id = data.tools['swh-metadata-detector']['id'] - tool2_id = data.tools['swh-metadata-detector2']['id'] + tool1_id = data.tools["swh-metadata-detector"]["id"] + tool2_id = data.tools["swh-metadata-detector2"]["id"] metadata1 = { - '@context': 'foo', - 'author': 'John Doe', + "@context": "foo", + "author": "John Doe", } metadata1_rev = { - 'id': data.revision_id_1, - 'metadata': metadata1, - 'mappings': ['npm'], - 'indexer_configuration_id': tool1_id, + "id": data.revision_id_1, + "metadata": metadata1, + "mappings": ["npm"], + "indexer_configuration_id": tool1_id, } metadata1_origin = { - 'id': data.origin_url_1, - 'metadata': metadata1, - 'mappings': ['npm'], - 'indexer_configuration_id': tool1_id, - 'from_revision': data.revision_id_1, + "id": data.origin_url_1, + "metadata": metadata1, + "mappings": ["npm"], + "indexer_configuration_id": tool1_id, + "from_revision": data.revision_id_1, } metadata2 = { - '@context': 'foo', - 'author': 'Jane Doe', + "@context": "foo", + "author": "Jane Doe", } metadata2_rev = { - 'id': data.revision_id_2, - 'metadata': metadata2, - 'mappings': ['npm', 'gemspec'], - 'indexer_configuration_id': tool2_id, + "id": data.revision_id_2, + "metadata": metadata2, + "mappings": ["npm", "gemspec"], + "indexer_configuration_id": tool2_id, } metadata2_origin = { - 'id': data.origin_url_2, - 'metadata': metadata2, - 'mappings': ['npm', 'gemspec'], - 'indexer_configuration_id': tool2_id, - 'from_revision': data.revision_id_2, + "id": data.origin_url_2, + "metadata": metadata2, + "mappings": ["npm", "gemspec"], + "indexer_configuration_id": tool2_id, + "from_revision": data.revision_id_2, } metadata3 = { - '@context': 'foo', + "@context": "foo", } metadata3_rev = { - 'id': data.revision_id_3, - 'metadata': metadata3, - 'mappings': ['npm', 'gemspec'], - 'indexer_configuration_id': tool2_id, + "id": data.revision_id_3, + "metadata": metadata3, + "mappings": ["npm", "gemspec"], + "indexer_configuration_id": tool2_id, } metadata3_origin = { - 'id': data.origin_url_3, - 'metadata': metadata3, - 'mappings': ['pkg-info'], - 'indexer_configuration_id': tool2_id, - 'from_revision': data.revision_id_3, + "id": data.origin_url_3, + "metadata": metadata3, + "mappings": ["pkg-info"], + "indexer_configuration_id": tool2_id, + "from_revision": data.revision_id_3, } storage.revision_intrinsic_metadata_add([metadata1_rev]) storage.origin_intrinsic_metadata_add([metadata1_origin]) storage.revision_intrinsic_metadata_add([metadata2_rev]) storage.origin_intrinsic_metadata_add([metadata2_origin]) storage.revision_intrinsic_metadata_add([metadata3_rev]) storage.origin_intrinsic_metadata_add([metadata3_origin]) def test_origin_intrinsic_metadata_search_by_producer( - self, swh_indexer_storage_with_data): + self, swh_indexer_storage_with_data + ): storage, data = swh_indexer_storage_with_data - self._fill_origin_intrinsic_metadata( - swh_indexer_storage_with_data) - tool1 = data.tools['swh-metadata-detector'] - tool2 = data.tools['swh-metadata-detector2'] + self._fill_origin_intrinsic_metadata(swh_indexer_storage_with_data) + tool1 = data.tools["swh-metadata-detector"] + tool2 = data.tools["swh-metadata-detector2"] endpoint = storage.origin_intrinsic_metadata_search_by_producer # test pagination # no 'page_token' param, return all origins result = endpoint(ids_only=True) - assert result['origins'] \ - == [data.origin_url_1, data.origin_url_2, data.origin_url_3] - assert 'next_page_token' not in result + assert result["origins"] == [ + data.origin_url_1, + data.origin_url_2, + data.origin_url_3, + ] + assert "next_page_token" not in result # 'page_token' is < than origin_1, return everything result = endpoint(page_token=data.origin_url_1[:-1], ids_only=True) - assert result['origins'] \ - == [data.origin_url_1, data.origin_url_2, data.origin_url_3] - assert 'next_page_token' not in result + assert result["origins"] == [ + data.origin_url_1, + data.origin_url_2, + data.origin_url_3, + ] + assert "next_page_token" not in result # 'page_token' is origin_3, return nothing result = endpoint(page_token=data.origin_url_3, ids_only=True) - assert not result['origins'] - assert 'next_page_token' not in result + assert not result["origins"] + assert "next_page_token" not in result # test limit argument - result = endpoint(page_token=data.origin_url_1[:-1], - limit=2, ids_only=True) - assert result['origins'] == [data.origin_url_1, data.origin_url_2] - assert result['next_page_token'] == result['origins'][-1] + result = endpoint(page_token=data.origin_url_1[:-1], limit=2, ids_only=True) + assert result["origins"] == [data.origin_url_1, data.origin_url_2] + assert result["next_page_token"] == result["origins"][-1] result = endpoint(page_token=data.origin_url_1, limit=2, ids_only=True) - assert result['origins'] == [data.origin_url_2, data.origin_url_3] - assert 'next_page_token' not in result + assert result["origins"] == [data.origin_url_2, data.origin_url_3] + assert "next_page_token" not in result result = endpoint(page_token=data.origin_url_2, limit=2, ids_only=True) - assert result['origins'] == [data.origin_url_3] - assert 'next_page_token' not in result + assert result["origins"] == [data.origin_url_3] + assert "next_page_token" not in result # test mappings filtering - result = endpoint(mappings=['npm'], ids_only=True) - assert result['origins'] == [data.origin_url_1, data.origin_url_2] - assert 'next_page_token' not in result + result = endpoint(mappings=["npm"], ids_only=True) + assert result["origins"] == [data.origin_url_1, data.origin_url_2] + assert "next_page_token" not in result - result = endpoint(mappings=['npm', 'gemspec'], ids_only=True) - assert result['origins'] == [data.origin_url_1, data.origin_url_2] - assert 'next_page_token' not in result + result = endpoint(mappings=["npm", "gemspec"], ids_only=True) + assert result["origins"] == [data.origin_url_1, data.origin_url_2] + assert "next_page_token" not in result - result = endpoint(mappings=['gemspec'], ids_only=True) - assert result['origins'] == [data.origin_url_2] - assert 'next_page_token' not in result + result = endpoint(mappings=["gemspec"], ids_only=True) + assert result["origins"] == [data.origin_url_2] + assert "next_page_token" not in result - result = endpoint(mappings=['pkg-info'], ids_only=True) - assert result['origins'] == [data.origin_url_3] - assert 'next_page_token' not in result + result = endpoint(mappings=["pkg-info"], ids_only=True) + assert result["origins"] == [data.origin_url_3] + assert "next_page_token" not in result - result = endpoint(mappings=['foobar'], ids_only=True) - assert not result['origins'] - assert 'next_page_token' not in result + result = endpoint(mappings=["foobar"], ids_only=True) + assert not result["origins"] + assert "next_page_token" not in result # test pagination + mappings - result = endpoint(mappings=['npm'], limit=1, ids_only=True) - assert result['origins'] == [data.origin_url_1] - assert result['next_page_token'] == result['origins'][-1] + result = endpoint(mappings=["npm"], limit=1, ids_only=True) + assert result["origins"] == [data.origin_url_1] + assert result["next_page_token"] == result["origins"][-1] # test tool filtering - result = endpoint(tool_ids=[tool1['id']], ids_only=True) - assert result['origins'] == [data.origin_url_1] - assert 'next_page_token' not in result - - result = endpoint(tool_ids=[tool2['id']], ids_only=True) - assert sorted(result['origins']) \ - == [data.origin_url_2, data.origin_url_3] - assert 'next_page_token' not in result - - result = endpoint(tool_ids=[tool1['id'], tool2['id']], ids_only=True) - assert sorted(result['origins']) \ - == [data.origin_url_1, data.origin_url_2, data.origin_url_3] - assert 'next_page_token' not in result + result = endpoint(tool_ids=[tool1["id"]], ids_only=True) + assert result["origins"] == [data.origin_url_1] + assert "next_page_token" not in result + + result = endpoint(tool_ids=[tool2["id"]], ids_only=True) + assert sorted(result["origins"]) == [data.origin_url_2, data.origin_url_3] + assert "next_page_token" not in result + + result = endpoint(tool_ids=[tool1["id"], tool2["id"]], ids_only=True) + assert sorted(result["origins"]) == [ + data.origin_url_1, + data.origin_url_2, + data.origin_url_3, + ] + assert "next_page_token" not in result # test ids_only=False - assert endpoint(mappings=['gemspec'])['origins'] \ - == [{ - 'id': data.origin_url_2, - 'metadata': { - '@context': 'foo', - 'author': 'Jane Doe', - }, - 'mappings': ['npm', 'gemspec'], - 'tool': tool2, - 'from_revision': data.revision_id_2, - }] + assert endpoint(mappings=["gemspec"])["origins"] == [ + { + "id": data.origin_url_2, + "metadata": {"@context": "foo", "author": "Jane Doe",}, + "mappings": ["npm", "gemspec"], + "tool": tool2, + "from_revision": data.revision_id_2, + } + ] - def test_origin_intrinsic_metadata_stats( - self, swh_indexer_storage_with_data): + def test_origin_intrinsic_metadata_stats(self, swh_indexer_storage_with_data): storage, data = swh_indexer_storage_with_data - self._fill_origin_intrinsic_metadata( - swh_indexer_storage_with_data) + self._fill_origin_intrinsic_metadata(swh_indexer_storage_with_data) result = storage.origin_intrinsic_metadata_stats() assert result == { - 'per_mapping': { - 'gemspec': 1, - 'npm': 2, - 'pkg-info': 1, - 'codemeta': 0, - 'maven': 0, + "per_mapping": { + "gemspec": 1, + "npm": 2, + "pkg-info": 1, + "codemeta": 0, + "maven": 0, }, - 'total': 3, - 'non_empty': 2, + "total": 3, + "non_empty": 2, } class TestIndexerStorageIndexerCondifuration: - def test_indexer_configuration_add( - self, swh_indexer_storage_with_data): + def test_indexer_configuration_add(self, swh_indexer_storage_with_data): storage, data = swh_indexer_storage_with_data tool = { - 'tool_name': 'some-unknown-tool', - 'tool_version': 'some-version', - 'tool_configuration': {"debian-package": "some-package"}, + "tool_name": "some-unknown-tool", + "tool_version": "some-version", + "tool_configuration": {"debian-package": "some-package"}, } actual_tool = storage.indexer_configuration_get(tool) assert actual_tool is None # does not exist # add it actual_tools = list(storage.indexer_configuration_add([tool])) assert len(actual_tools) == 1 actual_tool = actual_tools[0] assert actual_tool is not None # now it exists - new_id = actual_tool.pop('id') + new_id = actual_tool.pop("id") assert actual_tool == tool actual_tools2 = list(storage.indexer_configuration_add([tool])) actual_tool2 = actual_tools2[0] assert actual_tool2 is not None # now it exists - new_id2 = actual_tool2.pop('id') + new_id2 = actual_tool2.pop("id") assert new_id == new_id2 assert actual_tool == actual_tool2 - def test_indexer_configuration_add_multiple( - self, swh_indexer_storage_with_data): + def test_indexer_configuration_add_multiple(self, swh_indexer_storage_with_data): storage, data = swh_indexer_storage_with_data tool = { - 'tool_name': 'some-unknown-tool', - 'tool_version': 'some-version', - 'tool_configuration': {"debian-package": "some-package"}, + "tool_name": "some-unknown-tool", + "tool_version": "some-version", + "tool_configuration": {"debian-package": "some-package"}, } actual_tools = list(storage.indexer_configuration_add([tool])) assert len(actual_tools) == 1 - new_tools = [tool, { - 'tool_name': 'yet-another-tool', - 'tool_version': 'version', - 'tool_configuration': {}, - }] + new_tools = [ + tool, + { + "tool_name": "yet-another-tool", + "tool_version": "version", + "tool_configuration": {}, + }, + ] actual_tools = list(storage.indexer_configuration_add(new_tools)) assert len(actual_tools) == 2 # order not guaranteed, so we iterate over results to check for tool in actual_tools: - _id = tool.pop('id') + _id = tool.pop("id") assert _id is not None assert tool in new_tools - def test_indexer_configuration_get_missing( - self, swh_indexer_storage_with_data): + def test_indexer_configuration_get_missing(self, swh_indexer_storage_with_data): storage, data = swh_indexer_storage_with_data tool = { - 'tool_name': 'unknown-tool', - 'tool_version': '3.1.0rc2-31-ga2cbb8c', - 'tool_configuration': {"command_line": "nomossa "}, + "tool_name": "unknown-tool", + "tool_version": "3.1.0rc2-31-ga2cbb8c", + "tool_configuration": {"command_line": "nomossa "}, } actual_tool = storage.indexer_configuration_get(tool) assert actual_tool is None - def test_indexer_configuration_get( - self, swh_indexer_storage_with_data): + def test_indexer_configuration_get(self, swh_indexer_storage_with_data): storage, data = swh_indexer_storage_with_data tool = { - 'tool_name': 'nomos', - 'tool_version': '3.1.0rc2-31-ga2cbb8c', - 'tool_configuration': {"command_line": "nomossa "}, + "tool_name": "nomos", + "tool_version": "3.1.0rc2-31-ga2cbb8c", + "tool_configuration": {"command_line": "nomossa "}, } actual_tool = storage.indexer_configuration_get(tool) assert actual_tool expected_tool = tool.copy() - del actual_tool['id'] + del actual_tool["id"] assert expected_tool == actual_tool def test_indexer_configuration_metadata_get_missing_context( - self, swh_indexer_storage_with_data): + self, swh_indexer_storage_with_data + ): storage, data = swh_indexer_storage_with_data tool = { - 'tool_name': 'swh-metadata-translator', - 'tool_version': '0.0.1', - 'tool_configuration': {"context": "unknown-context"}, + "tool_name": "swh-metadata-translator", + "tool_version": "0.0.1", + "tool_configuration": {"context": "unknown-context"}, } actual_tool = storage.indexer_configuration_get(tool) assert actual_tool is None - def test_indexer_configuration_metadata_get( - self, swh_indexer_storage_with_data): + def test_indexer_configuration_metadata_get(self, swh_indexer_storage_with_data): storage, data = swh_indexer_storage_with_data tool = { - 'tool_name': 'swh-metadata-translator', - 'tool_version': '0.0.1', - 'tool_configuration': {"type": "local", "context": "NpmMapping"}, + "tool_name": "swh-metadata-translator", + "tool_version": "0.0.1", + "tool_configuration": {"type": "local", "context": "NpmMapping"}, } storage.indexer_configuration_add([tool]) actual_tool = storage.indexer_configuration_get(tool) assert actual_tool expected_tool = tool.copy() - expected_tool['id'] = actual_tool['id'] + expected_tool["id"] = actual_tool["id"] assert expected_tool == actual_tool diff --git a/swh/indexer/tests/tasks.py b/swh/indexer/tests/tasks.py index 64f5b32..46993b5 100644 --- a/swh/indexer/tests/tasks.py +++ b/swh/indexer/tests/tasks.py @@ -1,53 +1,46 @@ from celery import current_app as app -from swh.indexer.metadata import ( - OriginMetadataIndexer, RevisionMetadataIndexer -) +from swh.indexer.metadata import OriginMetadataIndexer, RevisionMetadataIndexer from .test_origin_head import OriginHeadTestIndexer from .test_metadata import ContentMetadataTestIndexer from .utils import BASE_TEST_CONFIG class RevisionMetadataTestIndexer(RevisionMetadataIndexer): """Specific indexer whose configuration is enough to satisfy the indexing tests. """ + ContentMetadataIndexer = ContentMetadataTestIndexer def parse_config_file(self, *args, **kwargs): return { **BASE_TEST_CONFIG, - 'tools': { - 'name': 'swh-metadata-detector', - 'version': '0.0.2', - 'configuration': { - 'type': 'local', - 'context': 'NpmMapping' - } - } + "tools": { + "name": "swh-metadata-detector", + "version": "0.0.2", + "configuration": {"type": "local", "context": "NpmMapping"}, + }, } class OriginMetadataTestIndexer(OriginMetadataIndexer): def parse_config_file(self, *args, **kwargs): - return { - **BASE_TEST_CONFIG, - 'tools': [] - } + return {**BASE_TEST_CONFIG, "tools": []} def _prepare_sub_indexers(self): self.origin_head_indexer = OriginHeadTestIndexer() self.revision_metadata_indexer = RevisionMetadataTestIndexer() @app.task def revision_intrinsic_metadata(*args, **kwargs): indexer = RevisionMetadataTestIndexer() indexer.run(*args, **kwargs) - print('REV RESULT=', indexer.results) + print("REV RESULT=", indexer.results) @app.task def origin_intrinsic_metadata(*args, **kwargs): indexer = OriginMetadataTestIndexer() indexer.run(*args, **kwargs) diff --git a/swh/indexer/tests/test_cli.py b/swh/indexer/tests/test_cli.py index 32e0e19..08ad477 100644 --- a/swh/indexer/tests/test_cli.py +++ b/swh/indexer/tests/test_cli.py @@ -1,359 +1,373 @@ # Copyright (C) 2019-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from functools import reduce import re import tempfile from unittest.mock import patch from click.testing import CliRunner from swh.journal.tests.utils import FakeKafkaMessage, MockedKafkaConsumer from swh.model.hashutil import hash_to_bytes from swh.indexer.cli import cli -CLI_CONFIG = ''' +CLI_CONFIG = """ scheduler: cls: foo args: {} storage: cls: memory indexer_storage: cls: memory args: {} -''' +""" def fill_idx_storage(idx_storage, nb_rows): tools = [ - { - 'tool_name': 'tool %d' % i, - 'tool_version': '0.0.1', - 'tool_configuration': {}, - } + {"tool_name": "tool %d" % i, "tool_version": "0.0.1", "tool_configuration": {},} for i in range(2) ] tools = idx_storage.indexer_configuration_add(tools) origin_metadata = [ { - 'id': 'file://dev/%04d' % origin_id, - 'from_revision': hash_to_bytes('abcd{:0>4}'.format(origin_id)), - 'indexer_configuration_id': tools[origin_id % 2]['id'], - 'metadata': {'name': 'origin %d' % origin_id}, - 'mappings': ['mapping%d' % (origin_id % 10)] + "id": "file://dev/%04d" % origin_id, + "from_revision": hash_to_bytes("abcd{:0>4}".format(origin_id)), + "indexer_configuration_id": tools[origin_id % 2]["id"], + "metadata": {"name": "origin %d" % origin_id}, + "mappings": ["mapping%d" % (origin_id % 10)], } for origin_id in range(nb_rows) ] revision_metadata = [ { - 'id': hash_to_bytes('abcd{:0>4}'.format(origin_id)), - 'indexer_configuration_id': tools[origin_id % 2]['id'], - 'metadata': {'name': 'origin %d' % origin_id}, - 'mappings': ['mapping%d' % (origin_id % 10)] + "id": hash_to_bytes("abcd{:0>4}".format(origin_id)), + "indexer_configuration_id": tools[origin_id % 2]["id"], + "metadata": {"name": "origin %d" % origin_id}, + "mappings": ["mapping%d" % (origin_id % 10)], } for origin_id in range(nb_rows) ] idx_storage.revision_intrinsic_metadata_add(revision_metadata) idx_storage.origin_intrinsic_metadata_add(origin_metadata) - return [tool['id'] for tool in tools] + return [tool["id"] for tool in tools] def _origins_in_task_args(tasks): """Returns the set of origins contained in the arguments of the provided tasks (assumed to be of type index-origin-metadata).""" return reduce( - set.union, - (set(task['arguments']['args'][0]) for task in tasks), - set() + set.union, (set(task["arguments"]["args"][0]) for task in tasks), set() ) def _assert_tasks_for_origins(tasks, origins): expected_kwargs = {"policy_update": "update-dups"} - assert {task['type'] for task in tasks} == {'index-origin-metadata'} - assert all(len(task['arguments']['args']) == 1 for task in tasks) + assert {task["type"] for task in tasks} == {"index-origin-metadata"} + assert all(len(task["arguments"]["args"]) == 1 for task in tasks) for task in tasks: - assert task['arguments']['kwargs'] == expected_kwargs, task - assert _origins_in_task_args(tasks) == set([ - 'file://dev/%04d' % i for i in origins]) + assert task["arguments"]["kwargs"] == expected_kwargs, task + assert _origins_in_task_args(tasks) == set(["file://dev/%04d" % i for i in origins]) def invoke(scheduler, catch_exceptions, args): runner = CliRunner() - with patch('swh.indexer.cli.get_scheduler') as get_scheduler_mock, \ - tempfile.NamedTemporaryFile('a', suffix='.yml') as config_fd: + with patch( + "swh.indexer.cli.get_scheduler" + ) as get_scheduler_mock, tempfile.NamedTemporaryFile( + "a", suffix=".yml" + ) as config_fd: config_fd.write(CLI_CONFIG) config_fd.seek(0) get_scheduler_mock.return_value = scheduler - result = runner.invoke(cli, ['-C' + config_fd.name] + args) + result = runner.invoke(cli, ["-C" + config_fd.name] + args) if not catch_exceptions and result.exception: print(result.output) raise result.exception return result def test_mapping_list(indexer_scheduler): - result = invoke(indexer_scheduler, False, [ - 'mapping', 'list', - ]) - expected_output = '\n'.join([ - 'codemeta', 'gemspec', 'maven', 'npm', 'pkg-info', '', - ]) + result = invoke(indexer_scheduler, False, ["mapping", "list",]) + expected_output = "\n".join( + ["codemeta", "gemspec", "maven", "npm", "pkg-info", "",] + ) assert result.exit_code == 0, result.output assert result.output == expected_output def test_mapping_list_terms(indexer_scheduler): - result = invoke(indexer_scheduler, False, [ - 'mapping', 'list-terms', - ]) + result = invoke(indexer_scheduler, False, ["mapping", "list-terms",]) assert result.exit_code == 0, result.output - assert re.search(r'http://schema.org/url:\n.*npm', result.output) - assert re.search(r'http://schema.org/url:\n.*codemeta', result.output) + assert re.search(r"http://schema.org/url:\n.*npm", result.output) + assert re.search(r"http://schema.org/url:\n.*codemeta", result.output) assert re.search( - r'https://codemeta.github.io/terms/developmentStatus:\n\tcodemeta', - result.output) + r"https://codemeta.github.io/terms/developmentStatus:\n\tcodemeta", + result.output, + ) def test_mapping_list_terms_exclude(indexer_scheduler): - result = invoke(indexer_scheduler, False, [ - 'mapping', 'list-terms', - '--exclude-mapping', 'codemeta' - ]) + result = invoke( + indexer_scheduler, + False, + ["mapping", "list-terms", "--exclude-mapping", "codemeta"], + ) assert result.exit_code == 0, result.output - assert re.search(r'http://schema.org/url:\n.*npm', result.output) - assert not re.search(r'http://schema.org/url:\n.*codemeta', result.output) + assert re.search(r"http://schema.org/url:\n.*npm", result.output) + assert not re.search(r"http://schema.org/url:\n.*codemeta", result.output) assert not re.search( - r'https://codemeta.github.io/terms/developmentStatus:\n\tcodemeta', - result.output) + r"https://codemeta.github.io/terms/developmentStatus:\n\tcodemeta", + result.output, + ) -@patch('swh.scheduler.cli.utils.TASK_BATCH_SIZE', 3) -@patch('swh.scheduler.cli_utils.TASK_BATCH_SIZE', 3) -def test_origin_metadata_reindex_empty_db( - indexer_scheduler, idx_storage, storage): - result = invoke(indexer_scheduler, False, [ - 'schedule', 'reindex_origin_metadata', - ]) - expected_output = ( - 'Nothing to do (no origin metadata matched the criteria).\n' - ) +@patch("swh.scheduler.cli.utils.TASK_BATCH_SIZE", 3) +@patch("swh.scheduler.cli_utils.TASK_BATCH_SIZE", 3) +def test_origin_metadata_reindex_empty_db(indexer_scheduler, idx_storage, storage): + result = invoke(indexer_scheduler, False, ["schedule", "reindex_origin_metadata",]) + expected_output = "Nothing to do (no origin metadata matched the criteria).\n" assert result.exit_code == 0, result.output assert result.output == expected_output tasks = indexer_scheduler.search_tasks() assert len(tasks) == 0 -@patch('swh.scheduler.cli.utils.TASK_BATCH_SIZE', 3) -@patch('swh.scheduler.cli_utils.TASK_BATCH_SIZE', 3) -def test_origin_metadata_reindex_divisor( - indexer_scheduler, idx_storage, storage): +@patch("swh.scheduler.cli.utils.TASK_BATCH_SIZE", 3) +@patch("swh.scheduler.cli_utils.TASK_BATCH_SIZE", 3) +def test_origin_metadata_reindex_divisor(indexer_scheduler, idx_storage, storage): """Tests the re-indexing when origin_batch_size*task_batch_size is a divisor of nb_origins.""" fill_idx_storage(idx_storage, 90) - result = invoke(indexer_scheduler, False, [ - 'schedule', 'reindex_origin_metadata', - ]) + result = invoke(indexer_scheduler, False, ["schedule", "reindex_origin_metadata",]) # Check the output expected_output = ( - 'Scheduled 3 tasks (30 origins).\n' - 'Scheduled 6 tasks (60 origins).\n' - 'Scheduled 9 tasks (90 origins).\n' - 'Done.\n' + "Scheduled 3 tasks (30 origins).\n" + "Scheduled 6 tasks (60 origins).\n" + "Scheduled 9 tasks (90 origins).\n" + "Done.\n" ) assert result.exit_code == 0, result.output assert result.output == expected_output # Check scheduled tasks tasks = indexer_scheduler.search_tasks() assert len(tasks) == 9 _assert_tasks_for_origins(tasks, range(90)) -@patch('swh.scheduler.cli.utils.TASK_BATCH_SIZE', 3) -@patch('swh.scheduler.cli_utils.TASK_BATCH_SIZE', 3) -def test_origin_metadata_reindex_dry_run( - indexer_scheduler, idx_storage, storage): +@patch("swh.scheduler.cli.utils.TASK_BATCH_SIZE", 3) +@patch("swh.scheduler.cli_utils.TASK_BATCH_SIZE", 3) +def test_origin_metadata_reindex_dry_run(indexer_scheduler, idx_storage, storage): """Tests the re-indexing when origin_batch_size*task_batch_size is a divisor of nb_origins.""" fill_idx_storage(idx_storage, 90) - result = invoke(indexer_scheduler, False, [ - 'schedule', '--dry-run', 'reindex_origin_metadata', - ]) + result = invoke( + indexer_scheduler, False, ["schedule", "--dry-run", "reindex_origin_metadata",] + ) # Check the output expected_output = ( - 'Scheduled 3 tasks (30 origins).\n' - 'Scheduled 6 tasks (60 origins).\n' - 'Scheduled 9 tasks (90 origins).\n' - 'Done.\n' + "Scheduled 3 tasks (30 origins).\n" + "Scheduled 6 tasks (60 origins).\n" + "Scheduled 9 tasks (90 origins).\n" + "Done.\n" ) assert result.exit_code == 0, result.output assert result.output == expected_output # Check scheduled tasks tasks = indexer_scheduler.search_tasks() assert len(tasks) == 0 -@patch('swh.scheduler.cli.utils.TASK_BATCH_SIZE', 3) -@patch('swh.scheduler.cli_utils.TASK_BATCH_SIZE', 3) -def test_origin_metadata_reindex_nondivisor( - indexer_scheduler, idx_storage, storage): +@patch("swh.scheduler.cli.utils.TASK_BATCH_SIZE", 3) +@patch("swh.scheduler.cli_utils.TASK_BATCH_SIZE", 3) +def test_origin_metadata_reindex_nondivisor(indexer_scheduler, idx_storage, storage): """Tests the re-indexing when neither origin_batch_size or task_batch_size is a divisor of nb_origins.""" fill_idx_storage(idx_storage, 70) - result = invoke(indexer_scheduler, False, [ - 'schedule', 'reindex_origin_metadata', - '--batch-size', '20', - ]) + result = invoke( + indexer_scheduler, + False, + ["schedule", "reindex_origin_metadata", "--batch-size", "20",], + ) # Check the output expected_output = ( - 'Scheduled 3 tasks (60 origins).\n' - 'Scheduled 4 tasks (70 origins).\n' - 'Done.\n' + "Scheduled 3 tasks (60 origins).\n" + "Scheduled 4 tasks (70 origins).\n" + "Done.\n" ) assert result.exit_code == 0, result.output assert result.output == expected_output # Check scheduled tasks tasks = indexer_scheduler.search_tasks() assert len(tasks) == 4 _assert_tasks_for_origins(tasks, range(70)) -@patch('swh.scheduler.cli.utils.TASK_BATCH_SIZE', 3) -@patch('swh.scheduler.cli_utils.TASK_BATCH_SIZE', 3) +@patch("swh.scheduler.cli.utils.TASK_BATCH_SIZE", 3) +@patch("swh.scheduler.cli_utils.TASK_BATCH_SIZE", 3) def test_origin_metadata_reindex_filter_one_mapping( - indexer_scheduler, idx_storage, storage): + indexer_scheduler, idx_storage, storage +): """Tests the re-indexing when origin_batch_size*task_batch_size is a divisor of nb_origins.""" fill_idx_storage(idx_storage, 110) - result = invoke(indexer_scheduler, False, [ - 'schedule', 'reindex_origin_metadata', - '--mapping', 'mapping1', - ]) + result = invoke( + indexer_scheduler, + False, + ["schedule", "reindex_origin_metadata", "--mapping", "mapping1",], + ) # Check the output - expected_output = ( - 'Scheduled 2 tasks (11 origins).\n' - 'Done.\n' - ) + expected_output = "Scheduled 2 tasks (11 origins).\n" "Done.\n" assert result.exit_code == 0, result.output assert result.output == expected_output # Check scheduled tasks tasks = indexer_scheduler.search_tasks() assert len(tasks) == 2 - _assert_tasks_for_origins( - tasks, - [1, 11, 21, 31, 41, 51, 61, 71, 81, 91, 101]) + _assert_tasks_for_origins(tasks, [1, 11, 21, 31, 41, 51, 61, 71, 81, 91, 101]) -@patch('swh.scheduler.cli.utils.TASK_BATCH_SIZE', 3) -@patch('swh.scheduler.cli_utils.TASK_BATCH_SIZE', 3) +@patch("swh.scheduler.cli.utils.TASK_BATCH_SIZE", 3) +@patch("swh.scheduler.cli_utils.TASK_BATCH_SIZE", 3) def test_origin_metadata_reindex_filter_two_mappings( - indexer_scheduler, idx_storage, storage): + indexer_scheduler, idx_storage, storage +): """Tests the re-indexing when origin_batch_size*task_batch_size is a divisor of nb_origins.""" fill_idx_storage(idx_storage, 110) - result = invoke(indexer_scheduler, False, [ - 'schedule', 'reindex_origin_metadata', - '--mapping', 'mapping1', '--mapping', 'mapping2', - ]) + result = invoke( + indexer_scheduler, + False, + [ + "schedule", + "reindex_origin_metadata", + "--mapping", + "mapping1", + "--mapping", + "mapping2", + ], + ) # Check the output - expected_output = ( - 'Scheduled 3 tasks (22 origins).\n' - 'Done.\n' - ) + expected_output = "Scheduled 3 tasks (22 origins).\n" "Done.\n" assert result.exit_code == 0, result.output assert result.output == expected_output # Check scheduled tasks tasks = indexer_scheduler.search_tasks() assert len(tasks) == 3 _assert_tasks_for_origins( tasks, - [1, 11, 21, 31, 41, 51, 61, 71, 81, 91, 101, - 2, 12, 22, 32, 42, 52, 62, 72, 82, 92, 102]) + [ + 1, + 11, + 21, + 31, + 41, + 51, + 61, + 71, + 81, + 91, + 101, + 2, + 12, + 22, + 32, + 42, + 52, + 62, + 72, + 82, + 92, + 102, + ], + ) -@patch('swh.scheduler.cli.utils.TASK_BATCH_SIZE', 3) -@patch('swh.scheduler.cli_utils.TASK_BATCH_SIZE', 3) +@patch("swh.scheduler.cli.utils.TASK_BATCH_SIZE", 3) +@patch("swh.scheduler.cli_utils.TASK_BATCH_SIZE", 3) def test_origin_metadata_reindex_filter_one_tool( - indexer_scheduler, idx_storage, storage): + indexer_scheduler, idx_storage, storage +): """Tests the re-indexing when origin_batch_size*task_batch_size is a divisor of nb_origins.""" tool_ids = fill_idx_storage(idx_storage, 110) - result = invoke(indexer_scheduler, False, [ - 'schedule', 'reindex_origin_metadata', - '--tool-id', str(tool_ids[0]), - ]) + result = invoke( + indexer_scheduler, + False, + ["schedule", "reindex_origin_metadata", "--tool-id", str(tool_ids[0]),], + ) # Check the output expected_output = ( - 'Scheduled 3 tasks (30 origins).\n' - 'Scheduled 6 tasks (55 origins).\n' - 'Done.\n' + "Scheduled 3 tasks (30 origins).\n" + "Scheduled 6 tasks (55 origins).\n" + "Done.\n" ) assert result.exit_code == 0, result.output assert result.output == expected_output # Check scheduled tasks tasks = indexer_scheduler.search_tasks() assert len(tasks) == 6 - _assert_tasks_for_origins( - tasks, - [x*2 for x in range(55)]) + _assert_tasks_for_origins(tasks, [x * 2 for x in range(55)]) def test_journal_client(storage, indexer_scheduler): """Test the 'swh indexer journal-client' cli tool.""" - message = FakeKafkaMessage('swh.journal.objects.origin_visit', 'bogus', { - 'status': 'full', - 'origin': { - 'url': 'file://dev/0000', - } - }) + message = FakeKafkaMessage( + "swh.journal.objects.origin_visit", + "bogus", + {"status": "full", "origin": {"url": "file://dev/0000",}}, + ) consumer = MockedKafkaConsumer([message]) - with patch('swh.journal.client.Consumer', - return_value=consumer): - result = invoke(indexer_scheduler, False, [ - 'journal-client', - '--stop-after-objects', '1', - '--broker', '192.0.2.1', - '--prefix', 'swh.journal.objects', - '--group-id', 'test-consumer', - ]) + with patch("swh.journal.client.Consumer", return_value=consumer): + result = invoke( + indexer_scheduler, + False, + [ + "journal-client", + "--stop-after-objects", + "1", + "--broker", + "192.0.2.1", + "--prefix", + "swh.journal.objects", + "--group-id", + "test-consumer", + ], + ) # Check the output - expected_output = ( - 'Done.\n' - ) + expected_output = "Done.\n" assert result.exit_code == 0, result.output assert result.output == expected_output # Check scheduled tasks tasks = indexer_scheduler.search_tasks() assert len(tasks) == 1 - _assert_tasks_for_origins( - tasks, - [0]) + _assert_tasks_for_origins(tasks, [0]) diff --git a/swh/indexer/tests/test_codemeta.py b/swh/indexer/tests/test_codemeta.py index c73b23a..dd632f1 100644 --- a/swh/indexer/tests/test_codemeta.py +++ b/swh/indexer/tests/test_codemeta.py @@ -1,283 +1,259 @@ # Copyright (C) 2018-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import pytest from swh.indexer.codemeta import merge_documents, merge_values from swh.indexer.codemeta import CROSSWALK_TABLE def test_crosstable(): - assert CROSSWALK_TABLE['NodeJS'] == { - 'repository': 'http://schema.org/codeRepository', - 'os': 'http://schema.org/operatingSystem', - 'cpu': 'http://schema.org/processorRequirements', - 'engines': - 'http://schema.org/processorRequirements', - 'author': 'http://schema.org/author', - 'author.email': 'http://schema.org/email', - 'author.name': 'http://schema.org/name', - 'contributor': 'http://schema.org/contributor', - 'keywords': 'http://schema.org/keywords', - 'license': 'http://schema.org/license', - 'version': 'http://schema.org/version', - 'description': 'http://schema.org/description', - 'name': 'http://schema.org/name', - 'bugs': 'https://codemeta.github.io/terms/issueTracker', - 'homepage': 'http://schema.org/url' + assert CROSSWALK_TABLE["NodeJS"] == { + "repository": "http://schema.org/codeRepository", + "os": "http://schema.org/operatingSystem", + "cpu": "http://schema.org/processorRequirements", + "engines": "http://schema.org/processorRequirements", + "author": "http://schema.org/author", + "author.email": "http://schema.org/email", + "author.name": "http://schema.org/name", + "contributor": "http://schema.org/contributor", + "keywords": "http://schema.org/keywords", + "license": "http://schema.org/license", + "version": "http://schema.org/version", + "description": "http://schema.org/description", + "name": "http://schema.org/name", + "bugs": "https://codemeta.github.io/terms/issueTracker", + "homepage": "http://schema.org/url", } def test_merge_values(): - assert merge_values('a', 'b') == ['a', 'b'] - assert merge_values(['a', 'b'], 'c') == ['a', 'b', 'c'] - assert merge_values('a', ['b', 'c']) == ['a', 'b', 'c'] + assert merge_values("a", "b") == ["a", "b"] + assert merge_values(["a", "b"], "c") == ["a", "b", "c"] + assert merge_values("a", ["b", "c"]) == ["a", "b", "c"] - assert merge_values({'@list': ['a']}, {'@list': ['b']}) \ - == {'@list': ['a', 'b']} - assert merge_values({'@list': ['a', 'b']}, {'@list': ['c']}) \ - == {'@list': ['a', 'b', 'c']} + assert merge_values({"@list": ["a"]}, {"@list": ["b"]}) == {"@list": ["a", "b"]} + assert merge_values({"@list": ["a", "b"]}, {"@list": ["c"]}) == { + "@list": ["a", "b", "c"] + } with pytest.raises(ValueError): - merge_values({'@list': ['a']}, 'b') + merge_values({"@list": ["a"]}, "b") with pytest.raises(ValueError): - merge_values('a', {'@list': ['b']}) + merge_values("a", {"@list": ["b"]}) with pytest.raises(ValueError): - merge_values({'@list': ['a']}, ['b']) + merge_values({"@list": ["a"]}, ["b"]) with pytest.raises(ValueError): - merge_values(['a'], {'@list': ['b']}) + merge_values(["a"], {"@list": ["b"]}) - assert merge_values('a', None) == 'a' - assert merge_values(['a', 'b'], None) == ['a', 'b'] - assert merge_values(None, ['b', 'c']) == ['b', 'c'] - assert merge_values({'@list': ['a']}, None) == {'@list': ['a']} - assert merge_values(None, {'@list': ['a']}) == {'@list': ['a']} + assert merge_values("a", None) == "a" + assert merge_values(["a", "b"], None) == ["a", "b"] + assert merge_values(None, ["b", "c"]) == ["b", "c"] + assert merge_values({"@list": ["a"]}, None) == {"@list": ["a"]} + assert merge_values(None, {"@list": ["a"]}) == {"@list": ["a"]} def test_merge_documents(): """ Test the creation of a coherent minimal metadata set """ # given - metadata_list = [{ - '@context': 'https://doi.org/10.5063/schema/codemeta-2.0', - 'name': 'test_1', - 'version': '0.0.2', - 'description': 'Simple package.json test for indexer', - 'codeRepository': - 'git+https://github.com/moranegg/metadata_test', - }, { - '@context': 'https://doi.org/10.5063/schema/codemeta-2.0', - 'name': 'test_0_1', - 'version': '0.0.2', - 'description': 'Simple package.json test for indexer', - 'codeRepository': - 'git+https://github.com/moranegg/metadata_test' - }, { - '@context': 'https://doi.org/10.5063/schema/codemeta-2.0', - 'name': 'test_metadata', - 'version': '0.0.2', - 'author': { - 'type': 'Person', - 'name': 'moranegg', + metadata_list = [ + { + "@context": "https://doi.org/10.5063/schema/codemeta-2.0", + "name": "test_1", + "version": "0.0.2", + "description": "Simple package.json test for indexer", + "codeRepository": "git+https://github.com/moranegg/metadata_test", + }, + { + "@context": "https://doi.org/10.5063/schema/codemeta-2.0", + "name": "test_0_1", + "version": "0.0.2", + "description": "Simple package.json test for indexer", + "codeRepository": "git+https://github.com/moranegg/metadata_test", + }, + { + "@context": "https://doi.org/10.5063/schema/codemeta-2.0", + "name": "test_metadata", + "version": "0.0.2", + "author": {"type": "Person", "name": "moranegg",}, }, - }] + ] # when results = merge_documents(metadata_list) # then expected_results = { - '@context': 'https://doi.org/10.5063/schema/codemeta-2.0', - "version": '0.0.2', - "description": 'Simple package.json test for indexer', - "name": ['test_1', 'test_0_1', 'test_metadata'], - "author": [{ - 'type': 'Person', - 'name': 'moranegg' - }], - "codeRepository": - 'git+https://github.com/moranegg/metadata_test', + "@context": "https://doi.org/10.5063/schema/codemeta-2.0", + "version": "0.0.2", + "description": "Simple package.json test for indexer", + "name": ["test_1", "test_0_1", "test_metadata"], + "author": [{"type": "Person", "name": "moranegg"}], + "codeRepository": "git+https://github.com/moranegg/metadata_test", } assert results == expected_results def test_merge_documents_ids(): # given - metadata_list = [{ - '@context': 'https://doi.org/10.5063/schema/codemeta-2.0', - 'id': 'http://example.org/test1', - 'name': 'test_1', - }, { - '@context': 'https://doi.org/10.5063/schema/codemeta-2.0', - 'id': 'http://example.org/test2', - 'name': 'test_2', - }] + metadata_list = [ + { + "@context": "https://doi.org/10.5063/schema/codemeta-2.0", + "id": "http://example.org/test1", + "name": "test_1", + }, + { + "@context": "https://doi.org/10.5063/schema/codemeta-2.0", + "id": "http://example.org/test2", + "name": "test_2", + }, + ] # when results = merge_documents(metadata_list) # then expected_results = { - '@context': 'https://doi.org/10.5063/schema/codemeta-2.0', - 'id': 'http://example.org/test1', - 'schema:sameAs': 'http://example.org/test2', - "name": ['test_1', 'test_2'] + "@context": "https://doi.org/10.5063/schema/codemeta-2.0", + "id": "http://example.org/test1", + "schema:sameAs": "http://example.org/test2", + "name": ["test_1", "test_2"], } assert results == expected_results def test_merge_documents_duplicate_ids(): # given - metadata_list = [{ - '@context': 'https://doi.org/10.5063/schema/codemeta-2.0', - 'id': 'http://example.org/test1', - 'name': 'test_1', - }, { - '@context': 'https://doi.org/10.5063/schema/codemeta-2.0', - 'id': 'http://example.org/test1', - 'name': 'test_1b', - }, { - '@context': 'https://doi.org/10.5063/schema/codemeta-2.0', - 'id': 'http://example.org/test2', - 'name': 'test_2', - }] + metadata_list = [ + { + "@context": "https://doi.org/10.5063/schema/codemeta-2.0", + "id": "http://example.org/test1", + "name": "test_1", + }, + { + "@context": "https://doi.org/10.5063/schema/codemeta-2.0", + "id": "http://example.org/test1", + "name": "test_1b", + }, + { + "@context": "https://doi.org/10.5063/schema/codemeta-2.0", + "id": "http://example.org/test2", + "name": "test_2", + }, + ] # when results = merge_documents(metadata_list) # then expected_results = { - '@context': 'https://doi.org/10.5063/schema/codemeta-2.0', - 'id': 'http://example.org/test1', - 'schema:sameAs': 'http://example.org/test2', - "name": ['test_1', 'test_1b', 'test_2'] + "@context": "https://doi.org/10.5063/schema/codemeta-2.0", + "id": "http://example.org/test1", + "schema:sameAs": "http://example.org/test2", + "name": ["test_1", "test_1b", "test_2"], } assert results == expected_results def test_merge_documents_lists(): """Tests merging two @list elements.""" # given - metadata_list = [{ - '@context': 'https://doi.org/10.5063/schema/codemeta-2.0', - 'author': { - '@list': [ - {'name': 'test_1'}, - ] + metadata_list = [ + { + "@context": "https://doi.org/10.5063/schema/codemeta-2.0", + "author": {"@list": [{"name": "test_1"},]}, }, - }, { - '@context': 'https://doi.org/10.5063/schema/codemeta-2.0', - 'author': { - '@list': [ - {'name': 'test_2'}, - ] + { + "@context": "https://doi.org/10.5063/schema/codemeta-2.0", + "author": {"@list": [{"name": "test_2"},]}, }, - }] + ] # when results = merge_documents(metadata_list) # then expected_results = { - '@context': 'https://doi.org/10.5063/schema/codemeta-2.0', - 'author': [ - {'name': 'test_1'}, - {'name': 'test_2'}, - ], + "@context": "https://doi.org/10.5063/schema/codemeta-2.0", + "author": [{"name": "test_1"}, {"name": "test_2"},], } assert results == expected_results def test_merge_documents_lists_duplicates(): """Tests merging two @list elements with a duplicate subelement.""" # given - metadata_list = [{ - '@context': 'https://doi.org/10.5063/schema/codemeta-2.0', - 'author': { - '@list': [ - {'name': 'test_1'}, - ] + metadata_list = [ + { + "@context": "https://doi.org/10.5063/schema/codemeta-2.0", + "author": {"@list": [{"name": "test_1"},]}, }, - }, { - '@context': 'https://doi.org/10.5063/schema/codemeta-2.0', - 'author': { - '@list': [ - {'name': 'test_2'}, - {'name': 'test_1'}, - ] + { + "@context": "https://doi.org/10.5063/schema/codemeta-2.0", + "author": {"@list": [{"name": "test_2"}, {"name": "test_1"},]}, }, - }] + ] # when results = merge_documents(metadata_list) # then expected_results = { - '@context': 'https://doi.org/10.5063/schema/codemeta-2.0', - 'author': [ - {'name': 'test_1'}, - {'name': 'test_2'}, - ], + "@context": "https://doi.org/10.5063/schema/codemeta-2.0", + "author": [{"name": "test_1"}, {"name": "test_2"},], } assert results == expected_results def test_merge_documents_list_left(): """Tests merging a singleton with an @list.""" # given - metadata_list = [{ - '@context': 'https://doi.org/10.5063/schema/codemeta-2.0', - 'author': {'name': 'test_1'}, - }, { - '@context': 'https://doi.org/10.5063/schema/codemeta-2.0', - 'author': { - '@list': [ - {'name': 'test_2'}, - ] + metadata_list = [ + { + "@context": "https://doi.org/10.5063/schema/codemeta-2.0", + "author": {"name": "test_1"}, }, - }] + { + "@context": "https://doi.org/10.5063/schema/codemeta-2.0", + "author": {"@list": [{"name": "test_2"},]}, + }, + ] # when results = merge_documents(metadata_list) # then expected_results = { - '@context': 'https://doi.org/10.5063/schema/codemeta-2.0', - 'author': [ - {'name': 'test_1'}, - {'name': 'test_2'}, - ], + "@context": "https://doi.org/10.5063/schema/codemeta-2.0", + "author": [{"name": "test_1"}, {"name": "test_2"},], } assert results == expected_results def test_merge_documents_list_right(): """Tests merging an @list with a singleton.""" # given - metadata_list = [{ - '@context': 'https://doi.org/10.5063/schema/codemeta-2.0', - 'author': { - '@list': [ - {'name': 'test_1'}, - ] + metadata_list = [ + { + "@context": "https://doi.org/10.5063/schema/codemeta-2.0", + "author": {"@list": [{"name": "test_1"},]}, + }, + { + "@context": "https://doi.org/10.5063/schema/codemeta-2.0", + "author": {"name": "test_2"}, }, - }, { - '@context': 'https://doi.org/10.5063/schema/codemeta-2.0', - 'author': {'name': 'test_2'}, - }] + ] # when results = merge_documents(metadata_list) # then expected_results = { - '@context': 'https://doi.org/10.5063/schema/codemeta-2.0', - 'author': [ - {'name': 'test_1'}, - {'name': 'test_2'}, - ], + "@context": "https://doi.org/10.5063/schema/codemeta-2.0", + "author": [{"name": "test_1"}, {"name": "test_2"},], } assert results == expected_results diff --git a/swh/indexer/tests/test_ctags.py b/swh/indexer/tests/test_ctags.py index e2bf5db..fca6498 100644 --- a/swh/indexer/tests/test_ctags.py +++ b/swh/indexer/tests/test_ctags.py @@ -1,183 +1,155 @@ # Copyright (C) 2017-2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import json import unittest from unittest.mock import patch import pytest import swh.indexer.ctags -from swh.indexer.ctags import ( - CtagsIndexer, run_ctags -) +from swh.indexer.ctags import CtagsIndexer, run_ctags from swh.indexer.tests.utils import ( CommonContentIndexerTest, - SHA1_TO_CTAGS, BASE_TEST_CONFIG, - OBJ_STORAGE_DATA, fill_storage, fill_obj_storage, + SHA1_TO_CTAGS, + BASE_TEST_CONFIG, + OBJ_STORAGE_DATA, + fill_storage, + fill_obj_storage, filter_dict, ) class BasicTest(unittest.TestCase): @patch("swh.indexer.ctags.subprocess") def test_run_ctags(self, mock_subprocess): """Computing licenses from a raw content should return results """ output0 = """ {"name":"defun","kind":"function","line":1,"language":"scheme"} {"name":"name","kind":"symbol","line":5,"language":"else"}""" output1 = """ {"name":"let","kind":"var","line":10,"language":"something"}""" expected_result0 = [ - { - 'name': 'defun', - 'kind': 'function', - 'line': 1, - 'lang': 'scheme' - }, - { - 'name': 'name', - 'kind': 'symbol', - 'line': 5, - 'lang': 'else' - } + {"name": "defun", "kind": "function", "line": 1, "lang": "scheme"}, + {"name": "name", "kind": "symbol", "line": 5, "lang": "else"}, ] expected_result1 = [ - { - 'name': 'let', - 'kind': 'var', - 'line': 10, - 'lang': 'something' - } + {"name": "let", "kind": "var", "line": 10, "lang": "something"} ] for path, lang, intermediary_result, expected_result in [ - (b'some/path', 'lisp', output0, expected_result0), - (b'some/path/2', 'markdown', output1, expected_result1) + (b"some/path", "lisp", output0, expected_result0), + (b"some/path/2", "markdown", output1, expected_result1), ]: mock_subprocess.check_output.return_value = intermediary_result actual_result = list(run_ctags(path, lang=lang)) self.assertEqual(actual_result, expected_result) class InjectCtagsIndexer: """Override ctags computations. """ + def compute_ctags(self, path, lang): """Inject fake ctags given path (sha1 identifier). """ - return { - 'lang': lang, - **SHA1_TO_CTAGS.get(path) - } + return {"lang": lang, **SHA1_TO_CTAGS.get(path)} CONFIG = { **BASE_TEST_CONFIG, - 'tools': { - 'name': 'universal-ctags', - 'version': '~git7859817b', - 'configuration': { - 'command_line': '''ctags --fields=+lnz --sort=no ''' - ''' --links=no ''', - 'max_content_size': 1000, + "tools": { + "name": "universal-ctags", + "version": "~git7859817b", + "configuration": { + "command_line": """ctags --fields=+lnz --sort=no """ + """ --links=no """, + "max_content_size": 1000, }, }, - 'languages': { - 'python': 'python', - 'haskell': 'haskell', - 'bar': 'bar', - }, - 'workdir': '/tmp', + "languages": {"python": "python", "haskell": "haskell", "bar": "bar",}, + "workdir": "/tmp", } class TestCtagsIndexer(CommonContentIndexerTest, unittest.TestCase): """Ctags indexer test scenarios: - Known sha1s in the input list have their data indexed - Unknown sha1 in the input list are not indexed """ legacy_get_format = True def get_indexer_results(self, ids): yield from self.idx_storage.content_ctags_get(ids) def setUp(self): super().setUp() self.indexer = CtagsIndexer(config=CONFIG) self.indexer.catch_exceptions = False self.idx_storage = self.indexer.idx_storage fill_storage(self.indexer.storage) fill_obj_storage(self.indexer.objstorage) # Prepare test input - self.id0 = '01c9379dfc33803963d07c1ccc748d3fe4c96bb5' - self.id1 = 'd4c647f0fc257591cc9ba1722484229780d1c607' - self.id2 = '688a5ef812c53907562fe379d4b3851e69c7cb15' + self.id0 = "01c9379dfc33803963d07c1ccc748d3fe4c96bb5" + self.id1 = "d4c647f0fc257591cc9ba1722484229780d1c607" + self.id2 = "688a5ef812c53907562fe379d4b3851e69c7cb15" - tool = {k.replace('tool_', ''): v - for (k, v) in self.indexer.tool.items()} + tool = {k.replace("tool_", ""): v for (k, v) in self.indexer.tool.items()} self.expected_results = { - self.id0: { - 'id': self.id0, - 'tool': tool, - **SHA1_TO_CTAGS[self.id0][0], - }, - self.id1: { - 'id': self.id1, - 'tool': tool, - **SHA1_TO_CTAGS[self.id1][0], - }, - self.id2: { - 'id': self.id2, - 'tool': tool, - **SHA1_TO_CTAGS[self.id2][0], - } + self.id0: {"id": self.id0, "tool": tool, **SHA1_TO_CTAGS[self.id0][0],}, + self.id1: {"id": self.id1, "tool": tool, **SHA1_TO_CTAGS[self.id1][0],}, + self.id2: {"id": self.id2, "tool": tool, **SHA1_TO_CTAGS[self.id2][0],}, } self._set_mocks() def _set_mocks(self): def find_ctags_for_content(raw_content): for (sha1, ctags) in SHA1_TO_CTAGS.items(): if OBJ_STORAGE_DATA[sha1] == raw_content: return ctags else: - raise ValueError(('%r not found in objstorage, can\'t mock ' - 'its ctags.') % raw_content) + raise ValueError( + ("%r not found in objstorage, can't mock " "its ctags.") + % raw_content + ) def fake_language(raw_content, *args, **kwargs): ctags = find_ctags_for_content(raw_content) - return {'lang': ctags[0]['lang']} + return {"lang": ctags[0]["lang"]} + self._real_compute_language = swh.indexer.ctags.compute_language swh.indexer.ctags.compute_language = fake_language def fake_check_output(cmd, *args, **kwargs): - id_ = cmd[-1].split('/')[-1] - return '\n'.join( - json.dumps({'language': ctag['lang'], **ctag}) - for ctag in SHA1_TO_CTAGS[id_]) + id_ = cmd[-1].split("/")[-1] + return "\n".join( + json.dumps({"language": ctag["lang"], **ctag}) + for ctag in SHA1_TO_CTAGS[id_] + ) + self._real_check_output = swh.indexer.ctags.subprocess.check_output swh.indexer.ctags.subprocess.check_output = fake_check_output def tearDown(self): swh.indexer.ctags.compute_language = self._real_compute_language swh.indexer.ctags.subprocess.check_output = self._real_check_output super().tearDown() def test_ctags_w_no_tool(): with pytest.raises(ValueError): - CtagsIndexer(config=filter_dict(CONFIG, 'tools')) + CtagsIndexer(config=filter_dict(CONFIG, "tools")) diff --git a/swh/indexer/tests/test_fossology_license.py b/swh/indexer/tests/test_fossology_license.py index 90bfa40..5ee22e3 100644 --- a/swh/indexer/tests/test_fossology_license.py +++ b/swh/indexer/tests/test_fossology_license.py @@ -1,181 +1,170 @@ # Copyright (C) 2017-2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import unittest import pytest from unittest.mock import patch from typing import Any, Dict from swh.indexer import fossology_license from swh.indexer.fossology_license import ( - FossologyLicenseIndexer, FossologyLicenseRangeIndexer, - compute_license + FossologyLicenseIndexer, + FossologyLicenseRangeIndexer, + compute_license, ) from swh.indexer.tests.utils import ( - SHA1_TO_LICENSES, CommonContentIndexerTest, CommonContentIndexerRangeTest, - BASE_TEST_CONFIG, fill_storage, fill_obj_storage, filter_dict, + SHA1_TO_LICENSES, + CommonContentIndexerTest, + CommonContentIndexerRangeTest, + BASE_TEST_CONFIG, + fill_storage, + fill_obj_storage, + filter_dict, ) class BasicTest(unittest.TestCase): @patch("swh.indexer.fossology_license.subprocess") def test_compute_license(self, mock_subprocess): """Computing licenses from a raw content should return results """ for path, intermediary_result, output in [ - (b'some/path', None, - []), - (b'some/path/2', [], - []), - (b'other/path', ' contains license(s) GPL,AGPL', - ['GPL', 'AGPL'])]: + (b"some/path", None, []), + (b"some/path/2", [], []), + (b"other/path", " contains license(s) GPL,AGPL", ["GPL", "AGPL"]), + ]: mock_subprocess.check_output.return_value = intermediary_result actual_result = compute_license(path) - self.assertEqual(actual_result, { - 'licenses': output, - 'path': path, - }) + self.assertEqual(actual_result, {"licenses": output, "path": path,}) def mock_compute_license(path): """path is the content identifier """ if isinstance(id, bytes): - path = path.decode('utf-8') + path = path.decode("utf-8") # path is something like /tmp/tmpXXX/ so we keep only the sha1 part - path = path.split('/')[-1] - return { - 'licenses': SHA1_TO_LICENSES.get(path) - } + path = path.split("/")[-1] + return {"licenses": SHA1_TO_LICENSES.get(path)} CONFIG = { **BASE_TEST_CONFIG, - 'workdir': '/tmp', - 'tools': { - 'name': 'nomos', - 'version': '3.1.0rc2-31-ga2cbb8c', - 'configuration': { - 'command_line': 'nomossa ', - }, + "workdir": "/tmp", + "tools": { + "name": "nomos", + "version": "3.1.0rc2-31-ga2cbb8c", + "configuration": {"command_line": "nomossa ",}, }, } # type: Dict[str, Any] -RANGE_CONFIG = dict(list(CONFIG.items()) + [('write_batch_size', 100)]) +RANGE_CONFIG = dict(list(CONFIG.items()) + [("write_batch_size", 100)]) class TestFossologyLicenseIndexer(CommonContentIndexerTest, unittest.TestCase): """Language indexer test scenarios: - Known sha1s in the input list have their data indexed - Unknown sha1 in the input list are not indexed """ def get_indexer_results(self, ids): yield from self.idx_storage.content_fossology_license_get(ids) def setUp(self): super().setUp() # replace actual license computation with a mock self.orig_compute_license = fossology_license.compute_license fossology_license.compute_license = mock_compute_license self.indexer = FossologyLicenseIndexer(CONFIG) self.indexer.catch_exceptions = False self.idx_storage = self.indexer.idx_storage fill_storage(self.indexer.storage) fill_obj_storage(self.indexer.objstorage) - self.id0 = '01c9379dfc33803963d07c1ccc748d3fe4c96bb5' - self.id1 = '688a5ef812c53907562fe379d4b3851e69c7cb15' - self.id2 = 'da39a3ee5e6b4b0d3255bfef95601890afd80709' # empty content + self.id0 = "01c9379dfc33803963d07c1ccc748d3fe4c96bb5" + self.id1 = "688a5ef812c53907562fe379d4b3851e69c7cb15" + self.id2 = "da39a3ee5e6b4b0d3255bfef95601890afd80709" # empty content - tool = {k.replace('tool_', ''): v - for (k, v) in self.indexer.tool.items()} + tool = {k.replace("tool_", ""): v for (k, v) in self.indexer.tool.items()} # then self.expected_results = { - self.id0: { - 'tool': tool, - 'licenses': SHA1_TO_LICENSES[self.id0], - }, - self.id1: { - 'tool': tool, - 'licenses': SHA1_TO_LICENSES[self.id1], - }, - self.id2: { - 'tool': tool, - 'licenses': SHA1_TO_LICENSES[self.id2], - } + self.id0: {"tool": tool, "licenses": SHA1_TO_LICENSES[self.id0],}, + self.id1: {"tool": tool, "licenses": SHA1_TO_LICENSES[self.id1],}, + self.id2: {"tool": tool, "licenses": SHA1_TO_LICENSES[self.id2],}, } def tearDown(self): super().tearDown() fossology_license.compute_license = self.orig_compute_license class TestFossologyLicenseRangeIndexer( - CommonContentIndexerRangeTest, unittest.TestCase): + CommonContentIndexerRangeTest, unittest.TestCase +): """Range Fossology License Indexer tests. - new data within range are indexed - no data outside a range are indexed - with filtering existing indexed data prior to compute new index - without filtering existing indexed data prior to compute new index """ + def setUp(self): super().setUp() # replace actual license computation with a mock self.orig_compute_license = fossology_license.compute_license fossology_license.compute_license = mock_compute_license self.indexer = FossologyLicenseRangeIndexer(config=RANGE_CONFIG) self.indexer.catch_exceptions = False fill_storage(self.indexer.storage) fill_obj_storage(self.indexer.objstorage) - self.id0 = '01c9379dfc33803963d07c1ccc748d3fe4c96bb5' - self.id1 = '02fb2c89e14f7fab46701478c83779c7beb7b069' - self.id2 = '103bc087db1d26afc3a0283f38663d081e9b01e6' - tool_id = self.indexer.tool['id'] + self.id0 = "01c9379dfc33803963d07c1ccc748d3fe4c96bb5" + self.id1 = "02fb2c89e14f7fab46701478c83779c7beb7b069" + self.id2 = "103bc087db1d26afc3a0283f38663d081e9b01e6" + tool_id = self.indexer.tool["id"] self.expected_results = { self.id0: { - 'id': self.id0, - 'indexer_configuration_id': tool_id, - 'licenses': SHA1_TO_LICENSES[self.id0] + "id": self.id0, + "indexer_configuration_id": tool_id, + "licenses": SHA1_TO_LICENSES[self.id0], }, self.id1: { - 'id': self.id1, - 'indexer_configuration_id': tool_id, - 'licenses': SHA1_TO_LICENSES[self.id1] + "id": self.id1, + "indexer_configuration_id": tool_id, + "licenses": SHA1_TO_LICENSES[self.id1], }, self.id2: { - 'id': self.id2, - 'indexer_configuration_id': tool_id, - 'licenses': SHA1_TO_LICENSES[self.id2] - } + "id": self.id2, + "indexer_configuration_id": tool_id, + "licenses": SHA1_TO_LICENSES[self.id2], + }, } def tearDown(self): super().tearDown() fossology_license.compute_license = self.orig_compute_license def test_fossology_w_no_tool(): with pytest.raises(ValueError): - FossologyLicenseIndexer(config=filter_dict(CONFIG, 'tools')) + FossologyLicenseIndexer(config=filter_dict(CONFIG, "tools")) def test_fossology_range_w_no_tool(): with pytest.raises(ValueError): - FossologyLicenseRangeIndexer(config=filter_dict(RANGE_CONFIG, 'tools')) + FossologyLicenseRangeIndexer(config=filter_dict(RANGE_CONFIG, "tools")) diff --git a/swh/indexer/tests/test_journal_client.py b/swh/indexer/tests/test_journal_client.py index 23f3cb5..2651ada 100644 --- a/swh/indexer/tests/test_journal_client.py +++ b/swh/indexer/tests/test_journal_client.py @@ -1,151 +1,154 @@ # Copyright (C) 2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import unittest from unittest.mock import Mock, patch from swh.indexer.journal_client import process_journal_objects class JournalClientTest(unittest.TestCase): def testOneOriginVisit(self): mock_scheduler = Mock() messages = { - 'origin_visit': [ - { - 'status': 'full', - 'origin': 'file:///dev/zero', - }, - ] + "origin_visit": [{"status": "full", "origin": "file:///dev/zero",},] } process_journal_objects( - messages, scheduler=mock_scheduler, - task_names={'origin_metadata': 'task-name'}) + messages, + scheduler=mock_scheduler, + task_names={"origin_metadata": "task-name"}, + ) self.assertTrue(mock_scheduler.create_tasks.called) call_args = mock_scheduler.create_tasks.call_args (args, kwargs) = call_args self.assertEqual(kwargs, {}) - del args[0][0]['next_run'] - self.assertEqual(args, ([ - { - 'arguments': { - 'kwargs': {'policy_update': 'update-dups'}, - 'args': (['file:///dev/zero'],) - }, - 'policy': 'oneshot', - 'type': 'task-name' - }, - ],)) + del args[0][0]["next_run"] + self.assertEqual( + args, + ( + [ + { + "arguments": { + "kwargs": {"policy_update": "update-dups"}, + "args": (["file:///dev/zero"],), + }, + "policy": "oneshot", + "type": "task-name", + }, + ], + ), + ) def testOriginVisitLegacy(self): mock_scheduler = Mock() messages = { - 'origin_visit': [ - { - 'status': 'full', - 'origin': { - 'url': 'file:///dev/zero', - } - }, + "origin_visit": [ + {"status": "full", "origin": {"url": "file:///dev/zero",}}, ] } process_journal_objects( - messages, scheduler=mock_scheduler, - task_names={'origin_metadata': 'task-name'}) + messages, + scheduler=mock_scheduler, + task_names={"origin_metadata": "task-name"}, + ) self.assertTrue(mock_scheduler.create_tasks.called) call_args = mock_scheduler.create_tasks.call_args (args, kwargs) = call_args self.assertEqual(kwargs, {}) - del args[0][0]['next_run'] - self.assertEqual(args, ([ - { - 'arguments': { - 'kwargs': {'policy_update': 'update-dups'}, - 'args': (['file:///dev/zero'],) - }, - 'policy': 'oneshot', - 'type': 'task-name' - }, - ],)) + del args[0][0]["next_run"] + self.assertEqual( + args, + ( + [ + { + "arguments": { + "kwargs": {"policy_update": "update-dups"}, + "args": (["file:///dev/zero"],), + }, + "policy": "oneshot", + "type": "task-name", + }, + ], + ), + ) def testOneOriginVisitBatch(self): mock_scheduler = Mock() messages = { - 'origin_visit': [ - { - 'status': 'full', - 'origin': 'file:///dev/zero', - }, - { - 'status': 'full', - 'origin': 'file:///tmp/foobar', - }, + "origin_visit": [ + {"status": "full", "origin": "file:///dev/zero",}, + {"status": "full", "origin": "file:///tmp/foobar",}, ] } process_journal_objects( - messages, scheduler=mock_scheduler, - task_names={'origin_metadata': 'task-name'}) + messages, + scheduler=mock_scheduler, + task_names={"origin_metadata": "task-name"}, + ) self.assertTrue(mock_scheduler.create_tasks.called) call_args = mock_scheduler.create_tasks.call_args (args, kwargs) = call_args self.assertEqual(kwargs, {}) - del args[0][0]['next_run'] - self.assertEqual(args, ([ - { - 'arguments': { - 'kwargs': {'policy_update': 'update-dups'}, - 'args': (['file:///dev/zero', 'file:///tmp/foobar'],) - }, - 'policy': 'oneshot', - 'type': 'task-name' - }, - ],)) + del args[0][0]["next_run"] + self.assertEqual( + args, + ( + [ + { + "arguments": { + "kwargs": {"policy_update": "update-dups"}, + "args": (["file:///dev/zero", "file:///tmp/foobar"],), + }, + "policy": "oneshot", + "type": "task-name", + }, + ], + ), + ) - @patch('swh.indexer.journal_client.MAX_ORIGINS_PER_TASK', 2) + @patch("swh.indexer.journal_client.MAX_ORIGINS_PER_TASK", 2) def testOriginVisitBatches(self): mock_scheduler = Mock() messages = { - 'origin_visit': [ - { - 'status': 'full', - 'origin': 'file:///dev/zero', - }, - { - 'status': 'full', - 'origin': 'file:///tmp/foobar', - }, - { - 'status': 'full', - 'origin': 'file:///tmp/spamegg', - }, + "origin_visit": [ + {"status": "full", "origin": "file:///dev/zero",}, + {"status": "full", "origin": "file:///tmp/foobar",}, + {"status": "full", "origin": "file:///tmp/spamegg",}, ] } process_journal_objects( - messages, scheduler=mock_scheduler, - task_names={'origin_metadata': 'task-name'}) + messages, + scheduler=mock_scheduler, + task_names={"origin_metadata": "task-name"}, + ) self.assertTrue(mock_scheduler.create_tasks.called) call_args = mock_scheduler.create_tasks.call_args (args, kwargs) = call_args self.assertEqual(kwargs, {}) - del args[0][0]['next_run'] - del args[0][1]['next_run'] - self.assertEqual(args, ([ - { - 'arguments': { - 'kwargs': {'policy_update': 'update-dups'}, - 'args': (['file:///dev/zero', 'file:///tmp/foobar'],) - }, - 'policy': 'oneshot', - 'type': 'task-name' - }, - { - 'arguments': { - 'kwargs': {'policy_update': 'update-dups'}, - 'args': (['file:///tmp/spamegg'],) - }, - 'policy': 'oneshot', - 'type': 'task-name' - }, - ],)) + del args[0][0]["next_run"] + del args[0][1]["next_run"] + self.assertEqual( + args, + ( + [ + { + "arguments": { + "kwargs": {"policy_update": "update-dups"}, + "args": (["file:///dev/zero", "file:///tmp/foobar"],), + }, + "policy": "oneshot", + "type": "task-name", + }, + { + "arguments": { + "kwargs": {"policy_update": "update-dups"}, + "args": (["file:///tmp/spamegg"],), + }, + "policy": "oneshot", + "type": "task-name", + }, + ], + ), + ) diff --git a/swh/indexer/tests/test_metadata.py b/swh/indexer/tests/test_metadata.py index 02981cc..c3ef250 100644 --- a/swh/indexer/tests/test_metadata.py +++ b/swh/indexer/tests/test_metadata.py @@ -1,1123 +1,1210 @@ # Copyright (C) 2017-2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import json import unittest import attr from hypothesis import given, strategies, settings, HealthCheck from swh.model.hashutil import hash_to_bytes from swh.indexer.codemeta import CODEMETA_TERMS from swh.indexer.metadata_dictionary import MAPPINGS from swh.indexer.metadata_dictionary.maven import MavenMapping from swh.indexer.metadata_dictionary.npm import NpmMapping from swh.indexer.metadata_dictionary.ruby import GemspecMapping -from swh.indexer.metadata_detector import ( - detect_metadata -) -from swh.indexer.metadata import ( - ContentMetadataIndexer, RevisionMetadataIndexer -) +from swh.indexer.metadata_detector import detect_metadata +from swh.indexer.metadata import ContentMetadataIndexer, RevisionMetadataIndexer from .utils import ( - BASE_TEST_CONFIG, fill_obj_storage, fill_storage, - YARN_PARSER_METADATA, json_document_strategy, + BASE_TEST_CONFIG, + fill_obj_storage, + fill_storage, + YARN_PARSER_METADATA, + json_document_strategy, xml_document_strategy, ) TRANSLATOR_TOOL = { - 'name': 'swh-metadata-translator', - 'version': '0.0.2', - 'configuration': { - 'type': 'local', - 'context': 'NpmMapping' - } + "name": "swh-metadata-translator", + "version": "0.0.2", + "configuration": {"type": "local", "context": "NpmMapping"}, } class ContentMetadataTestIndexer(ContentMetadataIndexer): """Specific Metadata whose configuration is enough to satisfy the indexing tests. """ + def parse_config_file(self, *args, **kwargs): - assert False, 'should not be called; the rev indexer configures it.' + assert False, "should not be called; the rev indexer configures it." REVISION_METADATA_CONFIG = { **BASE_TEST_CONFIG, - 'tools': TRANSLATOR_TOOL, + "tools": TRANSLATOR_TOOL, } class Metadata(unittest.TestCase): """ Tests metadata_mock_tool tool for Metadata detection """ + def setUp(self): """ shows the entire diff in the results """ self.maxDiff = None - self.npm_mapping = MAPPINGS['NpmMapping']() - self.codemeta_mapping = MAPPINGS['CodemetaMapping']() - self.maven_mapping = MAPPINGS['MavenMapping']() - self.pkginfo_mapping = MAPPINGS['PythonPkginfoMapping']() - self.gemspec_mapping = MAPPINGS['GemspecMapping']() + self.npm_mapping = MAPPINGS["NpmMapping"]() + self.codemeta_mapping = MAPPINGS["CodemetaMapping"]() + self.maven_mapping = MAPPINGS["MavenMapping"]() + self.pkginfo_mapping = MAPPINGS["PythonPkginfoMapping"]() + self.gemspec_mapping = MAPPINGS["GemspecMapping"]() def test_compute_metadata_none(self): """ testing content empty content is empty should return None """ # given content = b"" # None if no metadata was found or an error occurred declared_metadata = None # when result = self.npm_mapping.translate(content) # then self.assertEqual(declared_metadata, result) def test_compute_metadata_npm(self): """ testing only computation of metadata with hard_mapping_npm """ # given content = b""" { "name": "test_metadata", "version": "0.0.2", "description": "Simple package.json test for indexer", "repository": { "type": "git", "url": "https://github.com/moranegg/metadata_test" }, "author": { "email": "moranegg@example.com", "name": "Morane G" } } """ declared_metadata = { - '@context': 'https://doi.org/10.5063/schema/codemeta-2.0', - 'type': 'SoftwareSourceCode', - 'name': 'test_metadata', - 'version': '0.0.2', - 'description': 'Simple package.json test for indexer', - 'codeRepository': - 'git+https://github.com/moranegg/metadata_test', - 'author': [{ - 'type': 'Person', - 'name': 'Morane G', - 'email': 'moranegg@example.com', - }], + "@context": "https://doi.org/10.5063/schema/codemeta-2.0", + "type": "SoftwareSourceCode", + "name": "test_metadata", + "version": "0.0.2", + "description": "Simple package.json test for indexer", + "codeRepository": "git+https://github.com/moranegg/metadata_test", + "author": [ + {"type": "Person", "name": "Morane G", "email": "moranegg@example.com",} + ], } # when result = self.npm_mapping.translate(content) # then self.assertEqual(declared_metadata, result) def test_index_content_metadata_npm(self): """ testing NPM with package.json - one sha1 uses a file that can't be translated to metadata and should return None in the translated metadata """ # given sha1s = [ - hash_to_bytes('26a9f72a7c87cc9205725cfd879f514ff4f3d8d5'), - hash_to_bytes('d4c647f0fc257591cc9ba1722484229780d1c607'), - hash_to_bytes('02fb2c89e14f7fab46701478c83779c7beb7b069'), + hash_to_bytes("26a9f72a7c87cc9205725cfd879f514ff4f3d8d5"), + hash_to_bytes("d4c647f0fc257591cc9ba1722484229780d1c607"), + hash_to_bytes("02fb2c89e14f7fab46701478c83779c7beb7b069"), ] # this metadata indexer computes only metadata for package.json # in npm context with a hard mapping config = BASE_TEST_CONFIG.copy() - config['tools'] = [TRANSLATOR_TOOL] + config["tools"] = [TRANSLATOR_TOOL] metadata_indexer = ContentMetadataTestIndexer(config=config) fill_obj_storage(metadata_indexer.objstorage) fill_storage(metadata_indexer.storage) # when - metadata_indexer.run(sha1s, policy_update='ignore-dups') - results = list(metadata_indexer.idx_storage.content_metadata_get( - sha1s)) - - expected_results = [{ - 'metadata': { - '@context': 'https://doi.org/10.5063/schema/codemeta-2.0', - 'type': 'SoftwareSourceCode', - 'codeRepository': - 'git+https://github.com/moranegg/metadata_test', - 'description': 'Simple package.json test for indexer', - 'name': 'test_metadata', - 'version': '0.0.1' + metadata_indexer.run(sha1s, policy_update="ignore-dups") + results = list(metadata_indexer.idx_storage.content_metadata_get(sha1s)) + + expected_results = [ + { + "metadata": { + "@context": "https://doi.org/10.5063/schema/codemeta-2.0", + "type": "SoftwareSourceCode", + "codeRepository": "git+https://github.com/moranegg/metadata_test", + "description": "Simple package.json test for indexer", + "name": "test_metadata", + "version": "0.0.1", + }, + "id": hash_to_bytes("26a9f72a7c87cc9205725cfd879f514ff4f3d8d5"), }, - 'id': hash_to_bytes('26a9f72a7c87cc9205725cfd879f514ff4f3d8d5'), - }, { - 'metadata': { - '@context': 'https://doi.org/10.5063/schema/codemeta-2.0', - 'type': 'SoftwareSourceCode', - 'issueTracker': - 'https://github.com/npm/npm/issues', - 'author': [{ - 'type': 'Person', - 'name': 'Isaac Z. Schlueter', - 'email': 'i@izs.me', - 'url': 'http://blog.izs.me', - }], - 'codeRepository': - 'git+https://github.com/npm/npm', - 'description': 'a package manager for JavaScript', - 'license': 'https://spdx.org/licenses/Artistic-2.0', - 'version': '5.0.3', - 'name': 'npm', - 'keywords': [ - 'install', - 'modules', - 'package manager', - 'package.json' - ], - 'url': 'https://docs.npmjs.com/' + { + "metadata": { + "@context": "https://doi.org/10.5063/schema/codemeta-2.0", + "type": "SoftwareSourceCode", + "issueTracker": "https://github.com/npm/npm/issues", + "author": [ + { + "type": "Person", + "name": "Isaac Z. Schlueter", + "email": "i@izs.me", + "url": "http://blog.izs.me", + } + ], + "codeRepository": "git+https://github.com/npm/npm", + "description": "a package manager for JavaScript", + "license": "https://spdx.org/licenses/Artistic-2.0", + "version": "5.0.3", + "name": "npm", + "keywords": [ + "install", + "modules", + "package manager", + "package.json", + ], + "url": "https://docs.npmjs.com/", + }, + "id": hash_to_bytes("d4c647f0fc257591cc9ba1722484229780d1c607"), }, - 'id': hash_to_bytes('d4c647f0fc257591cc9ba1722484229780d1c607') - }] + ] for result in results: - del result['tool'] + del result["tool"] # The assertion below returns False sometimes because of nested lists self.assertEqual(expected_results, results) def test_npm_bugs_normalization(self): # valid dictionary package_json = b"""{ "name": "foo", "bugs": { "url": "https://github.com/owner/project/issues", "email": "foo@example.com" } }""" result = self.npm_mapping.translate(package_json) - self.assertEqual(result, { - '@context': 'https://doi.org/10.5063/schema/codemeta-2.0', - 'name': 'foo', - 'issueTracker': 'https://github.com/owner/project/issues', - 'type': 'SoftwareSourceCode', - }) + self.assertEqual( + result, + { + "@context": "https://doi.org/10.5063/schema/codemeta-2.0", + "name": "foo", + "issueTracker": "https://github.com/owner/project/issues", + "type": "SoftwareSourceCode", + }, + ) # "invalid" dictionary package_json = b"""{ "name": "foo", "bugs": { "email": "foo@example.com" } }""" result = self.npm_mapping.translate(package_json) - self.assertEqual(result, { - '@context': 'https://doi.org/10.5063/schema/codemeta-2.0', - 'name': 'foo', - 'type': 'SoftwareSourceCode', - }) + self.assertEqual( + result, + { + "@context": "https://doi.org/10.5063/schema/codemeta-2.0", + "name": "foo", + "type": "SoftwareSourceCode", + }, + ) # string package_json = b"""{ "name": "foo", "bugs": "https://github.com/owner/project/issues" }""" result = self.npm_mapping.translate(package_json) - self.assertEqual(result, { - '@context': 'https://doi.org/10.5063/schema/codemeta-2.0', - 'name': 'foo', - 'issueTracker': 'https://github.com/owner/project/issues', - 'type': 'SoftwareSourceCode', - }) + self.assertEqual( + result, + { + "@context": "https://doi.org/10.5063/schema/codemeta-2.0", + "name": "foo", + "issueTracker": "https://github.com/owner/project/issues", + "type": "SoftwareSourceCode", + }, + ) def test_npm_repository_normalization(self): # normal package_json = b"""{ "name": "foo", "repository": { "type" : "git", "url" : "https://github.com/npm/cli.git" } }""" result = self.npm_mapping.translate(package_json) - self.assertEqual(result, { - '@context': 'https://doi.org/10.5063/schema/codemeta-2.0', - 'name': 'foo', - 'codeRepository': 'git+https://github.com/npm/cli.git', - 'type': 'SoftwareSourceCode', - }) + self.assertEqual( + result, + { + "@context": "https://doi.org/10.5063/schema/codemeta-2.0", + "name": "foo", + "codeRepository": "git+https://github.com/npm/cli.git", + "type": "SoftwareSourceCode", + }, + ) # missing url package_json = b"""{ "name": "foo", "repository": { "type" : "git" } }""" result = self.npm_mapping.translate(package_json) - self.assertEqual(result, { - '@context': 'https://doi.org/10.5063/schema/codemeta-2.0', - 'name': 'foo', - 'type': 'SoftwareSourceCode', - }) + self.assertEqual( + result, + { + "@context": "https://doi.org/10.5063/schema/codemeta-2.0", + "name": "foo", + "type": "SoftwareSourceCode", + }, + ) # github shortcut package_json = b"""{ "name": "foo", "repository": "github:npm/cli" }""" result = self.npm_mapping.translate(package_json) expected_result = { - '@context': 'https://doi.org/10.5063/schema/codemeta-2.0', - 'name': 'foo', - 'codeRepository': 'git+https://github.com/npm/cli.git', - 'type': 'SoftwareSourceCode', + "@context": "https://doi.org/10.5063/schema/codemeta-2.0", + "name": "foo", + "codeRepository": "git+https://github.com/npm/cli.git", + "type": "SoftwareSourceCode", } self.assertEqual(result, expected_result) # github shortshortcut package_json = b"""{ "name": "foo", "repository": "npm/cli" }""" result = self.npm_mapping.translate(package_json) self.assertEqual(result, expected_result) # gitlab shortcut package_json = b"""{ "name": "foo", "repository": "gitlab:user/repo" }""" result = self.npm_mapping.translate(package_json) - self.assertEqual(result, { - '@context': 'https://doi.org/10.5063/schema/codemeta-2.0', - 'name': 'foo', - 'codeRepository': 'git+https://gitlab.com/user/repo.git', - 'type': 'SoftwareSourceCode', - }) + self.assertEqual( + result, + { + "@context": "https://doi.org/10.5063/schema/codemeta-2.0", + "name": "foo", + "codeRepository": "git+https://gitlab.com/user/repo.git", + "type": "SoftwareSourceCode", + }, + ) def test_detect_metadata_package_json(self): # given - df = [{ - 'sha1_git': b'abc', - 'name': b'index.js', - 'target': b'abc', - 'length': 897, - 'status': 'visible', - 'type': 'file', - 'perms': 33188, - 'dir_id': b'dir_a', - 'sha1': b'bcd' + df = [ + { + "sha1_git": b"abc", + "name": b"index.js", + "target": b"abc", + "length": 897, + "status": "visible", + "type": "file", + "perms": 33188, + "dir_id": b"dir_a", + "sha1": b"bcd", }, { - 'sha1_git': b'aab', - 'name': b'package.json', - 'target': b'aab', - 'length': 712, - 'status': 'visible', - 'type': 'file', - 'perms': 33188, - 'dir_id': b'dir_a', - 'sha1': b'cde' - }] + "sha1_git": b"aab", + "name": b"package.json", + "target": b"aab", + "length": 712, + "status": "visible", + "type": "file", + "perms": 33188, + "dir_id": b"dir_a", + "sha1": b"cde", + }, + ] # when results = detect_metadata(df) - expected_results = { - 'NpmMapping': [ - b'cde' - ] - } + expected_results = {"NpmMapping": [b"cde"]} # then self.assertEqual(expected_results, results) def test_compute_metadata_valid_codemeta(self): - raw_content = ( - b"""{ + raw_content = b"""{ "@context": "https://doi.org/10.5063/schema/codemeta-2.0", "@type": "SoftwareSourceCode", "identifier": "CodeMeta", "description": "CodeMeta is a concept vocabulary that can be used to standardize the exchange of software metadata across repositories and organizations.", "name": "CodeMeta: Minimal metadata schemas for science software and code, in JSON-LD", "codeRepository": "https://github.com/codemeta/codemeta", "issueTracker": "https://github.com/codemeta/codemeta/issues", "license": "https://spdx.org/licenses/Apache-2.0", "version": "2.0", "author": [ { "@type": "Person", "givenName": "Carl", "familyName": "Boettiger", "email": "cboettig@gmail.com", "@id": "http://orcid.org/0000-0002-1642-628X" }, { "@type": "Person", "givenName": "Matthew B.", "familyName": "Jones", "email": "jones@nceas.ucsb.edu", "@id": "http://orcid.org/0000-0003-0077-4738" } ], "maintainer": { "@type": "Person", "givenName": "Carl", "familyName": "Boettiger", "email": "cboettig@gmail.com", "@id": "http://orcid.org/0000-0002-1642-628X" }, "contIntegration": "https://travis-ci.org/codemeta/codemeta", "developmentStatus": "active", "downloadUrl": "https://github.com/codemeta/codemeta/archive/2.0.zip", "funder": { "@id": "https://doi.org/10.13039/100000001", "@type": "Organization", "name": "National Science Foundation" }, "funding":"1549758; Codemeta: A Rosetta Stone for Metadata in Scientific Software", "keywords": [ "metadata", "software" ], "version":"2.0", "dateCreated":"2017-06-05", "datePublished":"2017-06-05", "programmingLanguage": "JSON-LD" - }""") # noqa + }""" # noqa expected_result = { "@context": "https://doi.org/10.5063/schema/codemeta-2.0", "type": "SoftwareSourceCode", "identifier": "CodeMeta", - "description": - "CodeMeta is a concept vocabulary that can " - "be used to standardize the exchange of software metadata " - "across repositories and organizations.", - "name": - "CodeMeta: Minimal metadata schemas for science " - "software and code, in JSON-LD", + "description": "CodeMeta is a concept vocabulary that can " + "be used to standardize the exchange of software metadata " + "across repositories and organizations.", + "name": "CodeMeta: Minimal metadata schemas for science " + "software and code, in JSON-LD", "codeRepository": "https://github.com/codemeta/codemeta", "issueTracker": "https://github.com/codemeta/codemeta/issues", "license": "https://spdx.org/licenses/Apache-2.0", "version": "2.0", "author": [ - { + { + "type": "Person", + "givenName": "Carl", + "familyName": "Boettiger", + "email": "cboettig@gmail.com", + "id": "http://orcid.org/0000-0002-1642-628X", + }, + { + "type": "Person", + "givenName": "Matthew B.", + "familyName": "Jones", + "email": "jones@nceas.ucsb.edu", + "id": "http://orcid.org/0000-0003-0077-4738", + }, + ], + "maintainer": { "type": "Person", "givenName": "Carl", "familyName": "Boettiger", "email": "cboettig@gmail.com", - "id": "http://orcid.org/0000-0002-1642-628X" - }, - { - "type": "Person", - "givenName": "Matthew B.", - "familyName": "Jones", - "email": "jones@nceas.ucsb.edu", - "id": "http://orcid.org/0000-0003-0077-4738" - } - ], - "maintainer": { - "type": "Person", - "givenName": "Carl", - "familyName": "Boettiger", - "email": "cboettig@gmail.com", - "id": "http://orcid.org/0000-0002-1642-628X" + "id": "http://orcid.org/0000-0002-1642-628X", }, "contIntegration": "https://travis-ci.org/codemeta/codemeta", "developmentStatus": "active", - "downloadUrl": - "https://github.com/codemeta/codemeta/archive/2.0.zip", + "downloadUrl": "https://github.com/codemeta/codemeta/archive/2.0.zip", "funder": { "id": "https://doi.org/10.13039/100000001", "type": "Organization", - "name": "National Science Foundation" + "name": "National Science Foundation", }, "funding": "1549758; Codemeta: A Rosetta Stone for Metadata " - "in Scientific Software", - "keywords": [ - "metadata", - "software" - ], + "in Scientific Software", + "keywords": ["metadata", "software"], "version": "2.0", "dateCreated": "2017-06-05", "datePublished": "2017-06-05", - "programmingLanguage": "JSON-LD" - } + "programmingLanguage": "JSON-LD", + } result = self.codemeta_mapping.translate(raw_content) self.assertEqual(result, expected_result) def test_compute_metadata_codemeta_alternate_context(self): - raw_content = ( - b"""{ + raw_content = b"""{ "@context": "https://raw.githubusercontent.com/codemeta/codemeta/master/codemeta.jsonld", "@type": "SoftwareSourceCode", "identifier": "CodeMeta" - }""") # noqa + }""" # noqa expected_result = { "@context": "https://doi.org/10.5063/schema/codemeta-2.0", "type": "SoftwareSourceCode", "identifier": "CodeMeta", } result = self.codemeta_mapping.translate(raw_content) self.assertEqual(result, expected_result) def test_compute_metadata_maven(self): raw_content = b""" Maven Default Project 4.0.0 com.mycompany.app my-app 1.2.3 central Maven Repository Switchboard default http://repo1.maven.org/maven2 false Apache License, Version 2.0 https://www.apache.org/licenses/LICENSE-2.0.txt repo A business-friendly OSS license """ result = self.maven_mapping.translate(raw_content) - self.assertEqual(result, { - '@context': 'https://doi.org/10.5063/schema/codemeta-2.0', - 'type': 'SoftwareSourceCode', - 'name': 'Maven Default Project', - 'identifier': 'com.mycompany.app', - 'version': '1.2.3', - 'license': 'https://www.apache.org/licenses/LICENSE-2.0.txt', - 'codeRepository': - 'http://repo1.maven.org/maven2/com/mycompany/app/my-app', - }) + self.assertEqual( + result, + { + "@context": "https://doi.org/10.5063/schema/codemeta-2.0", + "type": "SoftwareSourceCode", + "name": "Maven Default Project", + "identifier": "com.mycompany.app", + "version": "1.2.3", + "license": "https://www.apache.org/licenses/LICENSE-2.0.txt", + "codeRepository": ( + "http://repo1.maven.org/maven2/com/mycompany/app/my-app" + ), + }, + ) def test_compute_metadata_maven_empty(self): raw_content = b""" """ result = self.maven_mapping.translate(raw_content) - self.assertEqual(result, { - '@context': 'https://doi.org/10.5063/schema/codemeta-2.0', - 'type': 'SoftwareSourceCode', - }) + self.assertEqual( + result, + { + "@context": "https://doi.org/10.5063/schema/codemeta-2.0", + "type": "SoftwareSourceCode", + }, + ) def test_compute_metadata_maven_almost_empty(self): raw_content = b""" """ result = self.maven_mapping.translate(raw_content) - self.assertEqual(result, { - '@context': 'https://doi.org/10.5063/schema/codemeta-2.0', - 'type': 'SoftwareSourceCode', - }) + self.assertEqual( + result, + { + "@context": "https://doi.org/10.5063/schema/codemeta-2.0", + "type": "SoftwareSourceCode", + }, + ) def test_compute_metadata_maven_invalid_xml(self): expected_warning = ( - 'WARNING:swh.indexer.metadata_dictionary.maven.MavenMapping:' - 'Error parsing XML from foo') + "WARNING:swh.indexer.metadata_dictionary.maven.MavenMapping:" + "Error parsing XML from foo" + ) raw_content = b""" """ - with self.assertLogs('swh.indexer.metadata_dictionary', - level='WARNING') as cm: - result = MAPPINGS["MavenMapping"]('foo').translate(raw_content) + with self.assertLogs("swh.indexer.metadata_dictionary", level="WARNING") as cm: + result = MAPPINGS["MavenMapping"]("foo").translate(raw_content) self.assertEqual(cm.output, [expected_warning]) self.assertEqual(result, None) raw_content = b""" """ - with self.assertLogs('swh.indexer.metadata_dictionary', - level='WARNING') as cm: - result = MAPPINGS["MavenMapping"]('foo').translate(raw_content) + with self.assertLogs("swh.indexer.metadata_dictionary", level="WARNING") as cm: + result = MAPPINGS["MavenMapping"]("foo").translate(raw_content) self.assertEqual(cm.output, [expected_warning]) self.assertEqual(result, None) def test_compute_metadata_maven_unknown_encoding(self): expected_warning = ( - 'WARNING:swh.indexer.metadata_dictionary.maven.MavenMapping:' - 'Error detecting XML encoding from foo') + "WARNING:swh.indexer.metadata_dictionary.maven.MavenMapping:" + "Error detecting XML encoding from foo" + ) raw_content = b""" """ - with self.assertLogs('swh.indexer.metadata_dictionary', - level='WARNING') as cm: - result = MAPPINGS["MavenMapping"]('foo').translate(raw_content) + with self.assertLogs("swh.indexer.metadata_dictionary", level="WARNING") as cm: + result = MAPPINGS["MavenMapping"]("foo").translate(raw_content) self.assertEqual(cm.output, [expected_warning]) self.assertEqual(result, None) raw_content = b""" """ - with self.assertLogs('swh.indexer.metadata_dictionary', - level='WARNING') as cm: - result = MAPPINGS["MavenMapping"]('foo').translate(raw_content) + with self.assertLogs("swh.indexer.metadata_dictionary", level="WARNING") as cm: + result = MAPPINGS["MavenMapping"]("foo").translate(raw_content) self.assertEqual(cm.output, [expected_warning]) self.assertEqual(result, None) def test_compute_metadata_maven_invalid_encoding(self): expected_warning = ( - 'WARNING:swh.indexer.metadata_dictionary.maven.MavenMapping:' - 'Error unidecoding XML from foo') + "WARNING:swh.indexer.metadata_dictionary.maven.MavenMapping:" + "Error unidecoding XML from foo" + ) raw_content = b""" """ - with self.assertLogs('swh.indexer.metadata_dictionary', - level='WARNING') as cm: - result = MAPPINGS["MavenMapping"]('foo').translate(raw_content) + with self.assertLogs("swh.indexer.metadata_dictionary", level="WARNING") as cm: + result = MAPPINGS["MavenMapping"]("foo").translate(raw_content) self.assertEqual(cm.output, [expected_warning]) self.assertEqual(result, None) def test_compute_metadata_maven_minimal(self): raw_content = b""" Maven Default Project 4.0.0 com.mycompany.app my-app 1.2.3 """ result = self.maven_mapping.translate(raw_content) - self.assertEqual(result, { - '@context': 'https://doi.org/10.5063/schema/codemeta-2.0', - 'type': 'SoftwareSourceCode', - 'name': 'Maven Default Project', - 'identifier': 'com.mycompany.app', - 'version': '1.2.3', - 'codeRepository': - 'https://repo.maven.apache.org/maven2/com/mycompany/app/my-app', - }) + self.assertEqual( + result, + { + "@context": "https://doi.org/10.5063/schema/codemeta-2.0", + "type": "SoftwareSourceCode", + "name": "Maven Default Project", + "identifier": "com.mycompany.app", + "version": "1.2.3", + "codeRepository": ( + "https://repo.maven.apache.org/maven2/com/mycompany/app/my-app" + ), + }, + ) def test_compute_metadata_maven_empty_nodes(self): raw_content = b""" Maven Default Project 4.0.0 com.mycompany.app my-app 1.2.3 """ result = self.maven_mapping.translate(raw_content) - self.assertEqual(result, { - '@context': 'https://doi.org/10.5063/schema/codemeta-2.0', - 'type': 'SoftwareSourceCode', - 'name': 'Maven Default Project', - 'identifier': 'com.mycompany.app', - 'version': '1.2.3', - 'codeRepository': - 'https://repo.maven.apache.org/maven2/com/mycompany/app/my-app', - }) + self.assertEqual( + result, + { + "@context": "https://doi.org/10.5063/schema/codemeta-2.0", + "type": "SoftwareSourceCode", + "name": "Maven Default Project", + "identifier": "com.mycompany.app", + "version": "1.2.3", + "codeRepository": ( + "https://repo.maven.apache.org/maven2/com/mycompany/app/my-app" + ), + }, + ) raw_content = b""" Maven Default Project 4.0.0 com.mycompany.app my-app """ result = self.maven_mapping.translate(raw_content) - self.assertEqual(result, { - '@context': 'https://doi.org/10.5063/schema/codemeta-2.0', - 'type': 'SoftwareSourceCode', - 'name': 'Maven Default Project', - 'identifier': 'com.mycompany.app', - 'codeRepository': - 'https://repo.maven.apache.org/maven2/com/mycompany/app/my-app', - }) + self.assertEqual( + result, + { + "@context": "https://doi.org/10.5063/schema/codemeta-2.0", + "type": "SoftwareSourceCode", + "name": "Maven Default Project", + "identifier": "com.mycompany.app", + "codeRepository": ( + "https://repo.maven.apache.org/maven2/com/mycompany/app/my-app" + ), + }, + ) raw_content = b""" 4.0.0 com.mycompany.app my-app 1.2.3 """ result = self.maven_mapping.translate(raw_content) - self.assertEqual(result, { - '@context': 'https://doi.org/10.5063/schema/codemeta-2.0', - 'type': 'SoftwareSourceCode', - 'identifier': 'com.mycompany.app', - 'version': '1.2.3', - 'codeRepository': - 'https://repo.maven.apache.org/maven2/com/mycompany/app/my-app', - }) + self.assertEqual( + result, + { + "@context": "https://doi.org/10.5063/schema/codemeta-2.0", + "type": "SoftwareSourceCode", + "identifier": "com.mycompany.app", + "version": "1.2.3", + "codeRepository": ( + "https://repo.maven.apache.org/maven2/com/mycompany/app/my-app" + ), + }, + ) raw_content = b""" Maven Default Project 4.0.0 com.mycompany.app my-app 1.2.3 """ result = self.maven_mapping.translate(raw_content) - self.assertEqual(result, { - '@context': 'https://doi.org/10.5063/schema/codemeta-2.0', - 'type': 'SoftwareSourceCode', - 'name': 'Maven Default Project', - 'identifier': 'com.mycompany.app', - 'version': '1.2.3', - 'codeRepository': - 'https://repo.maven.apache.org/maven2/com/mycompany/app/my-app', - }) + self.assertEqual( + result, + { + "@context": "https://doi.org/10.5063/schema/codemeta-2.0", + "type": "SoftwareSourceCode", + "name": "Maven Default Project", + "identifier": "com.mycompany.app", + "version": "1.2.3", + "codeRepository": ( + "https://repo.maven.apache.org/maven2/com/mycompany/app/my-app" + ), + }, + ) raw_content = b""" 1.2.3 """ result = self.maven_mapping.translate(raw_content) - self.assertEqual(result, { - '@context': 'https://doi.org/10.5063/schema/codemeta-2.0', - 'type': 'SoftwareSourceCode', - 'version': '1.2.3', - }) + self.assertEqual( + result, + { + "@context": "https://doi.org/10.5063/schema/codemeta-2.0", + "type": "SoftwareSourceCode", + "version": "1.2.3", + }, + ) def test_compute_metadata_maven_invalid_licenses(self): raw_content = b""" Maven Default Project 4.0.0 com.mycompany.app my-app 1.2.3 foo """ result = self.maven_mapping.translate(raw_content) - self.assertEqual(result, { - '@context': 'https://doi.org/10.5063/schema/codemeta-2.0', - 'type': 'SoftwareSourceCode', - 'name': 'Maven Default Project', - 'identifier': 'com.mycompany.app', - 'version': '1.2.3', - 'codeRepository': - 'https://repo.maven.apache.org/maven2/com/mycompany/app/my-app', - }) + self.assertEqual( + result, + { + "@context": "https://doi.org/10.5063/schema/codemeta-2.0", + "type": "SoftwareSourceCode", + "name": "Maven Default Project", + "identifier": "com.mycompany.app", + "version": "1.2.3", + "codeRepository": ( + "https://repo.maven.apache.org/maven2/com/mycompany/app/my-app" + ), + }, + ) def test_compute_metadata_maven_multiple(self): - '''Tests when there are multiple code repos and licenses.''' + """Tests when there are multiple code repos and licenses.""" raw_content = b""" Maven Default Project 4.0.0 com.mycompany.app my-app 1.2.3 central Maven Repository Switchboard default http://repo1.maven.org/maven2 false example Example Maven Repo default http://example.org/maven2 Apache License, Version 2.0 https://www.apache.org/licenses/LICENSE-2.0.txt repo A business-friendly OSS license MIT license https://opensource.org/licenses/MIT """ result = self.maven_mapping.translate(raw_content) - self.assertEqual(result, { - '@context': 'https://doi.org/10.5063/schema/codemeta-2.0', - 'type': 'SoftwareSourceCode', - 'name': 'Maven Default Project', - 'identifier': 'com.mycompany.app', - 'version': '1.2.3', - 'license': [ - 'https://www.apache.org/licenses/LICENSE-2.0.txt', - 'https://opensource.org/licenses/MIT', - ], - 'codeRepository': [ - 'http://repo1.maven.org/maven2/com/mycompany/app/my-app', - 'http://example.org/maven2/com/mycompany/app/my-app', - ] - }) + self.assertEqual( + result, + { + "@context": "https://doi.org/10.5063/schema/codemeta-2.0", + "type": "SoftwareSourceCode", + "name": "Maven Default Project", + "identifier": "com.mycompany.app", + "version": "1.2.3", + "license": [ + "https://www.apache.org/licenses/LICENSE-2.0.txt", + "https://opensource.org/licenses/MIT", + ], + "codeRepository": [ + "http://repo1.maven.org/maven2/com/mycompany/app/my-app", + "http://example.org/maven2/com/mycompany/app/my-app", + ], + }, + ) def test_compute_metadata_pkginfo(self): - raw_content = (b"""\ + raw_content = b"""\ Metadata-Version: 2.1 Name: swh.core Version: 0.0.49 Summary: Software Heritage core utilities Home-page: https://forge.softwareheritage.org/diffusion/DCORE/ Author: Software Heritage developers Author-email: swh-devel@inria.fr License: UNKNOWN Project-URL: Bug Reports, https://forge.softwareheritage.org/maniphest Project-URL: Funding, https://www.softwareheritage.org/donate Project-URL: Source, https://forge.softwareheritage.org/source/swh-core Description: swh-core ======== \x20 core library for swh's modules: - config parser - hash computations - serialization - logging mechanism \x20 Platform: UNKNOWN Classifier: Programming Language :: Python :: 3 Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3) Classifier: Operating System :: OS Independent Classifier: Development Status :: 5 - Production/Stable Description-Content-Type: text/markdown Provides-Extra: testing -""") # noqa +""" # noqa result = self.pkginfo_mapping.translate(raw_content) - self.assertCountEqual(result['description'], [ - 'Software Heritage core utilities', # note the comma here - 'swh-core\n' - '========\n' - '\n' - "core library for swh's modules:\n" - '- config parser\n' - '- hash computations\n' - '- serialization\n' - '- logging mechanism\n' - ''], - result) - del result['description'] - self.assertEqual(result, { - '@context': 'https://doi.org/10.5063/schema/codemeta-2.0', - 'type': 'SoftwareSourceCode', - 'url': 'https://forge.softwareheritage.org/diffusion/DCORE/', - 'name': 'swh.core', - 'author': [{ - 'type': 'Person', - 'name': 'Software Heritage developers', - 'email': 'swh-devel@inria.fr', - }], - 'version': '0.0.49', - }) + self.assertCountEqual( + result["description"], + [ + "Software Heritage core utilities", # note the comma here + "swh-core\n" + "========\n" + "\n" + "core library for swh's modules:\n" + "- config parser\n" + "- hash computations\n" + "- serialization\n" + "- logging mechanism\n" + "", + ], + result, + ) + del result["description"] + self.assertEqual( + result, + { + "@context": "https://doi.org/10.5063/schema/codemeta-2.0", + "type": "SoftwareSourceCode", + "url": "https://forge.softwareheritage.org/diffusion/DCORE/", + "name": "swh.core", + "author": [ + { + "type": "Person", + "name": "Software Heritage developers", + "email": "swh-devel@inria.fr", + } + ], + "version": "0.0.49", + }, + ) def test_compute_metadata_pkginfo_utf8(self): - raw_content = (b'''\ + raw_content = b"""\ Metadata-Version: 1.1 Name: snowpyt Description-Content-Type: UNKNOWN Description: foo Hydrology N\xc2\xb083 -''') # noqa +""" # noqa result = self.pkginfo_mapping.translate(raw_content) - self.assertEqual(result, { - '@context': 'https://doi.org/10.5063/schema/codemeta-2.0', - 'type': 'SoftwareSourceCode', - 'name': 'snowpyt', - 'description': 'foo\nHydrology N°83', - }) + self.assertEqual( + result, + { + "@context": "https://doi.org/10.5063/schema/codemeta-2.0", + "type": "SoftwareSourceCode", + "name": "snowpyt", + "description": "foo\nHydrology N°83", + }, + ) def test_compute_metadata_pkginfo_keywords(self): - raw_content = (b"""\ + raw_content = b"""\ Metadata-Version: 2.1 Name: foo Keywords: foo bar baz -""") # noqa +""" # noqa result = self.pkginfo_mapping.translate(raw_content) - self.assertEqual(result, { - '@context': 'https://doi.org/10.5063/schema/codemeta-2.0', - 'type': 'SoftwareSourceCode', - 'name': 'foo', - 'keywords': ['foo', 'bar', 'baz'], - }) + self.assertEqual( + result, + { + "@context": "https://doi.org/10.5063/schema/codemeta-2.0", + "type": "SoftwareSourceCode", + "name": "foo", + "keywords": ["foo", "bar", "baz"], + }, + ) def test_compute_metadata_pkginfo_license(self): - raw_content = (b"""\ + raw_content = b"""\ Metadata-Version: 2.1 Name: foo License: MIT -""") # noqa +""" # noqa result = self.pkginfo_mapping.translate(raw_content) - self.assertEqual(result, { - '@context': 'https://doi.org/10.5063/schema/codemeta-2.0', - 'type': 'SoftwareSourceCode', - 'name': 'foo', - 'license': 'MIT', - }) + self.assertEqual( + result, + { + "@context": "https://doi.org/10.5063/schema/codemeta-2.0", + "type": "SoftwareSourceCode", + "name": "foo", + "license": "MIT", + }, + ) def test_gemspec_base(self): raw_content = b""" Gem::Specification.new do |s| s.name = 'example' s.version = '0.1.0' s.licenses = ['MIT'] s.summary = "This is an example!" s.description = "Much longer explanation of the example!" s.authors = ["Ruby Coder"] s.email = 'rubycoder@example.com' s.files = ["lib/example.rb"] s.homepage = 'https://rubygems.org/gems/example' s.metadata = { "source_code_uri" => "https://github.com/example/example" } end""" result = self.gemspec_mapping.translate(raw_content) - self.assertCountEqual(result.pop('description'), [ - "This is an example!", - "Much longer explanation of the example!" - ]) - self.assertEqual(result, { - '@context': 'https://doi.org/10.5063/schema/codemeta-2.0', - 'type': 'SoftwareSourceCode', - 'author': [ - { - 'type': 'Person', - 'name': 'Ruby Coder' - } - ], - 'name': 'example', - 'license': 'https://spdx.org/licenses/MIT', - 'codeRepository': 'https://rubygems.org/gems/example', - 'email': 'rubycoder@example.com', - 'version': '0.1.0', - }) + self.assertCountEqual( + result.pop("description"), + ["This is an example!", "Much longer explanation of the example!"], + ) + self.assertEqual( + result, + { + "@context": "https://doi.org/10.5063/schema/codemeta-2.0", + "type": "SoftwareSourceCode", + "author": [{"type": "Person", "name": "Ruby Coder"}], + "name": "example", + "license": "https://spdx.org/licenses/MIT", + "codeRepository": "https://rubygems.org/gems/example", + "email": "rubycoder@example.com", + "version": "0.1.0", + }, + ) def test_gemspec_two_author_fields(self): raw_content = b""" Gem::Specification.new do |s| s.authors = ["Ruby Coder1"] s.author = "Ruby Coder2" end""" result = self.gemspec_mapping.translate(raw_content) - self.assertCountEqual(result.pop('author'), [ - { - 'type': 'Person', - 'name': 'Ruby Coder1' - }, + self.assertCountEqual( + result.pop("author"), + [ + {"type": "Person", "name": "Ruby Coder1"}, + {"type": "Person", "name": "Ruby Coder2"}, + ], + ) + self.assertEqual( + result, { - 'type': 'Person', - 'name': 'Ruby Coder2' + "@context": "https://doi.org/10.5063/schema/codemeta-2.0", + "type": "SoftwareSourceCode", }, - ]) - self.assertEqual(result, { - '@context': 'https://doi.org/10.5063/schema/codemeta-2.0', - 'type': 'SoftwareSourceCode', - }) + ) def test_gemspec_invalid_author(self): raw_content = b""" Gem::Specification.new do |s| s.author = ["Ruby Coder"] end""" result = self.gemspec_mapping.translate(raw_content) - self.assertEqual(result, { - '@context': 'https://doi.org/10.5063/schema/codemeta-2.0', - 'type': 'SoftwareSourceCode', - }) + self.assertEqual( + result, + { + "@context": "https://doi.org/10.5063/schema/codemeta-2.0", + "type": "SoftwareSourceCode", + }, + ) raw_content = b""" Gem::Specification.new do |s| s.author = "Ruby Coder1", end""" result = self.gemspec_mapping.translate(raw_content) - self.assertEqual(result, { - '@context': 'https://doi.org/10.5063/schema/codemeta-2.0', - 'type': 'SoftwareSourceCode', - }) + self.assertEqual( + result, + { + "@context": "https://doi.org/10.5063/schema/codemeta-2.0", + "type": "SoftwareSourceCode", + }, + ) raw_content = b""" Gem::Specification.new do |s| s.authors = ["Ruby Coder1", ["Ruby Coder2"]] end""" result = self.gemspec_mapping.translate(raw_content) - self.assertEqual(result, { - '@context': 'https://doi.org/10.5063/schema/codemeta-2.0', - 'type': 'SoftwareSourceCode', - 'author': [ - { - 'type': 'Person', - 'name': 'Ruby Coder1' - } - ], - }) + self.assertEqual( + result, + { + "@context": "https://doi.org/10.5063/schema/codemeta-2.0", + "type": "SoftwareSourceCode", + "author": [{"type": "Person", "name": "Ruby Coder1"}], + }, + ) def test_gemspec_alternative_header(self): raw_content = b""" require './lib/version' Gem::Specification.new { |s| s.name = 'rb-system-with-aliases' s.summary = 'execute system commands with aliases' } """ result = self.gemspec_mapping.translate(raw_content) - self.assertEqual(result, { - '@context': 'https://doi.org/10.5063/schema/codemeta-2.0', - 'type': 'SoftwareSourceCode', - 'name': 'rb-system-with-aliases', - 'description': 'execute system commands with aliases', - }) + self.assertEqual( + result, + { + "@context": "https://doi.org/10.5063/schema/codemeta-2.0", + "type": "SoftwareSourceCode", + "name": "rb-system-with-aliases", + "description": "execute system commands with aliases", + }, + ) @settings(suppress_health_check=[HealthCheck.too_slow]) @given(json_document_strategy(keys=list(NpmMapping.mapping))) def test_npm_adversarial(self, doc): raw = json.dumps(doc).encode() self.npm_mapping.translate(raw) @settings(suppress_health_check=[HealthCheck.too_slow]) @given(json_document_strategy(keys=CODEMETA_TERMS)) def test_codemeta_adversarial(self, doc): raw = json.dumps(doc).encode() self.codemeta_mapping.translate(raw) @settings(suppress_health_check=[HealthCheck.too_slow]) - @given(xml_document_strategy( - keys=list(MavenMapping.mapping), - root='project', - xmlns='http://maven.apache.org/POM/4.0.0')) + @given( + xml_document_strategy( + keys=list(MavenMapping.mapping), + root="project", + xmlns="http://maven.apache.org/POM/4.0.0", + ) + ) def test_maven_adversarial(self, doc): self.maven_mapping.translate(doc) @settings(suppress_health_check=[HealthCheck.too_slow]) - @given(strategies.dictionaries( - # keys - strategies.one_of( - strategies.text(), - *map(strategies.just, GemspecMapping.mapping) - ), - # values - strategies.recursive( - strategies.characters(), - lambda children: strategies.lists(children, min_size=1) + @given( + strategies.dictionaries( + # keys + strategies.one_of( + strategies.text(), *map(strategies.just, GemspecMapping.mapping) + ), + # values + strategies.recursive( + strategies.characters(), + lambda children: strategies.lists(children, min_size=1), + ), ) - )) + ) def test_gemspec_adversarial(self, doc): - parts = [b'Gem::Specification.new do |s|\n'] + parts = [b"Gem::Specification.new do |s|\n"] for (k, v) in doc.items(): - parts.append(' s.{} = {}\n'.format(k, repr(v)).encode()) - parts.append(b'end\n') - self.gemspec_mapping.translate(b''.join(parts)) + parts.append(" s.{} = {}\n".format(k, repr(v)).encode()) + parts.append(b"end\n") + self.gemspec_mapping.translate(b"".join(parts)) def test_revision_metadata_indexer(self): - metadata_indexer = RevisionMetadataIndexer( - config=REVISION_METADATA_CONFIG) + metadata_indexer = RevisionMetadataIndexer(config=REVISION_METADATA_CONFIG) fill_obj_storage(metadata_indexer.objstorage) fill_storage(metadata_indexer.storage) tool = metadata_indexer.idx_storage.indexer_configuration_get( - {'tool_'+k: v for (k, v) in TRANSLATOR_TOOL.items()}) + {"tool_" + k: v for (k, v) in TRANSLATOR_TOOL.items()} + ) assert tool is not None - metadata_indexer.idx_storage.content_metadata_add([{ - 'indexer_configuration_id': tool['id'], - 'id': b'cde', - 'metadata': YARN_PARSER_METADATA, - }]) + metadata_indexer.idx_storage.content_metadata_add( + [ + { + "indexer_configuration_id": tool["id"], + "id": b"cde", + "metadata": YARN_PARSER_METADATA, + } + ] + ) sha1_gits = [ - hash_to_bytes('8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f'), + hash_to_bytes("8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f"), ] - metadata_indexer.run(sha1_gits, 'update-dups') + metadata_indexer.run(sha1_gits, "update-dups") results = list( - metadata_indexer.idx_storage. - revision_intrinsic_metadata_get(sha1_gits)) + metadata_indexer.idx_storage.revision_intrinsic_metadata_get(sha1_gits) + ) - expected_results = [{ - 'id': hash_to_bytes('8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f'), - 'tool': TRANSLATOR_TOOL, - 'metadata': YARN_PARSER_METADATA, - 'mappings': ['npm'], - }] + expected_results = [ + { + "id": hash_to_bytes("8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f"), + "tool": TRANSLATOR_TOOL, + "metadata": YARN_PARSER_METADATA, + "mappings": ["npm"], + } + ] for result in results: - del result['tool']['id'] + del result["tool"]["id"] # then self.assertEqual(expected_results, results) def test_revision_metadata_indexer_single_root_dir(self): - metadata_indexer = RevisionMetadataIndexer( - config=REVISION_METADATA_CONFIG) + metadata_indexer = RevisionMetadataIndexer(config=REVISION_METADATA_CONFIG) fill_obj_storage(metadata_indexer.objstorage) fill_storage(metadata_indexer.storage) # Add a parent directory, that is the only directory at the root # of the revision - rev_id = hash_to_bytes('8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f') + rev_id = hash_to_bytes("8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f") rev = metadata_indexer.storage._revisions[rev_id] subdir_id = rev.directory - rev = attr.evolve(rev, directory=b'123456') - metadata_indexer.storage.directory_add([{ - 'id': b'123456', - 'entries': [{ - 'name': b'foobar-1.0.0', - 'type': 'dir', - 'target': subdir_id, - 'perms': 16384, - }], - }]) + rev = attr.evolve(rev, directory=b"123456") + metadata_indexer.storage.directory_add( + [ + { + "id": b"123456", + "entries": [ + { + "name": b"foobar-1.0.0", + "type": "dir", + "target": subdir_id, + "perms": 16384, + } + ], + } + ] + ) tool = metadata_indexer.idx_storage.indexer_configuration_get( - {'tool_'+k: v for (k, v) in TRANSLATOR_TOOL.items()}) + {"tool_" + k: v for (k, v) in TRANSLATOR_TOOL.items()} + ) assert tool is not None - metadata_indexer.idx_storage.content_metadata_add([{ - 'indexer_configuration_id': tool['id'], - 'id': b'cde', - 'metadata': YARN_PARSER_METADATA, - }]) + metadata_indexer.idx_storage.content_metadata_add( + [ + { + "indexer_configuration_id": tool["id"], + "id": b"cde", + "metadata": YARN_PARSER_METADATA, + } + ] + ) sha1_gits = [ - hash_to_bytes('8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f'), + hash_to_bytes("8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f"), ] - metadata_indexer.run(sha1_gits, 'update-dups') + metadata_indexer.run(sha1_gits, "update-dups") results = list( - metadata_indexer.idx_storage. - revision_intrinsic_metadata_get(sha1_gits)) + metadata_indexer.idx_storage.revision_intrinsic_metadata_get(sha1_gits) + ) - expected_results = [{ - 'id': hash_to_bytes('8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f'), - 'tool': TRANSLATOR_TOOL, - 'metadata': YARN_PARSER_METADATA, - 'mappings': ['npm'], - }] + expected_results = [ + { + "id": hash_to_bytes("8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f"), + "tool": TRANSLATOR_TOOL, + "metadata": YARN_PARSER_METADATA, + "mappings": ["npm"], + } + ] for result in results: - del result['tool']['id'] + del result["tool"]["id"] # then self.assertEqual(expected_results, results) diff --git a/swh/indexer/tests/test_mimetype.py b/swh/indexer/tests/test_mimetype.py index 91895c9..9a9e3b1 100644 --- a/swh/indexer/tests/test_mimetype.py +++ b/swh/indexer/tests/test_mimetype.py @@ -1,148 +1,150 @@ # Copyright (C) 2017-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import pytest import unittest from typing import Any, Dict from swh.indexer.mimetype import ( - MimetypeIndexer, MimetypeRangeIndexer, compute_mimetype_encoding + MimetypeIndexer, + MimetypeRangeIndexer, + compute_mimetype_encoding, ) from swh.indexer.tests.utils import ( - CommonContentIndexerTest, CommonContentIndexerRangeTest, - BASE_TEST_CONFIG, fill_storage, fill_obj_storage, filter_dict, + CommonContentIndexerTest, + CommonContentIndexerRangeTest, + BASE_TEST_CONFIG, + fill_storage, + fill_obj_storage, + filter_dict, ) def test_compute_mimetype_encoding(): """Compute mimetype encoding should return results""" for _input, _mimetype, _encoding in [ - ('du français'.encode(), 'text/plain', 'utf-8'), - (b'def __init__(self):', 'text/x-python', 'us-ascii'), - (b'\xff\xfe\x00\x00\x00\x00\xff\xfe\xff\xff', - 'application/octet-stream', '') + ("du français".encode(), "text/plain", "utf-8"), + (b"def __init__(self):", "text/x-python", "us-ascii"), + (b"\xff\xfe\x00\x00\x00\x00\xff\xfe\xff\xff", "application/octet-stream", ""), ]: actual_result = compute_mimetype_encoding(_input) - assert actual_result == { - 'mimetype': _mimetype, - 'encoding': _encoding - } + assert actual_result == {"mimetype": _mimetype, "encoding": _encoding} CONFIG = { **BASE_TEST_CONFIG, - 'tools': { - 'name': 'file', - 'version': '1:5.30-1+deb9u1', - 'configuration': { - "type": "library", - "debian-package": "python3-magic" - }, + "tools": { + "name": "file", + "version": "1:5.30-1+deb9u1", + "configuration": {"type": "library", "debian-package": "python3-magic"}, }, } # type: Dict[str, Any] class TestMimetypeIndexer(CommonContentIndexerTest, unittest.TestCase): """Mimetype indexer test scenarios: - Known sha1s in the input list have their data indexed - Unknown sha1 in the input list are not indexed """ + legacy_get_format = True def get_indexer_results(self, ids): yield from self.idx_storage.content_mimetype_get(ids) def setUp(self): self.indexer = MimetypeIndexer(config=CONFIG) self.indexer.catch_exceptions = False self.idx_storage = self.indexer.idx_storage fill_storage(self.indexer.storage) fill_obj_storage(self.indexer.objstorage) - self.id0 = '01c9379dfc33803963d07c1ccc748d3fe4c96bb5' - self.id1 = '688a5ef812c53907562fe379d4b3851e69c7cb15' - self.id2 = 'da39a3ee5e6b4b0d3255bfef95601890afd80709' + self.id0 = "01c9379dfc33803963d07c1ccc748d3fe4c96bb5" + self.id1 = "688a5ef812c53907562fe379d4b3851e69c7cb15" + self.id2 = "da39a3ee5e6b4b0d3255bfef95601890afd80709" - tool = {k.replace('tool_', ''): v - for (k, v) in self.indexer.tool.items()} + tool = {k.replace("tool_", ""): v for (k, v) in self.indexer.tool.items()} self.expected_results = { self.id0: { - 'id': self.id0, - 'tool': tool, - 'mimetype': 'text/plain', - 'encoding': 'us-ascii', + "id": self.id0, + "tool": tool, + "mimetype": "text/plain", + "encoding": "us-ascii", }, self.id1: { - 'id': self.id1, - 'tool': tool, - 'mimetype': 'text/plain', - 'encoding': 'us-ascii', + "id": self.id1, + "tool": tool, + "mimetype": "text/plain", + "encoding": "us-ascii", }, self.id2: { - 'id': self.id2, - 'tool': tool, - 'mimetype': 'application/x-empty', - 'encoding': 'binary', - } + "id": self.id2, + "tool": tool, + "mimetype": "application/x-empty", + "encoding": "binary", + }, } -RANGE_CONFIG = dict(list(CONFIG.items()) + [('write_batch_size', 100)]) +RANGE_CONFIG = dict(list(CONFIG.items()) + [("write_batch_size", 100)]) -class TestMimetypeRangeIndexer( - CommonContentIndexerRangeTest, unittest.TestCase): +class TestMimetypeRangeIndexer(CommonContentIndexerRangeTest, unittest.TestCase): """Range Mimetype Indexer tests. - new data within range are indexed - no data outside a range are indexed - with filtering existing indexed data prior to compute new index - without filtering existing indexed data prior to compute new index """ + def setUp(self): super().setUp() self.indexer = MimetypeRangeIndexer(config=RANGE_CONFIG) self.indexer.catch_exceptions = False fill_storage(self.indexer.storage) fill_obj_storage(self.indexer.objstorage) - self.id0 = '01c9379dfc33803963d07c1ccc748d3fe4c96bb5' - self.id1 = '02fb2c89e14f7fab46701478c83779c7beb7b069' - self.id2 = '103bc087db1d26afc3a0283f38663d081e9b01e6' - tool_id = self.indexer.tool['id'] + self.id0 = "01c9379dfc33803963d07c1ccc748d3fe4c96bb5" + self.id1 = "02fb2c89e14f7fab46701478c83779c7beb7b069" + self.id2 = "103bc087db1d26afc3a0283f38663d081e9b01e6" + tool_id = self.indexer.tool["id"] self.expected_results = { self.id0: { - 'encoding': 'us-ascii', - 'id': self.id0, - 'indexer_configuration_id': tool_id, - 'mimetype': 'text/plain'}, + "encoding": "us-ascii", + "id": self.id0, + "indexer_configuration_id": tool_id, + "mimetype": "text/plain", + }, self.id1: { - 'encoding': 'us-ascii', - 'id': self.id1, - 'indexer_configuration_id': tool_id, - 'mimetype': 'text/x-python'}, + "encoding": "us-ascii", + "id": self.id1, + "indexer_configuration_id": tool_id, + "mimetype": "text/x-python", + }, self.id2: { - 'encoding': 'us-ascii', - 'id': self.id2, - 'indexer_configuration_id': tool_id, - 'mimetype': 'text/plain'} + "encoding": "us-ascii", + "id": self.id2, + "indexer_configuration_id": tool_id, + "mimetype": "text/plain", + }, } def test_mimetype_w_no_tool(): with pytest.raises(ValueError): - MimetypeIndexer(config=filter_dict(CONFIG, 'tools')) + MimetypeIndexer(config=filter_dict(CONFIG, "tools")) def test_mimetype_range_w_no_tool(): with pytest.raises(ValueError): - MimetypeRangeIndexer(config=filter_dict(CONFIG, 'tools')) + MimetypeRangeIndexer(config=filter_dict(CONFIG, "tools")) diff --git a/swh/indexer/tests/test_origin_head.py b/swh/indexer/tests/test_origin_head.py index 758526c..483ebca 100644 --- a/swh/indexer/tests/test_origin_head.py +++ b/swh/indexer/tests/test_origin_head.py @@ -1,155 +1,171 @@ # Copyright (C) 2017-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import unittest from datetime import datetime from swh.indexer.origin_head import OriginHeadIndexer -from swh.indexer.tests.utils import ( - BASE_TEST_CONFIG, fill_storage -) +from swh.indexer.tests.utils import BASE_TEST_CONFIG, fill_storage ORIGIN_HEAD_CONFIG = { **BASE_TEST_CONFIG, - 'tools': { - 'name': 'origin-metadata', - 'version': '0.0.1', - 'configuration': {}, - }, - 'tasks': { - 'revision_intrinsic_metadata': None, - 'origin_intrinsic_metadata': None, - } + "tools": {"name": "origin-metadata", "version": "0.0.1", "configuration": {},}, + "tasks": {"revision_intrinsic_metadata": None, "origin_intrinsic_metadata": None,}, } class OriginHeadTestIndexer(OriginHeadIndexer): """Specific indexer whose configuration is enough to satisfy the indexing tests. """ + def parse_config_file(self, *args, **kwargs): return ORIGIN_HEAD_CONFIG def persist_index_computations(self, results, policy_update): self.results = results class OriginHead(unittest.TestCase): def setUp(self): self.indexer = OriginHeadTestIndexer() self.indexer.catch_exceptions = False fill_storage(self.indexer.storage) def test_git(self): - self.indexer.run( - ['https://github.com/SoftwareHeritage/swh-storage']) - self.assertEqual(self.indexer.results, [{ - 'revision_id': b'8K\x12\x00d\x03\xcc\xe4]bS\xe3\x8f{' - b'\xd7}\xac\xefrm', - 'origin_url': 'https://github.com/SoftwareHeritage/swh-storage'}]) + self.indexer.run(["https://github.com/SoftwareHeritage/swh-storage"]) + self.assertEqual( + self.indexer.results, + [ + { + "revision_id": b"8K\x12\x00d\x03\xcc\xe4]bS\xe3\x8f{" + b"\xd7}\xac\xefrm", + "origin_url": "https://github.com/SoftwareHeritage/swh-storage", + } + ], + ) def test_git_partial_snapshot(self): """Checks partial snapshots are ignored.""" - origin_url = 'https://github.com/SoftwareHeritage/swh-core' - self.indexer.storage.origin_add_one({ - 'url': origin_url, - }) + origin_url = "https://github.com/SoftwareHeritage/swh-core" + self.indexer.storage.origin_add_one( + {"url": origin_url,} + ) visit = self.indexer.storage.origin_visit_add( - origin_url, datetime(2019, 2, 27), type='git', + origin_url, datetime(2019, 2, 27), type="git", ) - self.indexer.storage.snapshot_add([{ - 'id': b'foo', - 'branches': { - b'foo': None, - b'HEAD': { - 'target_type': 'alias', - 'target': b'foo', + self.indexer.storage.snapshot_add( + [ + { + "id": b"foo", + "branches": { + b"foo": None, + b"HEAD": {"target_type": "alias", "target": b"foo",}, + }, } - } - }]) + ] + ) self.indexer.storage.origin_visit_update( - origin_url, visit.visit, status='partial', snapshot=b'foo') + origin_url, visit.visit, status="partial", snapshot=b"foo" + ) self.indexer.run([origin_url]) self.assertEqual(self.indexer.results, []) def test_vcs_missing_snapshot(self): - self.indexer.storage.origin_add([{ - 'url': 'https://github.com/SoftwareHeritage/swh-indexer', - }]) - self.indexer.run( - ['https://github.com/SoftwareHeritage/swh-indexer']) + self.indexer.storage.origin_add( + [{"url": "https://github.com/SoftwareHeritage/swh-indexer",}] + ) + self.indexer.run(["https://github.com/SoftwareHeritage/swh-indexer"]) self.assertEqual(self.indexer.results, []) def test_pypi_missing_branch(self): - origin_url = 'https://pypi.org/project/abcdef/' - self.indexer.storage.origin_add_one({ - 'url': origin_url, - }) + origin_url = "https://pypi.org/project/abcdef/" + self.indexer.storage.origin_add_one( + {"url": origin_url,} + ) visit = self.indexer.storage.origin_visit_add( - origin_url, datetime(2019, 2, 27), type='pypi') - self.indexer.storage.snapshot_add([{ - 'id': b'foo', - 'branches': { - b'foo': None, - b'HEAD': { - 'target_type': 'alias', - 'target': b'foo', + origin_url, datetime(2019, 2, 27), type="pypi" + ) + self.indexer.storage.snapshot_add( + [ + { + "id": b"foo", + "branches": { + b"foo": None, + b"HEAD": {"target_type": "alias", "target": b"foo",}, + }, } - } - }]) + ] + ) self.indexer.storage.origin_visit_update( - origin_url, visit.visit, status='full', snapshot=b'foo') - self.indexer.run(['https://pypi.org/project/abcdef/']) + origin_url, visit.visit, status="full", snapshot=b"foo" + ) + self.indexer.run(["https://pypi.org/project/abcdef/"]) self.assertEqual(self.indexer.results, []) def test_ftp(self): - self.indexer.run( - ['rsync://ftp.gnu.org/gnu/3dldf']) - self.assertEqual(self.indexer.results, [{ - 'revision_id': b'\x8e\xa9\x8e/\xea}\x9feF\xf4\x9f\xfd\xee' - b'\xcc\x1a\xb4`\x8c\x8by', - 'origin_url': 'rsync://ftp.gnu.org/gnu/3dldf'}]) + self.indexer.run(["rsync://ftp.gnu.org/gnu/3dldf"]) + self.assertEqual( + self.indexer.results, + [ + { + "revision_id": b"\x8e\xa9\x8e/\xea}\x9feF\xf4\x9f\xfd\xee" + b"\xcc\x1a\xb4`\x8c\x8by", + "origin_url": "rsync://ftp.gnu.org/gnu/3dldf", + } + ], + ) def test_ftp_missing_snapshot(self): - self.indexer.storage.origin_add([{ - 'url': 'rsync://ftp.gnu.org/gnu/foobar', - }]) - self.indexer.run( - ['rsync://ftp.gnu.org/gnu/foobar']) + self.indexer.storage.origin_add([{"url": "rsync://ftp.gnu.org/gnu/foobar",}]) + self.indexer.run(["rsync://ftp.gnu.org/gnu/foobar"]) self.assertEqual(self.indexer.results, []) def test_deposit(self): - self.indexer.run( - ['https://forge.softwareheritage.org/source/' - 'jesuisgpl/']) - self.assertEqual(self.indexer.results, [{ - 'revision_id': b'\xe7n\xa4\x9c\x9f\xfb\xb7\xf76\x11\x08{' - b'\xa6\xe9\x99\xb1\x9e]q\xeb', - 'origin_url': 'https://forge.softwareheritage.org/source/' - 'jesuisgpl/'}]) + self.indexer.run(["https://forge.softwareheritage.org/source/" "jesuisgpl/"]) + self.assertEqual( + self.indexer.results, + [ + { + "revision_id": b"\xe7n\xa4\x9c\x9f\xfb\xb7\xf76\x11\x08{" + b"\xa6\xe9\x99\xb1\x9e]q\xeb", + "origin_url": "https://forge.softwareheritage.org/source/" + "jesuisgpl/", + } + ], + ) def test_deposit_missing_snapshot(self): - self.indexer.storage.origin_add([{ - 'url': 'https://forge.softwareheritage.org/source/foobar', - }]) - self.indexer.run( - ['https://forge.softwareheritage.org/source/foobar']) + self.indexer.storage.origin_add( + [{"url": "https://forge.softwareheritage.org/source/foobar",}] + ) + self.indexer.run(["https://forge.softwareheritage.org/source/foobar"]) self.assertEqual(self.indexer.results, []) def test_pypi(self): - self.indexer.run( - ['https://pypi.org/project/limnoria/']) - self.assertEqual(self.indexer.results, [{ - 'revision_id': b'\x83\xb9\xb6\xc7\x05\xb1%\xd0\xfem\xd8k' - b'A\x10\x9d\xc5\xfa2\xf8t', - 'origin_url': 'https://pypi.org/project/limnoria/'}]) + self.indexer.run(["https://pypi.org/project/limnoria/"]) + self.assertEqual( + self.indexer.results, + [ + { + "revision_id": b"\x83\xb9\xb6\xc7\x05\xb1%\xd0\xfem\xd8k" + b"A\x10\x9d\xc5\xfa2\xf8t", + "origin_url": "https://pypi.org/project/limnoria/", + } + ], + ) def test_svn(self): - self.indexer.run( - ['http://0-512-md.googlecode.com/svn/']) - self.assertEqual(self.indexer.results, [{ - 'revision_id': b'\xe4?r\xe1,\x88\xab\xec\xe7\x9a\x87\xb8' - b'\xc9\xad#.\x1bw=\x18', - 'origin_url': 'http://0-512-md.googlecode.com/svn/'}]) + self.indexer.run(["http://0-512-md.googlecode.com/svn/"]) + self.assertEqual( + self.indexer.results, + [ + { + "revision_id": b"\xe4?r\xe1,\x88\xab\xec\xe7\x9a\x87\xb8" + b"\xc9\xad#.\x1bw=\x18", + "origin_url": "http://0-512-md.googlecode.com/svn/", + } + ], + ) diff --git a/swh/indexer/tests/test_origin_metadata.py b/swh/indexer/tests/test_origin_metadata.py index c7999f6..79e8de3 100644 --- a/swh/indexer/tests/test_origin_metadata.py +++ b/swh/indexer/tests/test_origin_metadata.py @@ -1,245 +1,224 @@ # Copyright (C) 2018-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from unittest.mock import patch from swh.model.hashutil import hash_to_bytes from swh.indexer.metadata import OriginMetadataIndexer from .utils import YARN_PARSER_METADATA from .test_metadata import REVISION_METADATA_CONFIG def test_origin_metadata_indexer(idx_storage, storage, obj_storage): indexer = OriginMetadataIndexer(config=REVISION_METADATA_CONFIG) indexer.run(["https://github.com/librariesio/yarn-parser"]) - origin = 'https://github.com/librariesio/yarn-parser' - rev_id = hash_to_bytes('8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f') + origin = "https://github.com/librariesio/yarn-parser" + rev_id = hash_to_bytes("8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f") rev_metadata = { - 'id': rev_id, - 'metadata': YARN_PARSER_METADATA, - 'mappings': ['npm'], + "id": rev_id, + "metadata": YARN_PARSER_METADATA, + "mappings": ["npm"], } origin_metadata = { - 'id': origin, - 'from_revision': rev_id, - 'metadata': YARN_PARSER_METADATA, - 'mappings': ['npm'], + "id": origin, + "from_revision": rev_id, + "metadata": YARN_PARSER_METADATA, + "mappings": ["npm"], } - results = list( - indexer.idx_storage.revision_intrinsic_metadata_get([rev_id])) + results = list(indexer.idx_storage.revision_intrinsic_metadata_get([rev_id])) for result in results: - del result['tool'] + del result["tool"] assert results == [rev_metadata] - results = list(indexer.idx_storage.origin_intrinsic_metadata_get([ - origin])) + results = list(indexer.idx_storage.origin_intrinsic_metadata_get([origin])) for result in results: - del result['tool'] + del result["tool"] assert results == [origin_metadata] -def test_origin_metadata_indexer_duplicate_origin( - idx_storage, storage, obj_storage): +def test_origin_metadata_indexer_duplicate_origin(idx_storage, storage, obj_storage): indexer = OriginMetadataIndexer(config=REVISION_METADATA_CONFIG) indexer.storage = storage indexer.idx_storage = idx_storage indexer.run(["https://github.com/librariesio/yarn-parser"]) - indexer.run(["https://github.com/librariesio/yarn-parser"]*2) + indexer.run(["https://github.com/librariesio/yarn-parser"] * 2) - origin = 'https://github.com/librariesio/yarn-parser' - rev_id = hash_to_bytes('8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f') + origin = "https://github.com/librariesio/yarn-parser" + rev_id = hash_to_bytes("8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f") - results = list( - indexer.idx_storage.revision_intrinsic_metadata_get([rev_id])) + results = list(indexer.idx_storage.revision_intrinsic_metadata_get([rev_id])) assert len(results) == 1 - results = list(indexer.idx_storage.origin_intrinsic_metadata_get([ - origin])) + results = list(indexer.idx_storage.origin_intrinsic_metadata_get([origin])) assert len(results) == 1 -def test_origin_metadata_indexer_missing_head( - idx_storage, storage, obj_storage): +def test_origin_metadata_indexer_missing_head(idx_storage, storage, obj_storage): - storage.origin_add([{ - 'url': 'https://example.com' - }]) + storage.origin_add([{"url": "https://example.com"}]) indexer = OriginMetadataIndexer(config=REVISION_METADATA_CONFIG) indexer.run(["https://example.com"]) - origin = 'https://example.com' + origin = "https://example.com" - results = list(indexer.idx_storage.origin_intrinsic_metadata_get([ - origin])) + results = list(indexer.idx_storage.origin_intrinsic_metadata_get([origin])) assert results == [] def test_origin_metadata_indexer_partial_missing_head( - idx_storage, storage, obj_storage): + idx_storage, storage, obj_storage +): - storage.origin_add([{ - 'url': 'https://example.com' - }]) + storage.origin_add([{"url": "https://example.com"}]) indexer = OriginMetadataIndexer(config=REVISION_METADATA_CONFIG) - indexer.run(["https://example.com", - "https://github.com/librariesio/yarn-parser"]) + indexer.run(["https://example.com", "https://github.com/librariesio/yarn-parser"]) - origin1 = 'https://example.com' - origin2 = 'https://github.com/librariesio/yarn-parser' - rev_id = hash_to_bytes('8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f') + origin1 = "https://example.com" + origin2 = "https://github.com/librariesio/yarn-parser" + rev_id = hash_to_bytes("8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f") rev_metadata = { - 'id': rev_id, - 'metadata': YARN_PARSER_METADATA, - 'mappings': ['npm'], + "id": rev_id, + "metadata": YARN_PARSER_METADATA, + "mappings": ["npm"], } origin_metadata = { - 'id': origin2, - 'from_revision': rev_id, - 'metadata': YARN_PARSER_METADATA, - 'mappings': ['npm'], + "id": origin2, + "from_revision": rev_id, + "metadata": YARN_PARSER_METADATA, + "mappings": ["npm"], } - results = list( - indexer.idx_storage.revision_intrinsic_metadata_get([rev_id])) + results = list(indexer.idx_storage.revision_intrinsic_metadata_get([rev_id])) for result in results: - del result['tool'] + del result["tool"] assert results == [rev_metadata] - results = list(indexer.idx_storage.origin_intrinsic_metadata_get([ - origin1, origin2])) + results = list( + indexer.idx_storage.origin_intrinsic_metadata_get([origin1, origin2]) + ) for result in results: - del result['tool'] + del result["tool"] assert results == [origin_metadata] -def test_origin_metadata_indexer_duplicate_revision( - idx_storage, storage, obj_storage): +def test_origin_metadata_indexer_duplicate_revision(idx_storage, storage, obj_storage): indexer = OriginMetadataIndexer(config=REVISION_METADATA_CONFIG) indexer.storage = storage indexer.idx_storage = idx_storage - indexer.run(["https://github.com/librariesio/yarn-parser", - "https://github.com/librariesio/yarn-parser.git"]) - - origin1 = 'https://github.com/librariesio/yarn-parser' - origin2 = 'https://github.com/librariesio/yarn-parser.git' - rev_id = hash_to_bytes('8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f') - - results = list( - indexer.idx_storage.revision_intrinsic_metadata_get([rev_id])) + indexer.run( + [ + "https://github.com/librariesio/yarn-parser", + "https://github.com/librariesio/yarn-parser.git", + ] + ) + + origin1 = "https://github.com/librariesio/yarn-parser" + origin2 = "https://github.com/librariesio/yarn-parser.git" + rev_id = hash_to_bytes("8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f") + + results = list(indexer.idx_storage.revision_intrinsic_metadata_get([rev_id])) assert len(results) == 1 - results = list(indexer.idx_storage.origin_intrinsic_metadata_get( - [origin1, origin2])) + results = list( + indexer.idx_storage.origin_intrinsic_metadata_get([origin1, origin2]) + ) assert len(results) == 2 -def test_origin_metadata_indexer_no_metadata_file( - idx_storage, storage, obj_storage): +def test_origin_metadata_indexer_no_metadata_file(idx_storage, storage, obj_storage): indexer = OriginMetadataIndexer(config=REVISION_METADATA_CONFIG) - with patch('swh.indexer.metadata_dictionary.npm.NpmMapping.filename', - b'foo.json'): + with patch("swh.indexer.metadata_dictionary.npm.NpmMapping.filename", b"foo.json"): indexer.run(["https://github.com/librariesio/yarn-parser"]) - origin = 'https://github.com/librariesio/yarn-parser' - rev_id = hash_to_bytes('8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f') + origin = "https://github.com/librariesio/yarn-parser" + rev_id = hash_to_bytes("8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f") - results = list( - indexer.idx_storage.revision_intrinsic_metadata_get([rev_id])) + results = list(indexer.idx_storage.revision_intrinsic_metadata_get([rev_id])) assert results == [] - results = list(indexer.idx_storage.origin_intrinsic_metadata_get([ - origin])) + results = list(indexer.idx_storage.origin_intrinsic_metadata_get([origin])) assert results == [] -def test_origin_metadata_indexer_no_metadata( - idx_storage, storage, obj_storage): +def test_origin_metadata_indexer_no_metadata(idx_storage, storage, obj_storage): indexer = OriginMetadataIndexer(config=REVISION_METADATA_CONFIG) - with patch('swh.indexer.metadata.RevisionMetadataIndexer' - '.translate_revision_intrinsic_metadata', - return_value=(['npm'], {'@context': 'foo'})): + with patch( + "swh.indexer.metadata.RevisionMetadataIndexer" + ".translate_revision_intrinsic_metadata", + return_value=(["npm"], {"@context": "foo"}), + ): indexer.run(["https://github.com/librariesio/yarn-parser"]) - origin = 'https://github.com/librariesio/yarn-parser' - rev_id = hash_to_bytes('8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f') + origin = "https://github.com/librariesio/yarn-parser" + rev_id = hash_to_bytes("8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f") - results = list( - indexer.idx_storage.revision_intrinsic_metadata_get([rev_id])) + results = list(indexer.idx_storage.revision_intrinsic_metadata_get([rev_id])) assert results == [] - results = list(indexer.idx_storage.origin_intrinsic_metadata_get([ - origin])) + results = list(indexer.idx_storage.origin_intrinsic_metadata_get([origin])) assert results == [] -def test_origin_metadata_indexer_error( - idx_storage, storage, obj_storage): +def test_origin_metadata_indexer_error(idx_storage, storage, obj_storage): indexer = OriginMetadataIndexer(config=REVISION_METADATA_CONFIG) - with patch('swh.indexer.metadata.RevisionMetadataIndexer' - '.translate_revision_intrinsic_metadata', - return_value=None): + with patch( + "swh.indexer.metadata.RevisionMetadataIndexer" + ".translate_revision_intrinsic_metadata", + return_value=None, + ): indexer.run(["https://github.com/librariesio/yarn-parser"]) - origin = 'https://github.com/librariesio/yarn-parser' - rev_id = hash_to_bytes('8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f') + origin = "https://github.com/librariesio/yarn-parser" + rev_id = hash_to_bytes("8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f") - results = list( - indexer.idx_storage.revision_intrinsic_metadata_get([rev_id])) + results = list(indexer.idx_storage.revision_intrinsic_metadata_get([rev_id])) assert results == [] - results = list(indexer.idx_storage.origin_intrinsic_metadata_get([ - origin])) + results = list(indexer.idx_storage.origin_intrinsic_metadata_get([origin])) assert results == [] -def test_origin_metadata_indexer_delete_metadata( - idx_storage, storage, obj_storage): +def test_origin_metadata_indexer_delete_metadata(idx_storage, storage, obj_storage): indexer = OriginMetadataIndexer(config=REVISION_METADATA_CONFIG) indexer.run(["https://github.com/librariesio/yarn-parser"]) - origin = 'https://github.com/librariesio/yarn-parser' - rev_id = hash_to_bytes('8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f') + origin = "https://github.com/librariesio/yarn-parser" + rev_id = hash_to_bytes("8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f") - results = list( - indexer.idx_storage.revision_intrinsic_metadata_get([rev_id])) + results = list(indexer.idx_storage.revision_intrinsic_metadata_get([rev_id])) assert results != [] - results = list(indexer.idx_storage.origin_intrinsic_metadata_get([ - origin])) + results = list(indexer.idx_storage.origin_intrinsic_metadata_get([origin])) assert results != [] - with patch('swh.indexer.metadata_dictionary.npm.NpmMapping.filename', - b'foo.json'): + with patch("swh.indexer.metadata_dictionary.npm.NpmMapping.filename", b"foo.json"): indexer.run(["https://github.com/librariesio/yarn-parser"]) - results = list( - indexer.idx_storage.revision_intrinsic_metadata_get([rev_id])) + results = list(indexer.idx_storage.revision_intrinsic_metadata_get([rev_id])) assert results == [] - results = list(indexer.idx_storage.origin_intrinsic_metadata_get([ - origin])) + results = list(indexer.idx_storage.origin_intrinsic_metadata_get([origin])) assert results == [] -def test_origin_metadata_indexer_unknown_origin( - idx_storage, storage, obj_storage): +def test_origin_metadata_indexer_unknown_origin(idx_storage, storage, obj_storage): indexer = OriginMetadataIndexer(config=REVISION_METADATA_CONFIG) result = indexer.index_list(["https://unknown.org/foo"]) assert not result diff --git a/swh/indexer/tests/utils.py b/swh/indexer/tests/utils.py index 17b4372..0dbe993 100644 --- a/swh/indexer/tests/utils.py +++ b/swh/indexer/tests/utils.py @@ -1,765 +1,724 @@ # Copyright (C) 2017-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import abc import datetime import functools import random from typing import Dict, Any import unittest from hypothesis import strategies from swh.model import hashutil from swh.model.hashutil import hash_to_bytes, hash_to_hex from swh.indexer.storage import INDEXER_CFG_KEY BASE_TEST_CONFIG: Dict[str, Dict[str, Any]] = { - 'storage': { - 'cls': 'pipeline', - 'steps': [ - {'cls': 'validate'}, - {'cls': 'memory'}, - ] - }, - 'objstorage': { - 'cls': 'memory', - 'args': { - }, - }, - INDEXER_CFG_KEY: { - 'cls': 'memory', - 'args': { - }, - }, + "storage": {"cls": "pipeline", "steps": [{"cls": "validate"}, {"cls": "memory"},]}, + "objstorage": {"cls": "memory", "args": {},}, + INDEXER_CFG_KEY: {"cls": "memory", "args": {},}, } ORIGIN_VISITS = [ - { - 'type': 'git', - 'url': 'https://github.com/SoftwareHeritage/swh-storage'}, - { - 'type': 'ftp', - 'url': 'rsync://ftp.gnu.org/gnu/3dldf'}, - { - 'type': 'deposit', - 'url': 'https://forge.softwareheritage.org/source/jesuisgpl/'}, - { - 'type': 'pypi', - 'url': 'https://pypi.org/project/limnoria/'}, - { - 'type': 'svn', - 'url': 'http://0-512-md.googlecode.com/svn/'}, - { - 'type': 'git', - 'url': 'https://github.com/librariesio/yarn-parser'}, - { - 'type': 'git', - 'url': 'https://github.com/librariesio/yarn-parser.git'}, - ] + {"type": "git", "url": "https://github.com/SoftwareHeritage/swh-storage"}, + {"type": "ftp", "url": "rsync://ftp.gnu.org/gnu/3dldf"}, + {"type": "deposit", "url": "https://forge.softwareheritage.org/source/jesuisgpl/"}, + {"type": "pypi", "url": "https://pypi.org/project/limnoria/"}, + {"type": "svn", "url": "http://0-512-md.googlecode.com/svn/"}, + {"type": "git", "url": "https://github.com/librariesio/yarn-parser"}, + {"type": "git", "url": "https://github.com/librariesio/yarn-parser.git"}, +] SNAPSHOTS = [ { - 'origin': 'https://github.com/SoftwareHeritage/swh-storage', - 'branches': { - b'refs/heads/add-revision-origin-cache': { - 'target': b'L[\xce\x1c\x88\x8eF\t\xf1"\x19\x1e\xfb\xc0' - b's\xe7/\xe9l\x1e', - 'target_type': 'revision'}, - b'refs/head/master': { - 'target': b'8K\x12\x00d\x03\xcc\xe4]bS\xe3\x8f{\xd7}' - b'\xac\xefrm', - 'target_type': 'revision'}, - b'HEAD': { - 'target': b'refs/head/master', - 'target_type': 'alias'}, - b'refs/tags/v0.0.103': { - 'target': b'\xb6"Im{\xfdLb\xb0\x94N\xea\x96m\x13x\x88+' - b'\x0f\xdd', - 'target_type': 'release'}, - }}, - { - 'origin': 'rsync://ftp.gnu.org/gnu/3dldf', - 'branches': { - b'3DLDF-1.1.4.tar.gz': { - 'target': b'dJ\xfb\x1c\x91\xf4\x82B%]6\xa2\x90|\xd3\xfc' - b'"G\x99\x11', - 'target_type': 'revision'}, - b'3DLDF-2.0.2.tar.gz': { - 'target': b'\xb6\x0e\xe7\x9e9\xac\xaa\x19\x9e=' - b'\xd1\xc5\x00\\\xc6\xfc\xe0\xa6\xb4V', - 'target_type': 'revision'}, - b'3DLDF-2.0.3-examples.tar.gz': { - 'target': b'!H\x19\xc0\xee\x82-\x12F1\xbd\x97' - b'\xfe\xadZ\x80\x80\xc1\x83\xff', - 'target_type': 'revision'}, - b'3DLDF-2.0.3.tar.gz': { - 'target': b'\x8e\xa9\x8e/\xea}\x9feF\xf4\x9f\xfd\xee' - b'\xcc\x1a\xb4`\x8c\x8by', - 'target_type': 'revision'}, - b'3DLDF-2.0.tar.gz': { - 'target': b'F6*\xff(?\x19a\xef\xb6\xc2\x1fv$S\xe3G' - b'\xd3\xd1m', - 'target_type': 'revision'} - }}, - { - 'origin': 'https://forge.softwareheritage.org/source/jesuisgpl/', - 'branches': { - b'master': { - 'target': b'\xe7n\xa4\x9c\x9f\xfb\xb7\xf76\x11\x08{' - b'\xa6\xe9\x99\xb1\x9e]q\xeb', - 'target_type': 'revision'} + "origin": "https://github.com/SoftwareHeritage/swh-storage", + "branches": { + b"refs/heads/add-revision-origin-cache": { + "target": b'L[\xce\x1c\x88\x8eF\t\xf1"\x19\x1e\xfb\xc0' + b"s\xe7/\xe9l\x1e", + "target_type": "revision", + }, + b"refs/head/master": { + "target": b"8K\x12\x00d\x03\xcc\xe4]bS\xe3\x8f{\xd7}" b"\xac\xefrm", + "target_type": "revision", + }, + b"HEAD": {"target": b"refs/head/master", "target_type": "alias"}, + b"refs/tags/v0.0.103": { + "target": b'\xb6"Im{\xfdLb\xb0\x94N\xea\x96m\x13x\x88+' b"\x0f\xdd", + "target_type": "release", + }, }, - 'id': b"h\xc0\xd2a\x04\xd4~'\x8d\xd6\xbe\x07\xeda\xfa\xfbV" - b"\x1d\r "}, - { - 'origin': 'https://pypi.org/project/limnoria/', - 'branches': { - b'HEAD': { - 'target': b'releases/2018.09.09', - 'target_type': 'alias'}, - b'releases/2018.09.01': { - 'target': b'<\xee1(\xe8\x8d_\xc1\xc9\xa6rT\xf1\x1d' - b'\xbb\xdfF\xfdw\xcf', - 'target_type': 'revision'}, - b'releases/2018.09.09': { - 'target': b'\x83\xb9\xb6\xc7\x05\xb1%\xd0\xfem\xd8k' - b'A\x10\x9d\xc5\xfa2\xf8t', - 'target_type': 'revision'}}, - 'id': b'{\xda\x8e\x84\x7fX\xff\x92\x80^\x93V\x18\xa3\xfay' - b'\x12\x9e\xd6\xb3'}, + }, { - 'origin': 'http://0-512-md.googlecode.com/svn/', - 'branches': { - b'master': { - 'target': b'\xe4?r\xe1,\x88\xab\xec\xe7\x9a\x87\xb8' - b'\xc9\xad#.\x1bw=\x18', - 'target_type': 'revision'}}, - 'id': b'\xa1\xa2\x8c\n\xb3\x87\xa8\xf9\xe0a\x8c\xb7' - b'\x05\xea\xb8\x1f\xc4H\xf4s'}, + "origin": "rsync://ftp.gnu.org/gnu/3dldf", + "branches": { + b"3DLDF-1.1.4.tar.gz": { + "target": b"dJ\xfb\x1c\x91\xf4\x82B%]6\xa2\x90|\xd3\xfc" b'"G\x99\x11', + "target_type": "revision", + }, + b"3DLDF-2.0.2.tar.gz": { + "target": b"\xb6\x0e\xe7\x9e9\xac\xaa\x19\x9e=" + b"\xd1\xc5\x00\\\xc6\xfc\xe0\xa6\xb4V", + "target_type": "revision", + }, + b"3DLDF-2.0.3-examples.tar.gz": { + "target": b"!H\x19\xc0\xee\x82-\x12F1\xbd\x97" + b"\xfe\xadZ\x80\x80\xc1\x83\xff", + "target_type": "revision", + }, + b"3DLDF-2.0.3.tar.gz": { + "target": b"\x8e\xa9\x8e/\xea}\x9feF\xf4\x9f\xfd\xee" + b"\xcc\x1a\xb4`\x8c\x8by", + "target_type": "revision", + }, + b"3DLDF-2.0.tar.gz": { + "target": b"F6*\xff(?\x19a\xef\xb6\xc2\x1fv$S\xe3G" b"\xd3\xd1m", + "target_type": "revision", + }, + }, + }, { - 'origin': 'https://github.com/librariesio/yarn-parser', - 'branches': { - b'HEAD': { - 'target': hash_to_bytes( - '8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f'), - 'target_type': 'revision'}}}, + "origin": "https://forge.softwareheritage.org/source/jesuisgpl/", + "branches": { + b"master": { + "target": b"\xe7n\xa4\x9c\x9f\xfb\xb7\xf76\x11\x08{" + b"\xa6\xe9\x99\xb1\x9e]q\xeb", + "target_type": "revision", + } + }, + "id": b"h\xc0\xd2a\x04\xd4~'\x8d\xd6\xbe\x07\xeda\xfa\xfbV" b"\x1d\r ", + }, { - 'origin': 'https://github.com/librariesio/yarn-parser.git', - 'branches': { - b'HEAD': { - 'target': hash_to_bytes( - '8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f'), - 'target_type': 'revision'}}}, -] - - -REVISIONS = [{ - 'id': hash_to_bytes('8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f'), - 'message': b'Improve search functionality', - 'author': { - 'name': b'Andrew Nesbitt', - 'fullname': b'Andrew Nesbitt ', - 'email': b'andrewnez@gmail.com' + "origin": "https://pypi.org/project/limnoria/", + "branches": { + b"HEAD": {"target": b"releases/2018.09.09", "target_type": "alias"}, + b"releases/2018.09.01": { + "target": b"<\xee1(\xe8\x8d_\xc1\xc9\xa6rT\xf1\x1d" + b"\xbb\xdfF\xfdw\xcf", + "target_type": "revision", + }, + b"releases/2018.09.09": { + "target": b"\x83\xb9\xb6\xc7\x05\xb1%\xd0\xfem\xd8k" + b"A\x10\x9d\xc5\xfa2\xf8t", + "target_type": "revision", + }, + }, + "id": b"{\xda\x8e\x84\x7fX\xff\x92\x80^\x93V\x18\xa3\xfay" b"\x12\x9e\xd6\xb3", }, - 'committer': { - 'name': b'Andrew Nesbitt', - 'fullname': b'Andrew Nesbitt ', - 'email': b'andrewnez@gmail.com' + { + "origin": "http://0-512-md.googlecode.com/svn/", + "branches": { + b"master": { + "target": b"\xe4?r\xe1,\x88\xab\xec\xe7\x9a\x87\xb8" + b"\xc9\xad#.\x1bw=\x18", + "target_type": "revision", + } + }, + "id": b"\xa1\xa2\x8c\n\xb3\x87\xa8\xf9\xe0a\x8c\xb7" + b"\x05\xea\xb8\x1f\xc4H\xf4s", }, - 'committer_date': { - 'negative_utc': False, - 'offset': 120, - 'timestamp': { - 'microseconds': 0, - 'seconds': 1380883849, - } + { + "origin": "https://github.com/librariesio/yarn-parser", + "branches": { + b"HEAD": { + "target": hash_to_bytes("8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f"), + "target_type": "revision", + } + }, }, - 'type': 'git', - 'synthetic': False, - 'date': { - 'negative_utc': False, - 'timestamp': { - 'seconds': 1487596456, - 'microseconds': 0, + { + "origin": "https://github.com/librariesio/yarn-parser.git", + "branches": { + b"HEAD": { + "target": hash_to_bytes("8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f"), + "target_type": "revision", + } }, - 'offset': 0 }, - 'directory': b'10' -}] +] -DIRECTORY_ID = b'10' -DIRECTORY_ENTRIES = [{ - 'name': b'index.js', - 'type': 'file', - 'target': b'abc', - 'perms': 33188, - }, +REVISIONS = [ { - 'name': b'package.json', - 'type': 'file', - 'target': b'cde', - 'perms': 33188, - }, - { - 'name': b'.github', - 'type': 'dir', - 'target': b'11', - 'perms': 16384, + "id": hash_to_bytes("8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f"), + "message": b"Improve search functionality", + "author": { + "name": b"Andrew Nesbitt", + "fullname": b"Andrew Nesbitt ", + "email": b"andrewnez@gmail.com", + }, + "committer": { + "name": b"Andrew Nesbitt", + "fullname": b"Andrew Nesbitt ", + "email": b"andrewnez@gmail.com", + }, + "committer_date": { + "negative_utc": False, + "offset": 120, + "timestamp": {"microseconds": 0, "seconds": 1380883849,}, + }, + "type": "git", + "synthetic": False, + "date": { + "negative_utc": False, + "timestamp": {"seconds": 1487596456, "microseconds": 0,}, + "offset": 0, + }, + "directory": b"10", } ] +DIRECTORY_ID = b"10" + +DIRECTORY_ENTRIES = [ + {"name": b"index.js", "type": "file", "target": b"abc", "perms": 33188,}, + {"name": b"package.json", "type": "file", "target": b"cde", "perms": 33188,}, + {"name": b".github", "type": "dir", "target": b"11", "perms": 16384,}, +] + SHA1_TO_LICENSES = { - '01c9379dfc33803963d07c1ccc748d3fe4c96bb5': ['GPL'], - '02fb2c89e14f7fab46701478c83779c7beb7b069': ['Apache2.0'], - '103bc087db1d26afc3a0283f38663d081e9b01e6': ['MIT'], - '688a5ef812c53907562fe379d4b3851e69c7cb15': ['AGPL'], - 'da39a3ee5e6b4b0d3255bfef95601890afd80709': [], + "01c9379dfc33803963d07c1ccc748d3fe4c96bb5": ["GPL"], + "02fb2c89e14f7fab46701478c83779c7beb7b069": ["Apache2.0"], + "103bc087db1d26afc3a0283f38663d081e9b01e6": ["MIT"], + "688a5ef812c53907562fe379d4b3851e69c7cb15": ["AGPL"], + "da39a3ee5e6b4b0d3255bfef95601890afd80709": [], } SHA1_TO_CTAGS = { - '01c9379dfc33803963d07c1ccc748d3fe4c96bb5': [{ - 'name': 'foo', - 'kind': 'str', - 'line': 10, - 'lang': 'bar', - }], - 'd4c647f0fc257591cc9ba1722484229780d1c607': [{ - 'name': 'let', - 'kind': 'int', - 'line': 100, - 'lang': 'haskell', - }], - '688a5ef812c53907562fe379d4b3851e69c7cb15': [{ - 'name': 'symbol', - 'kind': 'float', - 'line': 99, - 'lang': 'python', - }], + "01c9379dfc33803963d07c1ccc748d3fe4c96bb5": [ + {"name": "foo", "kind": "str", "line": 10, "lang": "bar",} + ], + "d4c647f0fc257591cc9ba1722484229780d1c607": [ + {"name": "let", "kind": "int", "line": 100, "lang": "haskell",} + ], + "688a5ef812c53907562fe379d4b3851e69c7cb15": [ + {"name": "symbol", "kind": "float", "line": 99, "lang": "python",} + ], } OBJ_STORAGE_DATA = { - '01c9379dfc33803963d07c1ccc748d3fe4c96bb5': b'this is some text', - '688a5ef812c53907562fe379d4b3851e69c7cb15': b'another text', - '8986af901dd2043044ce8f0d8fc039153641cf17': b'yet another text', - '02fb2c89e14f7fab46701478c83779c7beb7b069': b""" + "01c9379dfc33803963d07c1ccc748d3fe4c96bb5": b"this is some text", + "688a5ef812c53907562fe379d4b3851e69c7cb15": b"another text", + "8986af901dd2043044ce8f0d8fc039153641cf17": b"yet another text", + "02fb2c89e14f7fab46701478c83779c7beb7b069": b""" import unittest import logging from swh.indexer.mimetype import MimetypeIndexer from swh.indexer.tests.test_utils import MockObjStorage class MockStorage(): def content_mimetype_add(self, mimetypes): self.state = mimetypes self.conflict_update = conflict_update def indexer_configuration_add(self, tools): return [{ 'id': 10, }] """, - '103bc087db1d26afc3a0283f38663d081e9b01e6': b""" + "103bc087db1d26afc3a0283f38663d081e9b01e6": b""" #ifndef __AVL__ #define __AVL__ typedef struct _avl_tree avl_tree; typedef struct _data_t { int content; } data_t; """, - '93666f74f1cf635c8c8ac118879da6ec5623c410': b""" + "93666f74f1cf635c8c8ac118879da6ec5623c410": b""" (should 'pygments (recognize 'lisp 'easily)) """, - '26a9f72a7c87cc9205725cfd879f514ff4f3d8d5': b""" + "26a9f72a7c87cc9205725cfd879f514ff4f3d8d5": b""" { "name": "test_metadata", "version": "0.0.1", "description": "Simple package.json test for indexer", "repository": { "type": "git", "url": "https://github.com/moranegg/metadata_test" } } """, - 'd4c647f0fc257591cc9ba1722484229780d1c607': b""" + "d4c647f0fc257591cc9ba1722484229780d1c607": b""" { "version": "5.0.3", "name": "npm", "description": "a package manager for JavaScript", "keywords": [ "install", "modules", "package manager", "package.json" ], "preferGlobal": true, "config": { "publishtest": false }, "homepage": "https://docs.npmjs.com/", "author": "Isaac Z. Schlueter (http://blog.izs.me)", "repository": { "type": "git", "url": "https://github.com/npm/npm" }, "bugs": { "url": "https://github.com/npm/npm/issues" }, "dependencies": { "JSONStream": "~1.3.1", "abbrev": "~1.1.0", "ansi-regex": "~2.1.1", "ansicolors": "~0.3.2", "ansistyles": "~0.1.3" }, "devDependencies": { "tacks": "~1.2.6", "tap": "~10.3.2" }, "license": "Artistic-2.0" } """, - 'a7ab314d8a11d2c93e3dcf528ca294e7b431c449': b""" + "a7ab314d8a11d2c93e3dcf528ca294e7b431c449": b""" """, - 'da39a3ee5e6b4b0d3255bfef95601890afd80709': b'', + "da39a3ee5e6b4b0d3255bfef95601890afd80709": b"", # 626364 - hash_to_hex(b'bcd'): b'unimportant content for bcd', + hash_to_hex(b"bcd"): b"unimportant content for bcd", # 636465 - hash_to_hex(b'cde'): b""" + hash_to_hex( + b"cde" + ): b""" { "name": "yarn-parser", "version": "1.0.0", "description": "Tiny web service for parsing yarn.lock files", "main": "index.js", "scripts": { "start": "node index.js", "test": "mocha" }, "engines": { "node": "9.8.0" }, "repository": { "type": "git", "url": "git+https://github.com/librariesio/yarn-parser.git" }, "keywords": [ "yarn", "parse", "lock", "dependencies" ], "author": "Andrew Nesbitt", "license": "AGPL-3.0", "bugs": { "url": "https://github.com/librariesio/yarn-parser/issues" }, "homepage": "https://github.com/librariesio/yarn-parser#readme", "dependencies": { "@yarnpkg/lockfile": "^1.0.0", "body-parser": "^1.15.2", "express": "^4.14.0" }, "devDependencies": { "chai": "^4.1.2", "mocha": "^5.2.0", "request": "^2.87.0", "test": "^0.6.0" } } -""" +""", } YARN_PARSER_METADATA = { - '@context': 'https://doi.org/10.5063/schema/codemeta-2.0', - 'url': - 'https://github.com/librariesio/yarn-parser#readme', - 'codeRepository': - 'git+git+https://github.com/librariesio/yarn-parser.git', - 'author': [{ - 'type': 'Person', - 'name': 'Andrew Nesbitt' - }], - 'license': 'https://spdx.org/licenses/AGPL-3.0', - 'version': '1.0.0', - 'description': - "Tiny web service for parsing yarn.lock files", - 'issueTracker': - 'https://github.com/librariesio/yarn-parser/issues', - 'name': 'yarn-parser', - 'keywords': ['yarn', 'parse', 'lock', 'dependencies'], - 'type': 'SoftwareSourceCode', + "@context": "https://doi.org/10.5063/schema/codemeta-2.0", + "url": "https://github.com/librariesio/yarn-parser#readme", + "codeRepository": "git+git+https://github.com/librariesio/yarn-parser.git", + "author": [{"type": "Person", "name": "Andrew Nesbitt"}], + "license": "https://spdx.org/licenses/AGPL-3.0", + "version": "1.0.0", + "description": "Tiny web service for parsing yarn.lock files", + "issueTracker": "https://github.com/librariesio/yarn-parser/issues", + "name": "yarn-parser", + "keywords": ["yarn", "parse", "lock", "dependencies"], + "type": "SoftwareSourceCode", } json_dict_keys = strategies.one_of( strategies.characters(), - strategies.just('type'), - strategies.just('url'), - strategies.just('name'), - strategies.just('email'), - strategies.just('@id'), - strategies.just('@context'), - strategies.just('repository'), - strategies.just('license'), - strategies.just('repositories'), - strategies.just('licenses'), + strategies.just("type"), + strategies.just("url"), + strategies.just("name"), + strategies.just("email"), + strategies.just("@id"), + strategies.just("@context"), + strategies.just("repository"), + strategies.just("license"), + strategies.just("repositories"), + strategies.just("licenses"), ) """Hypothesis strategy that generates strings, with an emphasis on those that are often used as dictionary keys in metadata files.""" generic_json_document = strategies.recursive( - strategies.none() | strategies.booleans() | strategies.floats() | - strategies.characters(), + strategies.none() + | strategies.booleans() + | strategies.floats() + | strategies.characters(), lambda children: ( - strategies.lists(children, min_size=1) | - strategies.dictionaries(json_dict_keys, children, min_size=1) - ) + strategies.lists(children, min_size=1) + | strategies.dictionaries(json_dict_keys, children, min_size=1) + ), ) """Hypothesis strategy that generates possible values for values of JSON metadata files.""" def json_document_strategy(keys=None): """Generates an hypothesis strategy that generates metadata files for a JSON-based format that uses the given keys.""" if keys is None: keys = strategies.characters() else: keys = strategies.one_of(map(strategies.just, keys)) return strategies.dictionaries(keys, generic_json_document, min_size=1) def _tree_to_xml(root, xmlns, data): def encode(s): "Skips unpaired surrogates generated by json_document_strategy" - return s.encode('utf8', 'replace') + return s.encode("utf8", "replace") - def to_xml(data, indent=b' '): + def to_xml(data, indent=b" "): if data is None: - return b'' + return b"" elif isinstance(data, (bool, str, int, float)): return indent + encode(str(data)) elif isinstance(data, list): - return b'\n'.join(to_xml(v, indent=indent) for v in data) + return b"\n".join(to_xml(v, indent=indent) for v in data) elif isinstance(data, dict): lines = [] for (key, value) in data.items(): - lines.append(indent + encode('<{}>'.format(key))) - lines.append(to_xml(value, indent=indent+b' ')) - lines.append(indent + encode(''.format(key))) - return b'\n'.join(lines) + lines.append(indent + encode("<{}>".format(key))) + lines.append(to_xml(value, indent=indent + b" ")) + lines.append(indent + encode("".format(key))) + return b"\n".join(lines) else: raise TypeError(data) - return b'\n'.join([ - '<{} xmlns="{}">'.format(root, xmlns).encode(), - to_xml(data), - ''.format(root).encode(), - ]) + return b"\n".join( + [ + '<{} xmlns="{}">'.format(root, xmlns).encode(), + to_xml(data), + "".format(root).encode(), + ] + ) class TreeToXmlTest(unittest.TestCase): def test_leaves(self): self.assertEqual( - _tree_to_xml('root', 'http://example.com', None), - b'\n\n' + _tree_to_xml("root", "http://example.com", None), + b'\n\n', ) self.assertEqual( - _tree_to_xml('root', 'http://example.com', True), - b'\n True\n' + _tree_to_xml("root", "http://example.com", True), + b'\n True\n', ) self.assertEqual( - _tree_to_xml('root', 'http://example.com', 'abc'), - b'\n abc\n' + _tree_to_xml("root", "http://example.com", "abc"), + b'\n abc\n', ) self.assertEqual( - _tree_to_xml('root', 'http://example.com', 42), - b'\n 42\n' + _tree_to_xml("root", "http://example.com", 42), + b'\n 42\n', ) self.assertEqual( - _tree_to_xml('root', 'http://example.com', 3.14), - b'\n 3.14\n' + _tree_to_xml("root", "http://example.com", 3.14), + b'\n 3.14\n', ) def test_dict(self): self.assertIn( - _tree_to_xml('root', 'http://example.com', { - 'foo': 'bar', - 'baz': 'qux' - }), + _tree_to_xml("root", "http://example.com", {"foo": "bar", "baz": "qux"}), [ b'\n' - b' \n bar\n \n' - b' \n qux\n \n' - b'', + b" \n bar\n \n" + b" \n qux\n \n" + b"", b'\n' - b' \n qux\n \n' - b' \n bar\n \n' - b'' - ] + b" \n qux\n \n" + b" \n bar\n \n" + b"", + ], ) def test_list(self): self.assertEqual( - _tree_to_xml('root', 'http://example.com', [ - {'foo': 'bar'}, - {'foo': 'baz'}, - ]), + _tree_to_xml( + "root", "http://example.com", [{"foo": "bar"}, {"foo": "baz"},] + ), b'\n' - b' \n bar\n \n' - b' \n baz\n \n' - b'' + b" \n bar\n \n" + b" \n baz\n \n" + b"", ) def xml_document_strategy(keys, root, xmlns): """Generates an hypothesis strategy that generates metadata files for an XML format that uses the given keys.""" return strategies.builds( - functools.partial(_tree_to_xml, root, xmlns), - json_document_strategy(keys)) + functools.partial(_tree_to_xml, root, xmlns), json_document_strategy(keys) + ) def filter_dict(d, keys): - 'return a copy of the dict with keys deleted' + "return a copy of the dict with keys deleted" if not isinstance(keys, (list, tuple)): - keys = (keys, ) + keys = (keys,) return dict((k, v) for (k, v) in d.items() if k not in keys) def fill_obj_storage(obj_storage): """Add some content in an object storage.""" for (obj_id, content) in OBJ_STORAGE_DATA.items(): obj_storage.add(content, obj_id=hash_to_bytes(obj_id)) def fill_storage(storage): visit_types = {} for visit in ORIGIN_VISITS: - storage.origin_add_one({'url': visit['url']}) - visit_types[visit['url']] = visit['type'] + storage.origin_add_one({"url": visit["url"]}) + visit_types[visit["url"]] = visit["type"] for snap in SNAPSHOTS: - origin_url = snap['origin'] + origin_url = snap["origin"] visit = storage.origin_visit_add( - origin_url, - date=datetime.datetime.now(), - type=visit_types[origin_url]) - snap_id = snap.get('id') or \ - bytes([random.randint(0, 255) for _ in range(32)]) - storage.snapshot_add([{ - 'id': snap_id, - 'branches': snap['branches'] - }]) + origin_url, date=datetime.datetime.now(), type=visit_types[origin_url] + ) + snap_id = snap.get("id") or bytes([random.randint(0, 255) for _ in range(32)]) + storage.snapshot_add([{"id": snap_id, "branches": snap["branches"]}]) storage.origin_visit_update( - origin_url, visit.visit, status='full', snapshot=snap_id) + origin_url, visit.visit, status="full", snapshot=snap_id + ) storage.revision_add(REVISIONS) contents = [] for (obj_id, content) in OBJ_STORAGE_DATA.items(): content_hashes = hashutil.MultiHash.from_data(content).digest() - contents.append({ - 'data': content, - 'length': len(content), - 'status': 'visible', - 'sha1': hash_to_bytes(obj_id), - 'sha1_git': hash_to_bytes(obj_id), - 'sha256': content_hashes['sha256'], - 'blake2s256': content_hashes['blake2s256'] - }) + contents.append( + { + "data": content, + "length": len(content), + "status": "visible", + "sha1": hash_to_bytes(obj_id), + "sha1_git": hash_to_bytes(obj_id), + "sha256": content_hashes["sha256"], + "blake2s256": content_hashes["blake2s256"], + } + ) storage.content_add(contents) - storage.directory_add([{ - 'id': DIRECTORY_ID, - 'entries': DIRECTORY_ENTRIES, - }]) + storage.directory_add([{"id": DIRECTORY_ID, "entries": DIRECTORY_ENTRIES,}]) class CommonContentIndexerTest(metaclass=abc.ABCMeta): legacy_get_format = False """True if and only if the tested indexer uses the legacy format. see: https://forge.softwareheritage.org/T1433 """ + def get_indexer_results(self, ids): """Override this for indexers that don't have a mock storage.""" return self.indexer.idx_storage.state def assert_legacy_results_ok(self, sha1s, expected_results=None): # XXX old format, remove this when all endpoints are # updated to the new one # see: https://forge.softwareheritage.org/T1433 - sha1s = [sha1 if isinstance(sha1, bytes) else hash_to_bytes(sha1) - for sha1 in sha1s] + sha1s = [ + sha1 if isinstance(sha1, bytes) else hash_to_bytes(sha1) for sha1 in sha1s + ] actual_results = list(self.get_indexer_results(sha1s)) if expected_results is None: expected_results = self.expected_results - self.assertEqual(len(expected_results), len(actual_results), - (expected_results, actual_results)) + self.assertEqual( + len(expected_results), + len(actual_results), + (expected_results, actual_results), + ) for indexed_data in actual_results: - _id = indexed_data['id'] + _id = indexed_data["id"] expected_data = expected_results[hashutil.hash_to_hex(_id)].copy() - expected_data['id'] = _id + expected_data["id"] = _id self.assertEqual(indexed_data, expected_data) def assert_results_ok(self, sha1s, expected_results=None): if self.legacy_get_format: self.assert_legacy_results_ok(sha1s, expected_results) return - sha1s = [sha1 if isinstance(sha1, bytes) else hash_to_bytes(sha1) - for sha1 in sha1s] + sha1s = [ + sha1 if isinstance(sha1, bytes) else hash_to_bytes(sha1) for sha1 in sha1s + ] actual_results = list(self.get_indexer_results(sha1s)) if expected_results is None: expected_results = self.expected_results - self.assertEqual(len(expected_results), len(actual_results), - (expected_results, actual_results)) + self.assertEqual( + len(expected_results), + len(actual_results), + (expected_results, actual_results), + ) for indexed_data in actual_results: (_id, indexed_data) = list(indexed_data.items())[0] expected_data = expected_results[hashutil.hash_to_hex(_id)].copy() expected_data = [expected_data] self.assertEqual(indexed_data, expected_data) def test_index(self): """Known sha1 have their data indexed """ sha1s = [self.id0, self.id1, self.id2] # when - self.indexer.run(sha1s, policy_update='update-dups') + self.indexer.run(sha1s, policy_update="update-dups") self.assert_results_ok(sha1s) # 2nd pass - self.indexer.run(sha1s, policy_update='ignore-dups') + self.indexer.run(sha1s, policy_update="ignore-dups") self.assert_results_ok(sha1s) def test_index_one_unknown_sha1(self): """Unknown sha1 are not indexed""" - sha1s = [self.id1, - '799a5ef812c53907562fe379d4b3851e69c7cb15', # unknown - '800a5ef812c53907562fe379d4b3851e69c7cb15'] # unknown + sha1s = [ + self.id1, + "799a5ef812c53907562fe379d4b3851e69c7cb15", # unknown + "800a5ef812c53907562fe379d4b3851e69c7cb15", + ] # unknown # when - self.indexer.run(sha1s, policy_update='update-dups') + self.indexer.run(sha1s, policy_update="update-dups") # then expected_results = { k: v for k, v in self.expected_results.items() if k in sha1s } self.assert_results_ok(sha1s, expected_results) class CommonContentIndexerRangeTest: """Allows to factorize tests on range indexer. """ + def setUp(self): self.contents = sorted(OBJ_STORAGE_DATA) - def assert_results_ok(self, start, end, actual_results, - expected_results=None): + def assert_results_ok(self, start, end, actual_results, expected_results=None): if expected_results is None: expected_results = self.expected_results actual_results = list(actual_results) for indexed_data in actual_results: - _id = indexed_data['id'] + _id = indexed_data["id"] assert isinstance(_id, bytes) indexed_data = indexed_data.copy() - indexed_data['id'] = hash_to_hex(indexed_data['id']) + indexed_data["id"] = hash_to_hex(indexed_data["id"]) self.assertEqual(indexed_data, expected_results[hash_to_hex(_id)]) self.assertTrue(start <= _id <= end) - _tool_id = indexed_data['indexer_configuration_id'] - self.assertEqual(_tool_id, self.indexer.tool['id']) + _tool_id = indexed_data["indexer_configuration_id"] + self.assertEqual(_tool_id, self.indexer.tool["id"]) def test__index_contents(self): """Indexing contents without existing data results in indexed data """ _start, _end = [self.contents[0], self.contents[2]] # output hex ids start, end = map(hashutil.hash_to_bytes, (_start, _end)) # given - actual_results = list(self.indexer._index_contents( - start, end, indexed={})) + actual_results = list(self.indexer._index_contents(start, end, indexed={})) self.assert_results_ok(start, end, actual_results) def test__index_contents_with_indexed_data(self): """Indexing contents with existing data results in less indexed data """ _start, _end = [self.contents[0], self.contents[2]] # output hex ids start, end = map(hashutil.hash_to_bytes, (_start, _end)) data_indexed = [self.id0, self.id2] # given actual_results = self.indexer._index_contents( - start, end, indexed=set(map(hash_to_bytes, data_indexed))) + start, end, indexed=set(map(hash_to_bytes, data_indexed)) + ) # craft the expected results expected_results = self.expected_results.copy() for already_indexed_key in data_indexed: expected_results.pop(already_indexed_key) - self.assert_results_ok( - start, end, actual_results, expected_results) + self.assert_results_ok(start, end, actual_results, expected_results) def test_generate_content_get(self): """Optimal indexing should result in indexed data """ _start, _end = [self.contents[0], self.contents[2]] # output hex ids start, end = map(hashutil.hash_to_bytes, (_start, _end)) # given actual_results = self.indexer.run(start, end) # then - self.assertEqual(actual_results, {'status': 'uneventful'}) + self.assertEqual(actual_results, {"status": "uneventful"}) def test_generate_content_get_input_as_bytes(self): """Optimal indexing should result in indexed data Input are in bytes here. """ _start, _end = [self.contents[0], self.contents[2]] # output hex ids start, end = map(hashutil.hash_to_bytes, (_start, _end)) # given - actual_results = self.indexer.run( - start, end, skip_existing=False) + actual_results = self.indexer.run(start, end, skip_existing=False) # no already indexed data so same result as prior test # then - self.assertEqual(actual_results, {'status': 'uneventful'}) + self.assertEqual(actual_results, {"status": "uneventful"}) def test_generate_content_get_no_result(self): """No result indexed returns False""" - _start, _end = ['0000000000000000000000000000000000000000', - '0000000000000000000000000000000000000001'] + _start, _end = [ + "0000000000000000000000000000000000000000", + "0000000000000000000000000000000000000001", + ] start, end = map(hashutil.hash_to_bytes, (_start, _end)) # given actual_results = self.indexer.run(start, end, incremental=False) # then - self.assertEqual(actual_results, {'status': 'uneventful'}) + self.assertEqual(actual_results, {"status": "uneventful"}) diff --git a/tox.ini b/tox.ini index 7a30478..6bc37bb 100644 --- a/tox.ini +++ b/tox.ini @@ -1,32 +1,39 @@ [tox] -envlist=flake8,mypy,py3 +envlist=black,flake8,mypy,py3 [testenv] extras = testing deps = pytest-cov pifpaf dev: pdbpp commands = pifpaf run postgresql -- pytest --doctest-modules \ !slow: --hypothesis-profile=fast \ slow: --hypothesis-profile=slow \ {envsitepackagesdir}/swh/indexer \ --cov={envsitepackagesdir}/swh/indexer \ --cov-branch {posargs} +[testenv:black] +skip_install = true +deps = + black +commands = + {envpython} -m black --check swh + [testenv:flake8] skip_install = true deps = flake8 commands = {envpython} -m flake8 [testenv:mypy] extras = testing deps = mypy commands = mypy swh