diff --git a/README-dev.md b/README-dev.md index 8fcd4291..0ff05315 100644 --- a/README-dev.md +++ b/README-dev.md @@ -1,59 +1,57 @@ README-dev ========== # Run server Either use the django manage script directly (useful in development mode as it offers various commands). The configuration will be taken from the default configuration file: '~/.config/swh/webapp.yml'. ``` python3 -m swh.web.manage runserver ``` or use the following command: ``` ./bin/swh-web-dev --config ``` # modules' description ## Layers Folder swh/web/api/: - views main api endpoints definitions (html browsable + json + yaml) - service Orchestration layer used by views module. - In charge of communication with `backend` to retrieve + In charge of communication with swh storage to retrieve information and conversion for the upper layer. -- backend Lower layer in charge of communication with swh storage. - Used by `service` module. In short: -1. views -depends-> service -depends-> backend ----asks----> swh-storage -2. views <- service <- backend <----rets---- swh-storage +1. views -depends-> service ----asks----> swh-storage +2. views <- service <----rets---- swh-storage ## Utilities Folder swh/web/api/: - apidoc Browsable api functions. - apiresponse Api response utility functions - apiurls Api routes registration functions - exc Exception definitions - converters conversion layer to transform swh data to serializable data. Used by `service` to convert data before transmitting to `api` or `views`. - query Utilities to parse data from http endpoints. Used by `service` - utils Utilities used throughout swh-web-api. ### About apidoc This is a 'decorator tower' that stores the data associated with the documentation upon loading the apidoc module. The top decorator of any tower should be @apidoc.route(). Apidoc raises an exception if this decorator is missing, and flask raises an exception if it is present but not at the top of the tower. ## Graphics summary ![Summary dependencies](./docs/dependencies.png) diff --git a/swh/web/api/backend.py b/swh/web/api/backend.py deleted file mode 100644 index fbafc7e0..00000000 --- a/swh/web/api/backend.py +++ /dev/null @@ -1,357 +0,0 @@ -# Copyright (C) 2015-2016 The Software Heritage developers -# See the AUTHORS file at the top-level directory of this distribution -# License: GNU Affero General Public License version 3, or any later version -# See top-level LICENSE file for more information - -import os - -from swh.web import config - -MAX_LIMIT = 50 # Top limit the users can ask for - - -def content_get(sha1_bin): - """Lookup the content designed by {algo: hash_bin}. - - Args: - sha1_bin: content's binary sha1. - - Returns: - Content as dict with 'sha1' and 'data' keys. - data representing its raw data. - - """ - contents = config.storage().content_get([sha1_bin]) - if contents and len(contents) >= 1: - return contents[0] - return None - - -def content_find(algo, hash_bin): - """Retrieve the content with binary hash hash_bin - - Args: - algo: nature of the hash hash_bin. - hash_bin: content's hash searched for. - - Returns: - A dict with keys sha1, sha1_git, sha256, ... if the content - exist or None otherwise. - - """ - return config.storage().content_find({algo: hash_bin}) - - -def content_find_provenance(algo, hash_bin): - """Find the content's provenance information. - - Args: - algo: nature of the hash hash_bin. - hash_bin: content's hash corresponding to algo searched for. - - Yields: - Yields the list of provenance information for that content if - any (this can be empty if the cache is not populated) - - """ - return config.storage().content_find_provenance({algo: hash_bin}) - - -def content_ctags_get(id): - """Retrieve the list of ctags symbols for a specific sha1. - - Args: - id (bytes): content's hash identifier - - Returns: - dict of keys 'id' (bytes) and 'ctags' (list of dict) - - """ - return list(config.storage().content_ctags_get([id])) - - -def content_ctags_search(expression, last_sha1, limit=10): - """Lookup the content designed by {algo: hash_bin}. - - Args: - expression (str): Expression to lookup in indexed raw content - last_sha1 (str): Last hash - limit (int): Number of elements per page - - Returns: - sha1 whose indexed content match the expression - - """ - limit = min(limit, MAX_LIMIT) - return config.storage().content_ctags_search(expression, - last_sha1=last_sha1, - limit=limit) - - -def content_filetype_get(id): - """Retrieve content's filetype information. - - """ - r = list(config.storage().content_mimetype_get([id])) - if not r: - return None - return r[0] - - -def content_language_get(id): - """Retrieve content's language information. - - """ - r = list(config.storage().content_language_get([id])) - if not r: - return None - return r[0] - - -def content_license_get(id): - """Retrieve content's license information. - - """ - r = list(config.storage().content_fossology_license_get([id])) - if not r: - return None - return r[0] - - -def content_missing_per_sha1(sha1list): - """List content missing from storage based on sha1 - - Args: - sha1s: Iterable of sha1 to check for absence - Returns: - an iterable of sha1s missing from the storage - """ - return config.storage().content_missing_per_sha1(sha1list) - - -def directory_get(sha1_bin): - """Retrieve information on one directory. - - Args: - sha1_bin: Directory's identifier - - Returns: - The directory's information. - - """ - res = config.storage().directory_get([sha1_bin]) - if res and len(res) >= 1: - return res[0] - - -def origin_get(origin): - """Return information about the origin matching dict origin. - - Args: - origin: origin's dict with keys either 'id' or - ('type' AND 'url') - - Returns: - Origin information as dict. - - """ - return config.storage().origin_get(origin) - - -def person_get(person_id): - """Return information about the person with id person_id. - - Args: - person_id: person's identifier. - - Returns: - Person information as dict. - - """ - res = config.storage().person_get([person_id]) - if res and len(res) >= 1: - return res[0] - - -def directory_ls(sha1_git_bin, recursive=False): - """Return information about the directory with id sha1_git. - - Args: - sha1_git: directory's identifier. - recursive: Optional recursive flag default to False - - Returns: - Directory information as dict. - - """ - directory_entries = config.storage().directory_ls(sha1_git_bin, recursive) - if not directory_entries: - return [] - - return directory_entries - - -def release_get(sha1_git_bin): - """Return information about the release with sha1 sha1_git_bin. - - Args: - sha1_git_bin: The release's sha1 as bytes. - - Returns: - Release information as dict if found, None otherwise. - - Raises: - ValueError if the identifier provided is not of sha1 nature. - - """ - - res = config.storage().release_get([sha1_git_bin]) - - if res and len(res) >= 1: - return res[0] - return None - - -def revision_get(sha1_git_bin): - """Return information about the revision with sha1 sha1_git_bin. - - Args: - sha1_git_bin: The revision's sha1 as bytes. - - Returns: - Revision information as dict if found, None otherwise. - - Raises: - ValueError if the identifier provided is not of sha1 nature. - - """ - res = config.storage().revision_get([sha1_git_bin]) - if res and len(res) >= 1: - return res[0] - return None - - -def revision_get_multiple(sha1_git_bin_list): - """Return information about the revisions in sha1_git_bin_list - - Args: - sha1_git_bin_list: The revisions' sha1s as a list of bytes. - - Returns: - Revisions' information as an iterable of dicts if any found, - an empty list otherwise - - Raises: - ValueError if the identifier provided is not of sha1 nature. - """ - res = config.storage().revision_get(sha1_git_bin_list) - if res and len(res) >= 1: - return res - return [] - - -def revision_log(sha1_git_bin, limit): - """Return information about the revision with sha1 sha1_git_bin. - - Args: - sha1_git_bin: The revision's sha1 as bytes. - limit: the maximum number of revisions returned. - - Returns: - Revision information as dict if found, None otherwise. - - Raises: - ValueError if the identifier provided is not of sha1 nature. - - """ - return config.storage().revision_log([sha1_git_bin], limit) - - -def revision_log_by(origin_id, branch_name, ts, limit): - """Return information about the revision matching the timestamp - ts, from origin origin_id, in branch branch_name. - - Args: - origin_id: origin of the revision - - branch_name: revision's branch. - - timestamp: revision's time frame. - - Returns: - Information for the revision matching the criterions. - - """ - return config.storage().revision_log_by(origin_id, - branch_name, - ts, - limit=limit) - - -def stat_counters(): - """Return the stat counters for Software Heritage - - Returns: - A dict mapping textual labels to integer values. - """ - return config.storage().stat_counters() - - -def lookup_origin_visits(origin_id, last_visit=None, limit=10): - """Yields the origin origin_ids' visits. - - Args: - origin_id (int): origin to list visits for - last_visit (int): last visit to lookup from - limit (int): Number of elements max to display - - Yields: - Dictionaries of origin_visit for that origin - - """ - limit = min(limit, MAX_LIMIT) - yield from config.storage().origin_visit_get( - origin_id, last_visit=last_visit, limit=limit) - - -def lookup_origin_visit(origin_id, visit_id): - """Return information about visit visit_id with origin origin_id. - - Args: - origin_id: origin concerned by the visit - visit_id: the visit identifier to lookup - - Yields: - The dict origin_visit concerned - - """ - return config.storage().origin_visit_get_by(origin_id, visit_id) - - -def revision_get_by(origin_id, branch_name, timestamp): - """Return occurrence information matching the criterions origin_id, - branch_name, ts. - - """ - res = config.storage().revision_get_by(origin_id, - branch_name, - timestamp=timestamp, - limit=1) - if not res: - return None - return res[0] - - -def directory_entry_get_by_path(directory, path): - """Return a directory entry by its path. - - """ - paths = path.strip(os.path.sep).split(os.path.sep) - return config.storage().directory_entry_get_by_path( - directory, - list(map(lambda p: p.encode('utf-8'), paths))) - - -def entity_get(uuid): - """Retrieve the entity per its uuid. - - """ - return config.storage().entity_get(uuid) diff --git a/swh/web/api/service.py b/swh/web/api/service.py index a446b312..21957ae9 100644 --- a/swh/web/api/service.py +++ b/swh/web/api/service.py @@ -1,771 +1,815 @@ # Copyright (C) 2015-2017 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information +import os + from collections import defaultdict from swh.model import hashutil from swh.web.api import converters -from swh.web.api import query, backend +from swh.web.api import query from swh.web.api.exc import NotFoundExc +from swh.web import config + +storage = config.storage() + +MAX_LIMIT = 50 # Top limit the users can ask for + + +def _first_element(l): + """Returns the first element in the provided list or None + if it is empty or None""" + return next(iter(l or []), None) def lookup_multiple_hashes(hashes): """Lookup the passed hashes in a single DB connection, using batch processing. Args: An array of {filename: X, sha1: Y}, string X, hex sha1 string Y. Returns: The same array with elements updated with elem['found'] = true if the hash is present in storage, elem['found'] = false if not. """ hashlist = [hashutil.hash_to_bytes(elem['sha1']) for elem in hashes] - content_missing = backend.content_missing_per_sha1(hashlist) + content_missing = storage.content_missing_per_sha1(hashlist) missing = [hashutil.hash_to_hex(x) for x in content_missing] for x in hashes: x.update({'found': True}) for h in hashes: if h['sha1'] in missing: h['found'] = False return hashes def lookup_expression(expression, last_sha1, per_page): """Lookup expression in raw content. Args: expression (str): An expression to lookup through raw indexed content last_sha1 (str): Last sha1 seen per_page (int): Number of results per page Returns: List of ctags whose content match the expression """ - for ctag in backend.content_ctags_search(expression, last_sha1, per_page): + + limit = min(per_page, MAX_LIMIT) + ctags = storage.content_ctags_search(expression, + last_sha1=last_sha1, + limit=limit) + + for ctag in ctags: ctag = converters.from_swh(ctag, hashess={'id'}) ctag['sha1'] = ctag['id'] ctag.pop('id') yield ctag def lookup_hash(q): """Checks if the storage contains a given content checksum Args: query string of the form Returns: Dict with key found containing the hash info if the hash is present, None if not. """ algo, hash = query.parse_hash(q) - found = backend.content_find(algo, hash) + found = storage.content_find({algo: hash}) return {'found': found, 'algo': algo} def search_hash(q): """Checks if the storage contains a given content checksum Args: query string of the form Returns: Dict with key found to True or False, according to whether the checksum is present or not """ algo, hash = query.parse_hash(q) - found = backend.content_find(algo, hash) + found = storage.content_find({algo: hash}) return {'found': found is not None} def lookup_content_provenance(q): """Return provenance information from a specified content. Args: q: query string of the form Yields: provenance information (dict) list if the content is found. """ algo, hash = query.parse_hash(q) - provenances = backend.content_find_provenance(algo, hash) + provenances = storage.content_find_provenance({algo: hash}) if not provenances: return None return (converters.from_provenance(p) for p in provenances) def _lookup_content_sha1(q): """Given a possible input, query for the content's sha1. Args: q: query string of the form Returns: binary sha1 if found or None """ algo, hash = query.parse_hash(q) if algo != 'sha1': - hashes = backend.content_find(algo, hash) + hashes = storage.content_find({algo: hash}) if not hashes: return None return hashes['sha1'] return hash def lookup_content_ctags(q): """Return ctags information from a specified content. Args: q: query string of the form Yields: ctags information (dict) list if the content is found. """ sha1 = _lookup_content_sha1(q) + if not sha1: return None - ctags = backend.content_ctags_get(sha1) + ctags = list(storage.content_ctags_get([sha1])) if not ctags: return None for ctag in ctags: yield converters.from_swh(ctag, hashess={'id'}) def lookup_content_filetype(q): """Return filetype information from a specified content. Args: q: query string of the form Yields: filetype information (dict) list if the content is found. """ sha1 = _lookup_content_sha1(q) if not sha1: return None - filetype = backend.content_filetype_get(sha1) + filetype = _first_element(list(storage.content_mimetype_get([sha1]))) if not filetype: return None return converters.from_filetype(filetype) def lookup_content_language(q): """Return language information from a specified content. Args: q: query string of the form Yields: language information (dict) list if the content is found. """ sha1 = _lookup_content_sha1(q) if not sha1: return None - lang = backend.content_language_get(sha1) + lang = _first_element(list(storage.content_language_get([sha1]))) if not lang: return None return converters.from_swh(lang, hashess={'id'}) def lookup_content_license(q): """Return license information from a specified content. Args: q: query string of the form Yields: license information (dict) list if the content is found. """ sha1 = _lookup_content_sha1(q) if not sha1: return None - lang = backend.content_license_get(sha1) + lang = _first_element(storage.content_fossology_license_get([sha1])) if not lang: return None return converters.from_swh(lang, hashess={'id'}) def lookup_origin(origin): """Return information about the origin matching dict origin. Args: origin: origin's dict with keys either 'id' or ('type' AND 'url') Returns: origin information as dict. """ - return converters.from_origin(backend.origin_get(origin)) + return converters.from_origin(storage.origin_get(origin)) def lookup_person(person_id): """Return information about the person with id person_id. Args: person_id as string Returns: person information as dict. """ - person = backend.person_get(person_id) + person = _first_element(storage.person_get([person_id])) return converters.from_person(person) def lookup_directory(sha1_git): """Return information about the directory with id sha1_git. Args: sha1_git as string Returns: directory information as dict. """ _, sha1_git_bin = query.parse_hash_with_algorithms_or_throws( sha1_git, ['sha1'], # HACK: sha1_git really 'Only sha1_git is supported.') - dir = backend.directory_get(sha1_git_bin) + dir = _first_element(storage.directory_get([sha1_git_bin])) if not dir: return None - directory_entries = backend.directory_ls(sha1_git_bin) + directory_entries = storage.directory_ls(sha1_git_bin) or [] return map(converters.from_directory_entry, directory_entries) def lookup_directory_with_path(directory_sha1_git, path_string): """Return directory information for entry with path path_string w.r.t. root directory pointed by directory_sha1_git Args: - directory_sha1_git: sha1_git corresponding to the directory to which we append paths to (hopefully) find the entry - the relative path to the entry starting from the directory pointed by directory_sha1_git Raises: NotFoundExc if the directory entry is not found """ _, sha1_git_bin = query.parse_hash_with_algorithms_or_throws( directory_sha1_git, ['sha1'], 'Only sha1_git is supported.') - queried_dir = backend.directory_entry_get_by_path( - sha1_git_bin, path_string) + paths = path_string.strip(os.path.sep).split(os.path.sep) + queried_dir = storage.directory_entry_get_by_path( + sha1_git_bin, list(map(lambda p: p.encode('utf-8'), paths))) if not queried_dir: raise NotFoundExc(('Directory entry with path %s from %s not found') % (path_string, directory_sha1_git)) return converters.from_directory_entry(queried_dir) def lookup_release(release_sha1_git): """Return information about the release with sha1 release_sha1_git. Args: release_sha1_git: The release's sha1 as hexadecimal Returns: Release information as dict. Raises: ValueError if the identifier provided is not of sha1 nature. """ _, sha1_git_bin = query.parse_hash_with_algorithms_or_throws( release_sha1_git, ['sha1'], 'Only sha1_git is supported.') - res = backend.release_get(sha1_git_bin) + res = _first_element(storage.release_get([sha1_git_bin])) return converters.from_release(res) def lookup_revision(rev_sha1_git): """Return information about the revision with sha1 revision_sha1_git. Args: revision_sha1_git: The revision's sha1 as hexadecimal Returns: Revision information as dict. Raises: ValueError if the identifier provided is not of sha1 nature. """ _, sha1_git_bin = query.parse_hash_with_algorithms_or_throws( rev_sha1_git, ['sha1'], 'Only sha1_git is supported.') - - revision = backend.revision_get(sha1_git_bin) + revision = _first_element(storage.revision_get([sha1_git_bin])) return converters.from_revision(revision) def lookup_revision_multiple(sha1_git_list): """Return information about the revision with sha1 revision_sha1_git. Args: revision_sha1_git: The revision's sha1 as hexadecimal Returns: Revision information as dict. Raises: ValueError if the identifier provided is not of sha1 nature. """ def to_sha1_bin(sha1_hex): _, sha1_git_bin = query.parse_hash_with_algorithms_or_throws( sha1_hex, ['sha1'], 'Only sha1_git is supported.') return sha1_git_bin sha1_bin_list = (to_sha1_bin(x) for x in sha1_git_list) - revisions = backend.revision_get_multiple(sha1_bin_list) + revisions = storage.revision_get(sha1_bin_list) or [] return (converters.from_revision(x) for x in revisions) def lookup_revision_message(rev_sha1_git): """Return the raw message of the revision with sha1 revision_sha1_git. Args: revision_sha1_git: The revision's sha1 as hexadecimal Returns: Decoded revision message as dict {'message': } Raises: ValueError if the identifier provided is not of sha1 nature. NotFoundExc if the revision is not found, or if it has no message """ _, sha1_git_bin = query.parse_hash_with_algorithms_or_throws( rev_sha1_git, ['sha1'], 'Only sha1_git is supported.') - revision = backend.revision_get(sha1_git_bin) + revision = _first_element(storage.revision_get([sha1_git_bin])) if not revision: raise NotFoundExc('Revision with sha1_git %s not found.' % rev_sha1_git) if 'message' not in revision: raise NotFoundExc('No message for revision with sha1_git %s.' % rev_sha1_git) res = {'message': revision['message']} return res def lookup_revision_by(origin_id, branch_name="refs/heads/master", timestamp=None): """Lookup revisions by origin_id, branch_name and timestamp. If: - branch_name is not provided, lookup using 'refs/heads/master' as default. - ts is not provided, use the most recent Args: - origin_id: origin of the revision. - branch_name: revision's branch. - timestamp: revision's time frame. Yields: The revisions matching the criterions. """ - res = backend.revision_get_by(origin_id, branch_name, timestamp) + res = _first_element(storage.revision_get_by(origin_id, + branch_name, + timestamp=timestamp, + limit=1)) return converters.from_revision(res) def lookup_revision_log(rev_sha1_git, limit): """Return information about the revision with sha1 revision_sha1_git. Args: revision_sha1_git: The revision's sha1 as hexadecimal limit: the maximum number of revisions returned Returns: Revision information as dict. Raises: ValueError if the identifier provided is not of sha1 nature. """ _, sha1_git_bin = query.parse_hash_with_algorithms_or_throws( rev_sha1_git, ['sha1'], 'Only sha1_git is supported.') - revision_entries = backend.revision_log(sha1_git_bin, limit) + revision_entries = storage.revision_log([sha1_git_bin], limit) return map(converters.from_revision, revision_entries) def lookup_revision_log_by(origin_id, branch_name, timestamp, limit): """Return information about the revision with sha1 revision_sha1_git. Args: origin_id: origin of the revision branch_name: revision's branch timestamp: revision's time frame limit: the maximum number of revisions returned Returns: Revision information as dict. Raises: NotFoundExc if no revision corresponds to the criterion NotFoundExc if the corresponding revision has no log """ - revision_entries = backend.revision_log_by(origin_id, + revision_entries = storage.revision_log_by(origin_id, branch_name, timestamp, - limit) + limit=limit) if not revision_entries: return None return map(converters.from_revision, revision_entries) def lookup_revision_with_context_by(origin_id, branch_name, ts, sha1_git, limit=100): """Return information about revision sha1_git, limited to the sub-graph of all transitive parents of sha1_git_root. sha1_git_root being resolved through the lookup of a revision by origin_id, branch_name and ts. In other words, sha1_git is an ancestor of sha1_git_root. Args: - origin_id: origin of the revision. - branch_name: revision's branch. - timestamp: revision's time frame. - sha1_git: one of sha1_git_root's ancestors. - limit: limit the lookup to 100 revisions back. Returns: Pair of (root_revision, revision). Information on sha1_git if it is an ancestor of sha1_git_root including children leading to sha1_git_root Raises: - BadInputExc in case of unknown algo_hash or bad hash. - NotFoundExc if either revision is not found or if sha1_git is not an ancestor of sha1_git_root. """ - rev_root = backend.revision_get_by(origin_id, branch_name, ts) + rev_root = _first_element(storage.revision_get_by(origin_id, + branch_name, + timestamp=ts, + limit=1)) if not rev_root: raise NotFoundExc('Revision with (origin_id: %s, branch_name: %s' ', ts: %s) not found.' % (origin_id, branch_name, ts)) return (converters.from_revision(rev_root), lookup_revision_with_context(rev_root, sha1_git, limit)) def lookup_revision_with_context(sha1_git_root, sha1_git, limit=100): """Return information about revision sha1_git, limited to the sub-graph of all transitive parents of sha1_git_root. In other words, sha1_git is an ancestor of sha1_git_root. Args: sha1_git_root: latest revision. The type is either a sha1 (as an hex string) or a non converted dict. sha1_git: one of sha1_git_root's ancestors limit: limit the lookup to 100 revisions back Returns: Information on sha1_git if it is an ancestor of sha1_git_root including children leading to sha1_git_root Raises: BadInputExc in case of unknown algo_hash or bad hash NotFoundExc if either revision is not found or if sha1_git is not an ancestor of sha1_git_root """ _, sha1_git_bin = query.parse_hash_with_algorithms_or_throws( sha1_git, ['sha1'], 'Only sha1_git is supported.') - revision = backend.revision_get(sha1_git_bin) + revision = _first_element(storage.revision_get([sha1_git_bin])) if not revision: raise NotFoundExc('Revision %s not found' % sha1_git) if isinstance(sha1_git_root, str): _, sha1_git_root_bin = query.parse_hash_with_algorithms_or_throws( sha1_git_root, ['sha1'], 'Only sha1_git is supported.') - revision_root = backend.revision_get(sha1_git_root_bin) + revision_root = _first_element(storage.revision_get([sha1_git_root_bin])) # noqa if not revision_root: raise NotFoundExc('Revision root %s not found' % sha1_git_root) else: sha1_git_root_bin = sha1_git_root['id'] - revision_log = backend.revision_log(sha1_git_root_bin, limit) + revision_log = storage.revision_log([sha1_git_root_bin], limit) parents = {} children = defaultdict(list) for rev in revision_log: rev_id = rev['id'] parents[rev_id] = [] for parent_id in rev['parents']: parents[rev_id].append(parent_id) children[parent_id].append(rev_id) if revision['id'] not in parents: raise NotFoundExc('Revision %s is not an ancestor of %s' % (sha1_git, sha1_git_root)) revision['children'] = children[revision['id']] return converters.from_revision(revision) def lookup_directory_with_revision(sha1_git, dir_path=None, with_data=False): """Return information on directory pointed by revision with sha1_git. If dir_path is not provided, display top level directory. Otherwise, display the directory pointed by dir_path (if it exists). Args: sha1_git: revision's hash. dir_path: optional directory pointed to by that revision. with_data: boolean that indicates to retrieve the raw data if the path resolves to a content. Default to False (for the api) Returns: Information on the directory pointed to by that revision. Raises: BadInputExc in case of unknown algo_hash or bad hash. NotFoundExc either if the revision is not found or the path referenced does not exist. NotImplementedError in case of dir_path exists but do not reference a type 'dir' or 'file'. """ _, sha1_git_bin = query.parse_hash_with_algorithms_or_throws( sha1_git, ['sha1'], 'Only sha1_git is supported.') - revision = backend.revision_get(sha1_git_bin) + revision = _first_element(storage.revision_get([sha1_git_bin])) if not revision: raise NotFoundExc('Revision %s not found' % sha1_git) dir_sha1_git_bin = revision['directory'] if dir_path: - entity = backend.directory_entry_get_by_path(dir_sha1_git_bin, - dir_path) + paths = dir_path.strip(os.path.sep).split(os.path.sep) + entity = storage.directory_entry_get_by_path( + dir_sha1_git_bin, list(map(lambda p: p.encode('utf-8'), paths))) if not entity: raise NotFoundExc( "Directory or File '%s' pointed to by revision %s not found" % (dir_path, sha1_git)) else: entity = {'type': 'dir', 'target': dir_sha1_git_bin} if entity['type'] == 'dir': - directory_entries = backend.directory_ls(entity['target']) + directory_entries = storage.directory_ls(entity['target']) or [] return {'type': 'dir', 'path': '.' if not dir_path else dir_path, 'revision': sha1_git, 'content': map(converters.from_directory_entry, directory_entries)} elif entity['type'] == 'file': # content - content = backend.content_find('sha1_git', entity['target']) + content = storage.content_find({'sha1_git': entity['target']}) if with_data: - content['data'] = backend.content_get(content['sha1'])['data'] + c = _first_element(storage.content_get([content['sha1']])) + content['data'] = c['data'] return {'type': 'file', 'path': '.' if not dir_path else dir_path, 'revision': sha1_git, 'content': converters.from_content(content)} else: raise NotImplementedError('Entity of type %s not implemented.' % entity['type']) def lookup_content(q): """Lookup the content designed by q. Args: q: The release's sha1 as hexadecimal """ algo, hash = query.parse_hash(q) - c = backend.content_find(algo, hash) + c = storage.content_find({algo: hash}) return converters.from_content(c) def lookup_content_raw(q): """Lookup the content defined by q. Args: q: query string of the form Returns: dict with 'sha1' and 'data' keys. data representing its raw data decoded. """ algo, hash = query.parse_hash(q) - c = backend.content_find(algo, hash) + c = storage.content_find({algo: hash}) if not c: return None - - content = backend.content_get(c['sha1']) + content = _first_element(storage.content_get([c['sha1']])) return converters.from_content(content) def stat_counters(): """Return the stat counters for Software Heritage Returns: A dict mapping textual labels to integer values. """ - return backend.stat_counters() + return storage.stat_counters() + + +def _lookup_origin_visits(origin_id, last_visit=None, limit=10): + """Yields the origin origin_ids' visits. + + Args: + origin_id (int): origin to list visits for + last_visit (int): last visit to lookup from + limit (int): Number of elements max to display + + Yields: + Dictionaries of origin_visit for that origin + + """ + limit = min(limit, MAX_LIMIT) + yield from storage.origin_visit_get( + origin_id, last_visit=last_visit, limit=limit) def lookup_origin_visits(origin_id, last_visit=None, per_page=10): """Yields the origin origin_ids' visits. Args: origin_id: origin to list visits for Yields: Dictionaries of origin_visit for that origin """ - visits = backend.lookup_origin_visits( - origin_id, last_visit=last_visit, limit=per_page) + visits = _lookup_origin_visits(origin_id, last_visit=last_visit, + limit=per_page) for visit in visits: yield converters.from_origin_visit(visit) def lookup_origin_visit(origin_id, visit_id): """Return information about visit visit_id with origin origin_id. Args: origin_id: origin concerned by the visit visit_id: the visit identifier to lookup Yields: The dict origin_visit concerned """ - visit = backend.lookup_origin_visit(origin_id, visit_id) + visit = storage.origin_visit_get_by(origin_id, visit_id) return converters.from_origin_visit(visit) def lookup_entity_by_uuid(uuid): """Return the entity's hierarchy from its uuid. Args: uuid: entity's identifier. Returns: List of hierarchy entities from the entity with uuid. """ uuid = query.parse_uuid4(uuid) - for entity in backend.entity_get(uuid): + for entity in storage.entity_get(uuid): entity = converters.from_swh(entity, convert={'last_seen', 'uuid'}, convert_fn=lambda x: str(x)) yield entity def lookup_revision_through(revision, limit=100): """Retrieve a revision from the criterion stored in revision dictionary. Args: revision: Dictionary of criterion to lookup the revision with. Here are the supported combination of possible values: - origin_id, branch_name, ts, sha1_git - origin_id, branch_name, ts - sha1_git_root, sha1_git - sha1_git Returns: None if the revision is not found or the actual revision. """ if 'origin_id' in revision and \ 'branch_name' in revision and \ 'ts' in revision and \ 'sha1_git' in revision: return lookup_revision_with_context_by(revision['origin_id'], revision['branch_name'], revision['ts'], revision['sha1_git'], limit) if 'origin_id' in revision and \ 'branch_name' in revision and \ 'ts' in revision: return lookup_revision_by(revision['origin_id'], revision['branch_name'], revision['ts']) if 'sha1_git_root' in revision and \ 'sha1_git' in revision: return lookup_revision_with_context(revision['sha1_git_root'], revision['sha1_git'], limit) if 'sha1_git' in revision: return lookup_revision(revision['sha1_git']) # this should not happen raise NotImplementedError('Should not happen!') def lookup_directory_through_revision(revision, path=None, limit=100, with_data=False): """Retrieve the directory information from the revision. Args: revision: dictionary of criterion representing a revision to lookup path: directory's path to lookup. limit: optional query parameter to limit the revisions log (default to 100). For now, note that this limit could impede the transitivity conclusion about sha1_git not being an ancestor of. with_data: indicate to retrieve the content's raw data if path resolves to a content. Returns: The directory pointing to by the revision criterions at path. """ rev = lookup_revision_through(revision, limit) if not rev: raise NotFoundExc('Revision with criterion %s not found!' % revision) return (rev['id'], lookup_directory_with_revision(rev['id'], path, with_data)) diff --git a/swh/web/tests/api/test_backend.py b/swh/web/tests/api/test_backend.py deleted file mode 100644 index 877be081..00000000 --- a/swh/web/tests/api/test_backend.py +++ /dev/null @@ -1,938 +0,0 @@ -# Copyright (C) 2015-2017 The Software Heritage developers -# See the AUTHORS file at the top-level directory of this distribution -# License: GNU Affero General Public License version 3, or any later version -# See top-level LICENSE file for more information - -import datetime - -from nose.tools import istest -from unittest.mock import MagicMock - -from swh.model import hashutil - -from .swh_api_testcase import SWHApiTestCase -from swh.web.api import backend - - -class BackendTestCase(SWHApiTestCase): - - def setUp(self): - self.origin_visit1 = { - 'date': datetime.datetime( - 2015, 1, 1, 22, 0, 0, - tzinfo=datetime.timezone.utc), - 'origin': 1, - 'visit': 1 - } - - @istest - def content_get_ko_not_found_1(self): - # given - sha1_bin = hashutil.hash_to_bytes( - '456caf10e9535160d90e874b45aa426de762f777') - - self.storage().content_get = MagicMock(return_value=None) - - # when - actual_content = backend.content_get(sha1_bin) - - # then - self.assertIsNone(actual_content) - - self.storage().content_get.assert_called_once_with( - [sha1_bin]) - - @istest - def content_get_ko_not_found_empty_result(self): - # given - sha1_bin = hashutil.hash_to_bytes( - '456caf10e9535160d90e874b45aa426de762f19f') - self.storage().content_get = MagicMock(return_value=[]) - - # when - actual_content = backend.content_get(sha1_bin) - - # then - self.assertIsNone(actual_content) - - self.storage().content_get.assert_called_once_with( - [sha1_bin]) - - @istest - def content_ctags_search_1(self): - # given - self.storage().content_ctags_search = MagicMock( - return_value="some-result") - - # when - actual_ctags = backend.content_ctags_search( - 'foo', last_sha1='some-hash', limit=1) - - # then - self.assertEquals(actual_ctags, 'some-result') - self.storage().content_ctags_search.assert_called_once_with( - 'foo', last_sha1='some-hash', limit=1) - - @istest - def content_ctags_search_2(self): - # given - self.storage().content_ctags_search = MagicMock( - return_value="some other result") - - # when - actual_ctags = backend.content_ctags_search( - 'foo|bar', last_sha1='some-hash', limit=2) - - # then - self.assertEquals(actual_ctags, 'some other result') - self.storage().content_ctags_search.assert_called_once_with( - 'foo|bar', last_sha1='some-hash', limit=2) - - @istest - def content_ctags_search_3(self): - # given - self.storage().content_ctags_search = MagicMock( - return_value="yet another result") - - # when - actual_ctags = backend.content_ctags_search( - 'bar', last_sha1='some-hash', limit=1000) - - # then - self.assertEquals(actual_ctags, 'yet another result') - self.storage().content_ctags_search.assert_called_once_with( - 'bar', last_sha1='some-hash', limit=50) - - @istest - def content_get(self): - # given - sha1_bin = hashutil.hash_to_bytes( - '123caf10e9535160d90e874b45aa426de762f19f') - stub_contents = [{ - 'sha1': sha1_bin, - 'data': b'binary data', - }] - - self.storage().content_get = MagicMock(return_value=stub_contents) - - # when - actual_content = backend.content_get(sha1_bin) - - # then - self.assertEquals(actual_content, stub_contents[0]) - self.storage().content_get.assert_called_once_with( - [sha1_bin]) - - @istest - def content_find_ko_no_result(self): - # given - sha1_bin = hashutil.hash_to_bytes( - '123caf10e9535160d90e874b45aa426de762f19f') - self.storage().content_find = MagicMock(return_value=None) - - # when - actual_lookup = backend.content_find('sha1_git', sha1_bin) - - # then - self.assertIsNone(actual_lookup) - - self.storage().content_find.assert_called_once_with( - {'sha1_git': sha1_bin}) - - @istest - def content_find(self): - # given - sha1_bin = hashutil.hash_to_bytes( - '456caf10e9535160d90e874b45aa426de762f19f') - self.storage().content_find = MagicMock(return_value=(1, 2, 3)) - - # when - actual_content = backend.content_find('sha1', sha1_bin) - - # then - self.assertEquals(actual_content, (1, 2, 3)) - - # check the function has been called with parameters - self.storage().content_find.assert_called_with({'sha1': sha1_bin}) - - @istest - def content_find_provenance_ko_no_result(self): - # given - sha1_bin = hashutil.hash_to_bytes( - '123caf10e9535160d90e874b45aa426de762f19f') - self.storage().content_find_provenance = MagicMock( - return_value=(x for x in [])) - - # when - actual_lookup = backend.content_find_provenance('sha1_git', sha1_bin) - - # then - self.assertEquals(list(actual_lookup), []) - - self.storage().content_find_provenance.assert_called_once_with( - {'sha1_git': sha1_bin}) - - @istest - def content_ctags_get(self): - # given - sha1_bin = hashutil.hash_to_bytes( - '456caf10e9535160d90e874b45aa426de762f19f') - self.storage().content_ctags_get = MagicMock( - return_value=[1, 2, 3]) - - # when - actual_content = backend.content_ctags_get(sha1_bin) - - # then - self.assertEquals(actual_content, [1, 2, 3]) - - self.storage().content_ctags_get.assert_called_with( - [sha1_bin]) - - @istest - def content_ctags_get_no_result(self): - # given - sha1_bin = hashutil.hash_to_bytes( - '456caf10e9535160d90e874b45aa426de762f19f') - self.storage().content_ctags_get = MagicMock( - return_value=[]) - - # when - actual_content = backend.content_ctags_get(sha1_bin) - - # then - self.assertEquals(actual_content, []) - - self.storage().content_ctags_get.assert_called_with( - [sha1_bin]) - - @istest - def content_filetype_get(self): - # given - sha1_bin = hashutil.hash_to_bytes( - '456caf10e9535160d90e874b45aa426de762f19f') - self.storage().content_mimetype_get = MagicMock( - return_value=[1, 2, 3]) - - # when - actual_content = backend.content_filetype_get(sha1_bin) - - # then - self.assertEquals(actual_content, 1) - - self.storage().content_mimetype_get.assert_called_with( - [sha1_bin]) - - @istest - def content_filetype_get_no_result(self): - # given - sha1_bin = hashutil.hash_to_bytes( - '456caf10e9535160d90e874b45aa426de762f19f') - self.storage().content_mimetype_get = MagicMock( - return_value=[]) - - # when - actual_content = backend.content_filetype_get(sha1_bin) - - # then - self.assertIsNone(actual_content) - - self.storage().content_mimetype_get.assert_called_with( - [sha1_bin]) - - @istest - def content_language_get(self): - # given - sha1_bin = hashutil.hash_to_bytes( - '456caf10e9535160d90e874b45aa426de762f19f') - self.storage().content_language_get = MagicMock( - return_value=[1, 2, 3]) - - # when - actual_content = backend.content_language_get(sha1_bin) - - # then - self.assertEquals(actual_content, 1) - - self.storage().content_language_get.assert_called_with( - [sha1_bin]) - - @istest - def content_language_get_no_result(self): - # given - sha1_bin = hashutil.hash_to_bytes( - '456caf10e9535160d90e874b45aa426de762f19f') - self.storage().content_language_get = MagicMock( - return_value=[]) - - # when - actual_content = backend.content_language_get(sha1_bin) - - # then - self.assertIsNone(actual_content) - - self.storage().content_language_get.assert_called_with( - [sha1_bin]) - - @istest - def content_license_get(self): - # given - sha1_bin = hashutil.hash_to_bytes( - '456caf10e9535160d90e874b45aa426de762f19f') - self.storage().content_fossology_license_get = MagicMock( - return_value=[1, 2, 3]) - - # when - actual_content = backend.content_license_get(sha1_bin) - - # then - self.assertEquals(actual_content, 1) - - self.storage().content_fossology_license_get.assert_called_with( - [sha1_bin]) - - @istest - def content_license_get_no_result(self): - # given - sha1_bin = hashutil.hash_to_bytes( - '456caf10e9535160d90e874b45aa426de762f19f') - self.storage().content_fossology_license_get = MagicMock( - return_value=[]) - - # when - actual_content = backend.content_license_get(sha1_bin) - - # then - self.assertIsNone(actual_content) - - self.storage().content_fossology_license_get.assert_called_with( - [sha1_bin]) - - @istest - def content_find_provenance(self): - # given - sha1_bin = hashutil.hash_to_bytes( - '456caf10e9535160d90e874b45aa426de762f19f') - self.storage().content_find_provenance = MagicMock( - return_value=(x for x in (1, 2, 3))) - - # when - actual_content = backend.content_find_provenance('sha1', sha1_bin) - - # then - self.assertEquals(list(actual_content), [1, 2, 3]) - - # check the function has been called with parameters - self.storage().content_find_provenance.assert_called_with( - {'sha1': sha1_bin}) - - @istest - def content_missing_per_sha1_none(self): - # given - sha1s_bin = [hashutil.hash_to_bytes( - '456caf10e9535160d90e874b45aa426de762f19f'), - hashutil.hash_to_bytes( - '745bab676c8f3cec8016e0c39ea61cf57e518865' - )] - self.storage().content_missing_per_sha1 = MagicMock(return_value=[]) - - # when - actual_content = backend.content_missing_per_sha1(sha1s_bin) - - # then - self.assertEquals(actual_content, []) - self.storage().content_missing_per_sha1.assert_called_with(sha1s_bin) - - @istest - def content_missing_per_sha1_some(self): - # given - sha1s_bin = [hashutil.hash_to_bytes( - '456caf10e9535160d90e874b45aa426de762f19f'), - hashutil.hash_to_bytes( - '745bab676c8f3cec8016e0c39ea61cf57e518865' - )] - self.storage().content_missing_per_sha1 = MagicMock(return_value=[ - hashutil.hash_to_bytes( - '745bab676c8f3cec8016e0c39ea61cf57e518865' - )]) - - # when - actual_content = backend.content_missing_per_sha1(sha1s_bin) - - # then - self.assertEquals(actual_content, [hashutil.hash_to_bytes( - '745bab676c8f3cec8016e0c39ea61cf57e518865' - )]) - self.storage().content_missing_per_sha1.assert_called_with(sha1s_bin) - - @istest - def origin_get_by_id(self): - # given - self.storage().origin_get = MagicMock(return_value={ - 'id': 'origin-id', - 'lister': 'uuid-lister', - 'project': 'uuid-project', - 'url': 'ftp://some/url/to/origin', - 'type': 'ftp'}) - - # when - actual_origin = backend.origin_get({'id': 'origin-id'}) - - # then - self.assertEqual(actual_origin, {'id': 'origin-id', - 'lister': 'uuid-lister', - 'project': 'uuid-project', - 'url': 'ftp://some/url/to/origin', - 'type': 'ftp'}) - - self.storage().origin_get.assert_called_with({'id': 'origin-id'}) - - @istest - def origin_get_by_type_url(self): - # given - self.storage().origin_get = MagicMock(return_value={ - 'id': 'origin-id', - 'lister': 'uuid-lister', - 'project': 'uuid-project', - 'url': 'ftp://some/url/to/origin', - 'type': 'ftp'}) - - # when - actual_origin = backend.origin_get({'type': 'ftp', - 'url': 'ftp://some/url/to/origin'}) - - # then - self.assertEqual(actual_origin, {'id': 'origin-id', - 'lister': 'uuid-lister', - 'project': 'uuid-project', - 'url': 'ftp://some/url/to/origin', - 'type': 'ftp'}) - - self.storage().origin_get.assert_called_with( - {'type': 'ftp', - 'url': 'ftp://some/url/to/origin'}) - - @istest - def person_get(self): - # given - self.storage().person_get = MagicMock(return_value=[{ - 'id': 'person-id', - 'name': 'blah'}]) - - # when - actual_person = backend.person_get('person-id') - - # then - self.assertEqual(actual_person, {'id': 'person-id', - 'name': 'blah'}) - - self.storage().person_get.assert_called_with(['person-id']) - - @istest - def directory_get_not_found(self): - # given - sha1_bin = hashutil.hash_to_bytes( - '40e71b8614fcd89ccd17ca2b1d9e66c5b00a6d03') - self.storage().directory_get = MagicMock(return_value=None) - - # when - actual_directory = backend.directory_get(sha1_bin) - - # then - self.assertEquals(actual_directory, None) - - self.storage().directory_get.assert_called_with([sha1_bin]) - - @istest - def directory_get(self): - # given - sha1_bin = hashutil.hash_to_bytes( - '51f71b8614fcd89ccd17ca2b1d9e66c5b00a6d03') - sha1_bin2 = hashutil.hash_to_bytes( - '62071b8614fcd89ccd17ca2b1d9e66c5b00a6d03') - stub_dir = {'id': sha1_bin, 'revision': b'sha1-blah'} - stub_dir2 = {'id': sha1_bin2, 'revision': b'sha1-foobar'} - self.storage().directory_get = MagicMock(return_value=[stub_dir, - stub_dir2]) - - # when - actual_directory = backend.directory_get(sha1_bin) - - # then - self.assertEquals(actual_directory, stub_dir) - - self.storage().directory_get.assert_called_with([sha1_bin]) - - @istest - def directory_ls_empty_result(self): - # given - sha1_bin = hashutil.hash_to_bytes( - '40e71b8614fcd89ccd17ca2b1d9e66c5b00a6d03') - self.storage().directory_ls = MagicMock(return_value=[]) - - # when - actual_directory = backend.directory_ls(sha1_bin) - - # then - self.assertEquals(actual_directory, []) - - self.storage().directory_ls.assert_called_with(sha1_bin, False) - - @istest - def directory_ls(self): - # given - sha1_bin = hashutil.hash_to_bytes( - '40e71b8614fcd89ccd17ca2b1d9e66c5b00a6d03') - stub_dir_entries = [{ - 'sha1': hashutil.hash_to_bytes( - '5c6f0e2750f48fa0bd0c4cf5976ba0b9e02ebda5'), - 'sha256': hashutil.hash_to_bytes( - '39007420ca5de7cb3cfc15196335507e' - 'e76c98930e7e0afa4d2747d3bf96c926'), - 'sha1_git': hashutil.hash_to_bytes( - '40e71b8614fcd89ccd17ca2b1d9e66c5b00a6d03'), - 'target': hashutil.hash_to_bytes( - '40e71b8614fcd89ccd17ca2b1d9e66c5b00a6d03'), - 'dir_id': hashutil.hash_to_bytes( - '40e71b8614fcd89ccd17ca2b1d9e66c5b00a6d03'), - 'name': b'bob', - 'type': 10, - }] - - self.storage().directory_ls = MagicMock( - return_value=stub_dir_entries) - - actual_directory = backend.directory_ls(sha1_bin, recursive=True) - - # then - self.assertIsNotNone(actual_directory) - self.assertEqual(list(actual_directory), stub_dir_entries) - - self.storage().directory_ls.assert_called_with(sha1_bin, True) - - @istest - def release_get_not_found(self): - # given - sha1_bin = hashutil.hash_to_bytes( - '65a55bbdf3629f916219feb3dcc7393ded1bc8db') - - self.storage().release_get = MagicMock(return_value=[]) - - # when - actual_release = backend.release_get(sha1_bin) - - # then - self.assertIsNone(actual_release) - self.storage().release_get.assert_called_with([sha1_bin]) - - @istest - def release_get(self): - # given - sha1_bin = hashutil.hash_to_bytes( - '65a55bbdf3629f916219feb3dcc7393ded1bc8db') - - stub_releases = [{ - 'id': sha1_bin, - 'target': None, - 'date': datetime.datetime(2015, 1, 1, 22, 0, 0, - tzinfo=datetime.timezone.utc), - 'name': b'v0.0.1', - 'message': b'synthetic release', - 'synthetic': True, - }] - self.storage().release_get = MagicMock(return_value=stub_releases) - - # when - actual_release = backend.release_get(sha1_bin) - - # then - self.assertEqual(actual_release, stub_releases[0]) - - self.storage().release_get.assert_called_with([sha1_bin]) - - @istest - def revision_get_by_not_found(self): - # given - self.storage().revision_get_by = MagicMock(return_value=[]) - - # when - actual_revision = backend.revision_get_by(10, 'master', 'ts2') - - # then - self.assertIsNone(actual_revision) - - self.storage().revision_get_by.assert_called_with(10, 'master', - timestamp='ts2', - limit=1) - - @istest - def revision_get_by(self): - # given - self.storage().revision_get_by = MagicMock(return_value=[{'id': 1}]) - - # when - actual_revisions = backend.revision_get_by(100, 'dev', 'ts') - - # then - self.assertEquals(actual_revisions, {'id': 1}) - - self.storage().revision_get_by.assert_called_with(100, 'dev', - timestamp='ts', - limit=1) - - @istest - def revision_get_not_found(self): - # given - sha1_bin = hashutil.hash_to_bytes( - '18d8be353ed3480476f032475e7c233eff7371d5') - - self.storage().revision_get = MagicMock(return_value=[]) - - # when - actual_revision = backend.revision_get(sha1_bin) - - # then - self.assertIsNone(actual_revision) - - self.storage().revision_get.assert_called_with([sha1_bin]) - - @istest - def revision_get(self): - # given - sha1_bin = hashutil.hash_to_bytes( - '18d8be353ed3480476f032475e7c233eff7371d5') - - stub_revisions = [{ - 'id': sha1_bin, - 'directory': hashutil.hash_to_bytes( - '7834ef7e7c357ce2af928115c6c6a42b7e2a44e6'), - 'author': { - 'name': b'bill & boule', - 'email': b'bill@boule.org', - }, - 'committer': { - 'name': b'boule & bill', - 'email': b'boule@bill.org', - }, - 'message': b'elegant fix for bug 31415957', - 'date': datetime.datetime(2000, 1, 17, 11, 23, 54), - 'date_offset': 0, - 'committer_date': datetime.datetime(2000, 1, 17, 11, 23, 54), - 'committer_date_offset': 0, - 'synthetic': False, - 'type': 'git', - 'parents': [], - 'metadata': [], - }] - self.storage().revision_get = MagicMock(return_value=stub_revisions) - - # when - actual_revision = backend.revision_get(sha1_bin) - - # then - self.assertEqual(actual_revision, stub_revisions[0]) - - self.storage().revision_get.assert_called_with([sha1_bin]) - - @istest - def revision_get_multiple(self): - # given - sha1_bin = hashutil.hash_to_bytes( - '18d8be353ed3480476f032475e7c233eff7371d5') - sha1_other = hashutil.hash_to_bytes( - 'adc83b19e793491b1c6ea0fd8b46cd9f32e592fc') - - stub_revisions = [ - { - 'id': sha1_bin, - 'directory': hashutil.hash_to_bytes( - '7834ef7e7c357ce2af928115c6c6a42b7e2a44e6'), - 'author': { - 'name': b'bill & boule', - 'email': b'bill@boule.org', - }, - 'committer': { - 'name': b'boule & bill', - 'email': b'boule@bill.org', - }, - 'message': b'elegant fix for bug 31415957', - 'date': datetime.datetime(2000, 1, 17, 11, 23, 54), - 'date_offset': 0, - 'committer_date': datetime.datetime(2000, 1, 17, 11, 23, 54), - 'committer_date_offset': 0, - 'synthetic': False, - 'type': 'git', - 'parents': [], - 'metadata': [], - }, - { - 'id': sha1_other, - 'directory': hashutil.hash_to_bytes( - '7834ef7e7c357ce2af928115c6c6a42b7e2a44e6'), - 'author': { - 'name': b'name', - 'email': b'name@surname.org', - }, - 'committer': { - 'name': b'name', - 'email': b'name@surname.org', - }, - 'message': b'ugly fix for bug 42', - 'date': datetime.datetime(2000, 1, 12, 5, 23, 54), - 'date_offset': 0, - 'committer_date': datetime.datetime(2000, 1, 12, 5, 23, 54), - 'committer_date_offset': 0, - 'synthetic': False, - 'type': 'git', - 'parents': [], - 'metadata': [], - } - ] - self.storage().revision_get = MagicMock( - return_value=stub_revisions) - - # when - actual_revision = backend.revision_get_multiple([sha1_bin, sha1_other]) - - # then - self.assertEqual(actual_revision, stub_revisions) - - self.storage().revision_get.assert_called_with( - [sha1_bin, sha1_other]) - - @istest - def revision_get_multiple_none_found(self): - # given - sha1_bin = hashutil.hash_to_bytes( - '18d8be353ed3480476f032475e7c233eff7371d5') - sha1_other = hashutil.hash_to_bytes( - 'adc83b19e793491b1c6ea0fd8b46cd9f32e592fc') - - self.storage().revision_get = MagicMock( - return_value=[]) - - # when - actual_revision = backend.revision_get_multiple([sha1_bin, sha1_other]) - - # then - self.assertEqual(actual_revision, []) - - self.storage().revision_get.assert_called_with( - [sha1_bin, sha1_other]) - - @istest - def revision_log(self): - # given - sha1_bin = hashutil.hash_to_bytes( - '28d8be353ed3480476f032475e7c233eff7371d5') - stub_revision_log = [{ - 'id': sha1_bin, - 'directory': hashutil.hash_to_bytes( - '7834ef7e7c357ce2af928115c6c6a42b7e2a44e6'), - 'author': { - 'name': b'bill & boule', - 'email': b'bill@boule.org', - }, - 'committer': { - 'name': b'boule & bill', - 'email': b'boule@bill.org', - }, - 'message': b'elegant fix for bug 31415957', - 'date': datetime.datetime(2000, 1, 17, 11, 23, 54), - 'date_offset': 0, - 'committer_date': datetime.datetime(2000, 1, 17, 11, 23, 54), - 'committer_date_offset': 0, - 'synthetic': False, - 'type': 'git', - 'parents': [], - 'metadata': [], - }] - self.storage().revision_log = MagicMock(return_value=stub_revision_log) - - # when - actual_revision = backend.revision_log(sha1_bin, limit=1) - - # then - self.assertEqual(list(actual_revision), stub_revision_log) - - self.storage().revision_log.assert_called_with([sha1_bin], 1) - - @istest - def revision_log_by(self): - # given - sha1_bin = hashutil.hash_to_bytes( - '28d8be353ed3480476f032475e7c233eff7371d5') - stub_revision_log = [{ - 'id': sha1_bin, - 'directory': hashutil.hash_to_bytes( - '7834ef7e7c357ce2af928115c6c6a42b7e2a44e6'), - 'author': { - 'name': b'bill & boule', - 'email': b'bill@boule.org', - }, - 'committer': { - 'name': b'boule & bill', - 'email': b'boule@bill.org', - }, - 'message': b'elegant fix for bug 31415957', - 'date': datetime.datetime(2000, 1, 17, 11, 23, 54), - 'date_offset': 0, - 'committer_date': datetime.datetime(2000, 1, 17, 11, 23, 54), - 'committer_date_offset': 0, - 'synthetic': False, - 'type': 'git', - 'parents': [], - 'metadata': [], - }] - - self.storage().revision_log_by = MagicMock( - return_value=stub_revision_log) - - # when - actual_log = backend.revision_log_by(1, 'refs/heads/master', - None, limit=1) - - # then - self.assertEqual(actual_log, stub_revision_log) - self.storage().revision_log.assert_called_with([sha1_bin], 1) - - @istest - def revision_log_by_norev(self): - # given - sha1_bin = hashutil.hash_to_bytes( - '28d8be353ed3480476f032475e7c233eff7371d5') - - self.storage().revision_log_by = MagicMock(return_value=None) - - # when - actual_log = backend.revision_log_by(1, 'refs/heads/master', - None, limit=1) - - # then - self.assertEqual(actual_log, None) - self.storage().revision_log.assert_called_with([sha1_bin], 1) - - @istest - def stat_counters(self): - # given - input_stats = { - "content": 1770830, - "directory": 211683, - "directory_entry_dir": 209167, - "directory_entry_file": 1807094, - "directory_entry_rev": 0, - "entity": 0, - "entity_history": 0, - "occurrence": 0, - "occurrence_history": 19600, - "origin": 1096, - "person": 0, - "release": 8584, - "revision": 7792, - "revision_history": 0, - "skipped_content": 0 - } - self.storage().stat_counters = MagicMock(return_value=input_stats) - - # when - actual_stats = backend.stat_counters() - - # then - expected_stats = input_stats - self.assertEqual(actual_stats, expected_stats) - - self.storage().stat_counters.assert_called_with() - - @istest - def lookup_origin_visits(self): - # given - expected_origin_visits = [ - self.origin_visit1, { - 'date': datetime.datetime( - 2013, 7, 1, 20, 0, 0, - tzinfo=datetime.timezone.utc), - 'origin': 1, - 'visit': 2 - }, { - 'date': datetime.datetime( - 2015, 1, 1, 21, 0, 0, - tzinfo=datetime.timezone.utc), - 'origin': 1, - 'visit': 3 - }] - self.storage().origin_visit_get = MagicMock( - return_value=expected_origin_visits) - - # when - actual_origin_visits = backend.lookup_origin_visits(5) - - # then - self.assertEqual(list(actual_origin_visits), expected_origin_visits) - - self.storage().origin_visit_get.assert_called_with( - 5, last_visit=None, limit=10) - - @istest - def lookup_origin_visit(self): - # given - self.storage().origin_visit_get_by = MagicMock( - return_value=self.origin_visit1) - - # when - actual_origin_visit = backend.lookup_origin_visit(10, 1) - - # then - self.assertEqual(actual_origin_visit, self.origin_visit1) - - self.storage().origin_visit_get_by.assert_called_with(10, 1) - - @istest - def lookup_origin_visit_none(self): - # given - self.storage().origin_visit_get_by = MagicMock( - return_value=None) - - # when - actual_origin_visit = backend.lookup_origin_visit(1, 10) - - # then - self.assertIsNone(actual_origin_visit) - - self.storage().origin_visit_get_by.assert_called_with(1, 10) - - @istest - def directory_entry_get_by_path(self): - # given - stub_dir_entry = {'id': b'dir-id', - 'type': 'dir', - 'name': b'some/path/foo'} - self.storage().directory_entry_get_by_path = MagicMock( - return_value=stub_dir_entry) - - # when - actual_dir_entry = backend.directory_entry_get_by_path(b'dir-sha1', - 'some/path/foo') - - self.assertEquals(actual_dir_entry, stub_dir_entry) - self.storage().directory_entry_get_by_path.assert_called_once_with( - b'dir-sha1', - [b'some', b'path', b'foo']) - - @istest - def entity_get(self): - # given - stub_entities = [{'uuid': 'e8c3fc2e-a932-4fd7-8f8e-c40645eb35a7', - 'parent': 'aee991a0-f8d7-4295-a201-d1ce2efc9fb2'}, - {'uuid': 'aee991a0-f8d7-4295-a201-d1ce2efc9fb2', - 'parent': None}] - self.storage().entity_get = MagicMock(return_value=stub_entities) - - # when - actual_entities = backend.entity_get( - 'e8c3fc2e-a932-4fd7-8f8e-c40645eb35a7') - - # then - self.assertEquals(actual_entities, stub_entities) - - self.storage().entity_get.assert_called_once_with( - 'e8c3fc2e-a932-4fd7-8f8e-c40645eb35a7') diff --git a/swh/web/tests/api/test_service.py b/swh/web/tests/api/test_service.py index c6bbdabe..29a1c947 100644 --- a/swh/web/tests/api/test_service.py +++ b/swh/web/tests/api/test_service.py @@ -1,2050 +1,2058 @@ # Copyright (C) 2015-2017 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import datetime from nose.tools import istest from unittest.mock import MagicMock, patch, call from swh.model.hashutil import hash_to_bytes, hash_to_hex from .swh_api_testcase import SWHApiTestCase from swh.web.api import service from swh.web.api.exc import BadInputExc, NotFoundExc class ServiceTestCase(SWHApiTestCase): def setUp(self): self.SHA1_SAMPLE = '18d8be353ed3480476f032475e7c233eff7371d5' self.SHA1_SAMPLE_BIN = hash_to_bytes(self.SHA1_SAMPLE) self.SHA256_SAMPLE = ('39007420ca5de7cb3cfc15196335507e' 'e76c98930e7e0afa4d2747d3bf96c926') self.SHA256_SAMPLE_BIN = hash_to_bytes(self.SHA256_SAMPLE) self.SHA1GIT_SAMPLE = '40e71b8614fcd89ccd17ca2b1d9e66c5b00a6d03' self.SHA1GIT_SAMPLE_BIN = hash_to_bytes(self.SHA1GIT_SAMPLE) self.DIRECTORY_ID = '7834ef7e7c357ce2af928115c6c6a42b7e2a44e6' self.DIRECTORY_ID_BIN = hash_to_bytes(self.DIRECTORY_ID) self.AUTHOR_ID_BIN = { 'name': b'author', 'email': b'author@company.org', } self.AUTHOR_ID = { 'name': 'author', 'email': 'author@company.org', } self.COMMITTER_ID_BIN = { 'name': b'committer', 'email': b'committer@corp.org', } self.COMMITTER_ID = { 'name': 'committer', 'email': 'committer@corp.org', } self.SAMPLE_DATE_RAW = { 'timestamp': datetime.datetime( 2000, 1, 17, 11, 23, 54, tzinfo=datetime.timezone.utc, ).timestamp(), 'offset': 0, 'negative_utc': False, } self.SAMPLE_DATE = '2000-01-17T11:23:54+00:00' self.SAMPLE_MESSAGE_BIN = b'elegant fix for bug 31415957' self.SAMPLE_MESSAGE = 'elegant fix for bug 31415957' self.SAMPLE_REVISION = { 'id': self.SHA1_SAMPLE, 'directory': self.DIRECTORY_ID, 'author': self.AUTHOR_ID, 'committer': self.COMMITTER_ID, 'message': self.SAMPLE_MESSAGE, 'date': self.SAMPLE_DATE, 'committer_date': self.SAMPLE_DATE, 'synthetic': False, 'type': 'git', 'parents': [], 'metadata': {}, 'merge': False } self.SAMPLE_REVISION_RAW = { 'id': self.SHA1_SAMPLE_BIN, 'directory': self.DIRECTORY_ID_BIN, 'author': self.AUTHOR_ID_BIN, 'committer': self.COMMITTER_ID_BIN, 'message': self.SAMPLE_MESSAGE_BIN, 'date': self.SAMPLE_DATE_RAW, 'committer_date': self.SAMPLE_DATE_RAW, 'synthetic': False, 'type': 'git', 'parents': [], 'metadata': [], } self.SAMPLE_CONTENT = { 'sha1': self.SHA1_SAMPLE, 'sha256': self.SHA256_SAMPLE, 'sha1_git': self.SHA1GIT_SAMPLE, 'length': 190, 'status': 'absent' } self.SAMPLE_CONTENT_RAW = { 'sha1': self.SHA1_SAMPLE_BIN, 'sha256': self.SHA256_SAMPLE_BIN, 'sha1_git': self.SHA1GIT_SAMPLE_BIN, 'length': 190, 'status': 'hidden' } self.date_origin_visit1 = datetime.datetime( 2015, 1, 1, 22, 0, 0, tzinfo=datetime.timezone.utc) self.origin_visit1 = { 'date': self.date_origin_visit1, 'origin': 1, 'visit': 1 } - @patch('swh.web.api.service.backend') + @patch('swh.web.api.service.storage') @istest - def test_lookup_multiple_hashes_ball_missing(self, mock_backend): + def lookup_multiple_hashes_ball_missing(self, mock_storage): # given - mock_backend.content_missing_per_sha1 = MagicMock(return_value=[]) + mock_storage.content_missing_per_sha1 = MagicMock(return_value=[]) # when actual_lookup = service.lookup_multiple_hashes( [{'filename': 'a', 'sha1': '456caf10e9535160d90e874b45aa426de762f19f'}, {'filename': 'b', 'sha1': '745bab676c8f3cec8016e0c39ea61cf57e518865'}]) # then self.assertEquals(actual_lookup, [ {'filename': 'a', 'sha1': '456caf10e9535160d90e874b45aa426de762f19f', 'found': True}, {'filename': 'b', 'sha1': '745bab676c8f3cec8016e0c39ea61cf57e518865', 'found': True} ]) - @patch('swh.web.api.service.backend') + @patch('swh.web.api.service.storage') @istest - def test_lookup_multiple_hashes_some_missing(self, mock_backend): + def lookup_multiple_hashes_some_missing(self, mock_storage): # given - mock_backend.content_missing_per_sha1 = MagicMock(return_value=[ + mock_storage.content_missing_per_sha1 = MagicMock(return_value=[ hash_to_bytes('456caf10e9535160d90e874b45aa426de762f19f') ]) # when actual_lookup = service.lookup_multiple_hashes( [{'filename': 'a', 'sha1': '456caf10e9535160d90e874b45aa426de762f19f'}, {'filename': 'b', 'sha1': '745bab676c8f3cec8016e0c39ea61cf57e518865'}]) # then self.assertEquals(actual_lookup, [ {'filename': 'a', 'sha1': '456caf10e9535160d90e874b45aa426de762f19f', 'found': False}, {'filename': 'b', 'sha1': '745bab676c8f3cec8016e0c39ea61cf57e518865', 'found': True} ]) - @patch('swh.web.api.service.backend') + @patch('swh.web.api.service.storage') @istest - def lookup_hash_does_not_exist(self, mock_backend): + def lookup_hash_does_not_exist(self, mock_storage): # given - mock_backend.content_find = MagicMock(return_value=None) + mock_storage.content_find = MagicMock(return_value=None) # when actual_lookup = service.lookup_hash( 'sha1_git:123caf10e9535160d90e874b45aa426de762f19f') # then self.assertEquals({'found': None, 'algo': 'sha1_git'}, actual_lookup) # check the function has been called with parameters - mock_backend.content_find.assert_called_with( - 'sha1_git', - hash_to_bytes('123caf10e9535160d90e874b45aa426de762f19f')) + mock_storage.content_find.assert_called_with( + {'sha1_git': + hash_to_bytes('123caf10e9535160d90e874b45aa426de762f19f')}) - @patch('swh.web.api.service.backend') + @patch('swh.web.api.service.storage') @istest - def lookup_hash_exist(self, mock_backend): + def lookup_hash_exist(self, mock_storage): # given stub_content = { 'sha1': hash_to_bytes( '456caf10e9535160d90e874b45aa426de762f19f') } - mock_backend.content_find = MagicMock(return_value=stub_content) + mock_storage.content_find = MagicMock(return_value=stub_content) # when actual_lookup = service.lookup_hash( 'sha1:456caf10e9535160d90e874b45aa426de762f19f') # then self.assertEquals({'found': stub_content, 'algo': 'sha1'}, actual_lookup) - mock_backend.content_find.assert_called_with( - 'sha1', - hash_to_bytes('456caf10e9535160d90e874b45aa426de762f19f'), + mock_storage.content_find.assert_called_with( + {'sha1': + hash_to_bytes('456caf10e9535160d90e874b45aa426de762f19f')} ) - @patch('swh.web.api.service.backend') + @patch('swh.web.api.service.storage') @istest - def search_hash_does_not_exist(self, mock_backend): + def search_hash_does_not_exist(self, mock_storage): # given - mock_backend.content_find = MagicMock(return_value=None) + mock_storage.content_find = MagicMock(return_value=None) # when actual_lookup = service.search_hash( 'sha1_git:123caf10e9535160d90e874b45aa426de762f19f') # then self.assertEquals({'found': False}, actual_lookup) # check the function has been called with parameters - mock_backend.content_find.assert_called_with( - 'sha1_git', - hash_to_bytes('123caf10e9535160d90e874b45aa426de762f19f')) + mock_storage.content_find.assert_called_with( + {'sha1_git': + hash_to_bytes('123caf10e9535160d90e874b45aa426de762f19f')}) - @patch('swh.web.api.service.backend') + @patch('swh.web.api.service.storage') @istest - def search_hash_exist(self, mock_backend): + def search_hash_exist(self, mock_storage): # given stub_content = { 'sha1': hash_to_bytes( '456caf10e9535160d90e874b45aa426de762f19f') } - mock_backend.content_find = MagicMock(return_value=stub_content) + mock_storage.content_find = MagicMock(return_value=stub_content) # when actual_lookup = service.search_hash( 'sha1:456caf10e9535160d90e874b45aa426de762f19f') # then self.assertEquals({'found': True}, actual_lookup) - mock_backend.content_find.assert_called_with( - 'sha1', - hash_to_bytes('456caf10e9535160d90e874b45aa426de762f19f'), + mock_storage.content_find.assert_called_with( + {'sha1': + hash_to_bytes('456caf10e9535160d90e874b45aa426de762f19f')}, ) - @patch('swh.web.api.service.backend') + @patch('swh.web.api.service.storage') @istest - def lookup_content_ctags(self, mock_backend): + def lookup_content_ctags(self, mock_storage): # given - mock_backend.content_ctags_get = MagicMock( + mock_storage.content_ctags_get = MagicMock( return_value=[{ 'id': hash_to_bytes( '123caf10e9535160d90e874b45aa426de762f19f'), 'line': 100, 'name': 'hello', 'kind': 'function', 'tool_name': 'ctags', 'tool_version': 'some-version', }]) expected_ctags = [{ 'id': '123caf10e9535160d90e874b45aa426de762f19f', 'line': 100, 'name': 'hello', 'kind': 'function', 'tool_name': 'ctags', 'tool_version': 'some-version', }] # when actual_ctags = list(service.lookup_content_ctags( 'sha1:123caf10e9535160d90e874b45aa426de762f19f')) # then self.assertEqual(actual_ctags, expected_ctags) - mock_backend.content_ctags_get.assert_called_with( - hash_to_bytes('123caf10e9535160d90e874b45aa426de762f19f')) + mock_storage.content_ctags_get.assert_called_with( + [hash_to_bytes('123caf10e9535160d90e874b45aa426de762f19f')]) - @patch('swh.web.api.service.backend') + @patch('swh.web.api.service.storage') @istest - def lookup_content_ctags_no_hash(self, mock_backend): + def lookup_content_ctags_no_hash(self, mock_storage): # given - mock_backend.content_find.return_value = None - mock_backend.content_ctags_get = MagicMock( + mock_storage.content_find.return_value = None + mock_storage.content_ctags_get = MagicMock( return_value=None) # when actual_ctags = list(service.lookup_content_ctags( 'sha1_git:123caf10e9535160d90e874b45aa426de762f19f')) # then self.assertEqual(actual_ctags, []) - mock_backend.content_find.assert_called_once_with( - 'sha1_git', hash_to_bytes( - '123caf10e9535160d90e874b45aa426de762f19f')) + mock_storage.content_find.assert_called_once_with( + {'sha1_git': hash_to_bytes( + '123caf10e9535160d90e874b45aa426de762f19f')}) - @patch('swh.web.api.service.backend') + @patch('swh.web.api.service.storage') @istest - def lookup_content_filetype(self, mock_backend): + def lookup_content_filetype(self, mock_storage): # given - mock_backend.content_filetype_get = MagicMock( - return_value={ + mock_storage.content_mimetype_get = MagicMock( + return_value=[{ 'id': hash_to_bytes( '123caf10e9535160d90e874b45aa426de762f19f'), 'mimetype': b'text/x-c++', 'encoding': b'us-ascii', - }) + }]) expected_filetype = { 'id': '123caf10e9535160d90e874b45aa426de762f19f', 'mimetype': 'text/x-c++', 'encoding': 'us-ascii', } # when actual_filetype = service.lookup_content_filetype( 'sha1:123caf10e9535160d90e874b45aa426de762f19f') # then self.assertEqual(actual_filetype, expected_filetype) - mock_backend.content_filetype_get.assert_called_with( - hash_to_bytes('123caf10e9535160d90e874b45aa426de762f19f')) + mock_storage.content_mimetype_get.assert_called_with( + [hash_to_bytes('123caf10e9535160d90e874b45aa426de762f19f')]) - @patch('swh.web.api.service.backend') + @patch('swh.web.api.service.storage') @istest - def lookup_content_filetype_2(self, mock_backend): + def lookup_content_filetype_2(self, mock_storage): # given - mock_backend.content_find = MagicMock( + mock_storage.content_find = MagicMock( return_value={ 'sha1': hash_to_bytes( '123caf10e9535160d90e874b45aa426de762f19f') } ) - mock_backend.content_filetype_get = MagicMock( - return_value={ + mock_storage.content_mimetype_get = MagicMock( + return_value=[{ 'id': hash_to_bytes( '123caf10e9535160d90e874b45aa426de762f19f'), 'mimetype': b'text/x-python', 'encoding': b'us-ascii', - } + }] ) expected_filetype = { 'id': '123caf10e9535160d90e874b45aa426de762f19f', 'mimetype': 'text/x-python', 'encoding': 'us-ascii', } # when actual_filetype = service.lookup_content_filetype( 'sha1_git:456caf10e9535160d90e874b45aa426de762f19f') # then self.assertEqual(actual_filetype, expected_filetype) - mock_backend.content_find( + mock_storage.content_find( 'sha1_git', hash_to_bytes( '456caf10e9535160d90e874b45aa426de762f19f') ) - mock_backend.content_filetype_get.assert_called_with( - hash_to_bytes('123caf10e9535160d90e874b45aa426de762f19f')) + mock_storage.content_mimetype_get.assert_called_with( + [hash_to_bytes('123caf10e9535160d90e874b45aa426de762f19f')]) - @patch('swh.web.api.service.backend') + @patch('swh.web.api.service.storage') @istest - def lookup_content_language(self, mock_backend): + def lookup_content_language(self, mock_storage): # given - mock_backend.content_language_get = MagicMock( - return_value={ + mock_storage.content_language_get = MagicMock( + return_value=[{ 'id': hash_to_bytes( '123caf10e9535160d90e874b45aa426de762f19f'), 'lang': 'python', - }) + }]) expected_language = { 'id': '123caf10e9535160d90e874b45aa426de762f19f', 'lang': 'python', } # when actual_language = service.lookup_content_language( 'sha1:123caf10e9535160d90e874b45aa426de762f19f') # then self.assertEqual(actual_language, expected_language) - mock_backend.content_language_get.assert_called_with( - hash_to_bytes('123caf10e9535160d90e874b45aa426de762f19f')) + mock_storage.content_language_get.assert_called_with( + [hash_to_bytes('123caf10e9535160d90e874b45aa426de762f19f')]) - @patch('swh.web.api.service.backend') + @patch('swh.web.api.service.storage') @istest - def lookup_content_language_2(self, mock_backend): + def lookup_content_language_2(self, mock_storage): # given - mock_backend.content_find = MagicMock( + mock_storage.content_find = MagicMock( return_value={ 'sha1': hash_to_bytes( '123caf10e9535160d90e874b45aa426de762f19f') } ) - mock_backend.content_language_get = MagicMock( - return_value={ + mock_storage.content_language_get = MagicMock( + return_value=[{ 'id': hash_to_bytes( '123caf10e9535160d90e874b45aa426de762f19f'), 'lang': 'haskell', - } + }] ) expected_language = { 'id': '123caf10e9535160d90e874b45aa426de762f19f', 'lang': 'haskell', } # when actual_language = service.lookup_content_language( 'sha1_git:456caf10e9535160d90e874b45aa426de762f19f') # then self.assertEqual(actual_language, expected_language) - mock_backend.content_find( + mock_storage.content_find( 'sha1_git', hash_to_bytes( '456caf10e9535160d90e874b45aa426de762f19f') ) - mock_backend.content_language_get.assert_called_with( - hash_to_bytes('123caf10e9535160d90e874b45aa426de762f19f')) + mock_storage.content_language_get.assert_called_with( + [hash_to_bytes('123caf10e9535160d90e874b45aa426de762f19f')]) - @patch('swh.web.api.service.backend') + @patch('swh.web.api.service.storage') @istest - def lookup_expression(self, mock_backend): + def lookup_expression(self, mock_storage): # given - mock_backend.content_ctags_search = MagicMock( + mock_storage.content_ctags_search = MagicMock( return_value=[{ 'id': hash_to_bytes( '123caf10e9535160d90e874b45aa426de762f19f'), 'name': 'foobar', 'kind': 'variable', 'lang': 'C', 'line': 10 }]) expected_ctags = [{ 'sha1': '123caf10e9535160d90e874b45aa426de762f19f', 'name': 'foobar', 'kind': 'variable', 'lang': 'C', 'line': 10 }] # when actual_ctags = list(service.lookup_expression( 'foobar', last_sha1='hash', per_page=10)) # then self.assertEqual(actual_ctags, expected_ctags) - mock_backend.content_ctags_search.assert_called_with( - 'foobar', 'hash', 10) + mock_storage.content_ctags_search.assert_called_with( + 'foobar', last_sha1='hash', limit=10) - @patch('swh.web.api.service.backend') + @patch('swh.web.api.service.storage') @istest - def lookup_expression_no_result(self, mock_backend): + def lookup_expression_no_result(self, mock_storage): # given - mock_backend.content_ctags_search = MagicMock( + mock_storage.content_ctags_search = MagicMock( return_value=[]) expected_ctags = [] # when actual_ctags = list(service.lookup_expression( 'barfoo', last_sha1='hash', per_page=10)) # then self.assertEqual(actual_ctags, expected_ctags) - mock_backend.content_ctags_search.assert_called_with( - 'barfoo', 'hash', 10) + mock_storage.content_ctags_search.assert_called_with( + 'barfoo', last_sha1='hash', limit=10) - @patch('swh.web.api.service.backend') + @patch('swh.web.api.service.storage') @istest - def lookup_content_license(self, mock_backend): + def lookup_content_license(self, mock_storage): # given - mock_backend.content_license_get = MagicMock( - return_value={ + mock_storage.content_fossology_license_get = MagicMock( + return_value=[{ 'id': hash_to_bytes( '123caf10e9535160d90e874b45aa426de762f19f'), 'lang': 'python', - }) + }]) expected_license = { 'id': '123caf10e9535160d90e874b45aa426de762f19f', 'lang': 'python', } # when actual_license = service.lookup_content_license( 'sha1:123caf10e9535160d90e874b45aa426de762f19f') # then self.assertEqual(actual_license, expected_license) - mock_backend.content_license_get.assert_called_with( - hash_to_bytes('123caf10e9535160d90e874b45aa426de762f19f')) + mock_storage.content_fossology_license_get.assert_called_with( + [hash_to_bytes('123caf10e9535160d90e874b45aa426de762f19f')]) - @patch('swh.web.api.service.backend') + @patch('swh.web.api.service.storage') @istest - def lookup_content_license_2(self, mock_backend): + def lookup_content_license_2(self, mock_storage): # given - mock_backend.content_find = MagicMock( + mock_storage.content_find = MagicMock( return_value={ 'sha1': hash_to_bytes( '123caf10e9535160d90e874b45aa426de762f19f') } ) - mock_backend.content_license_get = MagicMock( - return_value={ + mock_storage.content_fossology_license_get = MagicMock( + return_value=[{ 'id': hash_to_bytes( '123caf10e9535160d90e874b45aa426de762f19f'), 'lang': 'haskell', - } + }] ) expected_license = { 'id': '123caf10e9535160d90e874b45aa426de762f19f', 'lang': 'haskell', } # when actual_license = service.lookup_content_license( 'sha1_git:456caf10e9535160d90e874b45aa426de762f19f') # then self.assertEqual(actual_license, expected_license) - mock_backend.content_find( + mock_storage.content_find( 'sha1_git', hash_to_bytes( '456caf10e9535160d90e874b45aa426de762f19f') ) - mock_backend.content_license_get.assert_called_with( - hash_to_bytes('123caf10e9535160d90e874b45aa426de762f19f')) + mock_storage.content_fossology_license_get.assert_called_with( + [hash_to_bytes('123caf10e9535160d90e874b45aa426de762f19f')]) - @patch('swh.web.api.service.backend') + @patch('swh.web.api.service.storage') @istest - def lookup_content_provenance(self, mock_backend): + def lookup_content_provenance(self, mock_storage): # given - mock_backend.content_find_provenance = MagicMock( + mock_storage.content_find_provenance = MagicMock( return_value=(p for p in [{ 'content': hash_to_bytes( '123caf10e9535160d90e874b45aa426de762f19f'), 'revision': hash_to_bytes( '456caf10e9535160d90e874b45aa426de762f19f'), 'origin': 100, 'visit': 1, 'path': b'octavio-3.4.0/octave.html/doc_002dS_005fISREG.html' }])) expected_provenances = [{ 'content': '123caf10e9535160d90e874b45aa426de762f19f', 'revision': '456caf10e9535160d90e874b45aa426de762f19f', 'origin': 100, 'visit': 1, 'path': 'octavio-3.4.0/octave.html/doc_002dS_005fISREG.html' }] # when actual_provenances = service.lookup_content_provenance( 'sha1_git:123caf10e9535160d90e874b45aa426de762f19f') # then self.assertEqual(list(actual_provenances), expected_provenances) - mock_backend.content_find_provenance.assert_called_with( - 'sha1_git', - hash_to_bytes('123caf10e9535160d90e874b45aa426de762f19f')) + mock_storage.content_find_provenance.assert_called_with( + {'sha1_git': + hash_to_bytes('123caf10e9535160d90e874b45aa426de762f19f')}) - @patch('swh.web.api.service.backend') + @patch('swh.web.api.service.storage') @istest - def lookup_content_provenance_not_found(self, mock_backend): + def lookup_content_provenance_not_found(self, mock_storage): # given - mock_backend.content_find_provenance = MagicMock(return_value=None) + mock_storage.content_find_provenance = MagicMock(return_value=None) # when actual_provenances = service.lookup_content_provenance( 'sha1_git:456caf10e9535160d90e874b45aa426de762f19f') # then self.assertIsNone(actual_provenances) - mock_backend.content_find_provenance.assert_called_with( - 'sha1_git', - hash_to_bytes('456caf10e9535160d90e874b45aa426de762f19f')) + mock_storage.content_find_provenance.assert_called_with( + {'sha1_git': + hash_to_bytes('456caf10e9535160d90e874b45aa426de762f19f')}) - @patch('swh.web.api.service.backend') + @patch('swh.web.api.service.storage') @istest - def stat_counters(self, mock_backend): + def stat_counters(self, mock_storage): # given input_stats = { "content": 1770830, "directory": 211683, "directory_entry_dir": 209167, "directory_entry_file": 1807094, "directory_entry_rev": 0, "entity": 0, "entity_history": 0, "occurrence": 0, "occurrence_history": 19600, "origin": 1096, "person": 0, "release": 8584, "revision": 7792, "revision_history": 0, "skipped_content": 0 } - mock_backend.stat_counters = MagicMock(return_value=input_stats) + mock_storage.stat_counters = MagicMock(return_value=input_stats) # when actual_stats = service.stat_counters() # then expected_stats = input_stats self.assertEqual(actual_stats, expected_stats) - mock_backend.stat_counters.assert_called_with() + mock_storage.stat_counters.assert_called_with() - @patch('swh.web.api.service.backend') + @patch('swh.web.api.service._lookup_origin_visits') @istest - def lookup_origin_visits(self, mock_backend): + def lookup_origin_visits(self, mock_lookup_visits): # given date_origin_visit2 = datetime.datetime( 2013, 7, 1, 20, 0, 0, tzinfo=datetime.timezone.utc) date_origin_visit3 = datetime.datetime( 2015, 1, 1, 21, 0, 0, tzinfo=datetime.timezone.utc) stub_result = [self.origin_visit1, { 'date': date_origin_visit2, 'origin': 1, 'visit': 2, 'target': hash_to_bytes( '65a55bbdf3629f916219feb3dcc7393ded1bc8db'), 'branch': b'master', 'target_type': 'release', 'metadata': None, }, { 'date': date_origin_visit3, 'origin': 1, 'visit': 3 }] - mock_backend.lookup_origin_visits.return_value = stub_result + mock_lookup_visits.return_value = stub_result # when expected_origin_visits = [{ 'date': self.origin_visit1['date'].isoformat(), 'origin': self.origin_visit1['origin'], 'visit': self.origin_visit1['visit'] }, { 'date': date_origin_visit2.isoformat(), 'origin': 1, 'visit': 2, 'target': '65a55bbdf3629f916219feb3dcc7393ded1bc8db', 'branch': 'master', 'target_type': 'release', 'metadata': {}, }, { 'date': date_origin_visit3.isoformat(), 'origin': 1, 'visit': 3 }] actual_origin_visits = service.lookup_origin_visits(6) # then self.assertEqual(list(actual_origin_visits), expected_origin_visits) - mock_backend.lookup_origin_visits.assert_called_once_with( + mock_lookup_visits.assert_called_once_with( 6, last_visit=None, limit=10) - @patch('swh.web.api.service.backend') + @patch('swh.web.api.service.storage') @istest - def lookup_origin_visit(self, mock_backend): + def lookup_origin_visit(self, mock_storage): # given stub_result = self.origin_visit1 - mock_backend.lookup_origin_visit.return_value = stub_result + mock_storage.origin_visit_get_by.return_value = stub_result expected_origin_visit = { 'date': self.origin_visit1['date'].isoformat(), 'origin': self.origin_visit1['origin'], 'visit': self.origin_visit1['visit'] } # when actual_origin_visit = service.lookup_origin_visit(1, 1) # then self.assertEqual(actual_origin_visit, expected_origin_visit) - mock_backend.lookup_origin_visit.assert_called_once_with(1, 1) + mock_storage.origin_visit_get_by.assert_called_once_with(1, 1) - @patch('swh.web.api.service.backend') + @patch('swh.web.api.service.storage') @istest - def lookup_origin(self, mock_backend): + def lookup_origin(self, mock_storage): # given - mock_backend.origin_get = MagicMock(return_value={ + mock_storage.origin_get = MagicMock(return_value={ 'id': 'origin-id', 'lister': 'uuid-lister', 'project': 'uuid-project', 'url': 'ftp://some/url/to/origin', 'type': 'ftp'}) # when actual_origin = service.lookup_origin({'id': 'origin-id'}) # then self.assertEqual(actual_origin, {'id': 'origin-id', 'lister': 'uuid-lister', 'project': 'uuid-project', 'url': 'ftp://some/url/to/origin', 'type': 'ftp'}) - mock_backend.origin_get.assert_called_with({'id': 'origin-id'}) + mock_storage.origin_get.assert_called_with({'id': 'origin-id'}) - @patch('swh.web.api.service.backend') + @patch('swh.web.api.service.storage') @istest def lookup_release_ko_id_checksum_not_ok_because_not_a_sha1(self, - mock_backend): + mock_storage): # given - mock_backend.release_get = MagicMock() + mock_storage.release_get = MagicMock() with self.assertRaises(BadInputExc) as cm: # when service.lookup_release('not-a-sha1') self.assertIn('invalid checksum', cm.exception.args[0]) - mock_backend.release_get.called = False + mock_storage.release_get.called = False - @patch('swh.web.api.service.backend') + @patch('swh.web.api.service.storage') @istest - def lookup_release_ko_id_checksum_ok_but_not_a_sha1(self, mock_backend): + def lookup_release_ko_id_checksum_ok_but_not_a_sha1(self, mock_storage): # given - mock_backend.release_get = MagicMock() + mock_storage.release_get = MagicMock() # when with self.assertRaises(BadInputExc) as cm: service.lookup_release( '13c1d34d138ec13b5ebad226dc2528dc7506c956e4646f62d4daf5' '1aea892abe') self.assertIn('sha1_git supported', cm.exception.args[0]) - mock_backend.release_get.called = False + mock_storage.release_get.called = False - @patch('swh.web.api.service.backend') + @patch('swh.web.api.service.storage') @istest - def lookup_directory_with_path_not_found(self, mock_backend): + def lookup_directory_with_path_not_found(self, mock_storage): # given - mock_backend.lookup_directory_with_path = MagicMock(return_value=None) + mock_storage.lookup_directory_with_path = MagicMock(return_value=None) sha1_git = '65a55bbdf3629f916219feb3dcc7393ded1bc8db' # when - actual_directory = mock_backend.lookup_directory_with_path( + actual_directory = mock_storage.lookup_directory_with_path( sha1_git, 'some/path/here') self.assertIsNone(actual_directory) - @patch('swh.web.api.service.backend') + @patch('swh.web.api.service.storage') @istest - def lookup_directory_with_path_found(self, mock_backend): + def lookup_directory_with_path_found(self, mock_storage): # given sha1_git = '65a55bbdf3629f916219feb3dcc7393ded1bc8db' entry = {'id': 'dir-id', 'type': 'dir', 'name': 'some/path/foo'} - mock_backend.lookup_directory_with_path = MagicMock(return_value=entry) + mock_storage.lookup_directory_with_path = MagicMock(return_value=entry) # when - actual_directory = mock_backend.lookup_directory_with_path( + actual_directory = mock_storage.lookup_directory_with_path( sha1_git, 'some/path/here') self.assertEqual(entry, actual_directory) - @patch('swh.web.api.service.backend') + @patch('swh.web.api.service.storage') @istest - def lookup_release(self, mock_backend): + def lookup_release(self, mock_storage): # given - mock_backend.release_get = MagicMock(return_value={ + mock_storage.release_get = MagicMock(return_value=[{ 'id': hash_to_bytes('65a55bbdf3629f916219feb3dcc7393ded1bc8db'), 'target': None, 'date': { 'timestamp': datetime.datetime( 2015, 1, 1, 22, 0, 0, tzinfo=datetime.timezone.utc).timestamp(), 'offset': 0, 'negative_utc': True, }, 'name': b'v0.0.1', 'message': b'synthetic release', 'synthetic': True, - }) + }]) # when actual_release = service.lookup_release( '65a55bbdf3629f916219feb3dcc7393ded1bc8db') # then self.assertEqual(actual_release, { 'id': '65a55bbdf3629f916219feb3dcc7393ded1bc8db', 'target': None, 'date': '2015-01-01T22:00:00-00:00', 'name': 'v0.0.1', 'message': 'synthetic release', 'synthetic': True, }) - mock_backend.release_get.assert_called_with( - hash_to_bytes('65a55bbdf3629f916219feb3dcc7393ded1bc8db')) + mock_storage.release_get.assert_called_with( + [hash_to_bytes('65a55bbdf3629f916219feb3dcc7393ded1bc8db')]) @istest def lookup_revision_with_context_ko_not_a_sha1_1(self): # given sha1_git = '13c1d34d138ec13b5ebad226dc2528dc7506c956e4646f62d4' \ 'daf51aea892abe' sha1_git_root = '65a55bbdf3629f916219feb3dcc7393ded1bc8db' # when with self.assertRaises(BadInputExc) as cm: service.lookup_revision_with_context(sha1_git_root, sha1_git) self.assertIn('Only sha1_git is supported', cm.exception.args[0]) @istest def lookup_revision_with_context_ko_not_a_sha1_2(self): # given sha1_git_root = '65a55bbdf3629f916219feb3dcc7393ded1bc8db' sha1_git = '13c1d34d138ec13b5ebad226dc2528dc7506c956e4646f6' \ '2d4daf51aea892abe' # when with self.assertRaises(BadInputExc) as cm: service.lookup_revision_with_context(sha1_git_root, sha1_git) self.assertIn('Only sha1_git is supported', cm.exception.args[0]) - @patch('swh.web.api.service.backend') + @patch('swh.web.api.service.storage') @istest def lookup_revision_with_context_ko_sha1_git_does_not_exist( self, - mock_backend): + mock_storage): # given sha1_git_root = '65a55bbdf3629f916219feb3dcc7393ded1bc8db' sha1_git = '777777bdf3629f916219feb3dcc7393ded1bc8db' sha1_git_bin = hash_to_bytes(sha1_git) - mock_backend.revision_get.return_value = None + mock_storage.revision_get.return_value = None # when with self.assertRaises(NotFoundExc) as cm: service.lookup_revision_with_context(sha1_git_root, sha1_git) self.assertIn('Revision 777777bdf3629f916219feb3dcc7393ded1bc8db' ' not found', cm.exception.args[0]) - mock_backend.revision_get.assert_called_once_with( - sha1_git_bin) + mock_storage.revision_get.assert_called_once_with( + [sha1_git_bin]) - @patch('swh.web.api.service.backend') + @patch('swh.web.api.service.storage') @istest def lookup_revision_with_context_ko_root_sha1_git_does_not_exist( self, - mock_backend): + mock_storage): # given sha1_git_root = '65a55bbdf3629f916219feb3dcc7393ded1bc8db' sha1_git = '777777bdf3629f916219feb3dcc7393ded1bc8db' sha1_git_root_bin = hash_to_bytes(sha1_git_root) sha1_git_bin = hash_to_bytes(sha1_git) - mock_backend.revision_get.side_effect = ['foo', None] + mock_storage.revision_get.side_effect = ['foo', None] # when with self.assertRaises(NotFoundExc) as cm: service.lookup_revision_with_context(sha1_git_root, sha1_git) self.assertIn('Revision 65a55bbdf3629f916219feb3dcc7393ded1bc8db' ' not found', cm.exception.args[0]) - mock_backend.revision_get.assert_has_calls([call(sha1_git_bin), - call(sha1_git_root_bin)]) + mock_storage.revision_get.assert_has_calls([call([sha1_git_bin]), + call([sha1_git_root_bin])]) - @patch('swh.web.api.service.backend') + @patch('swh.web.api.service.storage') @patch('swh.web.api.service.query') @istest - def lookup_revision_with_context(self, mock_query, mock_backend): + def lookup_revision_with_context(self, mock_query, mock_storage): # given sha1_git_root = '666' sha1_git = '883' sha1_git_root_bin = b'666' sha1_git_bin = b'883' sha1_git_root_dict = { 'id': sha1_git_root_bin, 'parents': [b'999'], } sha1_git_dict = { 'id': sha1_git_bin, 'parents': [], 'directory': b'278', } stub_revisions = [ sha1_git_root_dict, { 'id': b'999', 'parents': [b'777', b'883', b'888'], }, { 'id': b'777', 'parents': [b'883'], }, sha1_git_dict, { 'id': b'888', 'parents': [b'889'], }, { 'id': b'889', 'parents': [], }, ] # inputs ok mock_query.parse_hash_with_algorithms_or_throws.side_effect = [ ('sha1', sha1_git_bin), ('sha1', sha1_git_root_bin) ] # lookup revision first 883, then 666 (both exists) - mock_backend.revision_get.side_effect = [ + mock_storage.revision_get.return_value = [ sha1_git_dict, sha1_git_root_dict ] - mock_backend.revision_log = MagicMock( + mock_storage.revision_log = MagicMock( return_value=stub_revisions) # when actual_revision = service.lookup_revision_with_context( sha1_git_root, sha1_git) # then self.assertEquals(actual_revision, { 'id': hash_to_hex(sha1_git_bin), 'parents': [], 'children': [hash_to_hex(b'999'), hash_to_hex(b'777')], 'directory': hash_to_hex(b'278'), 'merge': False }) mock_query.parse_hash_with_algorithms_or_throws.assert_has_calls( [call(sha1_git, ['sha1'], 'Only sha1_git is supported.'), call(sha1_git_root, ['sha1'], 'Only sha1_git is supported.')]) - mock_backend.revision_log.assert_called_with( - sha1_git_root_bin, 100) + mock_storage.revision_log.assert_called_with( + [sha1_git_root_bin], 100) - @patch('swh.web.api.service.backend') + @patch('swh.web.api.service.storage') @patch('swh.web.api.service.query') @istest def lookup_revision_with_context_sha1_git_root_already_retrieved_as_dict( - self, mock_query, mock_backend): + self, mock_query, mock_storage): # given sha1_git = '883' sha1_git_root_bin = b'666' sha1_git_bin = b'883' sha1_git_root_dict = { 'id': sha1_git_root_bin, 'parents': [b'999'], } sha1_git_dict = { 'id': sha1_git_bin, 'parents': [], 'directory': b'278', } stub_revisions = [ sha1_git_root_dict, { 'id': b'999', 'parents': [b'777', b'883', b'888'], }, { 'id': b'777', 'parents': [b'883'], }, sha1_git_dict, { 'id': b'888', 'parents': [b'889'], }, { 'id': b'889', 'parents': [], }, ] # inputs ok mock_query.parse_hash_with_algorithms_or_throws.return_value = ( 'sha1', sha1_git_bin) # lookup only on sha1 - mock_backend.revision_get.return_value = sha1_git_dict + mock_storage.revision_get.return_value = [sha1_git_dict] - mock_backend.revision_log.return_value = stub_revisions + mock_storage.revision_log.return_value = stub_revisions # when actual_revision = service.lookup_revision_with_context( {'id': sha1_git_root_bin}, sha1_git) # then self.assertEquals(actual_revision, { 'id': hash_to_hex(sha1_git_bin), 'parents': [], 'children': [hash_to_hex(b'999'), hash_to_hex(b'777')], 'directory': hash_to_hex(b'278'), 'merge': False }) mock_query.parse_hash_with_algorithms_or_throws.assert_called_once_with( # noqa sha1_git, ['sha1'], 'Only sha1_git is supported.') - mock_backend.revision_get.assert_called_once_with(sha1_git_bin) + mock_storage.revision_get.assert_called_once_with([sha1_git_bin]) - mock_backend.revision_log.assert_called_with( - sha1_git_root_bin, 100) + mock_storage.revision_log.assert_called_with( + [sha1_git_root_bin], 100) - @patch('swh.web.api.service.backend') + @patch('swh.web.api.service.storage') @patch('swh.web.api.service.query') @istest def lookup_directory_with_revision_ko_revision_not_found(self, mock_query, - mock_backend): + mock_storage): # given mock_query.parse_hash_with_algorithms_or_throws.return_value = ('sha1', b'123') - mock_backend.revision_get.return_value = None + mock_storage.revision_get.return_value = None # when with self.assertRaises(NotFoundExc) as cm: service.lookup_directory_with_revision('123') self.assertIn('Revision 123 not found', cm.exception.args[0]) mock_query.parse_hash_with_algorithms_or_throws.assert_called_once_with ('123', ['sha1'], 'Only sha1_git is supported.') - mock_backend.revision_get.assert_called_once_with(b'123') + mock_storage.revision_get.assert_called_once_with([b'123']) - @patch('swh.web.api.service.backend') + @patch('swh.web.api.service.storage') @patch('swh.web.api.service.query') @istest def lookup_directory_with_revision_ko_revision_with_path_to_nowhere( self, mock_query, - mock_backend): + mock_storage): # given mock_query.parse_hash_with_algorithms_or_throws.return_value = ('sha1', b'123') dir_id = b'dir-id-as-sha1' - mock_backend.revision_get.return_value = { + mock_storage.revision_get.return_value = [{ 'directory': dir_id, - } + }] - mock_backend.directory_entry_get_by_path.return_value = None + mock_storage.directory_entry_get_by_path.return_value = None # when with self.assertRaises(NotFoundExc) as cm: service.lookup_directory_with_revision( '123', 'path/to/something/unknown') self.assertIn("Directory/File 'path/to/something/unknown' " + "pointed to by revision 123 not found", cm.exception.args[0]) mock_query.parse_hash_with_algorithms_or_throws.assert_called_once_with ('123', ['sha1'], 'Only sha1_git is supported.') - mock_backend.revision_get.assert_called_once_with(b'123') - mock_backend.directory_entry_get_by_path.assert_called_once_with( - b'dir-id-as-sha1', 'path/to/something/unknown') + mock_storage.revision_get.assert_called_once_with([b'123']) + mock_storage.directory_entry_get_by_path.assert_called_once_with( + b'dir-id-as-sha1', [b'path', b'to', b'something', b'unknown']) - @patch('swh.web.api.service.backend') + @patch('swh.web.api.service.storage') @patch('swh.web.api.service.query') @istest def lookup_directory_with_revision_ko_type_not_implemented( self, mock_query, - mock_backend): + mock_storage): # given mock_query.parse_hash_with_algorithms_or_throws.return_value = ('sha1', b'123') dir_id = b'dir-id-as-sha1' - mock_backend.revision_get.return_value = { + mock_storage.revision_get.return_value = [{ 'directory': dir_id, - } + }] - mock_backend.directory_entry_get_by_path.return_value = { + mock_storage.directory_entry_get_by_path.return_value = { 'type': 'rev', 'name': b'some/path/to/rev', 'target': b'456' } stub_content = { 'id': b'12', 'type': 'file' } - mock_backend.content_get.return_value = stub_content + mock_storage.content_get.return_value = stub_content # when with self.assertRaises(NotImplementedError) as cm: service.lookup_directory_with_revision( '123', 'some/path/to/rev') self.assertIn("Entity of type 'rev' not implemented.", cm.exception.args[0]) # then mock_query.parse_hash_with_algorithms_or_throws.assert_called_once_with ('123', ['sha1'], 'Only sha1_git is supported.') - mock_backend.revision_get.assert_called_once_with(b'123') - mock_backend.directory_entry_get_by_path.assert_called_once_with( - b'dir-id-as-sha1', 'some/path/to/rev') + mock_storage.revision_get.assert_called_once_with([b'123']) + mock_storage.directory_entry_get_by_path.assert_called_once_with( + b'dir-id-as-sha1', [b'some', b'path', b'to', b'rev']) - @patch('swh.web.api.service.backend') + @patch('swh.web.api.service.storage') @patch('swh.web.api.service.query') @istest def lookup_directory_with_revision_revision_without_path(self, mock_query, - mock_backend): + mock_storage): # given mock_query.parse_hash_with_algorithms_or_throws.return_value = ('sha1', b'123') dir_id = b'dir-id-as-sha1' - mock_backend.revision_get.return_value = { + mock_storage.revision_get.return_value = [{ 'directory': dir_id, - } + }] stub_dir_entries = [{ 'id': b'123', 'type': 'dir' }, { 'id': b'456', 'type': 'file' }] - mock_backend.directory_ls.return_value = stub_dir_entries + mock_storage.directory_ls.return_value = stub_dir_entries # when actual_directory_entries = service.lookup_directory_with_revision( '123') self.assertEqual(actual_directory_entries['type'], 'dir') self.assertEqual(list(actual_directory_entries['content']), stub_dir_entries) mock_query.parse_hash_with_algorithms_or_throws.assert_called_once_with ('123', ['sha1'], 'Only sha1_git is supported.') - mock_backend.revision_get.assert_called_once_with(b'123') - mock_backend.directory_ls.assert_called_once_with(dir_id) + mock_storage.revision_get.assert_called_once_with([b'123']) + mock_storage.directory_ls.assert_called_once_with(dir_id) - @patch('swh.web.api.service.backend') + @patch('swh.web.api.service.storage') @patch('swh.web.api.service.query') @istest def lookup_directory_with_revision_revision_with_path_to_dir(self, mock_query, - mock_backend): + mock_storage): # given mock_query.parse_hash_with_algorithms_or_throws.return_value = ('sha1', b'123') dir_id = b'dir-id-as-sha1' - mock_backend.revision_get.return_value = { + mock_storage.revision_get.return_value = [{ 'directory': dir_id, - } + }] stub_dir_entries = [{ 'id': b'12', 'type': 'dir' }, { 'id': b'34', 'type': 'file' }] - mock_backend.directory_entry_get_by_path.return_value = { + mock_storage.directory_entry_get_by_path.return_value = { 'type': 'dir', 'name': b'some/path', 'target': b'456' } - mock_backend.directory_ls.return_value = stub_dir_entries + mock_storage.directory_ls.return_value = stub_dir_entries # when actual_directory_entries = service.lookup_directory_with_revision( '123', 'some/path') self.assertEqual(actual_directory_entries['type'], 'dir') self.assertEqual(actual_directory_entries['revision'], '123') self.assertEqual(actual_directory_entries['path'], 'some/path') self.assertEqual(list(actual_directory_entries['content']), stub_dir_entries) mock_query.parse_hash_with_algorithms_or_throws.assert_called_once_with ('123', ['sha1'], 'Only sha1_git is supported.') - mock_backend.revision_get.assert_called_once_with(b'123') - mock_backend.directory_entry_get_by_path.assert_called_once_with( + mock_storage.revision_get.assert_called_once_with([b'123']) + mock_storage.directory_entry_get_by_path.assert_called_once_with( dir_id, - 'some/path') - mock_backend.directory_ls.assert_called_once_with(b'456') + [b'some', b'path']) + mock_storage.directory_ls.assert_called_once_with(b'456') - @patch('swh.web.api.service.backend') + @patch('swh.web.api.service.storage') @patch('swh.web.api.service.query') @istest def lookup_directory_with_revision_revision_with_path_to_file_without_data( self, mock_query, - mock_backend): + mock_storage): # given mock_query.parse_hash_with_algorithms_or_throws.return_value = ('sha1', b'123') dir_id = b'dir-id-as-sha1' - mock_backend.revision_get.return_value = { + mock_storage.revision_get.return_value = [{ 'directory': dir_id, - } + }] - mock_backend.directory_entry_get_by_path.return_value = { + mock_storage.directory_entry_get_by_path.return_value = { 'type': 'file', 'name': b'some/path/to/file', 'target': b'789' } stub_content = { 'status': 'visible', } - mock_backend.content_find.return_value = stub_content + mock_storage.content_find.return_value = stub_content # when actual_content = service.lookup_directory_with_revision( '123', 'some/path/to/file') # then self.assertEqual(actual_content, {'type': 'file', 'revision': '123', 'path': 'some/path/to/file', 'content': stub_content}) mock_query.parse_hash_with_algorithms_or_throws.assert_called_once_with ('123', ['sha1'], 'Only sha1_git is supported.') - mock_backend.revision_get.assert_called_once_with(b'123') - mock_backend.directory_entry_get_by_path.assert_called_once_with( - b'dir-id-as-sha1', 'some/path/to/file') - mock_backend.content_find.assert_called_once_with('sha1_git', b'789') + mock_storage.revision_get.assert_called_once_with([b'123']) + mock_storage.directory_entry_get_by_path.assert_called_once_with( + b'dir-id-as-sha1', [b'some', b'path', b'to', b'file']) + mock_storage.content_find.assert_called_once_with({'sha1_git': b'789'}) - @patch('swh.web.api.service.backend') + @patch('swh.web.api.service.storage') @patch('swh.web.api.service.query') @istest def lookup_directory_with_revision_revision_with_path_to_file_with_data( self, mock_query, - mock_backend): + mock_storage): # given mock_query.parse_hash_with_algorithms_or_throws.return_value = ('sha1', b'123') dir_id = b'dir-id-as-sha1' - mock_backend.revision_get.return_value = { + mock_storage.revision_get.return_value = [{ 'directory': dir_id, - } + }] - mock_backend.directory_entry_get_by_path.return_value = { + mock_storage.directory_entry_get_by_path.return_value = { 'type': 'file', 'name': b'some/path/to/file', 'target': b'789' } stub_content = { 'status': 'visible', 'sha1': b'content-sha1' } - mock_backend.content_find.return_value = stub_content - mock_backend.content_get.return_value = { + mock_storage.content_find.return_value = stub_content + mock_storage.content_get.return_value = [{ 'sha1': b'content-sha1', 'data': b'some raw data' - } + }] expected_content = { 'status': 'visible', 'sha1': hash_to_hex(b'content-sha1'), 'data': b'some raw data' } # when actual_content = service.lookup_directory_with_revision( '123', 'some/path/to/file', with_data=True) # then self.assertEqual(actual_content, {'type': 'file', 'revision': '123', 'path': 'some/path/to/file', 'content': expected_content}) mock_query.parse_hash_with_algorithms_or_throws.assert_called_once_with ('123', ['sha1'], 'Only sha1_git is supported.') - mock_backend.revision_get.assert_called_once_with(b'123') - mock_backend.directory_entry_get_by_path.assert_called_once_with( - b'dir-id-as-sha1', 'some/path/to/file') - mock_backend.content_find.assert_called_once_with('sha1_git', b'789') - mock_backend.content_get.assert_called_once_with(b'content-sha1') + mock_storage.revision_get.assert_called_once_with([b'123']) + mock_storage.directory_entry_get_by_path.assert_called_once_with( + b'dir-id-as-sha1', [b'some', b'path', b'to', b'file']) + mock_storage.content_find.assert_called_once_with({'sha1_git': b'789'}) + mock_storage.content_get.assert_called_once_with([b'content-sha1']) - @patch('swh.web.api.service.backend') + @patch('swh.web.api.service.storage') @istest - def lookup_revision(self, mock_backend): + def lookup_revision(self, mock_storage): # given - mock_backend.revision_get = MagicMock( - return_value=self.SAMPLE_REVISION_RAW) + mock_storage.revision_get = MagicMock( + return_value=[self.SAMPLE_REVISION_RAW]) # when actual_revision = service.lookup_revision( self.SHA1_SAMPLE) # then self.assertEqual(actual_revision, self.SAMPLE_REVISION) - mock_backend.revision_get.assert_called_with( - self.SHA1_SAMPLE_BIN) + mock_storage.revision_get.assert_called_with( + [self.SHA1_SAMPLE_BIN]) - @patch('swh.web.api.service.backend') + @patch('swh.web.api.service.storage') @istest - def lookup_revision_invalid_msg(self, mock_backend): + def lookup_revision_invalid_msg(self, mock_storage): # given stub_rev = self.SAMPLE_REVISION_RAW stub_rev['message'] = b'elegant fix for bug \xff' expected_revision = self.SAMPLE_REVISION expected_revision['message'] = None expected_revision['message_decoding_failed'] = True - mock_backend.revision_get = MagicMock(return_value=stub_rev) + mock_storage.revision_get = MagicMock(return_value=[stub_rev]) # when actual_revision = service.lookup_revision( self.SHA1_SAMPLE) # then self.assertEqual(actual_revision, expected_revision) - mock_backend.revision_get.assert_called_with( - self.SHA1_SAMPLE_BIN) + mock_storage.revision_get.assert_called_with( + [self.SHA1_SAMPLE_BIN]) - @patch('swh.web.api.service.backend') + @patch('swh.web.api.service.storage') @istest - def lookup_revision_msg_ok(self, mock_backend): + def lookup_revision_msg_ok(self, mock_storage): # given - mock_backend.revision_get.return_value = self.SAMPLE_REVISION_RAW + mock_storage.revision_get.return_value = [self.SAMPLE_REVISION_RAW] # when rv = service.lookup_revision_message( self.SHA1_SAMPLE) # then self.assertEquals(rv, {'message': self.SAMPLE_MESSAGE_BIN}) - mock_backend.revision_get.assert_called_with( - self.SHA1_SAMPLE_BIN) + mock_storage.revision_get.assert_called_with( + [self.SHA1_SAMPLE_BIN]) - @patch('swh.web.api.service.backend') + @patch('swh.web.api.service.storage') @istest - def lookup_revision_msg_absent(self, mock_backend): + def lookup_revision_msg_absent(self, mock_storage): # given stub_revision = self.SAMPLE_REVISION_RAW del stub_revision['message'] - mock_backend.revision_get.return_value = stub_revision + mock_storage.revision_get.return_value = stub_revision # when with self.assertRaises(NotFoundExc) as cm: service.lookup_revision_message( self.SHA1_SAMPLE) # then - mock_backend.revision_get.assert_called_with( + mock_storage.revision_get.assert_called_with( self.SHA1_SAMPLE_BIN) self.assertEqual(cm.exception.args[0], 'No message for revision ' 'with sha1_git ' '18d8be353ed3480476f032475e7c233eff7371d5.') - @patch('swh.web.api.service.backend') + @patch('swh.web.api.service.storage') @istest - def lookup_revision_msg_norev(self, mock_backend): + def lookup_revision_msg_norev(self, mock_storage): # given - mock_backend.revision_get.return_value = None + mock_storage.revision_get.return_value = None # when with self.assertRaises(NotFoundExc) as cm: service.lookup_revision_message( self.SHA1_SAMPLE) # then - mock_backend.revision_get.assert_called_with( + mock_storage.revision_get.assert_called_with( self.SHA1_SAMPLE_BIN) self.assertEqual(cm.exception.args[0], 'Revision with sha1_git ' '18d8be353ed3480476f032475e7c233eff7371d5 ' 'not found.') - @patch('swh.web.api.service.backend') + @patch('swh.web.api.service.storage') @istest - def lookup_revision_multiple(self, mock_backend): + def lookup_revision_multiple(self, mock_storage): # given sha1 = self.SHA1_SAMPLE sha1_other = 'adc83b19e793491b1c6ea0fd8b46cd9f32e592fc' stub_revisions = [ self.SAMPLE_REVISION_RAW, { 'id': hash_to_bytes(sha1_other), 'directory': 'abcdbe353ed3480476f032475e7c233eff7371d5', 'author': { 'name': b'name', 'email': b'name@surname.org', }, 'committer': { 'name': b'name', 'email': b'name@surname.org', }, 'message': b'ugly fix for bug 42', 'date': { 'timestamp': datetime.datetime( 2000, 1, 12, 5, 23, 54, tzinfo=datetime.timezone.utc).timestamp(), 'offset': 0, 'negative_utc': False }, 'date_offset': 0, 'committer_date': { 'timestamp': datetime.datetime( 2000, 1, 12, 5, 23, 54, tzinfo=datetime.timezone.utc).timestamp(), 'offset': 0, 'negative_utc': False }, 'committer_date_offset': 0, 'synthetic': False, 'type': 'git', 'parents': [], 'metadata': [], } ] - mock_backend.revision_get_multiple.return_value = stub_revisions + mock_storage.revision_get.return_value = stub_revisions # when actual_revisions = service.lookup_revision_multiple( [sha1, sha1_other]) # then self.assertEqual(list(actual_revisions), [ self.SAMPLE_REVISION, { 'id': sha1_other, 'directory': 'abcdbe353ed3480476f032475e7c233eff7371d5', 'author': { 'name': 'name', 'email': 'name@surname.org', }, 'committer': { 'name': 'name', 'email': 'name@surname.org', }, 'message': 'ugly fix for bug 42', 'date': '2000-01-12T05:23:54+00:00', 'date_offset': 0, 'committer_date': '2000-01-12T05:23:54+00:00', 'committer_date_offset': 0, 'synthetic': False, 'type': 'git', 'parents': [], 'metadata': {}, 'merge': False } ]) self.assertEqual( - list(mock_backend.revision_get_multiple.call_args[0][0]), + list(mock_storage.revision_get.call_args[0][0]), [hash_to_bytes(sha1), hash_to_bytes(sha1_other)]) - @patch('swh.web.api.service.backend') + @patch('swh.web.api.service.storage') @istest - def lookup_revision_multiple_none_found(self, mock_backend): + def lookup_revision_multiple_none_found(self, mock_storage): # given sha1_bin = self.SHA1_SAMPLE sha1_other = 'adc83b19e793491b1c6ea0fd8b46cd9f32e592fc' - mock_backend.revision_get_multiple.return_value = [] + mock_storage.revision_get.return_value = [] # then actual_revisions = service.lookup_revision_multiple( [sha1_bin, sha1_other]) self.assertEqual(list(actual_revisions), []) self.assertEqual( - list(mock_backend.revision_get_multiple.call_args[0][0]), + list(mock_storage.revision_get.call_args[0][0]), [hash_to_bytes(self.SHA1_SAMPLE), hash_to_bytes(sha1_other)]) - @patch('swh.web.api.service.backend') + @patch('swh.web.api.service.storage') @istest - def lookup_revision_log(self, mock_backend): + def lookup_revision_log(self, mock_storage): # given stub_revision_log = [self.SAMPLE_REVISION_RAW] - mock_backend.revision_log = MagicMock(return_value=stub_revision_log) + mock_storage.revision_log = MagicMock(return_value=stub_revision_log) # when actual_revision = service.lookup_revision_log( 'abcdbe353ed3480476f032475e7c233eff7371d5', limit=25) # then self.assertEqual(list(actual_revision), [self.SAMPLE_REVISION]) - mock_backend.revision_log.assert_called_with( - hash_to_bytes('abcdbe353ed3480476f032475e7c233eff7371d5'), 25) + mock_storage.revision_log.assert_called_with( + [hash_to_bytes('abcdbe353ed3480476f032475e7c233eff7371d5')], 25) - @patch('swh.web.api.service.backend') + @patch('swh.web.api.service.storage') @istest - def lookup_revision_log_by(self, mock_backend): + def lookup_revision_log_by(self, mock_storage): # given stub_revision_log = [self.SAMPLE_REVISION_RAW] - mock_backend.revision_log_by = MagicMock( + mock_storage.revision_log_by = MagicMock( return_value=stub_revision_log) # when actual_log = service.lookup_revision_log_by( 1, 'refs/heads/master', None, limit=100) # then self.assertEqual(list(actual_log), [self.SAMPLE_REVISION]) - mock_backend.revision_log_by.assert_called_with( - 1, 'refs/heads/master', None, 100) + mock_storage.revision_log_by.assert_called_with( + 1, 'refs/heads/master', None, limit=100) - @patch('swh.web.api.service.backend') + @patch('swh.web.api.service.storage') @istest - def lookup_revision_log_by_nolog(self, mock_backend): + def lookup_revision_log_by_nolog(self, mock_storage): # given - mock_backend.revision_log_by = MagicMock(return_value=None) + mock_storage.revision_log_by = MagicMock(return_value=None) # when res = service.lookup_revision_log_by( 1, 'refs/heads/master', None, limit=100) # then self.assertEquals(res, None) - mock_backend.revision_log_by.assert_called_with( - 1, 'refs/heads/master', None, 100) + mock_storage.revision_log_by.assert_called_with( + 1, 'refs/heads/master', None, limit=100) - @patch('swh.web.api.service.backend') + @patch('swh.web.api.service.storage') @istest - def lookup_content_raw_not_found(self, mock_backend): + def lookup_content_raw_not_found(self, mock_storage): # given - mock_backend.content_find = MagicMock(return_value=None) + mock_storage.content_find = MagicMock(return_value=None) # when actual_content = service.lookup_content_raw( 'sha1:18d8be353ed3480476f032475e7c233eff7371d5') # then self.assertIsNone(actual_content) - mock_backend.content_find.assert_called_with( - 'sha1', hash_to_bytes(self.SHA1_SAMPLE)) + mock_storage.content_find.assert_called_with( + {'sha1': hash_to_bytes(self.SHA1_SAMPLE)}) - @patch('swh.web.api.service.backend') + @patch('swh.web.api.service.storage') @istest - def lookup_content_raw(self, mock_backend): + def lookup_content_raw(self, mock_storage): # given - mock_backend.content_find = MagicMock(return_value={ + mock_storage.content_find = MagicMock(return_value={ 'sha1': self.SHA1_SAMPLE, }) - mock_backend.content_get = MagicMock(return_value={ - 'data': b'binary data'}) + mock_storage.content_get = MagicMock(return_value=[{ + 'data': b'binary data'}]) # when actual_content = service.lookup_content_raw( 'sha256:%s' % self.SHA256_SAMPLE) # then self.assertEquals(actual_content, {'data': b'binary data'}) - mock_backend.content_find.assert_called_once_with( - 'sha256', self.SHA256_SAMPLE_BIN) - mock_backend.content_get.assert_called_once_with( - self.SHA1_SAMPLE) + mock_storage.content_find.assert_called_once_with( + {'sha256': self.SHA256_SAMPLE_BIN}) + mock_storage.content_get.assert_called_once_with( + [self.SHA1_SAMPLE]) - @patch('swh.web.api.service.backend') + @patch('swh.web.api.service.storage') @istest - def lookup_content_not_found(self, mock_backend): + def lookup_content_not_found(self, mock_storage): # given - mock_backend.content_find = MagicMock(return_value=None) + mock_storage.content_find = MagicMock(return_value=None) # when actual_content = service.lookup_content( 'sha1:%s' % self.SHA1_SAMPLE) # then self.assertIsNone(actual_content) - mock_backend.content_find.assert_called_with( - 'sha1', self.SHA1_SAMPLE_BIN) + mock_storage.content_find.assert_called_with( + {'sha1': self.SHA1_SAMPLE_BIN}) - @patch('swh.web.api.service.backend') + @patch('swh.web.api.service.storage') @istest - def lookup_content_with_sha1(self, mock_backend): + def lookup_content_with_sha1(self, mock_storage): # given - mock_backend.content_find = MagicMock( + mock_storage.content_find = MagicMock( return_value=self.SAMPLE_CONTENT_RAW) # when actual_content = service.lookup_content( 'sha1:%s' % self.SHA1_SAMPLE) # then self.assertEqual(actual_content, self.SAMPLE_CONTENT) - mock_backend.content_find.assert_called_with( - 'sha1', hash_to_bytes(self.SHA1_SAMPLE)) + mock_storage.content_find.assert_called_with( + {'sha1': hash_to_bytes(self.SHA1_SAMPLE)}) - @patch('swh.web.api.service.backend') + @patch('swh.web.api.service.storage') @istest - def lookup_content_with_sha256(self, mock_backend): + def lookup_content_with_sha256(self, mock_storage): # given stub_content = self.SAMPLE_CONTENT_RAW stub_content['status'] = 'visible' expected_content = self.SAMPLE_CONTENT expected_content['status'] = 'visible' - mock_backend.content_find = MagicMock( + mock_storage.content_find = MagicMock( return_value=stub_content) # when actual_content = service.lookup_content( 'sha256:%s' % self.SHA256_SAMPLE) # then self.assertEqual(actual_content, expected_content) - mock_backend.content_find.assert_called_with( - 'sha256', self.SHA256_SAMPLE_BIN) + mock_storage.content_find.assert_called_with( + {'sha256': self.SHA256_SAMPLE_BIN}) - @patch('swh.web.api.service.backend') + @patch('swh.web.api.service.storage') @istest - def lookup_person(self, mock_backend): + def lookup_person(self, mock_storage): # given - mock_backend.person_get = MagicMock(return_value={ + mock_storage.person_get = MagicMock(return_value=[{ 'id': 'person_id', 'name': b'some_name', 'email': b'some-email', - }) + }]) # when actual_person = service.lookup_person('person_id') # then self.assertEqual(actual_person, { 'id': 'person_id', 'name': 'some_name', 'email': 'some-email', }) - mock_backend.person_get.assert_called_with('person_id') + mock_storage.person_get.assert_called_with(['person_id']) - @patch('swh.web.api.service.backend') + @patch('swh.web.api.service.storage') @istest - def lookup_directory_bad_checksum(self, mock_backend): + def lookup_directory_bad_checksum(self, mock_storage): # given - mock_backend.directory_ls = MagicMock() + mock_storage.directory_ls = MagicMock() # when with self.assertRaises(BadInputExc): service.lookup_directory('directory_id') # then - mock_backend.directory_ls.called = False + mock_storage.directory_ls.called = False - @patch('swh.web.api.service.backend') + @patch('swh.web.api.service.storage') @patch('swh.web.api.service.query') @istest - def lookup_directory_not_found(self, mock_query, mock_backend): + def lookup_directory_not_found(self, mock_query, mock_storage): # given mock_query.parse_hash_with_algorithms_or_throws.return_value = ( 'sha1', 'directory-id-bin') - mock_backend.directory_get.return_value = None + mock_storage.directory_get.return_value = None # when actual_dir = service.lookup_directory('directory_id') # then self.assertIsNone(actual_dir) mock_query.parse_hash_with_algorithms_or_throws.assert_called_with( 'directory_id', ['sha1'], 'Only sha1_git is supported.') - mock_backend.directory_get.assert_called_with('directory-id-bin') - mock_backend.directory_ls.called = False + mock_storage.directory_get.assert_called_with(['directory-id-bin']) + mock_storage.directory_ls.called = False - @patch('swh.web.api.service.backend') + @patch('swh.web.api.service.storage') @patch('swh.web.api.service.query') @istest - def lookup_directory(self, mock_query, mock_backend): + def lookup_directory(self, mock_query, mock_storage): mock_query.parse_hash_with_algorithms_or_throws.return_value = ( 'sha1', 'directory-sha1-bin') # something that exists is all that matters here - mock_backend.directory_get.return_value = {'id': b'directory-sha1-bin'} + mock_storage.directory_get.return_value = {'id': b'directory-sha1-bin'} # given stub_dir_entries = [{ 'sha1': self.SHA1_SAMPLE_BIN, 'sha256': self.SHA256_SAMPLE_BIN, 'sha1_git': self.SHA1GIT_SAMPLE_BIN, 'target': hash_to_bytes( '40e71b8614fcd89ccd17ca2b1d9e66c5b00a6d03'), 'dir_id': self.DIRECTORY_ID_BIN, 'name': b'bob', 'type': 10, }] expected_dir_entries = [{ 'sha1': self.SHA1_SAMPLE, 'sha256': self.SHA256_SAMPLE, 'sha1_git': self.SHA1GIT_SAMPLE, 'target': '40e71b8614fcd89ccd17ca2b1d9e66c5b00a6d03', 'dir_id': self.DIRECTORY_ID, 'name': 'bob', 'type': 10, }] - mock_backend.directory_ls.return_value = stub_dir_entries + mock_storage.directory_ls.return_value = stub_dir_entries # when actual_directory_ls = list(service.lookup_directory( 'directory-sha1')) # then self.assertEqual(actual_directory_ls, expected_dir_entries) mock_query.parse_hash_with_algorithms_or_throws.assert_called_with( 'directory-sha1', ['sha1'], 'Only sha1_git is supported.') - mock_backend.directory_ls.assert_called_with( + mock_storage.directory_ls.assert_called_with( 'directory-sha1-bin') - @patch('swh.web.api.service.backend') + @patch('swh.web.api.service.storage') @istest - def lookup_revision_by_nothing_found(self, mock_backend): + def lookup_revision_by_nothing_found(self, mock_storage): # given - mock_backend.revision_get_by.return_value = None + mock_storage.revision_get_by.return_value = None # when actual_revisions = service.lookup_revision_by(1) # then self.assertIsNone(actual_revisions) - mock_backend.revision_get_by(1, 'master', None) + mock_storage.revision_get_by.assert_called_with(1, 'refs/heads/master', + limit=1, + timestamp=None) - @patch('swh.web.api.service.backend') + @patch('swh.web.api.service.storage') @istest - def lookup_revision_by(self, mock_backend): + def lookup_revision_by(self, mock_storage): # given stub_rev = self.SAMPLE_REVISION_RAW expected_rev = self.SAMPLE_REVISION - mock_backend.revision_get_by.return_value = stub_rev + mock_storage.revision_get_by.return_value = [stub_rev] # when actual_revision = service.lookup_revision_by(10, 'master2', 'some-ts') # then self.assertEquals(actual_revision, expected_rev) - mock_backend.revision_get_by(1, 'master2', 'some-ts') + mock_storage.revision_get_by.assert_called_with(10, 'master2', + limit=1, + timestamp='some-ts') - @patch('swh.web.api.service.backend') + @patch('swh.web.api.service.storage') @istest - def lookup_revision_by_nomerge(self, mock_backend): + def lookup_revision_by_nomerge(self, mock_storage): # given stub_rev = self.SAMPLE_REVISION_RAW stub_rev['parents'] = [ hash_to_bytes('adc83b19e793491b1c6ea0fd8b46cd9f32e592fc')] expected_rev = self.SAMPLE_REVISION expected_rev['parents'] = ['adc83b19e793491b1c6ea0fd8b46cd9f32e592fc'] - mock_backend.revision_get_by.return_value = stub_rev + mock_storage.revision_get_by.return_value = [stub_rev] # when actual_revision = service.lookup_revision_by(10, 'master2', 'some-ts') # then self.assertEquals(actual_revision, expected_rev) - mock_backend.revision_get_by(1, 'master2', 'some-ts') + mock_storage.revision_get_by.assert_called_with(10, 'master2', + limit=1, + timestamp='some-ts') - @patch('swh.web.api.service.backend') + @patch('swh.web.api.service.storage') @istest - def lookup_revision_by_merge(self, mock_backend): + def lookup_revision_by_merge(self, mock_storage): # given stub_rev = self.SAMPLE_REVISION_RAW stub_rev['parents'] = [ hash_to_bytes('adc83b19e793491b1c6ea0fd8b46cd9f32e592fc'), hash_to_bytes('ffff3b19e793491b1c6db0fd8b46cd9f32e592fc') ] expected_rev = self.SAMPLE_REVISION expected_rev['parents'] = [ 'adc83b19e793491b1c6ea0fd8b46cd9f32e592fc', 'ffff3b19e793491b1c6db0fd8b46cd9f32e592fc' ] expected_rev['merge'] = True - mock_backend.revision_get_by.return_value = stub_rev + mock_storage.revision_get_by.return_value = [stub_rev] # when actual_revision = service.lookup_revision_by(10, 'master2', 'some-ts') # then self.assertEquals(actual_revision, expected_rev) - mock_backend.revision_get_by(1, 'master2', 'some-ts') + mock_storage.revision_get_by.assert_called_with(10, 'master2', + limit=1, + timestamp='some-ts') - @patch('swh.web.api.service.backend') + @patch('swh.web.api.service.storage') @istest - def lookup_revision_with_context_by_ko(self, mock_backend): + def lookup_revision_with_context_by_ko(self, mock_storage): # given - mock_backend.revision_get_by.return_value = None + mock_storage.revision_get_by.return_value = None # when with self.assertRaises(NotFoundExc) as cm: origin_id = 1 branch_name = 'master3' ts = None service.lookup_revision_with_context_by(origin_id, branch_name, ts, 'sha1') # then self.assertIn( 'Revision with (origin_id: %s, branch_name: %s' ', ts: %s) not found.' % (origin_id, branch_name, ts), cm.exception.args[0]) - mock_backend.revision_get_by.assert_called_once_with( + mock_storage.revision_get_by.assert_called_once_with( origin_id, branch_name, ts) @patch('swh.web.api.service.lookup_revision_with_context') - @patch('swh.web.api.service.backend') + @patch('swh.web.api.service.storage') @istest - def lookup_revision_with_context_by(self, mock_backend, + def lookup_revision_with_context_by(self, mock_storage, mock_lookup_revision_with_context): # given stub_root_rev = {'id': 'root-rev-id'} - mock_backend.revision_get_by.return_value = {'id': 'root-rev-id'} + mock_storage.revision_get_by.return_value = [{'id': 'root-rev-id'}] stub_rev = {'id': 'rev-found'} mock_lookup_revision_with_context.return_value = stub_rev # when origin_id = 1 branch_name = 'master3' ts = None sha1_git = 'sha1' actual_root_rev, actual_rev = service.lookup_revision_with_context_by( origin_id, branch_name, ts, sha1_git) # then self.assertEquals(actual_root_rev, stub_root_rev) self.assertEquals(actual_rev, stub_rev) - mock_backend.revision_get_by.assert_called_once_with( - origin_id, branch_name, ts) + mock_storage.revision_get_by.assert_called_once_with( + origin_id, branch_name, limit=1, timestamp=ts) mock_lookup_revision_with_context.assert_called_once_with( stub_root_rev, sha1_git, 100) - @patch('swh.web.api.service.backend') + @patch('swh.web.api.service.storage') @patch('swh.web.api.service.query') @istest - def lookup_entity_by_uuid(self, mock_query, mock_backend): + def lookup_entity_by_uuid(self, mock_query, mock_storage): # given uuid_test = 'correct-uuid' mock_query.parse_uuid4.return_value = uuid_test stub_entities = [{'uuid': uuid_test}] - mock_backend.entity_get.return_value = stub_entities + mock_storage.entity_get.return_value = stub_entities # when actual_entities = list(service.lookup_entity_by_uuid(uuid_test)) # then self.assertEquals(actual_entities, stub_entities) mock_query.parse_uuid4.assert_called_once_with(uuid_test) - mock_backend.entity_get.assert_called_once_with(uuid_test) + mock_storage.entity_get.assert_called_once_with(uuid_test) @istest def lookup_revision_through_ko_not_implemented(self): # then with self.assertRaises(NotImplementedError): service.lookup_revision_through({ 'something-unknown': 10, }) @patch('swh.web.api.service.lookup_revision_with_context_by') @istest def lookup_revision_through_with_context_by(self, mock_lookup): # given stub_rev = {'id': 'rev'} mock_lookup.return_value = stub_rev # when actual_revision = service.lookup_revision_through({ 'origin_id': 1, 'branch_name': 'master', 'ts': None, 'sha1_git': 'sha1-git' }, limit=1000) # then self.assertEquals(actual_revision, stub_rev) mock_lookup.assert_called_once_with( 1, 'master', None, 'sha1-git', 1000) @patch('swh.web.api.service.lookup_revision_by') @istest def lookup_revision_through_with_revision_by(self, mock_lookup): # given stub_rev = {'id': 'rev'} mock_lookup.return_value = stub_rev # when actual_revision = service.lookup_revision_through({ 'origin_id': 2, 'branch_name': 'master2', 'ts': 'some-ts', }, limit=10) # then self.assertEquals(actual_revision, stub_rev) mock_lookup.assert_called_once_with( 2, 'master2', 'some-ts') @patch('swh.web.api.service.lookup_revision_with_context') @istest def lookup_revision_through_with_context(self, mock_lookup): # given stub_rev = {'id': 'rev'} mock_lookup.return_value = stub_rev # when actual_revision = service.lookup_revision_through({ 'sha1_git_root': 'some-sha1-root', 'sha1_git': 'some-sha1', }) # then self.assertEquals(actual_revision, stub_rev) mock_lookup.assert_called_once_with( 'some-sha1-root', 'some-sha1', 100) @patch('swh.web.api.service.lookup_revision') @istest def lookup_revision_through_with_revision(self, mock_lookup): # given stub_rev = {'id': 'rev'} mock_lookup.return_value = stub_rev # when actual_revision = service.lookup_revision_through({ 'sha1_git': 'some-sha1', }) # then self.assertEquals(actual_revision, stub_rev) mock_lookup.assert_called_once_with( 'some-sha1') @patch('swh.web.api.service.lookup_revision_through') @istest def lookup_directory_through_revision_ko_not_found( self, mock_lookup_rev): # given mock_lookup_rev.return_value = None # when with self.assertRaises(NotFoundExc): service.lookup_directory_through_revision( {'id': 'rev'}, 'some/path', 100) mock_lookup_rev.assert_called_once_with({'id': 'rev'}, 100) @patch('swh.web.api.service.lookup_revision_through') @patch('swh.web.api.service.lookup_directory_with_revision') @istest def lookup_directory_through_revision_ok_with_data( self, mock_lookup_dir, mock_lookup_rev): # given mock_lookup_rev.return_value = {'id': 'rev-id'} mock_lookup_dir.return_value = {'type': 'dir', 'content': []} # when rev_id, dir_result = service.lookup_directory_through_revision( {'id': 'rev'}, 'some/path', 100) # then self.assertEquals(rev_id, 'rev-id') self.assertEquals(dir_result, {'type': 'dir', 'content': []}) mock_lookup_rev.assert_called_once_with({'id': 'rev'}, 100) mock_lookup_dir.assert_called_once_with('rev-id', 'some/path', False) @patch('swh.web.api.service.lookup_revision_through') @patch('swh.web.api.service.lookup_directory_with_revision') @istest def lookup_directory_through_revision_ok_with_content( self, mock_lookup_dir, mock_lookup_rev): # given mock_lookup_rev.return_value = {'id': 'rev-id'} stub_result = {'type': 'file', 'revision': 'rev-id', 'content': {'data': b'blah', 'sha1': 'sha1'}} mock_lookup_dir.return_value = stub_result # when rev_id, dir_result = service.lookup_directory_through_revision( {'id': 'rev'}, 'some/path', 10, with_data=True) # then self.assertEquals(rev_id, 'rev-id') self.assertEquals(dir_result, stub_result) mock_lookup_rev.assert_called_once_with({'id': 'rev'}, 10) mock_lookup_dir.assert_called_once_with('rev-id', 'some/path', True)