Changeset View
Changeset View
Standalone View
Standalone View
swh/web/common/service.py
| Show First 20 Lines • Show All 273 Lines • ▼ Show 20 Lines | def search_origin_metadata(fulltext, limit=50): | ||||
| """ | """ | ||||
| matches = idx_storage.origin_intrinsic_metadata_search_fulltext( | matches = idx_storage.origin_intrinsic_metadata_search_fulltext( | ||||
| conjunction=[fulltext], limit=limit) | conjunction=[fulltext], limit=limit) | ||||
| results = [] | results = [] | ||||
| for match in matches: | for match in matches: | ||||
| match['from_revision'] = hashutil.hash_to_hex(match['from_revision']) | match['from_revision'] = hashutil.hash_to_hex(match['from_revision']) | ||||
| result = converters.from_origin( | result = converters.from_origin( | ||||
| storage.origin_get({'id': match.pop('id')})) | storage.origin_get({'url': match.pop('origin_url')})) | ||||
| result['metadata'] = match | result['metadata'] = match | ||||
| results.append(result) | results.append(result) | ||||
| return results | return results | ||||
| def lookup_origin_intrinsic_metadata(origin_dict): | def lookup_origin_intrinsic_metadata(origin_dict): | ||||
| """Return intrinsic metadata for origin whose origin_id matches given | """Return intrinsic metadata for origin whose origin matches given | ||||
| origin_id. | origin. | ||||
| Args: | Args: | ||||
| origin_dict: origin's dict with keys ('type' AND 'url') | origin_dict: origin's dict with keys ('type' AND 'url') | ||||
| Returns: | Returns: | ||||
| origin metadata. | origin metadata. | ||||
| """ | """ | ||||
| origin_info = storage.origin_get(origin_dict) | origin_info = storage.origin_get(origin_dict) | ||||
| if not origin_info: | if not origin_info: | ||||
| msg = 'Origin with type %s and url %s not found!' % \ | msg = 'Origin with type %s and url %s not found!' % \ | ||||
| (origin_dict['type'], origin_dict['url']) | (origin_dict['type'], origin_dict['url']) | ||||
| raise NotFoundExc(msg) | raise NotFoundExc(msg) | ||||
| origin_ids = [origin_info['id']] | origins = [origin_info['url']] | ||||
| match = _first_element( | match = _first_element( | ||||
| idx_storage.origin_intrinsic_metadata_get(origin_ids)) | idx_storage.origin_intrinsic_metadata_get(origins)) | ||||
| result = {} | result = {} | ||||
| if match: | if match: | ||||
| result = match['metadata'] | result = match['metadata'] | ||||
| return result | return result | ||||
| def lookup_person(person_id): | def lookup_person(person_id): | ||||
| """Return information about the person with id person_id. | """Return information about the person with id person_id. | ||||
| ▲ Show 20 Lines • Show All 181 Lines • ▼ Show 20 Lines | if not revision: | ||||
| % rev_sha1_git) | % rev_sha1_git) | ||||
| if 'message' not in revision: | if 'message' not in revision: | ||||
| raise NotFoundExc('No message for revision with sha1_git %s.' | raise NotFoundExc('No message for revision with sha1_git %s.' | ||||
| % rev_sha1_git) | % rev_sha1_git) | ||||
| res = {'message': revision['message']} | res = {'message': revision['message']} | ||||
| return res | return res | ||||
| def _lookup_revision_id_by(origin_id, branch_name, timestamp): | def _lookup_revision_id_by(origin, branch_name, timestamp): | ||||
| def _get_snapshot_branch(snapshot, branch_name): | def _get_snapshot_branch(snapshot, branch_name): | ||||
| snapshot = lookup_snapshot(visit['snapshot'], | snapshot = lookup_snapshot(visit['snapshot'], | ||||
| branches_from=branch_name, | branches_from=branch_name, | ||||
| branches_count=10) | branches_count=10) | ||||
| branch = None | branch = None | ||||
| if branch_name in snapshot['branches']: | if branch_name in snapshot['branches']: | ||||
| branch = snapshot['branches'][branch_name] | branch = snapshot['branches'][branch_name] | ||||
| return branch | return branch | ||||
| visit = get_origin_visit({'id': origin_id}, visit_ts=timestamp) | if isinstance(origin, int): | ||||
| origin = {'id': origin} | |||||
| elif isinstance(origin, str): | |||||
| origin = {'url': origin} | |||||
| else: | |||||
| raise TypeError('"origin" must be an int or a string.') | |||||
| visit = get_origin_visit(origin, visit_ts=timestamp) | |||||
| branch = _get_snapshot_branch(visit['snapshot'], branch_name) | branch = _get_snapshot_branch(visit['snapshot'], branch_name) | ||||
| rev_id = None | rev_id = None | ||||
| if branch and branch['target_type'] == 'revision': | if branch and branch['target_type'] == 'revision': | ||||
| rev_id = branch['target'] | rev_id = branch['target'] | ||||
| elif branch and branch['target_type'] == 'alias': | elif branch and branch['target_type'] == 'alias': | ||||
| branch = _get_snapshot_branch(visit['snapshot'], branch['target']) | branch = _get_snapshot_branch(visit['snapshot'], branch['target']) | ||||
| if branch and branch['target_type'] == 'revision': | if branch and branch['target_type'] == 'revision': | ||||
| rev_id = branch['target'] | rev_id = branch['target'] | ||||
| if not rev_id: | if not rev_id: | ||||
| raise NotFoundExc('Revision for origin %s and branch %s not found.' | raise NotFoundExc('Revision for origin %s and branch %s not found.' | ||||
| % (origin_id, branch_name)) | % (origin.get('url') or origin['id'], branch_name)) | ||||
| return rev_id | return rev_id | ||||
| def lookup_revision_by(origin_id, | def lookup_revision_by(origin, | ||||
| branch_name='HEAD', | branch_name='HEAD', | ||||
| timestamp=None): | timestamp=None): | ||||
| """Lookup revision by origin id, snapshot branch name and visit timestamp. | """Lookup revision by origin, snapshot branch name and visit timestamp. | ||||
| If branch_name is not provided, lookup using 'HEAD' as default. | If branch_name is not provided, lookup using 'HEAD' as default. | ||||
| If timestamp is not provided, use the most recent. | If timestamp is not provided, use the most recent. | ||||
| Args: | Args: | ||||
| origin_id (int): origin of the revision | origin (Union[int,str]): origin of the revision | ||||
| branch_name (str): snapshot branch name | branch_name (str): snapshot branch name | ||||
| timestamp (str/int): origin visit time frame | timestamp (str/int): origin visit time frame | ||||
| Returns: | Returns: | ||||
| dict: The revision matching the criterions | dict: The revision matching the criterions | ||||
| Raises: | Raises: | ||||
| NotFoundExc if no revision corresponds to the criterion | NotFoundExc if no revision corresponds to the criterion | ||||
| """ | """ | ||||
| rev_id = _lookup_revision_id_by(origin_id, branch_name, timestamp) | rev_id = _lookup_revision_id_by(origin, branch_name, timestamp) | ||||
| return lookup_revision(rev_id) | return lookup_revision(rev_id) | ||||
| def lookup_revision_log(rev_sha1_git, limit): | def lookup_revision_log(rev_sha1_git, limit): | ||||
| """Lookup revision log by revision id. | """Lookup revision log by revision id. | ||||
| Args: | Args: | ||||
| rev_sha1_git (str): The revision's sha1 as hexadecimal | rev_sha1_git (str): The revision's sha1 as hexadecimal | ||||
| limit (int): the maximum number of revisions returned | limit (int): the maximum number of revisions returned | ||||
| Returns: | Returns: | ||||
| list: Revision log as list of revision dicts | list: Revision log as list of revision dicts | ||||
| Raises: | Raises: | ||||
| ValueError: if the identifier provided is not of sha1 nature. | ValueError: if the identifier provided is not of sha1 nature. | ||||
| NotFoundExc: if there is no revision with the provided sha1_git. | NotFoundExc: if there is no revision with the provided sha1_git. | ||||
| """ | """ | ||||
| lookup_revision(rev_sha1_git) | lookup_revision(rev_sha1_git) | ||||
| sha1_git_bin = _to_sha1_bin(rev_sha1_git) | sha1_git_bin = _to_sha1_bin(rev_sha1_git) | ||||
| revision_entries = storage.revision_log([sha1_git_bin], limit) | revision_entries = storage.revision_log([sha1_git_bin], limit) | ||||
| return map(converters.from_revision, revision_entries) | return map(converters.from_revision, revision_entries) | ||||
| def lookup_revision_log_by(origin_id, branch_name, timestamp, limit): | def lookup_revision_log_by(origin, branch_name, timestamp, limit): | ||||
| """Lookup revision by origin id, snapshot branch name and visit timestamp. | """Lookup revision by origin, snapshot branch name and visit timestamp. | ||||
| Args: | Args: | ||||
| origin_id (int): origin of the revision | origin (Union[int,str]): origin of the revision | ||||
| branch_name (str): snapshot branch | branch_name (str): snapshot branch | ||||
| timestamp (str/int): origin visit time frame | timestamp (str/int): origin visit time frame | ||||
| limit (int): the maximum number of revisions returned | limit (int): the maximum number of revisions returned | ||||
| Returns: | Returns: | ||||
| list: Revision log as list of revision dicts | list: Revision log as list of revision dicts | ||||
| Raises: | Raises: | ||||
| NotFoundExc: if no revision corresponds to the criterion | NotFoundExc: if no revision corresponds to the criterion | ||||
| """ | """ | ||||
| rev_id = _lookup_revision_id_by(origin_id, branch_name, timestamp) | rev_id = _lookup_revision_id_by(origin, branch_name, timestamp) | ||||
| return lookup_revision_log(rev_id, limit) | return lookup_revision_log(rev_id, limit) | ||||
| def lookup_revision_with_context_by(origin_id, branch_name, timestamp, | def lookup_revision_with_context_by(origin, branch_name, timestamp, | ||||
| sha1_git, limit=100): | sha1_git, limit=100): | ||||
| """Return information about revision sha1_git, limited to the | """Return information about revision sha1_git, limited to the | ||||
| sub-graph of all transitive parents of sha1_git_root. | sub-graph of all transitive parents of sha1_git_root. | ||||
| sha1_git_root being resolved through the lookup of a revision by origin_id, | sha1_git_root being resolved through the lookup of a revision by origin, | ||||
| branch_name and ts. | branch_name and ts. | ||||
| In other words, sha1_git is an ancestor of sha1_git_root. | In other words, sha1_git is an ancestor of sha1_git_root. | ||||
| Args: | Args: | ||||
| - origin_id: origin of the revision. | - origin: origin of the revision. | ||||
| - branch_name: revision's branch. | - branch_name: revision's branch. | ||||
| - timestamp: revision's time frame. | - timestamp: revision's time frame. | ||||
| - sha1_git: one of sha1_git_root's ancestors. | - sha1_git: one of sha1_git_root's ancestors. | ||||
| - limit: limit the lookup to 100 revisions back. | - limit: limit the lookup to 100 revisions back. | ||||
| Returns: | Returns: | ||||
| Pair of (root_revision, revision). | Pair of (root_revision, revision). | ||||
| Information on sha1_git if it is an ancestor of sha1_git_root | Information on sha1_git if it is an ancestor of sha1_git_root | ||||
| including children leading to sha1_git_root | including children leading to sha1_git_root | ||||
| Raises: | Raises: | ||||
| - BadInputExc in case of unknown algo_hash or bad hash. | - BadInputExc in case of unknown algo_hash or bad hash. | ||||
| - NotFoundExc if either revision is not found or if sha1_git is not an | - NotFoundExc if either revision is not found or if sha1_git is not an | ||||
| ancestor of sha1_git_root. | ancestor of sha1_git_root. | ||||
| """ | """ | ||||
| rev_root_id = _lookup_revision_id_by(origin_id, branch_name, timestamp) | rev_root_id = _lookup_revision_id_by(origin, branch_name, timestamp) | ||||
| rev_root_id_bin = hashutil.hash_to_bytes(rev_root_id) | rev_root_id_bin = hashutil.hash_to_bytes(rev_root_id) | ||||
| rev_root = _first_element(storage.revision_get([rev_root_id_bin])) | rev_root = _first_element(storage.revision_get([rev_root_id_bin])) | ||||
| return (converters.from_revision(rev_root), | return (converters.from_revision(rev_root), | ||||
| lookup_revision_with_context(rev_root, sha1_git, limit)) | lookup_revision_with_context(rev_root, sha1_git, limit)) | ||||
| ▲ Show 20 Lines • Show All 173 Lines • ▼ Show 20 Lines | def stat_counters(): | ||||
| Returns: | Returns: | ||||
| A dict mapping textual labels to integer values. | A dict mapping textual labels to integer values. | ||||
| """ | """ | ||||
| return storage.stat_counters() | return storage.stat_counters() | ||||
| def _lookup_origin_visits(origin_url, last_visit=None, limit=10): | def _lookup_origin_visits(origin_url, last_visit=None, limit=10): | ||||
| """Yields the origin origin_ids' visits. | """Yields the origin origins' visits. | ||||
| Args: | Args: | ||||
| origin_url (str): origin to list visits for | origin_url (str): origin to list visits for | ||||
| last_visit (int): last visit to lookup from | last_visit (int): last visit to lookup from | ||||
| limit (int): Number of elements max to display | limit (int): Number of elements max to display | ||||
| Yields: | Yields: | ||||
| Dictionaries of origin_visit for that origin | Dictionaries of origin_visit for that origin | ||||
| """ | """ | ||||
| limit = min(limit, MAX_LIMIT) | limit = min(limit, MAX_LIMIT) | ||||
| for visit in storage.origin_visit_get( | for visit in storage.origin_visit_get( | ||||
| origin_url, last_visit=last_visit, limit=limit): | origin_url, last_visit=last_visit, limit=limit): | ||||
| visit['origin'] = origin_url | visit['origin'] = origin_url | ||||
| yield visit | yield visit | ||||
| def lookup_origin_visits(origin_id, last_visit=None, per_page=10): | def lookup_origin_visits(origin, last_visit=None, per_page=10): | ||||
| """Yields the origin origin_ids' visits. | """Yields the origin origins' visits. | ||||
| Args: | Args: | ||||
| origin_id: origin to list visits for | origin: origin to list visits for | ||||
| Yields: | Yields: | ||||
| Dictionaries of origin_visit for that origin | Dictionaries of origin_visit for that origin | ||||
| """ | """ | ||||
| visits = _lookup_origin_visits(origin_id, last_visit=last_visit, | visits = _lookup_origin_visits(origin, last_visit=last_visit, | ||||
| limit=per_page) | limit=per_page) | ||||
| for visit in visits: | for visit in visits: | ||||
| yield converters.from_origin_visit(visit) | yield converters.from_origin_visit(visit) | ||||
| def lookup_origin_visit(origin_url, visit_id): | def lookup_origin_visit(origin_url, visit_id): | ||||
| """Return information about visit visit_id with origin origin_id. | """Return information about visit visit_id with origin origin. | ||||
| Args: | Args: | ||||
| origin (str): origin concerned by the visit | origin (str): origin concerned by the visit | ||||
| visit_id: the visit identifier to lookup | visit_id: the visit identifier to lookup | ||||
| Yields: | Yields: | ||||
| The dict origin_visit concerned | The dict origin_visit concerned | ||||
| ▲ Show 20 Lines • Show All 48 Lines • ▼ Show 20 Lines | def lookup_snapshot(snapshot_id, branches_from='', branches_count=1000, | ||||
| snapshot = storage.snapshot_get_branches(snapshot_id_bin, | snapshot = storage.snapshot_get_branches(snapshot_id_bin, | ||||
| branches_from.encode(), | branches_from.encode(), | ||||
| branches_count, target_types) | branches_count, target_types) | ||||
| if not snapshot: | if not snapshot: | ||||
| raise NotFoundExc('Snapshot with id %s not found!' % snapshot_id) | raise NotFoundExc('Snapshot with id %s not found!' % snapshot_id) | ||||
| return converters.from_snapshot(snapshot) | return converters.from_snapshot(snapshot) | ||||
| def lookup_latest_origin_snapshot(origin_id, allowed_statuses=None): | def lookup_latest_origin_snapshot(origin, allowed_statuses=None): | ||||
| """Return information about the latest snapshot of an origin. | """Return information about the latest snapshot of an origin. | ||||
| .. warning:: At most 1000 branches contained in the snapshot | .. warning:: At most 1000 branches contained in the snapshot | ||||
| will be returned for performance reasons. | will be returned for performance reasons. | ||||
| Args: | Args: | ||||
| origin_id: integer identifier of the origin | origin: URL or integer identifier of the origin | ||||
| allowed_statuses: list of visit statuses considered | allowed_statuses: list of visit statuses considered | ||||
| to find the latest snapshot for the visit. For instance, | to find the latest snapshot for the visit. For instance, | ||||
| ``allowed_statuses=['full']`` will only consider visits that | ``allowed_statuses=['full']`` will only consider visits that | ||||
| have successfully run to completion. | have successfully run to completion. | ||||
| Returns: | Returns: | ||||
| A dict filled with the snapshot content. | A dict filled with the snapshot content. | ||||
| """ | """ | ||||
| snapshot = storage.snapshot_get_latest(origin_id, allowed_statuses) | snapshot = storage.snapshot_get_latest(origin, allowed_statuses) | ||||
| return converters.from_snapshot(snapshot) | return converters.from_snapshot(snapshot) | ||||
| def lookup_revision_through(revision, limit=100): | def lookup_revision_through(revision, limit=100): | ||||
| """Retrieve a revision from the criterion stored in revision dictionary. | """Retrieve a revision from the criterion stored in revision dictionary. | ||||
| Args: | Args: | ||||
| revision: Dictionary of criterion to lookup the revision with. | revision: Dictionary of criterion to lookup the revision with. | ||||
| Here are the supported combination of possible values: | Here are the supported combination of possible values: | ||||
| - origin_id, branch_name, ts, sha1_git | - origin_id, branch_name, ts, sha1_git | ||||
| - origin_id, branch_name, ts | - origin_id, branch_name, ts | ||||
| - origin_url, branch_name, ts, sha1_git | |||||
| - origin_url, branch_name, ts | |||||
| - sha1_git_root, sha1_git | - sha1_git_root, sha1_git | ||||
| - sha1_git | - sha1_git | ||||
| Returns: | Returns: | ||||
| None if the revision is not found or the actual revision. | None if the revision is not found or the actual revision. | ||||
| """ | """ | ||||
anlambert: Here, I would prefer to put the test inside parenthesis instead of using backslashes… | |||||
| if 'origin_id' in revision and \ | if ( | ||||
| 'branch_name' in revision and \ | 'origin_url' in revision and | ||||
| 'ts' in revision and \ | 'branch_name' in revision and | ||||
| 'sha1_git' in revision: | 'ts' in revision and | ||||
| 'sha1_git' in revision): | |||||
| return lookup_revision_with_context_by(revision['origin_url'], | |||||
| revision['branch_name'], | |||||
| revision['ts'], | |||||
| revision['sha1_git'], | |||||
| limit) | |||||
| if ( | |||||
| 'origin_id' in revision and | |||||
| 'branch_name' in revision and | |||||
| 'ts' in revision and | |||||
| 'sha1_git' in revision): | |||||
| return lookup_revision_with_context_by(revision['origin_id'], | return lookup_revision_with_context_by(revision['origin_id'], | ||||
| revision['branch_name'], | revision['branch_name'], | ||||
| revision['ts'], | revision['ts'], | ||||
| revision['sha1_git'], | revision['sha1_git'], | ||||
| limit) | limit) | ||||
| if 'origin_id' in revision and \ | if ( | ||||
| 'branch_name' in revision and \ | 'origin_url' in revision and | ||||
| 'ts' in revision: | 'branch_name' in revision and | ||||
| 'ts' in revision): | |||||
| return lookup_revision_by(revision['origin_url'], | |||||
| revision['branch_name'], | |||||
| revision['ts']) | |||||
| if ( | |||||
| 'origin_id' in revision and | |||||
| 'branch_name' in revision and | |||||
| 'ts' in revision): | |||||
| return lookup_revision_by(revision['origin_id'], | return lookup_revision_by(revision['origin_id'], | ||||
| revision['branch_name'], | revision['branch_name'], | ||||
| revision['ts']) | revision['ts']) | ||||
| if 'sha1_git_root' in revision and \ | if ( | ||||
| 'sha1_git' in revision: | 'sha1_git_root' in revision and | ||||
| 'sha1_git' in revision): | |||||
| return lookup_revision_with_context(revision['sha1_git_root'], | return lookup_revision_with_context(revision['sha1_git_root'], | ||||
| revision['sha1_git'], | revision['sha1_git'], | ||||
| limit) | limit) | ||||
| if 'sha1_git' in revision: | if 'sha1_git' in revision: | ||||
| return lookup_revision(revision['sha1_git']) | return lookup_revision(revision['sha1_git']) | ||||
| # this should not happen | # this should not happen | ||||
| raise NotImplementedError('Should not happen!') | raise NotImplementedError('Should not happen!') | ||||
| ▲ Show 20 Lines • Show All 106 Lines • Show Last 20 Lines | |||||
Here, I would prefer to put the test inside parenthesis instead of using backslashes: