Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/storage.py
Show First 20 Lines • Show All 1,075 Lines • ▼ Show 20 Lines | def object_find_by_sha1_git(self, ids, db=None, cur=None): | ||||
ret[retval[0]].append(dict(zip(db.object_find_by_sha1_git_cols, | ret[retval[0]].append(dict(zip(db.object_find_by_sha1_git_cols, | ||||
retval))) | retval))) | ||||
return ret | return ret | ||||
origin_keys = ['id', 'type', 'url'] | origin_keys = ['id', 'type', 'url'] | ||||
@db_transaction(statement_timeout=500) | @db_transaction(statement_timeout=500) | ||||
def origin_get(self, origin, db=None, cur=None): | def origin_get(self, origins, db=None, cur=None): | ||||
"""Return the origin either identified by its id or its tuple | """Return the origin either identified by its id or its tuple | ||||
anlambert: Method description needs update here too | |||||
(type, url). | (type, url). | ||||
Args: | Args: | ||||
origin: dictionary representing the individual origin to find. | origin: a list of dictionaries representing the individual | ||||
This dict has either the keys type and url: | origins to find. | ||||
These dicts have either the keys type and url: | |||||
- type (FIXME: enum TBD): the origin type ('git', 'wget', ...) | - type (FIXME: enum TBD): the origin type ('git', 'wget', ...) | ||||
- url (bytes): the url the origin points to | - url (bytes): the url the origin points to | ||||
or the id: | or the id: | ||||
- id: the origin id | - id: the origin id | ||||
Returns: | Returns: | ||||
dict: the origin dictionary with the keys: | dict: the origin dictionary with the keys: | ||||
- id: origin's id | - id: origin's id | ||||
- type: origin's type | - type: origin's type | ||||
- url: origin's url | - url: origin's url | ||||
Raises: | Raises: | ||||
ValueError: if the keys does not match (url and type) nor id. | ValueError: if the keys does not match (url and type) nor id. | ||||
""" | """ | ||||
origin_id = origin.get('id') | if isinstance(origins, dict): | ||||
if origin_id: # check lookup per id first | # Old API | ||||
ori = db.origin_get(origin_id, cur) | return_single = True | ||||
elif 'type' in origin and 'url' in origin: # or lookup per type, url | origins = [origins] | ||||
ori = db.origin_get_with(origin['type'], origin['url'], cur) | elif len(origins) == 0: | ||||
return [] | |||||
else: | |||||
Not Done Inline Actionssame remark as above: else block could be removed anlambert: same remark as above: else block could be removed | |||||
return_single = False | |||||
origin_ids = [origin.get('id') for origin in origins] | |||||
origin_types_and_urls = [(origin.get('type'), origin.get('url')) | |||||
for origin in origins] | |||||
if any(origin_ids): | |||||
# Lookup per ID | |||||
if all(origin_ids): | |||||
results = db.origin_get(origin_ids, cur) | |||||
else: | |||||
raise ValueError( | |||||
'Either all origins or none at all should have an "id".') | |||||
elif any(type_ and url for (type_, url) in origin_types_and_urls): | |||||
# Lookup per type + URL | |||||
if all(type_ and url for (type_, url) in origin_types_and_urls): | |||||
results = db.origin_get_with(origin_types_and_urls, cur) | |||||
else: | |||||
raise ValueError( | |||||
'Either all origins or none at all should have a ' | |||||
'"type" and an "url".') | |||||
else: # unsupported lookup | else: # unsupported lookup | ||||
raise ValueError('Origin must have either id or (type and url).') | raise ValueError('Origin must have either id or (type and url).') | ||||
if ori: | results = [dict(zip(self.origin_keys, result)) | ||||
return dict(zip(self.origin_keys, ori)) | for result in results] | ||||
return None | if return_single: | ||||
assert len(results) == 1 | |||||
Done Inline ActionsI know None is returned by default when return is not called but I think that expression is clearer return results[0] if results[0]['id'] is not None else None anlambert: I know `None` is returned by default when return is not called but I think that expression is… | |||||
if results[0]['id'] is not None: | |||||
return results[0] | |||||
else: | |||||
return [None if res['id'] is None else res for res in results] | |||||
@db_transaction_generator() | @db_transaction_generator() | ||||
def origin_search(self, url_pattern, offset=0, limit=50, | def origin_search(self, url_pattern, offset=0, limit=50, | ||||
regexp=False, with_visit=False, db=None, cur=None): | regexp=False, with_visit=False, db=None, cur=None): | ||||
"""Search for origins whose urls contain a provided string pattern | """Search for origins whose urls contain a provided string pattern | ||||
or match a provided regular expression. | or match a provided regular expression. | ||||
The search is performed in a case insensitive way. | The search is performed in a case insensitive way. | ||||
▲ Show 20 Lines • Show All 77 Lines • ▼ Show 20 Lines | def origin_add_one(self, origin, db=None, cur=None): | ||||
- type (FIXME: enum TBD): the origin type ('git', 'wget', ...) | - type (FIXME: enum TBD): the origin type ('git', 'wget', ...) | ||||
- url (bytes): the url the origin points to | - url (bytes): the url the origin points to | ||||
Returns: | Returns: | ||||
the id of the added origin, or of the identical one that already | the id of the added origin, or of the identical one that already | ||||
exists. | exists. | ||||
""" | """ | ||||
data = db.origin_get_with(origin['type'], origin['url'], cur) | id_ = list(db.origin_get_with( | ||||
if data: | [(origin['type'], origin['url'])], cur))[0][0] | ||||
return data[0] | if id_: | ||||
return id_ | |||||
Done Inline Actionsorigin_id seems a better variable name here anlambert: `origin_id` seems a better variable name here | |||||
return db.origin_add(origin['type'], origin['url'], cur) | return db.origin_add(origin['type'], origin['url'], cur) | ||||
@db_transaction() | @db_transaction() | ||||
def fetch_history_start(self, origin_id, db=None, cur=None): | def fetch_history_start(self, origin_id, db=None, cur=None): | ||||
"""Add an entry for origin origin_id in fetch_history. Returns the id | """Add an entry for origin origin_id in fetch_history. Returns the id | ||||
of the added fetch_history entry | of the added fetch_history entry | ||||
""" | """ | ||||
▲ Show 20 Lines • Show All 261 Lines • Show Last 20 Lines |
Method description needs update here too