Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/storage.py
Show First 20 Lines • Show All 1,151 Lines • ▼ Show 20 Lines | |||||
def origin_visit_add(self, origin, date=None, type=None, | def origin_visit_add(self, origin, date=None, type=None, | ||||
db=None, cur=None, *, ts=None): | db=None, cur=None, *, ts=None): | ||||
"""Add an origin_visit for the origin at ts with status 'ongoing'. | """Add an origin_visit for the origin at ts with status 'ongoing'. | ||||
For backward compatibility, `type` is optional and defaults to | For backward compatibility, `type` is optional and defaults to | ||||
the origin's type. | the origin's type. | ||||
Args: | Args: | ||||
origin: Visited Origin id | origin (Union[int,str]): visited origin's identifier or URL | ||||
date: timestamp of such visit | date: timestamp of such visit | ||||
type (str): the type of loader used for the visit (hg, git, ...) | type (str): the type of loader used for the visit (hg, git, ...) | ||||
Returns: | Returns: | ||||
dict: dictionary with keys origin and visit where: | dict: dictionary with keys origin and visit where: | ||||
- origin: origin identifier | - origin: origin identifier | ||||
- visit: the visit identifier for the new visit occurrence | - visit: the visit identifier for the new visit occurrence | ||||
""" | """ | ||||
if ts is None: | if ts is None: | ||||
if date is None: | if date is None: | ||||
raise TypeError('origin_visit_add expected 2 arguments.') | raise TypeError('origin_visit_add expected 2 arguments.') | ||||
else: | else: | ||||
assert date is None | assert date is None | ||||
warnings.warn("argument 'ts' of origin_visit_add was renamed " | warnings.warn("argument 'ts' of origin_visit_add was renamed " | ||||
"to 'date' in v0.0.109.", | "to 'date' in v0.0.109.", | ||||
DeprecationWarning) | DeprecationWarning) | ||||
date = ts | date = ts | ||||
origin_id = origin # TODO: rename the argument | if isinstance(origin, str): | ||||
origin = self.origin_get({'url': origin}, db=db, cur=cur) | |||||
origin_id = origin['id'] | |||||
else: | |||||
origin = self.origin_get({'id': origin}, db=db, cur=cur) | |||||
origin_id = origin['id'] | |||||
if isinstance(date, str): | if isinstance(date, str): | ||||
date = dateutil.parser.parse(date) | date = dateutil.parser.parse(date) | ||||
if type is None: | if type is None: | ||||
origin = self.origin_get({'id': origin}) | |||||
type = origin['type'] | type = origin['type'] | ||||
visit_id = db.origin_visit_add(origin_id, date, type, cur) | visit_id = db.origin_visit_add(origin_id, date, type, cur) | ||||
if self.journal_writer: | if self.journal_writer: | ||||
# We can write to the journal only after inserting to the | # We can write to the journal only after inserting to the | ||||
# DB, because we want the id of the visit | # DB, because we want the id of the visit | ||||
origin = self.origin_get([{'id': origin_id}], db=db, cur=cur)[0] | |||||
del origin['id'] | del origin['id'] | ||||
self.journal_writer.write_addition('origin_visit', { | self.journal_writer.write_addition('origin_visit', { | ||||
'origin': origin, 'date': date, 'type': type, | 'origin': origin, 'date': date, 'type': type, | ||||
'visit': visit_id, | 'visit': visit_id, | ||||
'status': 'ongoing', 'metadata': None, 'snapshot': None}) | 'status': 'ongoing', 'metadata': None, 'snapshot': None}) | ||||
return { | return { | ||||
'origin': origin_id, | 'origin': origin_id, | ||||
'visit': visit_id, | 'visit': visit_id, | ||||
} | } | ||||
@db_transaction() | @db_transaction() | ||||
def origin_visit_update(self, origin, visit_id, status=None, | def origin_visit_update(self, origin, visit_id, status=None, | ||||
metadata=None, snapshot=None, | metadata=None, snapshot=None, | ||||
db=None, cur=None): | db=None, cur=None): | ||||
"""Update an origin_visit's status. | """Update an origin_visit's status. | ||||
Args: | Args: | ||||
origin: Visited Origin id | origin (Union[int,str]): visited origin's identifier or URL | ||||
visit_id: Visit's id | visit_id: Visit's id | ||||
status: Visit's new status | status: Visit's new status | ||||
metadata: Data associated to the visit | metadata: Data associated to the visit | ||||
snapshot (sha1_git): identifier of the snapshot to add to | snapshot (sha1_git): identifier of the snapshot to add to | ||||
the visit | the visit | ||||
Returns: | Returns: | ||||
None | None | ||||
""" | """ | ||||
origin_id = origin # TODO: rename the argument | if isinstance(origin, str): | ||||
ardumont: Add missing `db=db, cur=cur` parameters as in line 1198 (of this file) | |||||
origin_id = self.origin_get({'url': origin}, db=db, cur=cur)['id'] | |||||
else: | |||||
origin_id = origin | |||||
visit = db.origin_visit_get(origin_id, visit_id, cur=cur) | visit = db.origin_visit_get(origin_id, visit_id, cur=cur) | ||||
if not visit: | if not visit: | ||||
raise ValueError('Invalid visit_id for this origin.') | raise ValueError('Invalid visit_id for this origin.') | ||||
visit = dict(zip(db.origin_visit_get_cols, visit)) | visit = dict(zip(db.origin_visit_get_cols, visit)) | ||||
Show All 35 Lines | def origin_visit_upsert(self, visits, db=None, cur=None): | ||||
visits = copy.deepcopy(visits) | visits = copy.deepcopy(visits) | ||||
for visit in visits: | for visit in visits: | ||||
if isinstance(visit['date'], str): | if isinstance(visit['date'], str): | ||||
visit['date'] = dateutil.parser.parse(visit['date']) | visit['date'] = dateutil.parser.parse(visit['date']) | ||||
if self.journal_writer: | if self.journal_writer: | ||||
for visit in visits: | for visit in visits: | ||||
visit = visit.copy() | visit = visit.copy() | ||||
origin = self.origin_get([{'id': visit['origin']}])[0] | origin = self.origin_get( | ||||
[{'id': visit['origin']}], db=db, cur=cur)[0] | |||||
visit['origin'] = origin | visit['origin'] = origin | ||||
if visit.get('type') is None: | if visit.get('type') is None: | ||||
visit['type'] = origin['type'] | visit['type'] = origin['type'] | ||||
del visit['origin']['id'] | del visit['origin']['id'] | ||||
self.journal_writer.write_addition('origin_visit', visit) | self.journal_writer.write_addition('origin_visit', visit) | ||||
for visit in visits: | for visit in visits: | ||||
# TODO: upsert them all in a single query | # TODO: upsert them all in a single query | ||||
▲ Show 20 Lines • Show All 529 Lines • Show Last 20 Lines |
Add missing db=db, cur=cur parameters as in line 1198 (of this file)