Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/db.py
Show First 20 Lines • Show All 422 Lines • ▼ Show 20 Lines | revision_add_cols = [ | ||||
"committer_email", | "committer_email", | ||||
"metadata", | "metadata", | ||||
"synthetic", | "synthetic", | ||||
] | ] | ||||
revision_get_cols = revision_add_cols + ["parents"] | revision_get_cols = revision_add_cols + ["parents"] | ||||
def origin_visit_add(self, origin, ts, type, cur=None): | def origin_visit_add(self, origin, ts, type, cur=None): | ||||
"""Add a new origin_visit for origin origin at timestamp ts with | """Add a new origin_visit for origin origin at timestamp ts. | ||||
status 'ongoing'. | |||||
Args: | Args: | ||||
origin: origin concerned by the visit | origin: origin concerned by the visit | ||||
ts: the date of the visit | ts: the date of the visit | ||||
type: type of loader for the visit | type: type of loader for the visit | ||||
Returns: | Returns: | ||||
The new visit index step for that origin | The new visit index step for that origin | ||||
Show All 31 Lines | ) -> None: | ||||
f"VALUES ((select id from origin_id), " | f"VALUES ((select id from origin_id), " | ||||
f"{', '.join(['%s']*len(cols))}, %s) " | f"{', '.join(['%s']*len(cols))}, %s) " | ||||
f"ON CONFLICT (origin, visit, date) do nothing", | f"ON CONFLICT (origin, visit, date) do nothing", | ||||
[visit_status.origin] | [visit_status.origin] | ||||
+ [getattr(visit_status, key) for key in cols] | + [getattr(visit_status, key) for key in cols] | ||||
+ [jsonize(visit_status.metadata)], | + [jsonize(visit_status.metadata)], | ||||
) | ) | ||||
origin_visit_upsert_cols = [ | |||||
"origin", | |||||
"visit", | |||||
"date", | |||||
"type", | |||||
] | |||||
def origin_visit_upsert(self, origin_visit: OriginVisit, cur=None) -> None: | def origin_visit_upsert(self, origin_visit: OriginVisit, cur=None) -> None: | ||||
# doing an extra query like this is way simpler than trying to join | # doing an extra query like this is way simpler than trying to join | ||||
# the origin id in the query below | # the origin id in the query below | ||||
ov = origin_visit | ov = origin_visit | ||||
origin_id = next(self.origin_id_get_by_url([ov.origin])) | origin_id = next(self.origin_id_get_by_url([ov.origin])) | ||||
cur = self._cursor(cur) | cur = self._cursor(cur) | ||||
query = """INSERT INTO origin_visit ({cols}) VALUES ({values}) | query = """INSERT INTO origin_visit ({cols}) VALUES ({values}) | ||||
ON CONFLICT ON CONSTRAINT origin_visit_pkey DO | ON CONFLICT ON CONSTRAINT origin_visit_pkey DO | ||||
UPDATE SET {updates}""".format( | UPDATE SET {updates}""".format( | ||||
cols=", ".join(self.origin_visit_get_cols), | cols=", ".join(self.origin_visit_upsert_cols), | ||||
values=", ".join("%s" for col in self.origin_visit_get_cols), | values=", ".join("%s" for col in self.origin_visit_upsert_cols), | ||||
updates=", ".join( | updates=", ".join( | ||||
"{0}=excluded.{0}".format(col) for col in self.origin_visit_get_cols | "{0}=excluded.{0}".format(col) for col in self.origin_visit_upsert_cols | ||||
), | ), | ||||
) | ) | ||||
cur.execute( | cur.execute( | ||||
query, | query, (origin_id, ov.visit, ov.date, ov.type), | ||||
( | |||||
origin_id, | |||||
ov.visit, | |||||
ov.date, | |||||
ov.type, | |||||
ov.status, | |||||
ov.metadata, | |||||
ov.snapshot, | |||||
), | |||||
) | ) | ||||
origin_visit_get_cols = [ | origin_visit_get_cols = [ | ||||
"origin", | "origin", | ||||
"visit", | "visit", | ||||
"date", | "date", | ||||
"type", | "type", | ||||
"status", | "status", | ||||
▲ Show 20 Lines • Show All 745 Lines • Show Last 20 Lines |