Changeset View
Changeset View
Standalone View
Standalone View
swh/provenance/provenance.py
Show First 20 Lines • Show All 149 Lines • ▼ Show 20 Lines | def flush(self) -> None: | ||||
f"Data: {dates}. Retrying..." | f"Data: {dates}. Retrying..." | ||||
) | ) | ||||
# Origin-revision layer insertions ############################################# | # Origin-revision layer insertions ############################################# | ||||
# Origins urls should be inserted first so that internal ids' resolution works | # Origins urls should be inserted first so that internal ids' resolution works | ||||
# properly. | # properly. | ||||
urls = { | urls = { | ||||
sha1: date | sha1: url | ||||
for sha1, date in self.cache["origin"]["data"].items() | for sha1, url in self.cache["origin"]["data"].items() | ||||
if sha1 in self.cache["origin"]["added"] | if sha1 in self.cache["origin"]["added"] | ||||
} | } | ||||
while not self.storage.origin_set_url(urls): | while not self.storage.origin_set_url(urls): | ||||
logging.warning( | logging.warning( | ||||
f"Unable to write origins urls to the storage. " | f"Unable to write origins urls to the storage. " | ||||
f"Data: {urls}. Retrying..." | f"Data: {urls}. Retrying..." | ||||
) | ) | ||||
▲ Show 20 Lines • Show All 107 Lines • ▼ Show 20 Lines | class Provenance: | ||||
) -> Dict[Sha1Git, datetime]: | ) -> Dict[Sha1Git, datetime]: | ||||
cache = self.cache[entity] | cache = self.cache[entity] | ||||
missing_ids = set(id for id in ids if id not in cache) | missing_ids = set(id for id in ids if id not in cache) | ||||
if missing_ids: | if missing_ids: | ||||
if entity == "revision": | if entity == "revision": | ||||
updated = { | updated = { | ||||
id: rev.date | id: rev.date | ||||
for id, rev in self.storage.revision_get(missing_ids).items() | for id, rev in self.storage.revision_get(missing_ids).items() | ||||
if rev.date is not None | |||||
} | } | ||||
else: | else: | ||||
updated = getattr(self.storage, f"{entity}_get")(missing_ids) | updated = getattr(self.storage, f"{entity}_get")(missing_ids) | ||||
cache["data"].update(updated) | cache["data"].update(updated) | ||||
dates: Dict[Sha1Git, datetime] = {} | dates: Dict[Sha1Git, datetime] = {} | ||||
for sha1 in ids: | for sha1 in ids: | ||||
date = cache["data"].get(sha1) | date = cache["data"].setdefault(sha1, None) | ||||
if date is not None: | if date is not None: | ||||
dates[sha1] = date | dates[sha1] = date | ||||
return dates | return dates | ||||
def origin_add(self, origin: OriginEntry) -> None: | def origin_add(self, origin: OriginEntry) -> None: | ||||
self.cache["origin"]["data"][origin.id] = origin.url | self.cache["origin"]["data"][origin.id] = origin.url | ||||
self.cache["origin"]["added"].add(origin.id) | self.cache["origin"]["added"].add(origin.id) | ||||
▲ Show 20 Lines • Show All 50 Lines • Show Last 20 Lines |