Changeset View
Changeset View
Standalone View
Standalone View
swh/provenance/provenance.py
| Show First 20 Lines • Show All 149 Lines • ▼ Show 20 Lines | def flush(self) -> None: | ||||
| f"Data: {dates}. Retrying..." | f"Data: {dates}. Retrying..." | ||||
| ) | ) | ||||
| # Origin-revision layer insertions ############################################# | # Origin-revision layer insertions ############################################# | ||||
| # Origins urls should be inserted first so that internal ids' resolution works | # Origins urls should be inserted first so that internal ids' resolution works | ||||
| # properly. | # properly. | ||||
| urls = { | urls = { | ||||
| sha1: date | sha1: url | ||||
| for sha1, date in self.cache["origin"]["data"].items() | for sha1, url in self.cache["origin"]["data"].items() | ||||
| if sha1 in self.cache["origin"]["added"] | if sha1 in self.cache["origin"]["added"] | ||||
| } | } | ||||
| while not self.storage.origin_set_url(urls): | while not self.storage.origin_set_url(urls): | ||||
| logging.warning( | logging.warning( | ||||
| f"Unable to write origins urls to the storage. " | f"Unable to write origins urls to the storage. " | ||||
| f"Data: {urls}. Retrying..." | f"Data: {urls}. Retrying..." | ||||
| ) | ) | ||||
| ▲ Show 20 Lines • Show All 107 Lines • ▼ Show 20 Lines | class Provenance: | ||||
| ) -> Dict[Sha1Git, datetime]: | ) -> Dict[Sha1Git, datetime]: | ||||
| cache = self.cache[entity] | cache = self.cache[entity] | ||||
| missing_ids = set(id for id in ids if id not in cache) | missing_ids = set(id for id in ids if id not in cache) | ||||
| if missing_ids: | if missing_ids: | ||||
| if entity == "revision": | if entity == "revision": | ||||
| updated = { | updated = { | ||||
| id: rev.date | id: rev.date | ||||
| for id, rev in self.storage.revision_get(missing_ids).items() | for id, rev in self.storage.revision_get(missing_ids).items() | ||||
| if rev.date is not None | |||||
| } | } | ||||
| else: | else: | ||||
| updated = getattr(self.storage, f"{entity}_get")(missing_ids) | updated = getattr(self.storage, f"{entity}_get")(missing_ids) | ||||
| cache["data"].update(updated) | cache["data"].update(updated) | ||||
| dates: Dict[Sha1Git, datetime] = {} | dates: Dict[Sha1Git, datetime] = {} | ||||
| for sha1 in ids: | for sha1 in ids: | ||||
| date = cache["data"].get(sha1) | date = cache["data"].setdefault(sha1, None) | ||||
| if date is not None: | if date is not None: | ||||
| dates[sha1] = date | dates[sha1] = date | ||||
| return dates | return dates | ||||
| def origin_add(self, origin: OriginEntry) -> None: | def origin_add(self, origin: OriginEntry) -> None: | ||||
| self.cache["origin"]["data"][origin.id] = origin.url | self.cache["origin"]["data"][origin.id] = origin.url | ||||
| self.cache["origin"]["added"].add(origin.id) | self.cache["origin"]["added"].add(origin.id) | ||||
| ▲ Show 20 Lines • Show All 50 Lines • Show Last 20 Lines | |||||