Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/core/loader.py
Show First 20 Lines • Show All 390 Lines • ▼ Show 20 Lines | def load(self) -> Dict[str, str]: | ||||
snapshot=self.loaded_snapshot_id, | snapshot=self.loaded_snapshot_id, | ||||
) | ) | ||||
self.storage.origin_visit_status_add([visit_status]) | self.storage.origin_visit_status_add([visit_status]) | ||||
success = True | success = True | ||||
with self.statsd_timed( | with self.statsd_timed( | ||||
"post_load", tags={"success": success, "status": status} | "post_load", tags={"success": success, "status": status} | ||||
): | ): | ||||
self.post_load() | self.post_load() | ||||
except Exception as e: | except BaseException as e: | ||||
if isinstance(e, NotFound): | if isinstance(e, NotFound): | ||||
status = "not_found" | status = "not_found" | ||||
task_status = "uneventful" | task_status = "uneventful" | ||||
else: | else: | ||||
status = "partial" if self.loaded_snapshot_id else "failed" | status = "partial" if self.loaded_snapshot_id else "failed" | ||||
task_status = "failed" | task_status = "failed" | ||||
self.log.exception( | self.log.exception( | ||||
"Loading failure, updating to `%s` status", | "Loading failure, updating to `%s` status", | ||||
status, | status, | ||||
extra={ | extra={ | ||||
"swh_task_args": [], | "swh_task_args": [], | ||||
"swh_task_kwargs": { | "swh_task_kwargs": { | ||||
"origin": self.origin.url, | "origin": self.origin.url, | ||||
"lister_name": self.lister_name, | "lister_name": self.lister_name, | ||||
"lister_instance_name": self.lister_instance_name, | "lister_instance_name": self.lister_instance_name, | ||||
}, | }, | ||||
}, | }, | ||||
) | ) | ||||
if not isinstance(e, (SystemExit, KeyboardInterrupt)): | |||||
sentry_sdk.capture_exception() | sentry_sdk.capture_exception() | ||||
visit_status = OriginVisitStatus( | visit_status = OriginVisitStatus( | ||||
origin=self.origin.url, | origin=self.origin.url, | ||||
visit=self.visit.visit, | visit=self.visit.visit, | ||||
type=self.visit_type, | type=self.visit_type, | ||||
date=now(), | date=now(), | ||||
status=status, | status=status, | ||||
snapshot=self.loaded_snapshot_id, | snapshot=self.loaded_snapshot_id, | ||||
) | ) | ||||
self.storage.origin_visit_status_add([visit_status]) | self.storage.origin_visit_status_add([visit_status]) | ||||
with self.statsd_timed( | with self.statsd_timed( | ||||
"post_load", tags={"success": success, "status": status} | "post_load", tags={"success": success, "status": status} | ||||
): | ): | ||||
self.post_load(success=success) | self.post_load(success=success) | ||||
if not isinstance(e, Exception): | |||||
# e derives from BaseException but not Exception; this is most likely | |||||
# SystemExit or KeyboardInterrupt, so we should re-raise it. | |||||
raise | |||||
return {"status": task_status} | return {"status": task_status} | ||||
finally: | finally: | ||||
with self.statsd_timed( | with self.statsd_timed( | ||||
"flush", tags={"success": success, "status": status} | "flush", tags={"success": success, "status": status} | ||||
): | ): | ||||
self.flush() | self.flush() | ||||
with self.statsd_timed( | with self.statsd_timed( | ||||
"cleanup", tags={"success": success, "status": status} | "cleanup", tags={"success": success, "status": status} | ||||
▲ Show 20 Lines • Show All 167 Lines • Show Last 20 Lines |