Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/core/loader.py
| Show First 20 Lines • Show All 356 Lines • ▼ Show 20 Lines | def load(self) -> Dict[str, str]: | ||||
| }, | }, | ||||
| }, | }, | ||||
| ) | ) | ||||
| total_time_fetch_data = 0.0 | total_time_fetch_data = 0.0 | ||||
| total_time_store_data = 0.0 | total_time_store_data = 0.0 | ||||
| try: | try: | ||||
| # Initially not a success, will be True when actually one | |||||
| success = False | |||||
| with self.statsd_timed("prepare"): | with self.statsd_timed("prepare"): | ||||
| self.prepare() | self.prepare() | ||||
| while True: | while True: | ||||
| t1 = time.monotonic() | t1 = time.monotonic() | ||||
| more_data_to_fetch = self.fetch_data() | more_data_to_fetch = self.fetch_data() | ||||
| t2 = time.monotonic() | t2 = time.monotonic() | ||||
| total_time_fetch_data += t2 - t1 | total_time_fetch_data += t2 - t1 | ||||
| Show All 16 Lines | def load(self) -> Dict[str, str]: | ||||
| snapshot=self.loaded_snapshot_id, | snapshot=self.loaded_snapshot_id, | ||||
| ) | ) | ||||
| self.storage.origin_visit_status_add([visit_status]) | self.storage.origin_visit_status_add([visit_status]) | ||||
| success = True | success = True | ||||
| with self.statsd_timed( | with self.statsd_timed( | ||||
| "post_load", tags={"success": success, "status": status} | "post_load", tags={"success": success, "status": status} | ||||
| ): | ): | ||||
| self.post_load() | self.post_load() | ||||
| except Exception as e: | except BaseException as e: | ||||
| success = False | |||||
| if isinstance(e, NotFound): | if isinstance(e, NotFound): | ||||
| status = "not_found" | status = "not_found" | ||||
| task_status = "uneventful" | task_status = "uneventful" | ||||
| else: | else: | ||||
| status = "partial" if self.loaded_snapshot_id else "failed" | status = "partial" if self.loaded_snapshot_id else "failed" | ||||
| task_status = "failed" | task_status = "failed" | ||||
| self.log.exception( | self.log.exception( | ||||
| "Loading failure, updating to `%s` status", | "Loading failure, updating to `%s` status", | ||||
| status, | status, | ||||
| extra={ | extra={ | ||||
| "swh_task_args": [], | "swh_task_args": [], | ||||
| "swh_task_kwargs": { | "swh_task_kwargs": { | ||||
| "origin": self.origin.url, | "origin": self.origin.url, | ||||
| "lister_name": self.lister_name, | "lister_name": self.lister_name, | ||||
| "lister_instance_name": self.lister_instance_name, | "lister_instance_name": self.lister_instance_name, | ||||
| }, | }, | ||||
| }, | }, | ||||
| ) | ) | ||||
| if not isinstance(e, (SystemExit, KeyboardInterrupt)): | |||||
| sentry_sdk.capture_exception() | sentry_sdk.capture_exception() | ||||
| visit_status = OriginVisitStatus( | visit_status = OriginVisitStatus( | ||||
| origin=self.origin.url, | origin=self.origin.url, | ||||
| visit=self.visit.visit, | visit=self.visit.visit, | ||||
| type=self.visit_type, | type=self.visit_type, | ||||
| date=now(), | date=now(), | ||||
| status=status, | status=status, | ||||
| snapshot=self.loaded_snapshot_id, | snapshot=self.loaded_snapshot_id, | ||||
| ) | ) | ||||
| self.storage.origin_visit_status_add([visit_status]) | self.storage.origin_visit_status_add([visit_status]) | ||||
| with self.statsd_timed( | with self.statsd_timed( | ||||
| "post_load", tags={"success": success, "status": status} | "post_load", tags={"success": success, "status": status} | ||||
| ): | ): | ||||
| self.post_load(success=success) | self.post_load(success=success) | ||||
| if not isinstance(e, Exception): | |||||
| # e derives from BaseException but not Exception; this is most likely | |||||
| # SystemExit or KeyboardInterrupt, so we should re-raise it. | |||||
| raise | |||||
| return {"status": task_status} | return {"status": task_status} | ||||
| finally: | finally: | ||||
| with self.statsd_timed( | with self.statsd_timed( | ||||
| "flush", tags={"success": success, "status": status} | "flush", tags={"success": success, "status": status} | ||||
| ): | ): | ||||
| self.flush() | self.flush() | ||||
| with self.statsd_timed( | with self.statsd_timed( | ||||
| "cleanup", tags={"success": success, "status": status} | "cleanup", tags={"success": success, "status": status} | ||||
| ▲ Show 20 Lines • Show All 167 Lines • Show Last 20 Lines | |||||