Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/core/loader.py
| Show First 20 Lines • Show All 240 Lines • ▼ Show 20 Lines | def send_origin(self, origin): | ||||
| extra={ | extra={ | ||||
| 'swh_type': 'storage_send_end', | 'swh_type': 'storage_send_end', | ||||
| 'swh_content_type': 'origin', | 'swh_content_type': 'origin', | ||||
| 'swh_num': 1, | 'swh_num': 1, | ||||
| 'swh_id': log_id | 'swh_id': log_id | ||||
| }) | }) | ||||
| @retry(retry_on_exception=retry_loading, stop_max_attempt_number=3) | @retry(retry_on_exception=retry_loading, stop_max_attempt_number=3) | ||||
| def send_origin_visit(self, visit_date): | def send_origin_visit(self, visit_date, type_): | ||||
| log_id = str(uuid.uuid4()) | log_id = str(uuid.uuid4()) | ||||
| self.log.debug( | self.log.debug( | ||||
| 'Creating origin_visit for origin %s at time %s' % ( | 'Creating origin_visit for origin %s at time %s' % ( | ||||
| self.origin['url'], visit_date), | self.origin['url'], visit_date), | ||||
| extra={ | extra={ | ||||
| 'swh_type': 'storage_send_start', | 'swh_type': 'storage_send_start', | ||||
| 'swh_content_type': 'origin_visit', | 'swh_content_type': 'origin_visit', | ||||
| 'swh_num': 1, | 'swh_num': 1, | ||||
| 'swh_id': log_id | 'swh_id': log_id | ||||
| }) | }) | ||||
| origin_visit = self.storage.origin_visit_add( | origin_visit = self.storage.origin_visit_add( | ||||
| self.origin['url'], visit_date) | self.origin['url'], visit_date, type_) | ||||
| self.log.debug( | self.log.debug( | ||||
| 'Done Creating origin_visit for origin %s at time %s' % ( | 'Done Creating %s origin_visit for origin %s at time %s' % ( | ||||
| self.origin['url'], visit_date), | type_, self.origin['url'], visit_date), | ||||
| extra={ | extra={ | ||||
| 'swh_type': 'storage_send_end', | 'swh_type': 'storage_send_end', | ||||
| 'swh_content_type': 'origin_visit', | 'swh_content_type': 'origin_visit', | ||||
| 'swh_num': 1, | 'swh_num': 1, | ||||
| 'swh_id': log_id | 'swh_id': log_id | ||||
| }) | }) | ||||
| return origin_visit | return origin_visit | ||||
| ▲ Show 20 Lines • Show All 467 Lines • ▼ Show 20 Lines | def _store_origin_visit(self): | ||||
| if 'id' in origin: | if 'id' in origin: | ||||
| # TODO: remove the condition when we finished migrating away | # TODO: remove the condition when we finished migrating away | ||||
| # from origin ids | # from origin ids | ||||
| del origin['id'] | del origin['id'] | ||||
| self.send_origin(origin) | self.send_origin(origin) | ||||
| if not self.visit_date: # now as default visit_date if not provided | if not self.visit_date: # now as default visit_date if not provided | ||||
| self.visit_date = datetime.datetime.now(tz=datetime.timezone.utc) | self.visit_date = datetime.datetime.now(tz=datetime.timezone.utc) | ||||
| self.origin_visit = self.send_origin_visit(self.visit_date) | self.origin_visit = self.send_origin_visit( | ||||
| self.visit_date, self.visit_type) | |||||
| self.visit = self.origin_visit['visit'] | self.visit = self.origin_visit['visit'] | ||||
| @abstractmethod | @abstractmethod | ||||
| def prepare(self, *args, **kwargs): | def prepare(self, *args, **kwargs): | ||||
| """Second step executed by the loader to prepare some state needed by | """Second step executed by the loader to prepare some state needed by | ||||
| the loader. | the loader. | ||||
| """ | """ | ||||
| ▲ Show 20 Lines • Show All 223 Lines • Show Last 20 Lines | |||||