Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/core/loader.py
| Show First 20 Lines • Show All 395 Lines • ▼ Show 20 Lines | def send_contents(self, content_list): | ||||
| self.log.debug("Sending %d contents" % num_contents, | self.log.debug("Sending %d contents" % num_contents, | ||||
| extra={ | extra={ | ||||
| 'swh_type': 'storage_send_start', | 'swh_type': 'storage_send_start', | ||||
| 'swh_content_type': 'content', | 'swh_content_type': 'content', | ||||
| 'swh_num': num_contents, | 'swh_num': num_contents, | ||||
| 'swh_id': log_id, | 'swh_id': log_id, | ||||
| }) | }) | ||||
| result = self.storage.content_add(content_list) | result = self.storage.content_add(content_list) | ||||
| self.counters['contents'] += result['content:add'] | self.counters['contents'] += result.get('content:add', 0) | ||||
| self.log.debug("Done sending %d contents" % num_contents, | self.log.debug("Done sending %d contents" % num_contents, | ||||
| extra={ | extra={ | ||||
| 'swh_type': 'storage_send_end', | 'swh_type': 'storage_send_end', | ||||
| 'swh_content_type': 'content', | 'swh_content_type': 'content', | ||||
| 'swh_num': num_contents, | 'swh_num': num_contents, | ||||
| 'swh_id': log_id, | 'swh_id': log_id, | ||||
| }) | }) | ||||
| @retry(retry_on_exception=retry_loading, stop_max_attempt_number=3) | @retry(retry_on_exception=retry_loading, stop_max_attempt_number=3) | ||||
| def send_directories(self, directory_list): | def send_directories(self, directory_list): | ||||
| """Actually send properly formatted directories to the database. | """Actually send properly formatted directories to the database. | ||||
| """ | """ | ||||
| num_directories = len(directory_list) | num_directories = len(directory_list) | ||||
| if num_directories > 0: | if num_directories > 0: | ||||
| log_id = str(uuid.uuid4()) | log_id = str(uuid.uuid4()) | ||||
| self.log.debug("Sending %d directories" % num_directories, | self.log.debug("Sending %d directories" % num_directories, | ||||
| extra={ | extra={ | ||||
| 'swh_type': 'storage_send_start', | 'swh_type': 'storage_send_start', | ||||
| 'swh_content_type': 'directory', | 'swh_content_type': 'directory', | ||||
| 'swh_num': num_directories, | 'swh_num': num_directories, | ||||
| 'swh_id': log_id, | 'swh_id': log_id, | ||||
| }) | }) | ||||
| result = self.storage.directory_add(directory_list) | result = self.storage.directory_add(directory_list) | ||||
| self.counters['directories'] += result['directory:add'] | self.counters['directories'] += result.get('directory:add', 0) | ||||
| self.log.debug("Done sending %d directories" % num_directories, | self.log.debug("Done sending %d directories" % num_directories, | ||||
| extra={ | extra={ | ||||
| 'swh_type': 'storage_send_end', | 'swh_type': 'storage_send_end', | ||||
| 'swh_content_type': 'directory', | 'swh_content_type': 'directory', | ||||
| 'swh_num': num_directories, | 'swh_num': num_directories, | ||||
| 'swh_id': log_id, | 'swh_id': log_id, | ||||
| }) | }) | ||||
| @retry(retry_on_exception=retry_loading, stop_max_attempt_number=3) | @retry(retry_on_exception=retry_loading, stop_max_attempt_number=3) | ||||
| def send_revisions(self, revision_list): | def send_revisions(self, revision_list): | ||||
| """Actually send properly formatted revisions to the database. | """Actually send properly formatted revisions to the database. | ||||
| """ | """ | ||||
| num_revisions = len(revision_list) | num_revisions = len(revision_list) | ||||
| if num_revisions > 0: | if num_revisions > 0: | ||||
| log_id = str(uuid.uuid4()) | log_id = str(uuid.uuid4()) | ||||
| self.log.debug("Sending %d revisions" % num_revisions, | self.log.debug("Sending %d revisions" % num_revisions, | ||||
| extra={ | extra={ | ||||
| 'swh_type': 'storage_send_start', | 'swh_type': 'storage_send_start', | ||||
| 'swh_content_type': 'revision', | 'swh_content_type': 'revision', | ||||
| 'swh_num': num_revisions, | 'swh_num': num_revisions, | ||||
| 'swh_id': log_id, | 'swh_id': log_id, | ||||
| }) | }) | ||||
| result = self.storage.revision_add(revision_list) | result = self.storage.revision_add(revision_list) | ||||
| self.counters['revisions'] += result['revision:add'] | self.counters['revisions'] += result.get('revision:add', 0) | ||||
| self.log.debug("Done sending %d revisions" % num_revisions, | self.log.debug("Done sending %d revisions" % num_revisions, | ||||
| extra={ | extra={ | ||||
| 'swh_type': 'storage_send_end', | 'swh_type': 'storage_send_end', | ||||
| 'swh_content_type': 'revision', | 'swh_content_type': 'revision', | ||||
| 'swh_num': num_revisions, | 'swh_num': num_revisions, | ||||
| 'swh_id': log_id, | 'swh_id': log_id, | ||||
| }) | }) | ||||
| @retry(retry_on_exception=retry_loading, stop_max_attempt_number=3) | @retry(retry_on_exception=retry_loading, stop_max_attempt_number=3) | ||||
| def send_releases(self, release_list): | def send_releases(self, release_list): | ||||
| """Actually send properly formatted releases to the database. | """Actually send properly formatted releases to the database. | ||||
| """ | """ | ||||
| num_releases = len(release_list) | num_releases = len(release_list) | ||||
| if num_releases > 0: | if num_releases > 0: | ||||
| log_id = str(uuid.uuid4()) | log_id = str(uuid.uuid4()) | ||||
| self.log.debug("Sending %d releases" % num_releases, | self.log.debug("Sending %d releases" % num_releases, | ||||
| extra={ | extra={ | ||||
| 'swh_type': 'storage_send_start', | 'swh_type': 'storage_send_start', | ||||
| 'swh_content_type': 'release', | 'swh_content_type': 'release', | ||||
| 'swh_num': num_releases, | 'swh_num': num_releases, | ||||
| 'swh_id': log_id, | 'swh_id': log_id, | ||||
| }) | }) | ||||
| result = self.storage.release_add(release_list) | result = self.storage.release_add(release_list) | ||||
| self.counters['releases'] += result['release:add'] | self.counters['releases'] += result.get('release:add', 0) | ||||
| self.log.debug("Done sending %d releases" % num_releases, | self.log.debug("Done sending %d releases" % num_releases, | ||||
| extra={ | extra={ | ||||
| 'swh_type': 'storage_send_end', | 'swh_type': 'storage_send_end', | ||||
| 'swh_content_type': 'release', | 'swh_content_type': 'release', | ||||
| 'swh_num': num_releases, | 'swh_num': num_releases, | ||||
| 'swh_id': log_id, | 'swh_id': log_id, | ||||
| }) | }) | ||||
| ▲ Show 20 Lines • Show All 490 Lines • Show Last 20 Lines | |||||