diff --git a/swh/loader/core/loader.py b/swh/loader/core/loader.py --- a/swh/loader/core/loader.py +++ b/swh/loader/core/loader.py @@ -215,95 +215,6 @@ return origin_visit - @retry(retry_on_exception=retry_loading, stop_max_attempt_number=3) - def send_tool(self, tool: Dict[str, Any]) -> None: - log_id = str(uuid.uuid4()) - self.log.debug( - 'Creating tool with name %s version %s configuration %s' % ( - tool['name'], tool['version'], tool['configuration']), - extra={ - 'swh_type': 'storage_send_start', - 'swh_content_type': 'tool', - 'swh_num': 1, - 'swh_id': log_id - }) - - tools = self.storage.tool_add([tool]) - tool_id = tools[0]['id'] - - self.log.debug( - 'Done creating tool with name %s version %s and configuration %s' % ( # noqa - tool['name'], tool['version'], tool['configuration']), - extra={ - 'swh_type': 'storage_send_end', - 'swh_content_type': 'tool', - 'swh_num': 1, - 'swh_id': log_id - }) - return tool_id - - @retry(retry_on_exception=retry_loading, stop_max_attempt_number=3) - def send_provider(self, provider: Dict[str, Any]) -> None: - log_id = str(uuid.uuid4()) - self.log.debug( - 'Creating metadata_provider with name %s type %s url %s' % ( - provider['provider_name'], provider['provider_type'], - provider['provider_url']), - extra={ - 'swh_type': 'storage_send_start', - 'swh_content_type': 'metadata_provider', - 'swh_num': 1, - 'swh_id': log_id - }) - # FIXME: align metadata_provider_add with indexer_configuration_add - _provider = self.storage.metadata_provider_get_by(provider) - if _provider and 'id' in _provider: - provider_id = _provider['id'] - else: - provider_id = self.storage.metadata_provider_add( - provider['provider_name'], - provider['provider_type'], - provider['provider_url'], - provider['metadata']) - - self.log.debug( - 'Done creating metadata_provider with name %s type %s url %s' % ( - provider['provider_name'], provider['provider_type'], - provider['provider_url']), - extra={ - 'swh_type': 'storage_send_end', - 'swh_content_type': 'metadata_provider', - 'swh_num': 1, - 'swh_id': log_id - }) - return provider_id - - @retry(retry_on_exception=retry_loading, stop_max_attempt_number=3) - def send_origin_metadata(self, visit_date, provider_id, - tool_id, metadata): - log_id = str(uuid.uuid4()) - self.log.debug( - 'Creating origin_metadata for origin %s at time %s with provider_id %s and tool_id %s' % ( # noqa - self.origin['url'], visit_date, provider_id, tool_id), - extra={ - 'swh_type': 'storage_send_start', - 'swh_content_type': 'origin_metadata', - 'swh_num': 1, - 'swh_id': log_id - }) - - self.storage.origin_metadata_add( - self.origin['url'], visit_date, provider_id, tool_id, metadata) - self.log.debug( - 'Done Creating origin_metadata for origin %s at time %s with provider %s and tool %s' % ( # noqa - self.origin['url'], visit_date, provider_id, tool_id), - extra={ - 'swh_type': 'storage_send_end', - 'swh_content_type': 'origin_metadata', - 'swh_num': 1, - 'swh_id': log_id - }) - @retry(retry_on_exception=retry_loading, stop_max_attempt_number=3) def update_origin_visit(self, status: str) -> None: log_id = str(uuid.uuid4()) @@ -457,30 +368,6 @@ if hasattr(self.storage, 'flush'): self.storage.flush() - def prepare_metadata(self) -> None: - """First step for origin_metadata insertion, resolving the - provider_id and the tool_id by fetching data from the storage - or creating tool and provider on the fly if the data isn't available - - """ - origin_metadata = self.origin_metadata - - tool = origin_metadata['tool'] - try: - tool_id = self.send_tool(tool) - self.origin_metadata['tool']['tool_id'] = tool_id - except Exception: - self.log.exception('Problem when storing new tool') - raise - - provider = origin_metadata['provider'] - try: - provider_id = self.send_provider(provider) - self.origin_metadata['provider']['provider_id'] = provider_id - except Exception: - self.log.exception('Problem when storing new provider') - raise - @abstractmethod def cleanup(self) -> None: """Last step executed by the loader. diff --git a/swh/loader/core/tests/test_loader.py b/swh/loader/core/tests/test_loader.py --- a/swh/loader/core/tests/test_loader.py +++ b/swh/loader/core/tests/test_loader.py @@ -327,27 +327,6 @@ self.assertCountReleases(1) self.assertCountSnapshots(1) - def test_origin_metadata(self): - self.loader.load() - - provider_id = self.loader.send_provider(self.in_provider) - tool_id = self.loader.send_tool(self.in_tool) - - self.loader.send_origin_metadata( - self.loader.visit_date, provider_id, - tool_id, {'test_metadata': 'foobar'}) - - self.assertOriginMetadataContains(self.in_origin['url'], - {'test_metadata': 'foobar'}) - - with self.assertRaises(AssertionError): - self.assertOriginMetadataContains(self.in_origin['url'], - {'test_metadata': 'foobarbaz'}) - - with self.assertRaises(Exception): - self.assertOriginMetadataContains(self.in_origin['url'] + 'blah', - {'test_metadata': 'foobar'}) - def test_loader_logger_default_name(): loader = DummyBufferedLoader()