diff --git a/swh/loader/core/loader.py b/swh/loader/core/loader.py --- a/swh/loader/core/loader.py +++ b/swh/loader/core/loader.py @@ -467,17 +467,36 @@ ), "lister_instance_name is None, but lister_name is not" metadata = [] - for cls in get_fetchers_for_lister(self.lister_name): + + fetcher_classes = get_fetchers_for_lister(self.lister_name) + statsd.histogram( + f"{STATSD_PREFIX}.metadata.fetchers", + len(fetcher_classes), + tags={"visit_type": self.visit_type}, + ) + for cls in fetcher_classes: metadata_fetcher = cls( origin=self.origin, lister_name=self.lister_name, lister_instance_name=self.lister_instance_name, credentials=self.metadata_fetcher_credentials, ) - with self.statsd_timed("fetch_one_metadata"): + with self.statsd_timed( + "fetch_one_metadata", tags={"fetcher": cls.FETCHER_NAME} + ): metadata.extend(metadata_fetcher.get_origin_metadata()) if self.parent_origins is None: self.parent_origins = metadata_fetcher.get_parent_origins() + statsd.histogram( + f"{STATSD_PREFIX}.metadata.parent_origins", + len(self.parent_origins), + tags={"fetcher": cls.FETCHER_NAME, "visit_type": self.visit_type}, + ) + statsd.histogram( + f"{STATSD_PREFIX}.metadata.objects", + len(metadata), + tags={"visit_type": self.visit_type}, + ) return metadata diff --git a/swh/loader/core/metadata_fetchers.py b/swh/loader/core/metadata_fetchers.py --- a/swh/loader/core/metadata_fetchers.py +++ b/swh/loader/core/metadata_fetchers.py @@ -20,6 +20,7 @@ to loaders, via setuptools entrypoints.""" SUPPORTED_LISTERS: Set[str] + FETCHER_NAME: str def __init__( self, diff --git a/swh/loader/core/tests/test_loader.py b/swh/loader/core/tests/test_loader.py --- a/swh/loader/core/tests/test_loader.py +++ b/swh/loader/core/tests/test_loader.py @@ -97,7 +97,8 @@ class DummyMetadataFetcher: - SUPPORTED_LISTERS = {"fake-lister"} + SUPPORTED_LISTERS = {"fake-forge"} + FETCHER_NAME = "fake-forge" def __init__(self, origin, credentials, lister_name, lister_instance_name): pass @@ -110,7 +111,8 @@ class DummyMetadataFetcherWithFork: - SUPPORTED_LISTERS = {"fake-lister"} + SUPPORTED_LISTERS = {"fake-forge"} + FETCHER_NAME = "fake-forge" def __init__(self, origin, credentials, lister_name, lister_instance_name): pass @@ -151,7 +153,7 @@ ) loader = DummyBaseLoader( - swh_storage, lister_name="fake-lister", lister_instance_name="" + swh_storage, lister_name="fake-forge", lister_instance_name="" ) result = loader.load() assert result == {"status": "eventful"} @@ -160,7 +162,7 @@ fetcher_cls.assert_called_once_with( origin=ORIGIN, credentials={}, - lister_name="fake-lister", + lister_name="fake-forge", lister_instance_name="", ) assert swh_storage.raw_extrinsic_metadata_get( @@ -195,7 +197,7 @@ ) loader = DummyBaseLoader( - swh_storage, lister_name="fake-lister", lister_instance_name="" + swh_storage, lister_name="fake-forge", lister_instance_name="" ) result = loader.load() assert result == {"status": "eventful"} @@ -204,7 +206,7 @@ fetcher_cls.assert_called_once_with( origin=ORIGIN, credentials={}, - lister_name="fake-lister", + lister_name="fake-forge", lister_instance_name="", ) assert swh_storage.raw_extrinsic_metadata_get(