Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/core/tests/test_loader.py
| Show First 20 Lines • Show All 91 Lines • ▼ Show 20 Lines | |||||
| class DummyBaseLoader(DummyLoader, BaseLoader): | class DummyBaseLoader(DummyLoader, BaseLoader): | ||||
| """Buffered loader will send new data when threshold is reached""" | """Buffered loader will send new data when threshold is reached""" | ||||
| def store_data(self): | def store_data(self): | ||||
| pass | pass | ||||
| class DummyMetadataFetcher: | class DummyMetadataFetcher: | ||||
| SUPPORTED_LISTERS = {"fake-lister"} | SUPPORTED_LISTERS = {"fake-forge"} | ||||
| FETCHER_NAME = "fake-forge" | |||||
| def __init__(self, origin, credentials, lister_name, lister_instance_name): | def __init__(self, origin, credentials, lister_name, lister_instance_name): | ||||
| pass | pass | ||||
| def get_origin_metadata(self): | def get_origin_metadata(self): | ||||
| return [REMD] | return [REMD] | ||||
| def get_parent_origins(self): | def get_parent_origins(self): | ||||
| return [] | return [] | ||||
| class DummyMetadataFetcherWithFork: | class DummyMetadataFetcherWithFork: | ||||
| SUPPORTED_LISTERS = {"fake-lister"} | SUPPORTED_LISTERS = {"fake-forge"} | ||||
| FETCHER_NAME = "fake-forge" | |||||
| def __init__(self, origin, credentials, lister_name, lister_instance_name): | def __init__(self, origin, credentials, lister_name, lister_instance_name): | ||||
| pass | pass | ||||
| def get_origin_metadata(self): | def get_origin_metadata(self): | ||||
| return [REMD] | return [REMD] | ||||
| def get_parent_origins(self): | def get_parent_origins(self): | ||||
| Show All 19 Lines | def test_base_loader_with_config(swh_storage): | ||||
| loader = DummyBaseLoader(swh_storage, "logger-name") | loader = DummyBaseLoader(swh_storage, "logger-name") | ||||
| result = loader.load() | result = loader.load() | ||||
| assert result == {"status": "eventful"} | assert result == {"status": "eventful"} | ||||
| def test_base_loader_with_known_lister_name(swh_storage, mocker): | def test_base_loader_with_known_lister_name(swh_storage, mocker): | ||||
| fetcher_cls = MagicMock(wraps=DummyMetadataFetcher) | fetcher_cls = MagicMock(wraps=DummyMetadataFetcher) | ||||
| fetcher_cls.SUPPORTED_LISTERS = DummyMetadataFetcher.SUPPORTED_LISTERS | fetcher_cls.SUPPORTED_LISTERS = DummyMetadataFetcher.SUPPORTED_LISTERS | ||||
| fetcher_cls.FETCHER_NAME = "fake-forge" | |||||
| mocker.patch( | mocker.patch( | ||||
| "swh.loader.core.metadata_fetchers._fetchers", return_value=[fetcher_cls] | "swh.loader.core.metadata_fetchers._fetchers", return_value=[fetcher_cls] | ||||
| ) | ) | ||||
| statsd_report = mocker.patch("swh.core.statsd.statsd._report") | |||||
| loader = DummyBaseLoader( | loader = DummyBaseLoader( | ||||
| swh_storage, lister_name="fake-lister", lister_instance_name="" | swh_storage, lister_name="fake-forge", lister_instance_name="" | ||||
| ) | ) | ||||
| result = loader.load() | result = loader.load() | ||||
| assert result == {"status": "eventful"} | assert result == {"status": "eventful"} | ||||
| fetcher_cls.assert_called_once() | fetcher_cls.assert_called_once() | ||||
| fetcher_cls.assert_called_once_with( | fetcher_cls.assert_called_once_with( | ||||
| origin=ORIGIN, | origin=ORIGIN, | ||||
| credentials={}, | credentials={}, | ||||
| lister_name="fake-lister", | lister_name="fake-forge", | ||||
| lister_instance_name="", | lister_instance_name="", | ||||
| ) | ) | ||||
| assert swh_storage.raw_extrinsic_metadata_get( | assert swh_storage.raw_extrinsic_metadata_get( | ||||
| ORIGIN.swhid(), METADATA_AUTHORITY | ORIGIN.swhid(), METADATA_AUTHORITY | ||||
| ).results == [REMD] | ).results == [REMD] | ||||
| assert loader.parent_origins == [] | assert loader.parent_origins == [] | ||||
| assert [ | |||||
| call("swh_loader_metadata_fetchers_sum", "c", 1, {"visit_type": "git"}, 1), | |||||
| call("swh_loader_metadata_fetchers_count", "c", 1, {"visit_type": "git"}, 1), | |||||
| call( | |||||
| "swh_loader_metadata_parent_origins_sum", | |||||
| "c", | |||||
| 0, | |||||
| {"fetcher": "fake-forge", "visit_type": "git"}, | |||||
| 1, | |||||
| ), | |||||
| call( | |||||
| "swh_loader_metadata_parent_origins_count", | |||||
| "c", | |||||
| 1, | |||||
| {"fetcher": "fake-forge", "visit_type": "git"}, | |||||
| 1, | |||||
| ), | |||||
| call("swh_loader_metadata_objects_sum", "c", 1, {"visit_type": "git"}, 1), | |||||
| call("swh_loader_metadata_objects_count", "c", 1, {"visit_type": "git"}, 1), | |||||
| ] == [c for c in statsd_report.mock_calls if "_metadata_" in c[1][0]] | |||||
| def test_base_loader_with_unknown_lister_name(swh_storage, mocker): | def test_base_loader_with_unknown_lister_name(swh_storage, mocker): | ||||
| fetcher_cls = MagicMock(wraps=DummyMetadataFetcher) | fetcher_cls = MagicMock(wraps=DummyMetadataFetcher) | ||||
| fetcher_cls.SUPPORTED_LISTERS = DummyMetadataFetcher.SUPPORTED_LISTERS | fetcher_cls.SUPPORTED_LISTERS = DummyMetadataFetcher.SUPPORTED_LISTERS | ||||
| mocker.patch( | mocker.patch( | ||||
| "swh.loader.core.metadata_fetchers._fetchers", return_value=[fetcher_cls] | "swh.loader.core.metadata_fetchers._fetchers", return_value=[fetcher_cls] | ||||
| ) | ) | ||||
| loader = DummyBaseLoader( | loader = DummyBaseLoader( | ||||
| swh_storage, lister_name="other-lister", lister_instance_name="" | swh_storage, lister_name="other-lister", lister_instance_name="" | ||||
| ) | ) | ||||
| result = loader.load() | result = loader.load() | ||||
| assert result == {"status": "eventful"} | assert result == {"status": "eventful"} | ||||
| fetcher_cls.assert_not_called() | fetcher_cls.assert_not_called() | ||||
| with pytest.raises(swh.storage.exc.StorageArgumentException): | with pytest.raises(swh.storage.exc.StorageArgumentException): | ||||
| swh_storage.raw_extrinsic_metadata_get(ORIGIN.swhid(), METADATA_AUTHORITY) | swh_storage.raw_extrinsic_metadata_get(ORIGIN.swhid(), METADATA_AUTHORITY) | ||||
| def test_base_loader_forked_origin(swh_storage, mocker): | def test_base_loader_forked_origin(swh_storage, mocker): | ||||
| fetcher_cls = MagicMock(wraps=DummyMetadataFetcherWithFork) | fetcher_cls = MagicMock(wraps=DummyMetadataFetcherWithFork) | ||||
| fetcher_cls.SUPPORTED_LISTERS = DummyMetadataFetcherWithFork.SUPPORTED_LISTERS | fetcher_cls.SUPPORTED_LISTERS = DummyMetadataFetcherWithFork.SUPPORTED_LISTERS | ||||
| fetcher_cls.FETCHER_NAME = "fake-forge" | |||||
| mocker.patch( | mocker.patch( | ||||
| "swh.loader.core.metadata_fetchers._fetchers", return_value=[fetcher_cls] | "swh.loader.core.metadata_fetchers._fetchers", return_value=[fetcher_cls] | ||||
| ) | ) | ||||
| statsd_report = mocker.patch("swh.core.statsd.statsd._report") | |||||
| loader = DummyBaseLoader( | loader = DummyBaseLoader( | ||||
| swh_storage, lister_name="fake-lister", lister_instance_name="" | swh_storage, lister_name="fake-forge", lister_instance_name="" | ||||
| ) | ) | ||||
| result = loader.load() | result = loader.load() | ||||
| assert result == {"status": "eventful"} | assert result == {"status": "eventful"} | ||||
| fetcher_cls.assert_called_once() | fetcher_cls.assert_called_once() | ||||
| fetcher_cls.assert_called_once_with( | fetcher_cls.assert_called_once_with( | ||||
| origin=ORIGIN, | origin=ORIGIN, | ||||
| credentials={}, | credentials={}, | ||||
| lister_name="fake-lister", | lister_name="fake-forge", | ||||
| lister_instance_name="", | lister_instance_name="", | ||||
| ) | ) | ||||
| assert swh_storage.raw_extrinsic_metadata_get( | assert swh_storage.raw_extrinsic_metadata_get( | ||||
| ORIGIN.swhid(), METADATA_AUTHORITY | ORIGIN.swhid(), METADATA_AUTHORITY | ||||
| ).results == [REMD] | ).results == [REMD] | ||||
| assert loader.parent_origins == [PARENT_ORIGIN] | assert loader.parent_origins == [PARENT_ORIGIN] | ||||
| assert [ | |||||
| call("swh_loader_metadata_fetchers_sum", "c", 1, {"visit_type": "git"}, 1), | |||||
| call("swh_loader_metadata_fetchers_count", "c", 1, {"visit_type": "git"}, 1), | |||||
| call( | |||||
| "swh_loader_metadata_parent_origins_sum", | |||||
| "c", | |||||
| 1, | |||||
| {"fetcher": "fake-forge", "visit_type": "git"}, | |||||
| 1, | |||||
| ), | |||||
| call( | |||||
| "swh_loader_metadata_parent_origins_count", | |||||
| "c", | |||||
| 1, | |||||
| {"fetcher": "fake-forge", "visit_type": "git"}, | |||||
| 1, | |||||
| ), | |||||
| call("swh_loader_metadata_objects_sum", "c", 1, {"visit_type": "git"}, 1), | |||||
| call("swh_loader_metadata_objects_count", "c", 1, {"visit_type": "git"}, 1), | |||||
| ] == [c for c in statsd_report.mock_calls if "_metadata_" in c[1][0]] | |||||
| def test_dvcs_loader(swh_storage): | def test_dvcs_loader(swh_storage): | ||||
| loader = DummyDVCSLoader(swh_storage) | loader = DummyDVCSLoader(swh_storage) | ||||
| result = loader.load() | result = loader.load() | ||||
| assert result == {"status": "eventful"} | assert result == {"status": "eventful"} | ||||
| def test_dvcs_loader_with_config(swh_storage): | def test_dvcs_loader_with_config(swh_storage): | ||||
| ▲ Show 20 Lines • Show All 214 Lines • Show Last 20 Lines | |||||