Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/core/tests/test_loader.py
Show First 20 Lines • Show All 91 Lines • ▼ Show 20 Lines | |||||
class DummyBaseLoader(DummyLoader, BaseLoader): | class DummyBaseLoader(DummyLoader, BaseLoader): | ||||
"""Buffered loader will send new data when threshold is reached""" | """Buffered loader will send new data when threshold is reached""" | ||||
def store_data(self): | def store_data(self): | ||||
pass | pass | ||||
class DummyMetadataFetcher: | class DummyMetadataFetcher: | ||||
SUPPORTED_LISTERS = {"fake-lister"} | SUPPORTED_LISTERS = {"fake-forge"} | ||||
FETCHER_NAME = "fake-forge" | |||||
def __init__(self, origin, credentials, lister_name, lister_instance_name): | def __init__(self, origin, credentials, lister_name, lister_instance_name): | ||||
pass | pass | ||||
def get_origin_metadata(self): | def get_origin_metadata(self): | ||||
return [REMD] | return [REMD] | ||||
def get_parent_origins(self): | def get_parent_origins(self): | ||||
return [] | return [] | ||||
class DummyMetadataFetcherWithFork: | class DummyMetadataFetcherWithFork: | ||||
SUPPORTED_LISTERS = {"fake-lister"} | SUPPORTED_LISTERS = {"fake-forge"} | ||||
FETCHER_NAME = "fake-forge" | |||||
def __init__(self, origin, credentials, lister_name, lister_instance_name): | def __init__(self, origin, credentials, lister_name, lister_instance_name): | ||||
pass | pass | ||||
def get_origin_metadata(self): | def get_origin_metadata(self): | ||||
return [REMD] | return [REMD] | ||||
def get_parent_origins(self): | def get_parent_origins(self): | ||||
Show All 19 Lines | def test_base_loader_with_config(swh_storage): | ||||
loader = DummyBaseLoader(swh_storage, "logger-name") | loader = DummyBaseLoader(swh_storage, "logger-name") | ||||
result = loader.load() | result = loader.load() | ||||
assert result == {"status": "eventful"} | assert result == {"status": "eventful"} | ||||
def test_base_loader_with_known_lister_name(swh_storage, mocker): | def test_base_loader_with_known_lister_name(swh_storage, mocker): | ||||
fetcher_cls = MagicMock(wraps=DummyMetadataFetcher) | fetcher_cls = MagicMock(wraps=DummyMetadataFetcher) | ||||
fetcher_cls.SUPPORTED_LISTERS = DummyMetadataFetcher.SUPPORTED_LISTERS | fetcher_cls.SUPPORTED_LISTERS = DummyMetadataFetcher.SUPPORTED_LISTERS | ||||
fetcher_cls.FETCHER_NAME = "fake-forge" | |||||
mocker.patch( | mocker.patch( | ||||
"swh.loader.core.metadata_fetchers._fetchers", return_value=[fetcher_cls] | "swh.loader.core.metadata_fetchers._fetchers", return_value=[fetcher_cls] | ||||
) | ) | ||||
statsd_report = mocker.patch("swh.core.statsd.statsd._report") | |||||
loader = DummyBaseLoader( | loader = DummyBaseLoader( | ||||
swh_storage, lister_name="fake-lister", lister_instance_name="" | swh_storage, lister_name="fake-forge", lister_instance_name="" | ||||
) | ) | ||||
result = loader.load() | result = loader.load() | ||||
assert result == {"status": "eventful"} | assert result == {"status": "eventful"} | ||||
fetcher_cls.assert_called_once() | fetcher_cls.assert_called_once() | ||||
fetcher_cls.assert_called_once_with( | fetcher_cls.assert_called_once_with( | ||||
origin=ORIGIN, | origin=ORIGIN, | ||||
credentials={}, | credentials={}, | ||||
lister_name="fake-lister", | lister_name="fake-forge", | ||||
lister_instance_name="", | lister_instance_name="", | ||||
) | ) | ||||
assert swh_storage.raw_extrinsic_metadata_get( | assert swh_storage.raw_extrinsic_metadata_get( | ||||
ORIGIN.swhid(), METADATA_AUTHORITY | ORIGIN.swhid(), METADATA_AUTHORITY | ||||
).results == [REMD] | ).results == [REMD] | ||||
assert loader.parent_origins == [] | assert loader.parent_origins == [] | ||||
assert [ | |||||
call("swh_loader_metadata_fetchers_sum", "c", 1, {"visit_type": "git"}, 1), | |||||
call("swh_loader_metadata_fetchers_count", "c", 1, {"visit_type": "git"}, 1), | |||||
call( | |||||
"swh_loader_metadata_parent_origins_sum", | |||||
"c", | |||||
0, | |||||
{"fetcher": "fake-forge", "visit_type": "git"}, | |||||
1, | |||||
), | |||||
call( | |||||
"swh_loader_metadata_parent_origins_count", | |||||
"c", | |||||
1, | |||||
{"fetcher": "fake-forge", "visit_type": "git"}, | |||||
1, | |||||
), | |||||
call("swh_loader_metadata_objects_sum", "c", 1, {"visit_type": "git"}, 1), | |||||
call("swh_loader_metadata_objects_count", "c", 1, {"visit_type": "git"}, 1), | |||||
] == [c for c in statsd_report.mock_calls if "_metadata_" in c[1][0]] | |||||
def test_base_loader_with_unknown_lister_name(swh_storage, mocker): | def test_base_loader_with_unknown_lister_name(swh_storage, mocker): | ||||
fetcher_cls = MagicMock(wraps=DummyMetadataFetcher) | fetcher_cls = MagicMock(wraps=DummyMetadataFetcher) | ||||
fetcher_cls.SUPPORTED_LISTERS = DummyMetadataFetcher.SUPPORTED_LISTERS | fetcher_cls.SUPPORTED_LISTERS = DummyMetadataFetcher.SUPPORTED_LISTERS | ||||
mocker.patch( | mocker.patch( | ||||
"swh.loader.core.metadata_fetchers._fetchers", return_value=[fetcher_cls] | "swh.loader.core.metadata_fetchers._fetchers", return_value=[fetcher_cls] | ||||
) | ) | ||||
loader = DummyBaseLoader( | loader = DummyBaseLoader( | ||||
swh_storage, lister_name="other-lister", lister_instance_name="" | swh_storage, lister_name="other-lister", lister_instance_name="" | ||||
) | ) | ||||
result = loader.load() | result = loader.load() | ||||
assert result == {"status": "eventful"} | assert result == {"status": "eventful"} | ||||
fetcher_cls.assert_not_called() | fetcher_cls.assert_not_called() | ||||
with pytest.raises(swh.storage.exc.StorageArgumentException): | with pytest.raises(swh.storage.exc.StorageArgumentException): | ||||
swh_storage.raw_extrinsic_metadata_get(ORIGIN.swhid(), METADATA_AUTHORITY) | swh_storage.raw_extrinsic_metadata_get(ORIGIN.swhid(), METADATA_AUTHORITY) | ||||
def test_base_loader_forked_origin(swh_storage, mocker): | def test_base_loader_forked_origin(swh_storage, mocker): | ||||
fetcher_cls = MagicMock(wraps=DummyMetadataFetcherWithFork) | fetcher_cls = MagicMock(wraps=DummyMetadataFetcherWithFork) | ||||
fetcher_cls.SUPPORTED_LISTERS = DummyMetadataFetcherWithFork.SUPPORTED_LISTERS | fetcher_cls.SUPPORTED_LISTERS = DummyMetadataFetcherWithFork.SUPPORTED_LISTERS | ||||
fetcher_cls.FETCHER_NAME = "fake-forge" | |||||
mocker.patch( | mocker.patch( | ||||
"swh.loader.core.metadata_fetchers._fetchers", return_value=[fetcher_cls] | "swh.loader.core.metadata_fetchers._fetchers", return_value=[fetcher_cls] | ||||
) | ) | ||||
statsd_report = mocker.patch("swh.core.statsd.statsd._report") | |||||
loader = DummyBaseLoader( | loader = DummyBaseLoader( | ||||
swh_storage, lister_name="fake-lister", lister_instance_name="" | swh_storage, lister_name="fake-forge", lister_instance_name="" | ||||
) | ) | ||||
result = loader.load() | result = loader.load() | ||||
assert result == {"status": "eventful"} | assert result == {"status": "eventful"} | ||||
fetcher_cls.assert_called_once() | fetcher_cls.assert_called_once() | ||||
fetcher_cls.assert_called_once_with( | fetcher_cls.assert_called_once_with( | ||||
origin=ORIGIN, | origin=ORIGIN, | ||||
credentials={}, | credentials={}, | ||||
lister_name="fake-lister", | lister_name="fake-forge", | ||||
lister_instance_name="", | lister_instance_name="", | ||||
) | ) | ||||
assert swh_storage.raw_extrinsic_metadata_get( | assert swh_storage.raw_extrinsic_metadata_get( | ||||
ORIGIN.swhid(), METADATA_AUTHORITY | ORIGIN.swhid(), METADATA_AUTHORITY | ||||
).results == [REMD] | ).results == [REMD] | ||||
assert loader.parent_origins == [PARENT_ORIGIN] | assert loader.parent_origins == [PARENT_ORIGIN] | ||||
assert [ | |||||
call("swh_loader_metadata_fetchers_sum", "c", 1, {"visit_type": "git"}, 1), | |||||
call("swh_loader_metadata_fetchers_count", "c", 1, {"visit_type": "git"}, 1), | |||||
call( | |||||
"swh_loader_metadata_parent_origins_sum", | |||||
"c", | |||||
1, | |||||
{"fetcher": "fake-forge", "visit_type": "git"}, | |||||
1, | |||||
), | |||||
call( | |||||
"swh_loader_metadata_parent_origins_count", | |||||
"c", | |||||
1, | |||||
{"fetcher": "fake-forge", "visit_type": "git"}, | |||||
1, | |||||
), | |||||
call("swh_loader_metadata_objects_sum", "c", 1, {"visit_type": "git"}, 1), | |||||
call("swh_loader_metadata_objects_count", "c", 1, {"visit_type": "git"}, 1), | |||||
] == [c for c in statsd_report.mock_calls if "_metadata_" in c[1][0]] | |||||
def test_dvcs_loader(swh_storage): | def test_dvcs_loader(swh_storage): | ||||
loader = DummyDVCSLoader(swh_storage) | loader = DummyDVCSLoader(swh_storage) | ||||
result = loader.load() | result = loader.load() | ||||
assert result == {"status": "eventful"} | assert result == {"status": "eventful"} | ||||
def test_dvcs_loader_with_config(swh_storage): | def test_dvcs_loader_with_config(swh_storage): | ||||
▲ Show 20 Lines • Show All 214 Lines • Show Last 20 Lines |