Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/core/tests/test_loader.py
| Show First 20 Lines • Show All 146 Lines • ▼ Show 20 Lines | |||||
| def test_base_loader_with_known_lister_name(swh_storage, mocker): | def test_base_loader_with_known_lister_name(swh_storage, mocker): | ||||
| fetcher_cls = MagicMock(wraps=DummyMetadataFetcher) | fetcher_cls = MagicMock(wraps=DummyMetadataFetcher) | ||||
| fetcher_cls.SUPPORTED_LISTERS = DummyMetadataFetcher.SUPPORTED_LISTERS | fetcher_cls.SUPPORTED_LISTERS = DummyMetadataFetcher.SUPPORTED_LISTERS | ||||
| fetcher_cls.FETCHER_NAME = "fake-forge" | fetcher_cls.FETCHER_NAME = "fake-forge" | ||||
| mocker.patch( | mocker.patch( | ||||
| "swh.loader.core.metadata_fetchers._fetchers", return_value=[fetcher_cls] | "swh.loader.core.metadata_fetchers._fetchers", return_value=[fetcher_cls] | ||||
| ) | ) | ||||
| statsd_report = mocker.patch("swh.core.statsd.statsd._report") | |||||
| loader = DummyBaseLoader( | loader = DummyBaseLoader( | ||||
| swh_storage, lister_name="fake-forge", lister_instance_name="" | swh_storage, lister_name="fake-forge", lister_instance_name="" | ||||
| ) | ) | ||||
| statsd_report = mocker.patch.object(loader.statsd, "_report") | |||||
| result = loader.load() | result = loader.load() | ||||
| assert result == {"status": "eventful"} | assert result == {"status": "eventful"} | ||||
| fetcher_cls.assert_called_once() | fetcher_cls.assert_called_once() | ||||
| fetcher_cls.assert_called_once_with( | fetcher_cls.assert_called_once_with( | ||||
| origin=ORIGIN, | origin=ORIGIN, | ||||
| credentials={}, | credentials={}, | ||||
| lister_name="fake-forge", | lister_name="fake-forge", | ||||
| lister_instance_name="", | lister_instance_name="", | ||||
| ) | ) | ||||
| assert swh_storage.raw_extrinsic_metadata_get( | assert swh_storage.raw_extrinsic_metadata_get( | ||||
| ORIGIN.swhid(), METADATA_AUTHORITY | ORIGIN.swhid(), METADATA_AUTHORITY | ||||
| ).results == [REMD] | ).results == [REMD] | ||||
| assert loader.parent_origins == [] | assert loader.parent_origins == [] | ||||
| assert [ | assert [ | ||||
| call("swh_loader_metadata_fetchers_sum", "c", 1, {"visit_type": "git"}, 1), | call("metadata_fetchers_sum", "c", 1, {}, 1), | ||||
| call("swh_loader_metadata_fetchers_count", "c", 1, {"visit_type": "git"}, 1), | call("metadata_fetchers_count", "c", 1, {}, 1), | ||||
| call( | call("metadata_parent_origins_sum", "c", 0, {"fetcher": "fake-forge"}, 1), | ||||
| "swh_loader_metadata_parent_origins_sum", | call("metadata_parent_origins_count", "c", 1, {"fetcher": "fake-forge"}, 1), | ||||
| "c", | call("metadata_objects_sum", "c", 1, {}, 1), | ||||
| 0, | call("metadata_objects_count", "c", 1, {}, 1), | ||||
| {"fetcher": "fake-forge", "visit_type": "git"}, | ] == [c for c in statsd_report.mock_calls if "metadata_" in c[1][0]] | ||||
| 1, | assert loader.statsd.namespace == "swh_loader" | ||||
| ), | assert loader.statsd.constant_tags == {"visit_type": "git"} | ||||
| call( | |||||
| "swh_loader_metadata_parent_origins_count", | |||||
| "c", | |||||
| 1, | |||||
| {"fetcher": "fake-forge", "visit_type": "git"}, | |||||
| 1, | |||||
| ), | |||||
| call("swh_loader_metadata_objects_sum", "c", 1, {"visit_type": "git"}, 1), | |||||
| call("swh_loader_metadata_objects_count", "c", 1, {"visit_type": "git"}, 1), | |||||
| ] == [c for c in statsd_report.mock_calls if "_metadata_" in c[1][0]] | |||||
| def test_base_loader_with_unknown_lister_name(swh_storage, mocker): | def test_base_loader_with_unknown_lister_name(swh_storage, mocker): | ||||
| fetcher_cls = MagicMock(wraps=DummyMetadataFetcher) | fetcher_cls = MagicMock(wraps=DummyMetadataFetcher) | ||||
| fetcher_cls.SUPPORTED_LISTERS = DummyMetadataFetcher.SUPPORTED_LISTERS | fetcher_cls.SUPPORTED_LISTERS = DummyMetadataFetcher.SUPPORTED_LISTERS | ||||
| mocker.patch( | mocker.patch( | ||||
| "swh.loader.core.metadata_fetchers._fetchers", return_value=[fetcher_cls] | "swh.loader.core.metadata_fetchers._fetchers", return_value=[fetcher_cls] | ||||
| ) | ) | ||||
| Show All 11 Lines | |||||
| def test_base_loader_forked_origin(swh_storage, mocker): | def test_base_loader_forked_origin(swh_storage, mocker): | ||||
| fetcher_cls = MagicMock(wraps=DummyMetadataFetcherWithFork) | fetcher_cls = MagicMock(wraps=DummyMetadataFetcherWithFork) | ||||
| fetcher_cls.SUPPORTED_LISTERS = DummyMetadataFetcherWithFork.SUPPORTED_LISTERS | fetcher_cls.SUPPORTED_LISTERS = DummyMetadataFetcherWithFork.SUPPORTED_LISTERS | ||||
| fetcher_cls.FETCHER_NAME = "fake-forge" | fetcher_cls.FETCHER_NAME = "fake-forge" | ||||
| mocker.patch( | mocker.patch( | ||||
| "swh.loader.core.metadata_fetchers._fetchers", return_value=[fetcher_cls] | "swh.loader.core.metadata_fetchers._fetchers", return_value=[fetcher_cls] | ||||
| ) | ) | ||||
| statsd_report = mocker.patch("swh.core.statsd.statsd._report") | |||||
| loader = DummyBaseLoader( | loader = DummyBaseLoader( | ||||
| swh_storage, lister_name="fake-forge", lister_instance_name="" | swh_storage, lister_name="fake-forge", lister_instance_name="" | ||||
| ) | ) | ||||
| statsd_report = mocker.patch.object(loader.statsd, "_report") | |||||
| result = loader.load() | result = loader.load() | ||||
| assert result == {"status": "eventful"} | assert result == {"status": "eventful"} | ||||
| fetcher_cls.assert_called_once() | fetcher_cls.assert_called_once() | ||||
| fetcher_cls.assert_called_once_with( | fetcher_cls.assert_called_once_with( | ||||
| origin=ORIGIN, | origin=ORIGIN, | ||||
| credentials={}, | credentials={}, | ||||
| lister_name="fake-forge", | lister_name="fake-forge", | ||||
| lister_instance_name="", | lister_instance_name="", | ||||
| ) | ) | ||||
| assert swh_storage.raw_extrinsic_metadata_get( | assert swh_storage.raw_extrinsic_metadata_get( | ||||
| ORIGIN.swhid(), METADATA_AUTHORITY | ORIGIN.swhid(), METADATA_AUTHORITY | ||||
| ).results == [REMD] | ).results == [REMD] | ||||
| assert loader.parent_origins == [PARENT_ORIGIN] | assert loader.parent_origins == [PARENT_ORIGIN] | ||||
| assert [ | assert [ | ||||
| call("swh_loader_metadata_fetchers_sum", "c", 1, {"visit_type": "git"}, 1), | call("metadata_fetchers_sum", "c", 1, {}, 1), | ||||
| call("swh_loader_metadata_fetchers_count", "c", 1, {"visit_type": "git"}, 1), | call("metadata_fetchers_count", "c", 1, {}, 1), | ||||
| call( | call("metadata_parent_origins_sum", "c", 1, {"fetcher": "fake-forge"}, 1), | ||||
| "swh_loader_metadata_parent_origins_sum", | call("metadata_parent_origins_count", "c", 1, {"fetcher": "fake-forge"}, 1), | ||||
| "c", | call("metadata_objects_sum", "c", 1, {}, 1), | ||||
| 1, | call("metadata_objects_count", "c", 1, {}, 1), | ||||
| {"fetcher": "fake-forge", "visit_type": "git"}, | ] == [c for c in statsd_report.mock_calls if "metadata_" in c[1][0]] | ||||
| 1, | assert loader.statsd.namespace == "swh_loader" | ||||
| ), | assert loader.statsd.constant_tags == {"visit_type": "git"} | ||||
| call( | |||||
| "swh_loader_metadata_parent_origins_count", | |||||
| "c", | |||||
| 1, | |||||
| {"fetcher": "fake-forge", "visit_type": "git"}, | |||||
| 1, | |||||
| ), | |||||
| call("swh_loader_metadata_objects_sum", "c", 1, {"visit_type": "git"}, 1), | |||||
| call("swh_loader_metadata_objects_count", "c", 1, {"visit_type": "git"}, 1), | |||||
| ] == [c for c in statsd_report.mock_calls if "_metadata_" in c[1][0]] | |||||
| def test_dvcs_loader(swh_storage): | def test_dvcs_loader(swh_storage): | ||||
| loader = DummyDVCSLoader(swh_storage) | loader = DummyDVCSLoader(swh_storage) | ||||
| result = loader.load() | result = loader.load() | ||||
| assert result == {"status": "eventful"} | assert result == {"status": "eventful"} | ||||
| ▲ Show 20 Lines • Show All 88 Lines • ▼ Show 20 Lines | class TimedLoader(BaseLoader): | ||||
| return super().__getattribute__(method_name) | return super().__getattribute__(method_name) | ||||
| def meth(*args, **kwargs): | def meth(*args, **kwargs): | ||||
| nonlocal current_time | nonlocal current_time | ||||
| current_time += runtimes[method_name] | current_time += runtimes[method_name] | ||||
| return meth | return meth | ||||
| statsd_report = mocker.patch("swh.core.statsd.statsd._report") | |||||
| loader = TimedLoader(swh_storage, origin_url="http://example.org/hello.git") | loader = TimedLoader(swh_storage, origin_url="http://example.org/hello.git") | ||||
| statsd_report = mocker.patch.object(loader.statsd, "_report") | |||||
| loader.load() | loader.load() | ||||
| if success: | if success: | ||||
| expected_tags = { | expected_tags = { | ||||
| "post_load": {"success": True, "status": "full"}, | "post_load": {"success": True, "status": "full"}, | ||||
| "flush": {"success": True, "status": "full"}, | "flush": {"success": True, "status": "full"}, | ||||
| "cleanup": {"success": True, "status": "full"}, | "cleanup": {"success": True, "status": "full"}, | ||||
| } | } | ||||
| else: | else: | ||||
| expected_tags = { | expected_tags = { | ||||
| "post_load": {"success": False, "status": "failed"}, | "post_load": {"success": False, "status": "failed"}, | ||||
| "flush": {"success": False, "status": "failed"}, | "flush": {"success": False, "status": "failed"}, | ||||
| "cleanup": {"success": False, "status": "failed"}, | "cleanup": {"success": False, "status": "failed"}, | ||||
| } | } | ||||
| # note that this is a list equality, so order of entries in 'runtimes' matters. | # note that this is a list equality, so order of entries in 'runtimes' matters. | ||||
| # This is not perfect, but call() objects are not hashable so it's simpler this way, | # This is not perfect, but call() objects are not hashable so it's simpler this way, | ||||
| # even if not perfect. | # even if not perfect. | ||||
| assert statsd_report.mock_calls == [ | assert statsd_report.mock_calls == [ | ||||
| call( | call( | ||||
| "swh_loader_operation_duration_seconds", | "operation_duration_seconds", | ||||
| "ms", | "ms", | ||||
| value * 1000, | value * 1000, | ||||
| { | {"operation": key, **expected_tags.get(key, {})}, | ||||
| "visit_type": "my-visit-type", | |||||
| "operation": key, | |||||
| **expected_tags.get(key, {}), | |||||
| }, | |||||
| 1, | 1, | ||||
| ) | ) | ||||
| for (key, value) in runtimes.items() | for (key, value) in runtimes.items() | ||||
| ] | ] | ||||
| assert loader.statsd.namespace == "swh_loader" | |||||
| assert loader.statsd.constant_tags == {"visit_type": "my-visit-type"} | |||||
| class DummyDVCSLoaderExc(DummyDVCSLoader): | class DummyDVCSLoaderExc(DummyDVCSLoader): | ||||
| """A loader which raises an exception when loading some contents""" | """A loader which raises an exception when loading some contents""" | ||||
| def get_contents(self): | def get_contents(self): | ||||
| raise RuntimeError("Failed to get contents!") | raise RuntimeError("Failed to get contents!") | ||||
| ▲ Show 20 Lines • Show All 76 Lines • Show Last 20 Lines | |||||