Differential D7726 Diff 28040 swh/loader/core/tests/test_loader.py

Changeset View

Standalone View

swh/loader/core/tests/test_loader.py

# Copyright (C) 2018-2021 The Software Heritage developers		# Copyright (C) 2018-2021 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution		# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version		# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information		# See top-level LICENSE file for more information

import datetime		import datetime
import hashlib		import hashlib
import logging		import logging
from unittest.mock import MagicMock		import time
		from unittest.mock import MagicMock, call

import pytest		import pytest

from swh.loader.core.loader import BaseLoader, DVCSLoader		from swh.loader.core.loader import BaseLoader, DVCSLoader
from swh.loader.core.metadata_fetchers import MetadataFetcherProtocol		from swh.loader.core.metadata_fetchers import MetadataFetcherProtocol
from swh.loader.exception import NotFound		from swh.loader.exception import NotFound
from swh.loader.tests import assert_last_visit_matches		from swh.loader.tests import assert_last_visit_matches
from swh.model.hashutil import hash_to_bytes		from swh.model.hashutil import hash_to_bytes
▲ Show 20 Lines • Show All 254 Lines • ▼ Show 20 Lines	def _check_load_failure(caplog, loader, exc_class, exc_text, status="partial"):
# And confirm that the visit doesn't reference a snapshot		# And confirm that the visit doesn't reference a snapshot
visit = assert_last_visit_matches(loader.storage, ORIGIN.url, status)		visit = assert_last_visit_matches(loader.storage, ORIGIN.url, status)
if status != "partial":		if status != "partial":
assert visit.snapshot is None		assert visit.snapshot is None
# But that the snapshot didn't get loaded		# But that the snapshot didn't get loaded
assert loader.loaded_snapshot_id is None		assert loader.loaded_snapshot_id is None


		@pytest.mark.parametrize("success", [True, False])
		def test_loader_timings(swh_storage, mocker, success):
		current_time = time.time()
		mocker.patch("time.monotonic", side_effect=lambda: current_time)
		mocker.patch("swh.core.statsd.monotonic", side_effect=lambda: current_time)

		runtimes = {
		"pre_cleanup": 2.0,
		"build_extrinsic_origin_metadata": 3.0,
		"prepare": 5.0,
		"fetch_data": 7.0,
		"store_data": 11.0,
		"post_load": 13.0,
		"flush": 17.0,
		"cleanup": 23.0,
		}

		class TimedLoader(BaseLoader):
		visit_type = "my-visit-type"

		def __getattribute__(self, method_name):
		if method_name == "visit_status" and not success:

		def crashy():
		raise Exception("oh no")

		return crashy

		if method_name not in runtimes:
		return super().__getattribute__(method_name)

		def meth(args, *kwargs):
		nonlocal current_time
		current_time += runtimes[method_name]

		return meth

		statsd_report = mocker.patch("swh.core.statsd.statsd._report")

		loader = TimedLoader(swh_storage, origin_url="http://example.org/hello.git")
		loader.load()

		if success:
		expected_tags = {
		"post_load": {"success": True, "status": "full"},
		"flush": {"success": True, "status": "full"},
		"cleanup": {"success": True, "status": "full"},
		}
		else:
		expected_tags = {
		"post_load": {"success": False, "status": "failed"},
		"flush": {"success": False, "status": "failed"},
		"cleanup": {"success": False, "status": "failed"},
		}

		# note that this is a list equality, so order of entries in 'runtimes' matters.
		# This is not perfect, but call() objects are not hashable so it's simpler this way,
		# even if not perfect.
		assert statsd_report.mock_calls == [
		anlambertUnsubmitted Not Done Inline Actions You can directly use `mocker.call` to gain an import statement. anlambert: You can directly use `mocker.call` to gain an import statement.
		call(
		olasdUnsubmitted Not Done Inline Actions Hmpf. We should probably use either dots or underscores consistently (not that it really matters much, it's all getting underscored in the prometheus exporter). olasd: Hmpf. We should probably use either dots or underscores consistently (not that it really…
		"swh_loader_operation_duration_seconds",
		"ms",
		value * 1000,
		{
		"visit_type": "my-visit-type",
		"operation": key,
		**expected_tags.get(key, {}),
		},
		1,
		)
		for (key, value) in runtimes.items()
		]


class DummyDVCSLoaderExc(DummyDVCSLoader):		class DummyDVCSLoaderExc(DummyDVCSLoader):
"""A loader which raises an exception when loading some contents"""		"""A loader which raises an exception when loading some contents"""

def get_contents(self):		def get_contents(self):
raise RuntimeError("Failed to get contents!")		raise RuntimeError("Failed to get contents!")


def test_dvcs_loader_exc_partial_visit(swh_storage, caplog):		def test_dvcs_loader_exc_partial_visit(swh_storage, caplog):
▲ Show 20 Lines • Show All 74 Lines • Show Last 20 Lines