Differential D6455 Diff 23499 swh/web/tests/conftest.py

Changeset View

Standalone View

swh/web/tests/conftest.py

# Copyright (C) 2018-2021 The Software Heritage developers		# Copyright (C) 2018-2021 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution		# See the AUTHORS file at the top-level directory of this distribution
# License: GNU Affero General Public License version 3, or any later version		# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information		# See top-level LICENSE file for more information

from datetime import timedelta		from datetime import timedelta
import json		import json
import os		import os
		import random
import shutil		import shutil
from subprocess import PIPE, run		from subprocess import PIPE, run
import sys		import sys
from typing import Any, Dict, List, Optional		from typing import Any, Dict, List, Optional

from _pytest.python import Function		from _pytest.python import Function
from hypothesis import HealthCheck, settings		from hypothesis import HealthCheck, settings
import pytest		import pytest

from django.contrib.auth.models import User		from django.contrib.auth.models import User
from django.core.cache import cache		from django.core.cache import cache
from django.test.utils import setup_databases # type: ignore		from django.test.utils import setup_databases # type: ignore
from rest_framework.test import APIClient, APIRequestFactory		from rest_framework.test import APIClient, APIRequestFactory

from swh.model.hashutil import ALGORITHMS, hash_to_bytes		from swh.model.hashutil import (
		ALGORITHMS,
		DEFAULT_ALGORITHMS,
		hash_to_bytes,
		hash_to_hex,
		)
		from swh.model.model import Content
		from swh.model.swhids import ObjectType
from swh.scheduler.tests.common import TASK_TYPES		from swh.scheduler.tests.common import TASK_TYPES
from swh.storage.algos.origin import origin_get_latest_visit_status		from swh.storage.algos.origin import origin_get_latest_visit_status
from swh.storage.algos.snapshot import snapshot_get_all_branches, snapshot_get_latest		from swh.storage.algos.snapshot import snapshot_get_all_branches, snapshot_get_latest
from swh.web.auth.utils import OIDC_SWH_WEB_CLIENT_ID		from swh.web.auth.utils import OIDC_SWH_WEB_CLIENT_ID
from swh.web.common import converters		from swh.web.common import converters
from swh.web.common.origin_save import get_scheduler_load_task_types		from swh.web.common.origin_save import get_scheduler_load_task_types
from swh.web.common.typing import OriginVisitInfo		from swh.web.common.typing import OriginVisitInfo
		from swh.web.common.utils import browsers_supported_image_mimes
from swh.web.config import get_config		from swh.web.config import get_config
from swh.web.tests.data import get_tests_data, override_storages		from swh.web.tests.data import get_tests_data, override_storages

# Used to skip some tests		# Used to skip some tests
ctags_json_missing = (		ctags_json_missing = (
shutil.which("ctags") is None		shutil.which("ctags") is None
or b"+json" not in run(["ctags", "--version"], stdout=PIPE).stdout		or b"+json" not in run(["ctags", "--version"], stdout=PIPE).stdout
)		)
▲ Show 20 Lines • Show All 111 Lines • ▼ Show 20 Lines	def tests_data():
# Update swh-web configuration to use the in-memory storages		# Update swh-web configuration to use the in-memory storages
# instantiated in the tests.data module		# instantiated in the tests.data module
override_storages(		override_storages(
data["storage"], data["idx_storage"], data["search"], data["counters"]		data["storage"], data["idx_storage"], data["search"], data["counters"]
)		)
return data		return data


		def _known_swh_objects(tests_data, object_type):
		return tests_data[object_type]


		@pytest.fixture(scope="function")
		def content(tests_data):
		"""Fixture returning a random content ingested into the test archive.
		"""
		return random.choice(_known_swh_objects(tests_data, "contents"))


		@pytest.fixture(scope="function")
		def contents(tests_data):
		"""Fixture returning random contents ingested into the test archive.
		"""
		return random.choices(
		_known_swh_objects(tests_data, "contents"), k=random.randint(2, 8)
		)


		@pytest.fixture(scope="function")
		def empty_content():
		"""Fixture returning the empty content ingested into the test archive.
		"""
		empty_content = Content.from_data(data=b"").to_dict()
		for algo in DEFAULT_ALGORITHMS:
		empty_content[algo] = hash_to_hex(empty_content[algo])
		return empty_content


		@pytest.fixture(scope="function")
		def content_text(tests_data):
		"""
		Fixture returning a random textual content ingested into the test archive.
		"""
		return random.choice(
		list(
		filter(
		lambda c: c["mimetype"].startswith("text/"),
		_known_swh_objects(tests_data, "contents"),
		)
		)
		)


		@pytest.fixture(scope="function")
		def content_text_non_utf8(tests_data):
		"""Fixture returning a random textual content not encoded to UTF-8 ingested
		into the test archive.
		"""
		return random.choice(
		list(
		filter(
		lambda c: c["mimetype"].startswith("text/")
		and c["encoding"] not in ("utf-8", "us-ascii"),
		_known_swh_objects(tests_data, "contents"),
		)
		)
		)


		@pytest.fixture(scope="function")
		def content_application_no_highlight(tests_data):
		"""Fixture returning a random textual content with mimetype
		starting with application/ and no detected programming language to
		highlight ingested into the test archive.
		"""
		return random.choice(
		list(
		filter(
		lambda c: c["mimetype"].startswith("application/")
		and c["encoding"] != "binary"
		and c["hljs_language"] == "nohighlight",
		_known_swh_objects(tests_data, "contents"),
		)
		)
		)


		@pytest.fixture(scope="function")
		def content_text_no_highlight(tests_data):
		"""Fixture returning a random textual content with no detected
		programming language to highlight ingested into the test archive.
		"""
		return random.choice(
		list(
		filter(
		lambda c: c["mimetype"].startswith("text/")
		and c["hljs_language"] == "nohighlight",
		_known_swh_objects(tests_data, "contents"),
		)
		)
		)


		@pytest.fixture(scope="function")
		def content_image_type(tests_data):
		"""Fixture returning a random image content ingested into the test archive.
		"""
		return random.choice(
		list(
		filter(
		lambda c: c["mimetype"] in browsers_supported_image_mimes,
		_known_swh_objects(tests_data, "contents"),
		)
		)
		)


		@pytest.fixture(scope="function")
		def content_unsupported_image_type_rendering(tests_data):
		"""Fixture returning a random image content ingested into the test archive that
		can not be rendered by browsers.
		"""
		return random.choice(
		list(
		filter(
		lambda c: c["mimetype"].startswith("image/")
		and c["mimetype"] not in browsers_supported_image_mimes,
		_known_swh_objects(tests_data, "contents"),
		)
		)
		)


		@pytest.fixture(scope="function")
		def content_utf8_detected_as_binary(tests_data):
		"""Fixture returning a random textual content detected as binary
		by libmagic while they are valid UTF-8 encoded files.
		"""

		def utf8_binary_detected(content):
		if content["encoding"] != "binary":
		return False
		try:
		content["raw_data"].decode("utf-8")
		except Exception:
		return False
		else:
		return True

		return random.choice(
		list(filter(utf8_binary_detected, _known_swh_objects(tests_data, "contents")))
		)


		@pytest.fixture(scope="function")
		def contents_with_ctags():
		"""
		Fixture returning contents ingested into the test archive.
		Those contents are ctags compatible, that is running ctags on those lay results.
		"""
		return {
		"sha1s": [
		"0ab37c02043ebff946c1937523f60aadd0844351",
		"15554cf7608dde6bfefac7e3d525596343a85b6f",
		"2ce837f1489bdfb8faf3ebcc7e72421b5bea83bd",
		"30acd0b47fc25e159e27a980102ddb1c4bea0b95",
		"4f81f05aaea3efb981f9d90144f746d6b682285b",
		"5153aa4b6e4455a62525bc4de38ed0ff6e7dd682",
		"59d08bafa6a749110dfb65ba43a61963d5a5bf9f",
		"7568285b2d7f31ae483ae71617bd3db873deaa2c",
		"7ed3ee8e94ac52ba983dd7690bdc9ab7618247b4",
		"8ed7ef2e7ff9ed845e10259d08e4145f1b3b5b03",
		"9b3557f1ab4111c8607a4f2ea3c1e53c6992916c",
		"9c20da07ed14dc4fcd3ca2b055af99b2598d8bdd",
		"c20ceebd6ec6f7a19b5c3aebc512a12fbdc9234b",
		"e89e55a12def4cd54d5bff58378a3b5119878eb7",
		"e8c0654fe2d75ecd7e0b01bee8a8fc60a130097e",
		"eb6595e559a1d34a2b41e8d4835e0e4f98a5d2b5",
		],
		"symbol_name": "ABS",
		}


		@pytest.fixture(scope="function")
		def content_swhid(tests_data):
		"""
		Fixture returning a qualified SWHID for a random content object
		ingested into the test archive.
		"""
		return random.choice(
		list(
		filter(
		lambda swhid: swhid.object_type == ObjectType.CONTENT,
		_known_swh_objects(tests_data, "swhids"),
		)
		)
		)


# Fixture to manipulate data from a sample archive used in the tests		# Fixture to manipulate data from a sample archive used in the tests
@pytest.fixture(scope="function")		@pytest.fixture(scope="function")
def archive_data(tests_data):		def archive_data(tests_data):
return _ArchiveData(tests_data)		return _ArchiveData(tests_data)


# Fixture to manipulate indexer data from a sample archive used in the tests		# Fixture to manipulate indexer data from a sample archive used in the tests
@pytest.fixture(scope="function")		@pytest.fixture(scope="function")
▲ Show 20 Lines • Show All 368 Lines • Show Last 20 Lines