Changeset View
Changeset View
Standalone View
Standalone View
swh/web/tests/data.py
Show All 14 Lines | |||||
from swh.indexer.fossology_license import FossologyLicenseIndexer | from swh.indexer.fossology_license import FossologyLicenseIndexer | ||||
from swh.indexer.mimetype import MimetypeIndexer | from swh.indexer.mimetype import MimetypeIndexer | ||||
from swh.indexer.storage import get_indexer_storage | from swh.indexer.storage import get_indexer_storage | ||||
from swh.indexer.storage.model import OriginIntrinsicMetadataRow | from swh.indexer.storage.model import OriginIntrinsicMetadataRow | ||||
from swh.loader.git.from_disk import GitLoaderFromArchive | from swh.loader.git.from_disk import GitLoaderFromArchive | ||||
from swh.model.hashutil import DEFAULT_ALGORITHMS, hash_to_bytes, hash_to_hex | from swh.model.hashutil import DEFAULT_ALGORITHMS, hash_to_bytes, hash_to_hex | ||||
from swh.model.model import Content, Directory, Origin, OriginVisit, OriginVisitStatus | from swh.model.model import Content, Directory, Origin, OriginVisit, OriginVisitStatus | ||||
from swh.search import get_search | from swh.search import get_search | ||||
from swh.storage import get_storage | |||||
from swh.storage.algos.dir_iterators import dir_iterator | from swh.storage.algos.dir_iterators import dir_iterator | ||||
from swh.storage.algos.snapshot import snapshot_get_latest | from swh.storage.algos.snapshot import snapshot_get_latest | ||||
from swh.storage.interface import Sha1 | from swh.storage.interface import Sha1 | ||||
from swh.storage.utils import now | from swh.storage.utils import now | ||||
from swh.web import config | from swh.web import config | ||||
from swh.web.browse.utils import ( | from swh.web.browse.utils import ( | ||||
_re_encode_content, | _re_encode_content, | ||||
get_mimetype_and_encoding_for_content, | get_mimetype_and_encoding_for_content, | ||||
prepare_content_for_display, | prepare_content_for_display, | ||||
) | ) | ||||
from swh.web.common import archive | from swh.web.common import archive | ||||
# Module used to initialize data that will be provided as tests input | # Module used to initialize data that will be provided as tests input | ||||
# Configuration for git loader | |||||
_TEST_LOADER_CONFIG = { | |||||
"storage": {"cls": "memory",}, | |||||
"save_data": False, | |||||
"max_content_size": 100 * 1024 * 1024, | |||||
} | |||||
# Base content indexer configuration | # Base content indexer configuration | ||||
_TEST_INDEXER_BASE_CONFIG = { | _TEST_INDEXER_BASE_CONFIG = { | ||||
"storage": {"cls": "memory"}, | "storage": {"cls": "memory"}, | ||||
"objstorage": {"cls": "memory", "args": {},}, | "objstorage": {"cls": "memory", "args": {},}, | ||||
"indexer_storage": {"cls": "memory", "args": {},}, | "indexer_storage": {"cls": "memory", "args": {},}, | ||||
} | } | ||||
▲ Show 20 Lines • Show All 114 Lines • ▼ Show 20 Lines | |||||
ORIGIN_METADATA_VALUE = "git" | ORIGIN_METADATA_VALUE = "git" | ||||
ORIGIN_MASTER_REVISION = {} | ORIGIN_MASTER_REVISION = {} | ||||
# Tests data initialization | # Tests data initialization | ||||
def _init_tests_data(): | def _init_tests_data(): | ||||
# To hold reference to the memory storage | # To hold reference to the memory storage | ||||
storage = None | storage = get_storage("memory") | ||||
# Create search instance | # Create search instance | ||||
search = get_search("memory") | search = get_search("memory") | ||||
search.initialize() | search.initialize() | ||||
search.origin_update({"url": origin["url"]} for origin in _TEST_ORIGINS) | search.origin_update({"url": origin["url"]} for origin in _TEST_ORIGINS) | ||||
# Create indexer storage instance that will be shared by indexers | # Create indexer storage instance that will be shared by indexers | ||||
idx_storage = get_indexer_storage("memory") | idx_storage = get_indexer_storage("memory") | ||||
# Declare a test tool for origin intrinsic metadata tests | # Declare a test tool for origin intrinsic metadata tests | ||||
idx_tool = idx_storage.indexer_configuration_add([INDEXER_TOOL])[0] | idx_tool = idx_storage.indexer_configuration_add([INDEXER_TOOL])[0] | ||||
INDEXER_TOOL["id"] = idx_tool["id"] | INDEXER_TOOL["id"] = idx_tool["id"] | ||||
# Load git repositories from archives | # Load git repositories from archives | ||||
for origin in _TEST_ORIGINS: | for origin in _TEST_ORIGINS: | ||||
for i, archive_ in enumerate(origin["archives"]): | for i, archive_ in enumerate(origin["archives"]): | ||||
if i > 0: | if i > 0: | ||||
# ensure visit dates will be different when simulating | # ensure visit dates will be different when simulating | ||||
# multiple visits of an origin | # multiple visits of an origin | ||||
time.sleep(1) | time.sleep(1) | ||||
origin_repo_archive = os.path.join( | origin_repo_archive = os.path.join( | ||||
os.path.dirname(__file__), "resources/repos/%s" % archive_ | os.path.dirname(__file__), "resources/repos/%s" % archive_ | ||||
) | ) | ||||
loader = GitLoaderFromArchive( | loader = GitLoaderFromArchive( | ||||
origin["url"], | storage, origin["url"], archive_path=origin_repo_archive, | ||||
archive_path=origin_repo_archive, | ) | ||||
config=_TEST_LOADER_CONFIG, | |||||
) | |||||
if storage is None: | |||||
storage = loader.storage | |||||
else: | |||||
loader.storage = storage | |||||
result = loader.load() | result = loader.load() | ||||
assert result["status"] == "eventful" | assert result["status"] == "eventful" | ||||
ori = storage.origin_get([origin["url"]])[0] | ori = storage.origin_get([origin["url"]])[0] | ||||
origin.update(ori.to_dict()) # add an 'id' key if enabled | origin.update(ori.to_dict()) # add an 'id' key if enabled | ||||
search.origin_update([{"url": origin["url"], "has_visits": True}]) | search.origin_update([{"url": origin["url"], "has_visits": True}]) | ||||
for i in range(250): | for i in range(250): | ||||
▲ Show 20 Lines • Show All 193 Lines • Show Last 20 Lines |