Changeset View
Changeset View
Standalone View
Standalone View
swh/web/tests/data.py
Show All 11 Lines | |||||
from swh.core.config import merge_configs | from swh.core.config import merge_configs | ||||
from swh.indexer.ctags import CtagsIndexer | from swh.indexer.ctags import CtagsIndexer | ||||
from swh.indexer.fossology_license import FossologyLicenseIndexer | from swh.indexer.fossology_license import FossologyLicenseIndexer | ||||
from swh.indexer.mimetype import MimetypeIndexer | from swh.indexer.mimetype import MimetypeIndexer | ||||
from swh.indexer.storage import get_indexer_storage | from swh.indexer.storage import get_indexer_storage | ||||
from swh.indexer.storage.model import OriginIntrinsicMetadataRow | from swh.indexer.storage.model import OriginIntrinsicMetadataRow | ||||
from swh.loader.git.from_disk import GitLoaderFromArchive | from swh.loader.git.from_disk import GitLoaderFromArchive | ||||
from swh.model.hashutil import DEFAULT_ALGORITHMS, hash_to_bytes, hash_to_hex | from swh.model.hashutil import DEFAULT_ALGORITHMS, hash_to_hex | ||||
from swh.model.model import Content, Directory, Origin, OriginVisit, OriginVisitStatus | from swh.model.model import ( | ||||
Content, | |||||
Directory, | |||||
Origin, | |||||
OriginVisit, | |||||
OriginVisitStatus, | |||||
Snapshot, | |||||
) | |||||
from swh.search import get_search | from swh.search import get_search | ||||
from swh.storage import get_storage | from swh.storage import get_storage | ||||
from swh.storage.algos.dir_iterators import dir_iterator | from swh.storage.algos.dir_iterators import dir_iterator | ||||
from swh.storage.algos.snapshot import snapshot_get_latest | from swh.storage.algos.snapshot import snapshot_get_latest | ||||
from swh.storage.interface import Sha1 | from swh.storage.interface import Sha1 | ||||
from swh.storage.utils import now | from swh.storage.utils import now | ||||
from swh.web import config | from swh.web import config | ||||
from swh.web.browse.utils import ( | from swh.web.browse.utils import ( | ||||
▲ Show 20 Lines • Show All 131 Lines • ▼ Show 20 Lines | |||||
} | } | ||||
ORIGIN_METADATA_KEY = "vcs" | ORIGIN_METADATA_KEY = "vcs" | ||||
ORIGIN_METADATA_VALUE = "git" | ORIGIN_METADATA_VALUE = "git" | ||||
ORIGIN_MASTER_REVISION = {} | ORIGIN_MASTER_REVISION = {} | ||||
def _add_origin(storage, search, origin_url, visit_type="git", snapshot_branches={}): | |||||
storage.origin_add([Origin(url=origin_url)]) | |||||
search.origin_update( | |||||
[{"url": origin_url, "has_visits": True, "visit_types": [visit_type]}] | |||||
) | |||||
date = now() | |||||
visit = OriginVisit(origin=origin_url, date=date, type=visit_type) | |||||
visit = storage.origin_visit_add([visit])[0] | |||||
snapshot = Snapshot.from_dict({"branches": snapshot_branches}) | |||||
storage.snapshot_add([snapshot]) | |||||
visit_status = OriginVisitStatus( | |||||
origin=origin_url, | |||||
visit=visit.visit, | |||||
date=date + timedelta(minutes=1), | |||||
type=visit.type, | |||||
status="full", | |||||
snapshot=snapshot.id, | |||||
) | |||||
storage.origin_visit_status_add([visit_status]) | |||||
# Tests data initialization | # Tests data initialization | ||||
def _init_tests_data(): | def _init_tests_data(): | ||||
# To hold reference to the memory storage | # To hold reference to the memory storage | ||||
storage = get_storage("memory") | storage = get_storage("memory") | ||||
# Create search instance | # Create search instance | ||||
search = get_search("memory") | search = get_search("memory") | ||||
search.initialize() | search.initialize() | ||||
Show All 25 Lines | for origin in _TEST_ORIGINS: | ||||
ori = storage.origin_get([origin["url"]])[0] | ori = storage.origin_get([origin["url"]])[0] | ||||
origin.update(ori.to_dict()) # add an 'id' key if enabled | origin.update(ori.to_dict()) # add an 'id' key if enabled | ||||
search.origin_update( | search.origin_update( | ||||
[{"url": origin["url"], "has_visits": True, "visit_types": ["git"]}] | [{"url": origin["url"], "has_visits": True, "visit_types": ["git"]}] | ||||
) | ) | ||||
for i in range(250): | for i in range(250): | ||||
url = "https://many.origins/%d" % (i + 1) | _add_origin( | ||||
# storage.origin_add([{'url': url}]) | storage, search, origin_url=f"https://many.origins/{i+1}", visit_type="tar" | ||||
storage.origin_add([Origin(url=url)]) | |||||
search.origin_update([{"url": url, "has_visits": True, "visit_types": ["tar"]}]) | |||||
date = now() | |||||
visit = OriginVisit(origin=url, date=date, type="tar") | |||||
visit = storage.origin_visit_add([visit])[0] | |||||
visit_status = OriginVisitStatus( | |||||
origin=url, | |||||
visit=visit.visit, | |||||
date=date + timedelta(minutes=1), | |||||
type=visit.type, | |||||
status="full", | |||||
snapshot=hash_to_bytes("1a8893e6a86f444e8be8e7bda6cb34fb1735a00e"), | |||||
) | ) | ||||
storage.origin_visit_status_add([visit_status]) | |||||
sha1s: Set[Sha1] = set() | sha1s: Set[Sha1] = set() | ||||
directories = set() | directories = set() | ||||
revisions = set() | revisions = set() | ||||
releases = set() | releases = set() | ||||
snapshots = set() | snapshots = set() | ||||
content_path = {} | content_path = {} | ||||
▲ Show 20 Lines • Show All 85 Lines • ▼ Show 20 Lines | for content in result: | ||||
contents.append(content_metadata) | contents.append(content_metadata) | ||||
# Add the empty directory to the test archive | # Add the empty directory to the test archive | ||||
storage.directory_add([Directory(entries=())]) | storage.directory_add([Directory(entries=())]) | ||||
# Add empty content to the test archive | # Add empty content to the test archive | ||||
storage.content_add([Content.from_data(data=b"")]) | storage.content_add([Content.from_data(data=b"")]) | ||||
# Add fake git origin with pull request branches | |||||
_add_origin( | |||||
storage, | |||||
search, | |||||
origin_url="https://git.example.org/project", | |||||
snapshot_branches={ | |||||
b"refs/heads/master": { | |||||
"target_type": "revision", | |||||
"target": next(iter(revisions)), | |||||
}, | |||||
**{ | |||||
f"refs/pull/{i}".encode(): { | |||||
"target_type": "revision", | |||||
"target": next(iter(revisions)), | |||||
} | |||||
for i in range(300) | |||||
}, | |||||
}, | |||||
) | |||||
# Return tests data | # Return tests data | ||||
return { | return { | ||||
"search": search, | "search": search, | ||||
"storage": storage, | "storage": storage, | ||||
"idx_storage": idx_storage, | "idx_storage": idx_storage, | ||||
"origins": _TEST_ORIGINS, | "origins": _TEST_ORIGINS, | ||||
"contents": contents, | "contents": contents, | ||||
"directories": list(directories), | "directories": list(directories), | ||||
▲ Show 20 Lines • Show All 70 Lines • Show Last 20 Lines |