Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/core/tests/test_loader.py
Show All 15 Lines | from swh.loader.core.loader import ( | ||||
SENTRY_ORIGIN_URL_TAG_NAME, | SENTRY_ORIGIN_URL_TAG_NAME, | ||||
SENTRY_VISIT_TYPE_TAG_NAME, | SENTRY_VISIT_TYPE_TAG_NAME, | ||||
BaseLoader, | BaseLoader, | ||||
ContentLoader, | ContentLoader, | ||||
DirectoryLoader, | DirectoryLoader, | ||||
) | ) | ||||
from swh.loader.core.metadata_fetchers import MetadataFetcherProtocol | from swh.loader.core.metadata_fetchers import MetadataFetcherProtocol | ||||
from swh.loader.exception import NotFound, UnsupportedChecksumComputation | from swh.loader.exception import NotFound, UnsupportedChecksumComputation | ||||
from swh.loader.tests import assert_last_visit_matches | from swh.loader.tests import assert_last_visit_matches, get_stats | ||||
from swh.model.hashutil import hash_to_bytes | |||||
from swh.model.model import ( | from swh.model.model import ( | ||||
MetadataAuthority, | MetadataAuthority, | ||||
MetadataAuthorityType, | MetadataAuthorityType, | ||||
MetadataFetcher, | MetadataFetcher, | ||||
Origin, | Origin, | ||||
RawExtrinsicMetadata, | RawExtrinsicMetadata, | ||||
Snapshot, | |||||
SnapshotBranch, | |||||
TargetType, | |||||
) | ) | ||||
import swh.storage.exc | import swh.storage.exc | ||||
from .conftest import compute_hashes, compute_nar_hashes, nix_store_missing | from .conftest import compute_hashes, compute_nar_hashes, nix_store_missing | ||||
ORIGIN = Origin(url="some-url") | ORIGIN = Origin(url="some-url") | ||||
PARENT_ORIGIN = Origin(url="base-origin-url") | PARENT_ORIGIN = Origin(url="base-origin-url") | ||||
Show All 33 Lines | class DummyLoader: | ||||
def get_snapshot_id(self): | def get_snapshot_id(self): | ||||
return None | return None | ||||
class DummyBaseLoader(DummyLoader, BaseLoader): | class DummyBaseLoader(DummyLoader, BaseLoader): | ||||
"""Buffered loader will send new data when threshold is reached""" | """Buffered loader will send new data when threshold is reached""" | ||||
def store_data(self): | def store_data(self) -> None: | ||||
pass | pass | ||||
class DummyMetadataFetcher: | class DummyMetadataFetcher: | ||||
SUPPORTED_LISTERS = {"fake-forge"} | SUPPORTED_LISTERS = {"fake-forge"} | ||||
FETCHER_NAME = "fake-forge" | FETCHER_NAME = "fake-forge" | ||||
def __init__(self, origin, credentials, lister_name, lister_instance_name): | def __init__(self, origin, credentials, lister_name, lister_instance_name): | ||||
▲ Show 20 Lines • Show All 271 Lines • ▼ Show 20 Lines | assert statsd_report.mock_calls == [ | ||||
1, | 1, | ||||
) | ) | ||||
for (key, value) in runtimes.items() | for (key, value) in runtimes.items() | ||||
] | ] | ||||
assert loader.statsd.namespace == "swh_loader" | assert loader.statsd.namespace == "swh_loader" | ||||
assert loader.statsd.constant_tags == {"visit_type": "my-visit-type"} | assert loader.statsd.constant_tags == {"visit_type": "my-visit-type"} | ||||
class DummyLoaderWithPartialSnapshot(DummyBaseLoader): | |||||
call = 0 | |||||
def fetch_data(self): | |||||
self.call += 1 | |||||
# Let's have one call to fetch data and then another to fetch further data | |||||
return self.call == 1 | |||||
def store_data(self) -> None: | |||||
# First call does nothing and the last one flushes the final snapshot | |||||
if self.call != 1: | |||||
self.storage.snapshot_add([Snapshot(branches={})]) | |||||
def build_partial_snapshot(self): | |||||
"""Build partial snapshot to serialize during loading.""" | |||||
return Snapshot( | |||||
branches={ | |||||
b"alias": SnapshotBranch( | |||||
target=hash_to_bytes(b"0" * 20), | |||||
target_type=TargetType.DIRECTORY, | |||||
) | |||||
} | |||||
) | |||||
def test_loader_with_partial_snapshot(swh_storage, sentry_events): | |||||
"""Ensure loader can write partial snapshot when configured to.""" | |||||
loader = DummyLoaderWithPartialSnapshot( | |||||
swh_storage, "dummy-url", create_partial_snapshot=True | |||||
) | |||||
status = loader.load() | |||||
assert status == {"status": "eventful"} | |||||
actual_stats = get_stats(swh_storage) | |||||
expected_stats = { | |||||
"origin": 1, # only 1 origin | |||||
"origin_visit": 1, # with 1 visit | |||||
"snapshot": 1 + 1, # 1 partial snapshot and 1 final snapshot | |||||
} | |||||
for key in expected_stats.keys(): | |||||
assert actual_stats[key] == expected_stats[key] | |||||
class DummyLoaderWithError(DummyBaseLoader): | class DummyLoaderWithError(DummyBaseLoader): | ||||
def prepare(self, *args, **kwargs): | def prepare(self, *args, **kwargs): | ||||
raise Exception("error") | raise Exception("error") | ||||
def test_loader_sentry_tags_on_error(swh_storage, sentry_events): | def test_loader_sentry_tags_on_error(swh_storage, sentry_events): | ||||
loader = DummyLoaderWithError(swh_storage) | loader = DummyLoaderWithError(swh_storage) | ||||
loader.load() | loader.load() | ||||
▲ Show 20 Lines • Show All 317 Lines • Show Last 20 Lines |