Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/package/npm/tests/test_npm.py
Show All 13 Lines | from swh.loader.package.npm.loader import ( | ||||
_author_str, | _author_str, | ||||
extract_npm_package_author, | extract_npm_package_author, | ||||
) | ) | ||||
from swh.loader.tests import assert_last_visit_matches, check_snapshot, get_stats | from swh.loader.tests import assert_last_visit_matches, check_snapshot, get_stats | ||||
from swh.model.hashutil import hash_to_bytes | from swh.model.hashutil import hash_to_bytes | ||||
from swh.model.model import ( | from swh.model.model import ( | ||||
Person, | Person, | ||||
RawExtrinsicMetadata, | RawExtrinsicMetadata, | ||||
Release, | |||||
Snapshot, | Snapshot, | ||||
SnapshotBranch, | SnapshotBranch, | ||||
TargetType, | TargetType, | ||||
Timestamp, | |||||
TimestampWithTimezone, | |||||
) | ) | ||||
from swh.model.model import MetadataAuthority, MetadataAuthorityType, MetadataFetcher | from swh.model.model import MetadataAuthority, MetadataAuthorityType, MetadataFetcher | ||||
from swh.model.model import ObjectType as ModelObjectType | from swh.model.model import ObjectType as ModelObjectType | ||||
from swh.model.swhids import CoreSWHID, ExtendedObjectType, ExtendedSWHID, ObjectType | from swh.model.swhids import CoreSWHID, ExtendedObjectType, ExtendedSWHID, ObjectType | ||||
from swh.storage.interface import PagedResult | from swh.storage.interface import PagedResult | ||||
@pytest.fixture | @pytest.fixture | ||||
▲ Show 20 Lines • Show All 242 Lines • ▼ Show 20 Lines | [ | ||||
"202fafcd7c0f8230e89d5496ad7f44ab12b807bf", | "202fafcd7c0f8230e89d5496ad7f44ab12b807bf", | ||||
"775cc516543be86c15c1dc172f49c0d4e6e78235", | "775cc516543be86c15c1dc172f49c0d4e6e78235", | ||||
"ff3d1ead85a14f891e8b3fa3a89de39db1b8de2e", | "ff3d1ead85a14f891e8b3fa3a89de39db1b8de2e", | ||||
] | ] | ||||
) | ) | ||||
_expected_new_releases_first_visit = normalize_hashes( | _expected_new_releases_first_visit = normalize_hashes( | ||||
{ | { | ||||
"d25e722a32c145b3eb88b416049dd35d27759a87": ( | "adcc40ee87a3ebb1b5a82edd692cf52aa5099cee": ( | ||||
"42753c0c2ab00c4501b552ac4671c68f3cf5aece" | "42753c0c2ab00c4501b552ac4671c68f3cf5aece" | ||||
), | ), | ||||
"3522e846b97c0b8434c565fe891c0f082a357e5d": ( | "c781147df0e4963a0f9859134abd28296b702233": ( | ||||
"3370d20d6f96dc1c9e50f083e2134881db110f4f" | "3370d20d6f96dc1c9e50f083e2134881db110f4f" | ||||
), | ), | ||||
"54f6c1711c6aedb6de3cf2d6347b9f772e343784": ( | "f544812dac98e7589155be7dfaef64477a408ec0": ( | ||||
"d7895533ef5edbcffdea3f057d9fef3a1ef845ce" | "d7895533ef5edbcffdea3f057d9fef3a1ef845ce" | ||||
), | ), | ||||
} | } | ||||
) | ) | ||||
def package_url(package): | def package_url(package): | ||||
return "https://www.npmjs.com/package/%s" % package | return "https://www.npmjs.com/package/%s" % package | ||||
def package_metadata_url(package): | def package_metadata_url(package): | ||||
return "https://replicate.npmjs.com/%s/" % package | return "https://replicate.npmjs.com/%s/" % package | ||||
def test_npm_loader_first_visit(swh_storage, requests_mock_datadir, org_api_info): | def test_npm_loader_first_visit(swh_storage, requests_mock_datadir, org_api_info): | ||||
package = "org" | package = "org" | ||||
url = package_url(package) | url = package_url(package) | ||||
loader = NpmLoader(swh_storage, url) | loader = NpmLoader(swh_storage, url) | ||||
actual_load_status = loader.load() | actual_load_status = loader.load() | ||||
expected_snapshot_id = hash_to_bytes("ddaad89b0b4edb7eefe7c92e9b1166caa776ebbc") | expected_snapshot_id = hash_to_bytes("d24e3f10492ade1e9462ec701370fef4a79a40f1") | ||||
assert actual_load_status == { | assert actual_load_status == { | ||||
"status": "eventful", | "status": "eventful", | ||||
"snapshot_id": expected_snapshot_id.hex(), | "snapshot_id": expected_snapshot_id.hex(), | ||||
} | } | ||||
assert_last_visit_matches( | assert_last_visit_matches( | ||||
swh_storage, url, status="full", type="npm", snapshot=expected_snapshot_id | swh_storage, url, status="full", type="npm", snapshot=expected_snapshot_id | ||||
) | ) | ||||
versions = [ | versions = [ | ||||
("0.0.2", "d25e722a32c145b3eb88b416049dd35d27759a87"), | ("0.0.2", "adcc40ee87a3ebb1b5a82edd692cf52aa5099cee"), | ||||
("0.0.3", "3522e846b97c0b8434c565fe891c0f082a357e5d"), | ("0.0.3", "c781147df0e4963a0f9859134abd28296b702233"), | ||||
("0.0.4", "54f6c1711c6aedb6de3cf2d6347b9f772e343784"), | ("0.0.4", "f544812dac98e7589155be7dfaef64477a408ec0"), | ||||
] | ] | ||||
expected_snapshot = Snapshot( | expected_snapshot = Snapshot( | ||||
id=expected_snapshot_id, | id=expected_snapshot_id, | ||||
branches={ | branches={ | ||||
b"HEAD": SnapshotBranch( | b"HEAD": SnapshotBranch( | ||||
target=b"releases/0.0.4", target_type=TargetType.ALIAS | target=b"releases/0.0.4", target_type=TargetType.ALIAS | ||||
), | ), | ||||
**{ | **{ | ||||
b"releases/" | b"releases/" | ||||
+ version_name.encode(): SnapshotBranch( | + version_name.encode(): SnapshotBranch( | ||||
target=hash_to_bytes(version_id), target_type=TargetType.RELEASE, | target=hash_to_bytes(version_id), target_type=TargetType.RELEASE, | ||||
) | ) | ||||
for (version_name, version_id) in versions | for (version_name, version_id) in versions | ||||
}, | }, | ||||
}, | }, | ||||
) | ) | ||||
check_snapshot(expected_snapshot, swh_storage) | check_snapshot(expected_snapshot, swh_storage) | ||||
assert swh_storage.release_get( | |||||
[hash_to_bytes("adcc40ee87a3ebb1b5a82edd692cf52aa5099cee")] | |||||
)[0] == Release( | |||||
name=b"0.0.2", | |||||
message=b"Synthetic release for NPM source package org version 0.0.2", | |||||
target=hash_to_bytes("42753c0c2ab00c4501b552ac4671c68f3cf5aece"), | |||||
target_type=ModelObjectType.DIRECTORY, | |||||
synthetic=True, | |||||
author=Person( | |||||
fullname=b"mooz <stillpedant@gmail.com>", | |||||
name=b"mooz", | |||||
email=b"stillpedant@gmail.com", | |||||
), | |||||
date=TimestampWithTimezone( | |||||
timestamp=Timestamp(seconds=1388590833, microseconds=0), | |||||
offset=0, | |||||
negative_utc=False, | |||||
), | |||||
id=hash_to_bytes("adcc40ee87a3ebb1b5a82edd692cf52aa5099cee"), | |||||
) | |||||
contents = swh_storage.content_get(_expected_new_contents_first_visit) | contents = swh_storage.content_get(_expected_new_contents_first_visit) | ||||
count = sum(0 if content is None else 1 for content in contents) | count = sum(0 if content is None else 1 for content in contents) | ||||
assert count == len(_expected_new_contents_first_visit) | assert count == len(_expected_new_contents_first_visit) | ||||
assert ( | assert ( | ||||
list(swh_storage.directory_missing(_expected_new_directories_first_visit)) == [] | list(swh_storage.directory_missing(_expected_new_directories_first_visit)) == [] | ||||
) | ) | ||||
▲ Show 20 Lines • Show All 47 Lines • ▼ Show 20 Lines | def test_npm_loader_first_visit(swh_storage, requests_mock_datadir, org_api_info): | ||||
} == stats | } == stats | ||||
def test_npm_loader_incremental_visit(swh_storage, requests_mock_datadir_visits): | def test_npm_loader_incremental_visit(swh_storage, requests_mock_datadir_visits): | ||||
package = "org" | package = "org" | ||||
url = package_url(package) | url = package_url(package) | ||||
loader = NpmLoader(swh_storage, url) | loader = NpmLoader(swh_storage, url) | ||||
expected_snapshot_id = hash_to_bytes("ddaad89b0b4edb7eefe7c92e9b1166caa776ebbc") | expected_snapshot_id = hash_to_bytes("d24e3f10492ade1e9462ec701370fef4a79a40f1") | ||||
actual_load_status = loader.load() | actual_load_status = loader.load() | ||||
assert actual_load_status == { | assert actual_load_status == { | ||||
"status": "eventful", | "status": "eventful", | ||||
"snapshot_id": expected_snapshot_id.hex(), | "snapshot_id": expected_snapshot_id.hex(), | ||||
} | } | ||||
assert_last_visit_matches( | assert_last_visit_matches( | ||||
swh_storage, url, status="full", type="npm", snapshot=expected_snapshot_id | swh_storage, url, status="full", type="npm", snapshot=expected_snapshot_id | ||||
) | ) | ||||
▲ Show 20 Lines • Show All 46 Lines • ▼ Show 20 Lines | |||||
@pytest.mark.usefixtures("requests_mock_datadir") | @pytest.mark.usefixtures("requests_mock_datadir") | ||||
def test_npm_loader_version_divergence(swh_storage): | def test_npm_loader_version_divergence(swh_storage): | ||||
package = "@aller_shared" | package = "@aller_shared" | ||||
url = package_url(package) | url = package_url(package) | ||||
loader = NpmLoader(swh_storage, url) | loader = NpmLoader(swh_storage, url) | ||||
actual_load_status = loader.load() | actual_load_status = loader.load() | ||||
expected_snapshot_id = hash_to_bytes("7a89bc3cb51ff1d3213b2151c745d82c3b9d69b1") | expected_snapshot_id = hash_to_bytes("92ff37da8045f0088ed35bce0bc34e2025202825") | ||||
assert actual_load_status == { | assert actual_load_status == { | ||||
"status": "eventful", | "status": "eventful", | ||||
"snapshot_id": expected_snapshot_id.hex(), | "snapshot_id": expected_snapshot_id.hex(), | ||||
} | } | ||||
assert_last_visit_matches( | assert_last_visit_matches( | ||||
swh_storage, url, status="full", type="npm", snapshot=expected_snapshot_id | swh_storage, url, status="full", type="npm", snapshot=expected_snapshot_id | ||||
) | ) | ||||
expected_snapshot = Snapshot( | expected_snapshot = Snapshot( | ||||
id=expected_snapshot_id, | id=expected_snapshot_id, | ||||
branches={ | branches={ | ||||
b"HEAD": SnapshotBranch( | b"HEAD": SnapshotBranch( | ||||
target_type=TargetType.ALIAS, target=b"releases/0.1.0" | target_type=TargetType.ALIAS, target=b"releases/0.1.0" | ||||
), | ), | ||||
b"releases/0.1.0": SnapshotBranch( | b"releases/0.1.0": SnapshotBranch( | ||||
target_type=TargetType.RELEASE, | target_type=TargetType.RELEASE, | ||||
target=hash_to_bytes("103fa6d0a1abb405468e3590dcf634bcb77f67be"), | target=hash_to_bytes("c5e0f0e185660b6bdd694ca5c68babe5bab20e24"), | ||||
), | ), | ||||
b"releases/0.1.1-alpha.14": SnapshotBranch( | b"releases/0.1.1-alpha.14": SnapshotBranch( | ||||
target_type=TargetType.RELEASE, | target_type=TargetType.RELEASE, | ||||
target=hash_to_bytes("c00b54143582a4e963e0b86e8dfa58eedd260020"), | target=hash_to_bytes("2f89c709eacc974b587e13f90d10a826b23a550e"), | ||||
), | ), | ||||
}, | }, | ||||
) | ) | ||||
check_snapshot(expected_snapshot, swh_storage) | check_snapshot(expected_snapshot, swh_storage) | ||||
stats = get_stats(swh_storage) | stats = get_stats(swh_storage) | ||||
assert { # 1 new releases artifacts | assert { # 1 new releases artifacts | ||||
▲ Show 20 Lines • Show All 62 Lines • ▼ Show 20 Lines | def test_npm_artifact_use_mtime_if_no_time(swh_storage, requests_mock_datadir): | ||||
"""With no time upload, artifact is skipped | """With no time upload, artifact is skipped | ||||
""" | """ | ||||
package = "jammit-express" | package = "jammit-express" | ||||
url = package_url(package) | url = package_url(package) | ||||
loader = NpmLoader(swh_storage, url) | loader = NpmLoader(swh_storage, url) | ||||
actual_load_status = loader.load() | actual_load_status = loader.load() | ||||
expected_snapshot_id = hash_to_bytes("7f5e591dd3c4754abca4db1cc18355671e2c014c") | expected_snapshot_id = hash_to_bytes("2a7a67725f9c7134f56612281e8d1638f1386118") | ||||
assert actual_load_status == { | assert actual_load_status == { | ||||
"status": "eventful", | "status": "eventful", | ||||
"snapshot_id": expected_snapshot_id.hex(), | "snapshot_id": expected_snapshot_id.hex(), | ||||
} | } | ||||
# artifact is used | # artifact is used | ||||
expected_snapshot = Snapshot( | expected_snapshot = Snapshot( | ||||
id=expected_snapshot_id, | id=expected_snapshot_id, | ||||
branches={ | branches={ | ||||
b"HEAD": SnapshotBranch( | b"HEAD": SnapshotBranch( | ||||
target_type=TargetType.ALIAS, target=b"releases/0.0.1" | target_type=TargetType.ALIAS, target=b"releases/0.0.1" | ||||
), | ), | ||||
b"releases/0.0.1": SnapshotBranch( | b"releases/0.0.1": SnapshotBranch( | ||||
target_type=TargetType.RELEASE, | target_type=TargetType.RELEASE, | ||||
target=hash_to_bytes("199bf0ad020617357d608655e6549e526a65dc36"), | target=hash_to_bytes("68b2a100103cecec06b8dd780228bb751f2dc6f3"), | ||||
), | ), | ||||
}, | }, | ||||
) | ) | ||||
assert_last_visit_matches( | assert_last_visit_matches( | ||||
swh_storage, url, status="full", type="npm", snapshot=expected_snapshot.id | swh_storage, url, status="full", type="npm", snapshot=expected_snapshot.id | ||||
) | ) | ||||
Show All 27 Lines |