swh_config = '/tmp/pytest-of-jenkins/pytest-0/test_incremental_visit0/loader.yml' requests_mock_datadir_visits = <requests_mock.mocker.Mocker object at 0x7f4435d637b8> def test_incremental_visit(swh_config, requests_mock_datadir_visits): """With prior visit, 2nd load will result with a different snapshot """ url = "https://pypi.org/project/0805nexter" loader = PyPILoader(url) visit1_actual_load_status = loader.load() visit1_stats = get_stats(loader.storage) expected_snapshot_id = "ba6e158ada75d0b3cfb209ffdf6daa4ed34a227a" assert visit1_actual_load_status == { "status": "eventful", "snapshot_id": expected_snapshot_id, } assert_last_visit_matches(loader.storage, url, status="full", type="pypi") assert { "content": 6, "directory": 4, "origin": 1, "origin_visit": 1, "person": 1, "release": 0, "revision": 2, "skipped_content": 0, "snapshot": 1, } == visit1_stats # Reset internal state loader._info = None visit2_actual_load_status = loader.load() visit2_stats = get_stats(loader.storage) assert visit2_actual_load_status["status"] == "eventful" expected_snapshot_id2 = "2e5149a7b0725d18231a37b342e9b7c4e121f283" assert visit2_actual_load_status == { "status": "eventful", "snapshot_id": expected_snapshot_id2, } assert_last_visit_matches(loader.storage, url, status="full", type="pypi") assert { "content": 6 + 1, # 1 more content "directory": 4 + 2, # 2 more directories "origin": 1, "origin_visit": 1 + 1, "person": 1, "release": 0, "revision": 2 + 1, # 1 more revision "skipped_content": 0, "snapshot": 1 + 1, # 1 more snapshot } == visit2_stats expected_contents = map( hash_to_bytes, [ "a61e24cdfdab3bb7817f6be85d37a3e666b34566", "938c33483285fd8ad57f15497f538320df82aeb8", "a27576d60e08c94a05006d2e6d540c0fdb5f38c8", "405859113963cb7a797642b45f171d6360425d16", "e5686aa568fdb1d19d7f1329267082fe40482d31", "83ecf6ec1114fd260ca7a833a2d165e71258c338", "92689fa2b7fb4d4fc6fb195bf73a50c87c030639", ], ) assert list(loader.storage.content_missing_per_sha1(expected_contents)) == [] expected_dirs = map( hash_to_bytes, [ "05219ba38bc542d4345d5638af1ed56c7d43ca7d", "cf019eb456cf6f78d8c4674596f1c9a97ece8f44", "b178b66bd22383d5f16f4f5c923d39ca798861b4", "c3a58f8b57433a4b56caaa5033ae2e0931405338", "e226e7e4ad03b4fc1403d69a18ebdd6f2edd2b3a", "52604d46843b898f5a43208045d09fcf8731631b", ], ) assert list(loader.storage.directory_missing(expected_dirs)) == [] # {revision hash: directory hash} expected_revs = { hash_to_bytes("4c99891f93b81450385777235a37b5e966dd1571"): hash_to_bytes( "05219ba38bc542d4345d5638af1ed56c7d43ca7d" ), # noqa hash_to_bytes("e445da4da22b31bfebb6ffc4383dbf839a074d21"): hash_to_bytes( "b178b66bd22383d5f16f4f5c923d39ca798861b4" ), # noqa hash_to_bytes("51247143b01445c9348afa9edfae31bf7c5d86b1"): hash_to_bytes( "e226e7e4ad03b4fc1403d69a18ebdd6f2edd2b3a" ), # noqa } assert list(loader.storage.revision_missing(expected_revs)) == [] expected_branches = { "releases/1.1.0": { "target": "4c99891f93b81450385777235a37b5e966dd1571", "target_type": "revision", }, "releases/1.2.0": { "target": "e445da4da22b31bfebb6ffc4383dbf839a074d21", "target_type": "revision", }, "releases/1.3.0": { "target": "51247143b01445c9348afa9edfae31bf7c5d86b1", "target_type": "revision", }, "HEAD": {"target": "releases/1.3.0", "target_type": "alias",}, } expected_snapshot = { "id": expected_snapshot_id2, "branches": expected_branches, } check_snapshot(expected_snapshot, loader.storage) assert_last_visit_matches(loader.storage, url, status="full", type="pypi") urls = [ m.url for m in requests_mock_datadir_visits.request_history if m.url.startswith("https://files.pythonhosted.org") ] # visited each artifact once across 2 visits > assert len(urls) == len(set(urls)) E assert 5 == 3 E +5 E -3 .tox/py3/lib/python3.7/site-packages/swh/loader/package/pypi/tests/test_pypi.py:674: AssertionError