Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/package/deposit/tests/test_deposit.py
| Show First 20 Lines • Show All 43 Lines • ▼ Show 20 Lines | def requests_mock_datadir(requests_mock_datadir): | ||||
| requests_mock_datadir.put(re.compile("https")) | requests_mock_datadir.put(re.compile("https")) | ||||
| return requests_mock_datadir | return requests_mock_datadir | ||||
| def test_deposit_init_ok(swh_storage, deposit_client, swh_loader_config): | def test_deposit_init_ok(swh_storage, deposit_client, swh_loader_config): | ||||
| url = "some-url" | url = "some-url" | ||||
| deposit_id = 999 | deposit_id = 999 | ||||
| loader = DepositLoader( | loader = DepositLoader( | ||||
| swh_storage, url, deposit_id, deposit_client | swh_storage, url, deposit_id, deposit_client, default_filename="archive.zip" | ||||
| ) # Something that does not exist | ) # Something that does not exist | ||||
| assert loader.url == url | assert loader.url == url | ||||
| assert loader.client is not None | assert loader.client is not None | ||||
| assert loader.client.base_url == swh_loader_config["deposit"]["url"] | assert loader.client.base_url == swh_loader_config["deposit"]["url"] | ||||
| def test_deposit_from_configfile(swh_config): | def test_deposit_from_configfile(swh_config): | ||||
| """Ensure the deposit instantiation is ok | """Ensure the deposit instantiation is ok | ||||
| """ | """ | ||||
| loader = DepositLoader.from_configfile(url="some-url", deposit_id="666") | loader = DepositLoader.from_configfile( | ||||
| url="some-url", deposit_id="666", default_filename="archive.zip" | |||||
| ) | |||||
| assert isinstance(loader.client, ApiClient) | assert isinstance(loader.client, ApiClient) | ||||
| def test_deposit_loading_unknown_deposit( | def test_deposit_loading_unknown_deposit( | ||||
| swh_storage, deposit_client, requests_mock_datadir | swh_storage, deposit_client, requests_mock_datadir | ||||
| ): | ): | ||||
| """Loading an unknown deposit should fail | """Loading an unknown deposit should fail | ||||
| no origin, no visit, no snapshot | no origin, no visit, no snapshot | ||||
| """ | """ | ||||
| # private api url form: 'https://deposit.s.o/1/private/hal/666/raw/' | # private api url form: 'https://deposit.s.o/1/private/hal/666/raw/' | ||||
| url = "some-url" | url = "some-url" | ||||
| unknown_deposit_id = 667 | unknown_deposit_id = 667 | ||||
| loader = DepositLoader( | loader = DepositLoader( | ||||
| swh_storage, url, unknown_deposit_id, deposit_client | swh_storage, | ||||
| url, | |||||
| unknown_deposit_id, | |||||
| deposit_client, | |||||
| default_filename="archive.zip", | |||||
| ) # does not exist | ) # does not exist | ||||
| actual_load_status = loader.load() | actual_load_status = loader.load() | ||||
| assert actual_load_status == {"status": "failed"} | assert actual_load_status == {"status": "failed"} | ||||
| stats = get_stats(loader.storage) | stats = get_stats(loader.storage) | ||||
| assert { | assert { | ||||
| Show All 18 Lines | |||||
| ): | ): | ||||
| """Deposit with missing artifact ends up with an uneventful/partial visit | """Deposit with missing artifact ends up with an uneventful/partial visit | ||||
| """ | """ | ||||
| # private api url form: 'https://deposit.s.o/1/private/hal/666/raw/' | # private api url form: 'https://deposit.s.o/1/private/hal/666/raw/' | ||||
| url = "some-url-2" | url = "some-url-2" | ||||
| deposit_id = 666 | deposit_id = 666 | ||||
| requests_mock_datadir_missing_one.put(re.compile("https")) | requests_mock_datadir_missing_one.put(re.compile("https")) | ||||
| loader = DepositLoader(swh_storage, url, deposit_id, deposit_client) | loader = DepositLoader( | ||||
| swh_storage, url, deposit_id, deposit_client, default_filename="archive.zip" | |||||
| ) | |||||
| actual_load_status = loader.load() | actual_load_status = loader.load() | ||||
| assert actual_load_status["status"] == "uneventful" | assert actual_load_status["status"] == "uneventful" | ||||
| assert actual_load_status["snapshot_id"] is not None | assert actual_load_status["snapshot_id"] is not None | ||||
| assert_last_visit_matches(loader.storage, url, status="partial", type="deposit") | assert_last_visit_matches(loader.storage, url, status="partial", type="deposit") | ||||
| stats = get_stats(loader.storage) | stats = get_stats(loader.storage) | ||||
| Show All 26 Lines | ): | ||||
| assert body == expected_body | assert body == expected_body | ||||
| def test_deposit_revision_metadata_structure( | def test_deposit_revision_metadata_structure( | ||||
| swh_storage, deposit_client, requests_mock_datadir | swh_storage, deposit_client, requests_mock_datadir | ||||
| ): | ): | ||||
| url = "https://hal-test.archives-ouvertes.fr/some-external-id" | url = "https://hal-test.archives-ouvertes.fr/some-external-id" | ||||
| deposit_id = 666 | deposit_id = 666 | ||||
| loader = DepositLoader(swh_storage, url, deposit_id, deposit_client) | loader = DepositLoader( | ||||
| swh_storage, url, deposit_id, deposit_client, default_filename="archive.zip" | |||||
| ) | |||||
| actual_load_status = loader.load() | actual_load_status = loader.load() | ||||
| assert actual_load_status["status"] == "eventful" | assert actual_load_status["status"] == "eventful" | ||||
| assert actual_load_status["snapshot_id"] is not None | assert actual_load_status["snapshot_id"] is not None | ||||
| expected_revision_id = hash_to_bytes("637318680351f5d78856d13264faebbd91efe9bb") | expected_revision_id = hash_to_bytes("637318680351f5d78856d13264faebbd91efe9bb") | ||||
| revision = loader.storage.revision_get([expected_revision_id])[0] | revision = loader.storage.revision_get([expected_revision_id])[0] | ||||
| assert revision is not None | assert revision is not None | ||||
| Show All 15 Lines | for original_artifact in revision.metadata["original_artifact"]: | ||||
| original_artifact, | original_artifact, | ||||
| paths=[("filename", str), ("length", int), ("checksums", dict),], | paths=[("filename", str), ("length", int), ("checksums", dict),], | ||||
| ) | ) | ||||
| def test_deposit_loading_ok(swh_storage, deposit_client, requests_mock_datadir): | def test_deposit_loading_ok(swh_storage, deposit_client, requests_mock_datadir): | ||||
| url = "https://hal-test.archives-ouvertes.fr/some-external-id" | url = "https://hal-test.archives-ouvertes.fr/some-external-id" | ||||
| deposit_id = 666 | deposit_id = 666 | ||||
| loader = DepositLoader(swh_storage, url, deposit_id, deposit_client) | loader = DepositLoader( | ||||
| swh_storage, url, deposit_id, deposit_client, default_filename="archive.zip" | |||||
| ) | |||||
| actual_load_status = loader.load() | actual_load_status = loader.load() | ||||
| expected_snapshot_id = "b2b327b33dc85818bd23c3ccda8b7e675a66ecbd" | expected_snapshot_id = "b2b327b33dc85818bd23c3ccda8b7e675a66ecbd" | ||||
| assert actual_load_status == { | assert actual_load_status == { | ||||
| "status": "eventful", | "status": "eventful", | ||||
| "snapshot_id": expected_snapshot_id, | "snapshot_id": expected_snapshot_id, | ||||
| } | } | ||||
| ▲ Show 20 Lines • Show All 87 Lines • ▼ Show 20 Lines | |||||
| def test_deposit_loading_ok_2(swh_storage, deposit_client, requests_mock_datadir): | def test_deposit_loading_ok_2(swh_storage, deposit_client, requests_mock_datadir): | ||||
| """Field dates should be se appropriately | """Field dates should be se appropriately | ||||
| """ | """ | ||||
| external_id = "some-external-id" | external_id = "some-external-id" | ||||
| url = f"https://hal-test.archives-ouvertes.fr/{external_id}" | url = f"https://hal-test.archives-ouvertes.fr/{external_id}" | ||||
| deposit_id = 777 | deposit_id = 777 | ||||
| loader = DepositLoader(swh_storage, url, deposit_id, deposit_client) | loader = DepositLoader( | ||||
| swh_storage, url, deposit_id, deposit_client, default_filename="archive.zip" | |||||
| ) | |||||
| actual_load_status = loader.load() | actual_load_status = loader.load() | ||||
| expected_snapshot_id = "3e68440fdd7c81d283f8f3aebb6f0c8657864192" | expected_snapshot_id = "3e68440fdd7c81d283f8f3aebb6f0c8657864192" | ||||
| assert actual_load_status == { | assert actual_load_status == { | ||||
| "status": "eventful", | "status": "eventful", | ||||
| "snapshot_id": expected_snapshot_id, | "snapshot_id": expected_snapshot_id, | ||||
| } | } | ||||
| ▲ Show 20 Lines • Show All 182 Lines • ▼ Show 20 Lines | def test_deposit_loading_ok_3(swh_storage, deposit_client, requests_mock_datadir): | ||||
| """Deposit loading can happen on tarball artifacts as well | """Deposit loading can happen on tarball artifacts as well | ||||
| The latest deposit changes introduce the internal change. | The latest deposit changes introduce the internal change. | ||||
| """ | """ | ||||
| external_id = "hal-123456" | external_id = "hal-123456" | ||||
| url = f"https://hal-test.archives-ouvertes.fr/{external_id}" | url = f"https://hal-test.archives-ouvertes.fr/{external_id}" | ||||
| deposit_id = 888 | deposit_id = 888 | ||||
| loader = DepositLoader( | loader = DepositLoader(swh_storage, url, deposit_id, deposit_client) | ||||
| swh_storage, url, deposit_id, deposit_client, default_filename="archive.tar" | |||||
| ) | |||||
| actual_load_status = loader.load() | actual_load_status = loader.load() | ||||
| expected_snapshot_id = "0ac7b54c042a026389f2087dc16f1d5c644ed0e4" | expected_snapshot_id = "0ac7b54c042a026389f2087dc16f1d5c644ed0e4" | ||||
| assert actual_load_status == { | assert actual_load_status == { | ||||
| "status": "eventful", | "status": "eventful", | ||||
| "snapshot_id": expected_snapshot_id, | "snapshot_id": expected_snapshot_id, | ||||
| } | } | ||||
| assert_last_visit_matches(loader.storage, url, status="full", type="deposit") | assert_last_visit_matches(loader.storage, url, status="full", type="deposit") | ||||