Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/tests/storage_tests.py
| Show First 20 Lines • Show All 3,765 Lines • ▼ Show 20 Lines | def test_content_metadata_add(self, swh_storage, sample_data): | ||||
| ("metadata_authority", authority), | ("metadata_authority", authority), | ||||
| ("metadata_fetcher", fetcher), | ("metadata_fetcher", fetcher), | ||||
| ] + [("raw_extrinsic_metadata", item) for item in content_metadata] | ] + [("raw_extrinsic_metadata", item) for item in content_metadata] | ||||
| for obj in expected_objects: | for obj in expected_objects: | ||||
| assert obj in actual_objects | assert obj in actual_objects | ||||
| def test_content_metadata_add_duplicate(self, swh_storage, sample_data): | def test_content_metadata_add_duplicate(self, swh_storage, sample_data): | ||||
| """Duplicates should be silently updated.""" | """Duplicates should be silently ignored.""" | ||||
| content = sample_data.content | content = sample_data.content | ||||
| fetcher = sample_data.metadata_fetcher | fetcher = sample_data.metadata_fetcher | ||||
| authority = sample_data.metadata_authority | authority = sample_data.metadata_authority | ||||
| content_metadata, content_metadata2 = sample_data.content_metadata[:2] | content_metadata, content_metadata2 = sample_data.content_metadata[:2] | ||||
| new_content_metadata2 = RawExtrinsicMetadata.from_dict( | |||||
| { | |||||
| **remove_keys(content_metadata2.to_dict(), ("id",)), # recompute id | |||||
| "format": "new-format", | |||||
| "metadata": b"new-metadata", | |||||
| } | |||||
| ) | |||||
| swh_storage.metadata_fetcher_add([fetcher]) | swh_storage.metadata_fetcher_add([fetcher]) | ||||
| swh_storage.metadata_authority_add([authority]) | swh_storage.metadata_authority_add([authority]) | ||||
| swh_storage.raw_extrinsic_metadata_add([content_metadata, content_metadata2]) | swh_storage.raw_extrinsic_metadata_add([content_metadata, content_metadata2]) | ||||
| swh_storage.raw_extrinsic_metadata_add([new_content_metadata2]) | swh_storage.raw_extrinsic_metadata_add([content_metadata2, content_metadata]) | ||||
| result = swh_storage.raw_extrinsic_metadata_get( | result = swh_storage.raw_extrinsic_metadata_get( | ||||
| content.swhid().to_extended(), authority | content.swhid().to_extended(), authority | ||||
| ) | ) | ||||
| assert result.next_page_token is None | assert result.next_page_token is None | ||||
| expected_results1 = (content_metadata, new_content_metadata2) | expected_results = (content_metadata, content_metadata2) | ||||
| expected_results2 = (content_metadata, content_metadata2) | |||||
| assert tuple(sorted(result.results, key=lambda x: x.discovery_date,)) in ( | assert ( | ||||
| expected_results1, # cassandra | tuple(sorted(result.results, key=lambda x: x.discovery_date,)) | ||||
| expected_results2, # postgresql | == expected_results | ||||
| ) | ) | ||||
| def test_content_metadata_get(self, swh_storage, sample_data): | def test_content_metadata_get(self, swh_storage, sample_data): | ||||
| content, content2 = sample_data.contents[:2] | content, content2 = sample_data.contents[:2] | ||||
| fetcher, fetcher2 = sample_data.fetchers[:2] | fetcher, fetcher2 = sample_data.fetchers[:2] | ||||
| authority, authority2 = sample_data.authorities[:2] | authority, authority2 = sample_data.authorities[:2] | ||||
| ( | ( | ||||
| content1_metadata1, | content1_metadata1, | ||||
| ▲ Show 20 Lines • Show All 178 Lines • ▼ Show 20 Lines | |||||
| def test_origin_metadata_add_duplicate(self, swh_storage, sample_data): | def test_origin_metadata_add_duplicate(self, swh_storage, sample_data): | ||||
| """Duplicates should be silently updated.""" | """Duplicates should be silently updated.""" | ||||
| origin = sample_data.origin | origin = sample_data.origin | ||||
| fetcher = sample_data.metadata_fetcher | fetcher = sample_data.metadata_fetcher | ||||
| authority = sample_data.metadata_authority | authority = sample_data.metadata_authority | ||||
| origin_metadata, origin_metadata2 = sample_data.origin_metadata[:2] | origin_metadata, origin_metadata2 = sample_data.origin_metadata[:2] | ||||
| assert swh_storage.origin_add([origin]) == {"origin:add": 1} | assert swh_storage.origin_add([origin]) == {"origin:add": 1} | ||||
| new_origin_metadata2 = RawExtrinsicMetadata.from_dict( | |||||
| { | |||||
| **remove_keys(origin_metadata2.to_dict(), ("id",)), # recompute id | |||||
| "format": "new-format", | |||||
| "metadata": b"new-metadata", | |||||
| } | |||||
| ) | |||||
| swh_storage.metadata_fetcher_add([fetcher]) | swh_storage.metadata_fetcher_add([fetcher]) | ||||
| swh_storage.metadata_authority_add([authority]) | swh_storage.metadata_authority_add([authority]) | ||||
| swh_storage.raw_extrinsic_metadata_add([origin_metadata, origin_metadata2]) | swh_storage.raw_extrinsic_metadata_add([origin_metadata, origin_metadata2]) | ||||
| swh_storage.raw_extrinsic_metadata_add([new_origin_metadata2]) | swh_storage.raw_extrinsic_metadata_add([origin_metadata2, origin_metadata]) | ||||
| result = swh_storage.raw_extrinsic_metadata_get( | result = swh_storage.raw_extrinsic_metadata_get( | ||||
| Origin(origin.url).swhid(), authority | Origin(origin.url).swhid(), authority | ||||
| ) | ) | ||||
| assert result.next_page_token is None | assert result.next_page_token is None | ||||
| # which of the two behavior happens is backend-specific. | # which of the two behavior happens is backend-specific. | ||||
| expected_results1 = (origin_metadata, new_origin_metadata2) | expected_results = (origin_metadata, origin_metadata2) | ||||
| expected_results2 = (origin_metadata, origin_metadata2) | |||||
| assert tuple(sorted(result.results, key=lambda x: x.discovery_date,)) in ( | assert ( | ||||
| expected_results1, # cassandra | tuple(sorted(result.results, key=lambda x: x.discovery_date,)) | ||||
| expected_results2, # postgresql | == expected_results | ||||
| ) | ) | ||||
| def test_origin_metadata_get(self, swh_storage, sample_data): | def test_origin_metadata_get(self, swh_storage, sample_data): | ||||
| origin, origin2 = sample_data.origins[:2] | origin, origin2 = sample_data.origins[:2] | ||||
| fetcher, fetcher2 = sample_data.fetchers[:2] | fetcher, fetcher2 = sample_data.fetchers[:2] | ||||
| authority, authority2 = sample_data.authorities[:2] | authority, authority2 = sample_data.authorities[:2] | ||||
| ( | ( | ||||
| origin1_metadata1, | origin1_metadata1, | ||||
| ▲ Show 20 Lines • Show All 307 Lines • Show Last 20 Lines | |||||