Changeset View
Changeset View
Standalone View
Standalone View
swh/indexer/tests/storage/test_storage.py
| Show First 20 Lines • Show All 1,145 Lines • ▼ Show 20 Lines | class TestIndexerStorageOriginIntrinsicMetadata: | ||||
| def test_origin_intrinsic_metadata_add__deadlock( | def test_origin_intrinsic_metadata_add__deadlock( | ||||
| self, swh_indexer_storage_with_data: Tuple[IndexerStorageInterface, Any] | self, swh_indexer_storage_with_data: Tuple[IndexerStorageInterface, Any] | ||||
| ) -> None: | ) -> None: | ||||
| storage, data = swh_indexer_storage_with_data | storage, data = swh_indexer_storage_with_data | ||||
| # given | # given | ||||
| tool_id = data.tools["swh-metadata-detector"]["id"] | tool_id = data.tools["swh-metadata-detector"]["id"] | ||||
| ids = list(range(10)) | origins = ["file:///tmp/origin{:02d}".format(i) for i in range(100)] | ||||
| example_data1: Dict[str, Any] = { | example_data1: Dict[str, Any] = { | ||||
| "metadata": {"version": None, "name": None,}, | "metadata": {"version": None, "name": None,}, | ||||
| "mappings": [], | "mappings": [], | ||||
| } | } | ||||
| example_data2: Dict[str, Any] = { | example_data2: Dict[str, Any] = { | ||||
| "metadata": {"version": "v1.1.1", "name": "foo",}, | "metadata": {"version": "v1.1.1", "name": "foo",}, | ||||
| "mappings": [], | "mappings": [], | ||||
| } | } | ||||
| metadata_rev_v1 = RevisionIntrinsicMetadataRow( | metadata_rev_v1 = RevisionIntrinsicMetadataRow( | ||||
| id=data.revision_id_2, | id=data.revision_id_2, | ||||
| metadata={"version": None, "name": None,}, | metadata={"version": None, "name": None,}, | ||||
| mappings=[], | mappings=[], | ||||
| indexer_configuration_id=tool_id, | indexer_configuration_id=tool_id, | ||||
| ) | ) | ||||
| data_v1 = [ | data_v1 = [ | ||||
| OriginIntrinsicMetadataRow( | OriginIntrinsicMetadataRow( | ||||
| id="file:///tmp/origin%d" % id_, | id=origin, | ||||
| from_revision=data.revision_id_2, | from_revision=data.revision_id_2, | ||||
| indexer_configuration_id=tool_id, | indexer_configuration_id=tool_id, | ||||
| **example_data1, | **example_data1, | ||||
| ) | ) | ||||
| for id_ in ids | for origin in origins | ||||
| ] | ] | ||||
| data_v2 = [ | data_v2 = [ | ||||
| OriginIntrinsicMetadataRow( | OriginIntrinsicMetadataRow( | ||||
| id="file:///tmp/origin%d" % id_, | id=origin, | ||||
| from_revision=data.revision_id_2, | from_revision=data.revision_id_2, | ||||
| indexer_configuration_id=tool_id, | indexer_configuration_id=tool_id, | ||||
| **example_data2, | **example_data2, | ||||
| ) | ) | ||||
| for id_ in ids | for origin in origins | ||||
| ] | ] | ||||
| # Remove one item from each, so that both queries have to succeed for | # Remove one item from each, so that both queries have to succeed for | ||||
| # all items to be in the DB. | # all items to be in the DB. | ||||
| data_v2a = data_v2[1:] | data_v2a = data_v2[1:] | ||||
| data_v2b = list(reversed(data_v2[0:-1])) | data_v2b = list(reversed(data_v2[0:-1])) | ||||
| # given | # given | ||||
| storage.revision_intrinsic_metadata_add([metadata_rev_v1]) | storage.revision_intrinsic_metadata_add([metadata_rev_v1]) | ||||
| storage.origin_intrinsic_metadata_add(data_v1) | storage.origin_intrinsic_metadata_add(data_v1) | ||||
| # when | # when | ||||
| origins = ["file:///tmp/origin%d" % i for i in ids] | |||||
| actual_data = list(storage.origin_intrinsic_metadata_get(origins)) | actual_data = list(storage.origin_intrinsic_metadata_get(origins)) | ||||
| expected_data_v1 = [ | expected_data_v1 = [ | ||||
| OriginIntrinsicMetadataRow( | OriginIntrinsicMetadataRow( | ||||
| id="file:///tmp/origin%d" % id_, | id=origin, | ||||
| from_revision=data.revision_id_2, | from_revision=data.revision_id_2, | ||||
| tool=data.tools["swh-metadata-detector"], | tool=data.tools["swh-metadata-detector"], | ||||
| **example_data1, | **example_data1, | ||||
| ) | ) | ||||
| for id_ in ids | for origin in origins | ||||
| ] | ] | ||||
| # then | # then | ||||
| assert actual_data == expected_data_v1 | assert actual_data == expected_data_v1 | ||||
| # given | # given | ||||
| def f1() -> None: | def f1() -> None: | ||||
| storage.origin_intrinsic_metadata_add(data_v2a) | storage.origin_intrinsic_metadata_add(data_v2a) | ||||
| def f2() -> None: | def f2() -> None: | ||||
| storage.origin_intrinsic_metadata_add(data_v2b) | storage.origin_intrinsic_metadata_add(data_v2b) | ||||
| t1 = threading.Thread(target=f1) | t1 = threading.Thread(target=f1) | ||||
| t2 = threading.Thread(target=f2) | t2 = threading.Thread(target=f2) | ||||
| t2.start() | t2.start() | ||||
| t1.start() | t1.start() | ||||
| t1.join() | t1.join() | ||||
| t2.join() | t2.join() | ||||
| actual_data = list(storage.origin_intrinsic_metadata_get(origins)) | actual_data = list(storage.origin_intrinsic_metadata_get(origins)) | ||||
| expected_data_v2 = [ | expected_data_v2 = [ | ||||
| OriginIntrinsicMetadataRow( | OriginIntrinsicMetadataRow( | ||||
| id="file:///tmp/origin%d" % id_, | id=origin, | ||||
| from_revision=data.revision_id_2, | from_revision=data.revision_id_2, | ||||
| tool=data.tools["swh-metadata-detector"], | tool=data.tools["swh-metadata-detector"], | ||||
| **example_data2, | **example_data2, | ||||
| ) | ) | ||||
| for id_ in ids | for origin in origins | ||||
| ] | ] | ||||
| actual_data.sort(key=lambda item: item.id) | actual_data.sort(key=lambda item: item.id) | ||||
| assert len(actual_data) == len(expected_data_v1) == len(expected_data_v2) | assert len(actual_data) == len(expected_data_v1) == len(expected_data_v2) | ||||
| for (item, expected_item_v1, expected_item_v2) in zip( | for (item, expected_item_v1, expected_item_v2) in zip( | ||||
| actual_data, expected_data_v1, expected_data_v2 | actual_data, expected_data_v1, expected_data_v2 | ||||
| ): | ): | ||||
| assert item in (expected_item_v1, expected_item_v2) | assert item in (expected_item_v1, expected_item_v2) | ||||
| ▲ Show 20 Lines • Show All 457 Lines • Show Last 20 Lines | |||||