Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/tests/storage_tests.py
| Show All 32 Lines | from swh.model.model import ( | ||||
| RawExtrinsicMetadata, | RawExtrinsicMetadata, | ||||
| Revision, | Revision, | ||||
| SkippedContent, | SkippedContent, | ||||
| Snapshot, | Snapshot, | ||||
| SnapshotBranch, | SnapshotBranch, | ||||
| TargetType, | TargetType, | ||||
| ) | ) | ||||
| from swh.storage import get_storage | from swh.storage import get_storage | ||||
| from swh.storage.cassandra.storage import CassandraStorage | |||||
| from swh.storage.common import origin_url_to_sha1 as sha1 | from swh.storage.common import origin_url_to_sha1 as sha1 | ||||
| from swh.storage.exc import HashCollision, StorageArgumentException | from swh.storage.exc import HashCollision, StorageArgumentException | ||||
| from swh.storage.in_memory import InMemoryStorage | |||||
| from swh.storage.interface import ListOrder, PagedResult, StorageInterface | from swh.storage.interface import ListOrder, PagedResult, StorageInterface | ||||
| from swh.storage.tests.conftest import function_scoped_fixture_check | from swh.storage.tests.conftest import function_scoped_fixture_check | ||||
| from swh.storage.utils import ( | from swh.storage.utils import ( | ||||
| content_hex_hashes, | content_hex_hashes, | ||||
| now, | now, | ||||
| remove_keys, | remove_keys, | ||||
| round_to_milliseconds, | round_to_milliseconds, | ||||
| ) | ) | ||||
| ▲ Show 20 Lines • Show All 131 Lines • ▼ Show 20 Lines | def test_content_add(self, swh_storage, sample_data): | ||||
| if obj_type == "content" | if obj_type == "content" | ||||
| ] | ] | ||||
| assert len(contents) == 1 | assert len(contents) == 1 | ||||
| for obj in contents: | for obj in contents: | ||||
| assert insertion_start_time <= obj.ctime | assert insertion_start_time <= obj.ctime | ||||
| assert obj.ctime <= insertion_end_time | assert obj.ctime <= insertion_end_time | ||||
| assert obj == expected_cont | assert obj == expected_cont | ||||
| if isinstance(swh_storage, InMemoryStorage) or not isinstance( | |||||
| swh_storage, CassandraStorage | |||||
| ): | |||||
| swh_storage.refresh_stat_counters() | swh_storage.refresh_stat_counters() | ||||
| assert swh_storage.stat_counters()["content"] == 1 | assert swh_storage.stat_counters()["content"] == 1 | ||||
| def test_content_add_from_lazy_content(self, swh_storage, sample_data): | def test_content_add_from_lazy_content(self, swh_storage, sample_data): | ||||
| cont = sample_data.content | cont = sample_data.content | ||||
| lazy_content = LazyContent.from_dict(cont.to_dict()) | lazy_content = LazyContent.from_dict(cont.to_dict()) | ||||
| insertion_start_time = now() | insertion_start_time = now() | ||||
| actual_result = swh_storage.content_add([lazy_content]) | actual_result = swh_storage.content_add([lazy_content]) | ||||
| Show All 16 Lines | def test_content_add_from_lazy_content(self, swh_storage, sample_data): | ||||
| if obj_type == "content" | if obj_type == "content" | ||||
| ] | ] | ||||
| assert len(contents) == 1 | assert len(contents) == 1 | ||||
| for obj in contents: | for obj in contents: | ||||
| assert insertion_start_time <= obj.ctime | assert insertion_start_time <= obj.ctime | ||||
| assert obj.ctime <= insertion_end_time | assert obj.ctime <= insertion_end_time | ||||
| assert attr.evolve(obj, ctime=None).to_dict() == expected_cont.to_dict() | assert attr.evolve(obj, ctime=None).to_dict() == expected_cont.to_dict() | ||||
| if isinstance(swh_storage, InMemoryStorage) or not isinstance( | |||||
| swh_storage, CassandraStorage | |||||
| ): | |||||
| swh_storage.refresh_stat_counters() | swh_storage.refresh_stat_counters() | ||||
| assert swh_storage.stat_counters()["content"] == 1 | assert swh_storage.stat_counters()["content"] == 1 | ||||
| def test_content_get_data_missing(self, swh_storage, sample_data): | def test_content_get_data_missing(self, swh_storage, sample_data): | ||||
| cont, cont2 = sample_data.contents[:2] | cont, cont2 = sample_data.contents[:2] | ||||
| swh_storage.content_add([cont]) | swh_storage.content_add([cont]) | ||||
| # Query a single missing content | # Query a single missing content | ||||
| actual_content_data = swh_storage.content_get_data(cont2.sha1) | actual_content_data = swh_storage.content_get_data(cont2.sha1) | ||||
| ▲ Show 20 Lines • Show All 466 Lines • ▼ Show 20 Lines | def test_directory_add(self, swh_storage, sample_data): | ||||
| expected_data = list(transform_entries(swh_storage, directory)) | expected_data = list(transform_entries(swh_storage, directory)) | ||||
| for data in actual_data: | for data in actual_data: | ||||
| assert data in expected_data | assert data in expected_data | ||||
| after_missing = list(swh_storage.directory_missing([directory.id])) | after_missing = list(swh_storage.directory_missing([directory.id])) | ||||
| assert after_missing == [] | assert after_missing == [] | ||||
| if isinstance(swh_storage, InMemoryStorage) or not isinstance( | |||||
| swh_storage, CassandraStorage | |||||
| ): | |||||
| swh_storage.refresh_stat_counters() | swh_storage.refresh_stat_counters() | ||||
| assert swh_storage.stat_counters()["directory"] == 1 | assert swh_storage.stat_counters()["directory"] == 1 | ||||
| def test_directory_add_twice(self, swh_storage, sample_data): | def test_directory_add_twice(self, swh_storage, sample_data): | ||||
| directory = sample_data.directories[1] | directory = sample_data.directories[1] | ||||
| actual_result = swh_storage.directory_add([directory]) | actual_result = swh_storage.directory_add([directory]) | ||||
| assert actual_result == {"directory:add": 1} | assert actual_result == {"directory:add": 1} | ||||
| assert list(swh_storage.journal_writer.journal.objects) == [ | assert list(swh_storage.journal_writer.journal.objects) == [ | ||||
| ▲ Show 20 Lines • Show All 252 Lines • ▼ Show 20 Lines | def test_revision_add(self, swh_storage, sample_data): | ||||
| assert list(swh_storage.journal_writer.journal.objects) == [ | assert list(swh_storage.journal_writer.journal.objects) == [ | ||||
| ("revision", revision) | ("revision", revision) | ||||
| ] | ] | ||||
| # already there so nothing added | # already there so nothing added | ||||
| actual_result = swh_storage.revision_add([revision]) | actual_result = swh_storage.revision_add([revision]) | ||||
| assert actual_result == {"revision:add": 0} | assert actual_result == {"revision:add": 0} | ||||
| if isinstance(swh_storage, InMemoryStorage) or not isinstance( | |||||
| swh_storage, CassandraStorage | |||||
| ): | |||||
| swh_storage.refresh_stat_counters() | swh_storage.refresh_stat_counters() | ||||
| assert swh_storage.stat_counters()["revision"] == 1 | assert swh_storage.stat_counters()["revision"] == 1 | ||||
| def test_revision_add_twice(self, swh_storage, sample_data): | def test_revision_add_twice(self, swh_storage, sample_data): | ||||
| revision, revision2 = sample_data.revisions[:2] | revision, revision2 = sample_data.revisions[:2] | ||||
| actual_result = swh_storage.revision_add([revision]) | actual_result = swh_storage.revision_add([revision]) | ||||
| assert actual_result == {"revision:add": 1} | assert actual_result == {"revision:add": 1} | ||||
| assert list(swh_storage.journal_writer.journal.objects) == [ | assert list(swh_storage.journal_writer.journal.objects) == [ | ||||
| ▲ Show 20 Lines • Show All 383 Lines • ▼ Show 20 Lines | def test_release_add(self, swh_storage, sample_data): | ||||
| ("release", release), | ("release", release), | ||||
| ("release", release2), | ("release", release2), | ||||
| ] | ] | ||||
| # already present so nothing added | # already present so nothing added | ||||
| actual_result = swh_storage.release_add([release, release2]) | actual_result = swh_storage.release_add([release, release2]) | ||||
| assert actual_result == {"release:add": 0} | assert actual_result == {"release:add": 0} | ||||
| if isinstance(swh_storage, InMemoryStorage) or not isinstance( | |||||
| swh_storage, CassandraStorage | |||||
| ): | |||||
| swh_storage.refresh_stat_counters() | swh_storage.refresh_stat_counters() | ||||
| assert swh_storage.stat_counters()["release"] == 2 | assert swh_storage.stat_counters()["release"] == 2 | ||||
| def test_release_add_no_author_date(self, swh_storage, sample_data): | def test_release_add_no_author_date(self, swh_storage, sample_data): | ||||
| full_release = sample_data.release | full_release = sample_data.release | ||||
| release = attr.evolve(full_release, author=None, date=None) | release = attr.evolve(full_release, author=None, date=None) | ||||
| actual_result = swh_storage.release_add([release]) | actual_result = swh_storage.release_add([release]) | ||||
| assert actual_result == {"release:add": 1} | assert actual_result == {"release:add": 1} | ||||
| ▲ Show 20 Lines • Show All 88 Lines • ▼ Show 20 Lines | def test_origin_add(self, swh_storage, sample_data): | ||||
| actual_origins = swh_storage.origin_get(origin_urls) | actual_origins = swh_storage.origin_get(origin_urls) | ||||
| assert actual_origins == origins | assert actual_origins == origins | ||||
| assert set(swh_storage.journal_writer.journal.objects) == set( | assert set(swh_storage.journal_writer.journal.objects) == set( | ||||
| [("origin", origin) for origin in origins] | [("origin", origin) for origin in origins] | ||||
| ) | ) | ||||
| if isinstance(swh_storage, InMemoryStorage) or not isinstance( | |||||
| swh_storage, CassandraStorage | |||||
| ): | |||||
| swh_storage.refresh_stat_counters() | swh_storage.refresh_stat_counters() | ||||
| assert swh_storage.stat_counters()["origin"] == len(origins) | assert swh_storage.stat_counters()["origin"] == len(origins) | ||||
| def test_origin_add_twice(self, swh_storage, sample_data): | def test_origin_add_twice(self, swh_storage, sample_data): | ||||
| origin, origin2 = sample_data.origins[:2] | origin, origin2 = sample_data.origins[:2] | ||||
| add1 = swh_storage.origin_add([origin, origin2]) | add1 = swh_storage.origin_add([origin, origin2]) | ||||
| assert set(swh_storage.journal_writer.journal.objects) == set( | assert set(swh_storage.journal_writer.journal.objects) == set( | ||||
| [("origin", origin), ("origin", origin2),] | [("origin", origin), ("origin", origin2),] | ||||
| ) | ) | ||||
| ▲ Show 20 Lines • Show All 423 Lines • ▼ Show 20 Lines | def test_origin_visit_status_get_random(self, swh_storage, sample_data): | ||||
| status="full", | status="full", | ||||
| snapshot=hash_to_bytes( | snapshot=hash_to_bytes( | ||||
| "9b922e6d8d5b803c1582aabe5525b7b91150788e" | "9b922e6d8d5b803c1582aabe5525b7b91150788e" | ||||
| ), | ), | ||||
| ) | ) | ||||
| ] | ] | ||||
| ) | ) | ||||
| if isinstance(swh_storage, InMemoryStorage) or not isinstance( | |||||
| swh_storage, CassandraStorage | |||||
| ): | |||||
| swh_storage.refresh_stat_counters() | swh_storage.refresh_stat_counters() | ||||
| stats = swh_storage.stat_counters() | stats = swh_storage.stat_counters() | ||||
| assert stats["origin"] == len(origins) | assert stats["origin"] == len(origins) | ||||
| assert stats["origin_visit"] == len(origins) * len(visits) | assert stats["origin_visit"] == len(origins) * len(visits) | ||||
| random_ovs = swh_storage.origin_visit_status_get_random(visit_type) | random_ovs = swh_storage.origin_visit_status_get_random(visit_type) | ||||
| assert random_ovs | assert random_ovs | ||||
| assert random_ovs.origin is not None | assert random_ovs.origin is not None | ||||
| assert random_ovs.origin in [o.url for o in origins] | assert random_ovs.origin in [o.url for o in origins] | ||||
| assert random_ovs.type is not None | assert random_ovs.type is not None | ||||
| def test_origin_visit_status_get_random_nothing_found( | def test_origin_visit_status_get_random_nothing_found( | ||||
| ▲ Show 20 Lines • Show All 1,180 Lines • ▼ Show 20 Lines | def test_snapshot_add_many(self, swh_storage, sample_data): | ||||
| "next_branch": None, | "next_branch": None, | ||||
| } | } | ||||
| assert swh_storage.snapshot_get(snapshot.id) == { | assert swh_storage.snapshot_get(snapshot.id) == { | ||||
| **snapshot.to_dict(), | **snapshot.to_dict(), | ||||
| "next_branch": None, | "next_branch": None, | ||||
| } | } | ||||
| if isinstance(swh_storage, InMemoryStorage) or not isinstance( | |||||
| swh_storage, CassandraStorage | |||||
| ): | |||||
| swh_storage.refresh_stat_counters() | swh_storage.refresh_stat_counters() | ||||
| assert swh_storage.stat_counters()["snapshot"] == 2 | assert swh_storage.stat_counters()["snapshot"] == 2 | ||||
| def test_snapshot_add_many_incremental(self, swh_storage, sample_data): | def test_snapshot_add_many_incremental(self, swh_storage, sample_data): | ||||
| snapshot, _, complete_snapshot = sample_data.snapshots[:3] | snapshot, _, complete_snapshot = sample_data.snapshots[:3] | ||||
| actual_result = swh_storage.snapshot_add([complete_snapshot]) | actual_result = swh_storage.snapshot_add([complete_snapshot]) | ||||
| assert actual_result == {"snapshot:add": 1} | assert actual_result == {"snapshot:add": 1} | ||||
| actual_result2 = swh_storage.snapshot_add([snapshot, complete_snapshot]) | actual_result2 = swh_storage.snapshot_add([snapshot, complete_snapshot]) | ||||
| ▲ Show 20 Lines • Show All 483 Lines • ▼ Show 20 Lines | def test_snapshot_missing(self, swh_storage, sample_data): | ||||
| snapshots = [snapshot.id, missing_snapshot.id] | snapshots = [snapshot.id, missing_snapshot.id] | ||||
| swh_storage.snapshot_add([snapshot]) | swh_storage.snapshot_add([snapshot]) | ||||
| missing_snapshots = swh_storage.snapshot_missing(snapshots) | missing_snapshots = swh_storage.snapshot_missing(snapshots) | ||||
| assert list(missing_snapshots) == [missing_snapshot.id] | assert list(missing_snapshots) == [missing_snapshot.id] | ||||
| def test_stat_counters(self, swh_storage, sample_data): | def test_stat_counters(self, swh_storage, sample_data): | ||||
| if isinstance(swh_storage, CassandraStorage) and not isinstance( | |||||
| swh_storage, InMemoryStorage | |||||
| ): | |||||
| pytest.skip("Cassandra backend does not support stat counters") | |||||
| origin = sample_data.origin | origin = sample_data.origin | ||||
| snapshot = sample_data.snapshot | snapshot = sample_data.snapshot | ||||
| revision = sample_data.revision | revision = sample_data.revision | ||||
| release = sample_data.release | release = sample_data.release | ||||
| directory = sample_data.directory | directory = sample_data.directory | ||||
| content = sample_data.content | content = sample_data.content | ||||
| expected_keys = ["content", "directory", "origin", "revision"] | expected_keys = ["content", "directory", "origin", "revision"] | ||||
| ▲ Show 20 Lines • Show All 984 Lines • Show Last 20 Lines | |||||