Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/tests/storage_tests.py
| Show First 20 Lines • Show All 1,420 Lines • ▼ Show 20 Lines | def test_origin_visit_status_get_all(self, swh_storage, sample_data): | ||||
| [ | [ | ||||
| OriginVisit( | OriginVisit( | ||||
| origin=origin.url, date=date_visit1, type=sample_data.type_visit1, | origin=origin.url, date=date_visit1, type=sample_data.type_visit1, | ||||
| ), | ), | ||||
| ] | ] | ||||
| )[0] | )[0] | ||||
| ovs1 = OriginVisitStatus( | ovs1 = OriginVisitStatus( | ||||
| origin=origin.url, | origin=ov1.origin, | ||||
| visit=ov1.visit, | visit=ov1.visit, | ||||
| date=date_visit1, | date=date_visit1, | ||||
| type=ov1.type, | |||||
| status="created", | status="created", | ||||
| snapshot=None, | snapshot=None, | ||||
| ) | ) | ||||
| ovs2 = OriginVisitStatus( | ovs2 = OriginVisitStatus( | ||||
| origin=origin.url, | origin=ov1.origin, | ||||
| visit=ov1.visit, | visit=ov1.visit, | ||||
| date=date_visit2, | date=date_visit2, | ||||
| type=ov1.type, | |||||
| status="partial", | status="partial", | ||||
| snapshot=None, | snapshot=None, | ||||
| ) | ) | ||||
| ovs3 = OriginVisitStatus( | ovs3 = OriginVisitStatus( | ||||
| origin=origin.url, | origin=ov1.origin, | ||||
| visit=ov1.visit, | visit=ov1.visit, | ||||
| date=date_visit3, | date=date_visit3, | ||||
| type=ov1.type, | |||||
| status="full", | status="full", | ||||
| snapshot=sample_data.snapshot.id, | snapshot=sample_data.snapshot.id, | ||||
| metadata={}, | metadata={}, | ||||
| ) | ) | ||||
| swh_storage.origin_visit_status_add([ovs2, ovs3]) | swh_storage.origin_visit_status_add([ovs2, ovs3]) | ||||
| # order asc, no token, no limit | # order asc, no token, no limit | ||||
| ▲ Show 20 Lines • Show All 309 Lines • ▼ Show 20 Lines | def test_origin_visit_add(self, swh_storage, sample_data): | ||||
| # add once | # add once | ||||
| ov1, ov2 = swh_storage.origin_visit_add([visit1, visit2]) | ov1, ov2 = swh_storage.origin_visit_add([visit1, visit2]) | ||||
| # then again (will be ignored as they already exist) | # then again (will be ignored as they already exist) | ||||
| origin_visit1, origin_visit2 = swh_storage.origin_visit_add([ov1, ov2]) | origin_visit1, origin_visit2 = swh_storage.origin_visit_add([ov1, ov2]) | ||||
| assert ov1 == origin_visit1 | assert ov1 == origin_visit1 | ||||
| assert ov2 == origin_visit2 | assert ov2 == origin_visit2 | ||||
| ovs1 = OriginVisitStatus( | ovs1 = OriginVisitStatus( | ||||
| origin=origin1.url, | origin=ov1.origin, | ||||
| visit=ov1.visit, | visit=ov1.visit, | ||||
| date=date_visit, | date=date_visit, | ||||
| type=ov1.type, | |||||
| status="created", | status="created", | ||||
| snapshot=None, | snapshot=None, | ||||
| ) | ) | ||||
| ovs2 = OriginVisitStatus( | ovs2 = OriginVisitStatus( | ||||
| origin=origin1.url, | origin=ov2.origin, | ||||
| visit=ov2.visit, | visit=ov2.visit, | ||||
| date=date_visit2, | date=date_visit2, | ||||
| type=ov2.type, | |||||
| status="created", | status="created", | ||||
| snapshot=None, | snapshot=None, | ||||
| ) | ) | ||||
| actual_visits = swh_storage.origin_visit_get(origin1.url).results | actual_visits = swh_storage.origin_visit_get(origin1.url).results | ||||
| expected_visits = [ov1, ov2] | expected_visits = [ov1, ov2] | ||||
| assert len(expected_visits) == len(actual_visits) | assert len(expected_visits) == len(actual_visits) | ||||
| for visit in expected_visits: | for visit in expected_visits: | ||||
| ▲ Show 20 Lines • Show All 54 Lines • ▼ Show 20 Lines | def test_origin_visit_status_add(self, swh_storage, sample_data): | ||||
| origin=origin2.url, | origin=origin2.url, | ||||
| date=sample_data.date_visit2, | date=sample_data.date_visit2, | ||||
| type=sample_data.type_visit2, | type=sample_data.type_visit2, | ||||
| ), | ), | ||||
| ] | ] | ||||
| ) | ) | ||||
| ovs1 = OriginVisitStatus( | ovs1 = OriginVisitStatus( | ||||
| origin=origin1.url, | origin=ov1.origin, | ||||
| visit=ov1.visit, | visit=ov1.visit, | ||||
| date=sample_data.date_visit1, | date=sample_data.date_visit1, | ||||
| type=ov1.type, | |||||
| status="created", | status="created", | ||||
| snapshot=None, | snapshot=None, | ||||
| ) | ) | ||||
| ovs2 = OriginVisitStatus( | ovs2 = OriginVisitStatus( | ||||
| origin=origin2.url, | origin=ov2.origin, | ||||
| visit=ov2.visit, | visit=ov2.visit, | ||||
| date=sample_data.date_visit2, | date=sample_data.date_visit2, | ||||
| type=ov2.type, | |||||
| status="created", | status="created", | ||||
| snapshot=None, | snapshot=None, | ||||
| ) | ) | ||||
| date_visit_now = round_to_milliseconds(now()) | date_visit_now = round_to_milliseconds(now()) | ||||
| visit_status1 = OriginVisitStatus( | visit_status1 = OriginVisitStatus( | ||||
| origin=ov1.origin, | origin=ov1.origin, | ||||
| visit=ov1.visit, | visit=ov1.visit, | ||||
| date=date_visit_now, | date=date_visit_now, | ||||
| type=ov1.type, | |||||
| status="full", | status="full", | ||||
| snapshot=snapshot.id, | snapshot=snapshot.id, | ||||
| ) | ) | ||||
| date_visit_now = round_to_milliseconds(now()) | date_visit_now = round_to_milliseconds(now()) | ||||
| visit_status2 = OriginVisitStatus( | visit_status2 = OriginVisitStatus( | ||||
| origin=ov2.origin, | origin=ov2.origin, | ||||
| visit=ov2.visit, | visit=ov2.visit, | ||||
| date=date_visit_now, | date=date_visit_now, | ||||
| type=ov2.type, | |||||
| status="ongoing", | status="ongoing", | ||||
| snapshot=None, | snapshot=None, | ||||
| metadata={"intrinsic": "something"}, | metadata={"intrinsic": "something"}, | ||||
| ) | ) | ||||
| swh_storage.origin_visit_status_add([visit_status1, visit_status2]) | swh_storage.origin_visit_status_add([visit_status1, visit_status2]) | ||||
| visit = swh_storage.origin_visit_get_latest(origin1.url, require_snapshot=True) | visit = swh_storage.origin_visit_get_latest(origin1.url, require_snapshot=True) | ||||
| visit_status = swh_storage.origin_visit_status_get_latest( | visit_status = swh_storage.origin_visit_status_get_latest( | ||||
| Show All 36 Lines | def test_origin_visit_status_add_twice(self, swh_storage, sample_data): | ||||
| origin=origin1.url, | origin=origin1.url, | ||||
| date=sample_data.date_visit1, | date=sample_data.date_visit1, | ||||
| type=sample_data.type_visit1, | type=sample_data.type_visit1, | ||||
| ), | ), | ||||
| ] | ] | ||||
| )[0] | )[0] | ||||
| ovs1 = OriginVisitStatus( | ovs1 = OriginVisitStatus( | ||||
| origin=origin1.url, | origin=ov1.origin, | ||||
| visit=ov1.visit, | visit=ov1.visit, | ||||
| date=sample_data.date_visit1, | date=sample_data.date_visit1, | ||||
| type=ov1.type, | |||||
| status="created", | status="created", | ||||
| snapshot=None, | snapshot=None, | ||||
| ) | ) | ||||
| date_visit_now = round_to_milliseconds(now()) | date_visit_now = round_to_milliseconds(now()) | ||||
| visit_status1 = OriginVisitStatus( | visit_status1 = OriginVisitStatus( | ||||
| origin=ov1.origin, | origin=ov1.origin, | ||||
| visit=ov1.visit, | visit=ov1.visit, | ||||
| date=date_visit_now, | date=date_visit_now, | ||||
| type=ov1.type, | |||||
| status="full", | status="full", | ||||
| snapshot=snapshot.id, | snapshot=snapshot.id, | ||||
| ) | ) | ||||
| swh_storage.origin_visit_status_add([visit_status1]) | swh_storage.origin_visit_status_add([visit_status1]) | ||||
| # second call will ignore existing entries (will send to storage though) | # second call will ignore existing entries (will send to storage though) | ||||
| swh_storage.origin_visit_status_add([visit_status1]) | swh_storage.origin_visit_status_add([visit_status1]) | ||||
| ▲ Show 20 Lines • Show All 229 Lines • ▼ Show 20 Lines | def test_origin_visit_get_latest(self, swh_storage, sample_data): | ||||
| actual_visit = swh_storage.origin_visit_get_latest( | actual_visit = swh_storage.origin_visit_get_latest( | ||||
| origin.url, allowed_statuses=["created"] | origin.url, allowed_statuses=["created"] | ||||
| ) | ) | ||||
| assert actual_visit == ov3 | assert actual_visit == ov3 | ||||
| # Add snapshot to visit1; require_snapshot=True makes it return first visit | # Add snapshot to visit1; require_snapshot=True makes it return first visit | ||||
| swh_storage.snapshot_add([complete_snapshot]) | swh_storage.snapshot_add([complete_snapshot]) | ||||
| visit_status_with_snapshot = OriginVisitStatus( | visit_status_with_snapshot = OriginVisitStatus( | ||||
| origin=origin.url, | origin=ov1.origin, | ||||
| visit=ov1.visit, | visit=ov1.visit, | ||||
| date=round_to_milliseconds(now()), | date=round_to_milliseconds(now()), | ||||
| type=ov1.type, | |||||
| status="ongoing", | status="ongoing", | ||||
| snapshot=complete_snapshot.id, | snapshot=complete_snapshot.id, | ||||
| ) | ) | ||||
| swh_storage.origin_visit_status_add([visit_status_with_snapshot]) | swh_storage.origin_visit_status_add([visit_status_with_snapshot]) | ||||
| # only the first visit has a snapshot now | # only the first visit has a snapshot now | ||||
| actual_visit = swh_storage.origin_visit_get_latest( | actual_visit = swh_storage.origin_visit_get_latest( | ||||
| origin.url, require_snapshot=True | origin.url, require_snapshot=True | ||||
| ) | ) | ||||
| Show All 22 Lines | def test_origin_visit_get_latest(self, swh_storage, sample_data): | ||||
| # Status filter: all three visits are status=ongoing, so no visit | # Status filter: all three visits are status=ongoing, so no visit | ||||
| # returned | # returned | ||||
| actual_visit = swh_storage.origin_visit_get_latest( | actual_visit = swh_storage.origin_visit_get_latest( | ||||
| origin.url, allowed_statuses=["full"] | origin.url, allowed_statuses=["full"] | ||||
| ) | ) | ||||
| assert actual_visit is None | assert actual_visit is None | ||||
| visit_status1_full = OriginVisitStatus( | visit_status1_full = OriginVisitStatus( | ||||
| origin=origin.url, | origin=ov1.origin, | ||||
| visit=ov1.visit, | visit=ov1.visit, | ||||
| date=round_to_milliseconds(now()), | date=round_to_milliseconds(now()), | ||||
| type=ov1.type, | |||||
| status="full", | status="full", | ||||
| snapshot=complete_snapshot.id, | snapshot=complete_snapshot.id, | ||||
| ) | ) | ||||
| # Mark the first visit as completed and check status filter again | # Mark the first visit as completed and check status filter again | ||||
| swh_storage.origin_visit_status_add([visit_status1_full]) | swh_storage.origin_visit_status_add([visit_status1_full]) | ||||
| # only the first visit has the full status | # only the first visit has the full status | ||||
| actual_visit = swh_storage.origin_visit_get_latest( | actual_visit = swh_storage.origin_visit_get_latest( | ||||
| Show All 9 Lines | def test_origin_visit_get_latest(self, swh_storage, sample_data): | ||||
| # no specific filter, this returns as before the latest visit | # no specific filter, this returns as before the latest visit | ||||
| actual_visit = swh_storage.origin_visit_get_latest(origin.url) | actual_visit = swh_storage.origin_visit_get_latest(origin.url) | ||||
| assert actual_visit == ov3 | assert actual_visit == ov3 | ||||
| # Add snapshot to visit2 and check that the new snapshot is returned | # Add snapshot to visit2 and check that the new snapshot is returned | ||||
| swh_storage.snapshot_add([empty_snapshot]) | swh_storage.snapshot_add([empty_snapshot]) | ||||
| visit_status2_full = OriginVisitStatus( | visit_status2_full = OriginVisitStatus( | ||||
| origin=origin.url, | origin=ov2.origin, | ||||
| visit=ov2.visit, | visit=ov2.visit, | ||||
| date=round_to_milliseconds(now()), | date=round_to_milliseconds(now()), | ||||
| type=ov2.type, | |||||
| status="ongoing", | status="ongoing", | ||||
| snapshot=empty_snapshot.id, | snapshot=empty_snapshot.id, | ||||
| ) | ) | ||||
| swh_storage.origin_visit_status_add([visit_status2_full]) | swh_storage.origin_visit_status_add([visit_status2_full]) | ||||
| actual_visit = swh_storage.origin_visit_get_latest( | actual_visit = swh_storage.origin_visit_get_latest( | ||||
| origin.url, require_snapshot=True | origin.url, require_snapshot=True | ||||
| ) | ) | ||||
| # 2nd visit is most recent with a snapshot | # 2nd visit is most recent with a snapshot | ||||
| Show All 10 Lines | def test_origin_visit_get_latest(self, swh_storage, sample_data): | ||||
| # full status is still the first visit | # full status is still the first visit | ||||
| actual_visit = swh_storage.origin_visit_get_latest( | actual_visit = swh_storage.origin_visit_get_latest( | ||||
| origin.url, allowed_statuses=["full"] | origin.url, allowed_statuses=["full"] | ||||
| ) | ) | ||||
| assert actual_visit == ov1 | assert actual_visit == ov1 | ||||
| # Add snapshot to visit3 (same date as visit2) | # Add snapshot to visit3 (same date as visit2) | ||||
| visit_status3_with_snapshot = OriginVisitStatus( | visit_status3_with_snapshot = OriginVisitStatus( | ||||
| origin=origin.url, | origin=ov3.origin, | ||||
| visit=ov3.visit, | visit=ov3.visit, | ||||
| date=round_to_milliseconds(now()), | date=round_to_milliseconds(now()), | ||||
| type=ov3.type, | |||||
| status="ongoing", | status="ongoing", | ||||
| snapshot=complete_snapshot.id, | snapshot=complete_snapshot.id, | ||||
| ) | ) | ||||
| swh_storage.origin_visit_status_add([visit_status3_with_snapshot]) | swh_storage.origin_visit_status_add([visit_status3_with_snapshot]) | ||||
| # full status is still the first visit | # full status is still the first visit | ||||
| actual_visit = swh_storage.origin_visit_get_latest( | actual_visit = swh_storage.origin_visit_get_latest( | ||||
| origin.url, allowed_statuses=["full"], require_snapshot=True, | origin.url, allowed_statuses=["full"], require_snapshot=True, | ||||
| ▲ Show 20 Lines • Show All 131 Lines • ▼ Show 20 Lines | def test_origin_visit_status_get_latest(self, swh_storage, sample_data): | ||||
| ) | ) | ||||
| swh_storage.snapshot_add([snapshot]) | swh_storage.snapshot_add([snapshot]) | ||||
| date_now = round_to_milliseconds(now()) | date_now = round_to_milliseconds(now()) | ||||
| assert sample_data.date_visit1 < sample_data.date_visit2 | assert sample_data.date_visit1 < sample_data.date_visit2 | ||||
| assert sample_data.date_visit2 < date_now | assert sample_data.date_visit2 < date_now | ||||
| ovs1 = OriginVisitStatus( | ovs1 = OriginVisitStatus( | ||||
| origin=origin1.url, | origin=ov1.origin, | ||||
| visit=ov1.visit, | visit=ov1.visit, | ||||
| date=sample_data.date_visit1, | date=sample_data.date_visit1, | ||||
| type=ov1.type, | |||||
| status="partial", | status="partial", | ||||
| snapshot=None, | snapshot=None, | ||||
| ) | ) | ||||
| ovs2 = OriginVisitStatus( | ovs2 = OriginVisitStatus( | ||||
| origin=origin1.url, | origin=ov1.origin, | ||||
| visit=ov1.visit, | visit=ov1.visit, | ||||
| date=sample_data.date_visit2, | date=sample_data.date_visit2, | ||||
| type=ov1.type, | |||||
| status="ongoing", | status="ongoing", | ||||
| snapshot=None, | snapshot=None, | ||||
| ) | ) | ||||
| ovs3 = OriginVisitStatus( | ovs3 = OriginVisitStatus( | ||||
| origin=origin1.url, | origin=ov2.origin, | ||||
| visit=ov2.visit, | visit=ov2.visit, | ||||
| date=sample_data.date_visit2 | date=sample_data.date_visit2 | ||||
| + datetime.timedelta(minutes=1), # to not be ignored | + datetime.timedelta(minutes=1), # to not be ignored | ||||
| type=ov2.type, | |||||
| status="ongoing", | status="ongoing", | ||||
| snapshot=None, | snapshot=None, | ||||
| ) | ) | ||||
| ovs4 = OriginVisitStatus( | ovs4 = OriginVisitStatus( | ||||
| origin=origin1.url, | origin=ov2.origin, | ||||
| visit=ov2.visit, | visit=ov2.visit, | ||||
| date=date_now, | date=date_now, | ||||
| type=ov2.type, | |||||
| status="full", | status="full", | ||||
| snapshot=snapshot.id, | snapshot=snapshot.id, | ||||
| metadata={"something": "wicked"}, | metadata={"something": "wicked"}, | ||||
| ) | ) | ||||
| swh_storage.origin_visit_status_add([ovs1, ovs2, ovs3, ovs4]) | swh_storage.origin_visit_status_add([ovs1, ovs2, ovs3, ovs4]) | ||||
| # unknown origin so no result | # unknown origin so no result | ||||
| ▲ Show 20 Lines • Show All 84 Lines • ▼ Show 20 Lines | def test_snapshot_add_get_empty(self, swh_storage, sample_data): | ||||
| actual_result = swh_storage.snapshot_add([empty_snapshot]) | actual_result = swh_storage.snapshot_add([empty_snapshot]) | ||||
| assert actual_result == {"snapshot:add": 1} | assert actual_result == {"snapshot:add": 1} | ||||
| date_now = now() | date_now = now() | ||||
| swh_storage.origin_visit_status_add( | swh_storage.origin_visit_status_add( | ||||
| [ | [ | ||||
| OriginVisitStatus( | OriginVisitStatus( | ||||
| origin=origin.url, | origin=ov1.origin, | ||||
| visit=ov1.visit, | visit=ov1.visit, | ||||
| date=date_now, | date=date_now, | ||||
| type=ov1.type, | |||||
| status="full", | status="full", | ||||
| snapshot=empty_snapshot.id, | snapshot=empty_snapshot.id, | ||||
| ) | ) | ||||
| ] | ] | ||||
| ) | ) | ||||
| by_id = swh_storage.snapshot_get(empty_snapshot.id) | by_id = swh_storage.snapshot_get(empty_snapshot.id) | ||||
| assert by_id == {**empty_snapshot_dict, "next_branch": None} | assert by_id == {**empty_snapshot_dict, "next_branch": None} | ||||
| ovs1 = OriginVisitStatus.from_dict( | ovs1 = OriginVisitStatus.from_dict( | ||||
| { | { | ||||
| "origin": origin.url, | "origin": ov1.origin, | ||||
| "date": sample_data.date_visit1, | "date": sample_data.date_visit1, | ||||
| "type": ov1.type, | |||||
| "visit": ov1.visit, | "visit": ov1.visit, | ||||
| "status": "created", | "status": "created", | ||||
| "snapshot": None, | "snapshot": None, | ||||
| "metadata": None, | "metadata": None, | ||||
| } | } | ||||
| ) | ) | ||||
| ovs2 = OriginVisitStatus.from_dict( | ovs2 = OriginVisitStatus.from_dict( | ||||
| { | { | ||||
| "origin": origin.url, | "origin": ov1.origin, | ||||
| "date": date_now, | "date": date_now, | ||||
| "type": ov1.type, | |||||
| "visit": ov1.visit, | "visit": ov1.visit, | ||||
| "status": "full", | "status": "full", | ||||
| "metadata": None, | "metadata": None, | ||||
| "snapshot": empty_snapshot.id, | "snapshot": empty_snapshot.id, | ||||
| } | } | ||||
| ) | ) | ||||
| actual_objects = list(swh_storage.journal_writer.journal.objects) | actual_objects = list(swh_storage.journal_writer.journal.objects) | ||||
| ▲ Show 20 Lines • Show All 1,341 Lines • Show Last 20 Lines | |||||