diff --git a/swh/storage/cassandra/storage.py b/swh/storage/cassandra/storage.py --- a/swh/storage/cassandra/storage.py +++ b/swh/storage/cassandra/storage.py @@ -897,8 +897,7 @@ snapshot=snapshot or last_visit_update["snapshot"], metadata=metadata or last_visit_update["metadata"], ) - self._cql_runner.origin_visit_status_add_one(visit_status) - # self._origin_visit_status_add(visit_status) + self._origin_visit_status_add(visit_status) def _origin_visit_merge( self, visit: Dict[str, Any], visit_status: Dict[str, Any] diff --git a/swh/storage/in_memory.py b/swh/storage/in_memory.py --- a/swh/storage/in_memory.py +++ b/swh/storage/in_memory.py @@ -899,7 +899,6 @@ # Retrieve the previous visit status assert visit.visit is not None - visit_key = (origin_url, visit.visit) last_visit_status = self._origin_visit_get_updated(origin, visit_id) assert last_visit_status is not None @@ -913,10 +912,7 @@ snapshot=snapshot or last_visit_status.snapshot, metadata=metadata or last_visit_status.metadata, ) - visit_key = (visit_status.origin, visit_status.visit) - self._origin_visit_statuses.setdefault(visit_key, []) - self._origin_visit_statuses[visit_key].append(visit_status) - # self._origin_visit_status_add_one(visit_status) + self._origin_visit_status_add_one(visit_status) def origin_visit_upsert(self, visits: Iterable[OriginVisit]) -> None: for visit in visits: diff --git a/swh/storage/storage.py b/swh/storage/storage.py --- a/swh/storage/storage.py +++ b/swh/storage/storage.py @@ -954,13 +954,7 @@ snapshot=snapshot or last_visit_status["snapshot"], metadata=metadata or last_visit_status["metadata"], ) - db.origin_visit_status_add(visit_status, cur=cur) - send_metric( - "origin_visit_status:add", - count=1, - method_name="origin_visit_status", - ) - # self._origin_visit_status_add(visit_status, db=db, cur=cur) + self._origin_visit_status_add(visit_status, db=db, cur=cur) def _origin_visit_get_updated( self, origin: str, visit_id: int, db, cur diff --git a/swh/storage/tests/test_kafka_writer.py b/swh/storage/tests/test_kafka_writer.py --- a/swh/storage/tests/test_kafka_writer.py +++ b/swh/storage/tests/test_kafka_writer.py @@ -60,7 +60,7 @@ for k in ("visit", "origin", "date", "type"): del obj_d[k] storage.origin_visit_update(obj.origin, visit.visit, **obj_d) - expected_messages += 1 + expected_messages += 1 + 1 # 1 visit update + 1 visit status else: assert False, obj_type diff --git a/swh/storage/tests/test_storage.py b/swh/storage/tests/test_storage.py --- a/swh/storage/tests/test_storage.py +++ b/swh/storage/tests/test_storage.py @@ -42,7 +42,7 @@ from swh.storage.converters import origin_url_to_sha1 as sha1 from swh.storage.exc import HashCollision, StorageArgumentException from swh.storage.interface import StorageInterface -from swh.storage.utils import content_hex_hashes +from swh.storage.utils import content_hex_hashes, now from .storage_data import data @@ -103,10 +103,6 @@ return Content.from_dict({**self.to_dict(), "data": data.cont["data"]}) -def now(): - return datetime.datetime.now(tz=datetime.timezone.utc) - - class TestStorage: """Main class for Storage testing. @@ -2342,12 +2338,14 @@ actual_result = swh_storage.snapshot_add([data.empty_snapshot]) assert actual_result == {"snapshot:add": 1} + date_now = now() + swh_storage.origin_visit_update( origin_url, origin_visit1.visit, status="ongoing", snapshot=data.empty_snapshot["id"], - # date=data.date_visit2 + date=date_now, ) by_id = swh_storage.snapshot_get(data.empty_snapshot["id"]) @@ -2385,7 +2383,10 @@ "origin_visit", OriginVisit.from_dict({**data2, "type": data.type_visit1,}), ), - # ("origin_visit_status", OriginVisitStatus.from_dict(data2)), + ( + "origin_visit_status", + OriginVisitStatus.from_dict({**data2, "date": date_now}), + ), ] def test_snapshot_add_get_complete(self, swh_storage): @@ -2713,11 +2714,13 @@ ) visit1_id = origin_visit1.visit swh_storage.snapshot_add([data.snapshot]) + date_now2 = now() swh_storage.origin_visit_update( origin_url, origin_visit1.visit, status="ongoing", snapshot=data.snapshot["id"], + date=date_now2, ) by_ov1 = swh_storage.snapshot_get_by_origin_visit(origin_url, visit1_id) @@ -2729,11 +2732,13 @@ visit2_id = origin_visit2.visit swh_storage.snapshot_add([data.snapshot]) + date_now4 = now() swh_storage.origin_visit_update( origin_url, origin_visit2.visit, status="ongoing", snapshot=data.snapshot["id"], + date=date_now4, ) by_ov2 = swh_storage.snapshot_get_by_origin_visit(origin_url, visit2_id) @@ -2743,7 +2748,6 @@ "origin": origin_url, "date": data.date_visit1, "visit": origin_visit1.visit, - # "type": data.type_visit1, "status": "ongoing", "metadata": None, "snapshot": None, @@ -2752,7 +2756,6 @@ "origin": origin_url, "date": data.date_visit1, "visit": origin_visit1.visit, - "type": data.type_visit1, "status": "ongoing", "metadata": None, "snapshot": data.snapshot["id"], @@ -2761,7 +2764,6 @@ "origin": origin_url, "date": data.date_visit2, "visit": origin_visit2.visit, - # "type": data.type_visit2, "status": "ongoing", "metadata": None, "snapshot": None, @@ -2770,12 +2772,12 @@ "origin": origin_url, "date": data.date_visit2, "visit": origin_visit2.visit, - "type": data.type_visit2, "status": "ongoing", "metadata": None, "snapshot": data.snapshot["id"], } - assert list(swh_storage.journal_writer.journal.objects) == [ + actual_objects = list(swh_storage.journal_writer.journal.objects) + assert actual_objects == [ ("origin", Origin.from_dict(data.origin)), ( "origin_visit", @@ -2783,13 +2785,27 @@ ), ("origin_visit_status", OriginVisitStatus.from_dict(data1)), ("snapshot", Snapshot.from_dict(data.snapshot)), - ("origin_visit", OriginVisit.from_dict(data2)), + ( + "origin_visit", + OriginVisit.from_dict({**data2, "type": data.type_visit1}), + ), + ( + "origin_visit_status", + OriginVisitStatus.from_dict({**data2, "date": date_now2}), + ), ( "origin_visit", OriginVisit.from_dict({**data3, "type": data.type_visit2}), ), ("origin_visit_status", OriginVisitStatus.from_dict(data3)), - ("origin_visit", OriginVisit.from_dict(data4)), + ( + "origin_visit", + OriginVisit.from_dict({**data4, "type": data.type_visit2}), + ), + ( + "origin_visit_status", + OriginVisitStatus.from_dict({**data4, "date": date_now4}), + ), ] def test_snapshot_get_latest(self, swh_storage):