Changeset View
Changeset View
Standalone View
Standalone View
swh/scheduler/tests/test_scheduler.py
| Show First 20 Lines • Show All 941 Lines • ▼ Show 20 Lines | ): | ||||
| ) | ) | ||||
| self._check_grab_next_visit( | self._check_grab_next_visit( | ||||
| swh_scheduler, | swh_scheduler, | ||||
| visit_type=visit_type, | visit_type=visit_type, | ||||
| policy="oldest_scheduled_first", | policy="oldest_scheduled_first", | ||||
| expected=expected, | expected=expected, | ||||
| ) | ) | ||||
| @pytest.mark.parametrize("which_cooldown", ("scheduled", "failed", "not_found")) | @pytest.mark.parametrize( | ||||
| "which_cooldown", ("scheduled", "failed", "not_found", "absolute") | |||||
| ) | |||||
| @pytest.mark.parametrize("cooldown", (7, 15)) | @pytest.mark.parametrize("cooldown", (7, 15)) | ||||
| def test_grab_next_visits_cooldowns( | def test_grab_next_visits_cooldowns( | ||||
| self, | self, | ||||
| swh_scheduler, | swh_scheduler, | ||||
| listed_origins_by_type, | listed_origins_by_type, | ||||
| which_cooldown, | which_cooldown, | ||||
| cooldown, | cooldown, | ||||
| ): | ): | ||||
| visit_type, origins, expected = self._prepare_oldest_scheduled_first_origins( | visit_type, origins, expected = self._prepare_oldest_scheduled_first_origins( | ||||
| swh_scheduler, listed_origins_by_type | swh_scheduler, listed_origins_by_type | ||||
| ) | ) | ||||
| after = self._check_grab_next_visit_basic( | after = self._check_grab_next_visit_basic( | ||||
| swh_scheduler, | swh_scheduler, | ||||
| visit_type=visit_type, | visit_type=visit_type, | ||||
| policy="oldest_scheduled_first", | policy="oldest_scheduled_first", | ||||
| expected=expected, | expected=expected, | ||||
| ) | ) | ||||
| # Mark all the visits as scheduled, failed or notfound on the `after` timestamp | # Mark all the visits as scheduled, failed or not_found on the `after` timestamp. | ||||
| # If we're testing the `absolute_cooldown`, mark the visit as successful. | |||||
| ovs_args = { | ovs_args = { | ||||
| "last_visit": None, | "last_visit": None, | ||||
| "last_visit_status": None, | "last_visit_status": None, | ||||
| "last_scheduled": None, | "last_scheduled": None, | ||||
| "last_successful": None, | |||||
| "last_snapshot": None, | |||||
| } | } | ||||
| if which_cooldown == "scheduled": | if which_cooldown == "scheduled": | ||||
| ovs_args["last_scheduled"] = after | ovs_args["last_scheduled"] = after | ||||
| elif which_cooldown == "absolute": | |||||
| ovs_args["last_visit"] = after | |||||
| ovs_args["last_successful"] = after | |||||
| ovs_args["last_visit_status"] = LastVisitStatus.successful | |||||
| ovs_args["last_snapshot"] = b"\x00" * 20 | |||||
| else: | else: | ||||
| ovs_args["last_visit"] = after | ovs_args["last_visit"] = after | ||||
| ovs_args["last_visit_status"] = LastVisitStatus(which_cooldown) | ovs_args["last_visit_status"] = LastVisitStatus(which_cooldown) | ||||
| visit_stats = [ | visit_stats = [ | ||||
| OriginVisitStats( | OriginVisitStats( | ||||
| url=origin.url, | url=origin.url, | ||||
| visit_type=origin.visit_type, | visit_type=origin.visit_type, | ||||
| last_snapshot=None, | |||||
| last_successful=None, | |||||
| **ovs_args, | **ovs_args, | ||||
| ) | ) | ||||
| for i, origin in enumerate(origins) | for i, origin in enumerate(origins) | ||||
| ] | ] | ||||
| swh_scheduler.origin_visit_stats_upsert(visit_stats) | swh_scheduler.origin_visit_stats_upsert(visit_stats) | ||||
| cooldown_td = timedelta(days=cooldown) | cooldown_td = timedelta(days=cooldown) | ||||
| cooldown_args = { | cooldown_args = { | ||||
| "scheduled_cooldown": None, | "scheduled_cooldown": None, | ||||
| "failed_cooldown": None, | "failed_cooldown": None, | ||||
| "not_found_cooldown": None, | "not_found_cooldown": None, | ||||
| "absolute_cooldown": None, | |||||
| } | } | ||||
| cooldown_args[f"{which_cooldown}_cooldown"] = cooldown_td | cooldown_args[f"{which_cooldown}_cooldown"] = cooldown_td | ||||
| ret = swh_scheduler.grab_next_visits( | ret = swh_scheduler.grab_next_visits( | ||||
| visit_type=visit_type, | visit_type=visit_type, | ||||
| count=len(expected) + 1, | count=len(expected) + 1, | ||||
| policy="oldest_scheduled_first", | policy="oldest_scheduled_first", | ||||
| timestamp=after + cooldown_td - timedelta(seconds=1), | timestamp=after + cooldown_td - timedelta(seconds=1), | ||||
| ▲ Show 20 Lines • Show All 577 Lines • Show Last 20 Lines | |||||