Changeset View
Changeset View
Standalone View
Standalone View
swh/scheduler/tests/test_scheduler.py
| Show First 20 Lines • Show All 864 Lines • ▼ Show 20 Lines | ): | ||||
| ) | ) | ||||
| self._check_grab_next_visit( | self._check_grab_next_visit( | ||||
| swh_scheduler, | swh_scheduler, | ||||
| visit_type=visit_type, | visit_type=visit_type, | ||||
| policy="oldest_scheduled_first", | policy="oldest_scheduled_first", | ||||
| expected=expected, | expected=expected, | ||||
| ) | ) | ||||
| @pytest.mark.parametrize("which_cooldown", ("scheduled", "failed")) | @pytest.mark.parametrize("which_cooldown", ("scheduled", "failed", "notfound")) | ||||
| @pytest.mark.parametrize("cooldown", (7, 15)) | @pytest.mark.parametrize("cooldown", (7, 15)) | ||||
| def test_grab_next_visits_cooldowns( | def test_grab_next_visits_cooldowns( | ||||
| self, swh_scheduler, listed_origins_by_type, which_cooldown, cooldown, | self, swh_scheduler, listed_origins_by_type, which_cooldown, cooldown, | ||||
| ): | ): | ||||
| visit_type, origins, expected = self._prepare_oldest_scheduled_first_origins( | visit_type, origins, expected = self._prepare_oldest_scheduled_first_origins( | ||||
| swh_scheduler, listed_origins_by_type | swh_scheduler, listed_origins_by_type | ||||
| ) | ) | ||||
| after = self._check_grab_next_visit_basic( | after = self._check_grab_next_visit_basic( | ||||
| swh_scheduler, | swh_scheduler, | ||||
| visit_type=visit_type, | visit_type=visit_type, | ||||
| policy="oldest_scheduled_first", | policy="oldest_scheduled_first", | ||||
| expected=expected, | expected=expected, | ||||
| ) | ) | ||||
| # Mark all the visits as `{which_cooldown}` (scheduled, failed or notfound) on | # Mark all the visits as `{which_cooldown}` (scheduled, failed or notfound) on | ||||
| # the `after` timestamp | # the `after` timestamp | ||||
| ovs_args = {"last_failed": None, "last_scheduled": None} | ovs_args = {"last_failed": None, "last_notfound": None, "last_scheduled": None} | ||||
| ovs_args[f"last_{which_cooldown}"] = after | ovs_args[f"last_{which_cooldown}"] = after | ||||
| visit_stats = [ | visit_stats = [ | ||||
| OriginVisitStats( | OriginVisitStats( | ||||
| url=origin.url, | url=origin.url, | ||||
| visit_type=origin.visit_type, | visit_type=origin.visit_type, | ||||
| last_snapshot=None, | last_snapshot=None, | ||||
| last_eventful=None, | last_eventful=None, | ||||
| last_uneventful=None, | last_uneventful=None, | ||||
| last_notfound=None, | |||||
| **ovs_args, | **ovs_args, | ||||
| ) | ) | ||||
| for i, origin in enumerate(origins) | for i, origin in enumerate(origins) | ||||
| ] | ] | ||||
| swh_scheduler.origin_visit_stats_upsert(visit_stats) | swh_scheduler.origin_visit_stats_upsert(visit_stats) | ||||
| cooldown_td = datetime.timedelta(days=cooldown) | cooldown_td = datetime.timedelta(days=cooldown) | ||||
| cooldown_args = { | cooldown_args = { | ||||
| "scheduled_cooldown": None, | "scheduled_cooldown": None, | ||||
| "failed_cooldown": None, | "failed_cooldown": None, | ||||
| "notfound_cooldown": None, | |||||
| } | } | ||||
| cooldown_args[f"{which_cooldown}_cooldown"] = cooldown_td | cooldown_args[f"{which_cooldown}_cooldown"] = cooldown_td | ||||
| ret = swh_scheduler.grab_next_visits( | ret = swh_scheduler.grab_next_visits( | ||||
| visit_type=visit_type, | visit_type=visit_type, | ||||
| count=len(expected) + 1, | count=len(expected) + 1, | ||||
| policy="oldest_scheduled_first", | policy="oldest_scheduled_first", | ||||
| timestamp=after + cooldown_td - datetime.timedelta(seconds=1), | timestamp=after + cooldown_td - datetime.timedelta(seconds=1), | ||||
| ▲ Show 20 Lines • Show All 436 Lines • Show Last 20 Lines | |||||