Differential D5903 Diff 21222 swh/scheduler/tests/test_scheduler.py

Changeset View

Standalone View

swh/scheduler/tests/test_scheduler.py

Show First 20 Lines • Show All 864 Lines • ▼ Show 20 Lines	):
)		)
self._check_grab_next_visit(		self._check_grab_next_visit(
swh_scheduler,		swh_scheduler,
visit_type=visit_type,		visit_type=visit_type,
policy="oldest_scheduled_first",		policy="oldest_scheduled_first",
expected=expected,		expected=expected,
)		)

		@pytest.mark.parametrize("which_cooldown", ("scheduled", "failed"))
@pytest.mark.parametrize("cooldown", (7, 15))		@pytest.mark.parametrize("cooldown", (7, 15))
def test_grab_next_visits_oldest_scheduled_first_scheduled_cooldown(		def test_grab_next_visits_cooldowns(
self, swh_scheduler, listed_origins_by_type, cooldown		self, swh_scheduler, listed_origins_by_type, which_cooldown, cooldown,
):		):
visit_type, origins, expected = self._prepare_oldest_scheduled_first_origins(		visit_type, origins, expected = self._prepare_oldest_scheduled_first_origins(
swh_scheduler, listed_origins_by_type		swh_scheduler, listed_origins_by_type
)		)
after = self._check_grab_next_visit_basic(		after = self._check_grab_next_visit_basic(
swh_scheduler,		swh_scheduler,
visit_type=visit_type,		visit_type=visit_type,
policy="oldest_scheduled_first",		policy="oldest_scheduled_first",
expected=expected,		expected=expected,
)		)

		# Mark all the visits as `{which_cooldown}` (scheduled, failed or notfound) on
		# the `after` timestamp
		ovs_args = {"last_failed": None, "last_scheduled": None}
		ovs_args[f"last_{which_cooldown}"] = after

		visit_stats = [
		OriginVisitStats(
		url=origin.url,
		visit_type=origin.visit_type,
		last_snapshot=None,
		last_eventful=None,
		last_uneventful=None,
		last_notfound=None,
		**ovs_args,
		)
		for i, origin in enumerate(origins)
		]
		swh_scheduler.origin_visit_stats_upsert(visit_stats)

cooldown_td = datetime.timedelta(days=cooldown)		cooldown_td = datetime.timedelta(days=cooldown)
		cooldown_args = {
		"scheduled_cooldown": None,
		"failed_cooldown": None,
		}
		cooldown_args[f"{which_cooldown}_cooldown"] = cooldown_td

ret = swh_scheduler.grab_next_visits(		ret = swh_scheduler.grab_next_visits(
visit_type=visit_type,		visit_type=visit_type,
count=len(expected) + 1,		count=len(expected) + 1,
policy="oldest_scheduled_first",		policy="oldest_scheduled_first",
timestamp=after + cooldown_td - datetime.timedelta(seconds=1),		timestamp=after + cooldown_td - datetime.timedelta(seconds=1),
scheduled_cooldown=cooldown_td,		**cooldown_args,
)		)

assert ret == [], "Scheduled cooldown ignored"		assert ret == [], f"{which_cooldown}_cooldown ignored"

ret = swh_scheduler.grab_next_visits(		ret = swh_scheduler.grab_next_visits(
visit_type=visit_type,		visit_type=visit_type,
count=len(expected) + 1,		count=len(expected) + 1,
policy="oldest_scheduled_first",		policy="oldest_scheduled_first",
timestamp=after + cooldown_td + datetime.timedelta(seconds=1),		timestamp=after + cooldown_td + datetime.timedelta(seconds=1),
scheduled_cooldown=cooldown_td,		**cooldown_args,
)		)

assert sorted(ret) == sorted(		assert sorted(ret) == sorted(
expected		expected
), "grab_next_visits didn't reschedule visits after the configured cooldown"		), "grab_next_visits didn't reschedule visits after the configured cooldown"

def test_grab_next_visits_never_visited_oldest_update_first(		def test_grab_next_visits_never_visited_oldest_update_first(
self, swh_scheduler, listed_origins_by_type,		self, swh_scheduler, listed_origins_by_type,
Show All 39 Lines	):
# Update the visit stats with a known visit at a controlled date for		# Update the visit stats with a known visit at a controlled date for
# half the origins. Pick the date in the middle of the		# half the origins. Pick the date in the middle of the
# updated_origins' `last_update` range		# updated_origins' `last_update` range
visit_date = updated_origins[len(updated_origins) // 2].last_update		visit_date = updated_origins[len(updated_origins) // 2].last_update
visited_origins = updated_origins[::2]		visited_origins = updated_origins[::2]
visit_stats = [		visit_stats = [
OriginVisitStats(		OriginVisitStats(
url=origin.url,		url=origin.url,
visit_type=origin.visit_type,		visit_type=origin.visit_type,
		ardumontUnsubmitted Not Done Inline Actions Does it make sense to use both cooldown in one call? If not, can we code the exclusion logic directly within the grab_next_visits backend (making one none if the other is specified)? ardumont: Does it make sense to use both cooldown in one call? If not, can we code the exclusion logic…
		olasdAuthorUnsubmitted Done Inline Actions Yes, we will use them all (that's why they all have a non-zero default value too). It's just (much, much) easier to test them separately. olasd: Yes, we will use them all (that's why they all have a non-zero default value too). It's just…
		ardumontUnsubmitted Not Done Inline Actions ack, thx ;) ardumont: ack, thx ;)
last_snapshot=hash_to_bytes("d81cc0710eb6cf9efd5b920a8453e1e07157b6cd"),		last_snapshot=hash_to_bytes("d81cc0710eb6cf9efd5b920a8453e1e07157b6cd"),
last_eventful=visit_date,		last_eventful=visit_date,
last_uneventful=None,		last_uneventful=None,
last_failed=None,		last_failed=None,
last_notfound=None,		last_notfound=None,
)		)
for origin in visited_origins		for origin in visited_origins
]		]
▲ Show 20 Lines • Show All 361 Lines • Show Last 20 Lines