Page MenuHomeSoftware Heritage
Differential D4888 Diff 17451 swh/scheduler/tests/test_scheduler.py

Changeset View

Standalone View

View Options

swh/scheduler/tests/test_scheduler.py

Show First 20 Lines • Show All 766 Lines • ▼ Show 20 Lines def test_grab_next_visits(self, swh_scheduler, listed_origins_by_type, policy):
swh_scheduler.record_listed_origins(origins) swh_scheduler.record_listed_origins(origins)
before = utcnow() before = utcnow()
ret = swh_scheduler.grab_next_visits(visit_type, NUM_RESULTS, policy=policy) ret = swh_scheduler.grab_next_visits(visit_type, NUM_RESULTS, policy=policy)
after = utcnow() after = utcnow()
assert len(ret) == NUM_RESULTS assert len(ret) == NUM_RESULTS
for origin in ret: for origin in ret:
visit_stats = swh_scheduler.origin_visit_stats_get( pk = (origin.url, origin.visit_type)
origin.url, origin.visit_type visit_stats = swh_scheduler.origin_visit_stats_get([pk])[0]
)
assert visit_stats is not None assert visit_stats is not None
assert before <= visit_stats.last_scheduled <= after assert before <= visit_stats.last_scheduled <= after
@pytest.mark.parametrize("policy", ["oldest_scheduled_first"]) @pytest.mark.parametrize("policy", ["oldest_scheduled_first"])
def test_grab_next_visits_underflow( def test_grab_next_visits_underflow(
self, swh_scheduler, listed_origins_by_type, policy self, swh_scheduler, listed_origins_by_type, policy
): ):
NUM_RESULTS = 5 NUM_RESULTS = 5
Show All 14 Lines def test_grab_next_visits_unknown_policy(self, swh_scheduler):
swh_scheduler.grab_next_visits( swh_scheduler.grab_next_visits(
"type", NUM_RESULTS, policy="non_existing_policy" "type", NUM_RESULTS, policy="non_existing_policy"
) )
def _create_task_types(self, scheduler): def _create_task_types(self, scheduler):
for tt in TASK_TYPES.values(): for tt in TASK_TYPES.values():
scheduler.create_task_type(tt) scheduler.create_task_type(tt)
def test_origin_visit_stats_get_empty(self, swh_scheduler) -> None:
assert swh_scheduler.origin_visit_stats_get([]) == []
def test_origin_visit_stats_upsert(self, swh_scheduler) -> None: def test_origin_visit_stats_upsert(self, swh_scheduler) -> None:
eventful_date = utcnow() eventful_date = utcnow()
url = "https://github.com/test" url = "https://github.com/test"
visit_stats = OriginVisitStats( visit_stats = OriginVisitStats(
url=url, url=url,
visit_type="git", visit_type="git",
last_eventful=eventful_date, last_eventful=eventful_date,
last_uneventful=None, last_uneventful=None,
last_failed=None, last_failed=None,
last_notfound=None, last_notfound=None,
) )
swh_scheduler.origin_visit_stats_upsert([visit_stats]) swh_scheduler.origin_visit_stats_upsert([visit_stats])
swh_scheduler.origin_visit_stats_upsert([visit_stats]) swh_scheduler.origin_visit_stats_upsert([visit_stats])
assert swh_scheduler.origin_visit_stats_get(url, "git") == visit_stats assert swh_scheduler.origin_visit_stats_get([(url, "git")]) == [visit_stats]
assert swh_scheduler.origin_visit_stats_get(url, "svn") is None assert swh_scheduler.origin_visit_stats_get([(url, "svn")]) == []
uneventful_date = utcnow() uneventful_date = utcnow()
visit_stats = OriginVisitStats( visit_stats = OriginVisitStats(
url=url, url=url,
visit_type="git", visit_type="git",
last_eventful=None, last_eventful=None,
last_uneventful=uneventful_date, last_uneventful=uneventful_date,
last_failed=None, last_failed=None,
last_notfound=None, last_notfound=None,
) )
swh_scheduler.origin_visit_stats_upsert([visit_stats]) swh_scheduler.origin_visit_stats_upsert([visit_stats])
uneventful_visit = swh_scheduler.origin_visit_stats_get(url, "git") uneventful_visits = swh_scheduler.origin_visit_stats_get([(url, "git")])
expected_visit_stats = OriginVisitStats( expected_visit_stats = OriginVisitStats(
url=url, url=url,
visit_type="git", visit_type="git",
last_eventful=eventful_date, last_eventful=eventful_date,
last_uneventful=uneventful_date, last_uneventful=uneventful_date,
last_failed=None, last_failed=None,
last_notfound=None, last_notfound=None,
) )
assert uneventful_visit == expected_visit_stats assert uneventful_visits == [expected_visit_stats]
failed_date = utcnow() failed_date = utcnow()
visit_stats = OriginVisitStats( visit_stats = OriginVisitStats(
url=url, url=url,
visit_type="git", visit_type="git",
last_eventful=None, last_eventful=None,
last_uneventful=None, last_uneventful=None,
last_failed=failed_date, last_failed=failed_date,
last_notfound=None, last_notfound=None,
) )
swh_scheduler.origin_visit_stats_upsert([visit_stats]) swh_scheduler.origin_visit_stats_upsert([visit_stats])
failed_visit = swh_scheduler.origin_visit_stats_get(url, "git") failed_visits = swh_scheduler.origin_visit_stats_get([(url, "git")])
expected_visit_stats = OriginVisitStats( expected_visit_stats = OriginVisitStats(
url=url, url=url,
visit_type="git", visit_type="git",
last_eventful=eventful_date, last_eventful=eventful_date,
last_uneventful=uneventful_date, last_uneventful=uneventful_date,
last_failed=failed_date, last_failed=failed_date,
last_notfound=None, last_notfound=None,
) )
assert failed_visit == expected_visit_stats assert failed_visits == [expected_visit_stats]
def test_origin_visit_stats_upsert_with_snapshot(self, swh_scheduler) -> None: def test_origin_visit_stats_upsert_with_snapshot(self, swh_scheduler) -> None:
eventful_date = utcnow() eventful_date = utcnow()
url = "https://github.com/666/test" url = "https://github.com/666/test"
visit_stats = OriginVisitStats( visit_stats = OriginVisitStats(
url=url, url=url,
visit_type="git", visit_type="git",
last_eventful=eventful_date, last_eventful=eventful_date,
last_uneventful=None, last_uneventful=None,
last_failed=None, last_failed=None,
last_notfound=None, last_notfound=None,
last_snapshot=hash_to_bytes("d81cc0710eb6cf9efd5b920a8453e1e07157b6cd"), last_snapshot=hash_to_bytes("d81cc0710eb6cf9efd5b920a8453e1e07157b6cd"),
) )
swh_scheduler.origin_visit_stats_upsert([visit_stats]) swh_scheduler.origin_visit_stats_upsert([visit_stats])
assert swh_scheduler.origin_visit_stats_get(url, "git") == visit_stats assert swh_scheduler.origin_visit_stats_get([(url, "git")]) == [visit_stats]
assert swh_scheduler.origin_visit_stats_get(url, "svn") is None assert swh_scheduler.origin_visit_stats_get([(url, "svn")]) == []
def test_origin_visit_stats_upsert_messing_with_time(self, swh_scheduler) -> None: def test_origin_visit_stats_upsert_messing_with_time(self, swh_scheduler) -> None:
url = "interesting-origin" url = "interesting-origin"
# Let's play with dates... # Let's play with dates...
date2 = utcnow() date2 = utcnow()
date1 = date2 - ONEDAY date1 = date2 - ONEDAY
date0 = date1 - ONEDAY date0 = date1 - ONEDAY
assert date0 < date1 < date2 assert date0 < date1 < date2
snapshot2 = hash_to_bytes("d81cc0710eb6cf9efd5b920a8453e1e07157b6cd") snapshot2 = hash_to_bytes("d81cc0710eb6cf9efd5b920a8453e1e07157b6cd")
snapshot0 = hash_to_bytes("fffcc0710eb6cf9efd5b920a8453e1e07157bfff") snapshot0 = hash_to_bytes("fffcc0710eb6cf9efd5b920a8453e1e07157bfff")
visit_stats0 = OriginVisitStats( visit_stats0 = OriginVisitStats(
url=url, url=url,
visit_type="git", visit_type="git",
last_eventful=date2, last_eventful=date2,
last_uneventful=None, last_uneventful=None,
last_failed=None, last_failed=None,
last_notfound=None, last_notfound=None,
last_snapshot=snapshot2, last_snapshot=snapshot2,
) )
swh_scheduler.origin_visit_stats_upsert([visit_stats0]) swh_scheduler.origin_visit_stats_upsert([visit_stats0])
actual_visit_stats0 = swh_scheduler.origin_visit_stats_get(url, "git") actual_visit_stats0 = swh_scheduler.origin_visit_stats_get([(url, "git")])[0]
assert actual_visit_stats0 == visit_stats0 assert actual_visit_stats0 == visit_stats0
visit_stats2 = OriginVisitStats( visit_stats2 = OriginVisitStats(
url=url, url=url,
visit_type="git", visit_type="git",
last_eventful=None, last_eventful=None,
last_uneventful=date1, last_uneventful=date1,
last_notfound=None, last_notfound=None,
last_failed=None, last_failed=None,
) )
swh_scheduler.origin_visit_stats_upsert([visit_stats2]) swh_scheduler.origin_visit_stats_upsert([visit_stats2])
actual_visit_stats2 = swh_scheduler.origin_visit_stats_get(url, "git") actual_visit_stats2 = swh_scheduler.origin_visit_stats_get([(url, "git")])[0]
assert actual_visit_stats2 == attr.evolve( assert actual_visit_stats2 == attr.evolve(
actual_visit_stats0, last_uneventful=date1 actual_visit_stats0, last_uneventful=date1
) )
# a past date, what happens? # a past date, what happens?
# date0 < date2 so this ovs should be dismissed # date0 < date2 so this ovs should be dismissed
# the "eventful" associated snapshot should be dismissed as well # the "eventful" associated snapshot should be dismissed as well
visit_stats1 = OriginVisitStats( visit_stats1 = OriginVisitStats(
url=url, url=url,
visit_type="git", visit_type="git",
last_eventful=date0, last_eventful=date0,
last_uneventful=None, last_uneventful=None,
last_failed=None, last_failed=None,
last_notfound=None, last_notfound=None,
last_snapshot=snapshot0, last_snapshot=snapshot0,
) )
swh_scheduler.origin_visit_stats_upsert([visit_stats1]) swh_scheduler.origin_visit_stats_upsert([visit_stats1])
actual_visit_stats1 = swh_scheduler.origin_visit_stats_get(url, "git") actual_visit_stats1 = swh_scheduler.origin_visit_stats_get([(url, "git")])[0]
assert actual_visit_stats1 == attr.evolve( assert actual_visit_stats1 == attr.evolve(
actual_visit_stats2, last_eventful=date2 actual_visit_stats2, last_eventful=date2
) )
def test_origin_visit_stats_upsert_batch(self, swh_scheduler) -> None: def test_origin_visit_stats_upsert_batch(self, swh_scheduler) -> None:
"""Batch upsert is ok""" """Batch upsert is ok"""
visit_stats = [ visit_stats = [
Show All 14 Lines def test_origin_visit_stats_upsert_batch(self, swh_scheduler) -> None:
last_notfound=None, last_notfound=None,
last_failed=None, last_failed=None,
last_snapshot=hash_to_bytes("fffcc0710eb6cf9efd5b920a8453e1e07157bfff"), last_snapshot=hash_to_bytes("fffcc0710eb6cf9efd5b920a8453e1e07157bfff"),
), ),
] ]
swh_scheduler.origin_visit_stats_upsert(visit_stats) swh_scheduler.origin_visit_stats_upsert(visit_stats)
for visit_stat in visit_stats: for visit_stat in swh_scheduler.origin_visit_stats_get(
assert ( [(vs.url, vs.visit_type) for vs in visit_stats]
swh_scheduler.origin_visit_stats_get( ):
visit_stat.url, visit_stat.visit_type assert visit_stat is not None
)
is not None
)
def test_origin_visit_stats_upsert_cardinality_failing(self, swh_scheduler) -> None: def test_origin_visit_stats_upsert_cardinality_failing(self, swh_scheduler) -> None:
"""Batch upsert does not support altering multiple times the same origin-visit-status """Batch upsert does not support altering multiple times the same origin-visit-status
""" """
with pytest.raises(SchedulerException, match="CardinalityViolation"): with pytest.raises(SchedulerException, match="CardinalityViolation"):
swh_scheduler.origin_visit_stats_upsert( swh_scheduler.origin_visit_stats_upsert(
[ [
▲ Show 20 Lines • Show All 169 Lines • Show Last 20 Lines
About · Developer information · Legal