Page MenuHomeSoftware Heritage
Differential D4894 Diff 17446 swh/scheduler/tests/test_journal_client.py

Changeset View

Standalone View

View Options

swh/scheduler/tests/test_journal_client.py

Show First 20 Lines • Show All 105 Lines • ▼ Show 20 Lines visit_status = {
"type": "git", "type": "git",
"snapshot": None, "snapshot": None,
} }
process_journal_objects( process_journal_objects(
{"origin_visit_status": [visit_status]}, scheduler=swh_scheduler {"origin_visit_status": [visit_status]}, scheduler=swh_scheduler
) )
actual_origin_visit_stats = swh_scheduler.origin_visit_stats_get( actual_origin_visit_stats = swh_scheduler.origin_visit_stats_get("foo", "git")
visit_status["origin"], visit_status["type"]
)
assert actual_origin_visit_stats == OriginVisitStats( assert actual_origin_visit_stats == OriginVisitStats(
url=visit_status["origin"], url="foo",
visit_type=visit_status["type"], visit_type="git",
last_eventful=None, last_eventful=None,
last_uneventful=None, last_uneventful=None,
last_failed=None, last_failed=None,
last_notfound=visit_status["date"], last_notfound=visit_status["date"],
last_snapshot=None, last_snapshot=None,
) )
visit_statuses = [ visit_statuses = [
{ {
"origin": "foo", "origin": "foo",
"visit": 4, "visit": 3,
"status": "not_found", "status": "not_found",
"date": DATE3, "date": DATE2,
"type": "git", "type": "git",
"snapshot": None, "snapshot": None,
}, },
{ {
"origin": "foo", "origin": "foo",
"visit": 3, "visit": 4,
"status": "not_found", "status": "not_found",
"date": DATE2, "date": DATE3,
"type": "git", "type": "git",
"snapshot": None, "snapshot": None,
}, },
] ]
process_journal_objects( process_journal_objects(
{"origin_visit_status": visit_statuses}, scheduler=swh_scheduler {"origin_visit_status": visit_statuses}, scheduler=swh_scheduler
) )
for visit_status in visit_statuses: actual_origin_visit_stats = swh_scheduler.origin_visit_stats_get("foo", "git")
actual_origin_visit_stats = swh_scheduler.origin_visit_stats_get(
visit_status["origin"], visit_status["type"]
)
assert actual_origin_visit_stats is not None assert actual_origin_visit_stats is not None
assert actual_origin_visit_stats == OriginVisitStats( assert actual_origin_visit_stats == OriginVisitStats(
url=visit_status["origin"], url="foo",
visit_type=visit_status["type"], visit_type="git",
last_eventful=None, last_eventful=None,
last_uneventful=None, last_uneventful=None,
last_failed=None, last_failed=None,
last_notfound=DATE3, last_notfound=DATE3,
last_snapshot=None, last_snapshot=None,
) )
def test_journal_client_origin_visit_status_from_journal_last_failed(swh_scheduler): def test_journal_client_origin_visit_status_from_journal_last_failed(swh_scheduler):
visit_statuses = [ visit_statuses = [
{ {
"origin": "foo", "origin": "foo",
"visit": 1, "visit": 1,
"status": "partial", "status": "partial",
"date": utcnow(), "date": utcnow(),
"type": "git", "type": "git",
"snapshot": None, "snapshot": None,
}, },
{ {
"origin": "bar", "origin": "bar",
"visit": 2, "visit": 1,
"status": "full", "status": "full",
"date": DATE1, "date": DATE1,
"type": "git", "type": "git",
"snapshot": None, "snapshot": None,
}, },
]
process_journal_objects(
{"origin_visit_status": visit_statuses}, scheduler=swh_scheduler
)
# Ensure those visit status are ignored
for visit_status in visit_statuses:
actual_origin_visit_stats = swh_scheduler.origin_visit_stats_get(
visit_status["origin"], visit_status["type"]
)
assert actual_origin_visit_stats is not None
assert actual_origin_visit_stats == OriginVisitStats(
url=visit_status["origin"],
visit_type=visit_status["type"],
last_eventful=None,
last_uneventful=None,
last_failed=visit_status["date"],
last_notfound=None,
last_snapshot=None,
)
visit_statuses = [
{ {
"origin": "bar", "origin": "bar",
"visit": 3, "visit": 2,
"status": "full", "status": "full",
"date": DATE3, "date": DATE2,
"type": "git", "type": "git",
"snapshot": None, "snapshot": None,
}, },
{ {
"origin": "bar", "origin": "bar",
"visit": 4, "visit": 3,
"status": "full", "status": "full",
"date": DATE2, "date": DATE3,
"type": "git", "type": "git",
"snapshot": None, "snapshot": None,
}, },
] ]
process_journal_objects( process_journal_objects(
{"origin_visit_status": visit_statuses}, scheduler=swh_scheduler {"origin_visit_status": visit_statuses}, scheduler=swh_scheduler
) )
for visit_status in visit_statuses: actual_origin_visit_stats = swh_scheduler.origin_visit_stats_get("bar", "git")
actual_origin_visit_stats = swh_scheduler.origin_visit_stats_get(
visit_status["origin"], visit_status["type"]
)
assert actual_origin_visit_stats is not None assert actual_origin_visit_stats is not None
assert actual_origin_visit_stats == OriginVisitStats( assert actual_origin_visit_stats == OriginVisitStats(
url=visit_status["origin"], url="bar",
visit_type=visit_status["type"], visit_type="git",
last_eventful=None, last_eventful=None,
last_uneventful=None, last_uneventful=None,
last_failed=DATE3, last_failed=DATE3,
last_notfound=None, last_notfound=None,
last_snapshot=None, last_snapshot=None,
) )
def test_journal_client_origin_visit_status_from_journal_last_eventful(swh_scheduler): def test_journal_client_origin_visit_status_from_journal_last_eventful(swh_scheduler):
visit_statuses = [ visit_statuses = [
{ {
"origin": "bar", "origin": "bar",
"visit": 1, "visit": 1,
"status": "partial", "status": "partial",
"date": utcnow(), "date": utcnow(),
"type": "git", "type": "git",
"snapshot": hash_to_bytes("d81cc0710eb6cf9efd5b920a8453e1e07157b6cd"), "snapshot": hash_to_bytes("d81cc0710eb6cf9efd5b920a8453e1e07157b6cd"),
}, },
{ {
"origin": "foo", "origin": "foo",
"visit": 2, "visit": 1,
"status": "full", "status": "full",
"date": DATE1, "date": DATE1,
"type": "git", "type": "git",
"snapshot": hash_to_bytes("eeecc0710eb6cf9efd5b920a8453e1e07157bfff"), "snapshot": hash_to_bytes("eeecc0710eb6cf9efd5b920a8453e1e07157bfff"),
}, },
]
process_journal_objects(
{"origin_visit_status": visit_statuses}, scheduler=swh_scheduler
)
for visit_status in visit_statuses:
actual_origin_visit_stats = swh_scheduler.origin_visit_stats_get(
visit_status["origin"], visit_status["type"]
)
assert actual_origin_visit_stats is not None
assert actual_origin_visit_stats == OriginVisitStats(
url=visit_status["origin"],
visit_type=visit_status["type"],
last_eventful=visit_status["date"],
last_uneventful=None,
last_failed=None,
last_notfound=None,
last_snapshot=visit_status["snapshot"],
)
visit_statuses = [
{ {
"origin": "foo", "origin": "foo",
"visit": 4, "visit": 2,
"status": "full", "status": "partial",
"date": DATE3, "date": DATE2,
"type": "git", "type": "git",
"snapshot": hash_to_bytes("dddcc0710eb6cf9efd5b920a8453e1e07157bddd"), "snapshot": hash_to_bytes("aaacc0710eb6cf9efd5b920a8453e1e07157baaa"),
}, },
{ {
"origin": "foo", "origin": "foo",
"visit": 3, "visit": 3,
"status": "partial", "status": "full",
"date": DATE2, "date": DATE3,
"type": "git", "type": "git",
"snapshot": hash_to_bytes("aaacc0710eb6cf9efd5b920a8453e1e07157baaa"), "snapshot": hash_to_bytes("dddcc0710eb6cf9efd5b920a8453e1e07157bddd"),
}, },
] ]
process_journal_objects( process_journal_objects(
{"origin_visit_status": visit_statuses}, scheduler=swh_scheduler {"origin_visit_status": visit_statuses}, scheduler=swh_scheduler
) )
for visit_status in visit_statuses: actual_origin_visit_stats = swh_scheduler.origin_visit_stats_get("foo", "git")
actual_origin_visit_stats = swh_scheduler.origin_visit_stats_get(
visit_status["origin"], visit_status["type"]
)
assert actual_origin_visit_stats is not None assert actual_origin_visit_stats is not None
assert actual_origin_visit_stats == OriginVisitStats( assert actual_origin_visit_stats == OriginVisitStats(
url=visit_status["origin"], url="foo",
visit_type=visit_status["type"], visit_type="git",
last_eventful=DATE3, last_eventful=DATE3,
last_uneventful=None, last_uneventful=None,
last_failed=None, last_failed=None,
last_notfound=None, last_notfound=None,
last_snapshot=hash_to_bytes("dddcc0710eb6cf9efd5b920a8453e1e07157bddd"), last_snapshot=hash_to_bytes("dddcc0710eb6cf9efd5b920a8453e1e07157bddd"),
) )
def test_journal_client_origin_visit_status_from_journal_last_uneventful(swh_scheduler): def test_journal_client_origin_visit_status_from_journal_last_uneventful(swh_scheduler):
visit_status = { visit_status = {
"origin": "foo", "origin": "foo",
"visit": 1, "visit": 1,
"status": "full", "status": "full",
"date": DATE3 + ONE_DAY, "date": DATE3 + ONE_DAY,
▲ Show 20 Lines • Show All 91 Lines • ▼ Show 20 Lines expected_visit_stats = OriginVisitStats(
last_failed=None, last_failed=None,
last_notfound=None, last_notfound=None,
last_snapshot=hash_to_bytes("d81cc0710eb6cf9efd5b920a8453e1e07157b6cd"), last_snapshot=hash_to_bytes("d81cc0710eb6cf9efd5b920a8453e1e07157b6cd"),
) )
assert swh_scheduler.origin_visit_stats_get("foo", "git") == expected_visit_stats assert swh_scheduler.origin_visit_stats_get("foo", "git") == expected_visit_stats
VISIT_STATUSES1 = [ VISIT_STATUSES_1 = [
{**ovs, "date": DATE1 + n * ONE_DAY} {**ovs, "date": DATE1 + n * ONE_DAY}
for n, ovs in enumerate( for n, ovs in enumerate(
[ [
{ {
"origin": "cavabarder", "origin": "cavabarder",
"type": "hg", "type": "hg",
"visit": 1, "visit": 1,
"status": "partial", "status": "partial",
Show All 21 Lines for n, ovs in enumerate(
"snapshot": hash_to_bytes("aaaaaabbbeb6cf9efd5b920a8453e1e07157b6cd"), "snapshot": hash_to_bytes("aaaaaabbbeb6cf9efd5b920a8453e1e07157b6cd"),
}, },
] ]
) )
] ]
@pytest.mark.parametrize( @pytest.mark.parametrize(
"visit_statuses", permutations(VISIT_STATUSES1, len(VISIT_STATUSES1)) "visit_statuses", permutations(VISIT_STATUSES_1, len(VISIT_STATUSES_1))
) )
def test_journal_client_origin_visit_status_permutation1(visit_statuses, swh_scheduler): def test_journal_client_origin_visit_status_permutation1(visit_statuses, swh_scheduler):
"""Ensure out of order topic subscription ends up in the same final state """Ensure out of order topic subscription ends up in the same final state
""" """
process_journal_objects( process_journal_objects(
{"origin_visit_status": visit_statuses}, scheduler=swh_scheduler {"origin_visit_status": visit_statuses}, scheduler=swh_scheduler
) )
▲ Show 20 Lines • Show All 122 Lines • Show Last 20 Lines
About · Developer information · Legal