Differential D5919 Diff 21363 swh/scheduler/tests/test_journal_client.py

Changeset View

Standalone View

swh/scheduler/tests/test_journal_client.py

Show First 20 Lines • Show All 141 Lines • ▼ Show 20 Lines	assert actual_origin_visit_stats == [
OriginVisitStats(		OriginVisitStats(
url="foo",		url="foo",
visit_type="git",		visit_type="git",
last_eventful=None,		last_eventful=None,
last_uneventful=None,		last_uneventful=None,
last_failed=None,		last_failed=None,
last_notfound=visit_status["date"],		last_notfound=visit_status["date"],
last_snapshot=None,		last_snapshot=None,
		next_visit_queue_position=None,
		next_position_offset=5,
)		)
]		]

visit_statuses = [		visit_statuses = [
{		{
"origin": "foo",		"origin": "foo",
"visit": 3,		"visit": 3,
"status": "not_found",		"status": "not_found",
Show All 20 Lines	assert actual_origin_visit_stats == [
OriginVisitStats(		OriginVisitStats(
url="foo",		url="foo",
visit_type="git",		visit_type="git",
last_eventful=None,		last_eventful=None,
last_uneventful=None,		last_uneventful=None,
last_failed=None,		last_failed=None,
last_notfound=DATE3,		last_notfound=DATE3,
last_snapshot=None,		last_snapshot=None,
		next_visit_queue_position=None,
		next_position_offset=7,
)		)
]		]


def test_journal_client_origin_visit_status_from_journal_last_failed(swh_scheduler):		def test_journal_client_origin_visit_status_from_journal_last_failed(swh_scheduler):
visit_statuses = [		visit_statuses = [
{		{
"origin": "foo",		"origin": "foo",
Show All 38 Lines	assert actual_origin_visit_stats == [
OriginVisitStats(		OriginVisitStats(
url="bar",		url="bar",
visit_type="git",		visit_type="git",
last_eventful=None,		last_eventful=None,
last_uneventful=None,		last_uneventful=None,
last_failed=DATE3,		last_failed=DATE3,
last_notfound=None,		last_notfound=None,
last_snapshot=None,		last_snapshot=None,
		next_visit_queue_position=None,
		next_position_offset=7,
)		)
]		]


def test_journal_client_origin_visit_status_from_journal_last_failed2(swh_scheduler):		def test_journal_client_origin_visit_status_from_journal_last_failed2(swh_scheduler):
visit_statuses = [		visit_statuses = [
{		{
"origin": "bar",		"origin": "bar",
Show All 22 Lines	assert actual_origin_visit_stats == [
OriginVisitStats(		OriginVisitStats(
url="bar",		url="bar",
visit_type="git",		visit_type="git",
last_eventful=None,		last_eventful=None,
last_uneventful=None,		last_uneventful=None,
last_failed=DATE2,		last_failed=DATE2,
last_notfound=None,		last_notfound=None,
last_snapshot=None,		last_snapshot=None,
		next_visit_queue_position=None,
		next_position_offset=6,
)		)
]		]


def test_journal_client_origin_visit_status_from_journal_last_eventful(swh_scheduler):		def test_journal_client_origin_visit_status_from_journal_last_eventful(swh_scheduler):
visit_statuses = [		visit_statuses = [
{		{
"origin": "bar",		"origin": "bar",
Show All 38 Lines	assert actual_origin_visit_stats == [
OriginVisitStats(		OriginVisitStats(
url="foo",		url="foo",
visit_type="git",		visit_type="git",
last_eventful=DATE3,		last_eventful=DATE3,
last_uneventful=None,		last_uneventful=None,
last_failed=None,		last_failed=None,
last_notfound=None,		last_notfound=None,
last_snapshot=hash_to_bytes("dddcc0710eb6cf9efd5b920a8453e1e07157bddd"),		last_snapshot=hash_to_bytes("dddcc0710eb6cf9efd5b920a8453e1e07157bddd"),
		next_visit_queue_position=None,
		next_position_offset=0,
)		)
]		]


def test_journal_client_origin_visit_status_from_journal_last_uneventful(swh_scheduler):		def test_journal_client_origin_visit_status_from_journal_last_uneventful(swh_scheduler):
visit_status = {		visit_status = {
"origin": "foo",		"origin": "foo",
"visit": 1,		"visit": 1,
Show All 9 Lines	swh_scheduler.origin_visit_stats_upsert(
OriginVisitStats(		OriginVisitStats(
url=visit_status["origin"],		url=visit_status["origin"],
visit_type=visit_status["type"],		visit_type=visit_status["type"],
last_eventful=DATE1,		last_eventful=DATE1,
last_uneventful=DATE3,		last_uneventful=DATE3,
last_failed=DATE2,		last_failed=DATE2,
last_notfound=DATE1,		last_notfound=DATE1,
last_snapshot=visit_status["snapshot"],		last_snapshot=visit_status["snapshot"],
		next_visit_queue_position=None,
		next_position_offset=4,
)		)
]		]
)		)

process_journal_objects(		process_journal_objects(
{"origin_visit_status": [visit_status]}, scheduler=swh_scheduler		{"origin_visit_status": [visit_status]}, scheduler=swh_scheduler
)		)

actual_origin_visit_stats = swh_scheduler.origin_visit_stats_get(		actual_origin_visit_stats = swh_scheduler.origin_visit_stats_get(
[(visit_status["origin"], visit_status["type"])]		[(visit_status["origin"], visit_status["type"])]
)		)
assert actual_origin_visit_stats == [		assert actual_origin_visit_stats == [
OriginVisitStats(		OriginVisitStats(
url=visit_status["origin"],		url=visit_status["origin"],
visit_type=visit_status["type"],		visit_type=visit_status["type"],
last_eventful=DATE1,		last_eventful=DATE1,
last_uneventful=visit_status["date"], # most recent date but uneventful		last_uneventful=visit_status["date"], # most recent date but uneventful
last_failed=DATE2,		last_failed=DATE2,
last_notfound=DATE1,		last_notfound=DATE1,
last_snapshot=visit_status["snapshot"],		last_snapshot=visit_status["snapshot"],
		next_visit_queue_position=None,
		next_position_offset=5, # uneventful so visit less often
)		)
]		]


VISIT_STATUSES = [		VISIT_STATUSES = [
{*ovs, "date": DATE1 + n ONE_DAY}		{*ovs, "date": DATE1 + n ONE_DAY}
for n, ovs in enumerate(		for n, ovs in enumerate(
[		[
▲ Show 20 Lines • Show All 44 Lines • ▼ Show 20 Lines	def test_journal_client_origin_visit_status_permutation0(visit_statuses, swh_scheduler):
expected_visit_stats = OriginVisitStats(		expected_visit_stats = OriginVisitStats(
url="foo",		url="foo",
visit_type="git",		visit_type="git",
last_eventful=DATE1 + ONE_DAY,		last_eventful=DATE1 + ONE_DAY,
last_uneventful=DATE1 + 3 * ONE_DAY,		last_uneventful=DATE1 + 3 * ONE_DAY,
last_failed=None,		last_failed=None,
last_notfound=None,		last_notfound=None,
last_snapshot=hash_to_bytes("d81cc0710eb6cf9efd5b920a8453e1e07157b6cd"),		last_snapshot=hash_to_bytes("d81cc0710eb6cf9efd5b920a8453e1e07157b6cd"),
		next_visit_queue_position=None,
		next_position_offset=5, # uneventful, visit origin less often in the future
)		)

assert swh_scheduler.origin_visit_stats_get([("foo", "git")]) == [		assert swh_scheduler.origin_visit_stats_get([("foo", "git")]) == [
expected_visit_stats		expected_visit_stats
]		]


VISIT_STATUSES_1 = [		VISIT_STATUSES_1 = [
▲ Show 20 Lines • Show All 197 Lines • ▼ Show 20 Lines	def test_journal_client_origin_visit_status_duplicated_messages(swh_scheduler):
)		)

assert swh_scheduler.origin_visit_stats_get([("foo", "git")]) == [		assert swh_scheduler.origin_visit_stats_get([("foo", "git")]) == [
expected_visit_stats		expected_visit_stats
]		]


def test_journal_client_origin_visit_status_several_upsert(swh_scheduler):		def test_journal_client_origin_visit_status_several_upsert(swh_scheduler):
"""A duplicated message must be ignored		"""An old message updates old information

"""		"""
visit_status1 = {		visit_status1 = {
"origin": "foo",		"origin": "foo",
"visit": 1,		"visit": 1,
"status": "full",		"status": "full",
"date": DATE1,		"date": DATE1,
"type": "git",		"type": "git",
Show All 12 Lines	def test_journal_client_origin_visit_status_several_upsert(swh_scheduler):
process_journal_objects(		process_journal_objects(
{"origin_visit_status": [visit_status2]}, scheduler=swh_scheduler		{"origin_visit_status": [visit_status2]}, scheduler=swh_scheduler
)		)

process_journal_objects(		process_journal_objects(
{"origin_visit_status": [visit_status1]}, scheduler=swh_scheduler		{"origin_visit_status": [visit_status1]}, scheduler=swh_scheduler
)		)

expected_visit_stats = OriginVisitStats(		assert swh_scheduler.origin_visit_stats_get([("foo", "git")]) == [
		OriginVisitStats(
url="foo",		url="foo",
visit_type="git",		visit_type="git",
last_eventful=DATE1,		last_eventful=DATE1,
last_uneventful=DATE2,		last_uneventful=DATE2,
last_failed=None,		last_failed=None,
last_notfound=None,		last_notfound=None,
last_snapshot=hash_to_bytes("aaaaaabbbeb6cf9efd5b920a8453e1e07157b6cd"),		last_snapshot=hash_to_bytes("aaaaaabbbeb6cf9efd5b920a8453e1e07157b6cd"),
		next_visit_queue_position=None,
		next_position_offset=5,
)		)

assert swh_scheduler.origin_visit_stats_get([("foo", "git")]) == [
expected_visit_stats
]		]


VISIT_STATUSES_SAME_SNAPSHOT = [		VISIT_STATUSES_SAME_SNAPSHOT = [
{*ovs, "date": DATE1 + n ONE_YEAR}		{*ovs, "date": DATE1 + n ONE_YEAR}
for n, ovs in enumerate(		for n, ovs in enumerate(
[		[
{		{
Show All 31 Lines
):		):
"""Ensure out of order topic subscription ends up in the same final state		"""Ensure out of order topic subscription ends up in the same final state

"""		"""
process_journal_objects(		process_journal_objects(
{"origin_visit_status": visit_statuses}, scheduler=swh_scheduler		{"origin_visit_status": visit_statuses}, scheduler=swh_scheduler
)		)

expected_visit_stats = OriginVisitStats(		assert swh_scheduler.origin_visit_stats_get([("cavabarder", "hg")]) == [
		OriginVisitStats(
url="cavabarder",		url="cavabarder",
visit_type="hg",		visit_type="hg",
last_eventful=DATE1,		last_eventful=DATE1,
last_uneventful=DATE1 + 2 * ONE_YEAR,		last_uneventful=DATE1 + 2 * ONE_YEAR,
last_failed=None,		last_failed=None,
last_notfound=None,		last_notfound=None,
last_snapshot=hash_to_bytes("aaaaaabbbeb6cf9efd5b920a8453e1e07157b6cd"),		last_snapshot=hash_to_bytes("aaaaaabbbeb6cf9efd5b920a8453e1e07157b6cd"),
		next_visit_queue_position=None,
		next_position_offset=6, # 2 uneventful visits, whatever the permutation
)		)

assert swh_scheduler.origin_visit_stats_get([("cavabarder", "hg")]) == [
expected_visit_stats
]		]