Differential D5978 Diff 21544 swh/scheduler/tests/test_journal_client.py

Changeset View

Standalone View

swh/scheduler/tests/test_journal_client.py

Show First 20 Lines • Show All 124 Lines • ▼ Show 20 Lines	def test_journal_client_ignore_missing_type(swh_scheduler):
process_journal_objects(		process_journal_objects(
{"origin_visit_status": visit_statuses}, scheduler=swh_scheduler		{"origin_visit_status": visit_statuses}, scheduler=swh_scheduler
)		)

# The message has been ignored: no stats have been upserted		# The message has been ignored: no stats have been upserted
swh_scheduler.origin_visit_stats_upsert.assert_not_called()		swh_scheduler.origin_visit_stats_upsert.assert_not_called()


def assert_visit_stats_ok(actual_visit_stats, expected_visit_stats):		def assert_visit_stats_ok(
		actual_visit_stats, expected_visit_stats, compare_successive_visits=True
		):
"""Utility test function to ensure visits stats read from the backend are in the right		"""Utility test function to ensure visits stats read from the backend are in the right
shape. The comparison on the next_visit_queue_position will be dealt with in		shape. The comparison on the next_visit_queue_position will be dealt with in
dedicated tests so it's not tested in tests that are calling this function.		dedicated tests so it's not tested in tests that are calling this function.

"""		"""
assert len(actual_visit_stats) == len(expected_visit_stats)		assert len(actual_visit_stats) == len(expected_visit_stats)

for visit_stats in actual_visit_stats:		for visit_stats in actual_visit_stats:
visit_stats = attr.evolve(visit_stats, next_visit_queue_position=None)		visit_stats = attr.evolve(visit_stats, next_visit_queue_position=None)
		if not compare_successive_visits:
		visit_stats = attr.evolve(visit_stats, successive_visits=0)

assert visit_stats in expected_visit_stats		assert visit_stats in expected_visit_stats


def test_journal_client_origin_visit_status_from_journal_last_notfound(swh_scheduler):		def test_journal_client_origin_visit_status_from_journal_last_notfound(swh_scheduler):
visit_status = {		visit_status = {
"origin": "foo",		"origin": "foo",
"visit": 1,		"visit": 1,
Show All 15 Lines	assert_visit_stats_ok(
url="foo",		url="foo",
visit_type="git",		visit_type="git",
last_eventful=None,		last_eventful=None,
last_uneventful=None,		last_uneventful=None,
last_failed=None,		last_failed=None,
last_notfound=visit_status["date"],		last_notfound=visit_status["date"],
last_snapshot=None,		last_snapshot=None,
next_position_offset=5,		next_position_offset=5,
		successive_visits=1,
)		)
],		],
)		)

visit_statuses = [		visit_statuses = [
{		{
"origin": "foo",		"origin": "foo",
"visit": 3,		"visit": 3,
Show All 24 Lines	assert_visit_stats_ok(
url="foo",		url="foo",
visit_type="git",		visit_type="git",
last_eventful=None,		last_eventful=None,
last_uneventful=None,		last_uneventful=None,
last_failed=None,		last_failed=None,
last_notfound=DATE3,		last_notfound=DATE3,
last_snapshot=None,		last_snapshot=None,
next_position_offset=7,		next_position_offset=7,
		successive_visits=3,
)		)
],		],
)		)


def test_journal_client_origin_visit_status_from_journal_last_failed(swh_scheduler):		def test_journal_client_origin_visit_status_from_journal_last_failed(swh_scheduler):
visit_statuses = [		visit_statuses = [
{		{
▲ Show 20 Lines • Show All 42 Lines • ▼ Show 20 Lines	assert_visit_stats_ok(
url="bar",		url="bar",
visit_type="git",		visit_type="git",
last_eventful=None,		last_eventful=None,
last_uneventful=None,		last_uneventful=None,
last_failed=DATE3,		last_failed=DATE3,
last_notfound=None,		last_notfound=None,
last_snapshot=None,		last_snapshot=None,
next_position_offset=7,		next_position_offset=7,
		successive_visits=3,
)		)
],		],
)		)


def test_journal_client_origin_visit_status_from_journal_last_failed2(swh_scheduler):		def test_journal_client_origin_visit_status_from_journal_last_failed2(swh_scheduler):
visit_statuses = [		visit_statuses = [
{		{
Show All 26 Lines	assert_visit_stats_ok(
url="bar",		url="bar",
visit_type="git",		visit_type="git",
last_eventful=None,		last_eventful=None,
last_uneventful=None,		last_uneventful=None,
last_failed=DATE2,		last_failed=DATE2,
last_notfound=None,		last_notfound=None,
last_snapshot=None,		last_snapshot=None,
next_position_offset=6,		next_position_offset=6,
		successive_visits=2,
)		)
],		],
)		)


def test_journal_client_origin_visit_status_from_journal_last_eventful(swh_scheduler):		def test_journal_client_origin_visit_status_from_journal_last_eventful(swh_scheduler):
visit_statuses = [		visit_statuses = [
{		{
▲ Show 20 Lines • Show All 42 Lines • ▼ Show 20 Lines	assert_visit_stats_ok(
url="foo",		url="foo",
visit_type="git",		visit_type="git",
last_eventful=DATE3,		last_eventful=DATE3,
last_uneventful=None,		last_uneventful=None,
last_failed=None,		last_failed=None,
last_notfound=None,		last_notfound=None,
last_snapshot=hash_to_bytes("dddcc0710eb6cf9efd5b920a8453e1e07157bddd"),		last_snapshot=hash_to_bytes("dddcc0710eb6cf9efd5b920a8453e1e07157bddd"),
next_position_offset=0,		next_position_offset=0,
		successive_visits=3,
)		)
],		],
)		)


def test_journal_client_origin_visit_status_from_journal_last_uneventful(swh_scheduler):		def test_journal_client_origin_visit_status_from_journal_last_uneventful(swh_scheduler):
visit_status = {		visit_status = {
"origin": "foo",		"origin": "foo",
Show All 12 Lines	swh_scheduler.origin_visit_stats_upsert(
visit_type=visit_status["type"],		visit_type=visit_status["type"],
last_eventful=DATE1,		last_eventful=DATE1,
last_uneventful=DATE3,		last_uneventful=DATE3,
last_failed=DATE2,		last_failed=DATE2,
last_notfound=DATE1,		last_notfound=DATE1,
last_snapshot=visit_status["snapshot"],		last_snapshot=visit_status["snapshot"],
next_visit_queue_position=None,		next_visit_queue_position=None,
next_position_offset=4,		next_position_offset=4,
		successive_visits=1,
)		)
]		]
)		)

process_journal_objects(		process_journal_objects(
{"origin_visit_status": [visit_status]}, scheduler=swh_scheduler		{"origin_visit_status": [visit_status]}, scheduler=swh_scheduler
)		)

actual_origin_visit_stats = swh_scheduler.origin_visit_stats_get(		actual_origin_visit_stats = swh_scheduler.origin_visit_stats_get(
[(visit_status["origin"], visit_status["type"])]		[(visit_status["origin"], visit_status["type"])]
)		)

assert_visit_stats_ok(		assert_visit_stats_ok(
actual_origin_visit_stats,		actual_origin_visit_stats,
[		[
OriginVisitStats(		OriginVisitStats(
url=visit_status["origin"],		url=visit_status["origin"],
visit_type=visit_status["type"],		visit_type=visit_status["type"],
last_eventful=DATE1,		last_eventful=DATE1,
last_uneventful=visit_status["date"], # most recent date but uneventful		last_uneventful=visit_status["date"], # most recent date but uneventful
last_failed=DATE2,		last_failed=DATE2,
last_notfound=DATE1,		last_notfound=DATE1,
last_snapshot=visit_status["snapshot"],		last_snapshot=visit_status["snapshot"],
next_position_offset=5, # uneventful so visit less often		next_position_offset=5, # uneventful so visit less often
		successive_visits=2,
)		)
],		],
)		)


VISIT_STATUSES = [		VISIT_STATUSES = [
{*ovs, "date": DATE1 + n ONE_DAY}		{*ovs, "date": DATE1 + n ONE_DAY}
for n, ovs in enumerate(		for n, ovs in enumerate(
▲ Show 20 Lines • Show All 50 Lines • ▼ Show 20 Lines	assert_visit_stats_ok(
url="foo",		url="foo",
visit_type="git",		visit_type="git",
last_eventful=DATE1 + ONE_DAY,		last_eventful=DATE1 + ONE_DAY,
last_uneventful=DATE1 + 3 * ONE_DAY,		last_uneventful=DATE1 + 3 * ONE_DAY,
last_failed=None,		last_failed=None,
last_notfound=None,		last_notfound=None,
last_snapshot=hash_to_bytes("d81cc0710eb6cf9efd5b920a8453e1e07157b6cd"),		last_snapshot=hash_to_bytes("d81cc0710eb6cf9efd5b920a8453e1e07157b6cd"),
next_position_offset=5, # uneventful, visit origin less often in future		next_position_offset=5, # uneventful, visit origin less often in future
		successive_visits=1,
)		)
],		],
)		)


VISIT_STATUSES_1 = [		VISIT_STATUSES_1 = [
{*ovs, "date": DATE1 + n ONE_DAY}		{*ovs, "date": DATE1 + n ONE_DAY}
for n, ovs in enumerate(		for n, ovs in enumerate(
▲ Show 20 Lines • Show All 48 Lines • ▼ Show 20 Lines	def test_journal_client_origin_visit_status_permutation1(visit_statuses, swh_scheduler):
assert actual_visit_stats.visit_type == "hg"		assert actual_visit_stats.visit_type == "hg"
assert actual_visit_stats.last_eventful == DATE1 + 2 * ONE_DAY		assert actual_visit_stats.last_eventful == DATE1 + 2 * ONE_DAY
assert actual_visit_stats.last_uneventful == DATE1 + 3 * ONE_DAY		assert actual_visit_stats.last_uneventful == DATE1 + 3 * ONE_DAY
assert actual_visit_stats.last_failed is None		assert actual_visit_stats.last_failed is None
assert actual_visit_stats.last_notfound is None		assert actual_visit_stats.last_notfound is None
assert actual_visit_stats.last_snapshot == hash_to_bytes(		assert actual_visit_stats.last_snapshot == hash_to_bytes(
"aaaaaabbbeb6cf9efd5b920a8453e1e07157b6cd"		"aaaaaabbbeb6cf9efd5b920a8453e1e07157b6cd"
)		)
		assert actual_visit_stats.successive_visits == 1


VISIT_STATUSES_2 = [		VISIT_STATUSES_2 = [
{*ovs, "date": DATE1 + n ONE_DAY}		{*ovs, "date": DATE1 + n ONE_DAY}
for n, ovs in enumerate(		for n, ovs in enumerate(
[		[
{		{
"origin": "cavabarder",		"origin": "cavabarder",
▲ Show 20 Lines • Show All 127 Lines • ▼ Show 20 Lines	assert_visit_stats_ok(
OriginVisitStats(		OriginVisitStats(
url="foo",		url="foo",
visit_type="git",		visit_type="git",
last_eventful=DATE1,		last_eventful=DATE1,
last_uneventful=None,		last_uneventful=None,
last_failed=None,		last_failed=None,
last_notfound=None,		last_notfound=None,
last_snapshot=hash_to_bytes("aaaaaabbbeb6cf9efd5b920a8453e1e07157b6cd"),		last_snapshot=hash_to_bytes("aaaaaabbbeb6cf9efd5b920a8453e1e07157b6cd"),
		next_position_offset=4,
		successive_visits=1,
)		)
],		],
)		)


def test_journal_client_origin_visit_status_several_upsert(swh_scheduler):		def test_journal_client_origin_visit_status_several_upsert(swh_scheduler):
"""An old message updates old information		"""An old message updates old information

Show All 32 Lines	assert_visit_stats_ok(
url="foo",		url="foo",
visit_type="git",		visit_type="git",
last_eventful=DATE1,		last_eventful=DATE1,
last_uneventful=DATE2,		last_uneventful=DATE2,
last_failed=None,		last_failed=None,
last_notfound=None,		last_notfound=None,
last_snapshot=hash_to_bytes("aaaaaabbbeb6cf9efd5b920a8453e1e07157b6cd"),		last_snapshot=hash_to_bytes("aaaaaabbbeb6cf9efd5b920a8453e1e07157b6cd"),
next_position_offset=5,		next_position_offset=5,
		successive_visits=1,
)		)
],		],
)		)


VISIT_STATUSES_SAME_SNAPSHOT = [		VISIT_STATUSES_SAME_SNAPSHOT = [
{*ovs, "date": DATE1 + n ONE_YEAR}		{*ovs, "date": DATE1 + n ONE_YEAR}
for n, ovs in enumerate(		for n, ovs in enumerate(
▲ Show 20 Lines • Show All 50 Lines • ▼ Show 20 Lines	assert_visit_stats_ok(
last_eventful=DATE1,		last_eventful=DATE1,
last_uneventful=DATE1 + 2 * ONE_YEAR,		last_uneventful=DATE1 + 2 * ONE_YEAR,
last_failed=None,		last_failed=None,
last_notfound=None,		last_notfound=None,
last_snapshot=hash_to_bytes("aaaaaabbbeb6cf9efd5b920a8453e1e07157b6cd"),		last_snapshot=hash_to_bytes("aaaaaabbbeb6cf9efd5b920a8453e1e07157b6cd"),
next_position_offset=6, # 2 uneventful visits, whatever the permutation		next_position_offset=6, # 2 uneventful visits, whatever the permutation
)		)
],		],
		compare_successive_visits=False, # depending on permutations, result change
)		)


@pytest.mark.parametrize(		@pytest.mark.parametrize(
"position_offset, interval",		"position_offset, interval",
[		[
(0, 1),		(0, 1),
(1, 1),		(1, 1),
▲ Show 20 Lines • Show All 92 Lines • Show Last 20 Lines