Page MenuHomeSoftware Heritage
Differential D4457

test: Migrate parse_swhid test cases to pytest
ClosedPublic
Actions

Authored by ardumont on Nov 10 2020, 4:13 PM.

Details

Reviewers
vlorentz
Group Reviewers
Reviewers
Maniphest Tasks
T2769: Make function swh.model.identifiers.parse_swhid more strict
Commits
rDMOD47946aac0220: test: Migrate parse_swhid test cases to pytest
Summary

First step prior to rework the error messages and add more strictness
to the parse_swhid function.

Related to T2769

Test Plan

tox

Diff Detail

Repository
rDMOD Data model
Branch
master
Lint
No Linters Available
Unit
No Unit Test Coverage
Build Status
Buildable 17010
Build 26249: Phabricator diff pipeline on jenkinsJenkins console · Jenkins
Build 26248: arc lint + arc unit

Event Timeline

ardumont created this revision.Nov 10 2020, 4:13 PM
Herald added a reviewer: Reviewers. · View Herald TranscriptNov 10 2020, 4:13 PM
swh-public-ci added a comment.Nov 10 2020, 4:15 PM

Build is green

Patch application report for D4457 (id=15800)

Rebasing onto 4e3fdc062c...

Current branch diff-target is up to date.
Changes applied before test
commit 355dd50aa0b98ff904f73c37d7b38472ca41198a
Author: Antoine R. Dumont (@ardumont) <ardumont@softwareheritage.org>
Date:   Tue Nov 10 16:11:32 2020 +0100

    test: Check the detailed error message when swhid is invalid
    
    Related to T2769

See https://jenkins.softwareheritage.org/job/DMOD/job/tests-on-diff/161/ for more details.

Harbormaster completed remote builds in B17010: Diff 15800.Nov 10 2020, 4:15 PM
ardumont updated this revision to Diff 15805.Nov 10 2020, 6:37 PM

Drop the string check assertions part

ardumont retitled this revision from test: Check the detailed error message when swhid is invalid to test: Migrate parse_swhid test cases to pytest.Nov 10 2020, 6:37 PM
ardumont edited the summary of this revision. (Show Details)
ardumont edited the test plan for this revision. (Show Details)
swh-public-ci added a comment.Nov 10 2020, 6:38 PM

Build is green

Patch application report for D4457 (id=15805)

Rebasing onto 4e3fdc062c...

Current branch diff-target is up to date.
Changes applied before test
commit 47946aac0220fcfd55c7998908c71a6f402a5ddb
Author: Antoine R. Dumont (@ardumont) <ardumont@softwareheritage.org>
Date:   Tue Nov 10 16:11:32 2020 +0100

    test: Migrate parse_swhid test cases to pytest
    
    Related to T2769

See https://jenkins.softwareheritage.org/job/DMOD/job/tests-on-diff/163/ for more details.

Harbormaster completed remote builds in B17015: Diff 15805.Nov 10 2020, 6:38 PM
ardumont added a child revision: D4458: model.identifiers: Improve error messages in case of invalid SWHIDs.Nov 12 2020, 10:17 AM
vlorentz accepted this revision.Nov 12 2020, 11:17 AM
This revision is now accepted and ready to land.Nov 12 2020, 11:17 AM
Closed by commit rDMOD47946aac0220: test: Migrate parse_swhid test cases to pytest (authored by ardumont). · Explain WhyNov 12 2020, 11:38 AM
This revision was automatically updated to reflect the committed changes.
ardumont added a commit: rDMOD47946aac0220: test: Migrate parse_swhid test cases to pytest.

Revision Contents
Changeset List

PathSizeCoverage (All)Coverage (Touched)
swh/
model/
tests/
103 lines100%
Loading...

Diff 15800

View Options

swh/model/tests/test_identifiers.py

# Copyright (C) 2021 The Software Heritage developers # Copyright (C) 2021 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution # See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version # License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information # See top-level LICENSE file for more information
import datetime import datetime
from datetime import timedelta from datetime import timedelta
import functools import functools
from itertools import permutations from itertools import permutations
import random
from unittest.mock import Mock from unittest.mock import Mock
import attr import attr
import pytest import pytest
from swh.model.hashutil import hash_to_bytes from swh.model.hashutil import hash_to_bytes
from swh.scheduler.journal_client import ( from swh.scheduler.journal_client import (
INTERESTING_EVENTS,
compute_last_event,
from_position_offset_to_days, from_position_offset_to_days,
max_date, max_date,
next_visit_queue_position, next_visit_queue_position,
process_journal_objects, process_journal_objects,
) )
from swh.scheduler.model import ListedOrigin, OriginVisitStats from swh.scheduler.model import ListedOrigin, OriginVisitStats
from swh.scheduler.utils import utcnow from swh.scheduler.utils import utcnow
▲ Show 20 Lines • Show All 100 Lines • ▼ Show 20 Linesdef test_journal_client_ignore_missing_type(swh_scheduler):
process_journal_objects( process_journal_objects(
{"origin_visit_status": visit_statuses}, scheduler=swh_scheduler {"origin_visit_status": visit_statuses}, scheduler=swh_scheduler
) )
# The message has been ignored: no stats have been upserted # The message has been ignored: no stats have been upserted
swh_scheduler.origin_visit_stats_upsert.assert_not_called() swh_scheduler.origin_visit_stats_upsert.assert_not_called()
def assert_visit_stats_ok(actual_visit_stats, expected_visit_stats): def assert_visit_stats_ok(
actual_visit_stats, expected_visit_stats, compare_successive_visits=True
):
"""Utility test function to ensure visits stats read from the backend are in the right """Utility test function to ensure visits stats read from the backend are in the right
shape. The comparison on the next_visit_queue_position will be dealt with in shape. The comparison on the next_visit_queue_position will be dealt with in
dedicated tests so it's not tested in tests that are calling this function. dedicated tests so it's not tested in tests that are calling this function.
""" """
assert len(actual_visit_stats) == len(expected_visit_stats) assert len(actual_visit_stats) == len(expected_visit_stats)
for visit_stats in actual_visit_stats: for visit_stats in actual_visit_stats:
visit_stats = attr.evolve(visit_stats, next_visit_queue_position=None) visit_stats = attr.evolve(visit_stats, next_visit_queue_position=None)
if not compare_successive_visits:
visit_stats = attr.evolve(visit_stats, successive_visits=0)
assert visit_stats in expected_visit_stats assert visit_stats in expected_visit_stats
def test_journal_client_origin_visit_status_from_journal_last_notfound(swh_scheduler): def test_journal_client_origin_visit_status_from_journal_last_notfound(swh_scheduler):
visit_status = { visit_status = {
"origin": "foo", "origin": "foo",
"visit": 1, "visit": 1,
Show All 15 Lines assert_visit_stats_ok(
url="foo", url="foo",
visit_type="git", visit_type="git",
last_eventful=None, last_eventful=None,
last_uneventful=None, last_uneventful=None,
last_failed=None, last_failed=None,
last_notfound=visit_status["date"], last_notfound=visit_status["date"],
last_snapshot=None, last_snapshot=None,
next_position_offset=5, next_position_offset=5,
successive_visits=1,
) )
], ],
) )
visit_statuses = [ visit_statuses = [
{ {
"origin": "foo", "origin": "foo",
"visit": 3, "visit": 3,
Show All 24 Lines assert_visit_stats_ok(
url="foo", url="foo",
visit_type="git", visit_type="git",
last_eventful=None, last_eventful=None,
last_uneventful=None, last_uneventful=None,
last_failed=None, last_failed=None,
last_notfound=DATE3, last_notfound=DATE3,
last_snapshot=None, last_snapshot=None,
next_position_offset=7, next_position_offset=7,
successive_visits=3,
) )
], ],
) )
def test_journal_client_origin_visit_status_from_journal_last_failed(swh_scheduler): def test_journal_client_origin_visit_status_from_journal_last_failed(swh_scheduler):
visit_statuses = [ visit_statuses = [
{ {
▲ Show 20 Lines • Show All 42 Lines • ▼ Show 20 Lines assert_visit_stats_ok(
url="bar", url="bar",
visit_type="git", visit_type="git",
last_eventful=None, last_eventful=None,
last_uneventful=None, last_uneventful=None,
last_failed=DATE3, last_failed=DATE3,
last_notfound=None, last_notfound=None,
last_snapshot=None, last_snapshot=None,
next_position_offset=7, next_position_offset=7,
successive_visits=3,
) )
], ],
) )
def test_journal_client_origin_visit_status_from_journal_last_failed2(swh_scheduler): def test_journal_client_origin_visit_status_from_journal_last_failed2(swh_scheduler):
visit_statuses = [ visit_statuses = [
{ {
Show All 26 Lines assert_visit_stats_ok(
url="bar", url="bar",
visit_type="git", visit_type="git",
last_eventful=None, last_eventful=None,
last_uneventful=None, last_uneventful=None,
last_failed=DATE2, last_failed=DATE2,
last_notfound=None, last_notfound=None,
last_snapshot=None, last_snapshot=None,
next_position_offset=6, next_position_offset=6,
successive_visits=2,
) )
], ],
) )
def test_journal_client_origin_visit_status_from_journal_last_eventful(swh_scheduler): def test_journal_client_origin_visit_status_from_journal_last_eventful(swh_scheduler):
visit_statuses = [ visit_statuses = [
{ {
▲ Show 20 Lines • Show All 42 Lines • ▼ Show 20 Lines assert_visit_stats_ok(
url="foo", url="foo",
visit_type="git", visit_type="git",
last_eventful=DATE3, last_eventful=DATE3,
last_uneventful=None, last_uneventful=None,
last_failed=None, last_failed=None,
last_notfound=None, last_notfound=None,
last_snapshot=hash_to_bytes("dddcc0710eb6cf9efd5b920a8453e1e07157bddd"), last_snapshot=hash_to_bytes("dddcc0710eb6cf9efd5b920a8453e1e07157bddd"),
next_position_offset=0, next_position_offset=0,
successive_visits=3,
) )
], ],
) )
def test_journal_client_origin_visit_status_from_journal_last_uneventful(swh_scheduler): def test_journal_client_origin_visit_status_from_journal_last_uneventful(swh_scheduler):
visit_status = { visit_status = {
"origin": "foo", "origin": "foo",
Show All 12 Lines swh_scheduler.origin_visit_stats_upsert(
visit_type=visit_status["type"], visit_type=visit_status["type"],
last_eventful=DATE1, last_eventful=DATE1,
last_uneventful=DATE3, last_uneventful=DATE3,
last_failed=DATE2, last_failed=DATE2,
last_notfound=DATE1, last_notfound=DATE1,
last_snapshot=visit_status["snapshot"], last_snapshot=visit_status["snapshot"],
next_visit_queue_position=None, next_visit_queue_position=None,
next_position_offset=4, next_position_offset=4,
successive_visits=1,
) )
] ]
) )
process_journal_objects( process_journal_objects(
{"origin_visit_status": [visit_status]}, scheduler=swh_scheduler {"origin_visit_status": [visit_status]}, scheduler=swh_scheduler
) )
actual_origin_visit_stats = swh_scheduler.origin_visit_stats_get( actual_origin_visit_stats = swh_scheduler.origin_visit_stats_get(
[(visit_status["origin"], visit_status["type"])] [(visit_status["origin"], visit_status["type"])]
) )
assert_visit_stats_ok( assert_visit_stats_ok(
actual_origin_visit_stats, actual_origin_visit_stats,
[ [
OriginVisitStats( OriginVisitStats(
url=visit_status["origin"], url=visit_status["origin"],
visit_type=visit_status["type"], visit_type=visit_status["type"],
last_eventful=DATE1, last_eventful=DATE1,
last_uneventful=visit_status["date"], # most recent date but uneventful last_uneventful=visit_status["date"], # most recent date but uneventful
last_failed=DATE2, last_failed=DATE2,
last_notfound=DATE1, last_notfound=DATE1,
last_snapshot=visit_status["snapshot"], last_snapshot=visit_status["snapshot"],
next_position_offset=5, # uneventful so visit less often next_position_offset=5, # uneventful so visit less often
successive_visits=2,
) )
], ],
) )
VISIT_STATUSES = [ VISIT_STATUSES = [
{**ovs, "date": DATE1 + n * ONE_DAY} {**ovs, "date": DATE1 + n * ONE_DAY}
for n, ovs in enumerate( for n, ovs in enumerate(
▲ Show 20 Lines • Show All 50 Lines • ▼ Show 20 Lines assert_visit_stats_ok(
url="foo", url="foo",
visit_type="git", visit_type="git",
last_eventful=DATE1 + ONE_DAY, last_eventful=DATE1 + ONE_DAY,
last_uneventful=DATE1 + 3 * ONE_DAY, last_uneventful=DATE1 + 3 * ONE_DAY,
last_failed=None, last_failed=None,
last_notfound=None, last_notfound=None,
last_snapshot=hash_to_bytes("d81cc0710eb6cf9efd5b920a8453e1e07157b6cd"), last_snapshot=hash_to_bytes("d81cc0710eb6cf9efd5b920a8453e1e07157b6cd"),
next_position_offset=5, # uneventful, visit origin less often in future next_position_offset=5, # uneventful, visit origin less often in future
successive_visits=1,
) )
], ],
) )
VISIT_STATUSES_1 = [ VISIT_STATUSES_1 = [
{**ovs, "date": DATE1 + n * ONE_DAY} {**ovs, "date": DATE1 + n * ONE_DAY}
for n, ovs in enumerate( for n, ovs in enumerate(
▲ Show 20 Lines • Show All 48 Lines • ▼ Show 20 Linesdef test_journal_client_origin_visit_status_permutation1(visit_statuses, swh_scheduler):
assert actual_visit_stats.visit_type == "hg" assert actual_visit_stats.visit_type == "hg"
assert actual_visit_stats.last_eventful == DATE1 + 2 * ONE_DAY assert actual_visit_stats.last_eventful == DATE1 + 2 * ONE_DAY
assert actual_visit_stats.last_uneventful == DATE1 + 3 * ONE_DAY assert actual_visit_stats.last_uneventful == DATE1 + 3 * ONE_DAY
assert actual_visit_stats.last_failed is None assert actual_visit_stats.last_failed is None
assert actual_visit_stats.last_notfound is None assert actual_visit_stats.last_notfound is None
assert actual_visit_stats.last_snapshot == hash_to_bytes( assert actual_visit_stats.last_snapshot == hash_to_bytes(
"aaaaaabbbeb6cf9efd5b920a8453e1e07157b6cd" "aaaaaabbbeb6cf9efd5b920a8453e1e07157b6cd"
) )
assert actual_visit_stats.successive_visits == 1
VISIT_STATUSES_2 = [ VISIT_STATUSES_2 = [
{**ovs, "date": DATE1 + n * ONE_DAY} {**ovs, "date": DATE1 + n * ONE_DAY}
for n, ovs in enumerate( for n, ovs in enumerate(
[ [
{ {
"origin": "cavabarder", "origin": "cavabarder",
▲ Show 20 Lines • Show All 127 Lines • ▼ Show 20 Lines assert_visit_stats_ok(
OriginVisitStats( OriginVisitStats(
url="foo", url="foo",
visit_type="git", visit_type="git",
last_eventful=DATE1, last_eventful=DATE1,
last_uneventful=None, last_uneventful=None,
last_failed=None, last_failed=None,
last_notfound=None, last_notfound=None,
last_snapshot=hash_to_bytes("aaaaaabbbeb6cf9efd5b920a8453e1e07157b6cd"), last_snapshot=hash_to_bytes("aaaaaabbbeb6cf9efd5b920a8453e1e07157b6cd"),
next_position_offset=4,
successive_visits=1,
) )
], ],
) )
def test_journal_client_origin_visit_status_several_upsert(swh_scheduler): def test_journal_client_origin_visit_status_several_upsert(swh_scheduler):
"""An old message updates old information """An old message updates old information
Show All 32 Lines assert_visit_stats_ok(
url="foo", url="foo",
visit_type="git", visit_type="git",
last_eventful=DATE1, last_eventful=DATE1,
last_uneventful=DATE2, last_uneventful=DATE2,
last_failed=None, last_failed=None,
last_notfound=None, last_notfound=None,
last_snapshot=hash_to_bytes("aaaaaabbbeb6cf9efd5b920a8453e1e07157b6cd"), last_snapshot=hash_to_bytes("aaaaaabbbeb6cf9efd5b920a8453e1e07157b6cd"),
next_position_offset=5, next_position_offset=5,
successive_visits=1,
) )
], ],
) )
VISIT_STATUSES_SAME_SNAPSHOT = [ VISIT_STATUSES_SAME_SNAPSHOT = [
{**ovs, "date": DATE1 + n * ONE_YEAR} {**ovs, "date": DATE1 + n * ONE_YEAR}
for n, ovs in enumerate( for n, ovs in enumerate(
▲ Show 20 Lines • Show All 50 Lines • ▼ Show 20 Lines assert_visit_stats_ok(
last_eventful=DATE1, last_eventful=DATE1,
last_uneventful=DATE1 + 2 * ONE_YEAR, last_uneventful=DATE1 + 2 * ONE_YEAR,
last_failed=None, last_failed=None,
last_notfound=None, last_notfound=None,
last_snapshot=hash_to_bytes("aaaaaabbbeb6cf9efd5b920a8453e1e07157b6cd"), last_snapshot=hash_to_bytes("aaaaaabbbeb6cf9efd5b920a8453e1e07157b6cd"),
next_position_offset=6, # 2 uneventful visits, whatever the permutation next_position_offset=6, # 2 uneventful visits, whatever the permutation
) )
], ],
compare_successive_visits=False, # depending on permutations, result change
) )
@pytest.mark.parametrize( @pytest.mark.parametrize(
"position_offset, interval", "position_offset, interval",
[ [
(0, 1), (0, 1),
(1, 1), (1, 1),
▲ Show 20 Lines • Show All 84 Lines • ▼ Show 20 Lines actual_position = next_visit_queue_position(
}, },
) )
assert actual_position == date_now + timedelta( assert actual_position == date_now + timedelta(
days=from_position_offset_to_days(next_position_offset) * (1 + fudge_factor) days=from_position_offset_to_days(next_position_offset) * (1 + fudge_factor)
) )
assert mock_random.called assert mock_random.called
@pytest.mark.parametrize(
"last_event", INTERESTING_EVENTS,
)
def test_compute_last_event(last_event):
"""Compute last event of visit stats should always return most recent event if any
"""
maxdate = utcnow()
event_key = f"last_{last_event}"
visit_stats_d = {
event_key: maxdate,
}
other_events = set(INTERESTING_EVENTS) - set([last_event])
# For the remaining event, even have some empty dates or have some dates in the past
for i, event in enumerate(other_events):
empty_date = random.choice([True, False])
visit_stats_d[f"last_{event}"] = (
None if empty_date else (maxdate - timedelta(days=1 + i))
)
# Ensure our visit_stats_d is configured properly
assert max_date(*visit_stats_d.values()) == maxdate
actual_last_event = compute_last_event(visit_stats_d)
assert actual_last_event == event_key
def test_compute_last_event_none():
"""Compute last event out of visit stats without any date should return None
"""
visit_stats_d = {f"last_{event}": None for event in INTERESTING_EVENTS}
assert compute_last_event(visit_stats_d) is None
About · Developer information · Legal