Changeset View
Standalone View
swh/lister/fedora/tests/test_lister.py
- This file was added.
# Copyright (C) 2022 The Software Heritage developers | ||||||||||||||||
# See the AUTHORS file at the top-level directory of this distribution | ||||||||||||||||
# License: GNU General Public License version 3, or any later version | ||||||||||||||||
# See top-level LICENSE file for more information | ||||||||||||||||
from pathlib import Path | ||||||||||||||||
from typing import List | ||||||||||||||||
from unittest.mock import MagicMock | ||||||||||||||||
from swh.lister.fedora.lister import FedoraLister, Release, get_editions | ||||||||||||||||
from swh.scheduler.interface import SchedulerInterface | ||||||||||||||||
def mock_repomd(datadir, mocker, use_altered_fedora36=False): | ||||||||||||||||
"""Mocks the .xml files fetched by repomd for the next lister run""" | ||||||||||||||||
paths = ["repomd26.xml", "primary26.xml.gz", "repomd36.xml", "primary36.xml.gz"] | ||||||||||||||||
if use_altered_fedora36: | ||||||||||||||||
paths[3] = "primary36-altered.xml.gz" | ||||||||||||||||
cm = MagicMock() | ||||||||||||||||
cm.read.side_effect = list( | ||||||||||||||||
map( | ||||||||||||||||
lambda path: Path(datadir, "archives.fedoraproject.org", path).read_bytes(), | ||||||||||||||||
paths, | ||||||||||||||||
) | ||||||||||||||||
vlorentzUnsubmitted Not Done Inline Actions
vlorentz: | ||||||||||||||||
) | ||||||||||||||||
vlorentzUnsubmitted Not Done Inline Actions
vlorentz: | ||||||||||||||||
cm.__enter__.return_value = cm | ||||||||||||||||
mocker.patch("repomd.urllib.request.urlopen").return_value = cm | ||||||||||||||||
Not Done Inline Actionsplease add a docstring vlorentz: please add a docstring | ||||||||||||||||
def rpm_url(release, path): | ||||||||||||||||
return ( | ||||||||||||||||
"https://archives.fedoraproject.org/pub/archive/fedora/linux/releases/" | ||||||||||||||||
Not Done Inline ActionsCan you mock the backend requests rather than repomd? This allows catching misuses of repomd in our code and/or incompatibility with some versions vlorentz: Can you mock the backend requests rather than repomd? This allows catching misuses of repomd in… | ||||||||||||||||
Done Inline ActionsI tried but stopped since it would be ugly since repomd uses urllib.request.urlopen instead of requests library (requests_mock doesn't work) If you think it's still worth the shot, I'll do it :) KShivendu: I tried but stopped since it would be ugly since `repomd` uses `urllib.request.urlopen` instead… | ||||||||||||||||
Not Done Inline Actionsyes, please. There are already examples of it in swh-loader-core/swh/loader/package/tests/test_utils.py and swh-storage/swh/storage/tests/migrate_extrinsic_metadata/test_pypi.py vlorentz: yes, please.
There are already examples of it in `swh-loader… | ||||||||||||||||
f"{release}/Everything/source/tree/Packages/{path}" | ||||||||||||||||
) | ||||||||||||||||
_pkg_versions = { | ||||||||||||||||
"rpm://fedora/packages/0install": { | ||||||||||||||||
"26/Everything/2.11": { | ||||||||||||||||
"name": "0install", | ||||||||||||||||
"version": "2.11", | ||||||||||||||||
"release": 26, | ||||||||||||||||
"url": rpm_url(26, "0/0install-2.11-4.fc26.src.rpm"), | ||||||||||||||||
} | ||||||||||||||||
}, | ||||||||||||||||
"rpm://fedora/packages/0xFFFF": { | ||||||||||||||||
"36/Everything/0.9": { | ||||||||||||||||
"name": "0xFFFF", | ||||||||||||||||
"version": "0.9", | ||||||||||||||||
"release": 36, | ||||||||||||||||
"url": rpm_url(36, "0/0xFFFF-0.9-4.fc36.src.rpm"), | ||||||||||||||||
}, | ||||||||||||||||
Done Inline Actionsyou should test the actual values too vlorentz: you should test the actual values too | ||||||||||||||||
"26/Everything/0.3.9": { | ||||||||||||||||
"name": "0xFFFF", | ||||||||||||||||
"version": "0.3.9", | ||||||||||||||||
"release": 26, | ||||||||||||||||
"url": rpm_url(26, "0/0xFFFF-0.3.9-15.fc26.src.rpm"), | ||||||||||||||||
}, | ||||||||||||||||
}, | ||||||||||||||||
"rpm://fedora/packages/2ping": { | ||||||||||||||||
"36/Everything/4.5.1": { | ||||||||||||||||
"name": "2ping", | ||||||||||||||||
"version": "4.5.1", | ||||||||||||||||
"release": 36, | ||||||||||||||||
"url": rpm_url(36, "2/2ping-4.5.1-2.fc36.src.rpm"), | ||||||||||||||||
} | ||||||||||||||||
Not Done Inline ActionsPlease test the full URLs; and make it a list equality to pytest can display diffs, instead of length mismatches. And you are missing assertions for other attributes of listed origins (esp. extra_loader_arguments) vlorentz: Please test the full URLs; and make it a list equality to pytest can display diffs, instead of… | ||||||||||||||||
}, | ||||||||||||||||
} | ||||||||||||||||
def run_lister( | ||||||||||||||||
lister: FedoraLister, | ||||||||||||||||
swh_scheduler: SchedulerInterface, | ||||||||||||||||
releases: List[Release], | ||||||||||||||||
pkg_versions: dict, | ||||||||||||||||
origin_count: int, | ||||||||||||||||
): | ||||||||||||||||
"""Runs the lister and tests that the listed origins are correct.""" | ||||||||||||||||
stats = lister.run() | ||||||||||||||||
scheduler_origins = swh_scheduler.get_listed_origins(lister.lister_obj.id).results | ||||||||||||||||
lister_state = lister.get_state_from_scheduler() | ||||||||||||||||
state_pkg_versions = {k.split("/")[-1]: set(v) for k, v in pkg_versions.items()} | ||||||||||||||||
# One edition from each release (we mocked get_editions) | ||||||||||||||||
assert stats.pages == len(releases) | ||||||||||||||||
assert stats.origins == origin_count | ||||||||||||||||
Not Done Inline ActionsInstead of reusing the same lister instance, you should create a new one each time you want to execute a listing, def run_lister( swh_scheduler: SchedulerInterface, releases: List[Release], pkg_versions: dict, origin_count: int, ): """Runs the lister and tests that the listed origins are correct.""" lister = FedoraLister( scheduler=swh_scheduler, releases=releases, ) anlambert: Instead of reusing the same lister instance, you should create a new one each time you want to… | ||||||||||||||||
assert { | ||||||||||||||||
o.url: o.extra_loader_arguments["packages"] for o in scheduler_origins | ||||||||||||||||
} == pkg_versions | ||||||||||||||||
assert lister_state.package_versions == state_pkg_versions | ||||||||||||||||
assert lister.updated | ||||||||||||||||
def test_get_editions(): | ||||||||||||||||
assert get_editions(18) == ["Everything", "Fedora"] | ||||||||||||||||
assert get_editions(26) == ["Everything", "Server", "Workstation"] | ||||||||||||||||
assert get_editions(34) == ["Everything", "Server", "Workstation", "Modular"] | ||||||||||||||||
def test_full_lister_fedora( | ||||||||||||||||
swh_scheduler: SchedulerInterface, | ||||||||||||||||
mocker: MagicMock, | ||||||||||||||||
datadir: Path, | ||||||||||||||||
): | ||||||||||||||||
""" | ||||||||||||||||
Simulates a full listing of packages for fedora releases. | ||||||||||||||||
""" | ||||||||||||||||
releases = [26, 36] | ||||||||||||||||
lister = FedoraLister( | ||||||||||||||||
scheduler=swh_scheduler, | ||||||||||||||||
releases=releases, | ||||||||||||||||
) | ||||||||||||||||
get_editions_patch = mocker.patch("swh.lister.fedora.lister.get_editions") | ||||||||||||||||
get_editions_patch.return_value = ["Everything"] | ||||||||||||||||
pkg_versions = _pkg_versions.copy() | ||||||||||||||||
mock_repomd(datadir, mocker) | ||||||||||||||||
run_lister(lister, swh_scheduler, releases, pkg_versions, origin_count=3) | ||||||||||||||||
def test_incremental_lister( | ||||||||||||||||
swh_scheduler: SchedulerInterface, mocker: MagicMock, datadir: Path | ||||||||||||||||
): | ||||||||||||||||
""" | ||||||||||||||||
Simulates an incremental listing of packages for fedora releases. | ||||||||||||||||
""" | ||||||||||||||||
releases = [26, 36] | ||||||||||||||||
lister = FedoraLister( | ||||||||||||||||
scheduler=swh_scheduler, | ||||||||||||||||
releases=releases, | ||||||||||||||||
) | ||||||||||||||||
get_editions_patch = mocker.patch("swh.lister.fedora.lister.get_editions") | ||||||||||||||||
get_editions_patch.return_value = ["Everything"] | ||||||||||||||||
pkg_versions = _pkg_versions.copy() | ||||||||||||||||
# First run | ||||||||||||||||
mock_repomd(datadir, mocker) | ||||||||||||||||
run_lister(lister, swh_scheduler, releases, pkg_versions, origin_count=3) | ||||||||||||||||
# Second run (no updates) | ||||||||||||||||
mock_repomd(datadir, mocker) | ||||||||||||||||
run_lister(lister, swh_scheduler, releases, pkg_versions, origin_count=0) | ||||||||||||||||
# Use an altered version of primary36.xml in which we updated the version | ||||||||||||||||
# of package 0xFFFF to 0.10: | ||||||||||||||||
mock_repomd(datadir, mocker, use_altered_fedora36=True) | ||||||||||||||||
# Add new version to the set of expected pkg versions: | ||||||||||||||||
pkg_versions["rpm://fedora/packages/0xFFFF"].update( | ||||||||||||||||
{ | ||||||||||||||||
"36/Everything/0.10": { | ||||||||||||||||
"name": "0xFFFF", | ||||||||||||||||
"version": "0.10", | ||||||||||||||||
"release": 36, | ||||||||||||||||
# .rpm URL remains same as per primary.xml file: | ||||||||||||||||
"url": rpm_url(36, "0/0xFFFF-0.9-4.fc36.src.rpm"), | ||||||||||||||||
} | ||||||||||||||||
Not Done Inline ActionsI don't understand why changing pkg_versions affects the lister vlorentz: I don't understand why changing `pkg_versions` affects the lister | ||||||||||||||||
Done Inline ActionsChanging pkg._version_info.set("ver", "1.0.0") affected it (not pkg_versions) KShivendu: Changing `pkg._version_info.set("ver", "1.0.0")` affected it (not `pkg_versions`) | ||||||||||||||||
Not Done Inline ActionsAh, I see. Thanks for the extra comments vlorentz: Ah, I see. Thanks for the extra comments | ||||||||||||||||
} | ||||||||||||||||
) | ||||||||||||||||
# Third run (0xFFFF in fedora36 editions got updated, but no new origins were found) | ||||||||||||||||
run_lister(lister, swh_scheduler, releases, pkg_versions, origin_count=0) |