Changeset View
Standalone View
swh/lister/fedora/tests/test_lister.py
- This file was added.
# Copyright (C) 2022 The Software Heritage developers | ||||||||||||||||
# See the AUTHORS file at the top-level directory of this distribution | ||||||||||||||||
# License: GNU General Public License version 3, or any later version | ||||||||||||||||
# See top-level LICENSE file for more information | ||||||||||||||||
from pathlib import Path | ||||||||||||||||
from typing import List, Tuple | ||||||||||||||||
from unittest.mock import MagicMock | ||||||||||||||||
import defusedxml | ||||||||||||||||
import pytest | ||||||||||||||||
from repomd import Repo | ||||||||||||||||
from swh.lister.fedora.lister import FedoraLister, Release, get_editions | ||||||||||||||||
from swh.scheduler.interface import SchedulerInterface | ||||||||||||||||
def get_repo(datadir: Path, release: int) -> Repo: | ||||||||||||||||
"""Returns a repomd Repo object for the given release.""" | ||||||||||||||||
primary_xml = Path(datadir, "archives.fedoraproject.org", f"primary{release}.xml") | ||||||||||||||||
metadata = defusedxml.lxml.fromstring(primary_xml.read_bytes()) | ||||||||||||||||
return Repo(f"mocked.mirror.url/releases/{release}", metadata) | ||||||||||||||||
RepomdMock = Tuple[Repo, Repo] | ||||||||||||||||
vlorentzUnsubmitted Not Done Inline Actions
vlorentz: | ||||||||||||||||
Not Done Inline Actions
vlorentz: | ||||||||||||||||
@pytest.fixture | ||||||||||||||||
def repomd_mock(datadir, mocker, requests_mock) -> RepomdMock: | ||||||||||||||||
"""Mocks repomd.load function and returns a list of packages.""" | ||||||||||||||||
Not Done Inline Actionsplease add a docstring vlorentz: please add a docstring | ||||||||||||||||
fedora26 = get_repo(datadir, 26) | ||||||||||||||||
fedora36 = get_repo(datadir, 36) | ||||||||||||||||
repomd_mock = mocker.patch("swh.lister.fedora.lister.repomd") | ||||||||||||||||
Not Done Inline ActionsCan you mock the backend requests rather than repomd? This allows catching misuses of repomd in our code and/or incompatibility with some versions vlorentz: Can you mock the backend requests rather than repomd? This allows catching misuses of repomd in… | ||||||||||||||||
Done Inline ActionsI tried but stopped since it would be ugly since repomd uses urllib.request.urlopen instead of requests library (requests_mock doesn't work) If you think it's still worth the shot, I'll do it :) KShivendu: I tried but stopped since it would be ugly since `repomd` uses `urllib.request.urlopen` instead… | ||||||||||||||||
Not Done Inline Actionsyes, please. There are already examples of it in swh-loader-core/swh/loader/package/tests/test_utils.py and swh-storage/swh/storage/tests/migrate_extrinsic_metadata/test_pypi.py vlorentz: yes, please.
There are already examples of it in `swh-loader… | ||||||||||||||||
def side_effect(url): | ||||||||||||||||
if "36" in url: | ||||||||||||||||
return fedora36 | ||||||||||||||||
elif "26" in url: | ||||||||||||||||
return fedora26 | ||||||||||||||||
else: | ||||||||||||||||
raise Exception(f"Unexpected url: {url}") | ||||||||||||||||
repomd_mock.load.side_effect = side_effect | ||||||||||||||||
return fedora26, fedora36 | ||||||||||||||||
_pkg_versions = { | ||||||||||||||||
"rpm://fedora/packages/0install": { | ||||||||||||||||
"26/Everything/2.11": { | ||||||||||||||||
"name": "0install", | ||||||||||||||||
"version": "2.11", | ||||||||||||||||
"release": 26, | ||||||||||||||||
Done Inline Actionsyou should test the actual values too vlorentz: you should test the actual values too | ||||||||||||||||
"url": "mocked.mirror.url/releases/Packages/0/0install-2.11-4.fc26.src.rpm", | ||||||||||||||||
} | ||||||||||||||||
}, | ||||||||||||||||
"rpm://fedora/packages/0xFFFF": { | ||||||||||||||||
"36/Everything/0.9": { | ||||||||||||||||
"name": "0xFFFF", | ||||||||||||||||
"version": "0.9", | ||||||||||||||||
"release": 36, | ||||||||||||||||
"url": "mocked.mirror.url/releases/Packages/0/0xFFFF-0.9-4.fc36.src.rpm", | ||||||||||||||||
}, | ||||||||||||||||
"26/Everything/0.3.9": { | ||||||||||||||||
"name": "0xFFFF", | ||||||||||||||||
"version": "0.3.9", | ||||||||||||||||
"release": 26, | ||||||||||||||||
Not Done Inline ActionsPlease test the full URLs; and make it a list equality to pytest can display diffs, instead of length mismatches. And you are missing assertions for other attributes of listed origins (esp. extra_loader_arguments) vlorentz: Please test the full URLs; and make it a list equality to pytest can display diffs, instead of… | ||||||||||||||||
"url": "mocked.mirror.url/releases/Packages/0/0xFFFF-0.3.9-15.fc26.src.rpm", | ||||||||||||||||
}, | ||||||||||||||||
}, | ||||||||||||||||
"rpm://fedora/packages/2ping": { | ||||||||||||||||
"36/Everything/4.5.1": { | ||||||||||||||||
"name": "2ping", | ||||||||||||||||
"version": "4.5.1", | ||||||||||||||||
"release": 36, | ||||||||||||||||
"url": "mocked.mirror.url/releases/Packages/2/2ping-4.5.1-2.fc36.src.rpm", | ||||||||||||||||
} | ||||||||||||||||
}, | ||||||||||||||||
} | ||||||||||||||||
def run_lister( | ||||||||||||||||
lister: FedoraLister, | ||||||||||||||||
swh_scheduler: SchedulerInterface, | ||||||||||||||||
releases: List[Release], | ||||||||||||||||
pkg_versions: dict, | ||||||||||||||||
origin_count: int, | ||||||||||||||||
): | ||||||||||||||||
Not Done Inline ActionsInstead of reusing the same lister instance, you should create a new one each time you want to execute a listing, def run_lister( swh_scheduler: SchedulerInterface, releases: List[Release], pkg_versions: dict, origin_count: int, ): """Runs the lister and tests that the listed origins are correct.""" lister = FedoraLister( scheduler=swh_scheduler, releases=releases, ) anlambert: Instead of reusing the same lister instance, you should create a new one each time you want to… | ||||||||||||||||
"""Runs the lister and tests that the listed origins are correct.""" | ||||||||||||||||
stats = lister.run() | ||||||||||||||||
scheduler_origins = swh_scheduler.get_listed_origins(lister.lister_obj.id).results | ||||||||||||||||
# One edition from each release (we mocked get_editions) | ||||||||||||||||
assert stats.pages == len(releases) | ||||||||||||||||
assert stats.origins == origin_count | ||||||||||||||||
assert { | ||||||||||||||||
o.url: o.extra_loader_arguments["packages"] for o in scheduler_origins | ||||||||||||||||
} == pkg_versions | ||||||||||||||||
lister_state = lister.get_state_from_scheduler() | ||||||||||||||||
assert lister_state.package_versions == { | ||||||||||||||||
k.split("/")[-1]: set(v) for k, v in pkg_versions.items() | ||||||||||||||||
} | ||||||||||||||||
assert lister.updated | ||||||||||||||||
def test_get_editions(): | ||||||||||||||||
assert get_editions(18) == ["Everything", "Fedora"] | ||||||||||||||||
assert get_editions(26) == ["Everything", "Server", "Workstation"] | ||||||||||||||||
assert get_editions(34) == ["Everything", "Server", "Workstation", "Modular"] | ||||||||||||||||
def test_full_lister_fedora( | ||||||||||||||||
swh_scheduler: SchedulerInterface, | ||||||||||||||||
repomd_mock: RepomdMock, | ||||||||||||||||
mocker: MagicMock, | ||||||||||||||||
): | ||||||||||||||||
""" | ||||||||||||||||
Simulates a full listing of packages for fedora releases. | ||||||||||||||||
""" | ||||||||||||||||
releases = [26, 36] | ||||||||||||||||
lister = FedoraLister( | ||||||||||||||||
scheduler=swh_scheduler, | ||||||||||||||||
releases=releases, | ||||||||||||||||
) | ||||||||||||||||
get_editions_patch = mocker.patch("swh.lister.fedora.lister.get_editions") | ||||||||||||||||
get_editions_patch.return_value = ["Everything"] | ||||||||||||||||
pkg_versions = _pkg_versions.copy() | ||||||||||||||||
run_lister(lister, swh_scheduler, releases, pkg_versions, origin_count=3) | ||||||||||||||||
def test_incremental_lister( | ||||||||||||||||
swh_scheduler: SchedulerInterface, repomd_mock: RepomdMock, mocker: MagicMock | ||||||||||||||||
): | ||||||||||||||||
""" | ||||||||||||||||
Simulates an incremental listing of packages for fedora releases. | ||||||||||||||||
""" | ||||||||||||||||
releases = [26, 36] | ||||||||||||||||
lister = FedoraLister( | ||||||||||||||||
scheduler=swh_scheduler, | ||||||||||||||||
releases=releases, | ||||||||||||||||
) | ||||||||||||||||
get_editions_patch = mocker.patch("swh.lister.fedora.lister.get_editions") | ||||||||||||||||
get_editions_patch.return_value = ["Everything"] | ||||||||||||||||
pkg_versions = _pkg_versions.copy() | ||||||||||||||||
# First run | ||||||||||||||||
run_lister(lister, swh_scheduler, releases, pkg_versions, origin_count=3) | ||||||||||||||||
# Second run (no updates) | ||||||||||||||||
run_lister(lister, swh_scheduler, releases, pkg_versions, origin_count=0) | ||||||||||||||||
# Update version of the package 0xFFFF in Fedora 36: | ||||||||||||||||
_, fedora36 = repomd_mock | ||||||||||||||||
for pkg in fedora36: | ||||||||||||||||
if pkg.name == "0xFFFF": | ||||||||||||||||
pkg._version_info.set("ver", "0.10") | ||||||||||||||||
# Add new version to the set of expected pkg versions: | ||||||||||||||||
Not Done Inline ActionsI don't understand why changing pkg_versions affects the lister vlorentz: I don't understand why changing `pkg_versions` affects the lister | ||||||||||||||||
Done Inline ActionsChanging pkg._version_info.set("ver", "1.0.0") affected it (not pkg_versions) KShivendu: Changing `pkg._version_info.set("ver", "1.0.0")` affected it (not `pkg_versions`) | ||||||||||||||||
Not Done Inline ActionsAh, I see. Thanks for the extra comments vlorentz: Ah, I see. Thanks for the extra comments | ||||||||||||||||
pkg_versions["rpm://fedora/packages/0xFFFF"].update( | ||||||||||||||||
{ | ||||||||||||||||
"36/Everything/0.10": { | ||||||||||||||||
"name": "0xFFFF", | ||||||||||||||||
"version": "0.10", | ||||||||||||||||
"release": 36, | ||||||||||||||||
# .rpm URL remains same: | ||||||||||||||||
"url": "mocked.mirror.url/releases/Packages/0/0xFFFF-0.9-4.fc36.src.rpm", | ||||||||||||||||
} | ||||||||||||||||
} | ||||||||||||||||
) | ||||||||||||||||
# Third run (0xFFFF in fedora36 editions got updated, but no new origins were found) | ||||||||||||||||
run_lister(lister, swh_scheduler, releases, pkg_versions, origin_count=0) |