Differential D8386 Diff 30346 swh/lister/fedora/tests/test_lister.py

Changeset View

Standalone View

View Options

swh/lister/fedora/tests/test_lister.py

This file was added.

# See the AUTHORS file at the top-level directory of this distribution

# License: GNU General Public License version 3, or any later version

# See top-level LICENSE file for more information

from pathlib import Path

from unittest.mock import MagicMock

import defusedxml

import pytest

from repomd import Repo

from requests_mock import Mocker

from swh.lister.fedora.lister import FedoraLister

from swh.scheduler.interface import SchedulerInterface

mirror_url = "https://eu.edge.kernel.org/fedora/releases"

@pytest.fixture

def repomd_mock(requests_mock, datadir, mocker):

primary_xml = Path(datadir, "https_eu.edge.kernel.org_fedora_36", "primary.xml")

metadata = defusedxml.lxml.fromstring(primary_xml.read_bytes())

repomd_mock = mocker.patch("swh.lister.fedora.lister.repomd")

repomd_mock.load.return_value = Repo(mirror_url, metadata)

vlorentzUnsubmitted

Not Done

return Repo(f"mocked.mirror.url/releases/{release}", metadata)

- RepomdMock = Tuple[Tuple[MagicMock, MagicMock], Dict[str, Set[str]]]

+ RepomdMock = Tuple[Tuple[Repo, Repo], Dict[str, Set[str]]]

@pytest.fixture

vlorentz:

vlorentzUnsubmitted

Not Done

cm = MagicMock()

- cm.read.side_effect = list(

- map(

- lambda path: Path(datadir, "archives.fedoraproject.org", path).read_bytes(),

- paths,

- )

+ cm.read.side_effect = [

+ Path(datadir, "archives.fedoraproject.org", path).read_bytes(),

+ for path in paths

)

cm.__enter__.return_value = cm

vlorentz:

return repomd_mock

vlorentzUnsubmitted

Not Done

please add a docstring

vlorentz: please add a docstring

def test_full_lister_fedora(

swh_scheduler: SchedulerInterface,

repomd_mock: MagicMock,

requests_mock: Mocker,

vlorentzUnsubmitted

Not Done

Can you mock the backend requests rather than repomd? This allows catching misuses of repomd in our code and/or incompatibility with some versions

vlorentz: Can you mock the backend requests rather than repomd? This allows catching misuses of repomd in…

KShivenduAuthorUnsubmitted

Done

I tried but stopped since it would be ugly since repomd uses urllib.request.urlopen instead of requests library (requests_mock doesn't work)

If you think it's still worth the shot, I'll do it :)

KShivendu: I tried but stopped since it would be ugly since `repomd` uses `urllib.request.urlopen` instead…

vlorentzUnsubmitted

Not Done

yes, please.

There are already examples of it in swh-loader-core/swh/loader/package/tests/test_utils.py and swh-storage/swh/storage/tests/migrate_extrinsic_metadata/test_pypi.py

vlorentz: yes, please. There are already examples of it in `swh-loader…

"""

Simulate a full listing of packages for fedora releases.

"""

lister = FedoraLister(

scheduler=swh_scheduler,

url=mirror_url,

releases=[

"35",

"36",

], # here both repomd requests will return the same 14 packages

)

stats = lister.run()

scheduler_origins = swh_scheduler.get_listed_origins(lister.lister_obj.id).results

assert stats.pages == 2 # == no. of requested releases

assert stats.origins == 14

assert len(scheduler_origins) == 14

vlorentzUnsubmitted

Done

you should test the actual values too

vlorentz: you should test the actual values too

stats = lister.run()

scheduler_origins = swh_scheduler.get_listed_origins(lister.lister_obj.id).results

assert stats.pages == 2

assert stats.origins == 0

assert len(scheduler_origins) == 14

vlorentzUnsubmitted

Not Done

I don't understand why changing pkg_versions affects the lister

vlorentz: I don't understand why changing `pkg_versions` affects the lister

KShivenduAuthorUnsubmitted

Done

Changing pkg._version_info.set("ver", "1.0.0") affected it (not pkg_versions)

KShivendu: Changing `pkg._version_info.set("ver", "1.0.0")` affected it (not `pkg_versions`)

vlorentzUnsubmitted

Not Done

Ah, I see. Thanks for the extra comments

vlorentz: Ah, I see. Thanks for the extra comments

vlorentzUnsubmitted

Not Done

Please test the full URLs; and make it a list equality to pytest can display diffs, instead of length mismatches.

And you are missing assertions for other attributes of listed origins (esp. extra_loader_arguments)

vlorentz: Please test the full URLs; and make it a list equality to pytest can display diffs, instead of…

anlambertUnsubmitted

Not Done

Instead of reusing the same lister instance, you should create a new one each time you want to execute a listing,
this is the behavior we have in production.

def run_lister(
    swh_scheduler: SchedulerInterface,
    releases: List[Release],
    pkg_versions: dict,
    origin_count: int,
):
    """Runs the lister and tests that the listed origins are correct."""
    lister = FedoraLister(
        scheduler=swh_scheduler,
        releases=releases,
    )

anlambert: Instead of reusing the same lister instance, you should create a new one each time you want to…