Differential D8298 Diff 30139 swh/lister/golang/tests/test_lister.py

Changeset View

Standalone View

swh/lister/golang/tests/test_lister.py

# See the AUTHORS file at the top-level directory of this distribution # See the AUTHORS file at the top-level directory of this distribution

# License: GNU General Public License version 3, or any later version # License: GNU General Public License version 3, or any later version

# See top-level LICENSE file for more information # See top-level LICENSE file for more information

import datetime

from pathlib import Path from pathlib import Path

import iso8601 import iso8601

from swh.lister.golang.lister import GolangLister from swh.lister.golang.lister import GolangLister, GolangStateType

from swh.lister.tests.test_utils import assert_sleep_calls from swh.lister.tests.test_utils import assert_sleep_calls

from swh.lister.utils import WAIT_EXP_BASE from swh.lister.utils import WAIT_EXP_BASE

# https://pkg.go.dev prefix omitted # https://pkg.go.dev prefix omitted

expected_listed = [ expected_listed = [

("collectd.org", "2019-04-11T18:47:25.450546+00:00"), ("collectd.org", "2019-04-11T18:47:25.450546+00:00"),

("github.com/blang/semver", "2019-04-15T13:54:39.107258+00:00",), ("github.com/blang/semver", "2019-04-15T13:54:39.107258+00:00",),

("github.com/bmizerany/pat", "2019-04-11T18:47:29.390564+00:00",), ("github.com/bmizerany/pat", "2019-04-11T18:47:29.390564+00:00",),

▲ Show 20 Lines • Show All 64 Lines • ▼ Show 20 Lines def test_golang_lister(swh_scheduler, mocker, requests_mock, datadir):

# doing it all again (without incremental) should give us the same result # doing it all again (without incremental) should give us the same result

lister = GolangLister(scheduler=swh_scheduler) lister = GolangLister(scheduler=swh_scheduler)

mocked_sleep = mocker.patch.object(lister.api_request.retry, "sleep") mocked_sleep = mocker.patch.object(lister.api_request.retry, "sleep")

_generate_responses(datadir, requests_mock) _generate_responses(datadir, requests_mock)

stats = lister.run() stats = lister.run()

assert stats.pages == 3 assert stats.pages == 3

assert stats.origins == 18 assert stats.origins == 18

def test_golang_lister_incremental(swh_scheduler, requests_mock, datadir, mocker):

# first listing, should return one origin per package

lister = GolangLister(scheduler=swh_scheduler, incremental=True)

mock = mocker.spy(lister, "get_single_page")

responses = [

{"text": Path(datadir, "page-1.txt").read_text(), "status_code": 200},

]

requests_mock.get(GolangLister.GOLANG_MODULES_INDEX_URL, responses)

stats = lister.run()

page1_last_timestamp = datetime.datetime(

2019, 4, 11, 18, 47, 29, 390564, tzinfo=datetime.timezone.utc

)

page2_last_timestamp = datetime.datetime(

2019, 4, 15, 13, 54, 35, 250835, tzinfo=datetime.timezone.utc

)

page3_last_timestamp = datetime.datetime(

2019, 4, 18, 2, 7, 41, 336899, tzinfo=datetime.timezone.utc

)

mock.assert_has_calls(

[

# First call has no state

vlorentzUnsubmitted

Not Done

responses = [

- {"text": Path(datadir, "page-2.txt").read_text(), "status_code": 200},

+ {"text": (datadir / "page-2.txt").read_text(), "status_code": 200},

{"text": "", "status_code": 200},

;)

(ditto above)

vlorentz: ;) (ditto above)

AlphareAuthorUnsubmitted

Done

I had already tried that but datadir is a str and not a Path, maybe that's something that should be changed?

Alphare: I had already tried that but `datadir` is a `str` and not a `Path`, maybe that's something that…

mocker.call(since=None),

# Second call is the last timestamp in the listed page

mocker.call(since=page1_last_timestamp),

]

)

assert lister.get_state_from_scheduler() == GolangStateType(

last_seen=page1_last_timestamp

)

assert stats.pages == 1

assert stats.origins == 5

# Incremental should list nothing

lister = GolangLister(scheduler=swh_scheduler, incremental=True)

mock = mocker.spy(lister, "get_single_page")

stats = lister.run()

mock.assert_has_calls([mocker.call(since=page1_last_timestamp)])

assert stats.pages == 0

assert stats.origins == 0

# Add more responses

responses = [

{"text": Path(datadir, "page-2.txt").read_text(), "status_code": 200},

]

requests_mock.get(GolangLister.GOLANG_MODULES_INDEX_URL, responses)

# Incremental should list new page

lister = GolangLister(scheduler=swh_scheduler, incremental=True)

mock = mocker.spy(lister, "get_single_page")

stats = lister.run()

mock.assert_has_calls(

[

mocker.call(since=page1_last_timestamp),

mocker.call(since=page2_last_timestamp),

]

)

assert stats.pages == 1

assert stats.origins == 4

# Incremental should list nothing again

lister = GolangLister(scheduler=swh_scheduler, incremental=True)

mock = mocker.spy(lister, "get_single_page")

stats = lister.run()

assert stats.pages == 0

assert stats.origins == 0

mock.assert_has_calls([mocker.call(since=page2_last_timestamp)])

# Add yet more responses

responses = [

{"text": Path(datadir, "page-3.txt").read_text(), "status_code": 200},

]

requests_mock.get(GolangLister.GOLANG_MODULES_INDEX_URL, responses)

# Incremental should list new page again

lister = GolangLister(scheduler=swh_scheduler, incremental=True)

mock = mocker.spy(lister, "get_single_page")

stats = lister.run()

assert stats.pages == 1

assert stats.origins == 9

mock.assert_has_calls(

[

mocker.call(since=page2_last_timestamp),

mocker.call(since=page3_last_timestamp),

]

)

# Incremental should list nothing one last time

lister = GolangLister(scheduler=swh_scheduler, incremental=True)

mock = mocker.spy(lister, "get_single_page")

stats = lister.run()

assert stats.pages == 0

assert stats.origins == 0

mock.assert_has_calls([mocker.call(since=page3_last_timestamp)])