Changeset View
Changeset View
Standalone View
Standalone View
swh/lister/golang/tests/test_lister.py
Show First 20 Lines • Show All 92 Lines • ▼ Show 20 Lines | for file in Path(datadir).glob("page-*.txt"): | ||||
responses.append({"text": "", "status_code": 500}) | responses.append({"text": "", "status_code": 500}) | ||||
# Also test that the lister appropriately gets out of the infinite loop | # Also test that the lister appropriately gets out of the infinite loop | ||||
responses.append({"text": file.read_text(), "status_code": 200}) | responses.append({"text": file.read_text(), "status_code": 200}) | ||||
requests_mock.get(GolangLister.GOLANG_MODULES_INDEX_URL, responses) | requests_mock.get(GolangLister.GOLANG_MODULES_INDEX_URL, responses) | ||||
def test_golang_lister(swh_scheduler, mocker, requests_mock, datadir): | def test_golang_lister(swh_scheduler, mocker, requests_mock, datadir): | ||||
# first listing, should return one origin per package | |||||
lister = GolangLister(scheduler=swh_scheduler) | |||||
# Exponential retries take a long time, so stub time.sleep | # Exponential retries take a long time, so stub time.sleep | ||||
mocked_sleep = mocker.patch.object(lister.api_request.retry, "sleep") | mocked_sleep = mocker.patch.object(GolangLister.http_request.retry, "sleep") | ||||
# first listing, should return one origin per package | |||||
lister = GolangLister(scheduler=swh_scheduler) | |||||
_generate_responses(datadir, requests_mock) | _generate_responses(datadir, requests_mock) | ||||
stats = lister.run() | stats = lister.run() | ||||
assert stats.pages == 3 | assert stats.pages == 3 | ||||
# The two `golang.org/x/tools` versions are *not* listed as separate origins | # The two `golang.org/x/tools` versions are *not* listed as separate origins | ||||
assert stats.origins == 18 | assert stats.origins == 18 | ||||
Show All 12 Lines | def test_golang_lister(swh_scheduler, mocker, requests_mock, datadir): | ||||
# Test `time.sleep` is called with exponential retries | # Test `time.sleep` is called with exponential retries | ||||
assert_sleep_calls( | assert_sleep_calls( | ||||
mocker, mocked_sleep, [1, WAIT_EXP_BASE, 1, WAIT_EXP_BASE, 1, WAIT_EXP_BASE] | mocker, mocked_sleep, [1, WAIT_EXP_BASE, 1, WAIT_EXP_BASE, 1, WAIT_EXP_BASE] | ||||
) | ) | ||||
# doing it all again (without incremental) should give us the same result | # doing it all again (without incremental) should give us the same result | ||||
lister = GolangLister(scheduler=swh_scheduler) | lister = GolangLister(scheduler=swh_scheduler) | ||||
mocked_sleep = mocker.patch.object(lister.api_request.retry, "sleep") | |||||
_generate_responses(datadir, requests_mock) | _generate_responses(datadir, requests_mock) | ||||
stats = lister.run() | stats = lister.run() | ||||
assert stats.pages == 3 | assert stats.pages == 3 | ||||
assert stats.origins == 18 | assert stats.origins == 18 | ||||
def test_golang_lister_incremental(swh_scheduler, requests_mock, datadir, mocker): | def test_golang_lister_incremental(swh_scheduler, requests_mock, datadir, mocker): | ||||
▲ Show 20 Lines • Show All 99 Lines • Show Last 20 Lines |