Changeset View
Changeset View
Standalone View
Standalone View
swh/lister/golang/tests/test_lister.py
| Show First 20 Lines • Show All 101 Lines • ▼ Show 20 Lines | for scheduled, (url, timestamp) in zip(scheduler_origins, expected_listed): | ||||
| assert scheduled.visit_type == "golang" | assert scheduled.visit_type == "golang" | ||||
| assert len(scheduler_origins) == len(expected_listed) | assert len(scheduler_origins) == len(expected_listed) | ||||
| # Test `time.sleep` is called with exponential retries | # Test `time.sleep` is called with exponential retries | ||||
| assert_sleep_calls( | assert_sleep_calls( | ||||
| mocker, mocked_sleep, [1, WAIT_EXP_BASE, 1, WAIT_EXP_BASE, 1, WAIT_EXP_BASE] | mocker, mocked_sleep, [1, WAIT_EXP_BASE, 1, WAIT_EXP_BASE, 1, WAIT_EXP_BASE] | ||||
| ) | ) | ||||
| # Incremental should list nothing | |||||
| stats = lister.run() | |||||
| assert stats.pages == 0 | |||||
| assert stats.origins == 0 | |||||
| # Paranoid | |||||
| stats = lister.run() | |||||
| assert stats.pages == 0 | |||||
| assert stats.origins == 0 | |||||
| def test_golang_lister_incremental(swh_scheduler, requests_mock, datadir): | |||||
| # first listing, should return one origin per package | |||||
| lister = GolangLister(scheduler=swh_scheduler) | |||||
| responses = [ | |||||
| {"text": Path(datadir, "page-1.txt").read_text(), "status_code": 200}, | |||||
| # Returns empty text when the list is exhausted | |||||
| {"text": "", "status_code": 200}, | |||||
| ] | |||||
| requests_mock.get(GolangLister.GOLANG_MODULES_INDEX_URL, responses) | |||||
| stats = lister.run() | |||||
| assert stats.pages == 1 | |||||
| assert stats.origins == 5 | |||||
| # Incremental should list nothing | |||||
| stats = lister.run() | |||||
| assert stats.pages == 0 | |||||
| assert stats.origins == 0 | |||||
| # Add more responses | |||||
| responses = [ | |||||
| {"text": Path(datadir, "page-2.txt").read_text(), "status_code": 200}, | |||||
vlorentz: ;)
(ditto above) | |||||
AlphareAuthorUnsubmitted Done Inline ActionsI had already tried that but datadir is a str and not a Path, maybe that's something that should be changed? Alphare: I had already tried that but `datadir` is a `str` and not a `Path`, maybe that's something that… | |||||
| {"text": "", "status_code": 200}, | |||||
| ] | |||||
| requests_mock.get(GolangLister.GOLANG_MODULES_INDEX_URL, responses) | |||||
| # Incremental should list new page | |||||
| stats = lister.run() | |||||
| assert stats.pages == 1 | |||||
| assert stats.origins == 4 | |||||
| # Incremental should list nothing again | |||||
| stats = lister.run() | |||||
| assert stats.pages == 0 | |||||
| assert stats.origins == 0 | |||||
| # Add yet more responses | |||||
| responses = [ | |||||
| {"text": Path(datadir, "page-3.txt").read_text(), "status_code": 200}, | |||||
| {"text": "", "status_code": 200}, | |||||
| ] | |||||
| requests_mock.get(GolangLister.GOLANG_MODULES_INDEX_URL, responses) | |||||
| # Incremental should list new page again | |||||
| stats = lister.run() | |||||
| assert stats.pages == 1 | |||||
| assert stats.origins == 10 | |||||
| # Incremental should list nothing one last time | |||||
| stats = lister.run() | |||||
| assert stats.pages == 0 | |||||
| assert stats.origins == 0 | |||||
;)
(ditto above)