Differential D7329 Diff 26504 swh/lister/golang/tests/test_lister.py

Changeset View

Standalone View

swh/lister/golang/tests/test_lister.py

This file was added.

				# Copyright (C) 2022 The Software Heritage developers
				# See the AUTHORS file at the top-level directory of this distribution
				# License: GNU General Public License version 3, or any later version
				# See top-level LICENSE file for more information

				from pathlib import Path

				from dateutil import parser

				from swh.lister.golang.lister import GolangLister
				from swh.lister.tests.test_utils import assert_sleep_calls
				from swh.lister.utils import WAIT_EXP_BASE

				expected_listed = [
				("collectd.org/@v/v0.3.0.zip", "2019-04-11T18:47:25.450546+00:00"),
				(
				"github.com/blang/semver/@v/v3.5.1+incompatible.zip",
				"2019-04-15T13:54:39.107258+00:00",
				),
				(
				"github.com/bmizerany/pat/@v/v0.0.0-20170815010413-6226ea591a40.zip",
				"2019-04-11T18:47:29.390564+00:00",
				),
				("github.com/djherbis/buffer/@v/v1.0.0.zip", "2019-04-11T18:47:29.974874+00:00",),
				(
				"github.com/djherbis/nio/@v/v2.0.3+incompatible.zip",
				"2019-04-11T18:47:32.283312+00:00",
				),
				(
				"github.com/gobuffalo/buffalo-plugins/@v/v1.13.0.zip",
				"2019-04-15T13:54:34.222985+00:00",
				),
				(
				"github.com/gobuffalo/buffalo-pop/@v/v1.3.0.zip",
				"2019-04-15T13:54:39.135792+00:00",
				),
				("github.com/gobuffalo/clara/@v/v0.4.1.zip", "2019-04-15T13:54:40.651916+00:00",),
				(
				"github.com/gobuffalo/genny/@v/v0.0.0-20190104222617-a71664fc38e7.zip",
				"2019-04-15T13:54:37.841547+00:00",
				),
				("github.com/gobuffalo/packr/@v/v1.22.0.zip", "2019-04-15T13:54:35.688900+00:00",),
				("github.com/markbates/refresh/@v/v1.7.1.zip", "2019-04-15T13:54:35.250835+00:00",),
				(
				"github.com/mitchellh/go-homedir/@v/v1.1.0.zip",
				"2019-04-15T13:54:35.678214+00:00",
				),
				("github.com/nats-io/nuid/@v/v1.0.1.zip", "2019-04-11T18:47:28.102348+00:00",),
				("github.com/oklog/ulid/@v/v1.3.1.zip", "2019-04-11T18:47:23.234198+00:00",),
				(
				"github.com/pkg/errors/@v/v0.0.0-20161002052512-839d9e913e06.zip",
				"2019-04-18T02:07:41.336899+00:00",
				),
				(
				"golang.org/x/sys/@v/v0.0.0-20190220154126-629670e5acc5.zip",
				"2019-04-15T13:54:37.555525+00:00",
				),
				("golang.org/x/text/@v/v0.3.0.zip", "2019-04-10T19:08:52.997264+00:00"),
				(
				"golang.org/x/tools/@v/v0.0.0-20181213190329-bbccd8cae4a9.zip",
				"2019-04-15T13:54:41.905064+00:00",
				),
				(
				"golang.org/x/tools/@v/v0.0.0-20190131142011-8dbcc66f33bb.zip",
				"2019-04-15T13:54:39.250757+00:00",
				),
				]


				def test_golang_lister(swh_scheduler, mocker, requests_mock, datadir):
				# first listing, should return one origin per package
				lister = GolangLister(scheduler=swh_scheduler)

				# Exponential retries take a long time, so stub time.sleep
				mocked_sleep = mocker.patch.object(lister.api_request.retry, "sleep")

				responses = []
				for file in Path(datadir).glob("page-*.txt"):
				# Test that throttling and server errors are retries
				responses.append({"text": "", "status_code": 429})
				responses.append({"text": "", "status_code": 500})
				responses.append({"text": file.read_text(), "status_code": 200})

				# Returns empty text when the list is exhausted
				responses.append({"text": "", "status_code": 200})
				requests_mock.get(GolangLister.GOLANG_MODULES_INDEX_URL, responses)

				stats = lister.run()

				assert stats.pages == 3
				# The two `golang.org/x/tools` versions are listed as separate origins
				assert stats.origins == 19

				scheduler_origins = sorted(
				swh_scheduler.get_listed_origins(lister.lister_obj.id).results,
				key=lambda x: x.url,
				)

				for scheduled, (url, timestamp) in zip(scheduler_origins, expected_listed):
				assert scheduled.url == url
				assert scheduled.last_update == parser.isoparse(timestamp)
				assert scheduled.visit_type == "golang"

				assert len(scheduler_origins) == len(expected_listed)

				# Test `time.sleep` is called with exponential retries
				assert_sleep_calls(
				mocker, mocked_sleep, [1, WAIT_EXP_BASE, 1, WAIT_EXP_BASE, 1, WAIT_EXP_BASE]
				)