Differential D4843 Diff 17317 swh/lister/bitbucket/tests/test_lister.py

Changeset View

Standalone View

swh/lister/bitbucket/tests/test_lister.py

	# Copyright (C) 2017-2020 The Software Heritage developers			# Copyright (C) 2017-2021 The Software Heritage developers
	# See the AUTHORS file at the top-level directory of this distribution			# See the AUTHORS file at the top-level directory of this distribution
	# License: GNU General Public License version 3, or any later version			# License: GNU General Public License version 3, or any later version
	# See top-level LICENSE file for more information			# See top-level LICENSE file for more information

	from datetime import timedelta			from datetime import datetime
	import re			import json
	import unittest			import os
	from urllib.parse import unquote

	import iso8601
	import requests_mock

	from swh.lister.bitbucket.lister import BitBucketLister
	from swh.lister.core.tests.test_lister import HttpListerTester


	def _convert_type(req_index):
	"""Convert the req_index to its right type according to the model's
	"indexable" column.

	"""
	return iso8601.parse_date(unquote(req_index))


	class BitBucketListerTester(HttpListerTester, unittest.TestCase):
	Lister = BitBucketLister
	test_re = re.compile(r"/repositories\?after=([^?&]+)")
	lister_subdir = "bitbucket"
	good_api_response_file = "data/https_api.bitbucket.org/response.json"
	bad_api_response_file = "data/https_api.bitbucket.org/empty_response.json"
	first_index = _convert_type("2008-07-12T07:44:01.476818+00:00")
	last_index = _convert_type("2008-07-19T06:16:43.044743+00:00")
	entries_per_page = 10
	convert_type = _convert_type

	def request_index(self, request):
	"""(Override) This is needed to emulate the listing bootstrap
	when no min_bound is provided to run
	"""
	m = self.test_re.search(request.path_url)
	idx = _convert_type(m.group(1))
	if idx == self.Lister.default_min_bound:
	idx = self.first_index
	return idx

	@requests_mock.Mocker()
	def test_fetch_none_nodb(self, http_mocker):
	"""Overridden because index is not an integer nor a string

	"""
	http_mocker.get(self.test_re, text=self.mock_response)
	fl = self.get_fl()

	self.disable_scheduler(fl)
	self.disable_db(fl)

	# stores no results
	fl.run(
	min_bound=self.first_index - timedelta(days=3), max_bound=self.first_index
	)

	def test_is_within_bounds(self):			import pytest
	fl = self.get_fl()
	self.assertTrue(			from swh.lister.bitbucket.lister import BitbucketLister
	fl.is_within_bounds(
	iso8601.parse_date("2008-07-15"), self.first_index, self.last_index
	)			@pytest.fixture
	)			def bb_api_repositories_page1(datadir):
	self.assertFalse(			data_file_path = os.path.join(datadir, "bb_api_repositories_page1.json")
	fl.is_within_bounds(			with open(data_file_path, "r") as data_file:
	iso8601.parse_date("2008-07-20"), self.first_index, self.last_index			return json.load(data_file)


				@pytest.fixture
				def bb_api_repositories_page2(datadir):
				data_file_path = os.path.join(datadir, "bb_api_repositories_page2.json")
				with open(data_file_path, "r") as data_file:
				return json.load(data_file)


				def _check_listed_origins(lister_origins, scheduler_origins):
				"""Asserts that the two collections have the same origins from the point of view of
				the lister"""

				sorted_lister_origins = list(sorted(lister_origins))
				sorted_scheduler_origins = list(sorted(scheduler_origins))

				assert len(sorted_lister_origins) == len(sorted_scheduler_origins)

				for lo, so in zip(sorted_lister_origins, sorted_scheduler_origins):
				assert lo.url == so.url
				assert lo.last_update == so.last_update


				def test_bitbucket_incremental_lister(
				swh_scheduler,
				requests_mock,
				mocker,
				bb_api_repositories_page1,
				bb_api_repositories_page2,
				):
				"""Simple Bitbucket listing with two pages containing 10 origins"""

				requests_mock.get(
				BitbucketLister.API_URL,
				[{"json": bb_api_repositories_page1}, {"json": bb_api_repositories_page2},],
	)			)

				lister = BitbucketLister(scheduler=swh_scheduler, page_size=10)

				# First listing
				stats = lister.run()

				scheduler_origins = swh_scheduler.get_listed_origins(lister.lister_obj.id).origins

				assert stats.pages == 2
				assert stats.origins == 20
				assert len(scheduler_origins) == 20

				assert lister.updated
				lister_state = lister.get_state_from_scheduler()
				last_repo_cdate = lister_state.last_repo_cdate.isoformat()
				assert hasattr(lister_state, "last_repo_cdate")
				assert last_repo_cdate == bb_api_repositories_page2["values"][-1]["created_on"]

				# Second listing, restarting from last state
				lister.session.get = mocker.spy(lister.session, "get")

				lister.run()

				url_params = lister.url_params
				url_params["after"] = last_repo_cdate

				lister.session.get.assert_called_once_with(lister.API_URL, params=url_params)

				all_origins = (
				bb_api_repositories_page1["values"] + bb_api_repositories_page2["values"]
	)			)
	self.assertFalse(
	fl.is_within_bounds(			_check_listed_origins(lister.get_origins_from_page(all_origins), scheduler_origins)
	iso8601.parse_date("2008-07-11"), self.first_index, self.last_index

				def test_bitbucket_lister_rate_limit_hit(
				swh_scheduler,
				requests_mock,
				mocker,
				bb_api_repositories_page1,
				bb_api_repositories_page2,
				):
				"""Simple Bitbucket listing with two pages containing 10 origins"""

				requests_mock.get(
				BitbucketLister.API_URL,
				[
				{"json": bb_api_repositories_page1, "status_code": 200},
				{"json": None, "status_code": 429},
				{"json": None, "status_code": 429},
				{"json": bb_api_repositories_page2, "status_code": 200},
				],
	)			)

				lister = BitbucketLister(scheduler=swh_scheduler, page_size=10)

				mocker.patch.object(lister.page_request.retry, "sleep")

				stats = lister.run()

				assert stats.pages == 2
				assert stats.origins == 20
				assert len(swh_scheduler.get_listed_origins(lister.lister_obj.id).origins) == 20


				def test_bitbucket_full_lister(
				swh_scheduler,
				requests_mock,
				mocker,
				bb_api_repositories_page1,
				bb_api_repositories_page2,
				):
				"""Simple Bitbucket listing with two pages containing 10 origins"""

				requests_mock.get(
				BitbucketLister.API_URL,
				[
				{"json": bb_api_repositories_page1},
				{"json": bb_api_repositories_page2},
				{"json": bb_api_repositories_page1},
				{"json": bb_api_repositories_page2},
				],
	)			)

				lister = BitbucketLister(scheduler=swh_scheduler, page_size=10, incremental=True)

	def test_lister_bitbucket(lister_bitbucket, requests_mock_datadir):			# First do a incremental run to have an initial lister state
	"""Simple bitbucket listing should create scheduled tasks (git, hg)			stats = lister.run()

	"""			last_lister_state = lister.get_state_from_scheduler()
	lister_bitbucket.run()

	r = lister_bitbucket.scheduler.search_tasks(task_type="load-hg")			assert stats.origins == 20
	assert len(r) == 9

	for row in r:			# Then do the full run and verify lister state did not change
	args = row["arguments"]["args"]			# Modify last listed repo modification date to check it will be not saved
	kwargs = row["arguments"]["kwargs"]			# to lister state after its execution
				last_page2_repo = bb_api_repositories_page2["values"][-1]
				last_page2_repo["created_on"] = datetime.now().isoformat()
				last_page2_repo["updated_on"] = datetime.now().isoformat()

	assert len(args) == 0			lister = BitbucketLister(scheduler=swh_scheduler, page_size=10, incremental=False)
	assert len(kwargs) == 1
	url = kwargs["url"]

	assert url.startswith("https://bitbucket.org")			stats = lister.run()

	assert row["policy"] == "recurring"			assert stats.pages == 2
	assert row["priority"] is None			assert stats.origins == 20

	r = lister_bitbucket.scheduler.search_tasks(task_type="load-git")			scheduler_origins = swh_scheduler.get_listed_origins(lister.lister_obj.id).origins
	assert len(r) == 1

				vlorentzUnsubmitted Done Inline Actions isn't this a method? vlorentz: isn't this a method?
				tenmaAuthorUnsubmitted Done Inline Actions oops, will fix this. tenma: oops, will fix this.
	for row in r:			# 20 because scheduler upserts based on (id, type, url)
	args = row["arguments"]["args"]			assert len(scheduler_origins) == 20
	kwargs = row["arguments"]["kwargs"]
	assert len(args) == 0
	assert len(kwargs) == 1
	url = kwargs["url"]

	assert url.startswith("https://bitbucket.org")			# Modification on created_on SHOULD NOT impact lister state
				assert lister.get_state_from_scheduler() == last_lister_state

				# Modification on updated_on SHOULD impact lister state
				all_origins = (
				bb_api_repositories_page1["values"] + bb_api_repositories_page2["values"]
				)

	assert row["policy"] == "recurring"			_check_listed_origins(lister.get_origins_from_page(all_origins), scheduler_origins)
	assert row["priority"] is None