Differential D4843 Diff 17396 swh/lister/bitbucket/tests/test_lister.py

Changeset View

Standalone View

swh/lister/bitbucket/tests/test_lister.py

	# Copyright (C) 2017-2020 The Software Heritage developers			# Copyright (C) 2017-2021 The Software Heritage developers
	# See the AUTHORS file at the top-level directory of this distribution			# See the AUTHORS file at the top-level directory of this distribution
	# License: GNU General Public License version 3, or any later version			# License: GNU General Public License version 3, or any later version
	# See top-level LICENSE file for more information			# See top-level LICENSE file for more information

	from datetime import timedelta			from datetime import datetime
	import re			import json
	import unittest			import os
	from urllib.parse import unquote

	import iso8601			import pytest
	import requests_mock

	from swh.lister.bitbucket.lister import BitBucketLister			from swh.lister.bitbucket.lister import BitbucketLister
	from swh.lister.core.tests.test_lister import HttpListerTester


	def _convert_type(req_index):			@pytest.fixture
	"""Convert the req_index to its right type according to the model's			def bb_api_repositories_page1(datadir):
	"indexable" column.			data_file_path = os.path.join(datadir, "bb_api_repositories_page1.json")
				with open(data_file_path, "r") as data_file:
				return json.load(data_file)

	"""
	return iso8601.parse_date(unquote(req_index))

				@pytest.fixture
				def bb_api_repositories_page2(datadir):
				data_file_path = os.path.join(datadir, "bb_api_repositories_page2.json")
				with open(data_file_path, "r") as data_file:
				return json.load(data_file)

	class BitBucketListerTester(HttpListerTester, unittest.TestCase):
	Lister = BitBucketLister
	test_re = re.compile(r"/repositories\?after=([^?&]+)")
	lister_subdir = "bitbucket"
	good_api_response_file = "data/https_api.bitbucket.org/response.json"
	bad_api_response_file = "data/https_api.bitbucket.org/empty_response.json"
	first_index = _convert_type("2008-07-12T07:44:01.476818+00:00")
	last_index = _convert_type("2008-07-19T06:16:43.044743+00:00")
	entries_per_page = 10
	convert_type = _convert_type

	def request_index(self, request):			def _check_listed_origins(lister_origins, scheduler_origins):
	"""(Override) This is needed to emulate the listing bootstrap			"""Asserts that the two collections have the same origins from the point of view of
	when no min_bound is provided to run			the lister"""
	"""
	m = self.test_re.search(request.path_url)
	idx = _convert_type(m.group(1))
	if idx == self.Lister.default_min_bound:
	idx = self.first_index
	return idx

	@requests_mock.Mocker()			sorted_lister_origins = list(sorted(lister_origins))
	def test_fetch_none_nodb(self, http_mocker):			sorted_scheduler_origins = list(sorted(scheduler_origins))
	"""Overridden because index is not an integer nor a string

	"""			assert len(sorted_lister_origins) == len(sorted_scheduler_origins)
	http_mocker.get(self.test_re, text=self.mock_response)
	fl = self.get_fl()

	self.disable_scheduler(fl)			for lo, so in zip(sorted_lister_origins, sorted_scheduler_origins):
	self.disable_db(fl)			assert lo.url == so.url
				assert lo.last_update == so.last_update

	# stores no results
	fl.run(
	min_bound=self.first_index - timedelta(days=3), max_bound=self.first_index
	)

	def test_is_within_bounds(self):			def test_bitbucket_incremental_lister(
	fl = self.get_fl()			swh_scheduler,
	self.assertTrue(			requests_mock,
	fl.is_within_bounds(			mocker,
	iso8601.parse_date("2008-07-15"), self.first_index, self.last_index			bb_api_repositories_page1,
	)			bb_api_repositories_page2,
				):
				"""Simple Bitbucket listing with two pages containing 10 origins"""

				requests_mock.get(
				BitbucketLister.API_URL,
				[{"json": bb_api_repositories_page1}, {"json": bb_api_repositories_page2},],
	)			)
	self.assertFalse(
	fl.is_within_bounds(			lister = BitbucketLister(scheduler=swh_scheduler, page_size=10)
	iso8601.parse_date("2008-07-20"), self.first_index, self.last_index
				# First listing
				stats = lister.run()

				scheduler_origins = swh_scheduler.get_listed_origins(lister.lister_obj.id).origins

				assert stats.pages == 2
				assert stats.origins == 20
				assert len(scheduler_origins) == 20

				assert lister.updated
				lister_state = lister.get_state_from_scheduler()
				last_repo_cdate = lister_state.last_repo_cdate.isoformat()
				assert hasattr(lister_state, "last_repo_cdate")
				assert last_repo_cdate == bb_api_repositories_page2["values"][-1]["created_on"]

				# Second listing, restarting from last state
				lister.session.get = mocker.spy(lister.session, "get")

				lister.run()

				url_params = lister.url_params
				url_params["after"] = last_repo_cdate

				lister.session.get.assert_called_once_with(lister.API_URL, params=url_params)

				all_origins = (
				bb_api_repositories_page1["values"] + bb_api_repositories_page2["values"]
	)			)

				_check_listed_origins(lister.get_origins_from_page(all_origins), scheduler_origins)


				def test_bitbucket_lister_rate_limit_hit(
				swh_scheduler,
				requests_mock,
				mocker,
				bb_api_repositories_page1,
				bb_api_repositories_page2,
				):
				"""Simple Bitbucket listing with two pages containing 10 origins"""

				requests_mock.get(
				BitbucketLister.API_URL,
				[
				{"json": bb_api_repositories_page1, "status_code": 200},
				{"json": None, "status_code": 429},
				{"json": None, "status_code": 429},
				{"json": bb_api_repositories_page2, "status_code": 200},
				],
	)			)
	self.assertFalse(
	fl.is_within_bounds(			lister = BitbucketLister(scheduler=swh_scheduler, page_size=10)
	iso8601.parse_date("2008-07-11"), self.first_index, self.last_index
				mocker.patch.object(lister.page_request.retry, "sleep")

				stats = lister.run()

				assert stats.pages == 2
				assert stats.origins == 20
				assert len(swh_scheduler.get_listed_origins(lister.lister_obj.id).origins) == 20


				def test_bitbucket_full_lister(
				swh_scheduler,
				requests_mock,
				mocker,
				bb_api_repositories_page1,
				bb_api_repositories_page2,
				):
				"""Simple Bitbucket listing with two pages containing 10 origins"""

				requests_mock.get(
				BitbucketLister.API_URL,
				[
				{"json": bb_api_repositories_page1},
				{"json": bb_api_repositories_page2},
				{"json": bb_api_repositories_page1},
				{"json": bb_api_repositories_page2},
				],
	)			)

				credentials = {"bitbucket": {"bitbucket": [{"username": "u", "password": "p"}]}}
				lister = BitbucketLister(
				scheduler=swh_scheduler, page_size=10, incremental=True, credentials=credentials
	)			)
				assert lister.session.auth is not None

				# First do a incremental run to have an initial lister state
				stats = lister.run()

	def test_lister_bitbucket(lister_bitbucket, requests_mock_datadir):			last_lister_state = lister.get_state_from_scheduler()
	"""Simple bitbucket listing should create scheduled tasks (git, hg)

	"""			assert stats.origins == 20
	lister_bitbucket.run()

	r = lister_bitbucket.scheduler.search_tasks(task_type="load-hg")			# Then do the full run and verify lister state did not change
	assert len(r) == 9			# Modify last listed repo modification date to check it will be not saved
				# to lister state after its execution
				last_page2_repo = bb_api_repositories_page2["values"][-1]
				last_page2_repo["created_on"] = datetime.now().isoformat()
				last_page2_repo["updated_on"] = datetime.now().isoformat()

	for row in r:			lister = BitbucketLister(scheduler=swh_scheduler, page_size=10, incremental=False)
	args = row["arguments"]["args"]			assert lister.session.auth is None
	kwargs = row["arguments"]["kwargs"]

	assert len(args) == 0			stats = lister.run()
	assert len(kwargs) == 1
	url = kwargs["url"]

				vlorentzUnsubmitted Done Inline Actions isn't this a method? vlorentz: isn't this a method?
				tenmaAuthorUnsubmitted Done Inline Actions oops, will fix this. tenma: oops, will fix this.
	assert url.startswith("https://bitbucket.org")			assert stats.pages == 2
				assert stats.origins == 20

	assert row["policy"] == "recurring"			scheduler_origins = swh_scheduler.get_listed_origins(lister.lister_obj.id).origins
	assert row["priority"] is None

	r = lister_bitbucket.scheduler.search_tasks(task_type="load-git")			# 20 because scheduler upserts based on (id, type, url)
	assert len(r) == 1			assert len(scheduler_origins) == 20

	for row in r:			# Modification on created_on SHOULD NOT impact lister state
	args = row["arguments"]["args"]			assert lister.get_state_from_scheduler() == last_lister_state
	kwargs = row["arguments"]["kwargs"]
	assert len(args) == 0
	assert len(kwargs) == 1
	url = kwargs["url"]

	assert url.startswith("https://bitbucket.org")			# Modification on updated_on SHOULD impact lister state
				all_origins = (
				bb_api_repositories_page1["values"] + bb_api_repositories_page2["values"]
				)

	assert row["policy"] == "recurring"			_check_listed_origins(lister.get_origins_from_page(all_origins), scheduler_origins)
	assert row["priority"] is None