	Show All 16 Lines
	DEFAULT_BITBUCKET_PAGE = 10			DEFAULT_BITBUCKET_PAGE = 10


	class BitBucketLister(IndexingHttpLister):			class BitBucketLister(IndexingHttpLister):
	PATH_TEMPLATE = '/repositories?after=%s'			PATH_TEMPLATE = '/repositories?after=%s'
	MODEL = BitBucketModel			MODEL = BitBucketModel
	LISTER_NAME = 'bitbucket'			LISTER_NAME = 'bitbucket'
	instance = 'bitbucket'			instance = 'bitbucket'
	default_min_bound = datetime.utcfromtimestamp(0).isoformat()			default_min_bound = datetime.utcfromtimestamp(0)

	def __init__(self, api_baseurl, override_config=None, per_page=100):			def __init__(self, api_baseurl, override_config=None, per_page=100):
	super().__init__(			super().__init__(
	api_baseurl=api_baseurl, override_config=override_config)			api_baseurl=api_baseurl, override_config=override_config)
	if per_page != DEFAULT_BITBUCKET_PAGE:			if per_page != DEFAULT_BITBUCKET_PAGE:
	self.PATH_TEMPLATE = '%s&pagelen=%s' % (			self.PATH_TEMPLATE = '%s&pagelen=%s' % (
	self.PATH_TEMPLATE, per_page)			self.PATH_TEMPLATE, per_page)
	# to stay consistent with prior behavior (20 * 10 repositories then)			# to stay consistent with prior behavior (20 * 10 repositories then)
	self.flush_packet_db = int(			self.flush_packet_db = int(
	(self.flush_packet_db * DEFAULT_BITBUCKET_PAGE) / per_page)			(self.flush_packet_db * DEFAULT_BITBUCKET_PAGE) / per_page)

	def get_model_from_repo(self, repo):			def get_model_from_repo(self, repo):
	return {			return {
	'uid': repo['uuid'],			'uid': repo['uuid'],
	'indexable': repo['created_on'],			'indexable': iso8601.parse_date(repo['created_on']),
	'name': repo['name'],			'name': repo['name'],
	'full_name': repo['full_name'],			'full_name': repo['full_name'],
	'html_url': repo['links']['html']['href'],			'html_url': repo['links']['html']['href'],
	'origin_url': repo['links']['clone'][0]['href'],			'origin_url': repo['links']['clone'][0]['href'],
	'origin_type': repo['scm'],			'origin_type': repo['scm'],
	}			}

	def get_next_target_from_response(self, response):			def get_next_target_from_response(self, response):
	"""This will read the 'next' link from the api response if any			"""This will read the 'next' link from the api response if any
	and return it as an isoformatted date string.			and return it as a datetime.

				Args:
				reponse (Response): requests' response from api call

	Returns:			Returns:
	next date as an isoformatted date string			next date as a datetime

	"""			"""
	body = response.json()			body = response.json()
	next_ = body.get('next')			next_ = body.get('next')
	if next_ is not None:			if next_ is not None:
	next_ = parse.urlparse(next_)			next_ = parse.urlparse(next_)
	return parse.parse_qs(next_.query)['after'][0]			return iso8601.parse_date(parse.parse_qs(next_.query)['after'][0])

	def transport_response_simplified(self, response):			def transport_response_simplified(self, response):
	repos = response.json()['values']			repos = response.json()['values']
	return [self.get_model_from_repo(repo) for repo in repos]			return [self.get_model_from_repo(repo) for repo in repos]

	def request_uri(self, identifier):			def request_uri(self, identifier):
	identifier = parse.quote(identifier)			identifier = parse.quote(identifier.isoformat())
	return super().request_uri(identifier or '1970-01-01')			return super().request_uri(identifier or '1970-01-01')

	def is_within_bounds(self, inner, lower=None, upper=None):			def is_within_bounds(self, inner, lower=None, upper=None):
	# values are expected to be str dates			# values are expected to be datetimes
	try:
	inner = iso8601.parse_date(inner)
	if lower:
	lower = iso8601.parse_date(lower)
	if upper:
	upper = iso8601.parse_date(upper)
	if lower is None and upper is None:			if lower is None and upper is None:
	return True			ret = True
	elif lower is None:			elif lower is None:
	ret = inner <= upper			ret = inner <= upper
	elif upper is None:			elif upper is None:
	ret = inner >= lower			ret = inner >= lower
	else:			else:
	ret = lower <= inner <= upper			ret = lower <= inner <= upper
	except Exception as e:
	logger.error(str(e) + ': %s, %s, %s',
	('inner=%s%s' % (type(inner), inner)),
	('lower=%s%s' % (type(lower), lower)),
	('upper=%s%s' % (type(upper), upper)))
	raise

	return ret			return ret

View Options

swh/lister/bitbucket/models.py

	# Copyright (C) 2017-2019 the Software Heritage developers			# Copyright (C) 2017-2019 the Software Heritage developers
	# License: GNU General Public License version 3, or any later version			# License: GNU General Public License version 3, or any later version
	# See top-level LICENSE file for more information			# See top-level LICENSE file for more information

	from sqlalchemy import Column, String			from sqlalchemy import Column, String, DateTime

	from swh.lister.core.models import IndexingModelBase			from swh.lister.core.models import IndexingModelBase


	class BitBucketModel(IndexingModelBase):			class BitBucketModel(IndexingModelBase):
	"""a BitBucket repository"""			"""a BitBucket repository"""
	__tablename__ = 'bitbucket_repo'			__tablename__ = 'bitbucket_repo'

	uid = Column(String, primary_key=True)			uid = Column(String, primary_key=True)
	indexable = Column(String, index=True)			indexable = Column(DateTime(timezone=True), index=True)

View Options

swh/lister/bitbucket/tests/test_bb_lister.py

	# Copyright (C) 2017-2018 the Software Heritage developers			# Copyright (C) 2017-2019 the Software Heritage developers
	# License: GNU General Public License version 3, or any later version			# License: GNU General Public License version 3, or any later version
	# See top-level LICENSE file for more information			# See top-level LICENSE file for more information

	import re			import re
	import unittest			import unittest

				from datetime import timedelta

				from urllib.parse import unquote

				import iso8601
				import requests_mock

	from swh.lister.bitbucket.lister import BitBucketLister			from swh.lister.bitbucket.lister import BitBucketLister
	from swh.lister.core.tests.test_lister import HttpListerTester			from swh.lister.core.tests.test_lister import HttpListerTester


				def convert_type(req_index):
				return iso8601.parse_date(unquote(req_index))


	class BitBucketListerTester(HttpListerTester, unittest.TestCase):			class BitBucketListerTester(HttpListerTester, unittest.TestCase):
	Lister = BitBucketLister			Lister = BitBucketLister
	test_re = re.compile(r'/repositories\?after=([^?&]+)')			test_re = re.compile(r'/repositories\?after=([^?&]+)')
	lister_subdir = 'bitbucket'			lister_subdir = 'bitbucket'
	good_api_response_file = 'api_response.json'			good_api_response_file = 'api_response.json'
	bad_api_response_file = 'api_empty_response.json'			bad_api_response_file = 'api_empty_response.json'
	first_index = '2008-07-12T07:44:01.476818+00:00'			first_index = iso8601.parse_date('2008-07-12T07:44:01.476818+00:00')
	last_index = '2008-07-19T06:16:43.044743+00:00'			last_index = iso8601.parse_date('2008-07-19T06:16:43.044743+00:00')
	entries_per_page = 10			entries_per_page = 10
				indexable_type = staticmethod(convert_type)

				@requests_mock.Mocker()
				def test_fetch_none_nodb(self, http_mocker):
				"""Overridden because index is not an integer nor a string

				"""
				http_mocker.get(self.test_re, text=self.mock_response)
				fl = self.get_fl()

				self.disable_scheduler(fl)
				self.disable_db(fl)

				# stores no results
				fl.run(min_bound=self.first_index - timedelta(days=3),
				max_bound=self.first_index)

	def test_is_within_bounds(self):			def test_is_within_bounds(self):
	fl = self.get_fl()			fl = self.get_fl()
	self.assertTrue(fl.is_within_bounds(			self.assertTrue(fl.is_within_bounds(
	'2008-07-15', self.first_index, self.last_index))			iso8601.parse_date('2008-07-15'),
				self.first_index, self.last_index))
	self.assertFalse(fl.is_within_bounds(			self.assertFalse(fl.is_within_bounds(
	'2008-07-20', self.first_index, self.last_index))			iso8601.parse_date('2008-07-20'),
				self.first_index, self.last_index))
	self.assertFalse(fl.is_within_bounds(			self.assertFalse(fl.is_within_bounds(
	'2008-07-11', self.first_index, self.last_index))			iso8601.parse_date('2008-07-11'),
				self.first_index, self.last_index))

View Options

swh/lister/core/tests/test_lister.py

Show All 32 Lines	class HttpListerTesterBase(abc.ABC):
test_re = AbstractAttribute('Compiled regex matching the server url. Must'		test_re = AbstractAttribute('Compiled regex matching the server url. Must'
' capture the index value.')		' capture the index value.')
lister_subdir = AbstractAttribute('bitbucket, github, etc.')		lister_subdir = AbstractAttribute('bitbucket, github, etc.')
good_api_response_file = AbstractAttribute('Example good response body')		good_api_response_file = AbstractAttribute('Example good response body')
bad_api_response_file = AbstractAttribute('Example bad response body')		bad_api_response_file = AbstractAttribute('Example bad response body')
first_index = AbstractAttribute('First index in good_api_response')		first_index = AbstractAttribute('First index in good_api_response')
entries_per_page = AbstractAttribute('Number of results in good response')		entries_per_page = AbstractAttribute('Number of results in good response')
LISTER_NAME = 'fake-lister'		LISTER_NAME = 'fake-lister'
		indexable_type = str

# May need to override this if the headers are used for something		# May need to override this if the headers are used for something
def response_headers(self, request):		def response_headers(self, request):
return {}		return {}

# May need to override this if the server uses non-standard rate limiting		# May need to override this if the server uses non-standard rate limiting
# method.		# method.
# Please keep the requested retry delay reasonably low.		# Please keep the requested retry delay reasonably low.
Show All 12 Lines	def __init__(self, args, *kwargs):
if self.__class__ != HttpListerTesterBase:		if self.__class__ != HttpListerTesterBase:
self.run = TestCase.run.__get__(self, self.__class__)		self.run = TestCase.run.__get__(self, self.__class__)
else:		else:
self.run = noop		self.run = noop

def request_index(self, request):		def request_index(self, request):
m = self.test_re.search(request.path_url)		m = self.test_re.search(request.path_url)
if m and (len(m.groups()) > 0):		if m and (len(m.groups()) > 0):
return m.group(1)		return self.indexable_type(m.group(1))

def mock_response(self, request, context):		def mock_response(self, request, context):
self.fl.reset_backoff()		self.fl.reset_backoff()
self.rate_limit = 1		self.rate_limit = 1
context.status_code = 200		context.status_code = 200
custom_headers = self.response_headers(request)		custom_headers = self.response_headers(request)
context.headers.update(custom_headers)		context.headers.update(custom_headers)
req_index = self.request_index(request)		req_index = self.request_index(request)
▲ Show 20 Lines • Show All 156 Lines • Show Last 20 Lines