Differential D2408 Diff 8528 swh/lister/bitbucket/lister.py

Changeset View

Standalone View

swh/lister/bitbucket/lister.py

	# Copyright (C) 2017-2019 The Software Heritage developers			# Copyright (C) 2017-2019 The Software Heritage developers
	# See the AUTHORS file at the top-level directory of this distribution			# See the AUTHORS file at the top-level directory of this distribution
	# License: GNU General Public License version 3, or any later version			# License: GNU General Public License version 3, or any later version
	# See top-level LICENSE file for more information			# See top-level LICENSE file for more information

	import logging			import logging
	import iso8601			import iso8601

	from datetime import datetime, timezone			from datetime import datetime, timezone
	from typing import Any			from typing import Any
	from urllib import parse			from urllib import parse


	from swh.lister.bitbucket.models import BitBucketModel			from swh.lister.bitbucket.models import BitBucketModel
	from swh.lister.core.indexing_lister import IndexingHttpLister			from swh.lister.core.indexing_lister import IndexingHttpLister


	logger = logging.getLogger(__name__)			logger = logging.getLogger(__name__)


	class BitBucketLister(IndexingHttpLister):			class BitBucketLister(IndexingHttpLister):
	PATH_TEMPLATE = '/repositories?after=%s'			PATH_TEMPLATE = '/repositories?after=%s'
	MODEL = BitBucketModel			MODEL = BitBucketModel
	LISTER_NAME = 'bitbucket'			LISTER_NAME = 'bitbucket'
	DEFAULT_URL = 'https://api.bitbucket.org/2.0'			DEFAULT_URL = 'https://api.bitbucket.org/2.0'
	instance = 'bitbucket'			instance = 'bitbucket'
	default_min_bound = datetime.fromtimestamp(0, timezone.utc) # type: Any			default_min_bound = datetime.fromtimestamp(0, timezone.utc) # type: Any

	def __init__(self, url=None, override_config=None, per_page=100):			def __init__(self, url=None, override_config=None, per_page=100):
	super().__init__(url=url, override_config=override_config)			super().__init__(url=url, override_config=override_config)
	per_page = self.config.get('per_page', per_page)			per_page = self.config.get('per_page', per_page)

	self.PATH_TEMPLATE = '%s&pagelen=%s' % (			self.PATH_TEMPLATE = '%s&pagelen=%s' % (
	self.PATH_TEMPLATE, per_page)			self.PATH_TEMPLATE, per_page)

	def get_model_from_repo(self, repo):			def get_model_from_repo(self, repo):
				anlambertUnsubmitted Not Done Inline Actions s/Mapping/Dict/ anlambert: s/Mapping/Dict/
	return {			return {
	'uid': repo['uuid'],			'uid': repo['uuid'],
	'indexable': iso8601.parse_date(repo['created_on']),			'indexable': iso8601.parse_date(repo['created_on']),
	'name': repo['name'],			'name': repo['name'],
	'full_name': repo['full_name'],			'full_name': repo['full_name'],
	'html_url': repo['links']['html']['href'],			'html_url': repo['links']['html']['href'],
	'origin_url': repo['links']['clone'][0]['href'],			'origin_url': repo['links']['clone'][0]['href'],
	'origin_type': repo['scm'],			'origin_type': repo['scm'],
	}			}

	def get_next_target_from_response(self, response):			def get_next_target_from_response(self, response):
	"""This will read the 'next' link from the api response if any			"""This will read the 'next' link from the api response if any
	and return it as a datetime.			and return it as a datetime.

	Args:			Args:
	response (Response): requests' response from api call			response (Response): requests' response from api call
				anlambertUnsubmitted Not Done Inline Actions I think you should rather call the `utils.create_task_dict` function with keyword arguments directly in the `task_dict` method from the `ListerBase` class instead of overriding it here. anlambert: I think you should rather call the `utils.create_task_dict` function with keyword arguments…
				ardumontAuthorUnsubmitted Done Inline Actions Yes. Adapting right now. Relatedly, i also think i need to align the loader-git accordingly... D2409 ardumont: Yes. Adapting right now. Relatedly, i also think i need to align the loader-git accordingly...

	Returns:			Returns:
	next date as a datetime			next date as a datetime

	"""			"""
	body = response.json()			body = response.json()
	next_ = body.get('next')			next_ = body.get('next')
	if next_ is not None:			if next_ is not None:
	Show All 22 Lines