Differential D1629 Diff 5431 swh/lister/bitbucket/lister.py

Changeset View

Standalone View

swh/lister/bitbucket/lister.py

# Copyright (C) 2017-2019 the Software Heritage developers		# Copyright (C) 2017-2019 the Software Heritage developers
# License: GNU General Public License version 3, or any later version		# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information		# See top-level LICENSE file for more information

from urllib import parse		import datetime
import logging		import logging
import iso8601		import iso8601

		from urllib import parse

from swh.lister.bitbucket.models import BitBucketModel		from swh.lister.bitbucket.models import BitBucketModel
from swh.lister.core.indexing_lister import IndexingHttpLister		from swh.lister.core.indexing_lister import IndexingHttpLister


logger = logging.getLogger(__name__)		logger = logging.getLogger(__name__)


class BitBucketLister(IndexingHttpLister):		class BitBucketLister(IndexingHttpLister):
Show All 17 Lines	def get_next_target_from_response(self, response):
body = response.json()		body = response.json()
if 'next' in body:		if 'next' in body:
return parse.unquote(body['next'].split('after=')[1])		return parse.unquote(body['next'].split('after=')[1])

def transport_response_simplified(self, response):		def transport_response_simplified(self, response):
repos = response.json()['values']		repos = response.json()['values']
return [self.get_model_from_repo(repo) for repo in repos]		return [self.get_model_from_repo(repo) for repo in repos]

		def db_first_index(self):
		"""For the first time listing, there is no data in db, so fallback to the
		bitbucket starting year.

		"""
		return super().db_first_index() or '2008-01-01T00:00:00Z'

		def db_last_index(self):
		"""For the first time listing, there is no data in db, so fallback to the time
		of the first run as max date.

		"""
		ardumontAuthorUnsubmitted Done Inline Actions @olasd in the end, i kept the `db_last_index` because tests were annoying me... ardumont: @olasd in the end, i kept the `db_last_index` because tests were annoying me...
		return super().db_last_index() or datetime.datetime.now(
		tz=datetime.timezone.utc).isoformat()

def request_uri(self, identifier):		def request_uri(self, identifier):
return super().request_uri(identifier or '1970-01-01')		return super().request_uri(identifier or '1970-01-01')

def is_within_bounds(self, inner, lower=None, upper=None):		def is_within_bounds(self, inner, lower=None, upper=None):
# values are expected to be str dates		# values are expected to be str dates
try:		try:
inner = iso8601.parse_date(inner)		inner = iso8601.parse_date(inner)
if lower:		if lower:
Show All 20 Lines