Changeset View
Changeset View
Standalone View
Standalone View
swh/lister/bitbucket/lister.py
| # Copyright (C) 2017-2019 the Software Heritage developers | # Copyright (C) 2017-2019 the Software Heritage developers | ||||
| # License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
| # See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
| import logging | import logging | ||||
| import iso8601 | import iso8601 | ||||
| from datetime import datetime | from datetime import datetime | ||||
| from urllib import parse | from urllib import parse | ||||
| from swh.lister.bitbucket.models import BitBucketModel | from swh.lister.bitbucket.models import BitBucketModel | ||||
| from swh.lister.core.indexing_lister import IndexingHttpLister | from swh.lister.core.indexing_lister import IndexingHttpLister | ||||
| logger = logging.getLogger(__name__) | logger = logging.getLogger(__name__) | ||||
| DEFAULT_BITBUCKET_PAGE = 10 | |||||
| class BitBucketLister(IndexingHttpLister): | class BitBucketLister(IndexingHttpLister): | ||||
| PATH_TEMPLATE = '/repositories?after=%s' | PATH_TEMPLATE = '/repositories?after=%s' | ||||
| MODEL = BitBucketModel | MODEL = BitBucketModel | ||||
| LISTER_NAME = 'bitbucket' | LISTER_NAME = 'bitbucket' | ||||
| instance = 'bitbucket' | instance = 'bitbucket' | ||||
| default_min_bound = datetime.utcfromtimestamp(0) | default_min_bound = datetime.utcfromtimestamp(0) | ||||
| def __init__(self, api_baseurl, override_config=None, per_page=100): | def __init__(self, api_baseurl, override_config=None, per_page=100): | ||||
| super().__init__( | super().__init__( | ||||
| api_baseurl=api_baseurl, override_config=override_config) | api_baseurl=api_baseurl, override_config=override_config) | ||||
| if per_page != DEFAULT_BITBUCKET_PAGE: | per_page = self.config.get('per_page', per_page) | ||||
| self.PATH_TEMPLATE = '%s&pagelen=%s' % ( | self.PATH_TEMPLATE = '%s&pagelen=%s' % ( | ||||
| self.PATH_TEMPLATE, per_page) | self.PATH_TEMPLATE, per_page) | ||||
| # to stay consistent with prior behavior (20 * 10 repositories then) | |||||
| self.flush_packet_db = int( | |||||
| (self.flush_packet_db * DEFAULT_BITBUCKET_PAGE) / per_page) | |||||
| def get_model_from_repo(self, repo): | def get_model_from_repo(self, repo): | ||||
| return { | return { | ||||
| 'uid': repo['uuid'], | 'uid': repo['uuid'], | ||||
| 'indexable': iso8601.parse_date(repo['created_on']), | 'indexable': iso8601.parse_date(repo['created_on']), | ||||
| 'name': repo['name'], | 'name': repo['name'], | ||||
| 'full_name': repo['full_name'], | 'full_name': repo['full_name'], | ||||
| 'html_url': repo['links']['html']['href'], | 'html_url': repo['links']['html']['href'], | ||||
| Show All 40 Lines | |||||