Changeset View
Changeset View
Standalone View
Standalone View
swh/lister/gitlab/lister.py
| # Copyright (C) 2018 the Software Heritage developers | # Copyright (C) 2018 the Software Heritage developers | ||||
| # License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
| # See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
| import random | import random | ||||
| import re | import re | ||||
| import time | import time | ||||
| from ..core.paging_lister import SWHPagingHttpLister | from ..core.paging_lister import SWHPagingHttpLister | ||||
| from .models import GitLabModel | from .models import GitLabModel | ||||
| class GitLabLister(SWHPagingHttpLister): | class GitLabLister(SWHPagingHttpLister): | ||||
| # Template path expecting an integer that represents the page id | # Template path expecting an integer that represents the page id | ||||
| PATH_TEMPLATE = '/projects?page=%d&order_by=id&sort=asc&simple=true' | PATH_TEMPLATE = '/projects?page=%d&order_by=id' | ||||
| API_URL_INDEX_RE = re.compile(r'^.*/projects.*page=(\d+).*') | API_URL_INDEX_RE = re.compile(r'^.*/projects.*page=(\d+).*') | ||||
| MODEL = GitLabModel | MODEL = GitLabModel | ||||
| def __init__(self, lister_name=None, api_baseurl=None, | |||||
| override_config=None, | |||||
| sort='asc'): | |||||
| super().__init__(lister_name=lister_name, api_baseurl=api_baseurl, | |||||
| override_config=override_config) | |||||
| self.PATH_TEMPLATE = '%s&sort=%s' % (self.PATH_TEMPLATE, sort) | |||||
| @property | @property | ||||
| def CONFIG_BASE_FILENAME(self): | def CONFIG_BASE_FILENAME(self): | ||||
| """One gitlab lister for all instances. We discriminate between the | """One gitlab lister for all instances. We discriminate between the | ||||
| origin on a per instance basis in the table. | origin on a per instance basis in the table. | ||||
| """ | """ | ||||
| return 'lister-gitlab' | return 'lister-gitlab' | ||||
| @property | @property | ||||
| def ADDITIONAL_CONFIG(self): | def ADDITIONAL_CONFIG(self): | ||||
| """Override additional config as the 'credentials' structure change | """Override additional config as the 'credentials' structure change | ||||
| between the ancestor classes and this class. | between the ancestor classes and this class. | ||||
| ▲ Show 20 Lines • Show All 73 Lines • ▼ Show 20 Lines | def get_next_target_from_response(self, response): | ||||
| """ | """ | ||||
| if 'next' in response.links: | if 'next' in response.links: | ||||
| next_url = response.links['next']['url'] | next_url = response.links['next']['url'] | ||||
| return int(self.API_URL_INDEX_RE.match(next_url).group(1)) | return int(self.API_URL_INDEX_RE.match(next_url).group(1)) | ||||
| return None | return None | ||||
| def get_pages_information(self): | def get_pages_information(self): | ||||
| """Determine some pages information. | """Determine pages information. | ||||
| """ | """ | ||||
| response = self.transport_head(identifier=1) | response = self.transport_head(identifier=1) | ||||
| h = response.headers | h = response.headers | ||||
| total = h.get('x-total', h.get('X-Total')) | total = h.get('x-total', h.get('X-Total')) | ||||
| total_pages = h.get('x-total-pages', h.get('X-Total-Pages')) | total_pages = h.get('x-total-pages', h.get('X-Total-Pages')) | ||||
| per_page = h.get('x-per-page', h.get('X-Per-Page')) | per_page = h.get('x-per-page', h.get('X-Per-Page')) | ||||
| if total is not None: | if total is not None: | ||||
| Show All 10 Lines | |||||