Changeset View
Changeset View
Standalone View
Standalone View
swh/lister/github/lister.py
| # Copyright (C) 2017-2019 The Software Heritage developers | # Copyright (C) 2017-2020 The Software Heritage developers | ||||
| # See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
| # License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
| # See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
| import re | import re | ||||
| import time | |||||
| from typing import Any | from typing import Any | ||||
| from swh.lister.core.indexing_lister import IndexingHttpLister | from swh.lister.core.indexing_lister import IndexingHttpLister | ||||
| from swh.lister.github.models import GitHubModel | from swh.lister.github.models import GitHubModel | ||||
| class GitHubLister(IndexingHttpLister): | class GitHubLister(IndexingHttpLister): | ||||
| Show All 18 Lines | def get_model_from_repo(self, repo): | ||||
| } | } | ||||
| def transport_quota_check(self, response): | def transport_quota_check(self, response): | ||||
| x_rate_limit_remaining = response.headers.get('X-RateLimit-Remaining') | x_rate_limit_remaining = response.headers.get('X-RateLimit-Remaining') | ||||
| if not x_rate_limit_remaining: | if not x_rate_limit_remaining: | ||||
| return False, 0 | return False, 0 | ||||
| reqs_remaining = int(x_rate_limit_remaining) | reqs_remaining = int(x_rate_limit_remaining) | ||||
| if response.status_code == 403 and reqs_remaining == 0: | if response.status_code == 403 and reqs_remaining == 0: | ||||
| reset_at = int(response.headers['X-RateLimit-Reset']) | delay = int(response.headers['Retry-After']) | ||||
| delay = min(reset_at - time.time(), 3600) | |||||
| return True, delay | return True, delay | ||||
| return False, 0 | return False, 0 | ||||
| def get_next_target_from_response(self, response): | def get_next_target_from_response(self, response): | ||||
| if 'next' in response.links: | if 'next' in response.links: | ||||
| next_url = response.links['next']['url'] | next_url = response.links['next']['url'] | ||||
| return int(self.API_URL_INDEX_RE.match(next_url).group(1)) | return int(self.API_URL_INDEX_RE.match(next_url).group(1)) | ||||
| Show All 22 Lines | |||||