Changeset View
Changeset View
Standalone View
Standalone View
swh/lister/github/lister.py
# Copyright (C) 2017-2019 The Software Heritage developers | # Copyright (C) 2017-2020 The Software Heritage developers | ||||
# See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
# License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
# See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
import re | import re | ||||
import time | |||||
from typing import Any | from typing import Any | ||||
from swh.lister.core.indexing_lister import IndexingHttpLister | from swh.lister.core.indexing_lister import IndexingHttpLister | ||||
from swh.lister.github.models import GitHubModel | from swh.lister.github.models import GitHubModel | ||||
class GitHubLister(IndexingHttpLister): | class GitHubLister(IndexingHttpLister): | ||||
Show All 18 Lines | def get_model_from_repo(self, repo): | ||||
} | } | ||||
def transport_quota_check(self, response): | def transport_quota_check(self, response): | ||||
x_rate_limit_remaining = response.headers.get('X-RateLimit-Remaining') | x_rate_limit_remaining = response.headers.get('X-RateLimit-Remaining') | ||||
if not x_rate_limit_remaining: | if not x_rate_limit_remaining: | ||||
return False, 0 | return False, 0 | ||||
reqs_remaining = int(x_rate_limit_remaining) | reqs_remaining = int(x_rate_limit_remaining) | ||||
if response.status_code == 403 and reqs_remaining == 0: | if response.status_code == 403 and reqs_remaining == 0: | ||||
reset_at = int(response.headers['X-RateLimit-Reset']) | delay = int(response.headers['Retry-After']) | ||||
delay = min(reset_at - time.time(), 3600) | |||||
return True, delay | return True, delay | ||||
return False, 0 | return False, 0 | ||||
def get_next_target_from_response(self, response): | def get_next_target_from_response(self, response): | ||||
if 'next' in response.links: | if 'next' in response.links: | ||||
next_url = response.links['next']['url'] | next_url = response.links['next']['url'] | ||||
return int(self.API_URL_INDEX_RE.match(next_url).group(1)) | return int(self.API_URL_INDEX_RE.match(next_url).group(1)) | ||||
Show All 22 Lines |