Changeset View
Changeset View
Standalone View
Standalone View
swh/lister/gitlab/lister.py
| Show First 20 Lines • Show All 81 Lines • ▼ Show 20 Lines | def get_model_from_repo(self, repo): | ||||
| 'full_name': repo['path_with_namespace'], | 'full_name': repo['path_with_namespace'], | ||||
| 'html_url': repo['web_url'], | 'html_url': repo['web_url'], | ||||
| 'origin_url': repo['http_url_to_repo'], | 'origin_url': repo['http_url_to_repo'], | ||||
| 'origin_type': 'git', | 'origin_type': 'git', | ||||
| 'description': repo['description'], | 'description': repo['description'], | ||||
| } | } | ||||
| def transport_quota_check(self, response): | def transport_quota_check(self, response): | ||||
| """Deal with rate limit | """Deal with rate limit if any. | ||||
| """ | """ | ||||
| # not all gitlab instance have rate limit | |||||
| if 'RateLimit-Remaining' in response.headers: | |||||
| reqs_remaining = int(response.headers['RateLimit-Remaining']) | reqs_remaining = int(response.headers['RateLimit-Remaining']) | ||||
| if response.status_code == 403 and reqs_remaining == 0: | if response.status_code == 403 and reqs_remaining == 0: | ||||
| reset_at = int(response.headers['RateLimit-Reset']) | reset_at = int(response.headers['RateLimit-Reset']) | ||||
| delay = min(reset_at - time.time(), 3600) | delay = min(reset_at - time.time(), 3600) | ||||
| return True, delay | return True, delay | ||||
| return False, 0 | return False, 0 | ||||
| def get_next_target_from_response(self, response): | def get_next_target_from_response(self, response): | ||||
| """Deal with pagination | """Deal with pagination | ||||
| """ | """ | ||||
| if 'next' in response.links: | if 'next' in response.links: | ||||
| next_url = response.links['next']['url'] | next_url = response.links['next']['url'] | ||||
| return int(self.API_URL_INDEX_RE.match(next_url).group(1)) | return int(self.API_URL_INDEX_RE.match(next_url).group(1)) | ||||
| return None | return None | ||||
| def transport_response_simplified(self, response): | def transport_response_simplified(self, response): | ||||
| repos = response.json() | repos = response.json() | ||||
| return [self.get_model_from_repo(repo) for repo in repos] | return [self.get_model_from_repo(repo) for repo in repos] | ||||