Changeset View
Changeset View
Standalone View
Standalone View
swh/core/github/utils.py
| Show First 20 Lines • Show All 226 Lines • ▼ Show 20 Lines | def request(self, url) -> requests.Response: | ||||
| logger.info( | logger.info( | ||||
| "Rate limit exhausted for current user %s %s", | "Rate limit exhausted for current user %s %s", | ||||
| self.current_user, | self.current_user, | ||||
| reset_info, | reset_info, | ||||
| ) | ) | ||||
| # Use next token in line | # Use next token in line | ||||
| self.set_next_session_token() | self.set_next_session_token() | ||||
| # Wait one second to avoid triggering GitHub's abuse rate limits | # Wait one second to avoid triggering GitHub's abuse rate limits | ||||
| self.statsd.increment("sleep", 1) | self.statsd.increment("sleep_seconds_total", 1) | ||||
| time.sleep(1) | time.sleep(1) | ||||
| # All tokens have been rate-limited. What do we do? | # All tokens have been rate-limited. What do we do? | ||||
| if not reset_times: | if not reset_times: | ||||
| logger.warning( | logger.warning( | ||||
| "No X-Ratelimit-Reset value found in responses for any token; " | "No X-Ratelimit-Reset value found in responses for any token; " | ||||
| "Giving up." | "Giving up." | ||||
| ) | ) | ||||
| raise MissingRateLimitReset() | raise MissingRateLimitReset() | ||||
| sleep_time = max(reset_times.values()) - time.time() + 1 | sleep_time = max(reset_times.values()) - time.time() + 1 | ||||
| logger.info( | logger.info( | ||||
| "Rate limits exhausted for all tokens. Sleeping for %f seconds.", | "Rate limits exhausted for all tokens. Sleeping for %f seconds.", | ||||
| sleep_time, | sleep_time, | ||||
| ) | ) | ||||
| self.statsd.increment("sleep", sleep_time) | self.statsd.increment("sleep_seconds_total", sleep_time) | ||||
| time.sleep(sleep_time) | time.sleep(sleep_time) | ||||
| def get_canonical_url(self, url: str) -> Optional[str]: | def get_canonical_url(self, url: str) -> Optional[str]: | ||||
| """Retrieve canonical github url out of an url if any or None otherwise. | """Retrieve canonical github url out of an url if any or None otherwise. | ||||
| This triggers an http request to the github api url to determine the | This triggers an http request to the github api url to determine the | ||||
| canonical repository url. | canonical repository url. | ||||
| Show All 15 Lines | |||||