self = <swh.lister.github.lister.GitHubLister object at 0x7f4e623576d8> def get_pages(self) -> Iterator[List[Dict[str, Any]]]: current_id = 0 if self.first_id is not None: current_id = self.first_id elif self.state is not None: current_id = self.state.last_seen_id current_url = f"{self.API_URL}?since={current_id}&per_page={self.PAGE_SIZE}" while self.last_id is None or current_id < self.last_id: logger.debug("Getting page %s", current_url) # The following for/else loop handles rate limiting; if successful, # it provides the rest of the function with a `response` object. # # If all tokens are rate-limited, we sleep until the reset time, # then `continue` into another iteration of the outer while loop, # attempting to get data from the same URL again. max_attempts = 1 if self.anonymous else len(self.credentials) reset_times: Dict[int, int] = {} # token index -> time for attempt in range(max_attempts): try: > response = github_request(current_url, session=self.session) .tox/py3/lib/python3.7/site-packages/swh/lister/github/lister.py:216: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ args = ('https://api.github.com/repositories?per_page=1000&since=5000',) kw = {'session': <requests.sessions.Session object at 0x7f4e62357828>} @_utils.wraps(f) def wrapped_f(*args, **kw): > return self(f, *args, **kw) .tox/py3/lib/python3.7/site-packages/tenacity/__init__.py:333: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <Retrying object at 0x7f4e64531588 (stop=<tenacity.stop._stop_never object at 0x7f4e6478e780>, wait=<tenacity.wait.wai...0x7f4e645314a8>, before=<function before_nothing at 0x7f4e6478d9d8>, after=<function after_nothing at 0x7f4e6479f488>)> fn = <function github_request at 0x7f4e64628598> args = ('https://api.github.com/repositories?per_page=1000&since=5000',) kwargs = {'session': <requests.sessions.Session object at 0x7f4e62357828>} retry_state = <tenacity.RetryCallState object at 0x7f4e623ec6d8> do = <tenacity.DoAttempt object at 0x7f4e623ec9b0> def __call__(self, fn, *args, **kwargs): self.begin(fn) retry_state = RetryCallState( retry_object=self, fn=fn, args=args, kwargs=kwargs) while True: > do = self.iter(retry_state=retry_state) .tox/py3/lib/python3.7/site-packages/tenacity/__init__.py:423: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <Retrying object at 0x7f4e64531588 (stop=<tenacity.stop._stop_never object at 0x7f4e6478e780>, wait=<tenacity.wait.wai...0x7f4e645314a8>, before=<function before_nothing at 0x7f4e6478d9d8>, after=<function after_nothing at 0x7f4e6479f488>)> retry_state = <tenacity.RetryCallState object at 0x7f4e623ec6d8> def iter(self, retry_state): # noqa fut = retry_state.outcome if fut is None: if self.before is not None: self.before(retry_state) return DoAttempt() is_explicit_retry = retry_state.outcome.failed \ and isinstance(retry_state.outcome.exception(), TryAgain) if not (is_explicit_retry or self.retry(retry_state=retry_state)): > return fut.result() .tox/py3/lib/python3.7/site-packages/tenacity/__init__.py:360: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <Future at 0x7f4e623ec710 state=finished raised RateLimited> timeout = None def result(self, timeout=None): """Return the result of the call that the future represents. Args: timeout: The number of seconds to wait for the result if the future isn't done. If None, then there is no limit on the wait time. Returns: The result of the call that the future represents. Raises: CancelledError: If the future was cancelled. TimeoutError: If the future didn't finish executing before the given timeout. Exception: If the call raised then that exception will be raised. """ with self._condition: if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]: raise CancelledError() elif self._state == FINISHED: > return self.__get_result() /usr/lib/python3.7/concurrent/futures/_base.py:425: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <Future at 0x7f4e623ec710 state=finished raised RateLimited> def __get_result(self): if self._exception: > raise self._exception /usr/lib/python3.7/concurrent/futures/_base.py:384: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <Retrying object at 0x7f4e64531588 (stop=<tenacity.stop._stop_never object at 0x7f4e6478e780>, wait=<tenacity.wait.wai...0x7f4e645314a8>, before=<function before_nothing at 0x7f4e6478d9d8>, after=<function after_nothing at 0x7f4e6479f488>)> fn = <function github_request at 0x7f4e64628598> args = ('https://api.github.com/repositories?per_page=1000&since=5000',) kwargs = {'session': <requests.sessions.Session object at 0x7f4e62357828>} retry_state = <tenacity.RetryCallState object at 0x7f4e623ec6d8> do = <tenacity.DoAttempt object at 0x7f4e623ec9b0> def __call__(self, fn, *args, **kwargs): self.begin(fn) retry_state = RetryCallState( retry_object=self, fn=fn, args=args, kwargs=kwargs) while True: do = self.iter(retry_state=retry_state) if isinstance(do, DoAttempt): try: > result = fn(*args, **kwargs) .tox/py3/lib/python3.7/site-packages/tenacity/__init__.py:426: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ url = 'https://api.github.com/repositories?per_page=1000&since=5000' token = None, session = <requests.sessions.Session object at 0x7f4e62357828> @retry( wait=wait_exponential(multiplier=1, min=4, max=10), retry=retry_if_exception_type(requests.exceptions.ChunkedEncodingError), ) def github_request( url: str, token: Optional[str] = None, session: Optional[requests.Session] = None ) -> requests.Response: session = init_session(session) headers = {} if token: headers["Authorization"] = f"token {token}" response = session.get(url, headers=headers) anonymous = token is None and "Authorization" not in session.headers if ( # GitHub returns inconsistent status codes between unauthenticated # rate limit and authenticated rate limits. Handle both. response.status_code == 429 or (anonymous and response.status_code == 403) ): > raise RateLimited(response) E swh.lister.github.lister.RateLimited: <Response [429]> .tox/py3/lib/python3.7/site-packages/swh/lister/github/lister.py:85: RateLimited During handling of the above exception, another exception occurred: swh_scheduler = <swh.scheduler.backend.SchedulerBackend object at 0x7f4e626bad68> caplog = <_pytest.logging.LogCaptureFixture object at 0x7f4e62357198> requests_ratelimited = <requests_mock.mocker.Mocker object at 0x7f4e62357208> monkeypatch_sleep_calls = [], num_before_ratelimit = 5, ratelimit_reset = 123456 github_credentials = [{'password': 'token-legacy-0', 'username': 'swh-legacy0'}, {'password': 'token-legacy-1', 'username': 'swh-legacy1'},...n': 'token-0', 'username': 'swh0'}, {'token': 'token-1', 'username': 'swh1'}, {'token': 'token-2', 'username': 'swh2'}] lister_credentials = {'github': {'github': [{'password': 'token-legacy-0', 'username': 'swh-legacy0'}, {'password': 'token-legacy-1', 'user...: 'token-0', 'username': 'swh0'}, {'token': 'token-1', 'username': 'swh1'}, {'token': 'token-2', 'username': 'swh2'}]}} @pytest.mark.parametrize( # Do 5 successful requests, return 6 ratelimits (to exhaust the credentials) with a # set value for X-Ratelimit-Reset, then resume listing successfully. "num_before_ratelimit, num_ratelimit, ratelimit_reset", [(5, 6, 123456)], ) def test_ratelimit_reset_sleep( swh_scheduler, caplog, requests_ratelimited, monkeypatch_sleep_calls, num_before_ratelimit, ratelimit_reset, github_credentials, lister_credentials, ): """Check that the lister properly handles rate-limiting when providing it with authentication tokens""" caplog.set_level(logging.DEBUG, "swh.lister.github.lister") lister = GitHubLister(scheduler=swh_scheduler, credentials=lister_credentials) > res = lister.run() .tox/py3/lib/python3.7/site-packages/swh/lister/github/tests/test_lister.py:402: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .tox/py3/lib/python3.7/site-packages/swh/lister/pattern.py:121: in run for page in self.get_pages(): .tox/py3/lib/python3.7/site-packages/swh/lister/github/lister.py:228: in get_pages reset_info, /usr/lib/python3.7/logging/__init__.py:1383: in info self._log(INFO, msg, args, **kwargs) /usr/lib/python3.7/logging/__init__.py:1519: in _log self.handle(record) /usr/lib/python3.7/logging/__init__.py:1529: in handle self.callHandlers(record) /usr/lib/python3.7/logging/__init__.py:1591: in callHandlers hdlr.handle(record) /usr/lib/python3.7/logging/__init__.py:905: in handle self.emit(record) .tox/py3/lib/python3.7/site-packages/_pytest/logging.py:331: in emit super().emit(record) /usr/lib/python3.7/logging/__init__.py:1040: in emit self.handleError(record) /usr/lib/python3.7/logging/__init__.py:1034: in emit msg = self.format(record) /usr/lib/python3.7/logging/__init__.py:880: in format return fmt.format(record) .tox/py3/lib/python3.7/site-packages/_pytest/logging.py:92: in format return super().format(record) /usr/lib/python3.7/logging/__init__.py:619: in format record.message = record.getMessage() _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <LogRecord: swh.lister.github.lister, 20, /var/lib/jenkins/workspace/DLS/tests-on-diff/.tox/py3/lib/python3.7/site-packages/swh/lister/github/lister.py, 228, "Rate limit exhausted for current user %s"> def getMessage(self): """ Return the message for this LogRecord. Return the message for this LogRecord after merging any user-supplied arguments with the message. """ msg = str(self.msg) if self.args: > msg = msg % self.args E TypeError: not all arguments converted during string formatting /usr/lib/python3.7/logging/__init__.py:380: TypeError