diff --git a/swh/lister/pypi/lister.py b/swh/lister/pypi/lister.py
--- a/swh/lister/pypi/lister.py
+++ b/swh/lister/pypi/lister.py
@@ -8,11 +8,11 @@
 from datetime import datetime, timezone
 import logging
 from typing import Any, Dict, Iterator, List, Optional, Tuple
-from xmlrpc.client import ServerProxy
+from xmlrpc.client import Fault, ServerProxy
 
 from tenacity.before_sleep import before_sleep_log
 
-from swh.lister.utils import throttling_retry
+from swh.lister.utils import retry_attempt, throttling_retry
 from swh.scheduler.interface import SchedulerInterface
 from swh.scheduler.model import ListedOrigin
 
@@ -39,6 +39,16 @@
     """Last seen serial when visiting the pypi instance"""
 
 
+def _if_rate_limited(retry_state) -> bool:
+    """Custom tenacity retry predicate for handling xmlrpc client error
+    """
+    attempt = retry_attempt(retry_state)
+    if attempt.failed:
+        exc = attempt.exception()
+        return isinstance(exc, Fault)
+    return False
+
+
 def pypi_url(package_name: str) -> str:
     """Build pypi url out of a package name.
 
@@ -78,14 +88,18 @@
     def state_to_dict(self, state: PyPIListerState) -> Dict[str, Any]:
         return asdict(state)
 
-    @throttling_retry(before_sleep=before_sleep_log(logger, logging.WARNING))
+    @throttling_retry(
+        retry=_if_rate_limited, before_sleep=before_sleep_log(logger, logging.WARNING)
+    )
     def _changelog_last_serial(self, client: ServerProxy) -> int:
         """Internal detail to allow throttling when calling the changelog last entry"""
         serial = client.changelog_last_serial()
         assert isinstance(serial, int)
         return serial
 
-    @throttling_retry(before_sleep=before_sleep_log(logger, logging.WARNING))
+    @throttling_retry(
+        retry=_if_rate_limited, before_sleep=before_sleep_log(logger, logging.WARNING)
+    )
     def _changelog_since_serial(
         self, client: ServerProxy, serial: int
     ) -> List[ChangelogEntry]: