diff --git a/swh/core/api/__init__.py b/swh/core/api/__init__.py
--- a/swh/core/api/__init__.py
+++ b/swh/core/api/__init__.py
@@ -121,6 +121,12 @@
             return super().__str__()
 
 
+class TransientRemoteException(RemoteException):
+    """Subclass of RemoteException representing errors which are expected
+    to be temporary.
+    """
+
+
 F = TypeVar("F", bound=Callable)
 
 
@@ -334,8 +340,16 @@
                 exception = RemoteException(payload=exc_data, response=response)
 
         elif status_class == 5:
+            cls: Type[RemoteException]
+            if status_code == 503:
+                # This isn't a generic HTTP client and we know the server does
+                # not support the Retry-After header, so we do not implement
+                # it here either.
+                cls = TransientRemoteException
+            else:
+                cls = RemoteException
             exc_data = self._decode_response(response, check_status=False)
-            exception = RemoteException(payload=exc_data, response=response)
+            exception = cls(payload=exc_data, response=response)
 
         if exception:
             raise exception from None
diff --git a/swh/core/api/tests/test_rpc_client.py b/swh/core/api/tests/test_rpc_client.py
--- a/swh/core/api/tests/test_rpc_client.py
+++ b/swh/core/api/tests/test_rpc_client.py
@@ -8,7 +8,13 @@
 import pytest
 from requests.exceptions import ConnectionError
 
-from swh.core.api import APIError, RemoteException, RPCClient, remote_api_endpoint
+from swh.core.api import (
+    APIError,
+    RemoteException,
+    RPCClient,
+    TransientRemoteException,
+    remote_api_endpoint,
+)
 from swh.core.api.serializers import exception_to_dict, msgpack_dumps
 
 from .test_serializers import ExtraType, extra_decoders, extra_encoders
@@ -143,7 +149,7 @@
     assert str(exc_info.value) == error_message
 
 
-@pytest.mark.parametrize("status_code", [400, 500])
+@pytest.mark.parametrize("status_code", [400, 500, 503])
 def test_client_raise_remote_exception(rpc_client, requests_mock, status_code):
     """
     Exception caught server-side and not whitelisted will be wrapped and raised
@@ -165,3 +171,7 @@
 
     assert str(exc_info.value.args[0]["type"]) == "Exception"
     assert str(exc_info.value.args[0]["message"]) == error_message
+    if status_code == 503:
+        assert isinstance(exc_info.value, TransientRemoteException)
+    else:
+        assert not isinstance(exc_info.value, TransientRemoteException)