Changeset View
Changeset View
Standalone View
Standalone View
swh/deposit/client.py
# Copyright (C) 2017-2020 The Software Heritage developers | # Copyright (C) 2017-2020 The Software Heritage developers | ||||
# See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
# License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
# See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
"""Module in charge of defining an swh-deposit client | """Module in charge of defining an swh-deposit client | ||||
""" | """ | ||||
from abc import ABCMeta, abstractmethod | from abc import ABCMeta, abstractmethod | ||||
import hashlib | import hashlib | ||||
import logging | import logging | ||||
import os | import os | ||||
from typing import Any, Dict, Optional | from typing import Any, Dict, Optional, Tuple | ||||
from urllib.parse import urljoin | from urllib.parse import urljoin | ||||
import warnings | |||||
import requests | import requests | ||||
import xmltodict | import xmltodict | ||||
from swh.core.config import load_from_envvar | from swh.core.config import load_from_envvar | ||||
logger = logging.getLogger(__name__) | logger = logging.getLogger(__name__) | ||||
▲ Show 20 Lines • Show All 101 Lines • ▼ Show 20 Lines | def _parse_with_filter(stream, encoding="utf-8", keys=[]): | ||||
""" | """ | ||||
data = _parse(stream, encoding=encoding) | data = _parse(stream, encoding=encoding) | ||||
m = {} | m = {} | ||||
for key in keys: | for key in keys: | ||||
m[key] = data.get(key) | m[key] = data.get(key) | ||||
return m | return m | ||||
vlorentz: it doesn't just notify, but also computes a value. | |||||
def handle_deprecated_config(config: Dict) -> Tuple[str, Optional[Tuple[str, str]]]: | |||||
warnings.warn( | |||||
'"config" argument is deprecated, please ' | |||||
'use "url" and "auth" arguments instead; note that "auth" ' | |||||
"expects now a couple (username, password) and not a dict.", | |||||
DeprecationWarning, | |||||
) | |||||
url: str = config["url"] | |||||
auth: Optional[Tuple[str, str]] = None | |||||
if config.get("auth"): | |||||
auth = (config["auth"]["username"], config["auth"]["password"]) | |||||
return (url, auth) | |||||
class BaseApiDepositClient: | class BaseApiDepositClient: | ||||
"""Deposit client base class | """Deposit client base class | ||||
""" | """ | ||||
def __init__(self, config=None, _client=requests): | def __init__( | ||||
self.config: Dict[str, Any] = config or load_from_envvar() | self, | ||||
self._client = _client | config: Optional[Dict] = None, | ||||
self.base_url = self.config["url"].strip("/") + "/" | url: Optional[str] = None, | ||||
auth = self.config["auth"] | auth: Optional[Tuple[str, str]] = None, | ||||
if auth == {}: | ): | ||||
self.auth = None | if not url and not config: | ||||
else: | config = load_from_envvar() | ||||
self.auth = (auth["username"], auth["password"]) | if config: | ||||
url, auth = handle_deprecated_config(config) | |||||
Done Inline Actionsthese four lines look a lot like the code in notify_deprecated. (also, the or None is redundant) vlorentz: these four lines look a lot like the code in `notify_deprecated`.
(also, the `or None` is… | |||||
# needed to help mypy not be fooled by the Optional nature of url | |||||
assert url is not None | |||||
self.base_url = url.strip("/") + "/" | |||||
self.auth = auth | |||||
Done Inline Actionsmove the comment on the line before the assert, so black doesn't need to add newlines like this vlorentz: move the comment on the line before the assert, so black doesn't need to add newlines like this | |||||
def do(self, method, url, *args, **kwargs): | def do(self, method, url, *args, **kwargs): | ||||
"""Internal method to deal with requests, possibly with basic http | """Internal method to deal with requests, possibly with basic http | ||||
authentication. | authentication. | ||||
Args: | Args: | ||||
method (str): supported http methods as in self._methods' keys | method (str): supported http methods as in self._methods' keys | ||||
Returns: | Returns: | ||||
The request's execution | The request's execution | ||||
""" | """ | ||||
if hasattr(self._client, method): | method_fn = getattr(requests, method) | ||||
method_fn = getattr(self._client, method) | |||||
else: | |||||
raise ValueError("Development error, unsupported method %s" % (method)) | |||||
if self.auth: | if self.auth: | ||||
kwargs["auth"] = self.auth | kwargs["auth"] = self.auth | ||||
full_url = urljoin(self.base_url, url.lstrip("/")) | full_url = urljoin(self.base_url, url.lstrip("/")) | ||||
return method_fn(full_url, *args, **kwargs) | return method_fn(full_url, *args, **kwargs) | ||||
▲ Show 20 Lines • Show All 101 Lines • ▼ Show 20 Lines | def check(self, check_url): | ||||
raise ValueError(msg) | raise ValueError(msg) | ||||
class BaseDepositClient(BaseApiDepositClient, metaclass=ABCMeta): | class BaseDepositClient(BaseApiDepositClient, metaclass=ABCMeta): | ||||
"""Base Deposit client to access the public api. | """Base Deposit client to access the public api. | ||||
""" | """ | ||||
def __init__(self, config, error_msg=None, empty_result={}): | def __init__( | ||||
super().__init__(config) | self, config=None, url=None, auth=None, error_msg=None, empty_result={} | ||||
): | |||||
super().__init__(url=url, auth=auth, config=config) | |||||
self.error_msg = error_msg | self.error_msg = error_msg | ||||
self.empty_result = empty_result | self.empty_result = empty_result | ||||
@abstractmethod | @abstractmethod | ||||
def compute_url(self, *args, **kwargs): | def compute_url(self, *args, **kwargs): | ||||
"""Compute api url endpoint to query.""" | """Compute api url endpoint to query.""" | ||||
pass | pass | ||||
▲ Show 20 Lines • Show All 86 Lines • ▼ Show 20 Lines | def execute(self, *args, **kwargs) -> Dict[str, Any]: | ||||
return error | return error | ||||
class ServiceDocumentDepositClient(BaseDepositClient): | class ServiceDocumentDepositClient(BaseDepositClient): | ||||
"""Service Document information retrieval. | """Service Document information retrieval. | ||||
""" | """ | ||||
def __init__(self, config): | def __init__(self, config=None, url=None, auth=None): | ||||
super().__init__( | super().__init__( | ||||
config, | url=url, | ||||
auth=auth, | |||||
config=config, | |||||
error_msg="Service document failure at %s: %s", | error_msg="Service document failure at %s: %s", | ||||
empty_result={"collection": None}, | empty_result={"collection": None}, | ||||
) | ) | ||||
def compute_url(self, *args, **kwargs): | def compute_url(self, *args, **kwargs): | ||||
return "/servicedocument/" | return "/servicedocument/" | ||||
def compute_method(self, *args, **kwargs): | def compute_method(self, *args, **kwargs): | ||||
return "get" | return "get" | ||||
def parse_result_ok(self, xml_content): | def parse_result_ok(self, xml_content): | ||||
"""Parse service document's success response. | """Parse service document's success response. | ||||
""" | """ | ||||
return _parse(xml_content) | return _parse(xml_content) | ||||
class StatusDepositClient(BaseDepositClient): | class StatusDepositClient(BaseDepositClient): | ||||
"""Status information on a deposit. | """Status information on a deposit. | ||||
""" | """ | ||||
def __init__(self, config): | def __init__(self, config=None, url=None, auth=None): | ||||
super().__init__( | super().__init__( | ||||
config, | url=url, | ||||
auth=auth, | |||||
config=config, | |||||
error_msg="Status check failure at %s: %s", | error_msg="Status check failure at %s: %s", | ||||
empty_result={ | empty_result={ | ||||
"deposit_status": None, | "deposit_status": None, | ||||
"deposit_status_detail": None, | "deposit_status_detail": None, | ||||
"deposit_swh_id": None, | "deposit_swh_id": None, | ||||
}, | }, | ||||
) | ) | ||||
Show All 20 Lines | def parse_result_ok(self, xml_content): | ||||
) | ) | ||||
class BaseCreateDepositClient(BaseDepositClient): | class BaseCreateDepositClient(BaseDepositClient): | ||||
"""Deposit client base class to post new deposit. | """Deposit client base class to post new deposit. | ||||
""" | """ | ||||
def __init__(self, config): | def __init__(self, config=None, url=None, auth=None): | ||||
super().__init__( | super().__init__( | ||||
config, | url=url, | ||||
auth=auth, | |||||
config=config, | |||||
error_msg="Post Deposit failure at %s: %s", | error_msg="Post Deposit failure at %s: %s", | ||||
empty_result={"deposit_id": None, "deposit_status": None,}, | empty_result={"deposit_id": None, "deposit_status": None,}, | ||||
) | ) | ||||
def compute_url(self, collection, *args, **kwargs): | def compute_url(self, collection, *args, **kwargs): | ||||
return "/%s/" % collection | return "/%s/" % collection | ||||
def compute_method(self, *args, **kwargs): | def compute_method(self, *args, **kwargs): | ||||
▲ Show 20 Lines • Show All 141 Lines • ▼ Show 20 Lines | def compute_method(self, *args, replace=False, **kwargs): | ||||
return "put" if replace else "post" | return "put" if replace else "post" | ||||
class PublicApiDepositClient(BaseApiDepositClient): | class PublicApiDepositClient(BaseApiDepositClient): | ||||
"""Public api deposit client.""" | """Public api deposit client.""" | ||||
def service_document(self): | def service_document(self): | ||||
"""Retrieve service document endpoint's information.""" | """Retrieve service document endpoint's information.""" | ||||
return ServiceDocumentDepositClient(self.config).execute() | return ServiceDocumentDepositClient(url=self.base_url, auth=self.auth).execute() | ||||
def deposit_status(self, collection: str, deposit_id: int): | def deposit_status(self, collection: str, deposit_id: int): | ||||
"""Retrieve status information on a deposit.""" | """Retrieve status information on a deposit.""" | ||||
return StatusDepositClient(self.config).execute(collection, deposit_id) | return StatusDepositClient(url=self.base_url, auth=self.auth).execute( | ||||
collection, deposit_id | |||||
) | |||||
def deposit_create( | def deposit_create( | ||||
self, | self, | ||||
collection: str, | collection: str, | ||||
slug: str, | slug: str, | ||||
archive: Optional[str] = None, | archive: Optional[str] = None, | ||||
metadata: Optional[str] = None, | metadata: Optional[str] = None, | ||||
in_progress: bool = False, | in_progress: bool = False, | ||||
): | ): | ||||
"""Create a new deposit (archive, metadata, both as multipart).""" | """Create a new deposit (archive, metadata, both as multipart).""" | ||||
if archive and not metadata: | if archive and not metadata: | ||||
return CreateArchiveDepositClient(self.config).execute( | return CreateArchiveDepositClient( | ||||
collection, in_progress, slug, archive_path=archive | url=self.base_url, auth=self.auth | ||||
) | ).execute(collection, in_progress, slug, archive_path=archive) | ||||
elif not archive and metadata: | elif not archive and metadata: | ||||
return CreateMetadataDepositClient(self.config).execute( | return CreateMetadataDepositClient( | ||||
collection, in_progress, slug, metadata_path=metadata | url=self.base_url, auth=self.auth | ||||
) | ).execute(collection, in_progress, slug, metadata_path=metadata) | ||||
else: | else: | ||||
return CreateMultipartDepositClient(self.config).execute( | return CreateMultipartDepositClient( | ||||
url=self.base_url, auth=self.auth | |||||
).execute( | |||||
collection, | collection, | ||||
in_progress, | in_progress, | ||||
slug, | slug, | ||||
archive_path=archive, | archive_path=archive, | ||||
metadata_path=metadata, | metadata_path=metadata, | ||||
) | ) | ||||
def deposit_update( | def deposit_update( | ||||
Show All 23 Lines | ): | ||||
if swhid is not None and status != "done": | if swhid is not None and status != "done": | ||||
return { | return { | ||||
"error": "You can only update metadata on deposit with status 'done'", | "error": "You can only update metadata on deposit with status 'done'", | ||||
"detail": f"The deposit {deposit_id} has status '{status}'", | "detail": f"The deposit {deposit_id} has status '{status}'", | ||||
"deposit_status": status, | "deposit_status": status, | ||||
"deposit_id": deposit_id, | "deposit_id": deposit_id, | ||||
} | } | ||||
if archive and not metadata: | if archive and not metadata: | ||||
r = UpdateArchiveDepositClient(self.config).execute( | r = UpdateArchiveDepositClient(url=self.base_url, auth=self.auth).execute( | ||||
collection, | collection, | ||||
in_progress, | in_progress, | ||||
slug, | slug, | ||||
deposit_id=deposit_id, | deposit_id=deposit_id, | ||||
archive_path=archive, | archive_path=archive, | ||||
replace=replace, | replace=replace, | ||||
) | ) | ||||
elif not archive and metadata and swhid is None: | elif not archive and metadata and swhid is None: | ||||
r = UpdateMetadataOnPartialDepositClient(self.config).execute( | r = UpdateMetadataOnPartialDepositClient( | ||||
url=self.base_url, auth=self.auth | |||||
).execute( | |||||
collection, | collection, | ||||
in_progress, | in_progress, | ||||
slug, | slug, | ||||
deposit_id=deposit_id, | deposit_id=deposit_id, | ||||
metadata_path=metadata, | metadata_path=metadata, | ||||
replace=replace, | replace=replace, | ||||
) | ) | ||||
elif not archive and metadata and swhid is not None: | elif not archive and metadata and swhid is not None: | ||||
r = UpdateMetadataOnDoneDepositClient(self.config).execute( | r = UpdateMetadataOnDoneDepositClient( | ||||
url=self.base_url, auth=self.auth | |||||
).execute( | |||||
collection, | collection, | ||||
in_progress, | in_progress, | ||||
slug, | slug, | ||||
deposit_id=deposit_id, | deposit_id=deposit_id, | ||||
metadata_path=metadata, | metadata_path=metadata, | ||||
swhid=swhid, | swhid=swhid, | ||||
) | ) | ||||
else: | else: | ||||
r = UpdateMultipartDepositClient(self.config).execute( | r = UpdateMultipartDepositClient(url=self.base_url, auth=self.auth).execute( | ||||
collection, | collection, | ||||
in_progress, | in_progress, | ||||
slug, | slug, | ||||
deposit_id=deposit_id, | deposit_id=deposit_id, | ||||
archive_path=archive, | archive_path=archive, | ||||
metadata_path=metadata, | metadata_path=metadata, | ||||
replace=replace, | replace=replace, | ||||
) | ) | ||||
if "error" in r: | if "error" in r: | ||||
return r | return r | ||||
return self.deposit_status(collection, deposit_id) | return self.deposit_status(collection, deposit_id) |
it doesn't just notify, but also computes a value.