Changeset View
Changeset View
Standalone View
Standalone View
swh/deposit/client.py
Show All 10 Lines | |||||
import hashlib | import hashlib | ||||
import logging | import logging | ||||
import os | import os | ||||
from typing import Any, Dict, Optional, Tuple | from typing import Any, Dict, Optional, Tuple | ||||
from urllib.parse import urljoin | from urllib.parse import urljoin | ||||
import warnings | import warnings | ||||
import requests | import requests | ||||
import xmltodict | |||||
from swh.core.config import load_from_envvar | from swh.core.config import load_from_envvar | ||||
from swh.deposit import __version__ as swh_deposit_version | from swh.deposit import __version__ as swh_deposit_version | ||||
from swh.deposit.utils import parse_xml | |||||
logger = logging.getLogger(__name__) | logger = logging.getLogger(__name__) | ||||
def compute_unified_information( | def compute_unified_information( | ||||
collection: str, | collection: str, | ||||
in_progress: bool, | in_progress: bool, | ||||
slug: str, | slug: str, | ||||
▲ Show 20 Lines • Show All 53 Lines • ▼ Show 20 Lines | |||||
class MaintenanceError(ValueError): | class MaintenanceError(ValueError): | ||||
"""Informational maintenance error exception | """Informational maintenance error exception | ||||
""" | """ | ||||
pass | pass | ||||
def _parse(stream, encoding="utf-8"): | |||||
"""Given a xml stream, parse the result. | |||||
Args: | |||||
stream (bytes/text): The stream to parse | |||||
encoding (str): The encoding to use if to decode the bytes | |||||
stream | |||||
Returns: | |||||
A dict of values corresponding to the parsed xml | |||||
""" | |||||
if isinstance(stream, bytes): | |||||
stream = stream.decode(encoding) | |||||
data = xmltodict.parse(stream, encoding=encoding, process_namespaces=False) | |||||
if "entry" in data: | |||||
data = data["entry"] | |||||
if "sword:error" in data: | |||||
data = data["sword:error"] | |||||
return dict(data) | |||||
def _parse_with_filter(stream, encoding="utf-8", keys=[]): | |||||
"""Given a xml stream, parse the result and filter with keys. | |||||
Args: | |||||
stream (bytes/text): The stream to parse | |||||
encoding (str): The encoding to use if to decode the bytes | |||||
stream | |||||
keys ([str]): Keys to filter the parsed result | |||||
Returns: | |||||
A dict of values corresponding to the parsed xml filtered by | |||||
the keys provided. | |||||
""" | |||||
data = _parse(stream, encoding=encoding) | |||||
m = {} | |||||
for key in keys: | |||||
m[key] = data.get(key) | |||||
return m | |||||
def handle_deprecated_config(config: Dict) -> Tuple[str, Optional[Tuple[str, str]]]: | def handle_deprecated_config(config: Dict) -> Tuple[str, Optional[Tuple[str, str]]]: | ||||
ardumont: careful with this, i think that requires the DJANGO_SETTING_MODULE.... (or something) variable… | |||||
Done Inline ActionsI don't follow. That's client code, what does Django have to do with it? vlorentz: I don't follow. That's client code, what does Django have to do with it? | |||||
Not Done Inline ActionsI recall the code you are adding was the one I initially opened the client. But i had issues at the time, i recall it would not work because it required to have the env variable i mentioned. from rest_framework.parsers import BaseParser, FileUploadParser, MultiPartParser class SWHXMLParser(BaseParser) .... rest_framework pulls django deps iirc ¯\_(ツ)_/¯ It's probably fine with tests since everything is configured for it, django wise... I'm just saying, have you checked the actual runtime with those changes? ardumont: I recall the code you are adding was the one I initially opened the client.
But i had issues… | |||||
Done Inline Actionsoooh, ok, you're right. vlorentz: oooh, ok, you're right. | |||||
warnings.warn( | warnings.warn( | ||||
'"config" argument is deprecated, please ' | '"config" argument is deprecated, please ' | ||||
'use "url" and "auth" arguments instead; note that "auth" ' | 'use "url" and "auth" arguments instead; note that "auth" ' | ||||
"expects now a couple (username, password) and not a dict.", | "expects now a couple (username, password) and not a dict.", | ||||
DeprecationWarning, | DeprecationWarning, | ||||
) | ) | ||||
url: str = config["url"] | url: str = config["url"] | ||||
auth: Optional[Tuple[str, str]] = None | auth: Optional[Tuple[str, str]] = None | ||||
▲ Show 20 Lines • Show All 193 Lines • ▼ Show 20 Lines | def parse_result_error(self, xml_content): | ||||
Returns: | Returns: | ||||
dict with following keys: | dict with following keys: | ||||
'error': The error message | 'error': The error message | ||||
'detail': Some more detail about the error if any | 'detail': Some more detail about the error if any | ||||
""" | """ | ||||
return _parse_with_filter( | data = parse_xml(xml_content) | ||||
xml_content, keys=["summary", "detail", "sword:verboseDescription"] | return { | ||||
) | "summary": data["summary"], | ||||
"detail": data["detail"], | |||||
"sword:verboseDescription": data["sword:verboseDescription"], | |||||
} | |||||
def do_execute(self, method, url, info): | def do_execute(self, method, url, info): | ||||
"""Execute the http query to url using method and info information. | """Execute the http query to url using method and info information. | ||||
By default, execute a simple query to url with the http | By default, execute a simple query to url with the http | ||||
method. Override this in daughter class to improve the | method. Override this in daughter class to improve the | ||||
default behavior if needed. | default behavior if needed. | ||||
▲ Show 20 Lines • Show All 64 Lines • ▼ Show 20 Lines | class ServiceDocumentDepositClient(BaseDepositClient): | ||||
def compute_method(self, *args, **kwargs): | def compute_method(self, *args, **kwargs): | ||||
return "get" | return "get" | ||||
def parse_result_ok(self, xml_content): | def parse_result_ok(self, xml_content): | ||||
"""Parse service document's success response. | """Parse service document's success response. | ||||
""" | """ | ||||
return _parse(xml_content) | return parse_xml(xml_content) | ||||
class StatusDepositClient(BaseDepositClient): | class StatusDepositClient(BaseDepositClient): | ||||
"""Status information on a deposit. | """Status information on a deposit. | ||||
""" | """ | ||||
def __init__(self, config=None, url=None, auth=None): | def __init__(self, config=None, url=None, auth=None): | ||||
Show All 14 Lines | class StatusDepositClient(BaseDepositClient): | ||||
def compute_method(self, *args, **kwargs): | def compute_method(self, *args, **kwargs): | ||||
return "get" | return "get" | ||||
def parse_result_ok(self, xml_content): | def parse_result_ok(self, xml_content): | ||||
"""Given an xml content as string, returns a deposit dict. | """Given an xml content as string, returns a deposit dict. | ||||
""" | """ | ||||
return _parse_with_filter( | data = parse_xml(xml_content) | ||||
xml_content, | |||||
keys=[ | keys = [ | ||||
"deposit_id", | "deposit_id", | ||||
"deposit_status", | "deposit_status", | ||||
"deposit_status_detail", | "deposit_status_detail", | ||||
"deposit_swh_id", | "deposit_swh_id", | ||||
"deposit_swh_id_context", | "deposit_swh_id_context", | ||||
"deposit_external_id", | "deposit_external_id", | ||||
], | ] | ||||
) | return {key: data.get("swh:" + key) for key in keys} | ||||
class BaseCreateDepositClient(BaseDepositClient): | class BaseCreateDepositClient(BaseDepositClient): | ||||
"""Deposit client base class to post new deposit. | """Deposit client base class to post new deposit. | ||||
""" | """ | ||||
def __init__(self, config=None, url=None, auth=None): | def __init__(self, config=None, url=None, auth=None): | ||||
super().__init__( | super().__init__( | ||||
url=url, | url=url, | ||||
auth=auth, | auth=auth, | ||||
config=config, | config=config, | ||||
error_msg="Post Deposit failure at %s: %s", | error_msg="Post Deposit failure at %s: %s", | ||||
empty_result={"deposit_id": None, "deposit_status": None,}, | empty_result={"swh:deposit_id": None, "swh:deposit_status": None,}, | ||||
) | ) | ||||
def compute_url(self, collection, *args, **kwargs): | def compute_url(self, collection, *args, **kwargs): | ||||
return "/%s/" % collection | return "/%s/" % collection | ||||
def compute_method(self, *args, **kwargs): | def compute_method(self, *args, **kwargs): | ||||
return "post" | return "post" | ||||
def parse_result_ok(self, xml_content): | def parse_result_ok(self, xml_content): | ||||
"""Given an xml content as string, returns a deposit dict. | """Given an xml content as string, returns a deposit dict. | ||||
""" | """ | ||||
return _parse_with_filter( | data = parse_xml(xml_content) | ||||
xml_content, | |||||
keys=[ | keys = [ | ||||
"deposit_id", | "deposit_id", | ||||
"deposit_status", | "deposit_status", | ||||
"deposit_status_detail", | "deposit_status_detail", | ||||
"deposit_date", | "deposit_date", | ||||
], | ] | ||||
) | return {key: data.get("swh:" + key) for key in keys} | ||||
def compute_headers(self, info: Dict[str, Any]) -> Dict[str, Any]: | def compute_headers(self, info: Dict[str, Any]) -> Dict[str, Any]: | ||||
return info | return info | ||||
def do_execute(self, method, url, info): | def do_execute(self, method, url, info): | ||||
with open(info["filepath"], "rb") as f: | with open(info["filepath"], "rb") as f: | ||||
return self.do(method, url, data=f, headers=info["headers"]) | return self.do(method, url, data=f, headers=info["headers"]) | ||||
▲ Show 20 Lines • Show All 238 Lines • Show Last 20 Lines |
careful with this, i think that requires the DJANGO_SETTING_MODULE.... (or something) variable within the env for that to work.
Thus why, that's somehow duplicated here.