Changeset View
Changeset View
Standalone View
Standalone View
swh/deposit/api/common.py
| # Copyright (C) 2017-2021 The Software Heritage developers | # Copyright (C) 2017-2021 The Software Heritage developers | ||||
| # See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
| # License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
| # See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
| from abc import ABCMeta, abstractmethod | from abc import ABCMeta, abstractmethod | ||||
| import datetime | import datetime | ||||
| import hashlib | import hashlib | ||||
| import json | import json | ||||
| from typing import Any, Dict, Optional, Sequence, Tuple, Type, Union | from typing import Any, Dict, Optional, Sequence, Tuple, Type, Union | ||||
| import uuid | import uuid | ||||
| import attr | import attr | ||||
| from django.core.files.uploadedfile import UploadedFile | from django.core.files.uploadedfile import UploadedFile | ||||
| from django.http import FileResponse, HttpResponse | from django.http import FileResponse, HttpResponse | ||||
| from django.shortcuts import render | from django.shortcuts import render | ||||
| from django.template.loader import render_to_string | |||||
| from django.urls import reverse | from django.urls import reverse | ||||
| from django.utils import timezone | from django.utils import timezone | ||||
| from rest_framework import status | from rest_framework import status | ||||
| from rest_framework.authentication import BaseAuthentication, BasicAuthentication | from rest_framework.authentication import BaseAuthentication, BasicAuthentication | ||||
| from rest_framework.permissions import BasePermission, IsAuthenticated | from rest_framework.permissions import BasePermission, IsAuthenticated | ||||
| from rest_framework.request import Request | from rest_framework.request import Request | ||||
| from rest_framework.views import APIView | from rest_framework.views import APIView | ||||
| from swh.deposit.api.checks import check_metadata | from swh.deposit.api.checks import check_metadata | ||||
| from swh.deposit.api.converters import convert_status_detail | from swh.deposit.api.converters import convert_status_detail | ||||
| from swh.deposit.models import Deposit | from swh.deposit.models import Deposit | ||||
| from swh.deposit.utils import compute_metadata_context | from swh.deposit.utils import compute_metadata_context | ||||
| from swh.model import hashutil | from swh.model import hashutil | ||||
| from swh.model.identifiers import ( | from swh.model.identifiers import ( | ||||
| ExtendedObjectType, | ExtendedObjectType, | ||||
| ExtendedSWHID, | ExtendedSWHID, | ||||
| QualifiedSWHID, | QualifiedSWHID, | ||||
| ValidationError, | ValidationError, | ||||
| ) | ) | ||||
| from swh.model.model import ( | from swh.model.model import ( | ||||
| MetadataAuthority, | MetadataAuthority, | ||||
| MetadataAuthorityType, | MetadataAuthorityType, | ||||
| MetadataFetcher, | |||||
| Origin, | Origin, | ||||
| RawExtrinsicMetadata, | RawExtrinsicMetadata, | ||||
| ) | ) | ||||
| from swh.scheduler.utils import create_oneshot_task_dict | from swh.scheduler.utils import create_oneshot_task_dict | ||||
| from ..config import ( | from ..config import ( | ||||
| ARCHIVE_KEY, | ARCHIVE_KEY, | ||||
| ARCHIVE_TYPE, | ARCHIVE_TYPE, | ||||
| ▲ Show 20 Lines • Show All 607 Lines • ▼ Show 20 Lines | ) -> Tuple[ExtendedSWHID, Deposit, DepositRequest]: | ||||
| ) | ) | ||||
| metadata_authority = MetadataAuthority( | metadata_authority = MetadataAuthority( | ||||
| type=MetadataAuthorityType.DEPOSIT_CLIENT, | type=MetadataAuthorityType.DEPOSIT_CLIENT, | ||||
| url=deposit.client.provider_url, | url=deposit.client.provider_url, | ||||
| metadata={"name": deposit.client.last_name}, | metadata={"name": deposit.client.last_name}, | ||||
| ) | ) | ||||
| metadata_fetcher = MetadataFetcher( | metadata_fetcher = self.swh_deposit_fetcher() | ||||
| name=self.tool["name"], | |||||
| version=self.tool["version"], | |||||
| metadata=self.tool["configuration"], | |||||
| ) | |||||
| # replace metadata within the deposit backend | # replace metadata within the deposit backend | ||||
| deposit_request_data = { | deposit_request_data = { | ||||
| METADATA_KEY: metadata, | METADATA_KEY: metadata, | ||||
| RAW_METADATA_KEY: raw_metadata, | RAW_METADATA_KEY: raw_metadata, | ||||
| } | } | ||||
| # actually add the metadata to the completed deposit | # actually add the metadata to the completed deposit | ||||
| deposit_request = self._deposit_request_put(deposit, deposit_request_data) | deposit_request = self._deposit_request_put(deposit, deposit_request_data) | ||||
| target_swhid: ExtendedSWHID # origin URL or CoreSWHID | target_swhid: ExtendedSWHID # origin URL or CoreSWHID | ||||
| if isinstance(swhid_reference, str): | if isinstance(swhid_reference, str): | ||||
| target_swhid = Origin(swhid_reference).swhid() | target_swhid = Origin(swhid_reference).swhid() | ||||
| metadata_context = {} | metadata_context = {} | ||||
| else: | else: | ||||
| metadata_context = compute_metadata_context(swhid_reference) | metadata_context = compute_metadata_context(swhid_reference) | ||||
| if deposit_origin: # metadata deposit update on completed deposit | if deposit_origin: # metadata deposit update on completed deposit | ||||
| metadata_context["origin"] = deposit_origin | metadata_context["origin"] = deposit_origin | ||||
| target_swhid = extended_swhid_from_qualified(swhid_reference) | target_swhid = extended_swhid_from_qualified(swhid_reference) | ||||
| self._check_swhid_in_archive(target_swhid) | self._check_swhid_in_archive(target_swhid) | ||||
| # metadata deposited by the client | |||||
| metadata_object = RawExtrinsicMetadata( | metadata_object = RawExtrinsicMetadata( | ||||
| target=target_swhid, # core swhid or origin | target=target_swhid, # core swhid or origin | ||||
| discovery_date=deposit_request.date, | discovery_date=deposit_request.date, | ||||
| authority=metadata_authority, | authority=metadata_authority, | ||||
| fetcher=metadata_fetcher, | fetcher=metadata_fetcher, | ||||
| format="sword-v2-atom-codemeta", | format="sword-v2-atom-codemeta", | ||||
| metadata=raw_metadata, | metadata=raw_metadata, | ||||
| **metadata_context, | **metadata_context, | ||||
| ) | ) | ||||
| # metadata on the metadata object | |||||
| swh_deposit_authority = self.swh_deposit_authority() | |||||
| swh_deposit_fetcher = self.swh_deposit_fetcher() | |||||
| metametadata_object = RawExtrinsicMetadata( | |||||
| target=metadata_object.swhid(), | |||||
| discovery_date=deposit_request.date, | |||||
| authority=swh_deposit_authority, | |||||
| fetcher=swh_deposit_fetcher, | |||||
| format="xml-deposit-info", | |||||
| metadata=render_to_string( | |||||
| "deposit/deposit_info.xml", context={"deposit": deposit} | |||||
| ).encode(), | |||||
| ) | |||||
| # write to metadata storage | # write to metadata storage | ||||
| self.storage_metadata.metadata_authority_add([metadata_authority]) | self.storage_metadata.metadata_authority_add( | ||||
| self.storage_metadata.metadata_fetcher_add([metadata_fetcher]) | [metadata_authority, swh_deposit_authority] | ||||
| self.storage_metadata.raw_extrinsic_metadata_add([metadata_object]) | ) | ||||
| self.storage_metadata.metadata_fetcher_add( | |||||
| [metadata_fetcher, swh_deposit_fetcher] | |||||
| ) | |||||
| self.storage_metadata.raw_extrinsic_metadata_add( | |||||
| [metadata_object, metametadata_object] | |||||
| ) | |||||
| return (target_swhid, deposit, deposit_request) | return (target_swhid, deposit, deposit_request) | ||||
| def _check_swhid_in_archive(self, target_swhid: ExtendedSWHID) -> None: | def _check_swhid_in_archive(self, target_swhid: ExtendedSWHID) -> None: | ||||
| """Check the target object already exists in the archive, | """Check the target object already exists in the archive, | ||||
| and raises a BAD_REQUEST if it does not.""" | and raises a BAD_REQUEST if it does not.""" | ||||
| if target_swhid.object_type in (ExtendedObjectType.CONTENT,): | if target_swhid.object_type in (ExtendedObjectType.CONTENT,): | ||||
| if list( | if list( | ||||
| ▲ Show 20 Lines • Show All 534 Lines • Show Last 20 Lines | |||||