Page MenuHomeSoftware Heritage
Differential D399

deposit/loader: Add missing origin_url transmission step
ClosedPublic
Actions

Authored by ardumont on Jul 23 2018, 12:20 PM.

Details

Reviewers
moranegg
Commits
rDDEP2c4c4840ec38: deposit/loader: Add missing origin_url transmission step
Summary

We need to transmit the origin url needed for the contextual
persistent id computation.

This diff adds the missing step from D396 (merged a tad too fast).

Test Plan

Tests ok

Diff Detail

Repository
rDDEP Push deposit
Branch
master
Lint
No Linters Available
Unit
No Unit Test Coverage
Build Status
Buildable 1363
Build 1707: arc lint + arc unit

Event Timeline

ardumont created this revision.Jul 23 2018, 12:20 PM
Harbormaster completed remote builds in B1362: Diff 1243.Jul 23 2018, 12:20 PM
ardumont updated this revision to Diff 1244.Jul 23 2018, 1:27 PM

Update the loader's test which clarifies the url source use

Harbormaster completed remote builds in B1363: Diff 1244.Jul 23 2018, 1:27 PM
moranegg accepted this revision.Jul 23 2018, 1:37 PM
This revision is now accepted and ready to land.Jul 23 2018, 1:37 PM
Closed by commit rDDEP2c4c4840ec38: deposit/loader: Add missing origin_url transmission step (authored by ardumont). · Explain WhyJul 23 2018, 1:43 PM
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

PathSize
swh/
deposit/
client/
5 lines
loader/
11 lines
tests/
loader/
4 lines
28 lines

Diff 1244

View Options

swh/deposit/client/__init__.py

Show First 20 Lines • Show All 162 Lines • ▼ Show 20 Lines def metadata_get(self, metadata_url, log=None):
msg = 'Problem when retrieving metadata at %s' % metadata_url msg = 'Problem when retrieving metadata at %s' % metadata_url
if log: if log:
log.error(msg) log.error(msg)
raise ValueError(msg) raise ValueError(msg)
def status_update(self, update_status_url, status, def status_update(self, update_status_url, status,
revision_id=None, directory_id=None): revision_id=None, directory_id=None, origin_url=None):
"""Update the deposit's status. """Update the deposit's status.
Args: Args:
update_status_url (str): the full deposit's archive update_status_url (str): the full deposit's archive
status (str): The status to update the deposit with status (str): The status to update the deposit with
revision_id (str/None): the revision's identifier to update to revision_id (str/None): the revision's identifier to update to
directory_id (str/None): the directory's identifier to update to directory_id (str/None): the directory's identifier to update to
origin_url (str/None): deposit's associated origin url
""" """
payload = {'status': status} payload = {'status': status}
if revision_id: if revision_id:
payload['revision_id'] = revision_id payload['revision_id'] = revision_id
if directory_id: if directory_id:
payload['directory_id'] = directory_id payload['directory_id'] = directory_id
if origin_url:
payload['origin_url'] = origin_url
self.do('put', update_status_url, json=payload) self.do('put', update_status_url, json=payload)
def check(self, check_url, log=None): def check(self, check_url, log=None):
"""Check the deposit's associated data (metadata, archive(s)) """Check the deposit's associated data (metadata, archive(s))
Args: Args:
check_url (str): the full deposit's check url check_url (str): the full deposit's check url
▲ Show 20 Lines • Show All 378 Lines • Show Last 20 Lines
View Options

swh/deposit/loader/loader.py

Show First 20 Lines • Show All 110 Lines • ▼ Show 20 Lines def post_load(self, success=True):
return return
revisions = self.objects['revision'] revisions = self.objects['revision']
# Retrieve the revision # Retrieve the revision
[rev_id] = revisions.keys() [rev_id] = revisions.keys()
rev = revisions[rev_id] rev = revisions[rev_id]
if rev_id: if rev_id:
rev_id = hashutil.hash_to_hex(rev_id) rev_id = hashutil.hash_to_hex(rev_id)
dir_id = rev['directory'] dir_id = rev['directory']
if dir_id: if dir_id:
dir_id = hashutil.hash_to_hex(dir_id) dir_id = hashutil.hash_to_hex(dir_id)
# update the deposit's status to success with its # update the deposit's status to success with its
# revision-id and directory-id # revision-id and directory-id
self.client.status_update(self.deposit_update_url, self.client.status_update(
self.deposit_update_url,
status='done', status='done',
revision_id=rev_id, revision_id=rev_id,
directory_id=dir_id) directory_id=dir_id,
origin_url=self.origin['url'])
except Exception: except Exception:
self.log.exception( self.log.exception(
'Problem when trying to update the deposit\'s status') 'Problem when trying to update the deposit\'s status')
def cleanup(self): def cleanup(self):
"""Clean up temporary directory where we retrieved the tarball. """Clean up temporary directory where we retrieved the tarball.
""" """
super().cleanup() super().cleanup()
self.temporary_directory.cleanup() self.temporary_directory.cleanup()
View Options

swh/deposit/tests/loader/common.py

Show All 30 Lines def archive_get(self, archive_update_url, archive_path, log=None):
return archive_path return archive_path
def metadata_get(self, metadata_url, log=None): def metadata_get(self, metadata_url, log=None):
r = self.client.get(metadata_url) r = self.client.get(metadata_url)
return json.loads(r.content.decode('utf-8')) return json.loads(r.content.decode('utf-8'))
def status_update(self, update_status_url, status, def status_update(self, update_status_url, status,
revision_id=None, directory_id=None): revision_id=None, directory_id=None, origin_url=None):
payload = {'status': status} payload = {'status': status}
if revision_id: if revision_id:
payload['revision_id'] = revision_id payload['revision_id'] = revision_id
if directory_id: if directory_id:
payload['directory_id'] = directory_id payload['directory_id'] = directory_id
if origin_url:
payload['origin_url'] = origin_url
self.client.put(update_status_url, self.client.put(update_status_url,
content_type='application/json', content_type='application/json',
data=json.dumps(payload)) data=json.dumps(payload))
def check(self, check_url): def check(self, check_url):
r = self.client.get(check_url) r = self.client.get(check_url)
data = json.loads(r.content.decode('utf-8')) data = json.loads(r.content.decode('utf-8'))
return data['status'] return data['status']
View Options

swh/deposit/tests/loader/test_loader.py

# Copyright (C) 2017-2018 The Software Heritage developers # Copyright (C) 2017-2018 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution # See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version # License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information # See top-level LICENSE file for more information
import os import os
import unittest import unittest
import shutil import shutil
from nose.tools import istest from nose.tools import istest
from nose.plugins.attrib import attr from nose.plugins.attrib import attr
from rest_framework.test import APITestCase from rest_framework.test import APITestCase
from swh.model import hashutil from swh.model import hashutil
from swh.deposit.models import Deposit
from swh.deposit.loader import loader from swh.deposit.loader import loader
from swh.deposit.config import PRIVATE_GET_RAW_CONTENT from swh.deposit.config import (
from swh.deposit.config import PRIVATE_GET_DEPOSIT_METADATA PRIVATE_GET_RAW_CONTENT, PRIVATE_GET_DEPOSIT_METADATA, PRIVATE_PUT_DEPOSIT
from swh.deposit.config import PRIVATE_PUT_DEPOSIT )
from django.core.urlresolvers import reverse from django.core.urlresolvers import reverse
from .common import SWHDepositTestClient, CLIENT_TEST_CONFIG from .common import SWHDepositTestClient, CLIENT_TEST_CONFIG
from .. import TEST_LOADER_CONFIG from .. import TEST_LOADER_CONFIG
from ..common import BasicTestCase, WithAuthTestCase, CommonCreationRoutine from ..common import (BasicTestCase, WithAuthTestCase,
from ..common import FileSystemCreationRoutine CommonCreationRoutine,
FileSystemCreationRoutine)
TOOL_ID = 99 TOOL_ID = 99
PROVIDER_ID = 12 PROVIDER_ID = 12
class DepositLoaderInhibitsStorage: class DepositLoaderInhibitsStorage:
"""Mixin class to inhibit the persistence and keep in memory the data """Mixin class to inhibit the persistence and keep in memory the data
▲ Show 20 Lines • Show All 217 Lines • ▼ Show 20 Lines def inject_deposit_verify_metadata(self):
self.assertEquals(len(self.loader.state['revision']), 1) self.assertEquals(len(self.loader.state['revision']), 1)
self.assertEquals(len(self.loader.state['release']), 0) self.assertEquals(len(self.loader.state['release']), 0)
self.assertEquals(len(self.loader.state['snapshot']), 1) self.assertEquals(len(self.loader.state['snapshot']), 1)
self.assertEquals(len(self.loader.state['origin_metadata']), 1) self.assertEquals(len(self.loader.state['origin_metadata']), 1)
self.assertEquals(len(self.loader.state['tool']), 1) self.assertEquals(len(self.loader.state['tool']), 1)
self.assertEquals(len(self.loader.state['provider']), 1) self.assertEquals(len(self.loader.state['provider']), 1)
codemeta = 'codemeta:' codemeta = 'codemeta:'
origin_url = 'https://hal-test.archives-ouvertes.fr/hal-01243065'
expected_origin_metadata = { expected_origin_metadata = {
'@xmlns': 'http://www.w3.org/2005/Atom', '@xmlns': 'http://www.w3.org/2005/Atom',
'@xmlns:codemeta': 'https://doi.org/10.5063/SCHEMA/CODEMETA-2.0', '@xmlns:codemeta': 'https://doi.org/10.5063/SCHEMA/CODEMETA-2.0',
'author': { 'author': {
'email': 'hal@ccsd.cnrs.fr', 'email': 'hal@ccsd.cnrs.fr',
'name': 'HAL' 'name': 'HAL'
}, },
codemeta + 'url': codemeta + 'url': origin_url,
'https://hal-test.archives-ouvertes.fr/hal-01243065',
codemeta + 'runtimePlatform': 'phpstorm', codemeta + 'runtimePlatform': 'phpstorm',
codemeta + 'license': [ codemeta + 'license': [
{ {
codemeta + 'name': 'GNU General Public License v3.0 only' codemeta + 'name': 'GNU General Public License v3.0 only'
}, },
{ {
codemeta + 'name': 'CeCILL Free Software License Agreement v1.1' # noqa codemeta + 'name': 'CeCILL Free Software License Agreement v1.1' # noqa
} }
Show All 12 Lines def inject_deposit_verify_metadata(self):
'client': 'hal', 'client': 'hal',
codemeta + 'keywords': 'DSP programming,Web', codemeta + 'keywords': 'DSP programming,Web',
codemeta + 'developmentStatus': 'stable' codemeta + 'developmentStatus': 'stable'
} }
result = self.loader.state['origin_metadata'][0] result = self.loader.state['origin_metadata'][0]
self.assertEquals(result['metadata'], expected_origin_metadata) self.assertEquals(result['metadata'], expected_origin_metadata)
self.assertEquals(result['tool_id'], TOOL_ID) self.assertEquals(result['tool_id'], TOOL_ID)
self.assertEquals(result['provider_id'], PROVIDER_ID) self.assertEquals(result['provider_id'], PROVIDER_ID)
deposit = Deposit.objects.get(pk=self.deposit_id)
self.assertRegex(deposit.swh_id, r'^swh:1:dir:.*')
self.assertEquals(deposit.swh_id_context, '%s;origin=%s' % (
deposit.swh_id, origin_url
))
self.assertRegex(deposit.swh_anchor_id, r'^swh:1:rev:.*')
self.assertEquals(deposit.swh_anchor_id_context, '%s;origin=%s' % (
deposit.swh_anchor_id, origin_url
))
About · Developer information · Legal