diff --git a/swh/deposit/api/private/deposit_check.py b/swh/deposit/api/private/deposit_check.py --- a/swh/deposit/api/private/deposit_check.py +++ b/swh/deposit/api/private/deposit_check.py @@ -1,4 +1,4 @@ -# Copyright (C) 2017-2018 The Software Heritage developers +# Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -127,6 +127,7 @@ 'url': False, 'external_identifier': False, 'author': False, + 'codemeta:dateCreated': False, } alternate_fields = { ('name', 'title'): False, # alternate field, at least one diff --git a/swh/deposit/api/private/deposit_read.py b/swh/deposit/api/private/deposit_read.py --- a/swh/deposit/api/private/deposit_read.py +++ b/swh/deposit/api/private/deposit_read.py @@ -1,4 +1,4 @@ -# Copyright (C) 2017-2018 The Software Heritage developers +# Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -9,6 +9,7 @@ import tempfile from contextlib import contextmanager +from dateutil import parser from django.http import FileResponse from rest_framework import status @@ -171,10 +172,15 @@ fullname, deposit.id, deposit.collection.name) complete_date = identifiers.normalize_timestamp(deposit.complete_date) + commit_date = metadata['codemeta:dateCreated'] + if isinstance(commit_date, str): + commit_date = parser.parse(commit_date) + commit_date = identifiers.normalize_timestamp(commit_date) + data['revision'] = { 'synthetic': True, 'date': complete_date, - 'committer_date': complete_date, + 'committer_date': commit_date, 'author': author_committer, 'committer': author_committer, 'type': revision_type, diff --git a/swh/deposit/tests/api/test_deposit_check.py b/swh/deposit/tests/api/test_deposit_check.py --- a/swh/deposit/tests/api/test_deposit_check.py +++ b/swh/deposit/tests/api/test_deposit_check.py @@ -1,4 +1,4 @@ -# Copyright (C) 2017-2018 The Software Heritage developers +# Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -137,7 +137,8 @@ mandatory = details['metadata'][0] self.assertEqual(mandatory['summary'], MANDATORY_FIELDS_MISSING) self.assertEqual(set(mandatory['fields']), - set(['url', 'external_identifier', 'author'])) + set(['url', 'external_identifier', 'author', + 'codemeta:dateCreated'])) alternate = details['metadata'][1] self.assertEqual(alternate['summary'], ALTERNATE_FIELDS_MISSING) self.assertEqual(alternate['fields'], ['name or title']) @@ -180,6 +181,7 @@ 'external_identifier': 'something-else', 'name': 'foo', 'author': 'someone', + 'codemeta:dateCreated': 'now', }) self.assertTrue(actual_check) @@ -191,6 +193,7 @@ 'external_identifier': 'something-else', 'title': 'bar', 'author': 'someone', + 'codemeta:dateCreated': 'past', }) self.assertTrue(actual_check) @@ -204,6 +207,7 @@ 'url': 'something', 'external_identifier': 'something-else', 'author': 'someone', + 'codemeta:dateCreated': 'some-date', }) expected_error = { @@ -228,7 +232,7 @@ expected_error = { 'metadata': [{ 'summary': 'Mandatory fields are missing', - 'fields': ['author'], + 'fields': ['author', 'codemeta:dateCreated'], }] } diff --git a/swh/deposit/tests/api/test_deposit_read_metadata.py b/swh/deposit/tests/api/test_deposit_read_metadata.py --- a/swh/deposit/tests/api/test_deposit_read_metadata.py +++ b/swh/deposit/tests/api/test_deposit_read_metadata.py @@ -1,4 +1,4 @@ -# Copyright (C) 2017-2018 The Software Heritage developers +# Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -50,6 +50,7 @@ 'metadata': { '@xmlns': ['http://www.w3.org/2005/Atom'], 'author': ['some awesome author', 'another one', 'no one'], + 'codemeta:dateCreated': '2017-10-07T15:17:08Z', 'external_identifier': 'some-external-id', 'url': 'https://hal-test.archives-ouvertes.fr/' + 'some-external-id' @@ -70,7 +71,14 @@ }, 'revision': { 'synthetic': True, - 'committer_date': None, + 'committer_date': { + 'timestamp': { + 'seconds': 1507389428, + 'microseconds': 0 + }, + 'offset': 0, + 'negative_utc': False + }, 'message': 'hal: Deposit %s in collection hal' % deposit_id, 'author': SWH_PERSON, 'committer': SWH_PERSON, @@ -79,6 +87,7 @@ '@xmlns': ['http://www.w3.org/2005/Atom'], 'author': ['some awesome author', 'another one', 'no one'], 'external_identifier': 'some-external-id', + 'codemeta:dateCreated': '2017-10-07T15:17:08Z', 'url': 'https://hal-test.archives-ouvertes.fr/' + 'some-external-id' }, @@ -135,6 +144,7 @@ 'metadata': { '@xmlns': ['http://www.w3.org/2005/Atom'], 'author': ['some awesome author', 'another one', 'no one'], + 'codemeta:dateCreated': '2017-10-07T15:17:08Z', 'external_identifier': 'some-external-id', 'url': 'https://hal-test.archives-ouvertes.fr/' + 'some-external-id' @@ -156,7 +166,14 @@ 'revision': { 'synthetic': True, 'date': None, - 'committer_date': None, + 'committer_date': { + 'timestamp': { + 'seconds': 1507389428, + 'microseconds': 0 + }, + 'offset': 0, + 'negative_utc': False + }, 'author': SWH_PERSON, 'committer': SWH_PERSON, 'type': 'tar', @@ -164,6 +181,7 @@ 'metadata': { '@xmlns': ['http://www.w3.org/2005/Atom'], 'author': ['some awesome author', 'another one', 'no one'], + 'codemeta:dateCreated': '2017-10-07T15:17:08Z', 'external_identifier': 'some-external-id', 'url': 'https://hal-test.archives-ouvertes.fr/' + 'some-external-id' diff --git a/swh/deposit/tests/common.py b/swh/deposit/tests/common.py --- a/swh/deposit/tests/common.py +++ b/swh/deposit/tests/common.py @@ -1,4 +1,4 @@ -# Copyright (C) 2017-2018 The Software Heritage developers +# Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -346,6 +346,7 @@ another one no one + 2017-10-07T15:17:08Z """ self.atom_entry_data2 = b""" diff --git a/swh/deposit/tests/loader/test_loader.py b/swh/deposit/tests/loader/test_loader.py --- a/swh/deposit/tests/loader/test_loader.py +++ b/swh/deposit/tests/loader/test_loader.py @@ -1,4 +1,4 @@ -# Copyright (C) 2017-2018 The Software Heritage developers +# Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -59,12 +59,10 @@ # create the extraction dir used by the loader os.makedirs(TEST_LOADER_CONFIG['extraction_dir'], exist_ok=True) - # 1. create a deposit with archive and metadata - self.deposit_id = self.create_simple_binary_deposit() - # 2. Sets a basic client which accesses the test data + # Sets a basic client which accesses the test data loader_client = SWHDepositTestClient(self.client, config=CLIENT_TEST_CONFIG) - # 3. setup loader with that client + # Setup loader with that client self.loader = loader.DepositLoader(client=loader_client) self.storage = self.loader.storage @@ -77,7 +75,11 @@ """Load a deposit which is ready """ - args = [self.collection.name, self.deposit_id] + # create a deposit with archive and metadata + deposit_id = self.create_simple_binary_deposit() + self.update_binary_deposit(deposit_id, status_partial=False) + + args = [self.collection.name, deposit_id] archive_url = reverse(PRIVATE_GET_RAW_CONTENT, args=args) deposit_meta_url = reverse(PRIVATE_GET_DEPOSIT_METADATA, args=args) @@ -100,9 +102,9 @@ """Load a deposit with metadata, test metadata integrity """ - self.deposit_metadata_id = self.add_metadata_to_deposit( - self.deposit_id) - args = [self.collection.name, self.deposit_metadata_id] + deposit_id = self.create_simple_binary_deposit() + self.add_metadata_to_deposit(deposit_id, status_partial=False) + args = [self.collection.name, deposit_id] archive_url = reverse(PRIVATE_GET_RAW_CONTENT, args=args) deposit_meta_url = reverse(PRIVATE_GET_DEPOSIT_METADATA, args=args) @@ -157,7 +159,7 @@ self.assertOriginMetadataContains('deposit', origin_url, expected_origin_metadata) - deposit = Deposit.objects.get(pk=self.deposit_id) + deposit = Deposit.objects.get(pk=deposit_id) self.assertRegex(deposit.swh_id, r'^swh:1:dir:.*') self.assertEqual(deposit.swh_id_context, '%s;origin=%s' % (