diff --git a/mypy.ini b/mypy.ini --- a/mypy.ini +++ b/mypy.ini @@ -11,6 +11,9 @@ [mypy-confluent_kafka.*] ignore_missing_imports = True +[mypy-iso8601.*] +ignore_missing_imports = True + [mypy-magic.*] ignore_missing_imports = True diff --git a/requirements.txt b/requirements.txt --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,7 @@ # the version 2.1.2 is causing segmentation faults # cf https://forge.softwareheritage.org/T3815 frozendict != 2.1.2 +iso8601 pyld rdflib sentry-sdk diff --git a/swh/indexer/metadata_dictionary/codemeta.py b/swh/indexer/metadata_dictionary/codemeta.py --- a/swh/indexer/metadata_dictionary/codemeta.py +++ b/swh/indexer/metadata_dictionary/codemeta.py @@ -9,6 +9,7 @@ from typing import Any, Dict, List, Optional, Tuple, Union import xml.etree.ElementTree as ET +import iso8601 import xmltodict from swh.indexer.codemeta import CODEMETA_CONTEXT_URL, CODEMETA_TERMS, compact, expand @@ -19,6 +20,7 @@ _TAG_RE = re.compile(r"\{(?P.*?)\}(?P.*)") _IGNORED_NAMESPACES = ("http://www.w3.org/2005/Atom",) +_DATE_RE = re.compile("^[0-9]{4}-[0-9]{1,2}-[0-9]{1,2}$") class CodemetaMapping(SingleFileIntrinsicMapping): @@ -84,6 +86,20 @@ # expansion will convert it to a full URI based on # "@context": CODEMETA_CONTEXT_URL jsonld_child = self.xml_to_jsonld(child) + if ( + localname + in ( + "dateCreated", + "dateModified", + "datePublished", + ) + and isinstance(jsonld_child, str) + and _DATE_RE.match(jsonld_child) + ): + # Dates missing a leading zero for their day/month, used + # to be allowed by the deposit; so we need to reformat them + # to be valid ISO8601. + jsonld_child = iso8601.parse_date(jsonld_child).date().isoformat() doc[localname].append(jsonld_child) else: # Otherwise, we already know the URI diff --git a/swh/indexer/tests/metadata_dictionary/test_codemeta.py b/swh/indexer/tests/metadata_dictionary/test_codemeta.py --- a/swh/indexer/tests/metadata_dictionary/test_codemeta.py +++ b/swh/indexer/tests/metadata_dictionary/test_codemeta.py @@ -384,6 +384,28 @@ } +def test_sword_fix_date(): + content = """ + + Name + 2020-12-1 + 2020-12-2 + 2020-12-3 + + """ + + result = MAPPINGS["SwordCodemetaMapping"]().translate(content) + assert result == { + "@context": "https://doi.org/10.5063/schema/codemeta-2.0", + "name": "Name", + "dateModified": "2020-12-01", + "dateCreated": "2020-12-02", + "datePublished": "2020-12-03", + } + + def test_json_sword(): content = """{"id": "hal-01243573", "@xmlns": "http://www.w3.org/2005/Atom", "author": {"name": "Author 1", "email": "foo@example.org"}, "client": "hal", "codemeta:url": "http://example.org/", "codemeta:name": "The assignment problem", "@xmlns:codemeta": "https://doi.org/10.5063/SCHEMA/CODEMETA-2.0", "codemeta:author": {"codemeta:name": "Author 2"}, "codemeta:license": {"codemeta:name": "GNU General Public License v3.0 or later"}}""" # noqa result = MAPPINGS["JsonSwordCodemetaMapping"]().translate(content)