Page MenuHomeSoftware Heritage
Differential D1122 Diff 3553 swh/indexer/tests/test_metadata.py

Changeset View

Standalone View

View Options

swh/indexer/tests/test_metadata.py

# Copyright (C) 2017-2018 The Software Heritage developers # Copyright (C) 2017-2018 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution # See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version # License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information # See top-level LICENSE file for more information
import json import json
import unittest import unittest
from hypothesis import given, strategies from hypothesis import given, strategies
import xmltodict
from swh.model.hashutil import hash_to_bytes from swh.model.hashutil import hash_to_bytes
from swh.indexer.codemeta import CODEMETA_KEYS from swh.indexer.codemeta import CODEMETA_KEYS
from swh.indexer.metadata_dictionary import ( from swh.indexer.metadata_dictionary import (
CROSSWALK_TABLE, MAPPINGS, merge_values) CROSSWALK_TABLE, MAPPINGS, merge_values)
from swh.indexer.metadata_detector import ( from swh.indexer.metadata_detector import (
detect_metadata, extract_minimal_metadata_dict detect_metadata, extract_minimal_metadata_dict
) )
from swh.indexer.metadata import ( from swh.indexer.metadata import (
ContentMetadataIndexer, RevisionMetadataIndexer ContentMetadataIndexer, RevisionMetadataIndexer
) )
from .utils import ( from .utils import (
BASE_TEST_CONFIG, fill_obj_storage, fill_storage, BASE_TEST_CONFIG, fill_obj_storage, fill_storage,
YARN_PARSER_METADATA, json_document_strategy YARN_PARSER_METADATA, json_document_strategy,
xml_document_strategy,
) )
TRANSLATOR_TOOL = { TRANSLATOR_TOOL = {
'name': 'swh-metadata-translator', 'name': 'swh-metadata-translator',
'version': '0.0.2', 'version': '0.0.2',
'configuration': { 'configuration': {
'type': 'local', 'type': 'local',
▲ Show 20 Lines • Show All 532 Lines • ▼ Show 20 Lines def test_compute_metadata_codemeta_alternate_context(self):
"type": "SoftwareSourceCode", "type": "SoftwareSourceCode",
"identifier": "CodeMeta", "identifier": "CodeMeta",
} }
result = self.codemeta_mapping.translate(raw_content) result = self.codemeta_mapping.translate(raw_content)
self.assertEqual(result, expected_result) self.assertEqual(result, expected_result)
def test_compute_metadata_maven(self): def test_compute_metadata_maven(self):
raw_content = b""" raw_content = b"""
<project> <project xmlns="http://maven.apache.org/POM/4.0.0">
<name>Maven Default Project</name> <name>Maven Default Project</name>
<modelVersion>4.0.0</modelVersion> <modelVersion>4.0.0</modelVersion>
<groupId>com.mycompany.app</groupId> <groupId>com.mycompany.app</groupId>
<artifactId>my-app</artifactId> <artifactId>my-app</artifactId>
<version>1.2.3</version> <version>1.2.3</version>
<repositories> <repositories>
<repository> <repository>
<id>central</id> <id>central</id>
Show All 23 Lines def test_compute_metadata_maven(self):
'version': '1.2.3', 'version': '1.2.3',
'license': 'https://www.apache.org/licenses/LICENSE-2.0.txt', 'license': 'https://www.apache.org/licenses/LICENSE-2.0.txt',
'codeRepository': 'codeRepository':
'http://repo1.maven.org/maven2/com/mycompany/app/my-app', 'http://repo1.maven.org/maven2/com/mycompany/app/my-app',
}) })
def test_compute_metadata_maven_empty(self): def test_compute_metadata_maven_empty(self):
raw_content = b""" raw_content = b"""
<project> <project xmlns="http://maven.apache.org/POM/4.0.0">
</project>""" </project>"""
result = self.maven_mapping.translate(raw_content) result = self.maven_mapping.translate(raw_content)
self.assertEqual(result, { self.assertEqual(result, {
'@context': 'https://doi.org/10.5063/schema/codemeta-2.0', '@context': 'https://doi.org/10.5063/schema/codemeta-2.0',
'type': 'SoftwareSourceCode', 'type': 'SoftwareSourceCode',
}) })
def test_compute_metadata_maven_almost_empty(self): def test_compute_metadata_maven_almost_empty(self):
raw_content = b""" raw_content = b"""
<project> <project xmlns="http://maven.apache.org/POM/4.0.0">
<foo/> <foo/>
</project>""" </project>"""
result = self.maven_mapping.translate(raw_content) result = self.maven_mapping.translate(raw_content)
self.assertEqual(result, { self.assertEqual(result, {
'@context': 'https://doi.org/10.5063/schema/codemeta-2.0', '@context': 'https://doi.org/10.5063/schema/codemeta-2.0',
'type': 'SoftwareSourceCode', 'type': 'SoftwareSourceCode',
}) })
def test_compute_metadata_maven_unknown_project_ns(self):
raw_content = b"""
<foo:project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:foo="http://example.com/foo">
<name>Maven Default Project</name>
<version>1.2.3</version>
</foo:project>"""
result = self.maven_mapping.translate(raw_content)
self.assertEqual(result, {
'@context': 'https://doi.org/10.5063/schema/codemeta-2.0',
'type': 'SoftwareSourceCode',
})
def test_compute_metadata_maven_unknown_element_ns(self):
raw_content = b"""
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:foo="http://example.com/foo">
<foo:name>Maven Default Project</foo:name>
<version>1.2.3</version>
</project>"""
result = self.maven_mapping.translate(raw_content)
self.assertEqual(result, {
'@context': 'https://doi.org/10.5063/schema/codemeta-2.0',
'type': 'SoftwareSourceCode',
'version': '1.2.3',
})
def test_compute_metadata_maven_invalid_xml(self): def test_compute_metadata_maven_invalid_xml(self):
expected_warning = ( expected_warning = (
'WARNING:swh.indexer.metadata_dictionary.MavenMapping:' 'WARNING:swh.indexer.metadata_dictionary.MavenMapping:'
'Error parsing XML from foo') 'Error parsing XML from foo')
raw_content = b""" raw_content = b"""
<project>""" <project xmlns="http://maven.apache.org/POM/4.0.0">"""
with self.assertLogs('swh.indexer.metadata_dictionary', with self.assertLogs('swh.indexer.metadata_dictionary',
level='WARNING') as cm: level='WARNING') as cm:
result = MAPPINGS["MavenMapping"]('foo').translate(raw_content) result = MAPPINGS["MavenMapping"]('foo').translate(raw_content)
self.assertEqual(cm.output, [expected_warning]) self.assertEqual(cm.output, [expected_warning])
self.assertEqual(result, None) self.assertEqual(result, None)
raw_content = b""" raw_content = b"""
""" """
with self.assertLogs('swh.indexer.metadata_dictionary', with self.assertLogs('swh.indexer.metadata_dictionary',
level='WARNING') as cm: level='WARNING') as cm:
result = MAPPINGS["MavenMapping"]('foo').translate(raw_content) result = MAPPINGS["MavenMapping"]('foo').translate(raw_content)
self.assertEqual(cm.output, [expected_warning]) self.assertEqual(cm.output, [expected_warning])
self.assertEqual(result, None) self.assertEqual(result, None)
def test_compute_metadata_maven_unknown_encoding(self): def test_compute_metadata_maven_unknown_encoding(self):
expected_warning = ( expected_warning = (
'WARNING:swh.indexer.metadata_dictionary.MavenMapping:' 'WARNING:swh.indexer.metadata_dictionary.MavenMapping:'
'Error detecting XML encoding from foo') 'Error detecting XML encoding from foo')
raw_content = b"""<?xml version="1.0" encoding="foo"?> raw_content = b"""<?xml version="1.0" encoding="foo"?>
<project> <project xmlns="http://maven.apache.org/POM/4.0.0">
</project>""" </project>"""
with self.assertLogs('swh.indexer.metadata_dictionary', with self.assertLogs('swh.indexer.metadata_dictionary',
level='WARNING') as cm: level='WARNING') as cm:
result = MAPPINGS["MavenMapping"]('foo').translate(raw_content) result = MAPPINGS["MavenMapping"]('foo').translate(raw_content)
self.assertEqual(cm.output, [expected_warning]) self.assertEqual(cm.output, [expected_warning])
self.assertEqual(result, None) self.assertEqual(result, None)
raw_content = b"""<?xml version="1.0" encoding="UTF-7"?> raw_content = b"""<?xml version="1.0" encoding="UTF-7"?>
<project> <project xmlns="http://maven.apache.org/POM/4.0.0">
</project>""" </project>"""
with self.assertLogs('swh.indexer.metadata_dictionary', with self.assertLogs('swh.indexer.metadata_dictionary',
level='WARNING') as cm: level='WARNING') as cm:
result = MAPPINGS["MavenMapping"]('foo').translate(raw_content) result = MAPPINGS["MavenMapping"]('foo').translate(raw_content)
self.assertEqual(cm.output, [expected_warning]) self.assertEqual(cm.output, [expected_warning])
self.assertEqual(result, None) self.assertEqual(result, None)
def test_compute_metadata_maven_invalid_encoding(self): def test_compute_metadata_maven_invalid_encoding(self):
expected_warning = ( expected_warning = (
'WARNING:swh.indexer.metadata_dictionary.MavenMapping:' 'WARNING:swh.indexer.metadata_dictionary.MavenMapping:'
'Error unidecoding XML from foo') 'Error unidecoding XML from foo')
raw_content = b"""<?xml version="1.0" encoding="UTF-8"?> raw_content = b"""<?xml version="1.0" encoding="UTF-8"?>
<foo\xe5ct> <foo\xe5ct>
</foo>""" </foo>"""
with self.assertLogs('swh.indexer.metadata_dictionary', with self.assertLogs('swh.indexer.metadata_dictionary',
level='WARNING') as cm: level='WARNING') as cm:
result = MAPPINGS["MavenMapping"]('foo').translate(raw_content) result = MAPPINGS["MavenMapping"]('foo').translate(raw_content)
self.assertEqual(cm.output, [expected_warning]) self.assertEqual(cm.output, [expected_warning])
self.assertEqual(result, None) self.assertEqual(result, None)
def test_compute_metadata_maven_minimal(self): def test_compute_metadata_maven_minimal(self):
raw_content = b""" raw_content = b"""
<project> <project xmlns="http://maven.apache.org/POM/4.0.0">
<name>Maven Default Project</name> <name>Maven Default Project</name>
<modelVersion>4.0.0</modelVersion> <modelVersion>4.0.0</modelVersion>
<groupId>com.mycompany.app</groupId> <groupId>com.mycompany.app</groupId>
<artifactId>my-app</artifactId> <artifactId>my-app</artifactId>
<version>1.2.3</version> <version>1.2.3</version>
</project>""" </project>"""
result = self.maven_mapping.translate(raw_content) result = self.maven_mapping.translate(raw_content)
self.assertEqual(result, { self.assertEqual(result, {
'@context': 'https://doi.org/10.5063/schema/codemeta-2.0', '@context': 'https://doi.org/10.5063/schema/codemeta-2.0',
'type': 'SoftwareSourceCode', 'type': 'SoftwareSourceCode',
'name': 'Maven Default Project', 'name': 'Maven Default Project',
'identifier': 'com.mycompany.app', 'identifier': 'com.mycompany.app',
'version': '1.2.3', 'version': '1.2.3',
'codeRepository': 'codeRepository':
'https://repo.maven.apache.org/maven2/com/mycompany/app/my-app', 'https://repo.maven.apache.org/maven2/com/mycompany/app/my-app',
}) })
def test_compute_metadata_maven_empty_nodes(self): def test_compute_metadata_maven_empty_nodes(self):
raw_content = b""" raw_content = b"""
<project> <project xmlns="http://maven.apache.org/POM/4.0.0">
<name>Maven Default Project</name> <name>Maven Default Project</name>
<modelVersion>4.0.0</modelVersion> <modelVersion>4.0.0</modelVersion>
<groupId>com.mycompany.app</groupId> <groupId>com.mycompany.app</groupId>
<artifactId>my-app</artifactId> <artifactId>my-app</artifactId>
<version>1.2.3</version> <version>1.2.3</version>
<repositories> <repositories>
</repositories> </repositories>
</project>""" </project>"""
result = self.maven_mapping.translate(raw_content) result = self.maven_mapping.translate(raw_content)
self.assertEqual(result, { self.assertEqual(result, {
'@context': 'https://doi.org/10.5063/schema/codemeta-2.0', '@context': 'https://doi.org/10.5063/schema/codemeta-2.0',
'type': 'SoftwareSourceCode', 'type': 'SoftwareSourceCode',
'name': 'Maven Default Project', 'name': 'Maven Default Project',
'identifier': 'com.mycompany.app', 'identifier': 'com.mycompany.app',
'version': '1.2.3', 'version': '1.2.3',
'codeRepository': 'codeRepository':
'https://repo.maven.apache.org/maven2/com/mycompany/app/my-app', 'https://repo.maven.apache.org/maven2/com/mycompany/app/my-app',
}) })
raw_content = b""" raw_content = b"""
<project> <project xmlns="http://maven.apache.org/POM/4.0.0">
<name>Maven Default Project</name> <name>Maven Default Project</name>
<modelVersion>4.0.0</modelVersion> <modelVersion>4.0.0</modelVersion>
<groupId>com.mycompany.app</groupId> <groupId>com.mycompany.app</groupId>
<artifactId>my-app</artifactId> <artifactId>my-app</artifactId>
<version></version> <version></version>
</project>""" </project>"""
result = self.maven_mapping.translate(raw_content) result = self.maven_mapping.translate(raw_content)
self.assertEqual(result, { self.assertEqual(result, {
'@context': 'https://doi.org/10.5063/schema/codemeta-2.0', '@context': 'https://doi.org/10.5063/schema/codemeta-2.0',
'type': 'SoftwareSourceCode', 'type': 'SoftwareSourceCode',
'name': 'Maven Default Project', 'name': 'Maven Default Project',
'identifier': 'com.mycompany.app', 'identifier': 'com.mycompany.app',
'codeRepository': 'codeRepository':
'https://repo.maven.apache.org/maven2/com/mycompany/app/my-app', 'https://repo.maven.apache.org/maven2/com/mycompany/app/my-app',
}) })
raw_content = b""" raw_content = b"""
<project> <project xmlns="http://maven.apache.org/POM/4.0.0">
<name></name> <name></name>
<modelVersion>4.0.0</modelVersion> <modelVersion>4.0.0</modelVersion>
<groupId>com.mycompany.app</groupId> <groupId>com.mycompany.app</groupId>
<artifactId>my-app</artifactId> <artifactId>my-app</artifactId>
<version>1.2.3</version> <version>1.2.3</version>
</project>""" </project>"""
result = self.maven_mapping.translate(raw_content) result = self.maven_mapping.translate(raw_content)
self.assertEqual(result, { self.assertEqual(result, {
'@context': 'https://doi.org/10.5063/schema/codemeta-2.0', '@context': 'https://doi.org/10.5063/schema/codemeta-2.0',
'type': 'SoftwareSourceCode', 'type': 'SoftwareSourceCode',
'identifier': 'com.mycompany.app', 'identifier': 'com.mycompany.app',
'version': '1.2.3', 'version': '1.2.3',
'codeRepository': 'codeRepository':
'https://repo.maven.apache.org/maven2/com/mycompany/app/my-app', 'https://repo.maven.apache.org/maven2/com/mycompany/app/my-app',
}) })
raw_content = b""" raw_content = b"""
<project> <project xmlns="http://maven.apache.org/POM/4.0.0">
<name>Maven Default Project</name> <name>Maven Default Project</name>
<modelVersion>4.0.0</modelVersion> <modelVersion>4.0.0</modelVersion>
<groupId>com.mycompany.app</groupId> <groupId>com.mycompany.app</groupId>
<artifactId>my-app</artifactId> <artifactId>my-app</artifactId>
<version>1.2.3</version> <version>1.2.3</version>
<licenses> <licenses>
</licenses> </licenses>
</project>""" </project>"""
result = self.maven_mapping.translate(raw_content) result = self.maven_mapping.translate(raw_content)
self.assertEqual(result, { self.assertEqual(result, {
'@context': 'https://doi.org/10.5063/schema/codemeta-2.0', '@context': 'https://doi.org/10.5063/schema/codemeta-2.0',
'type': 'SoftwareSourceCode', 'type': 'SoftwareSourceCode',
'name': 'Maven Default Project', 'name': 'Maven Default Project',
'identifier': 'com.mycompany.app', 'identifier': 'com.mycompany.app',
'version': '1.2.3', 'version': '1.2.3',
'codeRepository': 'codeRepository':
'https://repo.maven.apache.org/maven2/com/mycompany/app/my-app', 'https://repo.maven.apache.org/maven2/com/mycompany/app/my-app',
}) })
raw_content = b""" raw_content = b"""
<project> <project xmlns="http://maven.apache.org/POM/4.0.0">
<groupId></groupId> <groupId></groupId>
<version>1.2.3</version> <version>1.2.3</version>
</project>""" </project>"""
result = self.maven_mapping.translate(raw_content) result = self.maven_mapping.translate(raw_content)
self.assertEqual(result, { self.assertEqual(result, {
'@context': 'https://doi.org/10.5063/schema/codemeta-2.0', '@context': 'https://doi.org/10.5063/schema/codemeta-2.0',
'type': 'SoftwareSourceCode', 'type': 'SoftwareSourceCode',
'version': '1.2.3', 'version': '1.2.3',
}) })
def test_compute_metadata_maven_invalid_licenses(self): def test_compute_metadata_maven_invalid_licenses(self):
raw_content = b""" raw_content = b"""
<project> <project xmlns="http://maven.apache.org/POM/4.0.0">
<name>Maven Default Project</name> <name>Maven Default Project</name>
<modelVersion>4.0.0</modelVersion> <modelVersion>4.0.0</modelVersion>
<groupId>com.mycompany.app</groupId> <groupId>com.mycompany.app</groupId>
<artifactId>my-app</artifactId> <artifactId>my-app</artifactId>
<version>1.2.3</version> <version>1.2.3</version>
<licenses> <licenses>
foo foo
</licenses> </licenses>
</project>""" </project>"""
result = self.maven_mapping.translate(raw_content) result = self.maven_mapping.translate(raw_content)
self.assertEqual(result, { self.assertEqual(result, {
'@context': 'https://doi.org/10.5063/schema/codemeta-2.0', '@context': 'https://doi.org/10.5063/schema/codemeta-2.0',
'type': 'SoftwareSourceCode', 'type': 'SoftwareSourceCode',
'name': 'Maven Default Project', 'name': 'Maven Default Project',
'identifier': 'com.mycompany.app', 'identifier': 'com.mycompany.app',
'version': '1.2.3', 'version': '1.2.3',
'codeRepository': 'codeRepository':
'https://repo.maven.apache.org/maven2/com/mycompany/app/my-app', 'https://repo.maven.apache.org/maven2/com/mycompany/app/my-app',
}) })
def test_compute_metadata_maven_multiple(self): def test_compute_metadata_maven_multiple(self):
'''Tests when there are multiple code repos and licenses.''' '''Tests when there are multiple code repos and licenses.'''
raw_content = b""" raw_content = b"""
<project> <project xmlns="http://maven.apache.org/POM/4.0.0">
<name>Maven Default Project</name> <name>Maven Default Project</name>
<modelVersion>4.0.0</modelVersion> <modelVersion>4.0.0</modelVersion>
<groupId>com.mycompany.app</groupId> <groupId>com.mycompany.app</groupId>
<artifactId>my-app</artifactId> <artifactId>my-app</artifactId>
<version>1.2.3</version> <version>1.2.3</version>
<repositories> <repositories>
<repository> <repository>
<id>central</id> <id>central</id>
▲ Show 20 Lines • Show All 226 Lines • ▼ Show 20 Lines def test_npm_adversarial(self, doc):
raw = json.dumps(doc).encode() raw = json.dumps(doc).encode()
self.npm_mapping.translate(raw) self.npm_mapping.translate(raw)
@given(json_document_strategy(keys=CODEMETA_KEYS)) @given(json_document_strategy(keys=CODEMETA_KEYS))
def test_codemeta_adversarial(self, doc): def test_codemeta_adversarial(self, doc):
raw = json.dumps(doc).encode() raw = json.dumps(doc).encode()
self.codemeta_mapping.translate(raw) self.codemeta_mapping.translate(raw)
@given(json_document_strategy( @given(xml_document_strategy(
keys=list(MAPPINGS['MavenMapping'].mapping))) keys=list(MAPPINGS['MavenMapping'].mapping),
root='project',
xmlns='http://maven.apache.org/POM/4.0.0'))
def test_maven_adversarial(self, doc): def test_maven_adversarial(self, doc):
raw = xmltodict.unparse({'project': doc}, pretty=True) self.maven_mapping.translate(doc)
self.maven_mapping.translate(raw)
@given(strategies.dictionaries( @given(strategies.dictionaries(
# keys # keys
strategies.one_of( strategies.one_of(
strategies.characters(), strategies.characters(),
*map(strategies.just, MAPPINGS['GemspecMapping'].mapping) *map(strategies.just, MAPPINGS['GemspecMapping'].mapping)
), ),
# values # values
▲ Show 20 Lines • Show All 48 Lines • Show Last 20 Lines
About · Developer information · Legal