Page MenuHomeSoftware Heritage
Differential D236

Refactor configuration for ContentMetadataIndexer
ClosedPublic
Actions

Authored by moranegg on Jul 28 2017, 1:27 PM.

Details

Reviewers
ardumont
Commits
rDCIDX71aecceae2c2: Refactor configuration for ContentMetadataIndexer

Diff Detail

Repository
rDCIDX Metadata indexer
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

moranegg created this revision.Jul 28 2017, 1:27 PM
Harbormaster completed remote builds in B974: Diff 776.Jul 28 2017, 1:27 PM
ardumont accepted this revision.Jul 28 2017, 1:56 PM
This revision is now accepted and ready to land.Jul 28 2017, 1:56 PM
Closed by commit rDCIDX71aecceae2c2: Refactor configuration for ContentMetadataIndexer (authored by moranegg). · Explain WhyJul 28 2017, 2:03 PM
This revision was automatically updated to reflect the committed changes.

Revision Contents
Changeset List

PathSize
swh/
indexer/
17 lines

Diff 777

View Options

swh/indexer/metadata.py

Show First 20 Lines • Show All 78 Lines • ▼ Show 20 Lines def setUp(self):
'type': 'local', 'type': 'local',
'context': 'NpmMapping' 'context': 'NpmMapping'
} }
} }
MockIndexerStorage.added_data = [] MockIndexerStorage.added_data = []
def test_crosstable(self): def test_crosstable(self):
self.assertEqual(CROSSWALK_TABLE['NodeJS'], { self.assertEqual(CROSSWALK_TABLE['NodeJS'], {
'repository': 'codeRepository', 'repository': 'https://codemeta.github.io/terms/codeRepository',
'os': 'operatingSystem', 'os': 'https://codemeta.github.io/terms/operatingSystem',
'cpu': 'processorRequirements', 'cpu': 'https://codemeta.github.io/terms/processorRequirements',
'engines': 'processorRequirements', 'engines':
'dependencies': 'softwareRequirements', 'https://codemeta.github.io/terms/processorRequirements',
'bundleDependencies': 'softwareRequirements', 'author': 'https://codemeta.github.io/terms/creator',
'bundledDependencies': 'softwareRequirements', 'author.email': 'https://codemeta.github.io/terms/email',
'peerDependencies': 'softwareRequirements', 'author.name': 'https://codemeta.github.io/terms/name',
'author': 'creator', 'contributor': 'https://codemeta.github.io/terms/contributor',
'author.email': 'email', 'keywords': 'https://codemeta.github.io/terms/keywords',
'author.name': 'name', 'license': 'https://codemeta.github.io/terms/license',
'contributor': 'contributor', 'version': 'https://codemeta.github.io/terms/version',
'keywords': 'keywords', 'description': 'https://codemeta.github.io/terms/description',
'license': 'license', 'name': 'https://codemeta.github.io/terms/name',
'version': 'version', 'bugs': 'https://codemeta.github.io/terms/issueTracker',
'description': 'description', 'homepage': 'https://codemeta.github.io/terms/url'
'name': 'name',
'devDependencies': 'softwareSuggestions',
'optionalDependencies': 'softwareSuggestions',
'bugs': 'issueTracker',
'homepage': 'url'
}) })
def test_compute_metadata_none(self): def test_compute_metadata_none(self):
""" """
testing content empty content is empty testing content empty content is empty
should return None should return None
""" """
# given # given
Show All 18 Lines def test_compute_metadata_npm(self):
"description": "Simple package.json test for indexer", "description": "Simple package.json test for indexer",
"repository": { "repository": {
"type": "git", "type": "git",
"url": "https://github.com/moranegg/metadata_test" "url": "https://github.com/moranegg/metadata_test"
} }
} }
""" """
declared_metadata = { declared_metadata = {
'name': 'test_metadata', '@context': 'https://doi.org/10.5063/schema/codemeta-2.0',
'version': '0.0.2', 'codemeta:name': 'test_metadata',
'description': 'Simple package.json test for indexer', 'codemeta:version': '0.0.2',
'codeRepository': { 'codemeta:description': 'Simple package.json test for indexer',
'type': 'git', 'codemeta:codeRepository':
'url': 'https://github.com/moranegg/metadata_test' 'git+https://github.com/moranegg/metadata_test',
},
'other': {}
} }
# when # when
result = MAPPINGS["NpmMapping"].translate(content) result = MAPPINGS["NpmMapping"].translate(content)
# then # then
self.assertEqual(declared_metadata, result) self.assertEqual(declared_metadata, result)
def test_extract_minimal_metadata_dict(self): def test_extract_minimal_metadata_dict(self):
""" """
Test the creation of a coherent minimal metadata set Test the creation of a coherent minimal metadata set
""" """
# given # given
metadata_list = [{ metadata_list = [{
'name': 'test_1', '@context': 'https://doi.org/10.5063/schema/codemeta-2.0',
'version': '0.0.2', 'codemeta:name': 'test_1',
'description': 'Simple package.json test for indexer', 'codemeta:version': '0.0.2',
'codeRepository': { 'codemeta:description': 'Simple package.json test for indexer',
'type': 'git', 'codemeta:codeRepository':
'url': 'https://github.com/moranegg/metadata_test' 'git+https://github.com/moranegg/metadata_test',
},
'other': {}
}, { }, {
'name': 'test_0_1', '@context': 'https://doi.org/10.5063/schema/codemeta-2.0',
'version': '0.0.2', 'codemeta:name': 'test_0_1',
'description': 'Simple package.json test for indexer', 'codemeta:version': '0.0.2',
'codeRepository': { 'codemeta:description': 'Simple package.json test for indexer',
'type': 'git', 'codemeta:codeRepository':
'url': 'https://github.com/moranegg/metadata_test' 'git+https://github.com/moranegg/metadata_test'
},
'other': {}
}, { }, {
'name': 'test_metadata', '@context': 'https://doi.org/10.5063/schema/codemeta-2.0',
'version': '0.0.2', 'codemeta:name': 'test_metadata',
'author': 'moranegg', 'codemeta:version': '0.0.2',
'other': {} 'codemeta:author': 'moranegg',
}] }]
# when # when
results = extract_minimal_metadata_dict(metadata_list) results = extract_minimal_metadata_dict(metadata_list)
# then # then
expected_results = { expected_results = {
"developmentStatus": None, '@context': 'https://doi.org/10.5063/schema/codemeta-2.0',
"version": ['0.0.2'], "codemeta:version": '0.0.2',
"operatingSystem": None, "codemeta:description": 'Simple package.json test for indexer',
"description": ['Simple package.json test for indexer'], "codemeta:name": ['test_1', 'test_0_1', 'test_metadata'],
"keywords": None, "codemeta:author": 'moranegg',
"issueTracker": None, "codemeta:codeRepository":
"name": ['test_1', 'test_0_1', 'test_metadata'], 'git+https://github.com/moranegg/metadata_test',
"author": ['moranegg'],
"relatedLink": None,
"url": None,
"license": None,
"maintainer": None,
"email": None,
"softwareRequirements": None,
"identifier": None,
"codeRepository": [{
'type': 'git',
'url': 'https://github.com/moranegg/metadata_test'
}]
} }
self.assertEqual(expected_results, results) self.assertEqual(expected_results, results)
def test_index_content_metadata_npm(self): def test_index_content_metadata_npm(self):
""" """
testing NPM with package.json testing NPM with package.json
- one sha1 uses a file that can't be translated to metadata and - one sha1 uses a file that can't be translated to metadata and
should return None in the translated metadata should return None in the translated metadata
Show All 9 Lines def test_index_content_metadata_npm(self):
# when # when
metadata_indexer.run(sha1s, policy_update='ignore-dups') metadata_indexer.run(sha1s, policy_update='ignore-dups')
results = metadata_indexer.idx_storage.added_data results = metadata_indexer.idx_storage.added_data
expected_results = [('content_metadata', False, [{ expected_results = [('content_metadata', False, [{
'indexer_configuration_id': 30, 'indexer_configuration_id': 30,
'translated_metadata': { 'translated_metadata': {
'other': {}, '@context': 'https://doi.org/10.5063/schema/codemeta-2.0',
'codeRepository': { 'codemeta:codeRepository':
'type': 'git', 'git+https://github.com/moranegg/metadata_test',
'url': 'https://github.com/moranegg/metadata_test' 'codemeta:description': 'Simple package.json test for indexer',
}, 'codemeta:name': 'test_metadata',
'description': 'Simple package.json test for indexer', 'codemeta:version': '0.0.1'
'name': 'test_metadata',
'version': '0.0.1'
}, },
'id': '26a9f72a7c87cc9205725cfd879f514ff4f3d8d5' 'id': '26a9f72a7c87cc9205725cfd879f514ff4f3d8d5'
}, { }, {
'indexer_configuration_id': 30, 'indexer_configuration_id': 30,
'translated_metadata': { 'translated_metadata': {
'softwareRequirements': { '@context': 'https://doi.org/10.5063/schema/codemeta-2.0',
'JSONStream': '~1.3.1', 'codemeta:issueTracker':
'abbrev': '~1.1.0', 'https://github.com/npm/npm/issues',
'ansi-regex': '~2.1.1', 'codemeta:creator':
'ansicolors': '~0.3.2',
'ansistyles': '~0.1.3'
},
'issueTracker': {
'url': 'https://github.com/npm/npm/issues'
},
'creator':
'Isaac Z. Schlueter <i@izs.me> (http://blog.izs.me)', 'Isaac Z. Schlueter <i@izs.me> (http://blog.izs.me)',
'codeRepository': { 'codemeta:codeRepository':
'type': 'git', 'git+https://github.com/npm/npm',
'url': 'https://github.com/npm/npm' 'codemeta:description': 'a package manager for JavaScript',
}, 'codemeta:license': 'Artistic-2.0',
'description': 'a package manager for JavaScript', 'codemeta:version': '5.0.3',
'softwareSuggestions': { 'codemeta:name': 'npm',
'tacks': '~1.2.6', 'codemeta:keywords': [
'tap': '~10.3.2'
},
'license': 'Artistic-2.0',
'version': '5.0.3',
'other': {
'preferGlobal': True,
'config': {
'publishtest': False
}
},
'name': 'npm',
'keywords': [
'install', 'install',
'modules', 'modules',
'package manager', 'package manager',
'package.json' 'package.json'
], ],
'url': 'https://docs.npmjs.com/' 'codemeta:url': 'https://docs.npmjs.com/'
}, },
'id': 'd4c647f0fc257591cc9ba1722484229780d1c607' 'id': 'd4c647f0fc257591cc9ba1722484229780d1c607'
}, { }, {
'indexer_configuration_id': 30, 'indexer_configuration_id': 30,
'translated_metadata': None, 'translated_metadata': None,
'id': '02fb2c89e14f7fab46701478c83779c7beb7b069' 'id': '02fb2c89e14f7fab46701478c83779c7beb7b069'
}])] }])]
▲ Show 20 Lines • Show All 43 Lines • ▼ Show 20 Lines def test_revision_metadata_indexer(self):
] ]
metadata_indexer.run(sha1_gits, 'update-dups') metadata_indexer.run(sha1_gits, 'update-dups')
results = metadata_indexer.idx_storage.added_data results = metadata_indexer.idx_storage.added_data
expected_results = [('revision_metadata', True, [{ expected_results = [('revision_metadata', True, [{
'id': '8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f', 'id': '8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f',
'translated_metadata': { 'translated_metadata': {
'identifier': None, '@context': 'https://doi.org/10.5063/schema/codemeta-2.0',
'maintainer': None, 'codemeta:url':
'url': [ 'https://github.com/librariesio/yarn-parser#readme',
'https://github.com/librariesio/yarn-parser#readme' 'codemeta:codeRepository':
], 'git+https://github.com/librariesio/yarn-parser.git',
'codeRepository': [{ 'codemeta:author': 'Andrew Nesbitt',
'type': 'git', 'codemeta:license': 'AGPL-3.0',
'url': 'git+https://github.com/librariesio/yarn-parser.git' 'codemeta:version': '1.0.0',
}], 'codemeta:description':
'author': ['Andrew Nesbitt'], 'Tiny web service for parsing yarn.lock files',
'license': ['AGPL-3.0'], 'codemeta:issueTracker':
'version': ['1.0.0'], 'https://github.com/librariesio/yarn-parser/issues',
'description': [ 'codemeta:name': 'yarn-parser',
'Tiny web service for parsing yarn.lock files' 'codemeta:keywords': ['yarn', 'parse', 'lock', 'dependencies'],
],
'relatedLink': None,
'developmentStatus': None,
'operatingSystem': None,
'issueTracker': [{
'url': 'https://github.com/librariesio/yarn-parser/issues'
}],
'softwareRequirements': [{
'express': '^4.14.0',
'yarn': '^0.21.0',
'body-parser': '^1.15.2'
}],
'name': ['yarn-parser'],
'keywords': [['yarn', 'parse', 'lock', 'dependencies']],
'email': None
}, },
'indexer_configuration_id': 7 'indexer_configuration_id': 7
}])] }])]
# then # then
self.assertEqual(expected_results, results) self.assertEqual(expected_results, results)
About · Developer information · Legal