Changeset View
Changeset View
Standalone View
Standalone View
swh/indexer/tests/test_origin_metadata.py
| Show All 11 Lines | |||||
| from .utils import YARN_PARSER_METADATA | from .utils import YARN_PARSER_METADATA | ||||
| from .test_metadata import REVISION_METADATA_CONFIG | from .test_metadata import REVISION_METADATA_CONFIG | ||||
| def test_origin_metadata_indexer( | def test_origin_metadata_indexer( | ||||
| idx_storage, storage, obj_storage): | idx_storage, storage, obj_storage): | ||||
| indexer = OriginMetadataIndexer(config=REVISION_METADATA_CONFIG) | indexer = OriginMetadataIndexer(config=REVISION_METADATA_CONFIG) | ||||
| indexer.run(["git+https://github.com/librariesio/yarn-parser"]) | indexer.run(["https://github.com/librariesio/yarn-parser"]) | ||||
| origin = storage.origin_get({ | origin = storage.origin_get({ | ||||
| 'type': 'git', | |||||
| 'url': 'https://github.com/librariesio/yarn-parser'}) | 'url': 'https://github.com/librariesio/yarn-parser'}) | ||||
| rev_id = hash_to_bytes('8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f') | rev_id = hash_to_bytes('8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f') | ||||
| rev_metadata = { | rev_metadata = { | ||||
| 'id': rev_id, | 'id': rev_id, | ||||
| 'metadata': YARN_PARSER_METADATA, | 'metadata': YARN_PARSER_METADATA, | ||||
| 'mappings': ['npm'], | 'mappings': ['npm'], | ||||
| } | } | ||||
| origin_metadata = { | origin_metadata = { | ||||
| 'id': origin['id'], | 'id': origin['id'], | ||||
| 'origin_url': origin['url'], | |||||
| 'from_revision': rev_id, | 'from_revision': rev_id, | ||||
| 'metadata': YARN_PARSER_METADATA, | 'metadata': YARN_PARSER_METADATA, | ||||
| 'mappings': ['npm'], | 'mappings': ['npm'], | ||||
| } | } | ||||
| results = list( | results = list( | ||||
| indexer.idx_storage.revision_intrinsic_metadata_get([rev_id])) | indexer.idx_storage.revision_intrinsic_metadata_get([rev_id])) | ||||
| for result in results: | for result in results: | ||||
| del result['tool'] | del result['tool'] | ||||
| assert results == [rev_metadata] | assert results == [rev_metadata] | ||||
| results = list(indexer.idx_storage.origin_intrinsic_metadata_get([ | results = list(indexer.idx_storage.origin_intrinsic_metadata_get([ | ||||
| origin['id']])) | origin['id']])) | ||||
| for result in results: | for result in results: | ||||
| del result['tool'] | del result['tool'] | ||||
| assert results == [origin_metadata] | assert results == [origin_metadata] | ||||
| def test_origin_metadata_indexer_duplicate_origin( | def test_origin_metadata_indexer_duplicate_origin( | ||||
| idx_storage, storage, obj_storage): | idx_storage, storage, obj_storage): | ||||
| indexer = OriginMetadataIndexer(config=REVISION_METADATA_CONFIG) | indexer = OriginMetadataIndexer(config=REVISION_METADATA_CONFIG) | ||||
| indexer.storage = storage | indexer.storage = storage | ||||
| indexer.idx_storage = idx_storage | indexer.idx_storage = idx_storage | ||||
| indexer.run(["git+https://github.com/librariesio/yarn-parser"]) | indexer.run(["https://github.com/librariesio/yarn-parser"]) | ||||
| indexer.run(["git+https://github.com/librariesio/yarn-parser"]*2) | indexer.run(["https://github.com/librariesio/yarn-parser"]*2) | ||||
| origin = storage.origin_get({ | origin = storage.origin_get({ | ||||
| 'type': 'git', | |||||
| 'url': 'https://github.com/librariesio/yarn-parser'}) | 'url': 'https://github.com/librariesio/yarn-parser'}) | ||||
| rev_id = hash_to_bytes('8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f') | rev_id = hash_to_bytes('8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f') | ||||
| results = list( | results = list( | ||||
| indexer.idx_storage.revision_intrinsic_metadata_get([rev_id])) | indexer.idx_storage.revision_intrinsic_metadata_get([rev_id])) | ||||
| assert len(results) == 1 | assert len(results) == 1 | ||||
| results = list(indexer.idx_storage.origin_intrinsic_metadata_get([ | results = list(indexer.idx_storage.origin_intrinsic_metadata_get([ | ||||
| origin['id']])) | origin['id']])) | ||||
| assert len(results) == 1 | assert len(results) == 1 | ||||
| def test_origin_metadata_indexer_missing_head( | def test_origin_metadata_indexer_missing_head( | ||||
| idx_storage, storage, obj_storage): | idx_storage, storage, obj_storage): | ||||
| storage.origin_add([{ | storage.origin_add([{ | ||||
| 'type': 'git', | 'type': 'git', | ||||
| 'url': 'https://example.com' | 'url': 'https://example.com' | ||||
| }]) | }]) | ||||
| indexer = OriginMetadataIndexer(config=REVISION_METADATA_CONFIG) | indexer = OriginMetadataIndexer(config=REVISION_METADATA_CONFIG) | ||||
| indexer.run(["git+https://example.com"]) | indexer.run(["https://example.com"]) | ||||
| origin = storage.origin_get({ | origin = storage.origin_get({ | ||||
| 'type': 'git', | |||||
| 'url': 'https://example.com'}) | 'url': 'https://example.com'}) | ||||
| results = list(indexer.idx_storage.origin_intrinsic_metadata_get([ | results = list(indexer.idx_storage.origin_intrinsic_metadata_get([ | ||||
| origin['id']])) | origin['id']])) | ||||
| assert results == [] | assert results == [] | ||||
| def test_origin_metadata_indexer_partial_missing_head( | def test_origin_metadata_indexer_partial_missing_head( | ||||
| idx_storage, storage, obj_storage): | idx_storage, storage, obj_storage): | ||||
| storage.origin_add([{ | storage.origin_add([{ | ||||
| 'type': 'git', | 'type': 'git', | ||||
| 'url': 'https://example.com' | 'url': 'https://example.com' | ||||
| }]) | }]) | ||||
| indexer = OriginMetadataIndexer(config=REVISION_METADATA_CONFIG) | indexer = OriginMetadataIndexer(config=REVISION_METADATA_CONFIG) | ||||
| indexer.run(["git+https://example.com", | indexer.run(["https://example.com", | ||||
| "git+https://github.com/librariesio/yarn-parser"]) | "https://github.com/librariesio/yarn-parser"]) | ||||
| origin1 = storage.origin_get({ | origin1 = storage.origin_get({ | ||||
| 'type': 'git', | |||||
| 'url': 'https://example.com'}) | 'url': 'https://example.com'}) | ||||
| origin2 = storage.origin_get({ | origin2 = storage.origin_get({ | ||||
| 'type': 'git', | |||||
| 'url': 'https://github.com/librariesio/yarn-parser'}) | 'url': 'https://github.com/librariesio/yarn-parser'}) | ||||
| rev_id = hash_to_bytes('8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f') | rev_id = hash_to_bytes('8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f') | ||||
| rev_metadata = { | rev_metadata = { | ||||
| 'id': rev_id, | 'id': rev_id, | ||||
| 'metadata': YARN_PARSER_METADATA, | 'metadata': YARN_PARSER_METADATA, | ||||
| 'mappings': ['npm'], | 'mappings': ['npm'], | ||||
| } | } | ||||
| origin_metadata = { | origin_metadata = { | ||||
| 'id': origin2['id'], | 'id': origin2['id'], | ||||
| 'origin_url': origin2['url'], | |||||
| 'from_revision': rev_id, | 'from_revision': rev_id, | ||||
| 'metadata': YARN_PARSER_METADATA, | 'metadata': YARN_PARSER_METADATA, | ||||
| 'mappings': ['npm'], | 'mappings': ['npm'], | ||||
| } | } | ||||
| results = list( | results = list( | ||||
| indexer.idx_storage.revision_intrinsic_metadata_get([rev_id])) | indexer.idx_storage.revision_intrinsic_metadata_get([rev_id])) | ||||
| for result in results: | for result in results: | ||||
| del result['tool'] | del result['tool'] | ||||
| assert results == [rev_metadata] | assert results == [rev_metadata] | ||||
| results = list(indexer.idx_storage.origin_intrinsic_metadata_get([ | results = list(indexer.idx_storage.origin_intrinsic_metadata_get([ | ||||
| origin1['id'], origin2['id']])) | origin1['id'], origin2['id']])) | ||||
| for result in results: | for result in results: | ||||
| del result['tool'] | del result['tool'] | ||||
| assert results == [origin_metadata] | assert results == [origin_metadata] | ||||
| def test_origin_metadata_indexer_duplicate_revision( | def test_origin_metadata_indexer_duplicate_revision( | ||||
| idx_storage, storage, obj_storage): | idx_storage, storage, obj_storage): | ||||
| indexer = OriginMetadataIndexer(config=REVISION_METADATA_CONFIG) | indexer = OriginMetadataIndexer(config=REVISION_METADATA_CONFIG) | ||||
| indexer.storage = storage | indexer.storage = storage | ||||
| indexer.idx_storage = idx_storage | indexer.idx_storage = idx_storage | ||||
| indexer.run(["git+https://github.com/librariesio/yarn-parser", | indexer.run(["https://github.com/librariesio/yarn-parser", | ||||
| "git+https://github.com/librariesio/yarn-parser.git"]) | "https://github.com/librariesio/yarn-parser.git"]) | ||||
| origin1 = storage.origin_get({ | origin1 = storage.origin_get({ | ||||
| 'type': 'git', | |||||
| 'url': 'https://github.com/librariesio/yarn-parser'}) | 'url': 'https://github.com/librariesio/yarn-parser'}) | ||||
| origin2 = storage.origin_get({ | origin2 = storage.origin_get({ | ||||
| 'type': 'git', | |||||
| 'url': 'https://github.com/librariesio/yarn-parser.git'}) | 'url': 'https://github.com/librariesio/yarn-parser.git'}) | ||||
| assert origin1['id'] != origin2['id'] | assert origin1['id'] != origin2['id'] | ||||
| rev_id = hash_to_bytes('8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f') | rev_id = hash_to_bytes('8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f') | ||||
| results = list( | results = list( | ||||
| indexer.idx_storage.revision_intrinsic_metadata_get([rev_id])) | indexer.idx_storage.revision_intrinsic_metadata_get([rev_id])) | ||||
| assert len(results) == 1 | assert len(results) == 1 | ||||
| results = list(indexer.idx_storage.origin_intrinsic_metadata_get([ | results = list(indexer.idx_storage.origin_intrinsic_metadata_get([ | ||||
| origin1['id'], origin2['id']])) | origin1['id'], origin2['id']])) | ||||
| assert len(results) == 2 | assert len(results) == 2 | ||||
| def test_origin_metadata_indexer_no_metadata_file( | def test_origin_metadata_indexer_no_metadata_file( | ||||
| idx_storage, storage, obj_storage): | idx_storage, storage, obj_storage): | ||||
| indexer = OriginMetadataIndexer(config=REVISION_METADATA_CONFIG) | indexer = OriginMetadataIndexer(config=REVISION_METADATA_CONFIG) | ||||
| with patch('swh.indexer.metadata_dictionary.npm.NpmMapping.filename', | with patch('swh.indexer.metadata_dictionary.npm.NpmMapping.filename', | ||||
| b'foo.json'): | b'foo.json'): | ||||
| indexer.run(["git+https://github.com/librariesio/yarn-parser"]) | indexer.run(["https://github.com/librariesio/yarn-parser"]) | ||||
| origin = storage.origin_get({ | origin = storage.origin_get({ | ||||
| 'type': 'git', | |||||
| 'url': 'https://github.com/librariesio/yarn-parser'}) | 'url': 'https://github.com/librariesio/yarn-parser'}) | ||||
| rev_id = hash_to_bytes('8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f') | rev_id = hash_to_bytes('8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f') | ||||
| results = list( | results = list( | ||||
| indexer.idx_storage.revision_intrinsic_metadata_get([rev_id])) | indexer.idx_storage.revision_intrinsic_metadata_get([rev_id])) | ||||
| assert results == [] | assert results == [] | ||||
| results = list(indexer.idx_storage.origin_intrinsic_metadata_get([ | results = list(indexer.idx_storage.origin_intrinsic_metadata_get([ | ||||
| origin['id']])) | origin['id']])) | ||||
| assert results == [] | assert results == [] | ||||
| def test_origin_metadata_indexer_no_metadata( | def test_origin_metadata_indexer_no_metadata( | ||||
| idx_storage, storage, obj_storage): | idx_storage, storage, obj_storage): | ||||
| indexer = OriginMetadataIndexer(config=REVISION_METADATA_CONFIG) | indexer = OriginMetadataIndexer(config=REVISION_METADATA_CONFIG) | ||||
| with patch('swh.indexer.metadata.RevisionMetadataIndexer' | with patch('swh.indexer.metadata.RevisionMetadataIndexer' | ||||
| '.translate_revision_intrinsic_metadata', | '.translate_revision_intrinsic_metadata', | ||||
| return_value=(['npm'], {'@context': 'foo'})): | return_value=(['npm'], {'@context': 'foo'})): | ||||
| indexer.run(["git+https://github.com/librariesio/yarn-parser"]) | indexer.run(["https://github.com/librariesio/yarn-parser"]) | ||||
| origin = storage.origin_get({ | origin = storage.origin_get({ | ||||
| 'type': 'git', | |||||
| 'url': 'https://github.com/librariesio/yarn-parser'}) | 'url': 'https://github.com/librariesio/yarn-parser'}) | ||||
| rev_id = hash_to_bytes('8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f') | rev_id = hash_to_bytes('8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f') | ||||
| results = list( | results = list( | ||||
| indexer.idx_storage.revision_intrinsic_metadata_get([rev_id])) | indexer.idx_storage.revision_intrinsic_metadata_get([rev_id])) | ||||
| assert results == [] | assert results == [] | ||||
| results = list(indexer.idx_storage.origin_intrinsic_metadata_get([ | results = list(indexer.idx_storage.origin_intrinsic_metadata_get([ | ||||
| origin['id']])) | origin['id']])) | ||||
| assert results == [] | assert results == [] | ||||
| def test_origin_metadata_indexer_error( | def test_origin_metadata_indexer_error( | ||||
| idx_storage, storage, obj_storage): | idx_storage, storage, obj_storage): | ||||
| indexer = OriginMetadataIndexer(config=REVISION_METADATA_CONFIG) | indexer = OriginMetadataIndexer(config=REVISION_METADATA_CONFIG) | ||||
| with patch('swh.indexer.metadata.RevisionMetadataIndexer' | with patch('swh.indexer.metadata.RevisionMetadataIndexer' | ||||
| '.translate_revision_intrinsic_metadata', | '.translate_revision_intrinsic_metadata', | ||||
| return_value=None): | return_value=None): | ||||
| indexer.run(["git+https://github.com/librariesio/yarn-parser"]) | indexer.run(["https://github.com/librariesio/yarn-parser"]) | ||||
| origin = storage.origin_get({ | origin = storage.origin_get({ | ||||
| 'type': 'git', | |||||
| 'url': 'https://github.com/librariesio/yarn-parser'}) | 'url': 'https://github.com/librariesio/yarn-parser'}) | ||||
| rev_id = hash_to_bytes('8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f') | rev_id = hash_to_bytes('8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f') | ||||
| results = list( | results = list( | ||||
| indexer.idx_storage.revision_intrinsic_metadata_get([rev_id])) | indexer.idx_storage.revision_intrinsic_metadata_get([rev_id])) | ||||
| assert results == [] | assert results == [] | ||||
| results = list(indexer.idx_storage.origin_intrinsic_metadata_get([ | results = list(indexer.idx_storage.origin_intrinsic_metadata_get([ | ||||
| origin['id']])) | origin['id']])) | ||||
| assert results == [] | assert results == [] | ||||
| def test_origin_metadata_indexer_delete_metadata( | def test_origin_metadata_indexer_delete_metadata( | ||||
| idx_storage, storage, obj_storage): | idx_storage, storage, obj_storage): | ||||
| indexer = OriginMetadataIndexer(config=REVISION_METADATA_CONFIG) | indexer = OriginMetadataIndexer(config=REVISION_METADATA_CONFIG) | ||||
| indexer.run(["git+https://github.com/librariesio/yarn-parser"]) | indexer.run(["https://github.com/librariesio/yarn-parser"]) | ||||
| origin = storage.origin_get({ | origin = storage.origin_get({ | ||||
| 'type': 'git', | |||||
| 'url': 'https://github.com/librariesio/yarn-parser'}) | 'url': 'https://github.com/librariesio/yarn-parser'}) | ||||
| rev_id = hash_to_bytes('8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f') | rev_id = hash_to_bytes('8dbb6aeb036e7fd80664eb8bfd1507881af1ba9f') | ||||
| results = list( | results = list( | ||||
| indexer.idx_storage.revision_intrinsic_metadata_get([rev_id])) | indexer.idx_storage.revision_intrinsic_metadata_get([rev_id])) | ||||
| assert results != [] | assert results != [] | ||||
| results = list(indexer.idx_storage.origin_intrinsic_metadata_get([ | results = list(indexer.idx_storage.origin_intrinsic_metadata_get([ | ||||
| origin['id']])) | origin['id']])) | ||||
| assert results != [] | assert results != [] | ||||
| with patch('swh.indexer.metadata_dictionary.npm.NpmMapping.filename', | with patch('swh.indexer.metadata_dictionary.npm.NpmMapping.filename', | ||||
| b'foo.json'): | b'foo.json'): | ||||
| indexer.run(["git+https://github.com/librariesio/yarn-parser"]) | indexer.run(["https://github.com/librariesio/yarn-parser"]) | ||||
| results = list( | results = list( | ||||
| indexer.idx_storage.revision_intrinsic_metadata_get([rev_id])) | indexer.idx_storage.revision_intrinsic_metadata_get([rev_id])) | ||||
| assert results == [] | assert results == [] | ||||
| results = list(indexer.idx_storage.origin_intrinsic_metadata_get([ | results = list(indexer.idx_storage.origin_intrinsic_metadata_get([ | ||||
| origin['id']])) | origin['id']])) | ||||
| assert results == [] | assert results == [] | ||||