Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/tests/test_storage.py
| Show First 20 Lines • Show All 145 Lines • ▼ Show 20 Lines | def test_content_add(self, swh_storage): | ||||
| 'content:add:bytes': cont['length'], | 'content:add:bytes': cont['length'], | ||||
| } | } | ||||
| assert list(swh_storage.content_get([cont['sha1']])) == \ | assert list(swh_storage.content_get([cont['sha1']])) == \ | ||||
| [{'sha1': cont['sha1'], 'data': cont['data']}] | [{'sha1': cont['sha1'], 'data': cont['data']}] | ||||
| expected_cont = data.cont | expected_cont = data.cont | ||||
| del expected_cont['data'] | del expected_cont['data'] | ||||
| journal_objects = list(swh_storage.journal_writer.objects) | journal_objects = list(swh_storage.journal_writer.journal.objects) | ||||
| for (obj_type, obj) in journal_objects: | for (obj_type, obj) in journal_objects: | ||||
| assert insertion_start_time <= obj['ctime'] | assert insertion_start_time <= obj['ctime'] | ||||
| assert obj['ctime'] <= insertion_end_time | assert obj['ctime'] <= insertion_end_time | ||||
| del obj['ctime'] | del obj['ctime'] | ||||
| assert journal_objects == [('content', expected_cont)] | assert journal_objects == [('content', expected_cont)] | ||||
| swh_storage.refresh_stat_counters() | swh_storage.refresh_stat_counters() | ||||
| assert swh_storage.stat_counters()['content'] == 1 | assert swh_storage.stat_counters()['content'] == 1 | ||||
| ▲ Show 20 Lines • Show All 73 Lines • ▼ Show 20 Lines | def test_content_add_different_input(self, swh_storage): | ||||
| } | } | ||||
| def test_content_add_twice(self, swh_storage): | def test_content_add_twice(self, swh_storage): | ||||
| actual_result = swh_storage.content_add([data.cont]) | actual_result = swh_storage.content_add([data.cont]) | ||||
| assert actual_result == { | assert actual_result == { | ||||
| 'content:add': 1, | 'content:add': 1, | ||||
| 'content:add:bytes': data.cont['length'], | 'content:add:bytes': data.cont['length'], | ||||
| } | } | ||||
| assert len(swh_storage.journal_writer.objects) == 1 | assert len(swh_storage.journal_writer.journal.objects) == 1 | ||||
| actual_result = swh_storage.content_add([data.cont, data.cont2]) | actual_result = swh_storage.content_add([data.cont, data.cont2]) | ||||
| assert actual_result == { | assert actual_result == { | ||||
| 'content:add': 1, | 'content:add': 1, | ||||
| 'content:add:bytes': data.cont2['length'], | 'content:add:bytes': data.cont2['length'], | ||||
| } | } | ||||
| assert 2 <= len(swh_storage.journal_writer.objects) <= 3 | assert 2 <= len(swh_storage.journal_writer.journal.objects) <= 3 | ||||
| assert len(swh_storage.content_find(data.cont)) == 1 | assert len(swh_storage.content_find(data.cont)) == 1 | ||||
| assert len(swh_storage.content_find(data.cont2)) == 1 | assert len(swh_storage.content_find(data.cont2)) == 1 | ||||
| def test_content_add_collision(self, swh_storage): | def test_content_add_collision(self, swh_storage): | ||||
| cont1 = data.cont | cont1 = data.cont | ||||
| # create (corrupted) content with same sha1{,_git} but != sha256 | # create (corrupted) content with same sha1{,_git} but != sha256 | ||||
| cont1b = cont1.copy() | cont1b = cont1.copy() | ||||
| sha256_array = bytearray(cont1b['sha256']) | sha256_array = bytearray(cont1b['sha256']) | ||||
| sha256_array[0] += 1 | sha256_array[0] += 1 | ||||
| cont1b['sha256'] = bytes(sha256_array) | cont1b['sha256'] = bytes(sha256_array) | ||||
| with pytest.raises(HashCollision) as cm: | with pytest.raises(HashCollision) as cm: | ||||
| swh_storage.content_add([cont1, cont1b]) | swh_storage.content_add([cont1, cont1b]) | ||||
| assert cm.value.args[0] in ['sha1', 'sha1_git', 'blake2s256'] | assert cm.value.args[0] in ['sha1', 'sha1_git', 'blake2s256'] | ||||
| def test_content_update(self, swh_storage): | def test_content_update(self, swh_storage): | ||||
| if hasattr(swh_storage, 'storage'): | if hasattr(swh_storage, 'storage'): | ||||
| swh_storage.storage.journal_writer = None # TODO, not supported | swh_storage.journal_writer.journal = None # TODO, not supported | ||||
| cont = copy.deepcopy(data.cont) | cont = copy.deepcopy(data.cont) | ||||
| swh_storage.content_add([cont]) | swh_storage.content_add([cont]) | ||||
| # alter the sha1_git for example | # alter the sha1_git for example | ||||
| cont['sha1_git'] = hash_to_bytes( | cont['sha1_git'] = hash_to_bytes( | ||||
| '3a60a5275d0333bf13468e8b3dcab90f4046e654') | '3a60a5275d0333bf13468e8b3dcab90f4046e654') | ||||
| Show All 14 Lines | def test_content_add_metadata(self, swh_storage): | ||||
| } | } | ||||
| expected_cont = cont.copy() | expected_cont = cont.copy() | ||||
| del expected_cont['ctime'] | del expected_cont['ctime'] | ||||
| assert swh_storage.content_get_metadata([cont['sha1']]) == { | assert swh_storage.content_get_metadata([cont['sha1']]) == { | ||||
| cont['sha1']: [expected_cont] | cont['sha1']: [expected_cont] | ||||
| } | } | ||||
| assert list(swh_storage.journal_writer.objects) == [('content', cont)] | assert list(swh_storage.journal_writer.journal.objects) == [ | ||||
| ('content', cont)] | |||||
| def test_content_add_metadata_different_input(self, swh_storage): | def test_content_add_metadata_different_input(self, swh_storage): | ||||
| cont = data.cont | cont = data.cont | ||||
| del cont['data'] | del cont['data'] | ||||
| cont['ctime'] = datetime.datetime.now() | cont['ctime'] = datetime.datetime.now() | ||||
| cont2 = data.cont2 | cont2 = data.cont2 | ||||
| del cont2['data'] | del cont2['data'] | ||||
| cont2['ctime'] = datetime.datetime.now() | cont2['ctime'] = datetime.datetime.now() | ||||
| ▲ Show 20 Lines • Show All 234 Lines • ▼ Show 20 Lines | class TestStorage: | ||||
| def test_directory_add(self, swh_storage): | def test_directory_add(self, swh_storage): | ||||
| init_missing = list(swh_storage.directory_missing([data.dir['id']])) | init_missing = list(swh_storage.directory_missing([data.dir['id']])) | ||||
| assert [data.dir['id']] == init_missing | assert [data.dir['id']] == init_missing | ||||
| actual_result = swh_storage.directory_add([data.dir]) | actual_result = swh_storage.directory_add([data.dir]) | ||||
| assert actual_result == {'directory:add': 1} | assert actual_result == {'directory:add': 1} | ||||
| assert list(swh_storage.journal_writer.objects) == \ | assert list(swh_storage.journal_writer.journal.objects) == \ | ||||
| [('directory', data.dir)] | [('directory', data.dir)] | ||||
| actual_data = list(swh_storage.directory_ls(data.dir['id'])) | actual_data = list(swh_storage.directory_ls(data.dir['id'])) | ||||
| expected_data = list(transform_entries(data.dir)) | expected_data = list(transform_entries(data.dir)) | ||||
| assert sorted(expected_data, key=cmpdir) \ | assert sorted(expected_data, key=cmpdir) \ | ||||
| == sorted(actual_data, key=cmpdir) | == sorted(actual_data, key=cmpdir) | ||||
| after_missing = list(swh_storage.directory_missing([data.dir['id']])) | after_missing = list(swh_storage.directory_missing([data.dir['id']])) | ||||
| assert after_missing == [] | assert after_missing == [] | ||||
| swh_storage.refresh_stat_counters() | swh_storage.refresh_stat_counters() | ||||
| assert swh_storage.stat_counters()['directory'] == 1 | assert swh_storage.stat_counters()['directory'] == 1 | ||||
| def test_directory_add_from_generator(self, swh_storage): | def test_directory_add_from_generator(self, swh_storage): | ||||
| def _dir_gen(): | def _dir_gen(): | ||||
| yield data.dir | yield data.dir | ||||
| actual_result = swh_storage.directory_add(directories=_dir_gen()) | actual_result = swh_storage.directory_add(directories=_dir_gen()) | ||||
| assert actual_result == {'directory:add': 1} | assert actual_result == {'directory:add': 1} | ||||
| assert list(swh_storage.journal_writer.objects) == \ | assert list(swh_storage.journal_writer.journal.objects) == \ | ||||
| [('directory', data.dir)] | [('directory', data.dir)] | ||||
| swh_storage.refresh_stat_counters() | swh_storage.refresh_stat_counters() | ||||
| assert swh_storage.stat_counters()['directory'] == 1 | assert swh_storage.stat_counters()['directory'] == 1 | ||||
| def test_directory_add_validation(self, swh_storage): | def test_directory_add_validation(self, swh_storage): | ||||
| dir_ = copy.deepcopy(data.dir) | dir_ = copy.deepcopy(data.dir) | ||||
| dir_['entries'][0]['type'] = 'foobar' | dir_['entries'][0]['type'] = 'foobar' | ||||
| Show All 9 Lines | def test_directory_add_validation(self, swh_storage): | ||||
| if type(cm.value) == psycopg2.IntegrityError: | if type(cm.value) == psycopg2.IntegrityError: | ||||
| assert cm.value.pgcode == psycopg2.errorcodes.NOT_NULL_VIOLATION | assert cm.value.pgcode == psycopg2.errorcodes.NOT_NULL_VIOLATION | ||||
| def test_directory_add_twice(self, swh_storage): | def test_directory_add_twice(self, swh_storage): | ||||
| actual_result = swh_storage.directory_add([data.dir]) | actual_result = swh_storage.directory_add([data.dir]) | ||||
| assert actual_result == {'directory:add': 1} | assert actual_result == {'directory:add': 1} | ||||
| assert list(swh_storage.journal_writer.objects) \ | assert list(swh_storage.journal_writer.journal.objects) \ | ||||
| == [('directory', data.dir)] | == [('directory', data.dir)] | ||||
| actual_result = swh_storage.directory_add([data.dir]) | actual_result = swh_storage.directory_add([data.dir]) | ||||
| assert actual_result == {'directory:add': 0} | assert actual_result == {'directory:add': 0} | ||||
| assert list(swh_storage.journal_writer.objects) \ | assert list(swh_storage.journal_writer.journal.objects) \ | ||||
| == [('directory', data.dir)] | == [('directory', data.dir)] | ||||
| def test_directory_get_recursive(self, swh_storage): | def test_directory_get_recursive(self, swh_storage): | ||||
| init_missing = list(swh_storage.directory_missing([data.dir['id']])) | init_missing = list(swh_storage.directory_missing([data.dir['id']])) | ||||
| assert init_missing == [data.dir['id']] | assert init_missing == [data.dir['id']] | ||||
| actual_result = swh_storage.directory_add( | actual_result = swh_storage.directory_add( | ||||
| [data.dir, data.dir2, data.dir3]) | [data.dir, data.dir2, data.dir3]) | ||||
| assert actual_result == {'directory:add': 3} | assert actual_result == {'directory:add': 3} | ||||
| assert list(swh_storage.journal_writer.objects) == [ | assert list(swh_storage.journal_writer.journal.objects) == [ | ||||
| ('directory', data.dir), | ('directory', data.dir), | ||||
| ('directory', data.dir2), | ('directory', data.dir2), | ||||
| ('directory', data.dir3)] | ('directory', data.dir3)] | ||||
| # List directory containing a file and an unknown subdirectory | # List directory containing a file and an unknown subdirectory | ||||
| actual_data = list(swh_storage.directory_ls( | actual_data = list(swh_storage.directory_ls( | ||||
| data.dir['id'], recursive=True)) | data.dir['id'], recursive=True)) | ||||
| expected_data = list(transform_entries(data.dir)) | expected_data = list(transform_entries(data.dir)) | ||||
| Show All 20 Lines | class TestStorage: | ||||
| def test_directory_get_non_recursive(self, swh_storage): | def test_directory_get_non_recursive(self, swh_storage): | ||||
| init_missing = list(swh_storage.directory_missing([data.dir['id']])) | init_missing = list(swh_storage.directory_missing([data.dir['id']])) | ||||
| assert init_missing == [data.dir['id']] | assert init_missing == [data.dir['id']] | ||||
| actual_result = swh_storage.directory_add( | actual_result = swh_storage.directory_add( | ||||
| [data.dir, data.dir2, data.dir3]) | [data.dir, data.dir2, data.dir3]) | ||||
| assert actual_result == {'directory:add': 3} | assert actual_result == {'directory:add': 3} | ||||
| assert list(swh_storage.journal_writer.objects) == [ | assert list(swh_storage.journal_writer.journal.objects) == [ | ||||
| ('directory', data.dir), | ('directory', data.dir), | ||||
| ('directory', data.dir2), | ('directory', data.dir2), | ||||
| ('directory', data.dir3)] | ('directory', data.dir3)] | ||||
| # List directory containing a file and an unknown subdirectory | # List directory containing a file and an unknown subdirectory | ||||
| actual_data = list(swh_storage.directory_ls(data.dir['id'])) | actual_data = list(swh_storage.directory_ls(data.dir['id'])) | ||||
| expected_data = list(transform_entries(data.dir)) | expected_data = list(transform_entries(data.dir)) | ||||
| assert sorted(expected_data, key=cmpdir) \ | assert sorted(expected_data, key=cmpdir) \ | ||||
| ▲ Show 20 Lines • Show All 97 Lines • ▼ Show 20 Lines | def test_revision_add(self, swh_storage): | ||||
| actual_result = swh_storage.revision_add([data.revision]) | actual_result = swh_storage.revision_add([data.revision]) | ||||
| assert actual_result == {'revision:add': 1} | assert actual_result == {'revision:add': 1} | ||||
| end_missing = swh_storage.revision_missing([data.revision['id']]) | end_missing = swh_storage.revision_missing([data.revision['id']]) | ||||
| assert list(end_missing) == [] | assert list(end_missing) == [] | ||||
| normalized_revision = Revision.from_dict(data.revision).to_dict() | normalized_revision = Revision.from_dict(data.revision).to_dict() | ||||
| assert list(swh_storage.journal_writer.objects) \ | assert list(swh_storage.journal_writer.journal.objects) \ | ||||
| == [('revision', normalized_revision)] | == [('revision', normalized_revision)] | ||||
| # already there so nothing added | # already there so nothing added | ||||
| actual_result = swh_storage.revision_add([data.revision]) | actual_result = swh_storage.revision_add([data.revision]) | ||||
| assert actual_result == {'revision:add': 0} | assert actual_result == {'revision:add': 0} | ||||
| swh_storage.refresh_stat_counters() | swh_storage.refresh_stat_counters() | ||||
| assert swh_storage.stat_counters()['revision'] == 1 | assert swh_storage.stat_counters()['revision'] == 1 | ||||
| ▲ Show 20 Lines • Show All 41 Lines • ▼ Show 20 Lines | class TestStorage: | ||||
| def test_revision_add_twice(self, swh_storage): | def test_revision_add_twice(self, swh_storage): | ||||
| actual_result = swh_storage.revision_add([data.revision]) | actual_result = swh_storage.revision_add([data.revision]) | ||||
| assert actual_result == {'revision:add': 1} | assert actual_result == {'revision:add': 1} | ||||
| normalized_revision = Revision.from_dict(data.revision).to_dict() | normalized_revision = Revision.from_dict(data.revision).to_dict() | ||||
| normalized_revision2 = Revision.from_dict(data.revision2).to_dict() | normalized_revision2 = Revision.from_dict(data.revision2).to_dict() | ||||
| assert list(swh_storage.journal_writer.objects) \ | assert list(swh_storage.journal_writer.journal.objects) \ | ||||
| == [('revision', normalized_revision)] | == [('revision', normalized_revision)] | ||||
| actual_result = swh_storage.revision_add( | actual_result = swh_storage.revision_add( | ||||
| [data.revision, data.revision2]) | [data.revision, data.revision2]) | ||||
| assert actual_result == {'revision:add': 1} | assert actual_result == {'revision:add': 1} | ||||
| assert list(swh_storage.journal_writer.objects) \ | assert list(swh_storage.journal_writer.journal.objects) \ | ||||
| == [('revision', normalized_revision), | == [('revision', normalized_revision), | ||||
| ('revision', normalized_revision2)] | ('revision', normalized_revision2)] | ||||
| def test_revision_add_name_clash(self, swh_storage): | def test_revision_add_name_clash(self, swh_storage): | ||||
| revision1 = data.revision | revision1 = data.revision | ||||
| revision2 = data.revision2 | revision2 = data.revision2 | ||||
| revision1['author'] = { | revision1['author'] = { | ||||
| Show All 28 Lines | def test_revision_log(self, swh_storage): | ||||
| assert len(actual_results) == 2 # rev4 -child-> rev3 | assert len(actual_results) == 2 # rev4 -child-> rev3 | ||||
| assert actual_results[0] == normalize_entity(data.revision4) | assert actual_results[0] == normalize_entity(data.revision4) | ||||
| assert actual_results[1] == normalize_entity(data.revision3) | assert actual_results[1] == normalize_entity(data.revision3) | ||||
| normalized_revision3 = Revision.from_dict(data.revision3).to_dict() | normalized_revision3 = Revision.from_dict(data.revision3).to_dict() | ||||
| normalized_revision4 = Revision.from_dict(data.revision4).to_dict() | normalized_revision4 = Revision.from_dict(data.revision4).to_dict() | ||||
| assert list(swh_storage.journal_writer.objects) == [ | assert list(swh_storage.journal_writer.journal.objects) == [ | ||||
| ('revision', normalized_revision3), | ('revision', normalized_revision3), | ||||
| ('revision', normalized_revision4)] | ('revision', normalized_revision4)] | ||||
| def test_revision_log_with_limit(self, swh_storage): | def test_revision_log_with_limit(self, swh_storage): | ||||
| # given | # given | ||||
| # data.revision4 -is-child-of-> data.revision3 | # data.revision4 -is-child-of-> data.revision3 | ||||
| swh_storage.revision_add([data.revision3, | swh_storage.revision_add([data.revision3, | ||||
| data.revision4]) | data.revision4]) | ||||
| ▲ Show 20 Lines • Show All 80 Lines • ▼ Show 20 Lines | def test_release_add(self, swh_storage): | ||||
| actual_result = swh_storage.release_add([data.release, data.release2]) | actual_result = swh_storage.release_add([data.release, data.release2]) | ||||
| assert actual_result == {'release:add': 2} | assert actual_result == {'release:add': 2} | ||||
| end_missing = swh_storage.release_missing([data.release['id'], | end_missing = swh_storage.release_missing([data.release['id'], | ||||
| data.release2['id']]) | data.release2['id']]) | ||||
| assert list(end_missing) == [] | assert list(end_missing) == [] | ||||
| assert list(swh_storage.journal_writer.objects) == [ | assert list(swh_storage.journal_writer.journal.objects) == [ | ||||
| ('release', normalized_release), | ('release', normalized_release), | ||||
| ('release', normalized_release2)] | ('release', normalized_release2)] | ||||
| # already present so nothing added | # already present so nothing added | ||||
| actual_result = swh_storage.release_add([data.release, data.release2]) | actual_result = swh_storage.release_add([data.release, data.release2]) | ||||
| assert actual_result == {'release:add': 0} | assert actual_result == {'release:add': 0} | ||||
| swh_storage.refresh_stat_counters() | swh_storage.refresh_stat_counters() | ||||
| assert swh_storage.stat_counters()['release'] == 2 | assert swh_storage.stat_counters()['release'] == 2 | ||||
| def test_release_add_from_generator(self, swh_storage): | def test_release_add_from_generator(self, swh_storage): | ||||
| def _rel_gen(): | def _rel_gen(): | ||||
| yield data.release | yield data.release | ||||
| yield data.release2 | yield data.release2 | ||||
| normalized_release = Release.from_dict(data.release).to_dict() | normalized_release = Release.from_dict(data.release).to_dict() | ||||
| normalized_release2 = Release.from_dict(data.release2).to_dict() | normalized_release2 = Release.from_dict(data.release2).to_dict() | ||||
| actual_result = swh_storage.release_add(_rel_gen()) | actual_result = swh_storage.release_add(_rel_gen()) | ||||
| assert actual_result == {'release:add': 2} | assert actual_result == {'release:add': 2} | ||||
| assert list(swh_storage.journal_writer.objects) == [ | assert list(swh_storage.journal_writer.journal.objects) == [ | ||||
| ('release', normalized_release), | ('release', normalized_release), | ||||
| ('release', normalized_release2)] | ('release', normalized_release2)] | ||||
| swh_storage.refresh_stat_counters() | swh_storage.refresh_stat_counters() | ||||
| assert swh_storage.stat_counters()['release'] == 2 | assert swh_storage.stat_counters()['release'] == 2 | ||||
| def test_release_add_no_author_date(self, swh_storage): | def test_release_add_no_author_date(self, swh_storage): | ||||
| release = data.release | release = data.release | ||||
| release['author'] = None | release['author'] = None | ||||
| release['date'] = None | release['date'] = None | ||||
| actual_result = swh_storage.release_add([release]) | actual_result = swh_storage.release_add([release]) | ||||
| assert actual_result == {'release:add': 1} | assert actual_result == {'release:add': 1} | ||||
| end_missing = swh_storage.release_missing([data.release['id']]) | end_missing = swh_storage.release_missing([data.release['id']]) | ||||
| assert list(end_missing) == [] | assert list(end_missing) == [] | ||||
| assert list(swh_storage.journal_writer.objects) \ | assert list(swh_storage.journal_writer.journal.objects) \ | ||||
| == [('release', release)] | == [('release', release)] | ||||
| def test_release_add_validation(self, swh_storage): | def test_release_add_validation(self, swh_storage): | ||||
| rel = copy.deepcopy(data.release) | rel = copy.deepcopy(data.release) | ||||
| rel['date']['offset'] = 2**16 | rel['date']['offset'] = 2**16 | ||||
| with pytest.raises(StorageArgumentException, match='offset') as cm: | with pytest.raises(StorageArgumentException, match='offset') as cm: | ||||
| swh_storage.release_add([rel]) | swh_storage.release_add([rel]) | ||||
| Show All 13 Lines | class TestStorage: | ||||
| def test_release_add_twice(self, swh_storage): | def test_release_add_twice(self, swh_storage): | ||||
| actual_result = swh_storage.release_add([data.release]) | actual_result = swh_storage.release_add([data.release]) | ||||
| assert actual_result == {'release:add': 1} | assert actual_result == {'release:add': 1} | ||||
| normalized_release = Release.from_dict(data.release).to_dict() | normalized_release = Release.from_dict(data.release).to_dict() | ||||
| normalized_release2 = Release.from_dict(data.release2).to_dict() | normalized_release2 = Release.from_dict(data.release2).to_dict() | ||||
| assert list(swh_storage.journal_writer.objects) \ | assert list(swh_storage.journal_writer.journal.objects) \ | ||||
| == [('release', normalized_release)] | == [('release', normalized_release)] | ||||
| actual_result = swh_storage.release_add([data.release, data.release2]) | actual_result = swh_storage.release_add([data.release, data.release2]) | ||||
| assert actual_result == {'release:add': 1} | assert actual_result == {'release:add': 1} | ||||
| assert list(swh_storage.journal_writer.objects) \ | assert list(swh_storage.journal_writer.journal.objects) \ | ||||
| == [('release', normalized_release), | == [('release', normalized_release), | ||||
| ('release', normalized_release2)] | ('release', normalized_release2)] | ||||
| def test_release_add_name_clash(self, swh_storage): | def test_release_add_name_clash(self, swh_storage): | ||||
| release1 = data.release.copy() | release1 = data.release.copy() | ||||
| release2 = data.release2.copy() | release2 = data.release2.copy() | ||||
| release1['author'] = { | release1['author'] = { | ||||
| ▲ Show 20 Lines • Show All 65 Lines • ▼ Show 20 Lines | def test_origin_add(self, swh_storage): | ||||
| 'url': data.origin2['url'], | 'url': data.origin2['url'], | ||||
| }])[0] | }])[0] | ||||
| assert actual_origin2['url'] == origin2['url'] | assert actual_origin2['url'] == origin2['url'] | ||||
| if 'id' in actual_origin: | if 'id' in actual_origin: | ||||
| del actual_origin['id'] | del actual_origin['id'] | ||||
| del actual_origin2['id'] | del actual_origin2['id'] | ||||
| assert list(swh_storage.journal_writer.objects) \ | assert list(swh_storage.journal_writer.journal.objects) \ | ||||
| == [('origin', actual_origin), | == [('origin', actual_origin), | ||||
| ('origin', actual_origin2)] | ('origin', actual_origin2)] | ||||
| swh_storage.refresh_stat_counters() | swh_storage.refresh_stat_counters() | ||||
| assert swh_storage.stat_counters()['origin'] == 2 | assert swh_storage.stat_counters()['origin'] == 2 | ||||
| def test_origin_add_from_generator(self, swh_storage): | def test_origin_add_from_generator(self, swh_storage): | ||||
| def _ori_gen(): | def _ori_gen(): | ||||
| Show All 11 Lines | def test_origin_add_from_generator(self, swh_storage): | ||||
| 'url': data.origin2['url'], | 'url': data.origin2['url'], | ||||
| }])[0] | }])[0] | ||||
| assert actual_origin2['url'] == origin2['url'] | assert actual_origin2['url'] == origin2['url'] | ||||
| if 'id' in actual_origin: | if 'id' in actual_origin: | ||||
| del actual_origin['id'] | del actual_origin['id'] | ||||
| del actual_origin2['id'] | del actual_origin2['id'] | ||||
| assert list(swh_storage.journal_writer.objects) \ | assert list(swh_storage.journal_writer.journal.objects) \ | ||||
| == [('origin', actual_origin), | == [('origin', actual_origin), | ||||
| ('origin', actual_origin2)] | ('origin', actual_origin2)] | ||||
| swh_storage.refresh_stat_counters() | swh_storage.refresh_stat_counters() | ||||
| assert swh_storage.stat_counters()['origin'] == 2 | assert swh_storage.stat_counters()['origin'] == 2 | ||||
| def test_origin_add_twice(self, swh_storage): | def test_origin_add_twice(self, swh_storage): | ||||
| add1 = swh_storage.origin_add([data.origin, data.origin2]) | add1 = swh_storage.origin_add([data.origin, data.origin2]) | ||||
| assert list(swh_storage.journal_writer.objects) \ | assert list(swh_storage.journal_writer.journal.objects) \ | ||||
| == [('origin', data.origin), | == [('origin', data.origin), | ||||
| ('origin', data.origin2)] | ('origin', data.origin2)] | ||||
| add2 = swh_storage.origin_add([data.origin, data.origin2]) | add2 = swh_storage.origin_add([data.origin, data.origin2]) | ||||
| assert list(swh_storage.journal_writer.objects) \ | assert list(swh_storage.journal_writer.journal.objects) \ | ||||
| == [('origin', data.origin), | == [('origin', data.origin), | ||||
| ('origin', data.origin2)] | ('origin', data.origin2)] | ||||
| assert add1 == add2 | assert add1 == add2 | ||||
| def test_origin_add_validation(self, swh_storage): | def test_origin_add_validation(self, swh_storage): | ||||
| with pytest.raises(StorageArgumentException, match='url'): | with pytest.raises(StorageArgumentException, match='url'): | ||||
| swh_storage.origin_add([{'type': 'git'}]) | swh_storage.origin_add([{'type': 'git'}]) | ||||
| ▲ Show 20 Lines • Show All 239 Lines • ▼ Show 20 Lines | def test_origin_visit_add(self, swh_storage): | ||||
| 'origin': origin_url, | 'origin': origin_url, | ||||
| 'date': date_visit, | 'date': date_visit, | ||||
| 'visit': origin_visit1['visit'], | 'visit': origin_visit1['visit'], | ||||
| 'type': data.type_visit1, | 'type': data.type_visit1, | ||||
| 'status': 'ongoing', | 'status': 'ongoing', | ||||
| 'metadata': None, | 'metadata': None, | ||||
| 'snapshot': None, | 'snapshot': None, | ||||
| } | } | ||||
| objects = list(swh_storage.journal_writer.objects) | objects = list(swh_storage.journal_writer.journal.objects) | ||||
| assert ('origin', data.origin2) in objects | assert ('origin', data.origin2) in objects | ||||
| assert ('origin_visit', origin_visit) in objects | assert ('origin_visit', origin_visit) in objects | ||||
| def test_origin_visit_get__unknown_origin(self, swh_storage): | def test_origin_visit_get__unknown_origin(self, swh_storage): | ||||
| assert [] == list(swh_storage.origin_visit_get('foo')) | assert [] == list(swh_storage.origin_visit_get('foo')) | ||||
| def test_origin_visit_add_default_type(self, swh_storage): | def test_origin_visit_add_default_type(self, swh_storage): | ||||
| # given | # given | ||||
| ▲ Show 20 Lines • Show All 46 Lines • ▼ Show 20 Lines | def test_origin_visit_add_default_type(self, swh_storage): | ||||
| 'status': 'ongoing', | 'status': 'ongoing', | ||||
| 'metadata': None, | 'metadata': None, | ||||
| 'snapshot': None, | 'snapshot': None, | ||||
| }, | }, | ||||
| ] | ] | ||||
| for visit in expected_visits: | for visit in expected_visits: | ||||
| assert visit in actual_origin_visits | assert visit in actual_origin_visits | ||||
| objects = list(swh_storage.journal_writer.objects) | objects = list(swh_storage.journal_writer.journal.objects) | ||||
| assert ('origin', data.origin2) in objects | assert ('origin', data.origin2) in objects | ||||
| for visit in expected_visits: | for visit in expected_visits: | ||||
| assert ('origin_visit', visit) in objects | assert ('origin_visit', visit) in objects | ||||
| def test_origin_visit_add_validation(self, swh_storage): | def test_origin_visit_add_validation(self, swh_storage): | ||||
| origin_url = swh_storage.origin_add_one(data.origin2) | origin_url = swh_storage.origin_add_one(data.origin2) | ||||
| ▲ Show 20 Lines • Show All 156 Lines • ▼ Show 20 Lines | def test_origin_visit_update(self, swh_storage): | ||||
| 'origin': origin_url2, | 'origin': origin_url2, | ||||
| 'date': date_visit2, | 'date': date_visit2, | ||||
| 'visit': origin_visit3['visit'], | 'visit': origin_visit3['visit'], | ||||
| 'type': data.type_visit3, | 'type': data.type_visit3, | ||||
| 'status': 'partial', | 'status': 'partial', | ||||
| 'metadata': None, | 'metadata': None, | ||||
| 'snapshot': None, | 'snapshot': None, | ||||
| } | } | ||||
| objects = list(swh_storage.journal_writer.objects) | objects = list(swh_storage.journal_writer.journal.objects) | ||||
| assert ('origin', data.origin) in objects | assert ('origin', data.origin) in objects | ||||
| assert ('origin', data.origin2) in objects | assert ('origin', data.origin2) in objects | ||||
| assert ('origin_visit', data1) in objects | assert ('origin_visit', data1) in objects | ||||
| assert ('origin_visit', data2) in objects | assert ('origin_visit', data2) in objects | ||||
| assert ('origin_visit', data3) in objects | assert ('origin_visit', data3) in objects | ||||
| assert ('origin_visit', data4) in objects | assert ('origin_visit', data4) in objects | ||||
| assert ('origin_visit', data5) in objects | assert ('origin_visit', data5) in objects | ||||
| ▲ Show 20 Lines • Show All 204 Lines • ▼ Show 20 Lines | def test_origin_visit_upsert_new(self, swh_storage): | ||||
| 'origin': origin_url, | 'origin': origin_url, | ||||
| 'date': data.date_visit3, | 'date': data.date_visit3, | ||||
| 'visit': 1234, | 'visit': 1234, | ||||
| 'type': data.type_visit2, | 'type': data.type_visit2, | ||||
| 'status': 'full', | 'status': 'full', | ||||
| 'metadata': None, | 'metadata': None, | ||||
| 'snapshot': None, | 'snapshot': None, | ||||
| } | } | ||||
| assert list(swh_storage.journal_writer.objects) == [ | assert list(swh_storage.journal_writer.journal.objects) == [ | ||||
| ('origin', data.origin2), | ('origin', data.origin2), | ||||
| ('origin_visit', data1), | ('origin_visit', data1), | ||||
| ('origin_visit', data2)] | ('origin_visit', data2)] | ||||
| def test_origin_visit_upsert_existing(self, swh_storage): | def test_origin_visit_upsert_existing(self, swh_storage): | ||||
| # given | # given | ||||
| swh_storage.origin_add_one(data.origin2) | swh_storage.origin_add_one(data.origin2) | ||||
| origin_url = data.origin2['url'] | origin_url = data.origin2['url'] | ||||
| ▲ Show 20 Lines • Show All 44 Lines • ▼ Show 20 Lines | def test_origin_visit_upsert_existing(self, swh_storage): | ||||
| 'origin': origin_url, | 'origin': origin_url, | ||||
| 'date': data.date_visit2, | 'date': data.date_visit2, | ||||
| 'visit': origin_visit1['visit'], | 'visit': origin_visit1['visit'], | ||||
| 'type': data.type_visit1, | 'type': data.type_visit1, | ||||
| 'status': 'full', | 'status': 'full', | ||||
| 'metadata': None, | 'metadata': None, | ||||
| 'snapshot': None, | 'snapshot': None, | ||||
| } | } | ||||
| assert list(swh_storage.journal_writer.objects) == [ | assert list(swh_storage.journal_writer.journal.objects) == [ | ||||
| ('origin', data.origin2), | ('origin', data.origin2), | ||||
| ('origin_visit', data1), | ('origin_visit', data1), | ||||
| ('origin_visit', data2)] | ('origin_visit', data2)] | ||||
| def test_origin_visit_get_by_no_result(self, swh_storage): | def test_origin_visit_get_by_no_result(self, swh_storage): | ||||
| swh_storage.origin_add([data.origin]) | swh_storage.origin_add([data.origin]) | ||||
| actual_origin_visit = swh_storage.origin_visit_get_by( | actual_origin_visit = swh_storage.origin_visit_get_by( | ||||
| data.origin['url'], 999) | data.origin['url'], 999) | ||||
| ▲ Show 20 Lines • Show All 160 Lines • ▼ Show 20 Lines | def test_snapshot_add_get_empty(self, swh_storage): | ||||
| 'origin': origin_url, | 'origin': origin_url, | ||||
| 'date': data.date_visit1, | 'date': data.date_visit1, | ||||
| 'visit': origin_visit1['visit'], | 'visit': origin_visit1['visit'], | ||||
| 'type': data.type_visit1, | 'type': data.type_visit1, | ||||
| 'status': 'ongoing', | 'status': 'ongoing', | ||||
| 'metadata': None, | 'metadata': None, | ||||
| 'snapshot': data.empty_snapshot['id'], | 'snapshot': data.empty_snapshot['id'], | ||||
| } | } | ||||
| assert list(swh_storage.journal_writer.objects) == \ | assert list(swh_storage.journal_writer.journal.objects) == \ | ||||
| [('origin', data.origin), | [('origin', data.origin), | ||||
| ('origin_visit', data1), | ('origin_visit', data1), | ||||
| ('snapshot', data.empty_snapshot), | ('snapshot', data.empty_snapshot), | ||||
| ('origin_visit', data2)] | ('origin_visit', data2)] | ||||
| def test_snapshot_add_get_complete(self, swh_storage): | def test_snapshot_add_get_complete(self, swh_storage): | ||||
| origin_url = data.origin['url'] | origin_url = data.origin['url'] | ||||
| swh_storage.origin_add_one(data.origin) | swh_storage.origin_add_one(data.origin) | ||||
| ▲ Show 20 Lines • Show All 53 Lines • ▼ Show 20 Lines | def test_snapshot_add_many_incremental(self, swh_storage): | ||||
| assert {**data.snapshot, 'next_branch': None} \ | assert {**data.snapshot, 'next_branch': None} \ | ||||
| == swh_storage.snapshot_get(data.snapshot['id']) | == swh_storage.snapshot_get(data.snapshot['id']) | ||||
| def test_snapshot_add_twice(self, swh_storage): | def test_snapshot_add_twice(self, swh_storage): | ||||
| actual_result = swh_storage.snapshot_add([data.empty_snapshot]) | actual_result = swh_storage.snapshot_add([data.empty_snapshot]) | ||||
| assert actual_result == {'snapshot:add': 1} | assert actual_result == {'snapshot:add': 1} | ||||
| assert list(swh_storage.journal_writer.objects) \ | assert list(swh_storage.journal_writer.journal.objects) \ | ||||
| == [('snapshot', data.empty_snapshot)] | == [('snapshot', data.empty_snapshot)] | ||||
| actual_result = swh_storage.snapshot_add([data.snapshot]) | actual_result = swh_storage.snapshot_add([data.snapshot]) | ||||
| assert actual_result == {'snapshot:add': 1} | assert actual_result == {'snapshot:add': 1} | ||||
| assert list(swh_storage.journal_writer.objects) \ | assert list(swh_storage.journal_writer.journal.objects) \ | ||||
| == [('snapshot', data.empty_snapshot), | == [('snapshot', data.empty_snapshot), | ||||
| ('snapshot', data.snapshot)] | ('snapshot', data.snapshot)] | ||||
| def test_snapshot_add_validation(self, swh_storage): | def test_snapshot_add_validation(self, swh_storage): | ||||
| snap = copy.deepcopy(data.snapshot) | snap = copy.deepcopy(data.snapshot) | ||||
| snap['branches'][b'foo'] = {'target_type': 'revision'} | snap['branches'][b'foo'] = {'target_type': 'revision'} | ||||
| with pytest.raises(StorageArgumentException, match='target'): | with pytest.raises(StorageArgumentException, match='target'): | ||||
| ▲ Show 20 Lines • Show All 219 Lines • ▼ Show 20 Lines | def test_snapshot_add_get(self, swh_storage): | ||||
| origin_url, visit_id) | origin_url, visit_id) | ||||
| assert origin_visit_info['snapshot'] == data.snapshot['id'] | assert origin_visit_info['snapshot'] == data.snapshot['id'] | ||||
| def test_snapshot_add_nonexistent_visit(self, swh_storage): | def test_snapshot_add_nonexistent_visit(self, swh_storage): | ||||
| origin_url = data.origin['url'] | origin_url = data.origin['url'] | ||||
| swh_storage.origin_add_one(data.origin) | swh_storage.origin_add_one(data.origin) | ||||
| visit_id = 54164461156 | visit_id = 54164461156 | ||||
| swh_storage.journal_writer.objects[:] = [] | swh_storage.journal_writer.journal.objects[:] = [] | ||||
| swh_storage.snapshot_add([data.snapshot]) | swh_storage.snapshot_add([data.snapshot]) | ||||
| with pytest.raises(StorageArgumentException): | with pytest.raises(StorageArgumentException): | ||||
| swh_storage.origin_visit_update( | swh_storage.origin_visit_update( | ||||
| origin_url, visit_id, snapshot=data.snapshot['id']) | origin_url, visit_id, snapshot=data.snapshot['id']) | ||||
| assert list(swh_storage.journal_writer.objects) == [ | assert list(swh_storage.journal_writer.journal.objects) == [ | ||||
| ('snapshot', data.snapshot)] | ('snapshot', data.snapshot)] | ||||
| def test_snapshot_add_twice__by_origin_visit(self, swh_storage): | def test_snapshot_add_twice__by_origin_visit(self, swh_storage): | ||||
| origin_url = data.origin['url'] | origin_url = data.origin['url'] | ||||
| swh_storage.origin_add_one(data.origin) | swh_storage.origin_add_one(data.origin) | ||||
| origin_visit1 = swh_storage.origin_visit_add( | origin_visit1 = swh_storage.origin_visit_add( | ||||
| origin=origin_url, | origin=origin_url, | ||||
| date=data.date_visit1, | date=data.date_visit1, | ||||
| ▲ Show 20 Lines • Show All 54 Lines • ▼ Show 20 Lines | def test_snapshot_add_twice__by_origin_visit(self, swh_storage): | ||||
| 'origin': origin_url, | 'origin': origin_url, | ||||
| 'date': data.date_visit2, | 'date': data.date_visit2, | ||||
| 'visit': origin_visit2['visit'], | 'visit': origin_visit2['visit'], | ||||
| 'type': data.type_visit2, | 'type': data.type_visit2, | ||||
| 'status': 'ongoing', | 'status': 'ongoing', | ||||
| 'metadata': None, | 'metadata': None, | ||||
| 'snapshot': data.snapshot['id'], | 'snapshot': data.snapshot['id'], | ||||
| } | } | ||||
| assert list(swh_storage.journal_writer.objects) \ | assert list(swh_storage.journal_writer.journal.objects) \ | ||||
| == [('origin', data.origin), | == [('origin', data.origin), | ||||
| ('origin_visit', data1), | ('origin_visit', data1), | ||||
| ('snapshot', data.snapshot), | ('snapshot', data.snapshot), | ||||
| ('origin_visit', data2), | ('origin_visit', data2), | ||||
| ('origin_visit', data3), | ('origin_visit', data3), | ||||
| ('origin_visit', data4)] | ('origin_visit', data4)] | ||||
| def test_snapshot_get_latest(self, swh_storage): | def test_snapshot_get_latest(self, swh_storage): | ||||
| ▲ Show 20 Lines • Show All 1,170 Lines • ▼ Show 20 Lines | class TestPgStorage: | ||||
| """This class is dedicated for the rare case where the schema needs to | """This class is dedicated for the rare case where the schema needs to | ||||
| be altered dynamically. | be altered dynamically. | ||||
| Otherwise, the tests could be blocking when ran altogether. | Otherwise, the tests could be blocking when ran altogether. | ||||
| """ | """ | ||||
| def test_content_update_with_new_cols(self, swh_storage): | def test_content_update_with_new_cols(self, swh_storage): | ||||
| swh_storage.storage.journal_writer = None # TODO, not supported | swh_storage.journal_writer.journal = None # TODO, not supported | ||||
| with db_transaction(swh_storage) as (_, cur): | with db_transaction(swh_storage) as (_, cur): | ||||
| cur.execute("""alter table content | cur.execute("""alter table content | ||||
| add column test text default null, | add column test text default null, | ||||
| add column test2 text default null""") | add column test2 text default null""") | ||||
| cont = copy.deepcopy(data.cont2) | cont = copy.deepcopy(data.cont2) | ||||
| swh_storage.content_add([cont]) | swh_storage.content_add([cont]) | ||||
| Show All 37 Lines | def test_content_add_db(self, swh_storage): | ||||
| (cont['sha1'],)) | (cont['sha1'],)) | ||||
| datum = cur.fetchone() | datum = cur.fetchone() | ||||
| assert datum == (cont['sha1'], cont['sha1_git'], cont['sha256'], | assert datum == (cont['sha1'], cont['sha1_git'], cont['sha256'], | ||||
| cont['length'], 'visible') | cont['length'], 'visible') | ||||
| expected_cont = cont.copy() | expected_cont = cont.copy() | ||||
| del expected_cont['data'] | del expected_cont['data'] | ||||
| journal_objects = list(swh_storage.journal_writer.objects) | journal_objects = list(swh_storage.journal_writer.journal.objects) | ||||
| for (obj_type, obj) in journal_objects: | for (obj_type, obj) in journal_objects: | ||||
| del obj['ctime'] | del obj['ctime'] | ||||
| assert journal_objects == [('content', expected_cont)] | assert journal_objects == [('content', expected_cont)] | ||||
| def test_content_add_metadata_db(self, swh_storage): | def test_content_add_metadata_db(self, swh_storage): | ||||
| cont = data.cont | cont = data.cont | ||||
| del cont['data'] | del cont['data'] | ||||
| cont['ctime'] = datetime.datetime.now() | cont['ctime'] = datetime.datetime.now() | ||||
| Show All 9 Lines | def test_content_add_metadata_db(self, swh_storage): | ||||
| with db_transaction(swh_storage) as (_, cur): | with db_transaction(swh_storage) as (_, cur): | ||||
| cur.execute('SELECT sha1, sha1_git, sha256, length, status' | cur.execute('SELECT sha1, sha1_git, sha256, length, status' | ||||
| ' FROM content WHERE sha1 = %s', | ' FROM content WHERE sha1 = %s', | ||||
| (cont['sha1'],)) | (cont['sha1'],)) | ||||
| datum = cur.fetchone() | datum = cur.fetchone() | ||||
| assert datum == (cont['sha1'], cont['sha1_git'], cont['sha256'], | assert datum == (cont['sha1'], cont['sha1_git'], cont['sha256'], | ||||
| cont['length'], 'visible') | cont['length'], 'visible') | ||||
| assert list(swh_storage.journal_writer.objects) == [('content', cont)] | assert list(swh_storage.journal_writer.journal.objects) == [ | ||||
| ('content', cont)] | |||||
| def test_skipped_content_add_db(self, swh_storage): | def test_skipped_content_add_db(self, swh_storage): | ||||
| cont = data.skipped_cont | cont = data.skipped_cont | ||||
| cont2 = data.skipped_cont2 | cont2 = data.skipped_cont2 | ||||
| cont2['blake2s256'] = None | cont2['blake2s256'] = None | ||||
| actual_result = swh_storage.skipped_content_add([cont, cont, cont2]) | actual_result = swh_storage.skipped_content_add([cont, cont, cont2]) | ||||
| Show All 18 Lines | |||||