Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/tests/test_storage.py
| Show First 20 Lines • Show All 209 Lines • ▼ Show 20 Lines | def test_content_add_twice(self, swh_storage): | ||||
| 'content:add:bytes': data.cont2['length'], | 'content:add:bytes': data.cont2['length'], | ||||
| 'skipped_content:add': 0 | 'skipped_content:add': 0 | ||||
| } | } | ||||
| assert len(swh_storage.journal_writer.objects) == 2 | assert len(swh_storage.journal_writer.objects) == 2 | ||||
| assert len(swh_storage.content_find(data.cont)) == 1 | assert len(swh_storage.content_find(data.cont)) == 1 | ||||
| assert len(swh_storage.content_find(data.cont2)) == 1 | assert len(swh_storage.content_find(data.cont2)) == 1 | ||||
| def test_content_add_db(self, swh_storage): | |||||
| cont = data.cont | |||||
| actual_result = swh_storage.content_add([cont]) | |||||
| assert actual_result == { | |||||
| 'content:add': 1, | |||||
| 'content:add:bytes': cont['length'], | |||||
| 'skipped_content:add': 0 | |||||
| } | |||||
| if hasattr(swh_storage, 'objstorage'): | |||||
| assert cont['sha1'] in swh_storage.objstorage | |||||
| with db_transaction(swh_storage) as (_, cur): | |||||
| cur.execute('SELECT sha1, sha1_git, sha256, length, status' | |||||
| ' FROM content WHERE sha1 = %s', | |||||
| (cont['sha1'],)) | |||||
| datum = cur.fetchone() | |||||
| assert datum == (cont['sha1'], cont['sha1_git'], cont['sha256'], | |||||
| cont['length'], 'visible') | |||||
| expected_cont = cont.copy() | |||||
| del expected_cont['data'] | |||||
| journal_objects = list(swh_storage.journal_writer.objects) | |||||
| for (obj_type, obj) in journal_objects: | |||||
| del obj['ctime'] | |||||
| assert journal_objects == [('content', expected_cont)] | |||||
| def test_content_add_collision(self, swh_storage): | def test_content_add_collision(self, swh_storage): | ||||
| cont1 = data.cont | cont1 = data.cont | ||||
| # create (corrupted) content with same sha1{,_git} but != sha256 | # create (corrupted) content with same sha1{,_git} but != sha256 | ||||
| cont1b = cont1.copy() | cont1b = cont1.copy() | ||||
| sha256_array = bytearray(cont1b['sha256']) | sha256_array = bytearray(cont1b['sha256']) | ||||
| sha256_array[0] += 1 | sha256_array[0] += 1 | ||||
| cont1b['sha256'] = bytes(sha256_array) | cont1b['sha256'] = bytes(sha256_array) | ||||
| ▲ Show 20 Lines • Show All 41 Lines • ▼ Show 20 Lines | def test_content_add_metadata_different_input(self, swh_storage): | ||||
| cont2['ctime'] = datetime.datetime.now() | cont2['ctime'] = datetime.datetime.now() | ||||
| actual_result = swh_storage.content_add_metadata([cont, cont2]) | actual_result = swh_storage.content_add_metadata([cont, cont2]) | ||||
| assert actual_result == { | assert actual_result == { | ||||
| 'content:add': 2, | 'content:add': 2, | ||||
| 'skipped_content:add': 0 | 'skipped_content:add': 0 | ||||
| } | } | ||||
| def test_content_add_metadata_db(self, swh_storage): | |||||
| cont = data.cont | |||||
| del cont['data'] | |||||
| cont['ctime'] = datetime.datetime.now() | |||||
| actual_result = swh_storage.content_add_metadata([cont]) | |||||
| assert actual_result == { | |||||
| 'content:add': 1, | |||||
| 'skipped_content:add': 0 | |||||
| } | |||||
| if hasattr(swh_storage, 'objstorage'): | |||||
| assert cont['sha1'] not in swh_storage.objstorage | |||||
| with db_transaction(swh_storage) as (_, cur): | |||||
| cur.execute('SELECT sha1, sha1_git, sha256, length, status' | |||||
| ' FROM content WHERE sha1 = %s', | |||||
| (cont['sha1'],)) | |||||
| datum = cur.fetchone() | |||||
| assert datum == (cont['sha1'], cont['sha1_git'], cont['sha256'], | |||||
| cont['length'], 'visible') | |||||
| assert list(swh_storage.journal_writer.objects) == [('content', cont)] | |||||
| def test_content_add_metadata_collision(self, swh_storage): | def test_content_add_metadata_collision(self, swh_storage): | ||||
| cont1 = data.cont | cont1 = data.cont | ||||
| del cont1['data'] | del cont1['data'] | ||||
| cont1['ctime'] = datetime.datetime.now() | cont1['ctime'] = datetime.datetime.now() | ||||
| # create (corrupted) content with same sha1{,_git} but != sha256 | # create (corrupted) content with same sha1{,_git} but != sha256 | ||||
| cont1b = cont1.copy() | cont1b = cont1.copy() | ||||
| sha256_array = bytearray(cont1b['sha256']) | sha256_array = bytearray(cont1b['sha256']) | ||||
| sha256_array[0] += 1 | sha256_array[0] += 1 | ||||
| cont1b['sha256'] = bytes(sha256_array) | cont1b['sha256'] = bytes(sha256_array) | ||||
| with pytest.raises(HashCollision) as cm: | with pytest.raises(HashCollision) as cm: | ||||
| swh_storage.content_add_metadata([cont1, cont1b]) | swh_storage.content_add_metadata([cont1, cont1b]) | ||||
| assert cm.value.args[0] in ['sha1', 'sha1_git', 'blake2s256'] | assert cm.value.args[0] in ['sha1', 'sha1_git', 'blake2s256'] | ||||
| def test_skipped_content_add_db(self, swh_storage): | |||||
| cont = data.skipped_cont | |||||
| cont2 = data.skipped_cont2 | |||||
| cont2['blake2s256'] = None | |||||
| actual_result = swh_storage.content_add([cont, cont, cont2]) | |||||
| assert actual_result == { | |||||
| 'content:add': 0, | |||||
| 'content:add:bytes': 0, | |||||
| 'skipped_content:add': 2, | |||||
| } | |||||
| with db_transaction(swh_storage) as (_, cur): | |||||
| cur.execute('SELECT sha1, sha1_git, sha256, blake2s256, ' | |||||
| 'length, status, reason ' | |||||
| 'FROM skipped_content ORDER BY sha1_git') | |||||
| datum = cur.fetchall() | |||||
| assert len(datum) == 2 | |||||
| assert datum[0] == (cont['sha1'], cont['sha1_git'], cont['sha256'], | |||||
| cont['blake2s256'], cont['length'], 'absent', | |||||
| 'Content too long') | |||||
| assert datum[1] == (cont2['sha1'], cont2['sha1_git'], cont2['sha256'], | |||||
| cont2['blake2s256'], cont2['length'], 'absent', | |||||
| 'Content too long') | |||||
| def test_skipped_content_add(self, swh_storage): | def test_skipped_content_add(self, swh_storage): | ||||
| cont = data.skipped_cont | cont = data.skipped_cont | ||||
| cont2 = data.skipped_cont2 | cont2 = data.skipped_cont2 | ||||
| cont2['blake2s256'] = None | cont2['blake2s256'] = None | ||||
| missing = list(swh_storage.skipped_content_missing([cont, cont2])) | missing = list(swh_storage.skipped_content_missing([cont, cont2])) | ||||
| assert len(missing) == 2 | assert len(missing) == 2 | ||||
| ▲ Show 20 Lines • Show All 3,004 Lines • ▼ Show 20 Lines | def test_content_add_race(self, swh_storage): | ||||
| with pytest.raises(queue.Empty): | with pytest.raises(queue.Empty): | ||||
| results.get(block=False) | results.get(block=False) | ||||
| assert r1[0] != r2[0] | assert r1[0] != r2[0] | ||||
| assert r1[1] == 'data', 'Got exception %r in Thread%s' % (r1[2], r1[0]) | assert r1[1] == 'data', 'Got exception %r in Thread%s' % (r1[2], r1[0]) | ||||
| assert r2[1] == 'data', 'Got exception %r in Thread%s' % (r2[2], r2[0]) | assert r2[1] == 'data', 'Got exception %r in Thread%s' % (r2[2], r2[0]) | ||||
| @pytest.mark.db | @pytest.mark.db | ||||
| class TestAlteringSchema: | class TestPgStorage: | ||||
| """This class is dedicated for the rare case where the schema needs to | """This class is dedicated for the rare case where the schema needs to | ||||
| be altered dynamically. | be altered dynamically. | ||||
| Otherwise, the tests could be blocking when ran altogether. | Otherwise, the tests could be blocking when ran altogether. | ||||
| """ | """ | ||||
| def test_content_update(self, swh_storage): | def test_content_update(self, swh_storage): | ||||
| swh_storage.journal_writer = None # TODO, not supported | swh_storage.journal_writer = None # TODO, not supported | ||||
| ▲ Show 20 Lines • Show All 41 Lines • ▼ Show 20 Lines | def test_content_update_with_new_cols(self, swh_storage): | ||||
| assert datum == (cont['sha1'], cont['sha1_git'], cont['sha256'], | assert datum == (cont['sha1'], cont['sha1_git'], cont['sha256'], | ||||
| cont['length'], 'visible', | cont['length'], 'visible', | ||||
| cont['test'], cont['test2']) | cont['test'], cont['test2']) | ||||
| with db_transaction(swh_storage) as (_, cur): | with db_transaction(swh_storage) as (_, cur): | ||||
| cur.execute("""alter table content drop column test, | cur.execute("""alter table content drop column test, | ||||
| drop column test2""") | drop column test2""") | ||||
| def test_content_add_db(self, swh_storage): | |||||
| cont = data.cont | |||||
| actual_result = swh_storage.content_add([cont]) | |||||
| assert actual_result == { | |||||
| 'content:add': 1, | |||||
| 'content:add:bytes': cont['length'], | |||||
| 'skipped_content:add': 0 | |||||
| } | |||||
| if hasattr(swh_storage, 'objstorage'): | |||||
| assert cont['sha1'] in swh_storage.objstorage | |||||
| with db_transaction(swh_storage) as (_, cur): | |||||
| cur.execute('SELECT sha1, sha1_git, sha256, length, status' | |||||
| ' FROM content WHERE sha1 = %s', | |||||
| (cont['sha1'],)) | |||||
| datum = cur.fetchone() | |||||
| assert datum == (cont['sha1'], cont['sha1_git'], cont['sha256'], | |||||
| cont['length'], 'visible') | |||||
| expected_cont = cont.copy() | |||||
| del expected_cont['data'] | |||||
| journal_objects = list(swh_storage.journal_writer.objects) | |||||
| for (obj_type, obj) in journal_objects: | |||||
| del obj['ctime'] | |||||
| assert journal_objects == [('content', expected_cont)] | |||||
| def test_content_add_metadata_db(self, swh_storage): | |||||
| cont = data.cont | |||||
| del cont['data'] | |||||
| cont['ctime'] = datetime.datetime.now() | |||||
| actual_result = swh_storage.content_add_metadata([cont]) | |||||
| assert actual_result == { | |||||
| 'content:add': 1, | |||||
| 'skipped_content:add': 0 | |||||
| } | |||||
| if hasattr(swh_storage, 'objstorage'): | |||||
| assert cont['sha1'] not in swh_storage.objstorage | |||||
| with db_transaction(swh_storage) as (_, cur): | |||||
| cur.execute('SELECT sha1, sha1_git, sha256, length, status' | |||||
| ' FROM content WHERE sha1 = %s', | |||||
| (cont['sha1'],)) | |||||
| datum = cur.fetchone() | |||||
| assert datum == (cont['sha1'], cont['sha1_git'], cont['sha256'], | |||||
| cont['length'], 'visible') | |||||
| assert list(swh_storage.journal_writer.objects) == [('content', cont)] | |||||
| def test_skipped_content_add_db(self, swh_storage): | |||||
| cont = data.skipped_cont | |||||
| cont2 = data.skipped_cont2 | |||||
| cont2['blake2s256'] = None | |||||
| actual_result = swh_storage.content_add([cont, cont, cont2]) | |||||
| assert actual_result == { | |||||
| 'content:add': 0, | |||||
| 'content:add:bytes': 0, | |||||
| 'skipped_content:add': 2, | |||||
| } | |||||
| with db_transaction(swh_storage) as (_, cur): | |||||
| cur.execute('SELECT sha1, sha1_git, sha256, blake2s256, ' | |||||
| 'length, status, reason ' | |||||
| 'FROM skipped_content ORDER BY sha1_git') | |||||
| dbdata = cur.fetchall() | |||||
| assert len(dbdata) == 2 | |||||
| assert dbdata[0] == (cont['sha1'], cont['sha1_git'], cont['sha256'], | |||||
| cont['blake2s256'], cont['length'], 'absent', | |||||
| 'Content too long') | |||||
| assert dbdata[1] == (cont2['sha1'], cont2['sha1_git'], cont2['sha256'], | |||||
| cont2['blake2s256'], cont2['length'], 'absent', | |||||
| 'Content too long') | |||||