diff --git a/swh/storage/in_memory.py b/swh/storage/in_memory.py --- a/swh/storage/in_memory.py +++ b/swh/storage/in_memory.py @@ -486,6 +486,7 @@ directory:add: Number of directories actually added """ + directories = list(directories) if self.journal_writer: self.journal_writer.write_additions( 'directory', @@ -652,6 +653,7 @@ revision_added: New objects actually stored in db """ + revisions = list(revisions) if self.journal_writer: self.journal_writer.write_additions( 'revision', @@ -768,6 +770,7 @@ release:add: New objects contents actually stored in db """ + releases = list(releases) if self.journal_writer: self.journal_writer.write_additions( 'release', @@ -1295,7 +1298,7 @@ list: given origins as dict updated with their id """ - origins = copy.deepcopy(origins) + origins = copy.deepcopy(list(origins)) for origin in origins: self.origin_add_one(origin) return origins diff --git a/swh/storage/storage.py b/swh/storage/storage.py --- a/swh/storage/storage.py +++ b/swh/storage/storage.py @@ -703,6 +703,7 @@ directory:add: Number of directories actually added """ + directories = list(directories) summary = {'directory:add': 0} dirs = set() @@ -874,6 +875,7 @@ revision:add: New objects actually stored in db """ + revisions = list(revisions) summary = {'revision:add': 0} revisions_missing = set(self.revision_missing( @@ -1031,6 +1033,7 @@ release:add: New objects contents actually stored in db """ + releases = list(releases) summary = {'release:add': 0} release_ids = set(release['id'] for release in releases) @@ -1845,7 +1848,7 @@ list: given origins as dict updated with their id """ - origins = copy.deepcopy(origins) + origins = copy.deepcopy(list(origins)) for origin in origins: self.origin_add_one(origin, db=db, cur=cur) diff --git a/swh/storage/tests/test_storage.py b/swh/storage/tests/test_storage.py --- a/swh/storage/tests/test_storage.py +++ b/swh/storage/tests/test_storage.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2019 The Software Heritage developers +# Copyright (C) 2015-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -125,6 +125,24 @@ del obj['ctime'] assert journal_objects == [('content', expected_cont)] + swh_storage.refresh_stat_counters() + assert swh_storage.stat_counters()['content'] == 1 + + def test_content_add_from_generator(self, swh_storage): + def _cnt_gen(): + yield data.cont + + actual_result = swh_storage.content_add(_cnt_gen()) + + assert actual_result == { + 'content:add': 1, + 'content:add:bytes': data.cont['length'], + 'skipped_content:add': 0 + } + + swh_storage.refresh_stat_counters() + assert swh_storage.stat_counters()['content'] == 1 + def test_content_add_validation(self, swh_storage): cont = data.cont @@ -496,6 +514,22 @@ after_missing = list(swh_storage.directory_missing([data.dir['id']])) assert after_missing == [] + swh_storage.refresh_stat_counters() + assert swh_storage.stat_counters()['directory'] == 1 + + def test_directory_add_from_generator(self, swh_storage): + def _dir_gen(): + yield data.dir + + actual_result = swh_storage.directory_add(directories=_dir_gen()) + assert actual_result == {'directory:add': 1} + + assert list(swh_storage.journal_writer.objects) == \ + [('directory', data.dir)] + + swh_storage.refresh_stat_counters() + assert swh_storage.stat_counters()['directory'] == 1 + def test_directory_add_validation(self, swh_storage): dir_ = copy.deepcopy(data.dir) dir_['entries'][0]['type'] = 'foobar' @@ -690,6 +724,19 @@ actual_result = swh_storage.revision_add([data.revision]) assert actual_result == {'revision:add': 0} + swh_storage.refresh_stat_counters() + assert swh_storage.stat_counters()['revision'] == 1 + + def test_revision_add_from_generator(self, swh_storage): + def _rev_gen(): + yield data.revision + + actual_result = swh_storage.revision_add(_rev_gen()) + assert actual_result == {'revision:add': 1} + + swh_storage.refresh_stat_counters() + assert swh_storage.stat_counters()['revision'] == 1 + def test_revision_add_validation(self, swh_storage): rev = copy.deepcopy(data.revision) rev['date']['offset'] = 2**16 @@ -879,6 +926,24 @@ actual_result = swh_storage.release_add([data.release, data.release2]) assert actual_result == {'release:add': 0} + swh_storage.refresh_stat_counters() + assert swh_storage.stat_counters()['release'] == 2 + + def test_release_add_from_generator(self, swh_storage): + def _rel_gen(): + yield data.release + yield data.release2 + + actual_result = swh_storage.release_add(_rel_gen()) + assert actual_result == {'release:add': 2} + + assert list(swh_storage.journal_writer.objects) == [ + ('release', data.release), + ('release', data.release2)] + + swh_storage.refresh_stat_counters() + assert swh_storage.stat_counters()['release'] == 2 + def test_release_add_no_author_date(self, swh_storage): release = data.release @@ -1012,6 +1077,37 @@ == [('origin', actual_origin), ('origin', actual_origin2)] + swh_storage.refresh_stat_counters() + assert swh_storage.stat_counters()['origin'] == 2 + + def test_origin_add_from_generator(self, swh_storage): + def _ori_gen(): + yield data.origin + yield data.origin2 + + origin1, origin2 = swh_storage.origin_add(_ori_gen()) + + actual_origin = swh_storage.origin_get([{ + 'url': data.origin['url'], + }])[0] + assert actual_origin['url'] == origin1['url'] + + actual_origin2 = swh_storage.origin_get([{ + 'url': data.origin2['url'], + }])[0] + assert actual_origin2['url'] == origin2['url'] + + if 'id' in actual_origin: + del actual_origin['id'] + del actual_origin2['id'] + + assert list(swh_storage.journal_writer.objects) \ + == [('origin', actual_origin), + ('origin', actual_origin2)] + + swh_storage.refresh_stat_counters() + assert swh_storage.stat_counters()['origin'] == 2 + def test_origin_add_twice(self, swh_storage): add1 = swh_storage.origin_add([data.origin, data.origin2]) assert list(swh_storage.journal_writer.objects) \ @@ -1988,6 +2084,20 @@ assert {**data.snapshot, 'next_branch': None} \ == swh_storage.snapshot_get(data.snapshot['id']) + swh_storage.refresh_stat_counters() + assert swh_storage.stat_counters()['snapshot'] == 2 + + def test_snapshot_add_many_from_generator(self, swh_storage): + def _snp_gen(): + yield data.snapshot + yield data.complete_snapshot + + actual_result = swh_storage.snapshot_add(_snp_gen()) + assert actual_result == {'snapshot:add': 2} + + swh_storage.refresh_stat_counters() + assert swh_storage.stat_counters()['snapshot'] == 2 + def test_snapshot_add_many_incremental(self, swh_storage): actual_result = swh_storage.snapshot_add([data.complete_snapshot]) assert actual_result == {'snapshot:add': 1}