diff --git a/swh/storage/in_memory.py b/swh/storage/in_memory.py --- a/swh/storage/in_memory.py +++ b/swh/storage/in_memory.py @@ -200,6 +200,48 @@ for c in self._content_to_model(content)] return self._content_add(content, with_data=True) + def content_update(self, content, keys=[]): + """Update content blobs to the storage. Does nothing for unknown + contents or skipped ones. + + Args: + content (iterable): iterable of dictionaries representing + individual pieces of content to update. Each dictionary has the + following keys: + + - data (bytes): the actual content + - length (int): content length (default: -1) + - one key for each checksum algorithm in + :data:`swh.model.hashutil.ALGORITHMS`, mapped to the + corresponding checksum + - status (str): one of visible, hidden, absent + + keys (list): List of keys (str) whose values needs an update, e.g., + new hash column + """ + if self.journal_writer: + raise NotImplementedError( + 'content_update is not yet supported with a journal_writer.') + + for cont_update in content: + cont_update = cont_update.copy() + sha1 = cont_update.pop('sha1') + for old_key in self._content_indexes['sha1'][sha1]: + old_cont = self._contents.pop(old_key) + + for algorithm in DEFAULT_ALGORITHMS: + hash_ = old_cont.get_hash(algorithm) + self._content_indexes[algorithm][hash_].remove(old_key) + + new_cont = attr.evolve(old_cont, **cont_update) + new_key = self._content_key(new_cont) + + self._contents[new_key] = new_cont + + for algorithm in DEFAULT_ALGORITHMS: + hash_ = new_cont.get_hash(algorithm) + self._content_indexes[algorithm][hash_].add(new_key) + def content_add_metadata(self, content): """Add content metadata to the storage (like `content_add`, but without inserting to the objstorage). @@ -369,6 +411,7 @@ for key in objs: d = self._contents[key].to_dict() del d['ctime'] + del d['data'] result[sha1].append(d) return result diff --git a/swh/storage/tests/test_api_client.py b/swh/storage/tests/test_api_client.py --- a/swh/storage/tests/test_api_client.py +++ b/swh/storage/tests/test_api_client.py @@ -3,13 +3,15 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +from unittest.mock import patch + import pytest from swh.storage.api.client import RemoteStorage import swh.storage.api.server as server import swh.storage.storage -from swh.storage.tests.test_storage import ( # noqa - TestStorage, TestStorageGeneratedData) +from swh.storage.tests.test_storage import TestStorageGeneratedData # noqa +from swh.storage.tests.test_storage import TestStorage as _TestStorage # tests are executed using imported classes (TestStorage and # TestStorageGeneratedData) using overloaded swh_storage fixture @@ -17,7 +19,7 @@ @pytest.fixture -def app(): +def app_server(): storage_config = { 'cls': 'memory', 'journal_writer': { @@ -25,12 +27,12 @@ }, } server.storage = swh.storage.get_storage(**storage_config) - # hack hack hack! - # We attach the journal storage to the app here to make it accessible to - # the test (as swh_storage.journal_writer); see swh_storage below. - server.app.journal_writer = server.storage.journal_writer - yield server.app - del server.app.journal_writer + yield server + + +@pytest.fixture +def app(app_server): + return app_server.app @pytest.fixture @@ -39,11 +41,11 @@ @pytest.fixture -def swh_storage(swh_rpc_client, app): +def swh_storage(swh_rpc_client, app_server): # This version of the swh_storage fixture uses the swh_rpc_client fixture # to instantiate a RemoteStorage (see swh_rpc_client_class above) that # proxies, via the swh.core RPC mechanism, the local (in memory) storage - # configured in the app fixture above. + # configured in the app_server fixture above. # # Also note that, for the sake of # making it easier to write tests, the in-memory journal writer of the @@ -51,6 +53,13 @@ # journal_writer attribute. storage = swh_rpc_client journal_writer = getattr(storage, 'journal_writer', None) - storage.journal_writer = app.journal_writer + storage.journal_writer = app_server.storage.journal_writer yield storage storage.journal_writer = journal_writer + + +class TestStorage(_TestStorage): + def test_content_update(self, swh_storage, app_server): + swh_storage.journal_writer = None # TODO, journal_writer not supported + with patch.object(server.storage, 'journal_writer', None): + super().test_content_update(swh_storage) diff --git a/swh/storage/tests/test_storage.py b/swh/storage/tests/test_storage.py --- a/swh/storage/tests/test_storage.py +++ b/swh/storage/tests/test_storage.py @@ -230,6 +230,22 @@ assert cm.value.args[0] in ['sha1', 'sha1_git', 'blake2s256'] + def test_content_update(self, swh_storage): + swh_storage.journal_writer = None # TODO, not supported + + cont = copy.deepcopy(data.cont) + + swh_storage.content_add([cont]) + # alter the sha1_git for example + cont['sha1_git'] = hash_to_bytes( + '3a60a5275d0333bf13468e8b3dcab90f4046e654') + + swh_storage.content_update([cont], keys=['sha1_git']) + + results = swh_storage.content_get_metadata([cont['sha1']]) + del cont['data'] + assert results == {cont['sha1']: [cont]} + def test_content_add_metadata(self, swh_storage): cont = data.cont del cont['data'] @@ -3618,26 +3634,6 @@ Otherwise, the tests could be blocking when ran altogether. """ - def test_content_update(self, swh_storage): - swh_storage.journal_writer = None # TODO, not supported - - cont = copy.deepcopy(data.cont) - - swh_storage.content_add([cont]) - # alter the sha1_git for example - cont['sha1_git'] = hash_to_bytes( - '3a60a5275d0333bf13468e8b3dcab90f4046e654') - - swh_storage.content_update([cont], keys=['sha1_git']) - - with db_transaction(swh_storage) as (_, cur): - cur.execute('SELECT sha1, sha1_git, sha256, length, status' - ' FROM content WHERE sha1 = %s', - (cont['sha1'],)) - datum = cur.fetchone() - - assert datum == (cont['sha1'], cont['sha1_git'], cont['sha256'], - cont['length'], 'visible') def test_content_update_with_new_cols(self, swh_storage): swh_storage.journal_writer = None # TODO, not supported