diff --git a/swh/storage/api/client.py b/swh/storage/api/client.py --- a/swh/storage/api/client.py +++ b/swh/storage/api/client.py @@ -34,6 +34,9 @@ def content_missing_per_sha1(self, contents): return self.post('content/missing/sha1', {'contents': contents}) + def skipped_content_missing(self, contents): + return self.post('content/skipped/missing', {'contents': contents}) + def content_get(self, content): return self.post('content/data', {'content': content}) diff --git a/swh/storage/api/server.py b/swh/storage/api/server.py --- a/swh/storage/api/server.py +++ b/swh/storage/api/server.py @@ -149,6 +149,13 @@ **decode_request(request))) +@app.route('/content/skipped/missing', methods=['POST']) +@timed +def skipped_content_missing(): + return encode_data(get_storage().skipped_content_missing( + **decode_request(request))) + + @app.route('/content/present', methods=['POST']) @timed def content_find(): diff --git a/swh/storage/storage.py b/swh/storage/storage.py --- a/swh/storage/storage.py +++ b/swh/storage/storage.py @@ -476,7 +476,7 @@ yield obj[0] @db_transaction_generator() - def skipped_content_missing(self, content, db=None, cur=None): + def skipped_content_missing(self, contents, db=None, cur=None): """List skipped_content missing from storage Args: @@ -490,7 +490,7 @@ keys = db.content_hash_keys db.mktemp('skipped_content', cur) - db.copy_to(content, 'tmp_skipped_content', + db.copy_to(contents, 'tmp_skipped_content', keys + ['length', 'reason'], cur) yield from db.skipped_content_missing_from_temp(cur) diff --git a/swh/storage/tests/test_in_memory.py b/swh/storage/tests/test_in_memory.py --- a/swh/storage/tests/test_in_memory.py +++ b/swh/storage/tests/test_in_memory.py @@ -29,13 +29,18 @@ pass @pytest.mark.skip('postgresql-specific test') - def test_skipped_content_add(self): + def test_skipped_content_add_db(self): pass @pytest.mark.skip('postgresql-specific test') def test_content_add_metadata_db(self): pass + @pytest.mark.skip( + 'not implemented, see https://forge.softwareheritage.org/T1633') + def test_skipped_content_add(self): + pass + @pytest.mark.db @pytest.mark.property_based diff --git a/swh/storage/tests/test_storage.py b/swh/storage/tests/test_storage.py --- a/swh/storage/tests/test_storage.py +++ b/swh/storage/tests/test_storage.py @@ -729,7 +729,7 @@ self.assertIn(cm.exception.args[0], ['sha1', 'sha1_git', 'blake2s256']) - def test_skipped_content_add(self): + def test_skipped_content_add_db(self): cont = self.skipped_cont.copy() cont2 = self.skipped_cont2.copy() cont2['blake2s256'] = None @@ -767,6 +767,27 @@ 'Content too long') ) + def test_skipped_content_add(self): + cont = self.skipped_cont.copy() + cont2 = self.skipped_cont2.copy() + cont2['blake2s256'] = None + + missing = list(self.storage.skipped_content_missing([cont, cont2])) + + self.assertEqual(len(missing), 2, missing) + + actual_result = self.storage.content_add([cont, cont, cont2]) + + self.assertEqual(actual_result, { + 'content:add': 0, + 'content:bytes:add': 0, + 'skipped_content:add': 2, + }) + + missing = list(self.storage.skipped_content_missing([cont, cont2])) + + self.assertEqual(missing, []) + @pytest.mark.property_based @given(strategies.sets( elements=strategies.sampled_from(