diff --git a/sql/swh-func.sql b/sql/swh-func.sql
--- a/sql/swh-func.sql
+++ b/sql/swh-func.sql
@@ -307,6 +307,35 @@
$$;
+-- Update content entries from temporary table.
+-- (columns are potential new columns added to the schema, this can be empty)
+--
+create or replace function swh_content_update(columns_update text[])
+ returns void
+ language plpgsql
+as $$
+declare
+ query text;
+ tmp_array text[];
+begin
+ if array_length(columns_update, 1) = 0 then
+ raise exception 'Please, provide the list of column names to update.';
+ end if;
+
+ tmp_array := array(select format('%1$s=t.%1$s', unnest) from unnest(columns_update));
+
+ query = format('update content set %s
+ from tmp_content t where t.sha1 = content.sha1',
+ array_to_string(tmp_array, ', '));
+
+ execute query;
+
+ return;
+end
+$$;
+
+comment on function swh_content_update(text[]) IS 'Update existing content''s columns';
+
-- check which entries of tmp_directory are missing from directory
--
-- operates in bulk: 0. swh_mktemp(directory), 1. COPY to tmp_directory,
diff --git a/sql/swh-schema.sql b/sql/swh-schema.sql
--- a/sql/swh-schema.sql
+++ b/sql/swh-schema.sql
@@ -14,7 +14,7 @@
);
insert into dbversion(version, release, description)
- values(100, now(), 'Work In Progress');
+ values(101, now(), 'Work In Progress');
-- a SHA1 checksum (not necessarily originating from Git)
create domain sha1 as bytea check (length(value) = 20);
diff --git a/sql/upgrades/101.sql b/sql/upgrades/101.sql
new file mode 100644
--- /dev/null
+++ b/sql/upgrades/101.sql
@@ -0,0 +1,36 @@
+-- SWH DB schema upgrade
+-- from_version: 100
+-- to_version: 101
+-- description: Open swh_content_update function
+
+insert into dbversion(version, release, description)
+ values(101, now(), 'Work In Progress');
+
+-- Update content entries from temporary table.
+-- (columns are potential new columns added to the schema, this can be empty)
+--
+create or replace function swh_content_update(columns_update text[])
+ returns void
+ language plpgsql
+as $$
+declare
+ query text;
+ tmp_array text[];
+begin
+ if array_length(columns_update, 1) = 0 then
+ raise exception 'Please, provide the list of column names to update.';
+ end if;
+
+ tmp_array := array(select format('%1$s=t.%1$s', unnest) from unnest(columns_update));
+
+ query = format('update content set %s
+ from tmp_content t where t.sha1 = content.sha1',
+ array_to_string(tmp_array, ', '));
+
+ execute query;
+
+ return;
+end
+$$;
+
+comment on function swh_content_update(text[]) IS 'Update existing content''s columns';
diff --git a/swh/storage/api/client.py b/swh/storage/api/client.py
--- a/swh/storage/api/client.py
+++ b/swh/storage/api/client.py
@@ -20,6 +20,10 @@
def content_add(self, content):
return self.post('content/add', {'content': content})
+ def content_update(self, content, keys=[]):
+ return self.post('content/update', {'content': content,
+ 'keys': keys})
+
def content_missing(self, content, key_hash='sha1'):
return self.post('content/missing', {'content': content,
'key_hash': key_hash})
diff --git a/swh/storage/api/server.py b/swh/storage/api/server.py
--- a/swh/storage/api/server.py
+++ b/swh/storage/api/server.py
@@ -82,6 +82,11 @@
return encode_data(g.storage.content_add(**decode_request(request)))
+@app.route('/content/update', methods=['POST'])
+def content_update():
+ return encode_data(g.storage.content_update(**decode_request(request)))
+
+
@app.route('/content/data', methods=['POST'])
def content_get():
return encode_data(g.storage.content_get(**decode_request(request)))
diff --git a/swh/storage/db.py b/swh/storage/db.py
--- a/swh/storage/db.py
+++ b/swh/storage/db.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2015-2016 The Software Heritage developers
+# Copyright (C) 2015-2017 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
@@ -263,6 +263,11 @@
self.copy_to(({'id': elem} for elem in ids), 'tmp_bytea',
['id'], cur)
+ def content_update_from_temp(self, keys_to_update, cur=None):
+ cur = self._cursor(cur)
+ cur.execute("""select swh_content_update(ARRAY[%s] :: text[])""" %
+ keys_to_update)
+
content_get_metadata_keys = ['sha1', 'sha1_git', 'sha256', 'length',
'status']
diff --git a/swh/storage/storage.py b/swh/storage/storage.py
--- a/swh/storage/storage.py
+++ b/swh/storage/storage.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2015-2016 The Software Heritage developers
+# Copyright (C) 2015-2017 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
@@ -135,6 +135,42 @@
# move metadata in place
db.skipped_content_add_from_temp(cur)
+ @db_transaction
+ def content_update(self, content, keys=[], cur=None):
+ """Update content blobs to the storage.
+
+ Note:
+ It's up to the content_update client caller to:
+ - only send existing contents for now.
+ - existing contents that are not skipped ones.
+
+ Args:
+ content: iterable of dictionaries representing individual pieces of
+ content to update. Each dictionary has the following keys:
+ - data (bytes): the actual content
+ - length (int): content length (default: -1)
+ - one key for each checksum algorithm in
+ swh.core.hashutil.ALGORITHMS, mapped to the corresponding
+ checksum
+ - status (str): one of visible, hidden, absent
+
+ keys ([str]): List of keys whose values needs an update (
+ e.g. new hash column)
+
+ """
+ db = self.db
+
+ # TODO: Add a check on input keys. How to properly implement
+ # this? We don't know yet the new columns.
+ # TODO2: Filter out unknown contents?
+ # TODO3: Filter out skipped contents?
+
+ db.mktemp('content')
+ select_keys = list(set(db.content_get_metadata_keys).union(set(keys)))
+ db.copy_to(content, 'tmp_content', select_keys, cur)
+ db.content_update_from_temp(keys_to_update=keys,
+ cur=cur)
+
def content_get(self, content):
"""Retrieve in bulk contents and their data.
diff --git a/swh/storage/tests/test_storage.py b/swh/storage/tests/test_storage.py
--- a/swh/storage/tests/test_storage.py
+++ b/swh/storage/tests/test_storage.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2015-2016 The Software Heritage developers
+# Copyright (C) 2015-2017 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
@@ -3040,3 +3040,60 @@
'email': person1['email'],
},
])
+
+
+class AlteringSchemaTest(AbstractTestStorage, unittest.TestCase):
+ """This class is dedicated for the rare case where the schema needs to
+ be altered dynamically.
+
+ Otherwise, the tests could be blocking when ran altogether.
+
+ """
+ @istest
+ def content_update(self):
+ cont = self.cont
+
+ self.storage.content_add([cont])
+ # alter the sha1_git for example
+ cont['sha1_git'] = hex_to_hash(
+ '3a60a5275d0333bf13468e8b3dcab90f4046e654')
+
+ self.storage.content_update([cont], keys=['sha1_git'])
+
+ self.cursor.execute('SELECT sha1, sha1_git, sha256, length, status'
+ ' FROM content WHERE sha1 = %s',
+ (cont['sha1'],))
+ datum = self.cursor.fetchone()
+ self.assertEqual(
+ (datum[0].tobytes(), datum[1].tobytes(), datum[2].tobytes(),
+ datum[3], datum[4]),
+ (cont['sha1'], cont['sha1_git'], cont['sha256'],
+ cont['length'], 'visible'))
+
+ @istest
+ def content_update_with_new_cols(self):
+ self.cursor.execute("""alter table content
+ add column test text default null,
+ add column test2 text default null""")
+
+ cont = self.cont2
+ self.storage.content_add([cont])
+ cont['test'] = 'value-1'
+ cont['test2'] = 'value-2'
+
+ self.storage.content_update([cont], keys=['test', 'test2'])
+
+ self.cursor.execute(
+ 'SELECT sha1, sha1_git, sha256, length, status, test, test2'
+ ' FROM content WHERE sha1 = %s',
+ (cont['sha1'],))
+
+ datum = self.cursor.fetchone()
+ self.assertEqual(
+ (datum[0].tobytes(), datum[1].tobytes(), datum[2].tobytes(),
+ datum[3], datum[4], datum[5], datum[6]),
+ (cont['sha1'], cont['sha1_git'], cont['sha256'],
+ cont['length'], 'visible', cont['test'], cont['test2']))
+
+ self.cursor.execute("""alter table content drop column test,
+ drop column test2""")