diff --git a/sql/upgrades/139.sql b/sql/upgrades/139.sql new file mode 100644 --- /dev/null +++ b/sql/upgrades/139.sql @@ -0,0 +1,19 @@ +-- SWH DB schema upgrade +-- from_version: 138 +-- to_version: 139 +-- description: Make fields target/name/perms of directory_entry_{dir,file,rev} not-null + +insert into dbversion(version, release, description) + values(139, now(), 'Work In Progress'); + +alter table directory_entry_dir alter column target set not null; +alter table directory_entry_dir alter column name set not null; +alter table directory_entry_dir alter column perms set not null; + +alter table directory_entry_file alter column target set not null; +alter table directory_entry_file alter column name set not null; +alter table directory_entry_file alter column perms set not null; + +alter table directory_entry_rev alter column target set not null; +alter table directory_entry_rev alter column name set not null; +alter table directory_entry_rev alter column perms set not null; diff --git a/swh/storage/in_memory.py b/swh/storage/in_memory.py --- a/swh/storage/in_memory.py +++ b/swh/storage/in_memory.py @@ -17,7 +17,7 @@ import attr -from swh.model.model import Content +from swh.model.model import Content, Directory from swh.model.hashutil import DEFAULT_ALGORITHMS from swh.model.identifiers import normalize_timestamp from swh.objstorage import get_objstorage @@ -435,13 +435,15 @@ if self.journal_writer: self.journal_writer.write_additions('directory', directories) + directories = [Directory.from_dict(d) for d in directories] + count = 0 for directory in directories: - if directory['id'] not in self._directories: + if directory.id not in self._directories: count += 1 - self._directories[directory['id']] = copy.deepcopy(directory) - self._objects[directory['id']].append( - ('directory', directory['id'])) + self._directories[directory.id] = directory + self._objects[directory.id].append( + ('directory', directory.id)) return {'directory:add': count} @@ -480,8 +482,8 @@ def _directory_ls(self, directory_id, recursive, prefix=b''): if directory_id in self._directories: - for entry in self._directories[directory_id]['entries']: - ret = self._join_dentry_to_content(entry) + for entry in self._directories[directory_id].entries: + ret = self._join_dentry_to_content(entry.to_dict()) ret['name'] = prefix + ret['name'] ret['dir_id'] = directory_id yield ret diff --git a/swh/storage/sql/30-swh-schema.sql b/swh/storage/sql/30-swh-schema.sql --- a/swh/storage/sql/30-swh-schema.sql +++ b/swh/storage/sql/30-swh-schema.sql @@ -17,7 +17,7 @@ -- latest schema version insert into dbversion(version, release, description) - values(138, now(), 'Work In Progress'); + values(139, now(), 'Work In Progress'); -- a SHA1 checksum create domain sha1 as bytea check (length(value) = 20); @@ -176,9 +176,9 @@ create table directory_entry_dir ( id bigserial, - target sha1_git, -- id of target directory - name unix_path, -- path name, relative to containing dir - perms file_perms -- unix-like permissions + target sha1_git not null, -- id of target directory + name unix_path not null, -- path name, relative to containing dir + perms file_perms not null -- unix-like permissions ); comment on table directory_entry_dir is 'Directory entry for directory'; @@ -192,9 +192,9 @@ create table directory_entry_file ( id bigserial, - target sha1_git, -- id of target file - name unix_path, -- path name, relative to containing dir - perms file_perms -- unix-like permissions + target sha1_git not null, -- id of target file + name unix_path not null, -- path name, relative to containing dir + perms file_perms not null -- unix-like permissions ); comment on table directory_entry_file is 'Directory entry for file'; @@ -208,9 +208,9 @@ create table directory_entry_rev ( id bigserial, - target sha1_git, -- id of target revision - name unix_path, -- path name, relative to containing dir - perms file_perms -- unix-like permissions + target sha1_git not null, -- id of target revision + name unix_path not null, -- path name, relative to containing dir + perms file_perms not null -- unix-like permissions ); comment on table directory_entry_rev is 'Directory entry for revision'; diff --git a/swh/storage/storage.py b/swh/storage/storage.py --- a/swh/storage/storage.py +++ b/swh/storage/storage.py @@ -597,6 +597,10 @@ for src_entry in cur_dir['entries']: entry = src_entry.copy() entry['dir_id'] = dir_id + if entry['type'] not in ('file', 'dir', 'rev'): + raise ValueError( + 'Entry type must be file, dir, or rev; not %s' + % entry['type']) dir_entries[entry['type']][dir_id].append(entry) dirs_missing = set(self.directory_missing(dirs, db=db, cur=cur)) diff --git a/swh/storage/tests/test_storage.py b/swh/storage/tests/test_storage.py --- a/swh/storage/tests/test_storage.py +++ b/swh/storage/tests/test_storage.py @@ -1007,6 +1007,20 @@ after_missing = list(self.storage.directory_missing([self.dir['id']])) self.assertEqual([], after_missing) + def test_directory_add_validation(self): + dir_ = copy.deepcopy(self.dir) + dir_['entries'][0]['type'] = 'foobar' + + with self.assertRaisesRegex(ValueError, 'type.*foobar'): + self.storage.directory_add([dir_]) + + dir_ = copy.deepcopy(self.dir) + del dir_['entries'][0]['target'] + + with self.assertRaisesRegex( + (TypeError, psycopg2.errors.NotNullViolation), 'target'): + self.storage.directory_add([dir_]) + def test_directory_get_recursive(self): init_missing = list(self.storage.directory_missing([self.dir['id']])) self.assertEqual([self.dir['id']], init_missing)