diff --git a/.coverage.paris.28594.210279 b/.coverage.paris.28594.210279 new file mode 100644 --- /dev/null +++ b/.coverage.paris.28594.210279 @@ -0,0 +1 @@ +!coverage.py: This is a private format, don't read it directly!{"arcs":{"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/__init__.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/cli.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/storage.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/converters.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/common.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/in_memory.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/listener.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/journal_writer.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/__init__.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/db.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/exc.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/schemata/__init__.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/schemata/distribution.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/api/server.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/api/__init__.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/api/client.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/api/wsgi.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/tests/storage_testing.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/tests/test_listener.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/tests/generate_data_test.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/tests/test_server.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/tests/test_api_client.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/tests/test_storage.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/tests/__init__.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/tests/test_converters.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/tests/test_in_memory.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/tests/test_db.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/tests/algos/test_snapshot.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/tests/algos/test_revisions_walker.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/tests/algos/test_diff.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/tests/algos/test_origin.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/tests/algos/test_dir_iterator.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/tests/algos/__init__.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/algos/diff.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/algos/__init__.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/algos/dir_iterators.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/algos/origin.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/algos/snapshot.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/algos/revisions_walker.py":[]}} \ No newline at end of file diff --git a/.coverage.paris.28623.351166 b/.coverage.paris.28623.351166 new file mode 100644 --- /dev/null +++ b/.coverage.paris.28623.351166 @@ -0,0 +1 @@ +!coverage.py: This is a private format, don't read it directly!{"arcs":{"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/__init__.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/cli.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/storage.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/converters.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/common.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/in_memory.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/listener.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/journal_writer.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/__init__.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/db.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/exc.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/schemata/__init__.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/schemata/distribution.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/api/server.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/api/__init__.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/api/client.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/api/wsgi.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/tests/storage_testing.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/tests/test_listener.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/tests/generate_data_test.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/tests/test_server.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/tests/test_api_client.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/tests/test_storage.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/tests/__init__.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/tests/test_converters.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/tests/test_in_memory.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/tests/test_db.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/tests/algos/test_snapshot.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/tests/algos/test_revisions_walker.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/tests/algos/test_diff.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/tests/algos/test_origin.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/tests/algos/test_dir_iterator.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/tests/algos/__init__.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/algos/diff.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/algos/__init__.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/algos/dir_iterators.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/algos/origin.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/algos/snapshot.py":[],"/home/nightstalker/Documents/cloned_repo/swh-storage/swh/storage/algos/revisions_walker.py":[]}} \ No newline at end of file diff --git a/swh/storage/in_memory.py b/swh/storage/in_memory.py --- a/swh/storage/in_memory.py +++ b/swh/storage/in_memory.py @@ -74,6 +74,7 @@ for content in contents: key = self._content_key(content) if key in self._contents: + self._objects[key].append(('skipped_content', content['sha1'])) continue for algorithm in DEFAULT_ALGORITHMS: if content[algorithm] in self._content_indexes[algorithm]: @@ -169,7 +170,7 @@ """ return self._content_add(contents, with_data=False) - def content_get(self, ids): + def content_get(self, ids, start=0, end=-1): """Retrieve in bulk contents and their data. This function may yield more blobs than provided sha1 identifiers, @@ -1170,7 +1171,8 @@ 'origin': origin_id, 'visit': visit_id, } - + key = (origin_id, visit_id) + self._objects[key].append(('origin_visit', origin_id)) if self.journal_writer: origin = self.origin_get([{'id': origin_id}])[0] del origin['id'] @@ -1514,12 +1516,12 @@ if key not in self._objects: person_id = len(self._persons) + 1 self._persons.append(dict(person)) - self._objects[key].append(('person', person_id)) else: person_id = self._objects[key][0][1] p = next(self.person_get([person_id])) person.update(p.items()) person['id'] = person_id + self._objects[key].append(('person', person_id)) @staticmethod def _content_key(content): diff --git a/swh/storage/storage.py b/swh/storage/storage.py --- a/swh/storage/storage.py +++ b/swh/storage/storage.py @@ -334,7 +334,7 @@ return summary - def content_get(self, content): + def content_get(self, content, start=0, end=-1): """Retrieve in bulk contents and their data. This generator yields exactly as many items than provided sha1 @@ -369,7 +369,10 @@ yield None continue - yield {'sha1': obj_id, 'data': data} + if end != -1: + yield {'sha1': obj_id, 'data': data[start:end]} + else: + yield {'sha1': obj_id, 'data': data} @db_transaction() def content_get_range(self, start, end, limit=1000, db=None, cur=None):