Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/in_memory.py
Show All 11 Lines | |||||
import copy | import copy | ||||
import datetime | import datetime | ||||
import itertools | import itertools | ||||
import random | import random | ||||
import warnings | import warnings | ||||
import attr | import attr | ||||
from swh.model.model import Content, Directory | from swh.model.model import Content, Directory, Revision, Release | ||||
from swh.model.hashutil import DEFAULT_ALGORITHMS | from swh.model.hashutil import DEFAULT_ALGORITHMS | ||||
from swh.model.identifiers import normalize_timestamp | |||||
from swh.objstorage import get_objstorage | from swh.objstorage import get_objstorage | ||||
from swh.objstorage.exc import ObjNotFoundError | from swh.objstorage.exc import ObjNotFoundError | ||||
from .journal_writer import get_journal_writer | from .journal_writer import get_journal_writer | ||||
# Max block size of contents to return | # Max block size of contents to return | ||||
BULK_BLOCK_CONTENT_LEN_MAX = 10000 | BULK_BLOCK_CONTENT_LEN_MAX = 10000 | ||||
▲ Show 20 Lines • Show All 551 Lines • ▼ Show 20 Lines | def revision_add(self, revisions): | ||||
Summary dict of keys with associated count as values | Summary dict of keys with associated count as values | ||||
revision_added: New objects actually stored in db | revision_added: New objects actually stored in db | ||||
""" | """ | ||||
if self.journal_writer: | if self.journal_writer: | ||||
self.journal_writer.write_additions('revision', revisions) | self.journal_writer.write_additions('revision', revisions) | ||||
revisions = [Revision.from_dict(rev) for rev in revisions] | |||||
count = 0 | count = 0 | ||||
for revision in revisions: | for revision in revisions: | ||||
if revision['id'] not in self._revisions: | if revision.id not in self._revisions: | ||||
self._revisions[revision['id']] = rev = copy.deepcopy(revision) | revision.committer = self._person_add(revision.committer) | ||||
self._person_add(rev['committer']) | revision.author = self._person_add(revision.author) | ||||
self._person_add(rev['author']) | self._revisions[revision.id] = revision | ||||
rev['date'] = normalize_timestamp(rev.get('date')) | self._objects[revision.id].append( | ||||
rev['committer_date'] = normalize_timestamp( | ('revision', revision.id)) | ||||
rev.get('committer_date')) | |||||
self._objects[revision['id']].append( | |||||
('revision', revision['id'])) | |||||
count += 1 | count += 1 | ||||
return {'revision:add': count} | return {'revision:add': count} | ||||
def revision_missing(self, revisions): | def revision_missing(self, revisions): | ||||
"""List revisions missing from storage | """List revisions missing from storage | ||||
Args: | Args: | ||||
revisions (iterable): revision ids | revisions (iterable): revision ids | ||||
Yields: | Yields: | ||||
missing revision ids | missing revision ids | ||||
""" | """ | ||||
for id in revisions: | for id in revisions: | ||||
if id not in self._revisions: | if id not in self._revisions: | ||||
yield id | yield id | ||||
def revision_get(self, revisions): | def revision_get(self, revisions): | ||||
for id in revisions: | for id in revisions: | ||||
yield copy.deepcopy(self._revisions.get(id)) | if id in self._revisions: | ||||
yield self._revisions.get(id).to_dict() | |||||
else: | |||||
yield None | |||||
def _get_parent_revs(self, rev_id, seen, limit): | def _get_parent_revs(self, rev_id, seen, limit): | ||||
if limit and len(seen) >= limit: | if limit and len(seen) >= limit: | ||||
return | return | ||||
if rev_id in seen or rev_id not in self._revisions: | if rev_id in seen or rev_id not in self._revisions: | ||||
return | return | ||||
seen.add(rev_id) | seen.add(rev_id) | ||||
yield self._revisions[rev_id] | yield self._revisions[rev_id].to_dict() | ||||
for parent in self._revisions[rev_id]['parents']: | for parent in self._revisions[rev_id].parents: | ||||
yield from self._get_parent_revs(parent, seen, limit) | yield from self._get_parent_revs(parent, seen, limit) | ||||
def revision_log(self, revisions, limit=None): | def revision_log(self, revisions, limit=None): | ||||
"""Fetch revision entry from the given root revisions. | """Fetch revision entry from the given root revisions. | ||||
Args: | Args: | ||||
revisions: array of root revision to lookup | revisions: array of root revision to lookup | ||||
limit: limitation on the output result. Default to None. | limit: limitation on the output result. Default to None. | ||||
▲ Show 20 Lines • Show All 44 Lines • ▼ Show 20 Lines | def release_add(self, releases): | ||||
Summary dict of keys with associated count as values | Summary dict of keys with associated count as values | ||||
release:add: New objects contents actually stored in db | release:add: New objects contents actually stored in db | ||||
""" | """ | ||||
if self.journal_writer: | if self.journal_writer: | ||||
self.journal_writer.write_additions('release', releases) | self.journal_writer.write_additions('release', releases) | ||||
releases = [Release.from_dict(rel) for rel in releases] | |||||
count = 0 | count = 0 | ||||
for rel in releases: | for rel in releases: | ||||
if rel['id'] not in self._releases: | if rel.id not in self._releases: | ||||
rel = copy.deepcopy(rel) | if rel.author: | ||||
rel['date'] = normalize_timestamp(rel['date']) | self._person_add(rel.author) | ||||
if rel['author']: | self._objects[rel.id].append( | ||||
self._person_add(rel['author']) | ('release', rel.id)) | ||||
self._objects[rel['id']].append( | self._releases[rel.id] = rel | ||||
('release', rel['id'])) | |||||
self._releases[rel['id']] = rel | |||||
count += 1 | count += 1 | ||||
return {'release:add': count} | return {'release:add': count} | ||||
def release_missing(self, releases): | def release_missing(self, releases): | ||||
"""List releases missing from storage | """List releases missing from storage | ||||
Args: | Args: | ||||
Show All 12 Lines | def release_get(self, releases): | ||||
releases: list of sha1s | releases: list of sha1s | ||||
Yields: | Yields: | ||||
dicts with the same keys as those given to `release_add` | dicts with the same keys as those given to `release_add` | ||||
(or ``None`` if a release does not exist) | (or ``None`` if a release does not exist) | ||||
""" | """ | ||||
for rel_id in releases: | for rel_id in releases: | ||||
yield copy.deepcopy(self._releases.get(rel_id)) | if rel_id in self._releases: | ||||
yield self._releases[rel_id].to_dict() | |||||
else: | |||||
yield None | |||||
def snapshot_add(self, snapshots): | def snapshot_add(self, snapshots): | ||||
"""Add a snapshot to the storage | """Add a snapshot to the storage | ||||
Args: | Args: | ||||
snapshot ([dict]): the snapshots to add, containing the | snapshot ([dict]): the snapshots to add, containing the | ||||
following keys: | following keys: | ||||
▲ Show 20 Lines • Show All 943 Lines • ▼ Show 20 Lines | def _person_add(self, person): | ||||
"""Add a person in storage. | """Add a person in storage. | ||||
Note: Private method, do not use outside of this class. | Note: Private method, do not use outside of this class. | ||||
Args: | Args: | ||||
person: dictionary with keys fullname, name and email. | person: dictionary with keys fullname, name and email. | ||||
""" | """ | ||||
key = ('person', person['fullname']) | key = ('person', person.fullname) | ||||
if key not in self._objects: | if key not in self._objects: | ||||
person_id = len(self._persons) + 1 | person_id = len(self._persons) + 1 | ||||
self._persons.append(dict(person)) | self._persons.append(person) | ||||
ardumont: person.to_dict()? | |||||
Done Inline ActionsNope, it stores instances of Person, like for all other objects. vlorentz: Nope, it stores instances of `Person`, like for all other objects. | |||||
self._objects[key].append(('person', person_id)) | self._objects[key].append(('person', person_id)) | ||||
else: | else: | ||||
person_id = self._objects[key][0][1] | person_id = self._objects[key][0][1] | ||||
p = self._persons[person_id-1] | person = self._persons[person_id-1] | ||||
person.update(p.items()) | return person | ||||
@staticmethod | @staticmethod | ||||
def _content_key(content): | def _content_key(content): | ||||
"""A stable key for a content""" | """A stable key for a content""" | ||||
return tuple(getattr(content, key) | return tuple(getattr(content, key) | ||||
for key in sorted(DEFAULT_ALGORITHMS)) | for key in sorted(DEFAULT_ALGORITHMS)) | ||||
@staticmethod | @staticmethod | ||||
Show All 15 Lines |
person.to_dict()?