Differential D1874 Diff 6459 swh/storage/in_memory.py

Changeset View

Standalone View

swh/storage/in_memory.py

Show All 11 Lines
import copy		import copy
import datetime		import datetime
import itertools		import itertools
import random		import random
import warnings		import warnings

import attr		import attr

from swh.model.model import Content, Directory		from swh.model.model import Content, Directory, Revision, Release
from swh.model.hashutil import DEFAULT_ALGORITHMS		from swh.model.hashutil import DEFAULT_ALGORITHMS
from swh.model.identifiers import normalize_timestamp
from swh.objstorage import get_objstorage		from swh.objstorage import get_objstorage
from swh.objstorage.exc import ObjNotFoundError		from swh.objstorage.exc import ObjNotFoundError

from .journal_writer import get_journal_writer		from .journal_writer import get_journal_writer

# Max block size of contents to return		# Max block size of contents to return
BULK_BLOCK_CONTENT_LEN_MAX = 10000		BULK_BLOCK_CONTENT_LEN_MAX = 10000

▲ Show 20 Lines • Show All 551 Lines • ▼ Show 20 Lines	def revision_add(self, revisions):
Summary dict of keys with associated count as values		Summary dict of keys with associated count as values

revision_added: New objects actually stored in db		revision_added: New objects actually stored in db

"""		"""
if self.journal_writer:		if self.journal_writer:
self.journal_writer.write_additions('revision', revisions)		self.journal_writer.write_additions('revision', revisions)

		revisions = [Revision.from_dict(rev) for rev in revisions]

count = 0		count = 0
for revision in revisions:		for revision in revisions:
if revision['id'] not in self._revisions:		if revision.id not in self._revisions:
self._revisions[revision['id']] = rev = copy.deepcopy(revision)		revision.committer = self._person_add(revision.committer)
self._person_add(rev['committer'])		revision.author = self._person_add(revision.author)
self._person_add(rev['author'])		self._revisions[revision.id] = revision
rev['date'] = normalize_timestamp(rev.get('date'))		self._objects[revision.id].append(
rev['committer_date'] = normalize_timestamp(		('revision', revision.id))
rev.get('committer_date'))
self._objects[revision['id']].append(
('revision', revision['id']))
count += 1		count += 1

return {'revision:add': count}		return {'revision:add': count}

def revision_missing(self, revisions):		def revision_missing(self, revisions):
"""List revisions missing from storage		"""List revisions missing from storage

Args:		Args:
revisions (iterable): revision ids		revisions (iterable): revision ids

Yields:		Yields:
missing revision ids		missing revision ids

"""		"""
for id in revisions:		for id in revisions:
if id not in self._revisions:		if id not in self._revisions:
yield id		yield id

def revision_get(self, revisions):		def revision_get(self, revisions):
for id in revisions:		for id in revisions:
yield copy.deepcopy(self._revisions.get(id))		if id in self._revisions:
		yield self._revisions.get(id).to_dict()
		else:
		yield None

def _get_parent_revs(self, rev_id, seen, limit):		def _get_parent_revs(self, rev_id, seen, limit):
if limit and len(seen) >= limit:		if limit and len(seen) >= limit:
return		return
if rev_id in seen or rev_id not in self._revisions:		if rev_id in seen or rev_id not in self._revisions:
return		return
seen.add(rev_id)		seen.add(rev_id)
yield self._revisions[rev_id]		yield self._revisions[rev_id].to_dict()
for parent in self._revisions[rev_id]['parents']:		for parent in self._revisions[rev_id].parents:
yield from self._get_parent_revs(parent, seen, limit)		yield from self._get_parent_revs(parent, seen, limit)

def revision_log(self, revisions, limit=None):		def revision_log(self, revisions, limit=None):
"""Fetch revision entry from the given root revisions.		"""Fetch revision entry from the given root revisions.

Args:		Args:
revisions: array of root revision to lookup		revisions: array of root revision to lookup
limit: limitation on the output result. Default to None.		limit: limitation on the output result. Default to None.
▲ Show 20 Lines • Show All 44 Lines • ▼ Show 20 Lines	def release_add(self, releases):
Summary dict of keys with associated count as values		Summary dict of keys with associated count as values

release:add: New objects contents actually stored in db		release:add: New objects contents actually stored in db

"""		"""
if self.journal_writer:		if self.journal_writer:
self.journal_writer.write_additions('release', releases)		self.journal_writer.write_additions('release', releases)

		releases = [Release.from_dict(rel) for rel in releases]

count = 0		count = 0
for rel in releases:		for rel in releases:
if rel['id'] not in self._releases:		if rel.id not in self._releases:
rel = copy.deepcopy(rel)		if rel.author:
rel['date'] = normalize_timestamp(rel['date'])		self._person_add(rel.author)
if rel['author']:		self._objects[rel.id].append(
self._person_add(rel['author'])		('release', rel.id))
self._objects[rel['id']].append(		self._releases[rel.id] = rel
('release', rel['id']))
self._releases[rel['id']] = rel
count += 1		count += 1

return {'release:add': count}		return {'release:add': count}

def release_missing(self, releases):		def release_missing(self, releases):
"""List releases missing from storage		"""List releases missing from storage

Args:		Args:
Show All 12 Lines	def release_get(self, releases):
releases: list of sha1s		releases: list of sha1s

Yields:		Yields:
dicts with the same keys as those given to `release_add`		dicts with the same keys as those given to `release_add`
(or ``None`` if a release does not exist)		(or ``None`` if a release does not exist)

"""		"""
for rel_id in releases:		for rel_id in releases:
yield copy.deepcopy(self._releases.get(rel_id))		if rel_id in self._releases:
		yield self._releases[rel_id].to_dict()
		else:
		yield None

def snapshot_add(self, snapshots):		def snapshot_add(self, snapshots):
"""Add a snapshot to the storage		"""Add a snapshot to the storage

Args:		Args:
snapshot ([dict]): the snapshots to add, containing the		snapshot ([dict]): the snapshots to add, containing the
following keys:		following keys:

▲ Show 20 Lines • Show All 943 Lines • ▼ Show 20 Lines	def _person_add(self, person):
"""Add a person in storage.		"""Add a person in storage.

Note: Private method, do not use outside of this class.		Note: Private method, do not use outside of this class.

Args:		Args:
person: dictionary with keys fullname, name and email.		person: dictionary with keys fullname, name and email.

"""		"""
key = ('person', person['fullname'])		key = ('person', person.fullname)
if key not in self._objects:		if key not in self._objects:
person_id = len(self._persons) + 1		person_id = len(self._persons) + 1
self._persons.append(dict(person))		self._persons.append(person)
		ardumontUnsubmitted Not Done Inline Actions person.to_dict()? ardumont: person.to_dict()?
		vlorentzAuthorUnsubmitted Done Inline Actions Nope, it stores instances of `Person`, like for all other objects. vlorentz: Nope, it stores instances of `Person`, like for all other objects.
self._objects[key].append(('person', person_id))		self._objects[key].append(('person', person_id))
else:		else:
person_id = self._objects[key][0][1]		person_id = self._objects[key][0][1]
p = self._persons[person_id-1]		person = self._persons[person_id-1]
person.update(p.items())		return person

@staticmethod		@staticmethod
def _content_key(content):		def _content_key(content):
"""A stable key for a content"""		"""A stable key for a content"""
return tuple(getattr(content, key)		return tuple(getattr(content, key)
for key in sorted(DEFAULT_ALGORITHMS))		for key in sorted(DEFAULT_ALGORITHMS))

@staticmethod		@staticmethod
Show All 15 Lines