Differential D2343 Diff 8064 swh/storage/in_memory.py

Changeset View

Standalone View

swh/storage/in_memory.py

Show All 9 Lines
from collections import defaultdict		from collections import defaultdict
import copy		import copy
import datetime		import datetime
import itertools		import itertools
import random		import random

import attr		import attr

from swh.model.model import \		from swh.model.model import (
Content, Directory, Revision, Release, Snapshot, OriginVisit, Origin		Content, Directory, Revision, Release, Snapshot, OriginVisit, Origin,
from swh.model.hashutil import DEFAULT_ALGORITHMS		SHA1_SIZE)
		from swh.model.hashutil import DEFAULT_ALGORITHMS, hash_to_bytes, hash_to_hex
from swh.objstorage import get_objstorage		from swh.objstorage import get_objstorage
from swh.objstorage.exc import ObjNotFoundError		from swh.objstorage.exc import ObjNotFoundError

from .storage import get_journal_writer		from .storage import get_journal_writer
		from .utils import get_partition_bounds_bytes

# Max block size of contents to return		# Max block size of contents to return
BULK_BLOCK_CONTENT_LEN_MAX = 10000		BULK_BLOCK_CONTENT_LEN_MAX = 10000


def now():		def now():
return datetime.datetime.now(tz=datetime.timezone.utc)		return datetime.datetime.now(tz=datetime.timezone.utc)

▲ Show 20 Lines • Show All 263 Lines • ▼ Show 20 Lines	def content_get_range(self, start, end, limit=1000, db=None, cur=None):
next_content = sha1		next_content = sha1
break		break
matched.append(self._contents[key].to_dict())		matched.append(self._contents[key].to_dict())
return {		return {
'contents': matched,		'contents': matched,
'next': next_content,		'next': next_content,
}		}

		def content_get_partition(
		self, partition_id, nb_partitions, limit=1000, page_token=None):
		ardumontUnsubmitted Done Inline Actions type? ardumont: type?
		"""Splits contents into nb_partitions, and returns one of these based on
		partition_id (which must be between 0 and nb_partitions-1 inclusive)
		ardumontUnsubmitted Done Inline Actions `must be in [0, nb_partitions[` or `must be in [0, nb_partitions - 1]` ? ardumont: `must be in [0, nb_partitions[` or `must be in [0, nb_partitions - 1]` ?

		There is no guarantee on how the partitioning is done, or the order
		ardumontUnsubmitted Done Inline Actions or the result order. ardumont: or the result order.
		of results.

		Args:
		partition_id (int): index of the partition to fetch
		nb_partitions (int): total number of partitions to split into
		limit (int): Limit result (default to 1000)
		page_token (Optional[str]): opaque token used for pagination.

		Returns:
		a dict with keys:
		- contents (List[dict]): iterable of contents in the partition.
		- next_page_token (Optional[str]): opaque token to be used as
		`page_token` for retrieveing the next page. if absent, there is
		ardumontUnsubmitted Done Inline Actions retrieving ardumont: retrieving
		no more pages to gather.
		"""
		if limit is None:
		raise ValueError('Development error: limit should not be None')
		(start, end) = get_partition_bounds_bytes(
		partition_id, nb_partitions, SHA1_SIZE)
		if page_token:
		start = hash_to_bytes(page_token)
		if end is None:
		end = b'\xff'*SHA1_SIZE
		result = self.content_get_range(start, end, limit)
		result2 = {
		'contents': result['contents'],
		'next_page_token': None,
		}
		if result['next']:
		result2['next_page_token'] = hash_to_hex(result['next'])
		return result2

def content_get_metadata(self, content):		def content_get_metadata(self, content):
"""Retrieve content metadata in bulk		"""Retrieve content metadata in bulk

Args:		Args:
content: iterable of content identifiers (sha1)		content: iterable of content identifiers (sha1)

Returns:		Returns:
an iterable with content metadata corresponding to the given ids		an iterable with content metadata corresponding to the given ids
▲ Show 20 Lines • Show All 1,378 Lines • Show Last 20 Lines