Differential D2583 Diff 9247 swh/storage/in_memory.py

Changeset View

Standalone View

swh/storage/in_memory.py

Show First 20 Lines • Show All 194 Lines • ▼ Show 20 Lines	def content_add(self, content):
skipped_content:add: New skipped contents (no data) added		skipped_content:add: New skipped contents (no data) added

"""		"""
now = datetime.datetime.now(tz=datetime.timezone.utc)		now = datetime.datetime.now(tz=datetime.timezone.utc)
content = [attr.evolve(c, ctime=now)		content = [attr.evolve(c, ctime=now)
for c in self._content_to_model(content)]		for c in self._content_to_model(content)]
return self._content_add(content, with_data=True)		return self._content_add(content, with_data=True)

		def content_update(self, content, keys=[]):
		"""Update content blobs to the storage. Does nothing for unknown
		contents or skipped ones.

		Args:
		content (iterable): iterable of dictionaries representing
		individual pieces of content to update. Each dictionary has the
		following keys:

		- data (bytes): the actual content
		- length (int): content length (default: -1)
		- one key for each checksum algorithm in
		:data:`swh.model.hashutil.ALGORITHMS`, mapped to the
		corresponding checksum
		- status (str): one of visible, hidden, absent

		keys (list): List of keys (str) whose values needs an update, e.g.,
		new hash column
		"""
		if self.journal_writer:
		raise NotImplementedError(
		'content_update is not yet supported with a journal_writer.')

		for cont_update in content:
		cont_update = cont_update.copy()
		sha1 = cont_update.pop('sha1')
		for old_key in self._content_indexes['sha1'][sha1]:
		old_cont = self._contents.pop(old_key)

		for algorithm in DEFAULT_ALGORITHMS:
		hash_ = old_cont.get_hash(algorithm)
		self._content_indexes[algorithm][hash_].remove(old_key)

		new_cont = attr.evolve(old_cont, **cont_update)
		new_key = self._content_key(new_cont)

		self._contents[new_key] = new_cont

		for algorithm in DEFAULT_ALGORITHMS:
		hash_ = new_cont.get_hash(algorithm)
		self._content_indexes[algorithm][hash_].add(new_key)

def content_add_metadata(self, content):		def content_add_metadata(self, content):
"""Add content metadata to the storage (like `content_add`, but		"""Add content metadata to the storage (like `content_add`, but
without inserting to the objstorage).		without inserting to the objstorage).

Args:		Args:
content (iterable): iterable of dictionaries representing		content (iterable): iterable of dictionaries representing
individual pieces of content to add. Each dictionary has the		individual pieces of content to add. Each dictionary has the
following keys:		following keys:
▲ Show 20 Lines • Show All 153 Lines • ▼ Show 20 Lines	def content_get_metadata(
for sha1 in contents:		for sha1 in contents:
if sha1 in self._content_indexes['sha1']:		if sha1 in self._content_indexes['sha1']:
objs = self._content_indexes['sha1'][sha1]		objs = self._content_indexes['sha1'][sha1]
# only 1 element as content_add_metadata would have raised a		# only 1 element as content_add_metadata would have raised a
# hash collision otherwise		# hash collision otherwise
for key in objs:		for key in objs:
d = self._contents[key].to_dict()		d = self._contents[key].to_dict()
del d['ctime']		del d['ctime']
		del d['data']
result[sha1].append(d)		result[sha1].append(d)
return result		return result

def content_find(self, content):		def content_find(self, content):
if not set(content).intersection(DEFAULT_ALGORITHMS):		if not set(content).intersection(DEFAULT_ALGORITHMS):
raise ValueError('content keys must contain at least one of: '		raise ValueError('content keys must contain at least one of: '
'%s' % ', '.join(sorted(DEFAULT_ALGORITHMS)))		'%s' % ', '.join(sorted(DEFAULT_ALGORITHMS)))
found = []		found = []
▲ Show 20 Lines • Show All 1,495 Lines • Show Last 20 Lines