Differential D95 Diff 320 swh/objstorage/objstorage.py

Changeset View

Standalone View

swh/objstorage/objstorage.py

	# Copyright (C) 2015-2016 The Software Heritage developers			# Copyright (C) 2015-2016 The Software Heritage developers
	# See the AUTHORS file at the top-level directory of this distribution			# See the AUTHORS file at the top-level directory of this distribution
	# License: GNU General Public License version 3, or any later version			# License: GNU General Public License version 3, or any later version
	# See top-level LICENSE file for more information			# See top-level LICENSE file for more information

				import abc

				from swh.core import hashutil

	from .exc import ObjNotFoundError			from .exc import ObjNotFoundError


	ID_HASH_ALGO = 'sha1'			ID_HASH_ALGO = 'sha1'
	ID_HASH_LENGTH = 40 # Size in bytes of the hash hexadecimal representation.			ID_HASH_LENGTH = 40 # Size in bytes of the hash hexadecimal representation.


	class ObjStorage():			def compute_hash(content):
				h = hashutil._new_hash(ID_HASH_ALGO, len(content))
				h.update(content)
				obj_id = h.digest()
				return obj_id
				zackUnsubmitted Not Done Inline Actions If we really don't have a helper like this one into hashutil, we probably should add one there, and use it here. There is no need to block this change due to this, but please check if this is the case. If it is, please submit a diff against swh.core to fix this. zack: If we really don't have a helper like this one into hashutil, we probably should add one there…
				olasdUnsubmitted Not Done Inline Actions That's `swh.core.hashutil.hashdata`. olasd: That's `swh.core.hashutil.hashdata`.
				qcamposAuthorUnsubmitted Not Done Inline Actions `swh.core.hashutil.hashdata` computes the hashcode for each algorithm (sha1, sha256, sha1_git). Should I use it anyway, or should I make a helper like `compute_hash(content, algorithm)` in `swh.core.hashutil`? qcampos: `swh.core.hashutil.hashdata` computes the hashcode for each algorithm (sha1, sha256, sha1_git).
				olasdUnsubmitted Done Inline Actions You can control which algos hashdata uses with an argument, that should be enough? olasd: You can control which algos hashdata uses with an argument, that should be enough?
				qcamposAuthorUnsubmitted Not Done Inline Actions Didn't see the default arg; that's exactly what's needed! qcampos: Didn't see the default arg; that's exactly what's needed!


				class ObjStorage(metaclass=abc.ABCMeta):
	""" High-level API to manipulate the Software Heritage object storage.			""" High-level API to manipulate the Software Heritage object storage.

	Conceptually, the object storage offers 5 methods:			Conceptually, the object storage offers 5 methods:

	- __contains__() check if an object is present, by object id			- __contains__() check if an object is present, by object id
	- add() add a new object, returning an object id			- add() add a new object, returning an object id
	- restore() same as add() but erase an already existed content			- restore() same as add() but erase an already existed content
	- get() retrieve the content of an object, by object id			- get() retrieve the content of an object, by object id
	- check() check the integrity of an object, by object id			- check() check the integrity of an object, by object id

	And some management methods:			And some management methods:

	- get_random() get random object id of existing contents (used for the			- get_random() get random object id of existing contents (used for the
	content integrity checker).			content integrity checker).

	Each implementation of this interface can have a different behavior and			Each implementation of this interface can have a different behavior and
	its own way to store the contents.			its own way to store the contents.
	"""			"""

	def __contains__(self, args, *kwargs):			@abc.abstractmethod
				def __contains__(self, obj_id, args, *kwargs):
				""" Indicates if the given object is present in the storage

				Returns:
				True iff the object is present in the current object storage.
				"""
	raise NotImplementedError(			raise NotImplementedError(
	"Implementations of ObjStorage must have a '__contains__' method"			"Implementations of ObjStorage must have a '__contains__' method"
	)			)

				@abc.abstractmethod
	def add(self, content, obj_id=None, check_presence=True, args, *kwargs):			def add(self, content, obj_id=None, check_presence=True, args, *kwargs):
	""" Add a new object to the object storage.			""" Add a new object to the object storage.

	Args:			Args:
	content: content of the object to be added to the storage.			content: content of the object to be added to the storage.
	obj_id: checksum of [bytes] using [ID_HASH_ALGO] algorithm. When			obj_id: checksum of [bytes] using [ID_HASH_ALGO] algorithm. When
	given, obj_id will be trusted to match the bytes. If missing,			given, obj_id will be trusted to match the bytes. If missing,
	obj_id will be computed on the fly.			obj_id will be computed on the fly.
	check_presence: indicate if the presence of the content should be			check_presence: indicate if the presence of the content should be
	verified before adding the file.			verified before adding the file.

	Returns:			Returns:
	the id of the object into the storage.			the id of the object into the storage.
	"""			"""
	raise NotImplementedError(			raise NotImplementedError(
	"Implementations of ObjStorage must have a 'add' method"			"Implementations of ObjStorage must have a 'add' method"
	)			)

	def restore(self, content, obj_id=None, args, *kwargs):			def restore(self, content, obj_id=None, args, *kwargs):
	""" Restore a content that have been corrupted.			""" Restore a content that have been corrupted.

	This function is identical to add_bytes but does not check if			This function is identical to add_bytes but does not check if
	the object id is already in the file system.			the object id is already in the file system.
				The default implementation provided by the current class is
				suitable for most cases.

	Args:			Args:
	content: content of the object to be added to the storage			content: content of the object to be added to the storage
	obj_id: checksums of `bytes` as computed by ID_HASH_ALGO. When			obj_id: checksums of `bytes` as computed by ID_HASH_ALGO. When
	given, obj_id will be trusted to match bytes. If missing,			given, obj_id will be trusted to match bytes. If missing,
	obj_id will be computed on the fly.			obj_id will be computed on the fly.
	"""			"""
	raise NotImplemented(			# check_presence to false will erase the potential previous content.
	"Implementations of ObjStorage must have a 'restore' method"			return self.add(content, obj_id, check_presence=False)
	)

				@abc.abstractmethod
	def get(self, obj_id, args, *kwargs):			def get(self, obj_id, args, *kwargs):
	""" Retrieve the content of a given object.			""" Retrieve the content of a given object.

	Args:			Args:
	obj_id: object id.			obj_id: object id.

	Returns:			Returns:
	the content of the requested object as bytes.			the content of the requested object as bytes.

	Raises:			Raises:
	ObjNotFoundError: if the requested object is missing.			ObjNotFoundError: if the requested object is missing.
	"""			"""
	raise NotImplementedError(			raise NotImplementedError(
	"Implementations of ObjStorage must have a 'get' method"			"Implementations of ObjStorage must have a 'get' method"
	)			)

	def get_batch(self, obj_ids, args, *kwargs):			def get_batch(self, obj_ids, args, *kwargs):
	""" Retrieve content in bulk.			""" Retrieve content in bulk.

	Note: This function does have a default implementation in ObjStorage			Note: This function does have a default implementation in ObjStorage
	that is suitable for most cases.			that is suitable for most cases.
				For object storages that needs to do the minimal number of requests
				possible (ex: remote object storages), that method can be overriden
				to perform a more efficient operation.

	Args:			Args:
	obj_ids: list of object ids.			obj_ids: list of object ids.

	Returns:			Returns:
	list of resulting contents, or None if the content could not			list of resulting contents, or None if the content could not
	be retrieved. Do not raise any exception as a fail for one content			be retrieved. Do not raise any exception as a fail for one content
	will not cancel the whole request.			will not cancel the whole request.
	"""			"""
	for obj_id in obj_ids:			for obj_id in obj_ids:
	try:			try:
	yield self.get(obj_id)			yield self.get(obj_id)
	except ObjNotFoundError:			except ObjNotFoundError:
	yield None			yield None

				@abc.abstractmethod
	def check(self, obj_id, args, *kwargs):			def check(self, obj_id, args, *kwargs):
	""" Perform an integrity check for a given object.			""" Perform an integrity check for a given object.

	Verify that the file object is in place and that the gziped content			Verify that the file object is in place and that the gziped content
	matches the object id.			matches the object id.

	Args:			Args:
	obj_id: object id.			obj_id: object id.

	Raises:			Raises:
	ObjNotFoundError: if the requested object is missing.			ObjNotFoundError: if the requested object is missing.
	Error: if the request object is corrupted.			Error: if the request object is corrupted.
	"""			"""
	raise NotImplementedError(			raise NotImplementedError(
	"Implementations of ObjStorage must have a 'check' method"			"Implementations of ObjStorage must have a 'check' method"
	)			)

	def get_random(self, batch_size, args, *kwargs):			def get_random(self, batch_size, args, *kwargs):
	""" Get random ids of existing contents			""" Get random ids of existing contents

	This method is used in order to get random ids to perform			This method is used in order to get random ids to perform
	content integrity verifications on random contents.			content integrity verifications on random contents.

	Attributes:			Args:
	batch_size (int): Number of ids that will be given			batch_size (int): Number of ids that will be given

	Yields:			Yields:
	An iterable of ids of contents that are in the current object			An iterable of ids of contents that are in the current object
	storage.			storage.
	"""			"""
	raise NotImplementedError(			raise NotImplementedError(
	"The current implementation of ObjStorage does not support "			"The current implementation of ObjStorage does not support "
	"'get_random' operation"			"'get_random' operation"
	)			)