Differential D95 Diff 330 swh/objstorage/objstorage.py

Changeset View

Standalone View

swh/objstorage/objstorage.py

# Copyright (C) 2015-2016 The Software Heritage developers		# Copyright (C) 2015-2016 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution		# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version		# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information		# See top-level LICENSE file for more information

from .exc import ObjNotFoundError		from .exc import ObjNotFoundError


ID_HASH_ALGO = 'sha1'		ID_HASH_ALGO = 'sha1'
ID_HASH_LENGTH = 40 # Size in bytes of the hash hexadecimal representation.		ID_HASH_LENGTH = 40 # Size in bytes of the hash hexadecimal representation.


class ObjStorage():		class ObjStorage():
""" High-level API to manipulate the Software Heritage object storage.		""" High-level API to manipulate the Software Heritage object storage.
		zackUnsubmitted Not Done Inline Actions If we really don't have a helper like this one into hashutil, we probably should add one there, and use it here. There is no need to block this change due to this, but please check if this is the case. If it is, please submit a diff against swh.core to fix this. zack: If we really don't have a helper like this one into hashutil, we probably should add one there…
		olasdUnsubmitted Not Done Inline Actions That's `swh.core.hashutil.hashdata`. olasd: That's `swh.core.hashutil.hashdata`.
		qcamposAuthorUnsubmitted Not Done Inline Actions `swh.core.hashutil.hashdata` computes the hashcode for each algorithm (sha1, sha256, sha1_git). Should I use it anyway, or should I make a helper like `compute_hash(content, algorithm)` in `swh.core.hashutil`? qcampos: `swh.core.hashutil.hashdata` computes the hashcode for each algorithm (sha1, sha256, sha1_git).
		olasdUnsubmitted Done Inline Actions You can control which algos hashdata uses with an argument, that should be enough? olasd: You can control which algos hashdata uses with an argument, that should be enough?
		qcamposAuthorUnsubmitted Not Done Inline Actions Didn't see the default arg; that's exactly what's needed! qcampos: Didn't see the default arg; that's exactly what's needed!

Conceptually, the object storage offers 5 methods:		Conceptually, the object storage offers 5 methods:

- __contains__() check if an object is present, by object id		- __contains__() check if an object is present, by object id
- add() add a new object, returning an object id		- add() add a new object, returning an object id
- restore() same as add() but erase an already existed content		- restore() same as add() but erase an already existed content
- get() retrieve the content of an object, by object id		- get() retrieve the content of an object, by object id
- check() check the integrity of an object, by object id		- check() check the integrity of an object, by object id

And some management methods:		And some management methods:

- get_random() get random object id of existing contents (used for the		- get_random() get random object id of existing contents (used for the
content integrity checker).		content integrity checker).

Each implementation of this interface can have a different behavior and		Each implementation of this interface can have a different behavior and
its own way to store the contents.		its own way to store the contents.
"""		"""

def __contains__(self, args, *kwargs):		def __contains__(self, args, *kwargs):
		def __contains__(self, obj_id, args, *kwargs):
		""" Indicates if the given object is present in the storage

		Returns:
		True iff the object is present in the current object storage.
		"""
raise NotImplementedError(		raise NotImplementedError(
"Implementations of ObjStorage must have a '__contains__' method"		"Implementations of ObjStorage must have a '__contains__' method"
)		)

def add(self, content, obj_id=None, check_presence=True, args, *kwargs):		def add(self, content, obj_id=None, check_presence=True, args, *kwargs):
""" Add a new object to the object storage.		""" Add a new object to the object storage.

Args:		Args:
Show All 11 Lines	def add(self, content, obj_id=None, check_presence=True, args, *kwargs):
"Implementations of ObjStorage must have a 'add' method"		"Implementations of ObjStorage must have a 'add' method"
)		)

def restore(self, content, obj_id=None, args, *kwargs):		def restore(self, content, obj_id=None, args, *kwargs):
""" Restore a content that have been corrupted.		""" Restore a content that have been corrupted.

This function is identical to add_bytes but does not check if		This function is identical to add_bytes but does not check if
the object id is already in the file system.		the object id is already in the file system.
		The default implementation provided by the current class is
		suitable for most cases.

Args:		Args:
content: content of the object to be added to the storage		content: content of the object to be added to the storage
obj_id: checksums of `bytes` as computed by ID_HASH_ALGO. When		obj_id: checksums of `bytes` as computed by ID_HASH_ALGO. When
given, obj_id will be trusted to match bytes. If missing,		given, obj_id will be trusted to match bytes. If missing,
obj_id will be computed on the fly.		obj_id will be computed on the fly.
"""		"""
raise NotImplemented(		# check_presence to false will erase the potential previous content.
"Implementations of ObjStorage must have a 'restore' method"		return self.add(content, obj_id, check_presence=False)
)

def get(self, obj_id, args, *kwargs):		def get(self, obj_id, args, *kwargs):
""" Retrieve the content of a given object.		""" Retrieve the content of a given object.

Args:		Args:
obj_id: object id.		obj_id: object id.

Returns:		Returns:
the content of the requested object as bytes.		the content of the requested object as bytes.

Raises:		Raises:
ObjNotFoundError: if the requested object is missing.		ObjNotFoundError: if the requested object is missing.
"""		"""
raise NotImplementedError(		raise NotImplementedError(
"Implementations of ObjStorage must have a 'get' method"		"Implementations of ObjStorage must have a 'get' method"
)		)

def get_batch(self, obj_ids, args, *kwargs):		def get_batch(self, obj_ids, args, *kwargs):
""" Retrieve content in bulk.		""" Retrieve content in bulk.

Note: This function does have a default implementation in ObjStorage		Note: This function does have a default implementation in ObjStorage
that is suitable for most cases.		that is suitable for most cases.
		For object storages that needs to do the minimal number of requests
		possible (ex: remote object storages), that method can be overriden
		to perform a more efficient operation.

Args:		Args:
obj_ids: list of object ids.		obj_ids: list of object ids.

Returns:		Returns:
list of resulting contents, or None if the content could not		list of resulting contents, or None if the content could not
be retrieved. Do not raise any exception as a fail for one content		be retrieved. Do not raise any exception as a fail for one content
will not cancel the whole request.		will not cancel the whole request.
Show All 22 Lines	def check(self, obj_id, args, *kwargs):
)		)

def get_random(self, batch_size, args, *kwargs):		def get_random(self, batch_size, args, *kwargs):
""" Get random ids of existing contents		""" Get random ids of existing contents

This method is used in order to get random ids to perform		This method is used in order to get random ids to perform
content integrity verifications on random contents.		content integrity verifications on random contents.

Attributes:		Args:
batch_size (int): Number of ids that will be given		batch_size (int): Number of ids that will be given

Yields:		Yields:
An iterable of ids of contents that are in the current object		An iterable of ids of contents that are in the current object
storage.		storage.
"""		"""
raise NotImplementedError(		raise NotImplementedError(
"The current implementation of ObjStorage does not support "		"The current implementation of ObjStorage does not support "
"'get_random' operation"		"'get_random' operation"
)		)