Diff 5469

swh/lister/core/indexing_lister.py

# Copyright (C) 2015-2017 The Software Heritage developers		# Copyright (C) 2015-2018 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution		# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version		# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information		# See top-level LICENSE file for more information

"""Module in charge of hashing function definitions. This is the base		"""Module in charge of hashing function definitions. This is the base
module use to compute swh's hashes.		module use to compute swh's hashes.

Only a subset of hashing algorithms is supported as defined in the		Only a subset of hashing algorithms is supported as defined in the
ALGORITHMS set. Any provided algorithms not in that list will result		ALGORITHMS set. Any provided algorithms not in that list will result
in a ValueError explaining the error.		in a ValueError explaining the error.

This modules defines the following hashing functions:		This module defines MultiHash class to ease the softwareheritage
		hashing algorithm. This allows as before (with hash_* function) to
		compute hashes from file object, path, data.

		Basic usage examples:

		- file object: MultiHash.from_file(file_object).digest()

		- path (filepath): MultiHash.from_path(b'foo').hexdigest()

		- data (bytes): MultiHash.from_data(b'foo').bytehexdigest()

		Complex usage (old use was through callback):

		- To compute length, integrate the length to the set of algorithms to
		compute, for example:

		h = MultiHash(hash_names=set({'length'}).union(DEFAULT_ALGORITHMS))

		- Write alongside computing hashing algorithms (from a stream), example:

		h = MultiHash(length=length)
		with open(filepath, 'wb') as f:
		for chunk in r.iter_content(): # r a stream of sort
		h.update(chunk)
		f.write(chunk)


		This module also defines the following (deprecated) hashing functions:

- hash_file: Hash the contents of the given file object with the given		- hash_file: Hash the contents of the given file object with the given
algorithms (defaulting to DEFAULT_ALGORITHMS if none provided).		algorithms (defaulting to DEFAULT_ALGORITHMS if none provided).

- hash_data: Hash the given binary blob with the given algorithms		- hash_data: Hash the given binary blob with the given algorithms
(defaulting to DEFAULT_ALGORITHMS if none provided).		(defaulting to DEFAULT_ALGORITHMS if none provided).

- hash_path: Hash the contents of the file at the given path with the		- hash_path: Hash the contents of the file at the given path with the
Show All 19 Lines
"""		"""

HASH_BLOCK_SIZE = 32768		HASH_BLOCK_SIZE = 32768
"""Block size for streaming hash computations made in this module"""		"""Block size for streaming hash computations made in this module"""

_blake2_hash_cache = {}		_blake2_hash_cache = {}


		class MultiHash:
		"""Hashutil class to support multiple hashes computation.

		Args:

		hash_names (set): Set of hash algorithms (+ optionally length)
		to compute hashes (cf. DEFAULT_ALGORITHMS)
		length (int): Length of the total sum of chunks to read

		If the length is provided as algorithm, the length is also
		computed and returned.

		"""
		def __init__(self, hash_names=DEFAULT_ALGORITHMS, length=None):
		self.state = {}
		self.track_length = False
		for name in hash_names:
		if name == 'length':
		self.state['length'] = 0
		self.track_length = True
		else:
		self.state[name] = _new_hash(name, length)

		@classmethod
		def from_state(cls, state, track_length):
		ret = cls([])
		ret.state = state
		ret.track_length = track_length

		@classmethod
		def from_file(cls, file, hash_names=DEFAULT_ALGORITHMS, length=None):
		ret = cls(length=length, hash_names=hash_names)
		for chunk in file:
		ret.update(chunk)
		return ret

		@classmethod
		def from_path(cls, path, hash_names=DEFAULT_ALGORITHMS, length=None,
		track_length=True):
		if not length:
		length = os.path.getsize(path)
		with open(path, 'rb') as f:
		ret = cls.from_file(f, hash_names=hash_names, length=length)
		# For compatibility reason with `hash_path`
		if track_length:
		ret.state['length'] = length
		return ret

		@classmethod
		def from_data(cls, data, hash_names=DEFAULT_ALGORITHMS, length=None):
		if not length:
		ardumontAuthorUnsubmitted Not Done Inline Actions I don't think this is used much. IIRC, it's only used once. So, i'm for removing that part later (when we will have migrated the base code). ardumont: I don't think this is used much. IIRC, it's only used once. So, i'm for removing that part…
		olasdUnsubmitted Done Inline Actions In any case, we should just use the length that we've retrieved from os.path.getsize rather than computing it again. olasd: In any case, we should just use the length that we've retrieved from os.path.getsize rather…
		ardumontAuthorUnsubmitted Not Done Inline Actions Right, fixing that as well. ardumont: Right, fixing that as well.
		length = len(data)
		fobj = BytesIO(data)
		return cls.from_file(fobj, hash_names=hash_names, length=length)

		def update(self, chunk):
		for name, h in self.state.items():
		if name == 'length':
		continue
		h.update(chunk)
		if self.track_length:
		self.state['length'] += len(chunk)

		def digest(self):
		return {
		name: h.digest() if name != 'length' else h
		for name, h in self.state.items()
		}

		def hexdigest(self):
		return {
		name: h.hexdigest() if name != 'length' else h
		for name, h in self.state.items()
		}

		def bytehexdigest(self):
		return {
		name: hash_to_bytehex(h.digest()) if name != 'length' else h
		for name, h in self.state.items()
		}

		def copy(self):
		copied_state = {
		name: h.copy() if name != 'length' else h
		for name, h in self.state.items()
		}
		return self.from_state(copied_state, self.track_length)


def _new_blake2_hash(algo):		def _new_blake2_hash(algo):
"""Return a function that initializes a blake2 hash.		"""Return a function that initializes a blake2 hash.

"""		"""
if algo in _blake2_hash_cache:		if algo in _blake2_hash_cache:
return _blake2_hash_cache[algo]()		return _blake2_hash_cache[algo]()

lalgo = algo.lower()		lalgo = algo.lower()
▲ Show 20 Lines • Show All 100 Lines • ▼ Show 20 Lines	if algo.endswith('_git'):
if length is None:		if length is None:
raise ValueError('Missing length for git hashing algorithm')		raise ValueError('Missing length for git hashing algorithm')
base_algo = algo[:-4]		base_algo = algo[:-4]
return _new_git_hash(base_algo, 'blob', length)		return _new_git_hash(base_algo, 'blob', length)

return _new_hashlib_hash(algo)		return _new_hashlib_hash(algo)


def hash_file(fobj, length=None, algorithms=DEFAULT_ALGORITHMS, chunk_cb=None):		def hash_file(fobj, length=None, algorithms=DEFAULT_ALGORITHMS,
"""Hash the contents of the given file object with the given algorithms.		chunk_cb=None):
		"""(Deprecated) cf. MultiHash.from_file

		Hash the contents of the given file object with the given algorithms.

Args:		Args:
fobj: a file-like object		fobj: a file-like object
length: the length of the contents of the file-like object (for the		length (int): the length of the contents of the file-like
git-specific algorithms)		object (for the git-specific algorithms)
algorithms: the hashing algorithms to be used, as an iterable over		algorithms (set): the hashing algorithms to be used, as an
strings		iterable over strings
		chunk_cb (fun): a callback function taking a chunk of data as
		parameter

Returns: a dict mapping each algorithm to a bytes digest.		Returns:
		a dict mapping each algorithm to a digest (bytes by default).

Raises:		Raises:
ValueError if algorithms contains an unknown hash algorithm.		ValueError if algorithms contains an unknown hash algorithm.
"""
hashes = {algo: _new_hash(algo, length) for algo in algorithms}

		"""
		h = MultiHash(algorithms, length)
while True:		while True:
chunk = fobj.read(HASH_BLOCK_SIZE)		chunk = fobj.read(HASH_BLOCK_SIZE)
if not chunk:		if not chunk:
break		break
for hash in hashes.values():		h.update(chunk)
hash.update(chunk)
if chunk_cb:		if chunk_cb:
chunk_cb(chunk)		chunk_cb(chunk)

return {algo: hash.digest() for algo, hash in hashes.items()}		return h.digest()


def hash_path(path, algorithms=DEFAULT_ALGORITHMS, chunk_cb=None):		def hash_path(path, algorithms=DEFAULT_ALGORITHMS, chunk_cb=None):
"""Hash the contents of the file at the given path with the given		"""(deprecated) cf. MultiHash.from_path

		Hash the contents of the file at the given path with the given
algorithms.		algorithms.

Args:		Args:
path: the path of the file to hash		path (str): the path of the file to hash
algorithms: the hashing algorithms used		algorithms (set): the hashing algorithms used
chunk_cb: a callback		chunk_cb (fun): a callback function taking a chunk of data as parameter

Returns: a dict mapping each algorithm to a bytes digest.		Returns: a dict mapping each algorithm to a bytes digest.

Raises:		Raises:
ValueError if algorithms contains an unknown hash algorithm.		ValueError if algorithms contains an unknown hash algorithm.
OSError on file access error		OSError on file access error

"""		"""
length = os.path.getsize(path)		length = os.path.getsize(path)
with open(path, 'rb') as fobj:		with open(path, 'rb') as fobj:
hash = hash_file(fobj, length, algorithms, chunk_cb)		hashes = hash_file(fobj, length, algorithms, chunk_cb=chunk_cb)
hash['length'] = length		hashes['length'] = length
return hash		return hashes


def hash_data(data, algorithms=DEFAULT_ALGORITHMS, with_length=False):		def hash_data(data, algorithms=DEFAULT_ALGORITHMS):
"""Hash the given binary blob with the given algorithms.		"""(deprecated) cf. MultiHash.from_data

		Hash the given binary blob with the given algorithms.

Args:		Args:
data (bytes): raw content to hash		data (bytes): raw content to hash
algorithms (list): the hashing algorithms used		algorithms (set): the hashing algorithms used
with_length (bool): add the length key in the resulting dict

Returns: a dict mapping each algorithm to a bytes digest		Returns: a dict mapping each algorithm to a bytes digest

Raises:		Raises:
TypeError if data does not support the buffer interface.		TypeError if data does not support the buffer interface.
ValueError if algorithms contains an unknown hash algorithm.		ValueError if algorithms contains an unknown hash algorithm.

"""		"""
fobj = BytesIO(data)		return MultiHash.from_data(data, hash_names=algorithms).digest()
length = len(data)
data = hash_file(fobj, length, algorithms)
if with_length:
data['length'] = length
return data


def hash_git_data(data, git_type, base_algo='sha1'):		def hash_git_data(data, git_type, base_algo='sha1'):
"""Hash the given data as a git object of type git_type.		"""Hash the given data as a git object of type git_type.

Args:		Args:
data: a bytes object		data: a bytes object
git_type: the git object type		git_type: the git object type
▲ Show 20 Lines • Show All 77 Lines • Show Last 20 Lines

View Options

swh/lister/github/lister.py

# Copyright (C) 2015-2017 The Software Heritage developers		# Copyright (C) 2015-2018 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution		# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version		# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information		# See top-level LICENSE file for more information

"""Module in charge of hashing function definitions. This is the base		"""Module in charge of hashing function definitions. This is the base
module use to compute swh's hashes.		module use to compute swh's hashes.

Only a subset of hashing algorithms is supported as defined in the		Only a subset of hashing algorithms is supported as defined in the
ALGORITHMS set. Any provided algorithms not in that list will result		ALGORITHMS set. Any provided algorithms not in that list will result
in a ValueError explaining the error.		in a ValueError explaining the error.

This modules defines the following hashing functions:		This module defines MultiHash class to ease the softwareheritage
		hashing algorithm. This allows as before (with hash_* function) to
		compute hashes from file object, path, data.

		Basic usage examples:

		- file object: MultiHash.from_file(file_object).digest()

		- path (filepath): MultiHash.from_path(b'foo').hexdigest()

		- data (bytes): MultiHash.from_data(b'foo').bytehexdigest()

		Complex usage (old use was through callback):

		- To compute length, integrate the length to the set of algorithms to
		compute, for example:

		h = MultiHash(hash_names=set({'length'}).union(DEFAULT_ALGORITHMS))

		- Write alongside computing hashing algorithms (from a stream), example:

		h = MultiHash(length=length)
		with open(filepath, 'wb') as f:
		for chunk in r.iter_content(): # r a stream of sort
		h.update(chunk)
		f.write(chunk)


		This module also defines the following (deprecated) hashing functions:

- hash_file: Hash the contents of the given file object with the given		- hash_file: Hash the contents of the given file object with the given
algorithms (defaulting to DEFAULT_ALGORITHMS if none provided).		algorithms (defaulting to DEFAULT_ALGORITHMS if none provided).

- hash_data: Hash the given binary blob with the given algorithms		- hash_data: Hash the given binary blob with the given algorithms
(defaulting to DEFAULT_ALGORITHMS if none provided).		(defaulting to DEFAULT_ALGORITHMS if none provided).

- hash_path: Hash the contents of the file at the given path with the		- hash_path: Hash the contents of the file at the given path with the
Show All 19 Lines
"""		"""

HASH_BLOCK_SIZE = 32768		HASH_BLOCK_SIZE = 32768
"""Block size for streaming hash computations made in this module"""		"""Block size for streaming hash computations made in this module"""

_blake2_hash_cache = {}		_blake2_hash_cache = {}


		class MultiHash:
		"""Hashutil class to support multiple hashes computation.

		Args:

		hash_names (set): Set of hash algorithms (+ optionally length)
		to compute hashes (cf. DEFAULT_ALGORITHMS)
		length (int): Length of the total sum of chunks to read

		If the length is provided as algorithm, the length is also
		computed and returned.

		"""
		def __init__(self, hash_names=DEFAULT_ALGORITHMS, length=None):
		self.state = {}
		self.track_length = False
		for name in hash_names:
		if name == 'length':
		self.state['length'] = 0
		self.track_length = True
		else:
		self.state[name] = _new_hash(name, length)

		@classmethod
		def from_state(cls, state, track_length):
		ret = cls([])
		ret.state = state
		ret.track_length = track_length

		@classmethod
		def from_file(cls, file, hash_names=DEFAULT_ALGORITHMS, length=None):
		ret = cls(length=length, hash_names=hash_names)
		for chunk in file:
		ret.update(chunk)
		return ret

		@classmethod
		def from_path(cls, path, hash_names=DEFAULT_ALGORITHMS, length=None,
		track_length=True):
		if not length:
		length = os.path.getsize(path)
		with open(path, 'rb') as f:
		ret = cls.from_file(f, hash_names=hash_names, length=length)
		# For compatibility reason with `hash_path`
		if track_length:
		ret.state['length'] = length
		return ret

		@classmethod
		def from_data(cls, data, hash_names=DEFAULT_ALGORITHMS, length=None):
		if not length:
		ardumontAuthorUnsubmitted Not Done Inline Actions I don't think this is used much. IIRC, it's only used once. So, i'm for removing that part later (when we will have migrated the base code). ardumont: I don't think this is used much. IIRC, it's only used once. So, i'm for removing that part…
		olasdUnsubmitted Done Inline Actions In any case, we should just use the length that we've retrieved from os.path.getsize rather than computing it again. olasd: In any case, we should just use the length that we've retrieved from os.path.getsize rather…
		ardumontAuthorUnsubmitted Not Done Inline Actions Right, fixing that as well. ardumont: Right, fixing that as well.
		length = len(data)
		fobj = BytesIO(data)
		return cls.from_file(fobj, hash_names=hash_names, length=length)

		def update(self, chunk):
		for name, h in self.state.items():
		if name == 'length':
		continue
		h.update(chunk)
		if self.track_length:
		self.state['length'] += len(chunk)

		def digest(self):
		return {
		name: h.digest() if name != 'length' else h
		for name, h in self.state.items()
		}

		def hexdigest(self):
		return {
		name: h.hexdigest() if name != 'length' else h
		for name, h in self.state.items()
		}

		def bytehexdigest(self):
		return {
		name: hash_to_bytehex(h.digest()) if name != 'length' else h
		for name, h in self.state.items()
		}

		def copy(self):
		copied_state = {
		name: h.copy() if name != 'length' else h
		for name, h in self.state.items()
		}
		return self.from_state(copied_state, self.track_length)


def _new_blake2_hash(algo):		def _new_blake2_hash(algo):
"""Return a function that initializes a blake2 hash.		"""Return a function that initializes a blake2 hash.

"""		"""
if algo in _blake2_hash_cache:		if algo in _blake2_hash_cache:
return _blake2_hash_cache[algo]()		return _blake2_hash_cache[algo]()

lalgo = algo.lower()		lalgo = algo.lower()
▲ Show 20 Lines • Show All 100 Lines • ▼ Show 20 Lines	if algo.endswith('_git'):
if length is None:		if length is None:
raise ValueError('Missing length for git hashing algorithm')		raise ValueError('Missing length for git hashing algorithm')
base_algo = algo[:-4]		base_algo = algo[:-4]
return _new_git_hash(base_algo, 'blob', length)		return _new_git_hash(base_algo, 'blob', length)

return _new_hashlib_hash(algo)		return _new_hashlib_hash(algo)


def hash_file(fobj, length=None, algorithms=DEFAULT_ALGORITHMS, chunk_cb=None):		def hash_file(fobj, length=None, algorithms=DEFAULT_ALGORITHMS,
"""Hash the contents of the given file object with the given algorithms.		chunk_cb=None):
		"""(Deprecated) cf. MultiHash.from_file

		Hash the contents of the given file object with the given algorithms.

Args:		Args:
fobj: a file-like object		fobj: a file-like object
length: the length of the contents of the file-like object (for the		length (int): the length of the contents of the file-like
git-specific algorithms)		object (for the git-specific algorithms)
algorithms: the hashing algorithms to be used, as an iterable over		algorithms (set): the hashing algorithms to be used, as an
strings		iterable over strings
		chunk_cb (fun): a callback function taking a chunk of data as
		parameter

Returns: a dict mapping each algorithm to a bytes digest.		Returns:
		a dict mapping each algorithm to a digest (bytes by default).

Raises:		Raises:
ValueError if algorithms contains an unknown hash algorithm.		ValueError if algorithms contains an unknown hash algorithm.
"""
hashes = {algo: _new_hash(algo, length) for algo in algorithms}

		"""
		h = MultiHash(algorithms, length)
while True:		while True:
chunk = fobj.read(HASH_BLOCK_SIZE)		chunk = fobj.read(HASH_BLOCK_SIZE)
if not chunk:		if not chunk:
break		break
for hash in hashes.values():		h.update(chunk)
hash.update(chunk)
if chunk_cb:		if chunk_cb:
chunk_cb(chunk)		chunk_cb(chunk)

return {algo: hash.digest() for algo, hash in hashes.items()}		return h.digest()


def hash_path(path, algorithms=DEFAULT_ALGORITHMS, chunk_cb=None):		def hash_path(path, algorithms=DEFAULT_ALGORITHMS, chunk_cb=None):
"""Hash the contents of the file at the given path with the given		"""(deprecated) cf. MultiHash.from_path

		Hash the contents of the file at the given path with the given
algorithms.		algorithms.

Args:		Args:
path: the path of the file to hash		path (str): the path of the file to hash
algorithms: the hashing algorithms used		algorithms (set): the hashing algorithms used
chunk_cb: a callback		chunk_cb (fun): a callback function taking a chunk of data as parameter

Returns: a dict mapping each algorithm to a bytes digest.		Returns: a dict mapping each algorithm to a bytes digest.

Raises:		Raises:
ValueError if algorithms contains an unknown hash algorithm.		ValueError if algorithms contains an unknown hash algorithm.
OSError on file access error		OSError on file access error

"""		"""
length = os.path.getsize(path)		length = os.path.getsize(path)
with open(path, 'rb') as fobj:		with open(path, 'rb') as fobj:
hash = hash_file(fobj, length, algorithms, chunk_cb)		hashes = hash_file(fobj, length, algorithms, chunk_cb=chunk_cb)
hash['length'] = length		hashes['length'] = length
return hash		return hashes


def hash_data(data, algorithms=DEFAULT_ALGORITHMS, with_length=False):		def hash_data(data, algorithms=DEFAULT_ALGORITHMS):
"""Hash the given binary blob with the given algorithms.		"""(deprecated) cf. MultiHash.from_data

		Hash the given binary blob with the given algorithms.

Args:		Args:
data (bytes): raw content to hash		data (bytes): raw content to hash
algorithms (list): the hashing algorithms used		algorithms (set): the hashing algorithms used
with_length (bool): add the length key in the resulting dict

Returns: a dict mapping each algorithm to a bytes digest		Returns: a dict mapping each algorithm to a bytes digest

Raises:		Raises:
TypeError if data does not support the buffer interface.		TypeError if data does not support the buffer interface.
ValueError if algorithms contains an unknown hash algorithm.		ValueError if algorithms contains an unknown hash algorithm.

"""		"""
fobj = BytesIO(data)		return MultiHash.from_data(data, hash_names=algorithms).digest()
length = len(data)
data = hash_file(fobj, length, algorithms)
if with_length:
data['length'] = length
return data


def hash_git_data(data, git_type, base_algo='sha1'):		def hash_git_data(data, git_type, base_algo='sha1'):
"""Hash the given data as a git object of type git_type.		"""Hash the given data as a git object of type git_type.

Args:		Args:
data: a bytes object		data: a bytes object
git_type: the git object type		git_type: the git object type
▲ Show 20 Lines • Show All 77 Lines • Show Last 20 Lines

View Options

swh/lister/phabricator/lister.py

# Copyright (C) 2015-2017 The Software Heritage developers		# Copyright (C) 2015-2018 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution		# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version		# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information		# See top-level LICENSE file for more information

"""Module in charge of hashing function definitions. This is the base		"""Module in charge of hashing function definitions. This is the base
module use to compute swh's hashes.		module use to compute swh's hashes.

Only a subset of hashing algorithms is supported as defined in the		Only a subset of hashing algorithms is supported as defined in the
ALGORITHMS set. Any provided algorithms not in that list will result		ALGORITHMS set. Any provided algorithms not in that list will result
in a ValueError explaining the error.		in a ValueError explaining the error.

This modules defines the following hashing functions:		This module defines MultiHash class to ease the softwareheritage
		hashing algorithm. This allows as before (with hash_* function) to
		compute hashes from file object, path, data.

		Basic usage examples:

		- file object: MultiHash.from_file(file_object).digest()

		- path (filepath): MultiHash.from_path(b'foo').hexdigest()

		- data (bytes): MultiHash.from_data(b'foo').bytehexdigest()

		Complex usage (old use was through callback):

		- To compute length, integrate the length to the set of algorithms to
		compute, for example:

		h = MultiHash(hash_names=set({'length'}).union(DEFAULT_ALGORITHMS))

		- Write alongside computing hashing algorithms (from a stream), example:

		h = MultiHash(length=length)
		with open(filepath, 'wb') as f:
		for chunk in r.iter_content(): # r a stream of sort
		h.update(chunk)
		f.write(chunk)


		This module also defines the following (deprecated) hashing functions:

- hash_file: Hash the contents of the given file object with the given		- hash_file: Hash the contents of the given file object with the given
algorithms (defaulting to DEFAULT_ALGORITHMS if none provided).		algorithms (defaulting to DEFAULT_ALGORITHMS if none provided).

- hash_data: Hash the given binary blob with the given algorithms		- hash_data: Hash the given binary blob with the given algorithms
(defaulting to DEFAULT_ALGORITHMS if none provided).		(defaulting to DEFAULT_ALGORITHMS if none provided).

- hash_path: Hash the contents of the file at the given path with the		- hash_path: Hash the contents of the file at the given path with the
Show All 19 Lines
"""		"""

HASH_BLOCK_SIZE = 32768		HASH_BLOCK_SIZE = 32768
"""Block size for streaming hash computations made in this module"""		"""Block size for streaming hash computations made in this module"""

_blake2_hash_cache = {}		_blake2_hash_cache = {}


		class MultiHash:
		"""Hashutil class to support multiple hashes computation.

		Args:

		hash_names (set): Set of hash algorithms (+ optionally length)
		to compute hashes (cf. DEFAULT_ALGORITHMS)
		length (int): Length of the total sum of chunks to read

		If the length is provided as algorithm, the length is also
		computed and returned.

		"""
		def __init__(self, hash_names=DEFAULT_ALGORITHMS, length=None):
		self.state = {}
		self.track_length = False
		for name in hash_names:
		if name == 'length':
		self.state['length'] = 0
		self.track_length = True
		else:
		self.state[name] = _new_hash(name, length)

		@classmethod
		def from_state(cls, state, track_length):
		ret = cls([])
		ret.state = state
		ret.track_length = track_length

		@classmethod
		def from_file(cls, file, hash_names=DEFAULT_ALGORITHMS, length=None):
		ret = cls(length=length, hash_names=hash_names)
		for chunk in file:
		ret.update(chunk)
		return ret

		@classmethod
		def from_path(cls, path, hash_names=DEFAULT_ALGORITHMS, length=None,
		track_length=True):
		if not length:
		length = os.path.getsize(path)
		with open(path, 'rb') as f:
		ret = cls.from_file(f, hash_names=hash_names, length=length)
		# For compatibility reason with `hash_path`
		if track_length:
		ret.state['length'] = length
		return ret

		@classmethod
		def from_data(cls, data, hash_names=DEFAULT_ALGORITHMS, length=None):
		if not length:
		ardumontAuthorUnsubmitted Not Done Inline Actions I don't think this is used much. IIRC, it's only used once. So, i'm for removing that part later (when we will have migrated the base code). ardumont: I don't think this is used much. IIRC, it's only used once. So, i'm for removing that part…
		olasdUnsubmitted Done Inline Actions In any case, we should just use the length that we've retrieved from os.path.getsize rather than computing it again. olasd: In any case, we should just use the length that we've retrieved from os.path.getsize rather…
		ardumontAuthorUnsubmitted Not Done Inline Actions Right, fixing that as well. ardumont: Right, fixing that as well.
		length = len(data)
		fobj = BytesIO(data)
		return cls.from_file(fobj, hash_names=hash_names, length=length)

		def update(self, chunk):
		for name, h in self.state.items():
		if name == 'length':
		continue
		h.update(chunk)
		if self.track_length:
		self.state['length'] += len(chunk)

		def digest(self):
		return {
		name: h.digest() if name != 'length' else h
		for name, h in self.state.items()
		}

		def hexdigest(self):
		return {
		name: h.hexdigest() if name != 'length' else h
		for name, h in self.state.items()
		}

		def bytehexdigest(self):
		return {
		name: hash_to_bytehex(h.digest()) if name != 'length' else h
		for name, h in self.state.items()
		}

		def copy(self):
		copied_state = {
		name: h.copy() if name != 'length' else h
		for name, h in self.state.items()
		}
		return self.from_state(copied_state, self.track_length)


def _new_blake2_hash(algo):		def _new_blake2_hash(algo):
"""Return a function that initializes a blake2 hash.		"""Return a function that initializes a blake2 hash.

"""		"""
if algo in _blake2_hash_cache:		if algo in _blake2_hash_cache:
return _blake2_hash_cache[algo]()		return _blake2_hash_cache[algo]()

lalgo = algo.lower()		lalgo = algo.lower()
▲ Show 20 Lines • Show All 100 Lines • ▼ Show 20 Lines	if algo.endswith('_git'):
if length is None:		if length is None:
raise ValueError('Missing length for git hashing algorithm')		raise ValueError('Missing length for git hashing algorithm')
base_algo = algo[:-4]		base_algo = algo[:-4]
return _new_git_hash(base_algo, 'blob', length)		return _new_git_hash(base_algo, 'blob', length)

return _new_hashlib_hash(algo)		return _new_hashlib_hash(algo)


def hash_file(fobj, length=None, algorithms=DEFAULT_ALGORITHMS, chunk_cb=None):		def hash_file(fobj, length=None, algorithms=DEFAULT_ALGORITHMS,
"""Hash the contents of the given file object with the given algorithms.		chunk_cb=None):
		"""(Deprecated) cf. MultiHash.from_file

		Hash the contents of the given file object with the given algorithms.

Args:		Args:
fobj: a file-like object		fobj: a file-like object
length: the length of the contents of the file-like object (for the		length (int): the length of the contents of the file-like
git-specific algorithms)		object (for the git-specific algorithms)
algorithms: the hashing algorithms to be used, as an iterable over		algorithms (set): the hashing algorithms to be used, as an
strings		iterable over strings
		chunk_cb (fun): a callback function taking a chunk of data as
		parameter

Returns: a dict mapping each algorithm to a bytes digest.		Returns:
		a dict mapping each algorithm to a digest (bytes by default).

Raises:		Raises:
ValueError if algorithms contains an unknown hash algorithm.		ValueError if algorithms contains an unknown hash algorithm.
"""
hashes = {algo: _new_hash(algo, length) for algo in algorithms}

		"""
		h = MultiHash(algorithms, length)
while True:		while True:
chunk = fobj.read(HASH_BLOCK_SIZE)		chunk = fobj.read(HASH_BLOCK_SIZE)
if not chunk:		if not chunk:
break		break
for hash in hashes.values():		h.update(chunk)
hash.update(chunk)
if chunk_cb:		if chunk_cb:
chunk_cb(chunk)		chunk_cb(chunk)

return {algo: hash.digest() for algo, hash in hashes.items()}		return h.digest()


def hash_path(path, algorithms=DEFAULT_ALGORITHMS, chunk_cb=None):		def hash_path(path, algorithms=DEFAULT_ALGORITHMS, chunk_cb=None):
"""Hash the contents of the file at the given path with the given		"""(deprecated) cf. MultiHash.from_path

		Hash the contents of the file at the given path with the given
algorithms.		algorithms.

Args:		Args:
path: the path of the file to hash		path (str): the path of the file to hash
algorithms: the hashing algorithms used		algorithms (set): the hashing algorithms used
chunk_cb: a callback		chunk_cb (fun): a callback function taking a chunk of data as parameter

Returns: a dict mapping each algorithm to a bytes digest.		Returns: a dict mapping each algorithm to a bytes digest.

Raises:		Raises:
ValueError if algorithms contains an unknown hash algorithm.		ValueError if algorithms contains an unknown hash algorithm.
OSError on file access error		OSError on file access error

"""		"""
length = os.path.getsize(path)		length = os.path.getsize(path)
with open(path, 'rb') as fobj:		with open(path, 'rb') as fobj:
hash = hash_file(fobj, length, algorithms, chunk_cb)		hashes = hash_file(fobj, length, algorithms, chunk_cb=chunk_cb)
hash['length'] = length		hashes['length'] = length
return hash		return hashes


def hash_data(data, algorithms=DEFAULT_ALGORITHMS, with_length=False):		def hash_data(data, algorithms=DEFAULT_ALGORITHMS):
"""Hash the given binary blob with the given algorithms.		"""(deprecated) cf. MultiHash.from_data

		Hash the given binary blob with the given algorithms.

Args:		Args:
data (bytes): raw content to hash		data (bytes): raw content to hash
algorithms (list): the hashing algorithms used		algorithms (set): the hashing algorithms used
with_length (bool): add the length key in the resulting dict

Returns: a dict mapping each algorithm to a bytes digest		Returns: a dict mapping each algorithm to a bytes digest

Raises:		Raises:
TypeError if data does not support the buffer interface.		TypeError if data does not support the buffer interface.
ValueError if algorithms contains an unknown hash algorithm.		ValueError if algorithms contains an unknown hash algorithm.

"""		"""
fobj = BytesIO(data)		return MultiHash.from_data(data, hash_names=algorithms).digest()
length = len(data)
data = hash_file(fobj, length, algorithms)
if with_length:
data['length'] = length
return data


def hash_git_data(data, git_type, base_algo='sha1'):		def hash_git_data(data, git_type, base_algo='sha1'):
"""Hash the given data as a git object of type git_type.		"""Hash the given data as a git object of type git_type.

Args:		Args:
data: a bytes object		data: a bytes object
git_type: the git object type		git_type: the git object type
▲ Show 20 Lines • Show All 77 Lines • Show Last 20 Lines

lister: Clean up code
ClosedPublic
Actions

Details

Diff Detail

Event Timeline

Revision Contents
Changeset List

Diff 5469

swh/lister/core/indexing_lister.py

swh/lister/github/lister.py

swh/lister/phabricator/lister.py

lister: Clean up codeClosedPublicActions

Details

Diff Detail

Event Timeline

Revision ContentsChangeset List

Diff 5469

swh/lister/core/indexing_lister.py

swh/lister/github/lister.py

swh/lister/phabricator/lister.py

lister: Clean up code
ClosedPublic
Actions

Revision Contents
Changeset List