Differential D546 Diff 1686 swh/storage/storage.py

Changeset View

Standalone View

swh/storage/storage.py

Show All 14 Lines
import psycopg2.pool		import psycopg2.pool

from . import converters		from . import converters
from .common import db_transaction_generator, db_transaction		from .common import db_transaction_generator, db_transaction
from .db import Db		from .db import Db
from .exc import StorageDBError		from .exc import StorageDBError
from .algos import diff		from .algos import diff

from swh.model.hashutil import ALGORITHMS		from swh.model.hashutil import ALGORITHMS, hash_to_bytes
from swh.objstorage import get_objstorage		from swh.objstorage import get_objstorage
from swh.objstorage.exc import ObjNotFoundError		from swh.objstorage.exc import ObjNotFoundError

# Max block size of contents to return		# Max block size of contents to return
BULK_BLOCK_CONTENT_LEN_MAX = 10000		BULK_BLOCK_CONTENT_LEN_MAX = 10000

		EMPTY_SNAPSHOT_ID = hash_to_bytes('1a8893e6a86f444e8be8e7bda6cb34fb1735a00e')
		"""Identifier for the empty snapshot"""


class Storage():		class Storage():
"""SWH storage proxy, encompassing DB and object storage		"""SWH storage proxy, encompassing DB and object storage

"""		"""

def __init__(self, db, objstorage, min_pool_conns=1, max_pool_conns=10):		def __init__(self, db, objstorage, min_pool_conns=1, max_pool_conns=10):
"""		"""
▲ Show 20 Lines • Show All 702 Lines • ▼ Show 20 Lines	def snapshot_get(self, snapshot_id, db=None, cur=None):
dict: a dict with three keys:		dict: a dict with three keys:
* id: identifier of the snapshot		* id: identifier of the snapshot
* branches: a dict of branches contained in the snapshot		* branches: a dict of branches contained in the snapshot
whose keys are the branches' names.		whose keys are the branches' names.
* next_branch: the name of the first branch not returned		* next_branch: the name of the first branch not returned
or :const:`None` if the snapshot has less than 1000		or :const:`None` if the snapshot has less than 1000
branches.		branches.
"""		"""
max_branches = 1000
branches = {}
next_branch = None
fetched_branches = list(db.snapshot_get_by_id(
snapshot_id, branches_count=max_branches+1, cur=cur))
for branch in fetched_branches[:max_branches]:
branch = dict(zip(db.snapshot_get_cols, branch))
del branch['snapshot_id']
name = branch.pop('name')
if branch == {'target': None, 'target_type': None}:
branch = None
branches[name] = branch

if len(fetched_branches) > max_branches:
branch = dict(zip(db.snapshot_get_cols, fetched_branches[-1]))
next_branch = branch['name']

if branches:		return self.snapshot_get_branches(snapshot_id, db=db, cur=cur)
return {
'id': snapshot_id,
'branches': branches,
'next_branch': next_branch
}

if db.snapshot_exists(snapshot_id, cur):
# empty snapshot
return {
'id': snapshot_id,
'branches': {},
'next_branch': None
}

return None

@db_transaction(statement_timeout=2000)		@db_transaction(statement_timeout=2000)
def snapshot_get_by_origin_visit(self, origin, visit, db=None, cur=None):		def snapshot_get_by_origin_visit(self, origin, visit, db=None, cur=None):
"""Get the content, possibly partial, of a snapshot for the given origin visit		"""Get the content, possibly partial, of a snapshot for the given origin visit

The branches of the snapshot are iterated in the lexicographical		The branches of the snapshot are iterated in the lexicographical
order of their names.		order of their names.

▲ Show 20 Lines • Show All 69 Lines • ▼ Show 20 Lines	def snapshot_count_branches(self, snapshot_id, db=None, cur=None):
dict: A dict whose keys are the target types of branches and		dict: A dict whose keys are the target types of branches and
values their corresponding amount		values their corresponding amount
"""		"""
return dict([bc for bc in		return dict([bc for bc in
db.snapshot_count_branches(snapshot_id, cur)])		db.snapshot_count_branches(snapshot_id, cur)])

@db_transaction(statement_timeout=2000)		@db_transaction(statement_timeout=2000)
def snapshot_get_branches(self, snapshot_id, branches_from=b'',		def snapshot_get_branches(self, snapshot_id, branches_from=b'',
branches_count=None, target_types=None,		branches_count=1000, target_types=None,
db=None, cur=None):		db=None, cur=None):
"""Get the content, possibly partial, of a snapshot with the given id		"""Get the content, possibly partial, of a snapshot with the given id

The branches of the snapshot are iterated in the lexicographical		The branches of the snapshot are iterated in the lexicographical
order of their names.		order of their names.

Args:		Args:
snapshot_id (bytes): identifier of the snapshot		snapshot_id (bytes): identifier of the snapshot
branches_from (bytes): optional parameter used to skip branches		branches_from (bytes): optional parameter used to skip branches
whose name is lesser than it before returning them		whose name is lesser than it before returning them
branches_count (int): optional parameter used to restrain		branches_count (int): optional parameter used to restrain
the amount of returned branches		the amount of returned branches
target_types (list): optional parameter used to filter the		target_types (list): optional parameter used to filter the
target types of branch to return (possible values that can be		target types of branch to return (possible values that can be
contained in that list are `'content', 'directory',		contained in that list are `'content', 'directory',
'revision', 'release', 'snapshot', 'alias'`)		'revision', 'release', 'snapshot', 'alias'`)
Returns:		Returns:
dict: a dict with two keys:		dict: a dict with three keys:
* id: identifier of the snapshot		* id: identifier of the snapshot
* branches: a dict of branches contained in the snapshot		* branches: a dict of branches contained in the snapshot
whose keys are the branches' names.		whose keys are the branches' names.
		* next_branch: the name of the first branch not returned
		or :const:`None` if the snapshot has less than
		`branches_count` branches after `branches_from` included.
"""		"""
		if snapshot_id == EMPTY_SNAPSHOT_ID:
		return {
		'id': snapshot_id,
		'branches': {},
		'next_branch': None,
		}

branches = {}		branches = {}
for branch in db.snapshot_get_by_id(snapshot_id, branches_from,		next_branch = None
branches_count, target_types, cur):
		fetched_branches = list(db.snapshot_get_by_id(
		snapshot_id, branches_from=branches_from,
		branches_count=branches_count+1, target_types=target_types,
		cur=cur,
		))
		for branch in fetched_branches[:branches_count]:
branch = dict(zip(db.snapshot_get_cols, branch))		branch = dict(zip(db.snapshot_get_cols, branch))
del branch['snapshot_id']		del branch['snapshot_id']
name = branch.pop('name')		name = branch.pop('name')
if branch == {'target': None, 'target_type': None}:		if branch == {'target': None, 'target_type': None}:
branch = None		branch = None
branches[name] = branch		branches[name] = branch

if branches:		if len(fetched_branches) > branches_count:
return {'id': snapshot_id, 'branches': branches}		branch = dict(zip(db.snapshot_get_cols, fetched_branches[-1]))
		next_branch = branch['name']

if db.snapshot_exists(snapshot_id, cur):		if branches:
return {'id': snapshot_id, 'branches': {}}		return {
		'id': snapshot_id,
		'branches': branches,
		'next_branch': next_branch,
		}

return None		return None

@db_transaction()		@db_transaction()
def origin_visit_add(self, origin, ts, db=None, cur=None):		def origin_visit_add(self, origin, ts, db=None, cur=None):
"""Add an origin_visit for the origin at ts with status 'ongoing'.		"""Add an origin_visit for the origin at ts with status 'ongoing'.

Args:		Args:
▲ Show 20 Lines • Show All 453 Lines • Show Last 20 Lines