Diff 1594

swh/indexer/storage/init.py

# Copyright (C) 2015-2018 The Software Heritage developers		# Copyright (C) 2015-2018 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution		# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version		# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information		# See top-level LICENSE file for more information


import json		import json
import psycopg2		import psycopg2

from collections import defaultdict		from collections import defaultdict

		from swh.core.api import remote_api_endpoint
from swh.storage.common import db_transaction_generator, db_transaction		from swh.storage.common import db_transaction_generator, db_transaction
from swh.storage.exc import StorageDBError		from swh.storage.exc import StorageDBError
from .db import Db		from .db import Db

from . import converters		from . import converters


INDEXER_CFG_KEY = 'indexer_storage'		INDEXER_CFG_KEY = 'indexer_storage'


def get_indexer_storage(cls, args):		def get_indexer_storage(cls, args):
		ardumontUnsubmitted Not Done Inline Actions Don't we want to keep the original docstring as well here (@functools.wraps)? ardumont: Don't we want to keep the original docstring as well here (@functools.wraps)?
		vlorentzAuthorUnsubmitted Done Inline Actions The original function is returned, so there is no wrapper. vlorentz: The original function is returned, so there is no wrapper.
		ardumontUnsubmitted Not Done Inline Actions Right! ardumont: Right!
"""Get an indexer storage object of class `storage_class` with		"""Get an indexer storage object of class `storage_class` with
arguments `storage_args`.		arguments `storage_args`.

Args:		Args:
args (dict): dictionary with keys:		args (dict): dictionary with keys:
- cls (str): storage's class, either 'local' or 'remote'		- cls (str): storage's class, either 'local' or 'remote'
- args (dict): dictionary with keys		- args (dict): dictionary with keys

Show All 9 Lines	def get_indexer_storage(cls, args):
elif cls == 'local':		elif cls == 'local':
from . import IndexerStorage		from . import IndexerStorage
else:		else:
raise ValueError('Unknown indexer storage class `%s`' % cls)		raise ValueError('Unknown indexer storage class `%s`' % cls)

return IndexerStorage(**args)		return IndexerStorage(**args)


class IndexerStorage():		class IndexerStorage:
"""SWH Indexer Storage		"""SWH Indexer Storage

"""		"""
def __init__(self, db, min_pool_conns=1, max_pool_conns=10):		def __init__(self, db, min_pool_conns=1, max_pool_conns=10):
"""		"""
Args:		Args:
db_conn: either a libpq connection string, or a psycopg2 connection		db_conn: either a libpq connection string, or a psycopg2 connection

Show All 10 Lines	def __init__(self, db, min_pool_conns=1, max_pool_conns=10):
except psycopg2.OperationalError as e:		except psycopg2.OperationalError as e:
raise StorageDBError(e)		raise StorageDBError(e)

def get_db(self):		def get_db(self):
if self._db:		if self._db:
return self._db		return self._db
return Db.from_pool(self._pool)		return Db.from_pool(self._pool)

		@remote_api_endpoint('check_config')
def check_config(self, *, check_write):		def check_config(self, *, check_write):
"""Check that the storage is configured and ready to go."""		"""Check that the storage is configured and ready to go."""
# Check permissions on one of the tables		# Check permissions on one of the tables
with self.get_db().transaction() as cur:		with self.get_db().transaction() as cur:
if check_write:		if check_write:
check = 'INSERT'		check = 'INSERT'
else:		else:
check = 'SELECT'		check = 'SELECT'

cur.execute(		cur.execute(
"select has_table_privilege(current_user, 'content_mimetype', %s)", # noqa		"select has_table_privilege(current_user, 'content_mimetype', %s)", # noqa
(check,)		(check,)
)		)
return cur.fetchone()[0]		return cur.fetchone()[0]

return True		return True

		@remote_api_endpoint('content_mimetype/missing')
@db_transaction_generator()		@db_transaction_generator()
def content_mimetype_missing(self, mimetypes, db=None, cur=None):		def content_mimetype_missing(self, mimetypes, db=None, cur=None):
"""List mimetypes missing from storage.		"""List mimetypes missing from storage.

Args:		Args:
mimetypes (iterable): iterable of dict with keys:		mimetypes (iterable): iterable of dict with keys:

id (bytes): sha1 identifier		id (bytes): sha1 identifier
indexer_configuration_id (int): tool used to compute		indexer_configuration_id (int): tool used to compute
the results		the results

Yields:		Yields:
an iterable of missing id for the tuple (id,		an iterable of missing id for the tuple (id,
indexer_configuration_id)		indexer_configuration_id)

"""		"""
for obj in db.content_mimetype_missing_from_list(mimetypes, cur):		for obj in db.content_mimetype_missing_from_list(mimetypes, cur):
yield obj[0]		yield obj[0]

		@remote_api_endpoint('content_mimetype/add')
@db_transaction()		@db_transaction()
def content_mimetype_add(self, mimetypes, conflict_update=False, db=None,		def content_mimetype_add(self, mimetypes, conflict_update=False, db=None,
cur=None):		cur=None):
"""Add mimetypes not present in storage.		"""Add mimetypes not present in storage.

Args:		Args:
mimetypes (iterable): dictionaries with keys:		mimetypes (iterable): dictionaries with keys:

id (bytes): sha1 identifier		id (bytes): sha1 identifier
mimetype (bytes): raw content's mimetype		mimetype (bytes): raw content's mimetype
encoding (bytes): raw content's encoding		encoding (bytes): raw content's encoding
indexer_configuration_id (int): tool's id used to		indexer_configuration_id (int): tool's id used to
compute the results		compute the results
conflict_update (bool): Flag to determine if we want to		conflict_update (bool): Flag to determine if we want to
overwrite (true) or skip duplicates		overwrite (true) or skip duplicates
(false, the default)		(false, the default)

"""		"""
db.mktemp_content_mimetype(cur)		db.mktemp_content_mimetype(cur)
db.copy_to(mimetypes, 'tmp_content_mimetype',		db.copy_to(mimetypes, 'tmp_content_mimetype',
['id', 'mimetype', 'encoding', 'indexer_configuration_id'],		['id', 'mimetype', 'encoding', 'indexer_configuration_id'],
cur)		cur)
db.content_mimetype_add_from_temp(conflict_update, cur)		db.content_mimetype_add_from_temp(conflict_update, cur)

		@remote_api_endpoint('content_mimetype')
@db_transaction_generator()		@db_transaction_generator()
def content_mimetype_get(self, ids, db=None, cur=None):		def content_mimetype_get(self, ids, db=None, cur=None):
"""Retrieve full content mimetype per ids.		"""Retrieve full content mimetype per ids.

Args:		Args:
ids (iterable): sha1 identifier		ids (iterable): sha1 identifier

Yields:		Yields:
mimetypes (iterable): dictionaries with keys:		mimetypes (iterable): dictionaries with keys:

id (bytes): sha1 identifier		id (bytes): sha1 identifier
mimetype (bytes): raw content's mimetype		mimetype (bytes): raw content's mimetype
encoding (bytes): raw content's encoding		encoding (bytes): raw content's encoding
tool (dict): Tool used to compute the language		tool (dict): Tool used to compute the language

"""		"""
for c in db.content_mimetype_get_from_list(ids, cur):		for c in db.content_mimetype_get_from_list(ids, cur):
yield converters.db_to_mimetype(		yield converters.db_to_mimetype(
dict(zip(db.content_mimetype_cols, c)))		dict(zip(db.content_mimetype_cols, c)))

		@remote_api_endpoint('content_language/missing')
@db_transaction_generator()		@db_transaction_generator()
def content_language_missing(self, languages, db=None, cur=None):		def content_language_missing(self, languages, db=None, cur=None):
"""List languages missing from storage.		"""List languages missing from storage.

Args:		Args:
languages (iterable): dictionaries with keys:		languages (iterable): dictionaries with keys:

id (bytes): sha1 identifier		id (bytes): sha1 identifier
indexer_configuration_id (int): tool used to compute		indexer_configuration_id (int): tool used to compute
the results		the results

Yields:		Yields:
an iterable of missing id for the tuple (id,		an iterable of missing id for the tuple (id,
indexer_configuration_id)		indexer_configuration_id)

"""		"""
for obj in db.content_language_missing_from_list(languages, cur):		for obj in db.content_language_missing_from_list(languages, cur):
yield obj[0]		yield obj[0]

		@remote_api_endpoint('content_language')
@db_transaction_generator()		@db_transaction_generator()
def content_language_get(self, ids, db=None, cur=None):		def content_language_get(self, ids, db=None, cur=None):
"""Retrieve full content language per ids.		"""Retrieve full content language per ids.

Args:		Args:
ids (iterable): sha1 identifier		ids (iterable): sha1 identifier

Yields:		Yields:
languages (iterable): dictionaries with keys:		languages (iterable): dictionaries with keys:

id (bytes): sha1 identifier		id (bytes): sha1 identifier
lang (bytes): raw content's language		lang (bytes): raw content's language
tool (dict): Tool used to compute the language		tool (dict): Tool used to compute the language

"""		"""
for c in db.content_language_get_from_list(ids, cur):		for c in db.content_language_get_from_list(ids, cur):
yield converters.db_to_language(		yield converters.db_to_language(
dict(zip(db.content_language_cols, c)))		dict(zip(db.content_language_cols, c)))

		@remote_api_endpoint('content_language/add')
@db_transaction()		@db_transaction()
def content_language_add(self, languages, conflict_update=False, db=None,		def content_language_add(self, languages, conflict_update=False, db=None,
cur=None):		cur=None):
"""Add languages not present in storage.		"""Add languages not present in storage.

Args:		Args:
languages (iterable): dictionaries with keys:		languages (iterable): dictionaries with keys:

Show All 13 Lines	def content_language_add(self, languages, conflict_update=False, db=None,
'lang': 'unknown' if not l['lang'] else l['lang'],		'lang': 'unknown' if not l['lang'] else l['lang'],
'indexer_configuration_id': l['indexer_configuration_id'],		'indexer_configuration_id': l['indexer_configuration_id'],
} for l in languages),		} for l in languages),
'tmp_content_language',		'tmp_content_language',
['id', 'lang', 'indexer_configuration_id'], cur)		['id', 'lang', 'indexer_configuration_id'], cur)

db.content_language_add_from_temp(conflict_update, cur)		db.content_language_add_from_temp(conflict_update, cur)

		@remote_api_endpoint('content/ctags/missing')
@db_transaction_generator()		@db_transaction_generator()
def content_ctags_missing(self, ctags, db=None, cur=None):		def content_ctags_missing(self, ctags, db=None, cur=None):
"""List ctags missing from storage.		"""List ctags missing from storage.

Args:		Args:
ctags (iterable): dicts with keys:		ctags (iterable): dicts with keys:

id (bytes): sha1 identifier		id (bytes): sha1 identifier
indexer_configuration_id (int): tool used to compute		indexer_configuration_id (int): tool used to compute
the results		the results

Yields:		Yields:
an iterable of missing id for the tuple (id,		an iterable of missing id for the tuple (id,
indexer_configuration_id)		indexer_configuration_id)

"""		"""
for obj in db.content_ctags_missing_from_list(ctags, cur):		for obj in db.content_ctags_missing_from_list(ctags, cur):
yield obj[0]		yield obj[0]

		@remote_api_endpoint('content/ctags')
@db_transaction_generator()		@db_transaction_generator()
def content_ctags_get(self, ids, db=None, cur=None):		def content_ctags_get(self, ids, db=None, cur=None):
"""Retrieve ctags per id.		"""Retrieve ctags per id.

Args:		Args:
ids (iterable): sha1 checksums		ids (iterable): sha1 checksums

Yields:		Yields:
Dictionaries with keys:		Dictionaries with keys:

id (bytes): content's identifier		id (bytes): content's identifier
name (str): symbol's name		name (str): symbol's name
kind (str): symbol's kind		kind (str): symbol's kind
language (str): language for that content		language (str): language for that content
tool (dict): tool used to compute the ctags' info		tool (dict): tool used to compute the ctags' info


"""		"""
for c in db.content_ctags_get_from_list(ids, cur):		for c in db.content_ctags_get_from_list(ids, cur):
yield converters.db_to_ctags(dict(zip(db.content_ctags_cols, c)))		yield converters.db_to_ctags(dict(zip(db.content_ctags_cols, c)))

		@remote_api_endpoint('content/ctags/add')
@db_transaction()		@db_transaction()
def content_ctags_add(self, ctags, conflict_update=False, db=None,		def content_ctags_add(self, ctags, conflict_update=False, db=None,
cur=None):		cur=None):
"""Add ctags not present in storage		"""Add ctags not present in storage

Args:		Args:
ctags (iterable): dictionaries with keys:		ctags (iterable): dictionaries with keys:

Show All 13 Lines	def content_ctags_add(self, ctags, conflict_update=False, db=None,
db.copy_to(list(_convert_ctags(ctags)),		db.copy_to(list(_convert_ctags(ctags)),
tblname='tmp_content_ctags',		tblname='tmp_content_ctags',
columns=['id', 'name', 'kind', 'line',		columns=['id', 'name', 'kind', 'line',
'lang', 'indexer_configuration_id'],		'lang', 'indexer_configuration_id'],
cur=cur)		cur=cur)

db.content_ctags_add_from_temp(conflict_update, cur)		db.content_ctags_add_from_temp(conflict_update, cur)

		@remote_api_endpoint('content/ctags/search')
@db_transaction_generator()		@db_transaction_generator()
def content_ctags_search(self, expression,		def content_ctags_search(self, expression,
limit=10, last_sha1=None, db=None, cur=None):		limit=10, last_sha1=None, db=None, cur=None):
"""Search through content's raw ctags symbols.		"""Search through content's raw ctags symbols.

Args:		Args:
expression (str): Expression to search for		expression (str): Expression to search for
limit (int): Number of rows to return (default to 10).		limit (int): Number of rows to return (default to 10).
last_sha1 (str): Offset from which retrieving data (default to '').		last_sha1 (str): Offset from which retrieving data (default to '').

Yields:		Yields:
rows of ctags including id, name, lang, kind, line, etc...		rows of ctags including id, name, lang, kind, line, etc...

"""		"""
for obj in db.content_ctags_search(expression, last_sha1, limit,		for obj in db.content_ctags_search(expression, last_sha1, limit,
cur=cur):		cur=cur):
yield converters.db_to_ctags(dict(zip(db.content_ctags_cols, obj)))		yield converters.db_to_ctags(dict(zip(db.content_ctags_cols, obj)))

		@remote_api_endpoint('content/fossology_license')
@db_transaction_generator()		@db_transaction_generator()
def content_fossology_license_get(self, ids, db=None, cur=None):		def content_fossology_license_get(self, ids, db=None, cur=None):
"""Retrieve licenses per id.		"""Retrieve licenses per id.

Args:		Args:
ids (iterable): sha1 checksums		ids (iterable): sha1 checksums

Yields:		Yields:
Show All 9 Lines	def content_fossology_license_get(self, ids, db=None, cur=None):
license = dict(zip(db.content_fossology_license_cols, c))		license = dict(zip(db.content_fossology_license_cols, c))

id_ = license['id']		id_ = license['id']
d[id_].append(converters.db_to_fossology_license(license))		d[id_].append(converters.db_to_fossology_license(license))

for id_, facts in d.items():		for id_, facts in d.items():
yield {id_: facts}		yield {id_: facts}

		@remote_api_endpoint('content/fossology_license/add')
@db_transaction()		@db_transaction()
def content_fossology_license_add(self, licenses, conflict_update=False,		def content_fossology_license_add(self, licenses, conflict_update=False,
db=None, cur=None):		db=None, cur=None):
"""Add licenses not present in storage.		"""Add licenses not present in storage.

Args:		Args:
licenses (iterable): dictionaries with keys:		licenses (iterable): dictionaries with keys:

Show All 17 Lines	def content_fossology_license_add(self, licenses, conflict_update=False,
'license': license,		'license': license,
} for sha1 in licenses		} for sha1 in licenses
for license in sha1['licenses']),		for license in sha1['licenses']),
tblname='tmp_content_fossology_license',		tblname='tmp_content_fossology_license',
columns=['id', 'license', 'indexer_configuration_id'],		columns=['id', 'license', 'indexer_configuration_id'],
cur=cur)		cur=cur)
db.content_fossology_license_add_from_temp(conflict_update, cur)		db.content_fossology_license_add_from_temp(conflict_update, cur)

		@remote_api_endpoint('content_metadata/missing')
@db_transaction_generator()		@db_transaction_generator()
def content_metadata_missing(self, metadata, db=None, cur=None):		def content_metadata_missing(self, metadata, db=None, cur=None):
"""List metadata missing from storage.		"""List metadata missing from storage.

Args:		Args:
metadata (iterable): dictionaries with keys:		metadata (iterable): dictionaries with keys:

id (bytes): sha1 identifier		id (bytes): sha1 identifier
indexer_configuration_id (int): tool used to compute		indexer_configuration_id (int): tool used to compute
the results		the results

Yields:		Yields:
an iterable of missing id for the tuple (id,		an iterable of missing id for the tuple (id,
indexer_configuration_id)		indexer_configuration_id)

"""		"""
for obj in db.content_metadata_missing_from_list(metadata, cur):		for obj in db.content_metadata_missing_from_list(metadata, cur):
yield obj[0]		yield obj[0]

		@remote_api_endpoint('content_metadata')
@db_transaction_generator()		@db_transaction_generator()
def content_metadata_get(self, ids, db=None, cur=None):		def content_metadata_get(self, ids, db=None, cur=None):
"""Retrieve metadata per id.		"""Retrieve metadata per id.

Args:		Args:
ids (iterable): sha1 checksums		ids (iterable): sha1 checksums

Yields:		Yields:
list: dictionaries with the following keys:		list: dictionaries with the following keys:

id (bytes)		id (bytes)
translated_metadata (str): associated metadata		translated_metadata (str): associated metadata
tool (dict): tool used to compute metadata		tool (dict): tool used to compute metadata

"""		"""
for c in db.content_metadata_get_from_list(ids, cur):		for c in db.content_metadata_get_from_list(ids, cur):
yield converters.db_to_metadata(		yield converters.db_to_metadata(
dict(zip(db.content_metadata_cols, c)))		dict(zip(db.content_metadata_cols, c)))

		@remote_api_endpoint('content_metadata/add')
@db_transaction()		@db_transaction()
def content_metadata_add(self, metadata, conflict_update=False, db=None,		def content_metadata_add(self, metadata, conflict_update=False, db=None,
cur=None):		cur=None):
"""Add metadata not present in storage.		"""Add metadata not present in storage.

Args:		Args:
metadata (iterable): dictionaries with keys:		metadata (iterable): dictionaries with keys:

id: sha1		id: sha1
translated_metadata: bytes / jsonb ?		translated_metadata: bytes / jsonb ?

conflict_update: Flag to determine if we want to overwrite (true)		conflict_update: Flag to determine if we want to overwrite (true)
or skip duplicates (false, the default)		or skip duplicates (false, the default)

"""		"""
db.mktemp_content_metadata(cur)		db.mktemp_content_metadata(cur)
# empty metadata is mapped to 'unknown'		# empty metadata is mapped to 'unknown'

db.copy_to(metadata, 'tmp_content_metadata',		db.copy_to(metadata, 'tmp_content_metadata',
['id', 'translated_metadata', 'indexer_configuration_id'],		['id', 'translated_metadata', 'indexer_configuration_id'],
cur)		cur)
db.content_metadata_add_from_temp(conflict_update, cur)		db.content_metadata_add_from_temp(conflict_update, cur)

		@remote_api_endpoint('revision_metadata/missing')
@db_transaction_generator()		@db_transaction_generator()
def revision_metadata_missing(self, metadata, db=None, cur=None):		def revision_metadata_missing(self, metadata, db=None, cur=None):
"""List metadata missing from storage.		"""List metadata missing from storage.

Args:		Args:
metadata (iterable): dictionaries with keys:		metadata (iterable): dictionaries with keys:

id (bytes): sha1_git revision identifier		id (bytes): sha1_git revision identifier
indexer_configuration_id (int): tool used to compute		indexer_configuration_id (int): tool used to compute
the results		the results

Returns:		Returns:
iterable: missing ids		iterable: missing ids

"""		"""
for obj in db.revision_metadata_missing_from_list(metadata, cur):		for obj in db.revision_metadata_missing_from_list(metadata, cur):
yield obj[0]		yield obj[0]

		@remote_api_endpoint('revision_metadata')
@db_transaction_generator()		@db_transaction_generator()
def revision_metadata_get(self, ids, db=None, cur=None):		def revision_metadata_get(self, ids, db=None, cur=None):
"""Retrieve revision metadata per id.		"""Retrieve revision metadata per id.

Args:		Args:
ids (iterable): sha1 checksums		ids (iterable): sha1 checksums

Yields:		Yields:
list: dictionaries with the following keys:		list: dictionaries with the following keys:

id (bytes)		id (bytes)
translated_metadata (str): associated metadata		translated_metadata (str): associated metadata
tool (dict): tool used to compute metadata		tool (dict): tool used to compute metadata

"""		"""
for c in db.revision_metadata_get_from_list(ids, cur):		for c in db.revision_metadata_get_from_list(ids, cur):
yield converters.db_to_metadata(		yield converters.db_to_metadata(
dict(zip(db.revision_metadata_cols, c)))		dict(zip(db.revision_metadata_cols, c)))

		@remote_api_endpoint('revision_metadata/add')
@db_transaction()		@db_transaction()
def revision_metadata_add(self, metadata, conflict_update=False, db=None,		def revision_metadata_add(self, metadata, conflict_update=False, db=None,
cur=None):		cur=None):
"""Add metadata not present in storage.		"""Add metadata not present in storage.

Args:		Args:
metadata (iterable): dictionaries with keys:		metadata (iterable): dictionaries with keys:

- id: sha1_git of revision		- id: sha1_git of revision
- translated_metadata: bytes / jsonb ?		- translated_metadata: bytes / jsonb ?

conflict_update: Flag to determine if we want to overwrite (true)		conflict_update: Flag to determine if we want to overwrite (true)
or skip duplicates (false, the default)		or skip duplicates (false, the default)

"""		"""
db.mktemp_revision_metadata(cur)		db.mktemp_revision_metadata(cur)
# empty metadata is mapped to 'unknown'		# empty metadata is mapped to 'unknown'

db.copy_to(metadata, 'tmp_revision_metadata',		db.copy_to(metadata, 'tmp_revision_metadata',
['id', 'translated_metadata', 'indexer_configuration_id'],		['id', 'translated_metadata', 'indexer_configuration_id'],
cur)		cur)
db.revision_metadata_add_from_temp(conflict_update, cur)		db.revision_metadata_add_from_temp(conflict_update, cur)

		@remote_api_endpoint('indexer_configuration/add')
@db_transaction_generator()		@db_transaction_generator()
def indexer_configuration_add(self, tools, db=None, cur=None):		def indexer_configuration_add(self, tools, db=None, cur=None):
"""Add new tools to the storage.		"""Add new tools to the storage.

Args:		Args:
tools ([dict]): List of dictionary representing tool to		tools ([dict]): List of dictionary representing tool to
insert in the db. Dictionary with the following keys::		insert in the db. Dictionary with the following keys::

Show All 12 Lines	def indexer_configuration_add(self, tools, db=None, cur=None):
db.copy_to(tools, 'tmp_indexer_configuration',		db.copy_to(tools, 'tmp_indexer_configuration',
['tool_name', 'tool_version', 'tool_configuration'],		['tool_name', 'tool_version', 'tool_configuration'],
cur)		cur)

tools = db.indexer_configuration_add_from_temp(cur)		tools = db.indexer_configuration_add_from_temp(cur)
for line in tools:		for line in tools:
yield dict(zip(db.indexer_configuration_cols, line))		yield dict(zip(db.indexer_configuration_cols, line))

		@remote_api_endpoint('indexer_configuration/data')
@db_transaction()		@db_transaction()
def indexer_configuration_get(self, tool, db=None, cur=None):		def indexer_configuration_get(self, tool, db=None, cur=None):
"""Retrieve tool information.		"""Retrieve tool information.

Args:		Args:
tool (dict): Dictionary representing a tool with the		tool (dict): Dictionary representing a tool with the
following keys::		following keys::

Show All 18 Lines

View Options

swh/indexer/storage/api/client.py

	# Copyright (C) 2015-2018 The Software Heritage developers			# Copyright (C) 2015-2018 The Software Heritage developers
	# See the AUTHORS file at the top-level directory of this distribution			# See the AUTHORS file at the top-level directory of this distribution
	# License: GNU General Public License version 3, or any later version			# License: GNU General Public License version 3, or any later version
	# See top-level LICENSE file for more information			# See top-level LICENSE file for more information


	from swh.core.api import SWHRemoteAPI			from swh.core.api import SWHRemoteAPI

	from swh.storage.exc import StorageAPIError			from swh.storage.exc import StorageAPIError

				from .. import IndexerStorage


	class RemoteStorage(SWHRemoteAPI):			class RemoteStorage(SWHRemoteAPI):
	"""Proxy to a remote storage API"""			"""Proxy to a remote storage API"""

				backend_class = IndexerStorage

	def __init__(self, url, timeout=None):			def __init__(self, url, timeout=None):
	super().__init__(			super().__init__(
	api_exception=StorageAPIError, url=url, timeout=timeout)			api_exception=StorageAPIError, url=url, timeout=timeout)
				ardumontUnsubmitted Done Inline Actions I do not see that snippet being called. If it's not, can you please remove it? ardumont: I do not see that snippet being called. If it's not, can you please remove it?
				vlorentzAuthorUnsubmitted Done Inline Actions Good catch; that was for an intermediate version of the Diff. vlorentz: Good catch; that was for an intermediate version of the Diff.

	def check_config(self, *, check_write):
	return self.post('check_config', {'check_write': check_write})

	def content_mimetype_add(self, mimetypes, conflict_update=False):
	return self.post('content_mimetype/add', {
	'mimetypes': mimetypes,
	'conflict_update': conflict_update,
	})

	def content_mimetype_missing(self, mimetypes):
	return self.post('content_mimetype/missing', {'mimetypes': mimetypes})

	def content_mimetype_get(self, ids):
	return self.post('content_mimetype', {'ids': ids})

	def content_language_add(self, languages, conflict_update=False):
	return self.post('content_language/add', {
	'languages': languages,
	'conflict_update': conflict_update,
	})

	def content_language_missing(self, languages):
	return self.post('content_language/missing', {'languages': languages})

	def content_language_get(self, ids):
	return self.post('content_language', {'ids': ids})

	def content_ctags_add(self, ctags, conflict_update=False):
	return self.post('content/ctags/add', {
	'ctags': ctags,
	'conflict_update': conflict_update,
	})

	def content_ctags_missing(self, ctags):
	return self.post('content/ctags/missing', {'ctags': ctags})

	def content_ctags_get(self, ids):
	return self.post('content/ctags', {'ids': ids})

	def content_ctags_search(self, expression, limit=10, last_sha1=None):
	return self.post('content/ctags/search', {
	'expression': expression,
	'limit': limit,
	'last_sha1': last_sha1,
	})

	def content_fossology_license_add(self, licenses, conflict_update=False):
	return self.post('content/fossology_license/add', {
	'licenses': licenses,
	'conflict_update': conflict_update,
	})

	def content_fossology_license_get(self, ids):
	return self.post('content/fossology_license', {'ids': ids})

	def content_metadata_add(self, metadata, conflict_update=False):
	return self.post('content_metadata/add', {
	'metadata': metadata,
	'conflict_update': conflict_update,
	})

	def content_metadata_missing(self, metadata):
	return self.post('content_metadata/missing', {'metadata': metadata})

	def content_metadata_get(self, ids):
	return self.post('content_metadata', {'ids': ids})

	def revision_metadata_add(self, metadata, conflict_update=False):
	return self.post('revision_metadata/add', {
	'metadata': metadata,
	'conflict_update': conflict_update,
	})

	def revision_metadata_missing(self, metadata):
	return self.post('revision_metadata/missing', {'metadata': metadata})

	def revision_metadata_get(self, ids):
	return self.post('revision_metadata', {'ids': ids})

	def indexer_configuration_add(self, tools):
	return self.post('indexer_configuration/add', {'tools': tools})

	def indexer_configuration_get(self, tool):
	return self.post('indexer_configuration/data', {'tool': tool})

View Options

swh/indexer/storage/api/server.py

	# Copyright (C) 2015-2018 The Software Heritage developers			# Copyright (C) 2015-2018 The Software Heritage developers
	# See the AUTHORS file at the top-level directory of this distribution			# See the AUTHORS file at the top-level directory of this distribution
	# License: GNU General Public License version 3, or any later version			# License: GNU General Public License version 3, or any later version
	# See top-level LICENSE file for more information			# See top-level LICENSE file for more information

	import logging			import logging
	import click			import click

	from flask import request

	from swh.core import config			from swh.core import config
	from swh.core.api import (SWHServerAPIApp, decode_request,			from swh.core.api import (SWHServerAPIApp, error_handler,
	error_handler,
	encode_data_server as encode_data)			encode_data_server as encode_data)
	from swh.indexer.storage import get_indexer_storage, INDEXER_CFG_KEY			from swh.indexer.storage import get_indexer_storage, INDEXER_CFG_KEY

				from .. import IndexerStorage


	DEFAULT_CONFIG_PATH = 'storage/indexer'			DEFAULT_CONFIG_PATH = 'storage/indexer'
	DEFAULT_CONFIG = {			DEFAULT_CONFIG = {
	INDEXER_CFG_KEY: ('dict', {			INDEXER_CFG_KEY: ('dict', {
	'cls': 'local',			'cls': 'local',
	'args': {			'args': {
	'db': 'dbname=softwareheritage-indexer-dev',			'db': 'dbname=softwareheritage-indexer-dev',
	},			},
	})			})
	}			}


	app = SWHServerAPIApp(__name__)
	storage = None


	@app.errorhandler(Exception)
	def my_error_handler(exception):
	return error_handler(exception, encode_data)


	def get_storage():			def get_storage():
	global storage			global storage
	if not storage:			if not storage:
	storage = get_indexer_storage(**app.config[INDEXER_CFG_KEY])			storage = get_indexer_storage(**app.config[INDEXER_CFG_KEY])

	return storage			return storage


	@app.route('/')			app = SWHServerAPIApp(__name__,
	def index():			backend_class=IndexerStorage,
	return 'SWH Indexer Storage API server'			backend_factory=get_storage)
				storage = None

	@app.route('/check_config', methods=['POST'])
	def check_config():
	return encode_data(get_storage().check_config(**decode_request(request)))


	@app.route('/content_mimetype/add', methods=['POST'])
	def content_mimetype_add():
	return encode_data(
	get_storage().content_mimetype_add(**decode_request(request)))


	@app.route('/content_mimetype/missing', methods=['POST'])
	def content_mimetype_missing():
	return encode_data(
	get_storage().content_mimetype_missing(**decode_request(request)))


	@app.route('/content_mimetype', methods=['POST'])
	def content_mimetype_get():
	return encode_data(
	get_storage().content_mimetype_get(**decode_request(request)))


	@app.route('/content_language/add', methods=['POST'])
	def content_language_add():
	return encode_data(
	get_storage().content_language_add(**decode_request(request)))


	@app.route('/content_language/missing', methods=['POST'])
	def content_language_missing():
	return encode_data(
	get_storage().content_language_missing(**decode_request(request)))


	@app.route('/content_language', methods=['POST'])
	def content_language_get():
	return encode_data(
	get_storage().content_language_get(**decode_request(request)))


	@app.route('/content/ctags/add', methods=['POST'])
	def content_ctags_add():
	return encode_data(
	get_storage().content_ctags_add(**decode_request(request)))


	@app.route('/content/ctags/search', methods=['POST'])
	def content_ctags_search():
	return encode_data(
	get_storage().content_ctags_search(**decode_request(request)))


	@app.route('/content/ctags/missing', methods=['POST'])
	def content_ctags_missing():
	return encode_data(
	get_storage().content_ctags_missing(**decode_request(request)))


	@app.route('/content/ctags', methods=['POST'])
	def content_ctags_get():
	return encode_data(
	get_storage().content_ctags_get(**decode_request(request)))


	@app.route('/content/fossology_license/add', methods=['POST'])
	def content_fossology_license_add():
	return encode_data(
	get_storage().content_fossology_license_add(**decode_request(request)))


	@app.route('/content/fossology_license', methods=['POST'])
	def content_fossology_license_get():
	return encode_data(
	get_storage().content_fossology_license_get(**decode_request(request)))


	@app.route('/indexer_configuration/data', methods=['POST'])
	def indexer_configuration_get():
	return encode_data(get_storage().indexer_configuration_get(
	**decode_request(request)))


	@app.route('/indexer_configuration/add', methods=['POST'])
	def indexer_configuration_add():
	return encode_data(get_storage().indexer_configuration_add(
	**decode_request(request)))


	@app.route('/content_metadata/add', methods=['POST'])
	def content_metadata_add():
	return encode_data(
	get_storage().content_metadata_add(**decode_request(request)))


	@app.route('/content_metadata/missing', methods=['POST'])
	def content_metadata_missing():
	return encode_data(
	get_storage().content_metadata_missing(**decode_request(request)))


	@app.route('/content_metadata', methods=['POST'])
	def content_metadata_get():
	return encode_data(
	get_storage().content_metadata_get(**decode_request(request)))


	@app.route('/revision_metadata/add', methods=['POST'])
	def revision_metadata_add():
	return encode_data(
	get_storage().revision_metadata_add(**decode_request(request)))


	@app.route('/revision_metadata/missing', methods=['POST'])			@app.errorhandler(Exception)
	def revision_metadata_missing():			def my_error_handler(exception):
	return encode_data(			return error_handler(exception, encode_data)
	get_storage().revision_metadata_missing(**decode_request(request)))


	@app.route('/revision_metadata', methods=['POST'])			@app.route('/')
	def revision_metadata_get():			def index():
	return encode_data(			return 'SWH Indexer Storage API server'
	get_storage().revision_metadata_get(**decode_request(request)))


	def run_from_webserver(environ, start_response,			def run_from_webserver(environ, start_response,
	config_path=DEFAULT_CONFIG_PATH):			config_path=DEFAULT_CONFIG_PATH):
	"""Run the WSGI app from the webserver, loading the configuration."""			"""Run the WSGI app from the webserver, loading the configuration."""
	cfg = config.load_named_config(config_path, DEFAULT_CONFIG)			cfg = config.load_named_config(config_path, DEFAULT_CONFIG)
	app.config.update(cfg)			app.config.update(cfg)
	handler = logging.StreamHandler()			handler = logging.StreamHandler()
	Show All 18 Lines

Autogenerate the Indexer Storage HTTP API.
ClosedPublic
Actions

Details

Diff Detail

Event Timeline

Revision Contents
Changeset List

Diff 1594

swh/indexer/storage/init.py

swh/indexer/storage/api/client.py

swh/indexer/storage/api/server.py

Autogenerate the Indexer Storage HTTP API.ClosedPublicActions

Details

Diff Detail

Event Timeline

Revision ContentsChangeset List

Diff 1594

swh/indexer/storage/__init__.py

swh/indexer/storage/api/client.py

swh/indexer/storage/api/server.py

Autogenerate the Indexer Storage HTTP API.
ClosedPublic
Actions

Revision Contents
Changeset List

swh/indexer/storage/init.py