Diff 2488

swh/indexer/fossology_license.py

# Copyright (C) 2015-2018 The Software Heritage developers		# Copyright (C) 2015-2018 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution		# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version		# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information		# See top-level LICENSE file for more information


import json		import json
import psycopg2		import psycopg2

from collections import defaultdict		from collections import defaultdict

		from swh.core.api import remote_api_endpoint
from swh.storage.common import db_transaction_generator, db_transaction		from swh.storage.common import db_transaction_generator, db_transaction
from swh.storage.exc import StorageDBError		from swh.storage.exc import StorageDBError
from .db import Db		from .db import Db

from . import converters		from . import converters


INDEXER_CFG_KEY = 'indexer_storage'		INDEXER_CFG_KEY = 'indexer_storage'


def get_indexer_storage(cls, args):		def get_indexer_storage(cls, args):
		ardumontUnsubmitted Not Done Inline Actions Don't we want to keep the original docstring as well here (@functools.wraps)? ardumont: Don't we want to keep the original docstring as well here (@functools.wraps)?
		vlorentzAuthorUnsubmitted Done Inline Actions The original function is returned, so there is no wrapper. vlorentz: The original function is returned, so there is no wrapper.
		ardumontUnsubmitted Not Done Inline Actions Right! ardumont: Right!
"""Get an indexer storage object of class `storage_class` with		"""Get an indexer storage object of class `storage_class` with
arguments `storage_args`.		arguments `storage_args`.

Args:		Args:
args (dict): dictionary with keys:		args (dict): dictionary with keys:
- cls (str): storage's class, either 'local' or 'remote'		- cls (str): storage's class, either 'local' or 'remote'
- args (dict): dictionary with keys		- args (dict): dictionary with keys

Show All 9 Lines	def get_indexer_storage(cls, args):
elif cls == 'local':		elif cls == 'local':
from . import IndexerStorage		from . import IndexerStorage
else:		else:
raise ValueError('Unknown indexer storage class `%s`' % cls)		raise ValueError('Unknown indexer storage class `%s`' % cls)

return IndexerStorage(**args)		return IndexerStorage(**args)


class IndexerStorage():		class IndexerStorage:
"""SWH Indexer Storage		"""SWH Indexer Storage

"""		"""
def __init__(self, db, min_pool_conns=1, max_pool_conns=10):		def __init__(self, db, min_pool_conns=1, max_pool_conns=10):
"""		"""
Args:		Args:
db_conn: either a libpq connection string, or a psycopg2 connection		db_conn: either a libpq connection string, or a psycopg2 connection

Show All 10 Lines	def __init__(self, db, min_pool_conns=1, max_pool_conns=10):
except psycopg2.OperationalError as e:		except psycopg2.OperationalError as e:
raise StorageDBError(e)		raise StorageDBError(e)

def get_db(self):		def get_db(self):
if self._db:		if self._db:
return self._db		return self._db
return Db.from_pool(self._pool)		return Db.from_pool(self._pool)

		@remote_api_endpoint('check_config')
def check_config(self, *, check_write):		def check_config(self, *, check_write):
"""Check that the storage is configured and ready to go."""		"""Check that the storage is configured and ready to go."""
# Check permissions on one of the tables		# Check permissions on one of the tables
with self.get_db().transaction() as cur:		with self.get_db().transaction() as cur:
if check_write:		if check_write:
check = 'INSERT'		check = 'INSERT'
else:		else:
check = 'SELECT'		check = 'SELECT'

cur.execute(		cur.execute(
"select has_table_privilege(current_user, 'content_mimetype', %s)", # noqa		"select has_table_privilege(current_user, 'content_mimetype', %s)", # noqa
(check,)		(check,)
)		)
return cur.fetchone()[0]		return cur.fetchone()[0]

return True		return True

		@remote_api_endpoint('content_mimetype/missing')
@db_transaction_generator()		@db_transaction_generator()
def content_mimetype_missing(self, mimetypes, db=None, cur=None):		def content_mimetype_missing(self, mimetypes, db=None, cur=None):
"""List mimetypes missing from storage.		"""List mimetypes missing from storage.

Args:		Args:
mimetypes (iterable): iterable of dict with keys:		mimetypes (iterable): iterable of dict with keys:

id (bytes): sha1 identifier		id (bytes): sha1 identifier
indexer_configuration_id (int): tool used to compute		indexer_configuration_id (int): tool used to compute
the results		the results

Yields:		Yields:
an iterable of missing id for the tuple (id,		an iterable of missing id for the tuple (id,
indexer_configuration_id)		indexer_configuration_id)

"""		"""
for obj in db.content_mimetype_missing_from_list(mimetypes, cur):		for obj in db.content_mimetype_missing_from_list(mimetypes, cur):
yield obj[0]		yield obj[0]

		@remote_api_endpoint('content_mimetype/add')
@db_transaction()		@db_transaction()
def content_mimetype_add(self, mimetypes, conflict_update=False, db=None,		def content_mimetype_add(self, mimetypes, conflict_update=False, db=None,
cur=None):		cur=None):
"""Add mimetypes not present in storage.		"""Add mimetypes not present in storage.

Args:		Args:
mimetypes (iterable): dictionaries with keys:		mimetypes (iterable): dictionaries with keys:

id (bytes): sha1 identifier		id (bytes): sha1 identifier
mimetype (bytes): raw content's mimetype		mimetype (bytes): raw content's mimetype
encoding (bytes): raw content's encoding		encoding (bytes): raw content's encoding
indexer_configuration_id (int): tool's id used to		indexer_configuration_id (int): tool's id used to
compute the results		compute the results
conflict_update (bool): Flag to determine if we want to		conflict_update (bool): Flag to determine if we want to
overwrite (true) or skip duplicates		overwrite (true) or skip duplicates
(false, the default)		(false, the default)

"""		"""
db.mktemp_content_mimetype(cur)		db.mktemp_content_mimetype(cur)
db.copy_to(mimetypes, 'tmp_content_mimetype',		db.copy_to(mimetypes, 'tmp_content_mimetype',
['id', 'mimetype', 'encoding', 'indexer_configuration_id'],		['id', 'mimetype', 'encoding', 'indexer_configuration_id'],
cur)		cur)
db.content_mimetype_add_from_temp(conflict_update, cur)		db.content_mimetype_add_from_temp(conflict_update, cur)

		@remote_api_endpoint('content_mimetype')
@db_transaction_generator()		@db_transaction_generator()
def content_mimetype_get(self, ids, db=None, cur=None):		def content_mimetype_get(self, ids, db=None, cur=None):
"""Retrieve full content mimetype per ids.		"""Retrieve full content mimetype per ids.

Args:		Args:
ids (iterable): sha1 identifier		ids (iterable): sha1 identifier

Yields:		Yields:
mimetypes (iterable): dictionaries with keys:		mimetypes (iterable): dictionaries with keys:

id (bytes): sha1 identifier		id (bytes): sha1 identifier
mimetype (bytes): raw content's mimetype		mimetype (bytes): raw content's mimetype
encoding (bytes): raw content's encoding		encoding (bytes): raw content's encoding
tool (dict): Tool used to compute the language		tool (dict): Tool used to compute the language

"""		"""
for c in db.content_mimetype_get_from_list(ids, cur):		for c in db.content_mimetype_get_from_list(ids, cur):
yield converters.db_to_mimetype(		yield converters.db_to_mimetype(
dict(zip(db.content_mimetype_cols, c)))		dict(zip(db.content_mimetype_cols, c)))

		@remote_api_endpoint('content_language/missing')
@db_transaction_generator()		@db_transaction_generator()
def content_language_missing(self, languages, db=None, cur=None):		def content_language_missing(self, languages, db=None, cur=None):
"""List languages missing from storage.		"""List languages missing from storage.

Args:		Args:
languages (iterable): dictionaries with keys:		languages (iterable): dictionaries with keys:

id (bytes): sha1 identifier		id (bytes): sha1 identifier
indexer_configuration_id (int): tool used to compute		indexer_configuration_id (int): tool used to compute
the results		the results

Yields:		Yields:
an iterable of missing id for the tuple (id,		an iterable of missing id for the tuple (id,
indexer_configuration_id)		indexer_configuration_id)

"""		"""
for obj in db.content_language_missing_from_list(languages, cur):		for obj in db.content_language_missing_from_list(languages, cur):
yield obj[0]		yield obj[0]

		@remote_api_endpoint('content_language')
@db_transaction_generator()		@db_transaction_generator()
def content_language_get(self, ids, db=None, cur=None):		def content_language_get(self, ids, db=None, cur=None):
"""Retrieve full content language per ids.		"""Retrieve full content language per ids.

Args:		Args:
ids (iterable): sha1 identifier		ids (iterable): sha1 identifier

Yields:		Yields:
languages (iterable): dictionaries with keys:		languages (iterable): dictionaries with keys:

id (bytes): sha1 identifier		id (bytes): sha1 identifier
lang (bytes): raw content's language		lang (bytes): raw content's language
tool (dict): Tool used to compute the language		tool (dict): Tool used to compute the language

"""		"""
for c in db.content_language_get_from_list(ids, cur):		for c in db.content_language_get_from_list(ids, cur):
yield converters.db_to_language(		yield converters.db_to_language(
dict(zip(db.content_language_cols, c)))		dict(zip(db.content_language_cols, c)))

		@remote_api_endpoint('content_language/add')
@db_transaction()		@db_transaction()
def content_language_add(self, languages, conflict_update=False, db=None,		def content_language_add(self, languages, conflict_update=False, db=None,
cur=None):		cur=None):
"""Add languages not present in storage.		"""Add languages not present in storage.

Args:		Args:
languages (iterable): dictionaries with keys:		languages (iterable): dictionaries with keys:

Show All 13 Lines	def content_language_add(self, languages, conflict_update=False, db=None,
'lang': 'unknown' if not l['lang'] else l['lang'],		'lang': 'unknown' if not l['lang'] else l['lang'],
'indexer_configuration_id': l['indexer_configuration_id'],		'indexer_configuration_id': l['indexer_configuration_id'],
} for l in languages),		} for l in languages),
'tmp_content_language',		'tmp_content_language',
['id', 'lang', 'indexer_configuration_id'], cur)		['id', 'lang', 'indexer_configuration_id'], cur)

db.content_language_add_from_temp(conflict_update, cur)		db.content_language_add_from_temp(conflict_update, cur)

		@remote_api_endpoint('content/ctags/missing')
@db_transaction_generator()		@db_transaction_generator()
def content_ctags_missing(self, ctags, db=None, cur=None):		def content_ctags_missing(self, ctags, db=None, cur=None):
"""List ctags missing from storage.		"""List ctags missing from storage.

Args:		Args:
ctags (iterable): dicts with keys:		ctags (iterable): dicts with keys:

id (bytes): sha1 identifier		id (bytes): sha1 identifier
indexer_configuration_id (int): tool used to compute		indexer_configuration_id (int): tool used to compute
the results		the results

Yields:		Yields:
an iterable of missing id for the tuple (id,		an iterable of missing id for the tuple (id,
indexer_configuration_id)		indexer_configuration_id)

"""		"""
for obj in db.content_ctags_missing_from_list(ctags, cur):		for obj in db.content_ctags_missing_from_list(ctags, cur):
yield obj[0]		yield obj[0]

		@remote_api_endpoint('content/ctags')
@db_transaction_generator()		@db_transaction_generator()
def content_ctags_get(self, ids, db=None, cur=None):		def content_ctags_get(self, ids, db=None, cur=None):
"""Retrieve ctags per id.		"""Retrieve ctags per id.

Args:		Args:
ids (iterable): sha1 checksums		ids (iterable): sha1 checksums

Yields:		Yields:
Dictionaries with keys:		Dictionaries with keys:

id (bytes): content's identifier		id (bytes): content's identifier
name (str): symbol's name		name (str): symbol's name
kind (str): symbol's kind		kind (str): symbol's kind
language (str): language for that content		language (str): language for that content
tool (dict): tool used to compute the ctags' info		tool (dict): tool used to compute the ctags' info


"""		"""
for c in db.content_ctags_get_from_list(ids, cur):		for c in db.content_ctags_get_from_list(ids, cur):
yield converters.db_to_ctags(dict(zip(db.content_ctags_cols, c)))		yield converters.db_to_ctags(dict(zip(db.content_ctags_cols, c)))

		@remote_api_endpoint('content/ctags/add')
@db_transaction()		@db_transaction()
def content_ctags_add(self, ctags, conflict_update=False, db=None,		def content_ctags_add(self, ctags, conflict_update=False, db=None,
cur=None):		cur=None):
"""Add ctags not present in storage		"""Add ctags not present in storage

Args:		Args:
ctags (iterable): dictionaries with keys:		ctags (iterable): dictionaries with keys:

Show All 13 Lines	def content_ctags_add(self, ctags, conflict_update=False, db=None,
db.copy_to(list(_convert_ctags(ctags)),		db.copy_to(list(_convert_ctags(ctags)),
tblname='tmp_content_ctags',		tblname='tmp_content_ctags',
columns=['id', 'name', 'kind', 'line',		columns=['id', 'name', 'kind', 'line',
'lang', 'indexer_configuration_id'],		'lang', 'indexer_configuration_id'],
cur=cur)		cur=cur)

db.content_ctags_add_from_temp(conflict_update, cur)		db.content_ctags_add_from_temp(conflict_update, cur)

		@remote_api_endpoint('content/ctags/search')
@db_transaction_generator()		@db_transaction_generator()
def content_ctags_search(self, expression,		def content_ctags_search(self, expression,
limit=10, last_sha1=None, db=None, cur=None):		limit=10, last_sha1=None, db=None, cur=None):
"""Search through content's raw ctags symbols.		"""Search through content's raw ctags symbols.

Args:		Args:
expression (str): Expression to search for		expression (str): Expression to search for
limit (int): Number of rows to return (default to 10).		limit (int): Number of rows to return (default to 10).
last_sha1 (str): Offset from which retrieving data (default to '').		last_sha1 (str): Offset from which retrieving data (default to '').

Yields:		Yields:
rows of ctags including id, name, lang, kind, line, etc...		rows of ctags including id, name, lang, kind, line, etc...

"""		"""
for obj in db.content_ctags_search(expression, last_sha1, limit,		for obj in db.content_ctags_search(expression, last_sha1, limit,
cur=cur):		cur=cur):
yield converters.db_to_ctags(dict(zip(db.content_ctags_cols, obj)))		yield converters.db_to_ctags(dict(zip(db.content_ctags_cols, obj)))

		@remote_api_endpoint('content/fossology_license')
@db_transaction_generator()		@db_transaction_generator()
def content_fossology_license_get(self, ids, db=None, cur=None):		def content_fossology_license_get(self, ids, db=None, cur=None):
"""Retrieve licenses per id.		"""Retrieve licenses per id.

Args:		Args:
ids (iterable): sha1 checksums		ids (iterable): sha1 checksums

Yields:		Yields:
Show All 9 Lines	def content_fossology_license_get(self, ids, db=None, cur=None):
license = dict(zip(db.content_fossology_license_cols, c))		license = dict(zip(db.content_fossology_license_cols, c))

id_ = license['id']		id_ = license['id']
d[id_].append(converters.db_to_fossology_license(license))		d[id_].append(converters.db_to_fossology_license(license))

for id_, facts in d.items():		for id_, facts in d.items():
yield {id_: facts}		yield {id_: facts}

		@remote_api_endpoint('content/fossology_license/add')
@db_transaction()		@db_transaction()
def content_fossology_license_add(self, licenses, conflict_update=False,		def content_fossology_license_add(self, licenses, conflict_update=False,
db=None, cur=None):		db=None, cur=None):
"""Add licenses not present in storage.		"""Add licenses not present in storage.

Args:		Args:
licenses (iterable): dictionaries with keys:		licenses (iterable): dictionaries with keys:

Show All 17 Lines	def content_fossology_license_add(self, licenses, conflict_update=False,
'license': license,		'license': license,
} for sha1 in licenses		} for sha1 in licenses
for license in sha1['licenses']),		for license in sha1['licenses']),
tblname='tmp_content_fossology_license',		tblname='tmp_content_fossology_license',
columns=['id', 'license', 'indexer_configuration_id'],		columns=['id', 'license', 'indexer_configuration_id'],
cur=cur)		cur=cur)
db.content_fossology_license_add_from_temp(conflict_update, cur)		db.content_fossology_license_add_from_temp(conflict_update, cur)

		@remote_api_endpoint('content_metadata/missing')
@db_transaction_generator()		@db_transaction_generator()
def content_metadata_missing(self, metadata, db=None, cur=None):		def content_metadata_missing(self, metadata, db=None, cur=None):
"""List metadata missing from storage.		"""List metadata missing from storage.

Args:		Args:
metadata (iterable): dictionaries with keys:		metadata (iterable): dictionaries with keys:

id (bytes): sha1 identifier		id (bytes): sha1 identifier
indexer_configuration_id (int): tool used to compute		indexer_configuration_id (int): tool used to compute
the results		the results

Yields:		Yields:
an iterable of missing id for the tuple (id,		an iterable of missing id for the tuple (id,
indexer_configuration_id)		indexer_configuration_id)

"""		"""
for obj in db.content_metadata_missing_from_list(metadata, cur):		for obj in db.content_metadata_missing_from_list(metadata, cur):
yield obj[0]		yield obj[0]

		@remote_api_endpoint('content_metadata')
@db_transaction_generator()		@db_transaction_generator()
def content_metadata_get(self, ids, db=None, cur=None):		def content_metadata_get(self, ids, db=None, cur=None):
"""Retrieve metadata per id.		"""Retrieve metadata per id.

Args:		Args:
ids (iterable): sha1 checksums		ids (iterable): sha1 checksums

Yields:		Yields:
list: dictionaries with the following keys:		list: dictionaries with the following keys:

id (bytes)		id (bytes)
translated_metadata (str): associated metadata		translated_metadata (str): associated metadata
tool (dict): tool used to compute metadata		tool (dict): tool used to compute metadata

"""		"""
for c in db.content_metadata_get_from_list(ids, cur):		for c in db.content_metadata_get_from_list(ids, cur):
yield converters.db_to_metadata(		yield converters.db_to_metadata(
dict(zip(db.content_metadata_cols, c)))		dict(zip(db.content_metadata_cols, c)))

		@remote_api_endpoint('content_metadata/add')
@db_transaction()		@db_transaction()
def content_metadata_add(self, metadata, conflict_update=False, db=None,		def content_metadata_add(self, metadata, conflict_update=False, db=None,
cur=None):		cur=None):
"""Add metadata not present in storage.		"""Add metadata not present in storage.

Args:		Args:
metadata (iterable): dictionaries with keys:		metadata (iterable): dictionaries with keys:

id: sha1		id: sha1
translated_metadata: bytes / jsonb ?		translated_metadata: bytes / jsonb ?

conflict_update: Flag to determine if we want to overwrite (true)		conflict_update: Flag to determine if we want to overwrite (true)
or skip duplicates (false, the default)		or skip duplicates (false, the default)

"""		"""
db.mktemp_content_metadata(cur)		db.mktemp_content_metadata(cur)
# empty metadata is mapped to 'unknown'		# empty metadata is mapped to 'unknown'

db.copy_to(metadata, 'tmp_content_metadata',		db.copy_to(metadata, 'tmp_content_metadata',
['id', 'translated_metadata', 'indexer_configuration_id'],		['id', 'translated_metadata', 'indexer_configuration_id'],
cur)		cur)
db.content_metadata_add_from_temp(conflict_update, cur)		db.content_metadata_add_from_temp(conflict_update, cur)

		@remote_api_endpoint('revision_metadata/missing')
@db_transaction_generator()		@db_transaction_generator()
def revision_metadata_missing(self, metadata, db=None, cur=None):		def revision_metadata_missing(self, metadata, db=None, cur=None):
"""List metadata missing from storage.		"""List metadata missing from storage.

Args:		Args:
metadata (iterable): dictionaries with keys:		metadata (iterable): dictionaries with keys:

id (bytes): sha1_git revision identifier		id (bytes): sha1_git revision identifier
indexer_configuration_id (int): tool used to compute		indexer_configuration_id (int): tool used to compute
the results		the results

Returns:		Returns:
iterable: missing ids		iterable: missing ids

"""		"""
for obj in db.revision_metadata_missing_from_list(metadata, cur):		for obj in db.revision_metadata_missing_from_list(metadata, cur):
yield obj[0]		yield obj[0]

		@remote_api_endpoint('revision_metadata')
@db_transaction_generator()		@db_transaction_generator()
def revision_metadata_get(self, ids, db=None, cur=None):		def revision_metadata_get(self, ids, db=None, cur=None):
"""Retrieve revision metadata per id.		"""Retrieve revision metadata per id.

Args:		Args:
ids (iterable): sha1 checksums		ids (iterable): sha1 checksums

Yields:		Yields:
list: dictionaries with the following keys:		list: dictionaries with the following keys:

id (bytes)		id (bytes)
translated_metadata (str): associated metadata		translated_metadata (str): associated metadata
tool (dict): tool used to compute metadata		tool (dict): tool used to compute metadata

"""		"""
for c in db.revision_metadata_get_from_list(ids, cur):		for c in db.revision_metadata_get_from_list(ids, cur):
yield converters.db_to_metadata(		yield converters.db_to_metadata(
dict(zip(db.revision_metadata_cols, c)))		dict(zip(db.revision_metadata_cols, c)))

		@remote_api_endpoint('revision_metadata/add')
@db_transaction()		@db_transaction()
def revision_metadata_add(self, metadata, conflict_update=False, db=None,		def revision_metadata_add(self, metadata, conflict_update=False, db=None,
cur=None):		cur=None):
"""Add metadata not present in storage.		"""Add metadata not present in storage.

Args:		Args:
metadata (iterable): dictionaries with keys:		metadata (iterable): dictionaries with keys:

- id: sha1_git of revision		- id: sha1_git of revision
- translated_metadata: bytes / jsonb ?		- translated_metadata: bytes / jsonb ?

conflict_update: Flag to determine if we want to overwrite (true)		conflict_update: Flag to determine if we want to overwrite (true)
or skip duplicates (false, the default)		or skip duplicates (false, the default)

"""		"""
db.mktemp_revision_metadata(cur)		db.mktemp_revision_metadata(cur)
# empty metadata is mapped to 'unknown'		# empty metadata is mapped to 'unknown'

db.copy_to(metadata, 'tmp_revision_metadata',		db.copy_to(metadata, 'tmp_revision_metadata',
['id', 'translated_metadata', 'indexer_configuration_id'],		['id', 'translated_metadata', 'indexer_configuration_id'],
cur)		cur)
db.revision_metadata_add_from_temp(conflict_update, cur)		db.revision_metadata_add_from_temp(conflict_update, cur)

		@remote_api_endpoint('indexer_configuration/add')
@db_transaction_generator()		@db_transaction_generator()
def indexer_configuration_add(self, tools, db=None, cur=None):		def indexer_configuration_add(self, tools, db=None, cur=None):
"""Add new tools to the storage.		"""Add new tools to the storage.

Args:		Args:
tools ([dict]): List of dictionary representing tool to		tools ([dict]): List of dictionary representing tool to
insert in the db. Dictionary with the following keys::		insert in the db. Dictionary with the following keys::

Show All 12 Lines	def indexer_configuration_add(self, tools, db=None, cur=None):
db.copy_to(tools, 'tmp_indexer_configuration',		db.copy_to(tools, 'tmp_indexer_configuration',
['tool_name', 'tool_version', 'tool_configuration'],		['tool_name', 'tool_version', 'tool_configuration'],
cur)		cur)

tools = db.indexer_configuration_add_from_temp(cur)		tools = db.indexer_configuration_add_from_temp(cur)
for line in tools:		for line in tools:
yield dict(zip(db.indexer_configuration_cols, line))		yield dict(zip(db.indexer_configuration_cols, line))

		@remote_api_endpoint('indexer_configuration/data')
@db_transaction()		@db_transaction()
def indexer_configuration_get(self, tool, db=None, cur=None):		def indexer_configuration_get(self, tool, db=None, cur=None):
"""Retrieve tool information.		"""Retrieve tool information.

Args:		Args:
tool (dict): Dictionary representing a tool with the		tool (dict): Dictionary representing a tool with the
following keys::		following keys::

Show All 18 Lines

View Options

swh/indexer/indexer.py

# Copyright (C) 2015-2018 The Software Heritage developers		# Copyright (C) 2015-2018 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution		# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version		# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information		# See top-level LICENSE file for more information


import json		import json
import psycopg2		import psycopg2

from collections import defaultdict		from collections import defaultdict

		from swh.core.api import remote_api_endpoint
from swh.storage.common import db_transaction_generator, db_transaction		from swh.storage.common import db_transaction_generator, db_transaction
from swh.storage.exc import StorageDBError		from swh.storage.exc import StorageDBError
from .db import Db		from .db import Db

from . import converters		from . import converters


INDEXER_CFG_KEY = 'indexer_storage'		INDEXER_CFG_KEY = 'indexer_storage'


def get_indexer_storage(cls, args):		def get_indexer_storage(cls, args):
		ardumontUnsubmitted Not Done Inline Actions Don't we want to keep the original docstring as well here (@functools.wraps)? ardumont: Don't we want to keep the original docstring as well here (@functools.wraps)?
		vlorentzAuthorUnsubmitted Done Inline Actions The original function is returned, so there is no wrapper. vlorentz: The original function is returned, so there is no wrapper.
		ardumontUnsubmitted Not Done Inline Actions Right! ardumont: Right!
"""Get an indexer storage object of class `storage_class` with		"""Get an indexer storage object of class `storage_class` with
arguments `storage_args`.		arguments `storage_args`.

Args:		Args:
args (dict): dictionary with keys:		args (dict): dictionary with keys:
- cls (str): storage's class, either 'local' or 'remote'		- cls (str): storage's class, either 'local' or 'remote'
- args (dict): dictionary with keys		- args (dict): dictionary with keys

Show All 9 Lines	def get_indexer_storage(cls, args):
elif cls == 'local':		elif cls == 'local':
from . import IndexerStorage		from . import IndexerStorage
else:		else:
raise ValueError('Unknown indexer storage class `%s`' % cls)		raise ValueError('Unknown indexer storage class `%s`' % cls)

return IndexerStorage(**args)		return IndexerStorage(**args)


class IndexerStorage():		class IndexerStorage:
"""SWH Indexer Storage		"""SWH Indexer Storage

"""		"""
def __init__(self, db, min_pool_conns=1, max_pool_conns=10):		def __init__(self, db, min_pool_conns=1, max_pool_conns=10):
"""		"""
Args:		Args:
db_conn: either a libpq connection string, or a psycopg2 connection		db_conn: either a libpq connection string, or a psycopg2 connection

Show All 10 Lines	def __init__(self, db, min_pool_conns=1, max_pool_conns=10):
except psycopg2.OperationalError as e:		except psycopg2.OperationalError as e:
raise StorageDBError(e)		raise StorageDBError(e)

def get_db(self):		def get_db(self):
if self._db:		if self._db:
return self._db		return self._db
return Db.from_pool(self._pool)		return Db.from_pool(self._pool)

		@remote_api_endpoint('check_config')
def check_config(self, *, check_write):		def check_config(self, *, check_write):
"""Check that the storage is configured and ready to go."""		"""Check that the storage is configured and ready to go."""
# Check permissions on one of the tables		# Check permissions on one of the tables
with self.get_db().transaction() as cur:		with self.get_db().transaction() as cur:
if check_write:		if check_write:
check = 'INSERT'		check = 'INSERT'
else:		else:
check = 'SELECT'		check = 'SELECT'

cur.execute(		cur.execute(
"select has_table_privilege(current_user, 'content_mimetype', %s)", # noqa		"select has_table_privilege(current_user, 'content_mimetype', %s)", # noqa
(check,)		(check,)
)		)
return cur.fetchone()[0]		return cur.fetchone()[0]

return True		return True

		@remote_api_endpoint('content_mimetype/missing')
@db_transaction_generator()		@db_transaction_generator()
def content_mimetype_missing(self, mimetypes, db=None, cur=None):		def content_mimetype_missing(self, mimetypes, db=None, cur=None):
"""List mimetypes missing from storage.		"""List mimetypes missing from storage.

Args:		Args:
mimetypes (iterable): iterable of dict with keys:		mimetypes (iterable): iterable of dict with keys:

id (bytes): sha1 identifier		id (bytes): sha1 identifier
indexer_configuration_id (int): tool used to compute		indexer_configuration_id (int): tool used to compute
the results		the results

Yields:		Yields:
an iterable of missing id for the tuple (id,		an iterable of missing id for the tuple (id,
indexer_configuration_id)		indexer_configuration_id)

"""		"""
for obj in db.content_mimetype_missing_from_list(mimetypes, cur):		for obj in db.content_mimetype_missing_from_list(mimetypes, cur):
yield obj[0]		yield obj[0]

		@remote_api_endpoint('content_mimetype/add')
@db_transaction()		@db_transaction()
def content_mimetype_add(self, mimetypes, conflict_update=False, db=None,		def content_mimetype_add(self, mimetypes, conflict_update=False, db=None,
cur=None):		cur=None):
"""Add mimetypes not present in storage.		"""Add mimetypes not present in storage.

Args:		Args:
mimetypes (iterable): dictionaries with keys:		mimetypes (iterable): dictionaries with keys:

id (bytes): sha1 identifier		id (bytes): sha1 identifier
mimetype (bytes): raw content's mimetype		mimetype (bytes): raw content's mimetype
encoding (bytes): raw content's encoding		encoding (bytes): raw content's encoding
indexer_configuration_id (int): tool's id used to		indexer_configuration_id (int): tool's id used to
compute the results		compute the results
conflict_update (bool): Flag to determine if we want to		conflict_update (bool): Flag to determine if we want to
overwrite (true) or skip duplicates		overwrite (true) or skip duplicates
(false, the default)		(false, the default)

"""		"""
db.mktemp_content_mimetype(cur)		db.mktemp_content_mimetype(cur)
db.copy_to(mimetypes, 'tmp_content_mimetype',		db.copy_to(mimetypes, 'tmp_content_mimetype',
['id', 'mimetype', 'encoding', 'indexer_configuration_id'],		['id', 'mimetype', 'encoding', 'indexer_configuration_id'],
cur)		cur)
db.content_mimetype_add_from_temp(conflict_update, cur)		db.content_mimetype_add_from_temp(conflict_update, cur)

		@remote_api_endpoint('content_mimetype')
@db_transaction_generator()		@db_transaction_generator()
def content_mimetype_get(self, ids, db=None, cur=None):		def content_mimetype_get(self, ids, db=None, cur=None):
"""Retrieve full content mimetype per ids.		"""Retrieve full content mimetype per ids.

Args:		Args:
ids (iterable): sha1 identifier		ids (iterable): sha1 identifier

Yields:		Yields:
mimetypes (iterable): dictionaries with keys:		mimetypes (iterable): dictionaries with keys:

id (bytes): sha1 identifier		id (bytes): sha1 identifier
mimetype (bytes): raw content's mimetype		mimetype (bytes): raw content's mimetype
encoding (bytes): raw content's encoding		encoding (bytes): raw content's encoding
tool (dict): Tool used to compute the language		tool (dict): Tool used to compute the language

"""		"""
for c in db.content_mimetype_get_from_list(ids, cur):		for c in db.content_mimetype_get_from_list(ids, cur):
yield converters.db_to_mimetype(		yield converters.db_to_mimetype(
dict(zip(db.content_mimetype_cols, c)))		dict(zip(db.content_mimetype_cols, c)))

		@remote_api_endpoint('content_language/missing')
@db_transaction_generator()		@db_transaction_generator()
def content_language_missing(self, languages, db=None, cur=None):		def content_language_missing(self, languages, db=None, cur=None):
"""List languages missing from storage.		"""List languages missing from storage.

Args:		Args:
languages (iterable): dictionaries with keys:		languages (iterable): dictionaries with keys:

id (bytes): sha1 identifier		id (bytes): sha1 identifier
indexer_configuration_id (int): tool used to compute		indexer_configuration_id (int): tool used to compute
the results		the results

Yields:		Yields:
an iterable of missing id for the tuple (id,		an iterable of missing id for the tuple (id,
indexer_configuration_id)		indexer_configuration_id)

"""		"""
for obj in db.content_language_missing_from_list(languages, cur):		for obj in db.content_language_missing_from_list(languages, cur):
yield obj[0]		yield obj[0]

		@remote_api_endpoint('content_language')
@db_transaction_generator()		@db_transaction_generator()
def content_language_get(self, ids, db=None, cur=None):		def content_language_get(self, ids, db=None, cur=None):
"""Retrieve full content language per ids.		"""Retrieve full content language per ids.

Args:		Args:
ids (iterable): sha1 identifier		ids (iterable): sha1 identifier

Yields:		Yields:
languages (iterable): dictionaries with keys:		languages (iterable): dictionaries with keys:

id (bytes): sha1 identifier		id (bytes): sha1 identifier
lang (bytes): raw content's language		lang (bytes): raw content's language
tool (dict): Tool used to compute the language		tool (dict): Tool used to compute the language

"""		"""
for c in db.content_language_get_from_list(ids, cur):		for c in db.content_language_get_from_list(ids, cur):
yield converters.db_to_language(		yield converters.db_to_language(
dict(zip(db.content_language_cols, c)))		dict(zip(db.content_language_cols, c)))

		@remote_api_endpoint('content_language/add')
@db_transaction()		@db_transaction()
def content_language_add(self, languages, conflict_update=False, db=None,		def content_language_add(self, languages, conflict_update=False, db=None,
cur=None):		cur=None):
"""Add languages not present in storage.		"""Add languages not present in storage.

Args:		Args:
languages (iterable): dictionaries with keys:		languages (iterable): dictionaries with keys:

Show All 13 Lines	def content_language_add(self, languages, conflict_update=False, db=None,
'lang': 'unknown' if not l['lang'] else l['lang'],		'lang': 'unknown' if not l['lang'] else l['lang'],
'indexer_configuration_id': l['indexer_configuration_id'],		'indexer_configuration_id': l['indexer_configuration_id'],
} for l in languages),		} for l in languages),
'tmp_content_language',		'tmp_content_language',
['id', 'lang', 'indexer_configuration_id'], cur)		['id', 'lang', 'indexer_configuration_id'], cur)

db.content_language_add_from_temp(conflict_update, cur)		db.content_language_add_from_temp(conflict_update, cur)

		@remote_api_endpoint('content/ctags/missing')
@db_transaction_generator()		@db_transaction_generator()
def content_ctags_missing(self, ctags, db=None, cur=None):		def content_ctags_missing(self, ctags, db=None, cur=None):
"""List ctags missing from storage.		"""List ctags missing from storage.

Args:		Args:
ctags (iterable): dicts with keys:		ctags (iterable): dicts with keys:

id (bytes): sha1 identifier		id (bytes): sha1 identifier
indexer_configuration_id (int): tool used to compute		indexer_configuration_id (int): tool used to compute
the results		the results

Yields:		Yields:
an iterable of missing id for the tuple (id,		an iterable of missing id for the tuple (id,
indexer_configuration_id)		indexer_configuration_id)

"""		"""
for obj in db.content_ctags_missing_from_list(ctags, cur):		for obj in db.content_ctags_missing_from_list(ctags, cur):
yield obj[0]		yield obj[0]

		@remote_api_endpoint('content/ctags')
@db_transaction_generator()		@db_transaction_generator()
def content_ctags_get(self, ids, db=None, cur=None):		def content_ctags_get(self, ids, db=None, cur=None):
"""Retrieve ctags per id.		"""Retrieve ctags per id.

Args:		Args:
ids (iterable): sha1 checksums		ids (iterable): sha1 checksums

Yields:		Yields:
Dictionaries with keys:		Dictionaries with keys:

id (bytes): content's identifier		id (bytes): content's identifier
name (str): symbol's name		name (str): symbol's name
kind (str): symbol's kind		kind (str): symbol's kind
language (str): language for that content		language (str): language for that content
tool (dict): tool used to compute the ctags' info		tool (dict): tool used to compute the ctags' info


"""		"""
for c in db.content_ctags_get_from_list(ids, cur):		for c in db.content_ctags_get_from_list(ids, cur):
yield converters.db_to_ctags(dict(zip(db.content_ctags_cols, c)))		yield converters.db_to_ctags(dict(zip(db.content_ctags_cols, c)))

		@remote_api_endpoint('content/ctags/add')
@db_transaction()		@db_transaction()
def content_ctags_add(self, ctags, conflict_update=False, db=None,		def content_ctags_add(self, ctags, conflict_update=False, db=None,
cur=None):		cur=None):
"""Add ctags not present in storage		"""Add ctags not present in storage

Args:		Args:
ctags (iterable): dictionaries with keys:		ctags (iterable): dictionaries with keys:

Show All 13 Lines	def content_ctags_add(self, ctags, conflict_update=False, db=None,
db.copy_to(list(_convert_ctags(ctags)),		db.copy_to(list(_convert_ctags(ctags)),
tblname='tmp_content_ctags',		tblname='tmp_content_ctags',
columns=['id', 'name', 'kind', 'line',		columns=['id', 'name', 'kind', 'line',
'lang', 'indexer_configuration_id'],		'lang', 'indexer_configuration_id'],
cur=cur)		cur=cur)

db.content_ctags_add_from_temp(conflict_update, cur)		db.content_ctags_add_from_temp(conflict_update, cur)

		@remote_api_endpoint('content/ctags/search')
@db_transaction_generator()		@db_transaction_generator()
def content_ctags_search(self, expression,		def content_ctags_search(self, expression,
limit=10, last_sha1=None, db=None, cur=None):		limit=10, last_sha1=None, db=None, cur=None):
"""Search through content's raw ctags symbols.		"""Search through content's raw ctags symbols.

Args:		Args:
expression (str): Expression to search for		expression (str): Expression to search for
limit (int): Number of rows to return (default to 10).		limit (int): Number of rows to return (default to 10).
last_sha1 (str): Offset from which retrieving data (default to '').		last_sha1 (str): Offset from which retrieving data (default to '').

Yields:		Yields:
rows of ctags including id, name, lang, kind, line, etc...		rows of ctags including id, name, lang, kind, line, etc...

"""		"""
for obj in db.content_ctags_search(expression, last_sha1, limit,		for obj in db.content_ctags_search(expression, last_sha1, limit,
cur=cur):		cur=cur):
yield converters.db_to_ctags(dict(zip(db.content_ctags_cols, obj)))		yield converters.db_to_ctags(dict(zip(db.content_ctags_cols, obj)))

		@remote_api_endpoint('content/fossology_license')
@db_transaction_generator()		@db_transaction_generator()
def content_fossology_license_get(self, ids, db=None, cur=None):		def content_fossology_license_get(self, ids, db=None, cur=None):
"""Retrieve licenses per id.		"""Retrieve licenses per id.

Args:		Args:
ids (iterable): sha1 checksums		ids (iterable): sha1 checksums

Yields:		Yields:
Show All 9 Lines	def content_fossology_license_get(self, ids, db=None, cur=None):
license = dict(zip(db.content_fossology_license_cols, c))		license = dict(zip(db.content_fossology_license_cols, c))

id_ = license['id']		id_ = license['id']
d[id_].append(converters.db_to_fossology_license(license))		d[id_].append(converters.db_to_fossology_license(license))

for id_, facts in d.items():		for id_, facts in d.items():
yield {id_: facts}		yield {id_: facts}

		@remote_api_endpoint('content/fossology_license/add')
@db_transaction()		@db_transaction()
def content_fossology_license_add(self, licenses, conflict_update=False,		def content_fossology_license_add(self, licenses, conflict_update=False,
db=None, cur=None):		db=None, cur=None):
"""Add licenses not present in storage.		"""Add licenses not present in storage.

Args:		Args:
licenses (iterable): dictionaries with keys:		licenses (iterable): dictionaries with keys:

Show All 17 Lines	def content_fossology_license_add(self, licenses, conflict_update=False,
'license': license,		'license': license,
} for sha1 in licenses		} for sha1 in licenses
for license in sha1['licenses']),		for license in sha1['licenses']),
tblname='tmp_content_fossology_license',		tblname='tmp_content_fossology_license',
columns=['id', 'license', 'indexer_configuration_id'],		columns=['id', 'license', 'indexer_configuration_id'],
cur=cur)		cur=cur)
db.content_fossology_license_add_from_temp(conflict_update, cur)		db.content_fossology_license_add_from_temp(conflict_update, cur)

		@remote_api_endpoint('content_metadata/missing')
@db_transaction_generator()		@db_transaction_generator()
def content_metadata_missing(self, metadata, db=None, cur=None):		def content_metadata_missing(self, metadata, db=None, cur=None):
"""List metadata missing from storage.		"""List metadata missing from storage.

Args:		Args:
metadata (iterable): dictionaries with keys:		metadata (iterable): dictionaries with keys:

id (bytes): sha1 identifier		id (bytes): sha1 identifier
indexer_configuration_id (int): tool used to compute		indexer_configuration_id (int): tool used to compute
the results		the results

Yields:		Yields:
an iterable of missing id for the tuple (id,		an iterable of missing id for the tuple (id,
indexer_configuration_id)		indexer_configuration_id)

"""		"""
for obj in db.content_metadata_missing_from_list(metadata, cur):		for obj in db.content_metadata_missing_from_list(metadata, cur):
yield obj[0]		yield obj[0]

		@remote_api_endpoint('content_metadata')
@db_transaction_generator()		@db_transaction_generator()
def content_metadata_get(self, ids, db=None, cur=None):		def content_metadata_get(self, ids, db=None, cur=None):
"""Retrieve metadata per id.		"""Retrieve metadata per id.

Args:		Args:
ids (iterable): sha1 checksums		ids (iterable): sha1 checksums

Yields:		Yields:
list: dictionaries with the following keys:		list: dictionaries with the following keys:

id (bytes)		id (bytes)
translated_metadata (str): associated metadata		translated_metadata (str): associated metadata
tool (dict): tool used to compute metadata		tool (dict): tool used to compute metadata

"""		"""
for c in db.content_metadata_get_from_list(ids, cur):		for c in db.content_metadata_get_from_list(ids, cur):
yield converters.db_to_metadata(		yield converters.db_to_metadata(
dict(zip(db.content_metadata_cols, c)))		dict(zip(db.content_metadata_cols, c)))

		@remote_api_endpoint('content_metadata/add')
@db_transaction()		@db_transaction()
def content_metadata_add(self, metadata, conflict_update=False, db=None,		def content_metadata_add(self, metadata, conflict_update=False, db=None,
cur=None):		cur=None):
"""Add metadata not present in storage.		"""Add metadata not present in storage.

Args:		Args:
metadata (iterable): dictionaries with keys:		metadata (iterable): dictionaries with keys:

id: sha1		id: sha1
translated_metadata: bytes / jsonb ?		translated_metadata: bytes / jsonb ?

conflict_update: Flag to determine if we want to overwrite (true)		conflict_update: Flag to determine if we want to overwrite (true)
or skip duplicates (false, the default)		or skip duplicates (false, the default)

"""		"""
db.mktemp_content_metadata(cur)		db.mktemp_content_metadata(cur)
# empty metadata is mapped to 'unknown'		# empty metadata is mapped to 'unknown'

db.copy_to(metadata, 'tmp_content_metadata',		db.copy_to(metadata, 'tmp_content_metadata',
['id', 'translated_metadata', 'indexer_configuration_id'],		['id', 'translated_metadata', 'indexer_configuration_id'],
cur)		cur)
db.content_metadata_add_from_temp(conflict_update, cur)		db.content_metadata_add_from_temp(conflict_update, cur)

		@remote_api_endpoint('revision_metadata/missing')
@db_transaction_generator()		@db_transaction_generator()
def revision_metadata_missing(self, metadata, db=None, cur=None):		def revision_metadata_missing(self, metadata, db=None, cur=None):
"""List metadata missing from storage.		"""List metadata missing from storage.

Args:		Args:
metadata (iterable): dictionaries with keys:		metadata (iterable): dictionaries with keys:

id (bytes): sha1_git revision identifier		id (bytes): sha1_git revision identifier
indexer_configuration_id (int): tool used to compute		indexer_configuration_id (int): tool used to compute
the results		the results

Returns:		Returns:
iterable: missing ids		iterable: missing ids

"""		"""
for obj in db.revision_metadata_missing_from_list(metadata, cur):		for obj in db.revision_metadata_missing_from_list(metadata, cur):
yield obj[0]		yield obj[0]

		@remote_api_endpoint('revision_metadata')
@db_transaction_generator()		@db_transaction_generator()
def revision_metadata_get(self, ids, db=None, cur=None):		def revision_metadata_get(self, ids, db=None, cur=None):
"""Retrieve revision metadata per id.		"""Retrieve revision metadata per id.

Args:		Args:
ids (iterable): sha1 checksums		ids (iterable): sha1 checksums

Yields:		Yields:
list: dictionaries with the following keys:		list: dictionaries with the following keys:

id (bytes)		id (bytes)
translated_metadata (str): associated metadata		translated_metadata (str): associated metadata
tool (dict): tool used to compute metadata		tool (dict): tool used to compute metadata

"""		"""
for c in db.revision_metadata_get_from_list(ids, cur):		for c in db.revision_metadata_get_from_list(ids, cur):
yield converters.db_to_metadata(		yield converters.db_to_metadata(
dict(zip(db.revision_metadata_cols, c)))		dict(zip(db.revision_metadata_cols, c)))

		@remote_api_endpoint('revision_metadata/add')
@db_transaction()		@db_transaction()
def revision_metadata_add(self, metadata, conflict_update=False, db=None,		def revision_metadata_add(self, metadata, conflict_update=False, db=None,
cur=None):		cur=None):
"""Add metadata not present in storage.		"""Add metadata not present in storage.

Args:		Args:
metadata (iterable): dictionaries with keys:		metadata (iterable): dictionaries with keys:

- id: sha1_git of revision		- id: sha1_git of revision
- translated_metadata: bytes / jsonb ?		- translated_metadata: bytes / jsonb ?

conflict_update: Flag to determine if we want to overwrite (true)		conflict_update: Flag to determine if we want to overwrite (true)
or skip duplicates (false, the default)		or skip duplicates (false, the default)

"""		"""
db.mktemp_revision_metadata(cur)		db.mktemp_revision_metadata(cur)
# empty metadata is mapped to 'unknown'		# empty metadata is mapped to 'unknown'

db.copy_to(metadata, 'tmp_revision_metadata',		db.copy_to(metadata, 'tmp_revision_metadata',
['id', 'translated_metadata', 'indexer_configuration_id'],		['id', 'translated_metadata', 'indexer_configuration_id'],
cur)		cur)
db.revision_metadata_add_from_temp(conflict_update, cur)		db.revision_metadata_add_from_temp(conflict_update, cur)

		@remote_api_endpoint('indexer_configuration/add')
@db_transaction_generator()		@db_transaction_generator()
def indexer_configuration_add(self, tools, db=None, cur=None):		def indexer_configuration_add(self, tools, db=None, cur=None):
"""Add new tools to the storage.		"""Add new tools to the storage.

Args:		Args:
tools ([dict]): List of dictionary representing tool to		tools ([dict]): List of dictionary representing tool to
insert in the db. Dictionary with the following keys::		insert in the db. Dictionary with the following keys::

Show All 12 Lines	def indexer_configuration_add(self, tools, db=None, cur=None):
db.copy_to(tools, 'tmp_indexer_configuration',		db.copy_to(tools, 'tmp_indexer_configuration',
['tool_name', 'tool_version', 'tool_configuration'],		['tool_name', 'tool_version', 'tool_configuration'],
cur)		cur)

tools = db.indexer_configuration_add_from_temp(cur)		tools = db.indexer_configuration_add_from_temp(cur)
for line in tools:		for line in tools:
yield dict(zip(db.indexer_configuration_cols, line))		yield dict(zip(db.indexer_configuration_cols, line))

		@remote_api_endpoint('indexer_configuration/data')
@db_transaction()		@db_transaction()
def indexer_configuration_get(self, tool, db=None, cur=None):		def indexer_configuration_get(self, tool, db=None, cur=None):
"""Retrieve tool information.		"""Retrieve tool information.

Args:		Args:
tool (dict): Dictionary representing a tool with the		tool (dict): Dictionary representing a tool with the
following keys::		following keys::

Show All 18 Lines

View Options

swh/indexer/storage/init.py

# Copyright (C) 2015-2018 The Software Heritage developers		# Copyright (C) 2015-2018 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution		# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version		# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information		# See top-level LICENSE file for more information


import json		import json
import psycopg2		import psycopg2

from collections import defaultdict		from collections import defaultdict

		from swh.core.api import remote_api_endpoint
from swh.storage.common import db_transaction_generator, db_transaction		from swh.storage.common import db_transaction_generator, db_transaction
from swh.storage.exc import StorageDBError		from swh.storage.exc import StorageDBError
from .db import Db		from .db import Db

from . import converters		from . import converters


INDEXER_CFG_KEY = 'indexer_storage'		INDEXER_CFG_KEY = 'indexer_storage'


def get_indexer_storage(cls, args):		def get_indexer_storage(cls, args):
		ardumontUnsubmitted Not Done Inline Actions Don't we want to keep the original docstring as well here (@functools.wraps)? ardumont: Don't we want to keep the original docstring as well here (@functools.wraps)?
		vlorentzAuthorUnsubmitted Done Inline Actions The original function is returned, so there is no wrapper. vlorentz: The original function is returned, so there is no wrapper.
		ardumontUnsubmitted Not Done Inline Actions Right! ardumont: Right!
"""Get an indexer storage object of class `storage_class` with		"""Get an indexer storage object of class `storage_class` with
arguments `storage_args`.		arguments `storage_args`.

Args:		Args:
args (dict): dictionary with keys:		args (dict): dictionary with keys:
- cls (str): storage's class, either 'local' or 'remote'		- cls (str): storage's class, either 'local' or 'remote'
- args (dict): dictionary with keys		- args (dict): dictionary with keys

Show All 9 Lines	def get_indexer_storage(cls, args):
elif cls == 'local':		elif cls == 'local':
from . import IndexerStorage		from . import IndexerStorage
else:		else:
raise ValueError('Unknown indexer storage class `%s`' % cls)		raise ValueError('Unknown indexer storage class `%s`' % cls)

return IndexerStorage(**args)		return IndexerStorage(**args)


class IndexerStorage():		class IndexerStorage:
"""SWH Indexer Storage		"""SWH Indexer Storage

"""		"""
def __init__(self, db, min_pool_conns=1, max_pool_conns=10):		def __init__(self, db, min_pool_conns=1, max_pool_conns=10):
"""		"""
Args:		Args:
db_conn: either a libpq connection string, or a psycopg2 connection		db_conn: either a libpq connection string, or a psycopg2 connection

Show All 10 Lines	def __init__(self, db, min_pool_conns=1, max_pool_conns=10):
except psycopg2.OperationalError as e:		except psycopg2.OperationalError as e:
raise StorageDBError(e)		raise StorageDBError(e)

def get_db(self):		def get_db(self):
if self._db:		if self._db:
return self._db		return self._db
return Db.from_pool(self._pool)		return Db.from_pool(self._pool)

		@remote_api_endpoint('check_config')
def check_config(self, *, check_write):		def check_config(self, *, check_write):
"""Check that the storage is configured and ready to go."""		"""Check that the storage is configured and ready to go."""
# Check permissions on one of the tables		# Check permissions on one of the tables
with self.get_db().transaction() as cur:		with self.get_db().transaction() as cur:
if check_write:		if check_write:
check = 'INSERT'		check = 'INSERT'
else:		else:
check = 'SELECT'		check = 'SELECT'

cur.execute(		cur.execute(
"select has_table_privilege(current_user, 'content_mimetype', %s)", # noqa		"select has_table_privilege(current_user, 'content_mimetype', %s)", # noqa
(check,)		(check,)
)		)
return cur.fetchone()[0]		return cur.fetchone()[0]

return True		return True

		@remote_api_endpoint('content_mimetype/missing')
@db_transaction_generator()		@db_transaction_generator()
def content_mimetype_missing(self, mimetypes, db=None, cur=None):		def content_mimetype_missing(self, mimetypes, db=None, cur=None):
"""List mimetypes missing from storage.		"""List mimetypes missing from storage.

Args:		Args:
mimetypes (iterable): iterable of dict with keys:		mimetypes (iterable): iterable of dict with keys:

id (bytes): sha1 identifier		id (bytes): sha1 identifier
indexer_configuration_id (int): tool used to compute		indexer_configuration_id (int): tool used to compute
the results		the results

Yields:		Yields:
an iterable of missing id for the tuple (id,		an iterable of missing id for the tuple (id,
indexer_configuration_id)		indexer_configuration_id)

"""		"""
for obj in db.content_mimetype_missing_from_list(mimetypes, cur):		for obj in db.content_mimetype_missing_from_list(mimetypes, cur):
yield obj[0]		yield obj[0]

		@remote_api_endpoint('content_mimetype/add')
@db_transaction()		@db_transaction()
def content_mimetype_add(self, mimetypes, conflict_update=False, db=None,		def content_mimetype_add(self, mimetypes, conflict_update=False, db=None,
cur=None):		cur=None):
"""Add mimetypes not present in storage.		"""Add mimetypes not present in storage.

Args:		Args:
mimetypes (iterable): dictionaries with keys:		mimetypes (iterable): dictionaries with keys:

id (bytes): sha1 identifier		id (bytes): sha1 identifier
mimetype (bytes): raw content's mimetype		mimetype (bytes): raw content's mimetype
encoding (bytes): raw content's encoding		encoding (bytes): raw content's encoding
indexer_configuration_id (int): tool's id used to		indexer_configuration_id (int): tool's id used to
compute the results		compute the results
conflict_update (bool): Flag to determine if we want to		conflict_update (bool): Flag to determine if we want to
overwrite (true) or skip duplicates		overwrite (true) or skip duplicates
(false, the default)		(false, the default)

"""		"""
db.mktemp_content_mimetype(cur)		db.mktemp_content_mimetype(cur)
db.copy_to(mimetypes, 'tmp_content_mimetype',		db.copy_to(mimetypes, 'tmp_content_mimetype',
['id', 'mimetype', 'encoding', 'indexer_configuration_id'],		['id', 'mimetype', 'encoding', 'indexer_configuration_id'],
cur)		cur)
db.content_mimetype_add_from_temp(conflict_update, cur)		db.content_mimetype_add_from_temp(conflict_update, cur)

		@remote_api_endpoint('content_mimetype')
@db_transaction_generator()		@db_transaction_generator()
def content_mimetype_get(self, ids, db=None, cur=None):		def content_mimetype_get(self, ids, db=None, cur=None):
"""Retrieve full content mimetype per ids.		"""Retrieve full content mimetype per ids.

Args:		Args:
ids (iterable): sha1 identifier		ids (iterable): sha1 identifier

Yields:		Yields:
mimetypes (iterable): dictionaries with keys:		mimetypes (iterable): dictionaries with keys:

id (bytes): sha1 identifier		id (bytes): sha1 identifier
mimetype (bytes): raw content's mimetype		mimetype (bytes): raw content's mimetype
encoding (bytes): raw content's encoding		encoding (bytes): raw content's encoding
tool (dict): Tool used to compute the language		tool (dict): Tool used to compute the language

"""		"""
for c in db.content_mimetype_get_from_list(ids, cur):		for c in db.content_mimetype_get_from_list(ids, cur):
yield converters.db_to_mimetype(		yield converters.db_to_mimetype(
dict(zip(db.content_mimetype_cols, c)))		dict(zip(db.content_mimetype_cols, c)))

		@remote_api_endpoint('content_language/missing')
@db_transaction_generator()		@db_transaction_generator()
def content_language_missing(self, languages, db=None, cur=None):		def content_language_missing(self, languages, db=None, cur=None):
"""List languages missing from storage.		"""List languages missing from storage.

Args:		Args:
languages (iterable): dictionaries with keys:		languages (iterable): dictionaries with keys:

id (bytes): sha1 identifier		id (bytes): sha1 identifier
indexer_configuration_id (int): tool used to compute		indexer_configuration_id (int): tool used to compute
the results		the results

Yields:		Yields:
an iterable of missing id for the tuple (id,		an iterable of missing id for the tuple (id,
indexer_configuration_id)		indexer_configuration_id)

"""		"""
for obj in db.content_language_missing_from_list(languages, cur):		for obj in db.content_language_missing_from_list(languages, cur):
yield obj[0]		yield obj[0]

		@remote_api_endpoint('content_language')
@db_transaction_generator()		@db_transaction_generator()
def content_language_get(self, ids, db=None, cur=None):		def content_language_get(self, ids, db=None, cur=None):
"""Retrieve full content language per ids.		"""Retrieve full content language per ids.

Args:		Args:
ids (iterable): sha1 identifier		ids (iterable): sha1 identifier

Yields:		Yields:
languages (iterable): dictionaries with keys:		languages (iterable): dictionaries with keys:

id (bytes): sha1 identifier		id (bytes): sha1 identifier
lang (bytes): raw content's language		lang (bytes): raw content's language
tool (dict): Tool used to compute the language		tool (dict): Tool used to compute the language

"""		"""
for c in db.content_language_get_from_list(ids, cur):		for c in db.content_language_get_from_list(ids, cur):
yield converters.db_to_language(		yield converters.db_to_language(
dict(zip(db.content_language_cols, c)))		dict(zip(db.content_language_cols, c)))

		@remote_api_endpoint('content_language/add')
@db_transaction()		@db_transaction()
def content_language_add(self, languages, conflict_update=False, db=None,		def content_language_add(self, languages, conflict_update=False, db=None,
cur=None):		cur=None):
"""Add languages not present in storage.		"""Add languages not present in storage.

Args:		Args:
languages (iterable): dictionaries with keys:		languages (iterable): dictionaries with keys:

Show All 13 Lines	def content_language_add(self, languages, conflict_update=False, db=None,
'lang': 'unknown' if not l['lang'] else l['lang'],		'lang': 'unknown' if not l['lang'] else l['lang'],
'indexer_configuration_id': l['indexer_configuration_id'],		'indexer_configuration_id': l['indexer_configuration_id'],
} for l in languages),		} for l in languages),
'tmp_content_language',		'tmp_content_language',
['id', 'lang', 'indexer_configuration_id'], cur)		['id', 'lang', 'indexer_configuration_id'], cur)

db.content_language_add_from_temp(conflict_update, cur)		db.content_language_add_from_temp(conflict_update, cur)

		@remote_api_endpoint('content/ctags/missing')
@db_transaction_generator()		@db_transaction_generator()
def content_ctags_missing(self, ctags, db=None, cur=None):		def content_ctags_missing(self, ctags, db=None, cur=None):
"""List ctags missing from storage.		"""List ctags missing from storage.

Args:		Args:
ctags (iterable): dicts with keys:		ctags (iterable): dicts with keys:

id (bytes): sha1 identifier		id (bytes): sha1 identifier
indexer_configuration_id (int): tool used to compute		indexer_configuration_id (int): tool used to compute
the results		the results

Yields:		Yields:
an iterable of missing id for the tuple (id,		an iterable of missing id for the tuple (id,
indexer_configuration_id)		indexer_configuration_id)

"""		"""
for obj in db.content_ctags_missing_from_list(ctags, cur):		for obj in db.content_ctags_missing_from_list(ctags, cur):
yield obj[0]		yield obj[0]

		@remote_api_endpoint('content/ctags')
@db_transaction_generator()		@db_transaction_generator()
def content_ctags_get(self, ids, db=None, cur=None):		def content_ctags_get(self, ids, db=None, cur=None):
"""Retrieve ctags per id.		"""Retrieve ctags per id.

Args:		Args:
ids (iterable): sha1 checksums		ids (iterable): sha1 checksums

Yields:		Yields:
Dictionaries with keys:		Dictionaries with keys:

id (bytes): content's identifier		id (bytes): content's identifier
name (str): symbol's name		name (str): symbol's name
kind (str): symbol's kind		kind (str): symbol's kind
language (str): language for that content		language (str): language for that content
tool (dict): tool used to compute the ctags' info		tool (dict): tool used to compute the ctags' info


"""		"""
for c in db.content_ctags_get_from_list(ids, cur):		for c in db.content_ctags_get_from_list(ids, cur):
yield converters.db_to_ctags(dict(zip(db.content_ctags_cols, c)))		yield converters.db_to_ctags(dict(zip(db.content_ctags_cols, c)))

		@remote_api_endpoint('content/ctags/add')
@db_transaction()		@db_transaction()
def content_ctags_add(self, ctags, conflict_update=False, db=None,		def content_ctags_add(self, ctags, conflict_update=False, db=None,
cur=None):		cur=None):
"""Add ctags not present in storage		"""Add ctags not present in storage

Args:		Args:
ctags (iterable): dictionaries with keys:		ctags (iterable): dictionaries with keys:

Show All 13 Lines	def content_ctags_add(self, ctags, conflict_update=False, db=None,
db.copy_to(list(_convert_ctags(ctags)),		db.copy_to(list(_convert_ctags(ctags)),
tblname='tmp_content_ctags',		tblname='tmp_content_ctags',
columns=['id', 'name', 'kind', 'line',		columns=['id', 'name', 'kind', 'line',
'lang', 'indexer_configuration_id'],		'lang', 'indexer_configuration_id'],
cur=cur)		cur=cur)

db.content_ctags_add_from_temp(conflict_update, cur)		db.content_ctags_add_from_temp(conflict_update, cur)

		@remote_api_endpoint('content/ctags/search')
@db_transaction_generator()		@db_transaction_generator()
def content_ctags_search(self, expression,		def content_ctags_search(self, expression,
limit=10, last_sha1=None, db=None, cur=None):		limit=10, last_sha1=None, db=None, cur=None):
"""Search through content's raw ctags symbols.		"""Search through content's raw ctags symbols.

Args:		Args:
expression (str): Expression to search for		expression (str): Expression to search for
limit (int): Number of rows to return (default to 10).		limit (int): Number of rows to return (default to 10).
last_sha1 (str): Offset from which retrieving data (default to '').		last_sha1 (str): Offset from which retrieving data (default to '').

Yields:		Yields:
rows of ctags including id, name, lang, kind, line, etc...		rows of ctags including id, name, lang, kind, line, etc...

"""		"""
for obj in db.content_ctags_search(expression, last_sha1, limit,		for obj in db.content_ctags_search(expression, last_sha1, limit,
cur=cur):		cur=cur):
yield converters.db_to_ctags(dict(zip(db.content_ctags_cols, obj)))		yield converters.db_to_ctags(dict(zip(db.content_ctags_cols, obj)))

		@remote_api_endpoint('content/fossology_license')
@db_transaction_generator()		@db_transaction_generator()
def content_fossology_license_get(self, ids, db=None, cur=None):		def content_fossology_license_get(self, ids, db=None, cur=None):
"""Retrieve licenses per id.		"""Retrieve licenses per id.

Args:		Args:
ids (iterable): sha1 checksums		ids (iterable): sha1 checksums

Yields:		Yields:
Show All 9 Lines	def content_fossology_license_get(self, ids, db=None, cur=None):
license = dict(zip(db.content_fossology_license_cols, c))		license = dict(zip(db.content_fossology_license_cols, c))

id_ = license['id']		id_ = license['id']
d[id_].append(converters.db_to_fossology_license(license))		d[id_].append(converters.db_to_fossology_license(license))

for id_, facts in d.items():		for id_, facts in d.items():
yield {id_: facts}		yield {id_: facts}

		@remote_api_endpoint('content/fossology_license/add')
@db_transaction()		@db_transaction()
def content_fossology_license_add(self, licenses, conflict_update=False,		def content_fossology_license_add(self, licenses, conflict_update=False,
db=None, cur=None):		db=None, cur=None):
"""Add licenses not present in storage.		"""Add licenses not present in storage.

Args:		Args:
licenses (iterable): dictionaries with keys:		licenses (iterable): dictionaries with keys:

Show All 17 Lines	def content_fossology_license_add(self, licenses, conflict_update=False,
'license': license,		'license': license,
} for sha1 in licenses		} for sha1 in licenses
for license in sha1['licenses']),		for license in sha1['licenses']),
tblname='tmp_content_fossology_license',		tblname='tmp_content_fossology_license',
columns=['id', 'license', 'indexer_configuration_id'],		columns=['id', 'license', 'indexer_configuration_id'],
cur=cur)		cur=cur)
db.content_fossology_license_add_from_temp(conflict_update, cur)		db.content_fossology_license_add_from_temp(conflict_update, cur)

		@remote_api_endpoint('content_metadata/missing')
@db_transaction_generator()		@db_transaction_generator()
def content_metadata_missing(self, metadata, db=None, cur=None):		def content_metadata_missing(self, metadata, db=None, cur=None):
"""List metadata missing from storage.		"""List metadata missing from storage.

Args:		Args:
metadata (iterable): dictionaries with keys:		metadata (iterable): dictionaries with keys:

id (bytes): sha1 identifier		id (bytes): sha1 identifier
indexer_configuration_id (int): tool used to compute		indexer_configuration_id (int): tool used to compute
the results		the results

Yields:		Yields:
an iterable of missing id for the tuple (id,		an iterable of missing id for the tuple (id,
indexer_configuration_id)		indexer_configuration_id)

"""		"""
for obj in db.content_metadata_missing_from_list(metadata, cur):		for obj in db.content_metadata_missing_from_list(metadata, cur):
yield obj[0]		yield obj[0]

		@remote_api_endpoint('content_metadata')
@db_transaction_generator()		@db_transaction_generator()
def content_metadata_get(self, ids, db=None, cur=None):		def content_metadata_get(self, ids, db=None, cur=None):
"""Retrieve metadata per id.		"""Retrieve metadata per id.

Args:		Args:
ids (iterable): sha1 checksums		ids (iterable): sha1 checksums

Yields:		Yields:
list: dictionaries with the following keys:		list: dictionaries with the following keys:

id (bytes)		id (bytes)
translated_metadata (str): associated metadata		translated_metadata (str): associated metadata
tool (dict): tool used to compute metadata		tool (dict): tool used to compute metadata

"""		"""
for c in db.content_metadata_get_from_list(ids, cur):		for c in db.content_metadata_get_from_list(ids, cur):
yield converters.db_to_metadata(		yield converters.db_to_metadata(
dict(zip(db.content_metadata_cols, c)))		dict(zip(db.content_metadata_cols, c)))

		@remote_api_endpoint('content_metadata/add')
@db_transaction()		@db_transaction()
def content_metadata_add(self, metadata, conflict_update=False, db=None,		def content_metadata_add(self, metadata, conflict_update=False, db=None,
cur=None):		cur=None):
"""Add metadata not present in storage.		"""Add metadata not present in storage.

Args:		Args:
metadata (iterable): dictionaries with keys:		metadata (iterable): dictionaries with keys:

id: sha1		id: sha1
translated_metadata: bytes / jsonb ?		translated_metadata: bytes / jsonb ?

conflict_update: Flag to determine if we want to overwrite (true)		conflict_update: Flag to determine if we want to overwrite (true)
or skip duplicates (false, the default)		or skip duplicates (false, the default)

"""		"""
db.mktemp_content_metadata(cur)		db.mktemp_content_metadata(cur)
# empty metadata is mapped to 'unknown'		# empty metadata is mapped to 'unknown'

db.copy_to(metadata, 'tmp_content_metadata',		db.copy_to(metadata, 'tmp_content_metadata',
['id', 'translated_metadata', 'indexer_configuration_id'],		['id', 'translated_metadata', 'indexer_configuration_id'],
cur)		cur)
db.content_metadata_add_from_temp(conflict_update, cur)		db.content_metadata_add_from_temp(conflict_update, cur)

		@remote_api_endpoint('revision_metadata/missing')
@db_transaction_generator()		@db_transaction_generator()
def revision_metadata_missing(self, metadata, db=None, cur=None):		def revision_metadata_missing(self, metadata, db=None, cur=None):
"""List metadata missing from storage.		"""List metadata missing from storage.

Args:		Args:
metadata (iterable): dictionaries with keys:		metadata (iterable): dictionaries with keys:

id (bytes): sha1_git revision identifier		id (bytes): sha1_git revision identifier
indexer_configuration_id (int): tool used to compute		indexer_configuration_id (int): tool used to compute
the results		the results

Returns:		Returns:
iterable: missing ids		iterable: missing ids

"""		"""
for obj in db.revision_metadata_missing_from_list(metadata, cur):		for obj in db.revision_metadata_missing_from_list(metadata, cur):
yield obj[0]		yield obj[0]

		@remote_api_endpoint('revision_metadata')
@db_transaction_generator()		@db_transaction_generator()
def revision_metadata_get(self, ids, db=None, cur=None):		def revision_metadata_get(self, ids, db=None, cur=None):
"""Retrieve revision metadata per id.		"""Retrieve revision metadata per id.

Args:		Args:
ids (iterable): sha1 checksums		ids (iterable): sha1 checksums

Yields:		Yields:
list: dictionaries with the following keys:		list: dictionaries with the following keys:

id (bytes)		id (bytes)
translated_metadata (str): associated metadata		translated_metadata (str): associated metadata
tool (dict): tool used to compute metadata		tool (dict): tool used to compute metadata

"""		"""
for c in db.revision_metadata_get_from_list(ids, cur):		for c in db.revision_metadata_get_from_list(ids, cur):
yield converters.db_to_metadata(		yield converters.db_to_metadata(
dict(zip(db.revision_metadata_cols, c)))		dict(zip(db.revision_metadata_cols, c)))

		@remote_api_endpoint('revision_metadata/add')
@db_transaction()		@db_transaction()
def revision_metadata_add(self, metadata, conflict_update=False, db=None,		def revision_metadata_add(self, metadata, conflict_update=False, db=None,
cur=None):		cur=None):
"""Add metadata not present in storage.		"""Add metadata not present in storage.

Args:		Args:
metadata (iterable): dictionaries with keys:		metadata (iterable): dictionaries with keys:

- id: sha1_git of revision		- id: sha1_git of revision
- translated_metadata: bytes / jsonb ?		- translated_metadata: bytes / jsonb ?

conflict_update: Flag to determine if we want to overwrite (true)		conflict_update: Flag to determine if we want to overwrite (true)
or skip duplicates (false, the default)		or skip duplicates (false, the default)

"""		"""
db.mktemp_revision_metadata(cur)		db.mktemp_revision_metadata(cur)
# empty metadata is mapped to 'unknown'		# empty metadata is mapped to 'unknown'

db.copy_to(metadata, 'tmp_revision_metadata',		db.copy_to(metadata, 'tmp_revision_metadata',
['id', 'translated_metadata', 'indexer_configuration_id'],		['id', 'translated_metadata', 'indexer_configuration_id'],
cur)		cur)
db.revision_metadata_add_from_temp(conflict_update, cur)		db.revision_metadata_add_from_temp(conflict_update, cur)

		@remote_api_endpoint('indexer_configuration/add')
@db_transaction_generator()		@db_transaction_generator()
def indexer_configuration_add(self, tools, db=None, cur=None):		def indexer_configuration_add(self, tools, db=None, cur=None):
"""Add new tools to the storage.		"""Add new tools to the storage.

Args:		Args:
tools ([dict]): List of dictionary representing tool to		tools ([dict]): List of dictionary representing tool to
insert in the db. Dictionary with the following keys::		insert in the db. Dictionary with the following keys::

Show All 12 Lines	def indexer_configuration_add(self, tools, db=None, cur=None):
db.copy_to(tools, 'tmp_indexer_configuration',		db.copy_to(tools, 'tmp_indexer_configuration',
['tool_name', 'tool_version', 'tool_configuration'],		['tool_name', 'tool_version', 'tool_configuration'],
cur)		cur)

tools = db.indexer_configuration_add_from_temp(cur)		tools = db.indexer_configuration_add_from_temp(cur)
for line in tools:		for line in tools:
yield dict(zip(db.indexer_configuration_cols, line))		yield dict(zip(db.indexer_configuration_cols, line))

		@remote_api_endpoint('indexer_configuration/data')
@db_transaction()		@db_transaction()
def indexer_configuration_get(self, tool, db=None, cur=None):		def indexer_configuration_get(self, tool, db=None, cur=None):
"""Retrieve tool information.		"""Retrieve tool information.

Args:		Args:
tool (dict): Dictionary representing a tool with the		tool (dict): Dictionary representing a tool with the
following keys::		following keys::

Show All 18 Lines

View Options

swh/indexer/storage/in_memory.py

# Copyright (C) 2015-2018 The Software Heritage developers		# Copyright (C) 2015-2018 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution		# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version		# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information		# See top-level LICENSE file for more information


import json		import json
import psycopg2		import psycopg2

from collections import defaultdict		from collections import defaultdict

		from swh.core.api import remote_api_endpoint
from swh.storage.common import db_transaction_generator, db_transaction		from swh.storage.common import db_transaction_generator, db_transaction
from swh.storage.exc import StorageDBError		from swh.storage.exc import StorageDBError
from .db import Db		from .db import Db

from . import converters		from . import converters


INDEXER_CFG_KEY = 'indexer_storage'		INDEXER_CFG_KEY = 'indexer_storage'


def get_indexer_storage(cls, args):		def get_indexer_storage(cls, args):
		ardumontUnsubmitted Not Done Inline Actions Don't we want to keep the original docstring as well here (@functools.wraps)? ardumont: Don't we want to keep the original docstring as well here (@functools.wraps)?
		vlorentzAuthorUnsubmitted Done Inline Actions The original function is returned, so there is no wrapper. vlorentz: The original function is returned, so there is no wrapper.
		ardumontUnsubmitted Not Done Inline Actions Right! ardumont: Right!
"""Get an indexer storage object of class `storage_class` with		"""Get an indexer storage object of class `storage_class` with
arguments `storage_args`.		arguments `storage_args`.

Args:		Args:
args (dict): dictionary with keys:		args (dict): dictionary with keys:
- cls (str): storage's class, either 'local' or 'remote'		- cls (str): storage's class, either 'local' or 'remote'
- args (dict): dictionary with keys		- args (dict): dictionary with keys

Show All 9 Lines	def get_indexer_storage(cls, args):
elif cls == 'local':		elif cls == 'local':
from . import IndexerStorage		from . import IndexerStorage
else:		else:
raise ValueError('Unknown indexer storage class `%s`' % cls)		raise ValueError('Unknown indexer storage class `%s`' % cls)

return IndexerStorage(**args)		return IndexerStorage(**args)


class IndexerStorage():		class IndexerStorage:
"""SWH Indexer Storage		"""SWH Indexer Storage

"""		"""
def __init__(self, db, min_pool_conns=1, max_pool_conns=10):		def __init__(self, db, min_pool_conns=1, max_pool_conns=10):
"""		"""
Args:		Args:
db_conn: either a libpq connection string, or a psycopg2 connection		db_conn: either a libpq connection string, or a psycopg2 connection

Show All 10 Lines	def __init__(self, db, min_pool_conns=1, max_pool_conns=10):
except psycopg2.OperationalError as e:		except psycopg2.OperationalError as e:
raise StorageDBError(e)		raise StorageDBError(e)

def get_db(self):		def get_db(self):
if self._db:		if self._db:
return self._db		return self._db
return Db.from_pool(self._pool)		return Db.from_pool(self._pool)

		@remote_api_endpoint('check_config')
def check_config(self, *, check_write):		def check_config(self, *, check_write):
"""Check that the storage is configured and ready to go."""		"""Check that the storage is configured and ready to go."""
# Check permissions on one of the tables		# Check permissions on one of the tables
with self.get_db().transaction() as cur:		with self.get_db().transaction() as cur:
if check_write:		if check_write:
check = 'INSERT'		check = 'INSERT'
else:		else:
check = 'SELECT'		check = 'SELECT'

cur.execute(		cur.execute(
"select has_table_privilege(current_user, 'content_mimetype', %s)", # noqa		"select has_table_privilege(current_user, 'content_mimetype', %s)", # noqa
(check,)		(check,)
)		)
return cur.fetchone()[0]		return cur.fetchone()[0]

return True		return True

		@remote_api_endpoint('content_mimetype/missing')
@db_transaction_generator()		@db_transaction_generator()
def content_mimetype_missing(self, mimetypes, db=None, cur=None):		def content_mimetype_missing(self, mimetypes, db=None, cur=None):
"""List mimetypes missing from storage.		"""List mimetypes missing from storage.

Args:		Args:
mimetypes (iterable): iterable of dict with keys:		mimetypes (iterable): iterable of dict with keys:

id (bytes): sha1 identifier		id (bytes): sha1 identifier
indexer_configuration_id (int): tool used to compute		indexer_configuration_id (int): tool used to compute
the results		the results

Yields:		Yields:
an iterable of missing id for the tuple (id,		an iterable of missing id for the tuple (id,
indexer_configuration_id)		indexer_configuration_id)

"""		"""
for obj in db.content_mimetype_missing_from_list(mimetypes, cur):		for obj in db.content_mimetype_missing_from_list(mimetypes, cur):
yield obj[0]		yield obj[0]

		@remote_api_endpoint('content_mimetype/add')
@db_transaction()		@db_transaction()
def content_mimetype_add(self, mimetypes, conflict_update=False, db=None,		def content_mimetype_add(self, mimetypes, conflict_update=False, db=None,
cur=None):		cur=None):
"""Add mimetypes not present in storage.		"""Add mimetypes not present in storage.

Args:		Args:
mimetypes (iterable): dictionaries with keys:		mimetypes (iterable): dictionaries with keys:

id (bytes): sha1 identifier		id (bytes): sha1 identifier
mimetype (bytes): raw content's mimetype		mimetype (bytes): raw content's mimetype
encoding (bytes): raw content's encoding		encoding (bytes): raw content's encoding
indexer_configuration_id (int): tool's id used to		indexer_configuration_id (int): tool's id used to
compute the results		compute the results
conflict_update (bool): Flag to determine if we want to		conflict_update (bool): Flag to determine if we want to
overwrite (true) or skip duplicates		overwrite (true) or skip duplicates
(false, the default)		(false, the default)

"""		"""
db.mktemp_content_mimetype(cur)		db.mktemp_content_mimetype(cur)
db.copy_to(mimetypes, 'tmp_content_mimetype',		db.copy_to(mimetypes, 'tmp_content_mimetype',
['id', 'mimetype', 'encoding', 'indexer_configuration_id'],		['id', 'mimetype', 'encoding', 'indexer_configuration_id'],
cur)		cur)
db.content_mimetype_add_from_temp(conflict_update, cur)		db.content_mimetype_add_from_temp(conflict_update, cur)

		@remote_api_endpoint('content_mimetype')
@db_transaction_generator()		@db_transaction_generator()
def content_mimetype_get(self, ids, db=None, cur=None):		def content_mimetype_get(self, ids, db=None, cur=None):
"""Retrieve full content mimetype per ids.		"""Retrieve full content mimetype per ids.

Args:		Args:
ids (iterable): sha1 identifier		ids (iterable): sha1 identifier

Yields:		Yields:
mimetypes (iterable): dictionaries with keys:		mimetypes (iterable): dictionaries with keys:

id (bytes): sha1 identifier		id (bytes): sha1 identifier
mimetype (bytes): raw content's mimetype		mimetype (bytes): raw content's mimetype
encoding (bytes): raw content's encoding		encoding (bytes): raw content's encoding
tool (dict): Tool used to compute the language		tool (dict): Tool used to compute the language

"""		"""
for c in db.content_mimetype_get_from_list(ids, cur):		for c in db.content_mimetype_get_from_list(ids, cur):
yield converters.db_to_mimetype(		yield converters.db_to_mimetype(
dict(zip(db.content_mimetype_cols, c)))		dict(zip(db.content_mimetype_cols, c)))

		@remote_api_endpoint('content_language/missing')
@db_transaction_generator()		@db_transaction_generator()
def content_language_missing(self, languages, db=None, cur=None):		def content_language_missing(self, languages, db=None, cur=None):
"""List languages missing from storage.		"""List languages missing from storage.

Args:		Args:
languages (iterable): dictionaries with keys:		languages (iterable): dictionaries with keys:

id (bytes): sha1 identifier		id (bytes): sha1 identifier
indexer_configuration_id (int): tool used to compute		indexer_configuration_id (int): tool used to compute
the results		the results

Yields:		Yields:
an iterable of missing id for the tuple (id,		an iterable of missing id for the tuple (id,
indexer_configuration_id)		indexer_configuration_id)

"""		"""
for obj in db.content_language_missing_from_list(languages, cur):		for obj in db.content_language_missing_from_list(languages, cur):
yield obj[0]		yield obj[0]

		@remote_api_endpoint('content_language')
@db_transaction_generator()		@db_transaction_generator()
def content_language_get(self, ids, db=None, cur=None):		def content_language_get(self, ids, db=None, cur=None):
"""Retrieve full content language per ids.		"""Retrieve full content language per ids.

Args:		Args:
ids (iterable): sha1 identifier		ids (iterable): sha1 identifier

Yields:		Yields:
languages (iterable): dictionaries with keys:		languages (iterable): dictionaries with keys:

id (bytes): sha1 identifier		id (bytes): sha1 identifier
lang (bytes): raw content's language		lang (bytes): raw content's language
tool (dict): Tool used to compute the language		tool (dict): Tool used to compute the language

"""		"""
for c in db.content_language_get_from_list(ids, cur):		for c in db.content_language_get_from_list(ids, cur):
yield converters.db_to_language(		yield converters.db_to_language(
dict(zip(db.content_language_cols, c)))		dict(zip(db.content_language_cols, c)))

		@remote_api_endpoint('content_language/add')
@db_transaction()		@db_transaction()
def content_language_add(self, languages, conflict_update=False, db=None,		def content_language_add(self, languages, conflict_update=False, db=None,
cur=None):		cur=None):
"""Add languages not present in storage.		"""Add languages not present in storage.

Args:		Args:
languages (iterable): dictionaries with keys:		languages (iterable): dictionaries with keys:

Show All 13 Lines	def content_language_add(self, languages, conflict_update=False, db=None,
'lang': 'unknown' if not l['lang'] else l['lang'],		'lang': 'unknown' if not l['lang'] else l['lang'],
'indexer_configuration_id': l['indexer_configuration_id'],		'indexer_configuration_id': l['indexer_configuration_id'],
} for l in languages),		} for l in languages),
'tmp_content_language',		'tmp_content_language',
['id', 'lang', 'indexer_configuration_id'], cur)		['id', 'lang', 'indexer_configuration_id'], cur)

db.content_language_add_from_temp(conflict_update, cur)		db.content_language_add_from_temp(conflict_update, cur)

		@remote_api_endpoint('content/ctags/missing')
@db_transaction_generator()		@db_transaction_generator()
def content_ctags_missing(self, ctags, db=None, cur=None):		def content_ctags_missing(self, ctags, db=None, cur=None):
"""List ctags missing from storage.		"""List ctags missing from storage.

Args:		Args:
ctags (iterable): dicts with keys:		ctags (iterable): dicts with keys:

id (bytes): sha1 identifier		id (bytes): sha1 identifier
indexer_configuration_id (int): tool used to compute		indexer_configuration_id (int): tool used to compute
the results		the results

Yields:		Yields:
an iterable of missing id for the tuple (id,		an iterable of missing id for the tuple (id,
indexer_configuration_id)		indexer_configuration_id)

"""		"""
for obj in db.content_ctags_missing_from_list(ctags, cur):		for obj in db.content_ctags_missing_from_list(ctags, cur):
yield obj[0]		yield obj[0]

		@remote_api_endpoint('content/ctags')
@db_transaction_generator()		@db_transaction_generator()
def content_ctags_get(self, ids, db=None, cur=None):		def content_ctags_get(self, ids, db=None, cur=None):
"""Retrieve ctags per id.		"""Retrieve ctags per id.

Args:		Args:
ids (iterable): sha1 checksums		ids (iterable): sha1 checksums

Yields:		Yields:
Dictionaries with keys:		Dictionaries with keys:

id (bytes): content's identifier		id (bytes): content's identifier
name (str): symbol's name		name (str): symbol's name
kind (str): symbol's kind		kind (str): symbol's kind
language (str): language for that content		language (str): language for that content
tool (dict): tool used to compute the ctags' info		tool (dict): tool used to compute the ctags' info


"""		"""
for c in db.content_ctags_get_from_list(ids, cur):		for c in db.content_ctags_get_from_list(ids, cur):
yield converters.db_to_ctags(dict(zip(db.content_ctags_cols, c)))		yield converters.db_to_ctags(dict(zip(db.content_ctags_cols, c)))

		@remote_api_endpoint('content/ctags/add')
@db_transaction()		@db_transaction()
def content_ctags_add(self, ctags, conflict_update=False, db=None,		def content_ctags_add(self, ctags, conflict_update=False, db=None,
cur=None):		cur=None):
"""Add ctags not present in storage		"""Add ctags not present in storage

Args:		Args:
ctags (iterable): dictionaries with keys:		ctags (iterable): dictionaries with keys:

Show All 13 Lines	def content_ctags_add(self, ctags, conflict_update=False, db=None,
db.copy_to(list(_convert_ctags(ctags)),		db.copy_to(list(_convert_ctags(ctags)),
tblname='tmp_content_ctags',		tblname='tmp_content_ctags',
columns=['id', 'name', 'kind', 'line',		columns=['id', 'name', 'kind', 'line',
'lang', 'indexer_configuration_id'],		'lang', 'indexer_configuration_id'],
cur=cur)		cur=cur)

db.content_ctags_add_from_temp(conflict_update, cur)		db.content_ctags_add_from_temp(conflict_update, cur)

		@remote_api_endpoint('content/ctags/search')
@db_transaction_generator()		@db_transaction_generator()
def content_ctags_search(self, expression,		def content_ctags_search(self, expression,
limit=10, last_sha1=None, db=None, cur=None):		limit=10, last_sha1=None, db=None, cur=None):
"""Search through content's raw ctags symbols.		"""Search through content's raw ctags symbols.

Args:		Args:
expression (str): Expression to search for		expression (str): Expression to search for
limit (int): Number of rows to return (default to 10).		limit (int): Number of rows to return (default to 10).
last_sha1 (str): Offset from which retrieving data (default to '').		last_sha1 (str): Offset from which retrieving data (default to '').

Yields:		Yields:
rows of ctags including id, name, lang, kind, line, etc...		rows of ctags including id, name, lang, kind, line, etc...

"""		"""
for obj in db.content_ctags_search(expression, last_sha1, limit,		for obj in db.content_ctags_search(expression, last_sha1, limit,
cur=cur):		cur=cur):
yield converters.db_to_ctags(dict(zip(db.content_ctags_cols, obj)))		yield converters.db_to_ctags(dict(zip(db.content_ctags_cols, obj)))

		@remote_api_endpoint('content/fossology_license')
@db_transaction_generator()		@db_transaction_generator()
def content_fossology_license_get(self, ids, db=None, cur=None):		def content_fossology_license_get(self, ids, db=None, cur=None):
"""Retrieve licenses per id.		"""Retrieve licenses per id.

Args:		Args:
ids (iterable): sha1 checksums		ids (iterable): sha1 checksums

Yields:		Yields:
Show All 9 Lines	def content_fossology_license_get(self, ids, db=None, cur=None):
license = dict(zip(db.content_fossology_license_cols, c))		license = dict(zip(db.content_fossology_license_cols, c))

id_ = license['id']		id_ = license['id']
d[id_].append(converters.db_to_fossology_license(license))		d[id_].append(converters.db_to_fossology_license(license))

for id_, facts in d.items():		for id_, facts in d.items():
yield {id_: facts}		yield {id_: facts}

		@remote_api_endpoint('content/fossology_license/add')
@db_transaction()		@db_transaction()
def content_fossology_license_add(self, licenses, conflict_update=False,		def content_fossology_license_add(self, licenses, conflict_update=False,
db=None, cur=None):		db=None, cur=None):
"""Add licenses not present in storage.		"""Add licenses not present in storage.

Args:		Args:
licenses (iterable): dictionaries with keys:		licenses (iterable): dictionaries with keys:

Show All 17 Lines	def content_fossology_license_add(self, licenses, conflict_update=False,
'license': license,		'license': license,
} for sha1 in licenses		} for sha1 in licenses
for license in sha1['licenses']),		for license in sha1['licenses']),
tblname='tmp_content_fossology_license',		tblname='tmp_content_fossology_license',
columns=['id', 'license', 'indexer_configuration_id'],		columns=['id', 'license', 'indexer_configuration_id'],
cur=cur)		cur=cur)
db.content_fossology_license_add_from_temp(conflict_update, cur)		db.content_fossology_license_add_from_temp(conflict_update, cur)

		@remote_api_endpoint('content_metadata/missing')
@db_transaction_generator()		@db_transaction_generator()
def content_metadata_missing(self, metadata, db=None, cur=None):		def content_metadata_missing(self, metadata, db=None, cur=None):
"""List metadata missing from storage.		"""List metadata missing from storage.

Args:		Args:
metadata (iterable): dictionaries with keys:		metadata (iterable): dictionaries with keys:

id (bytes): sha1 identifier		id (bytes): sha1 identifier
indexer_configuration_id (int): tool used to compute		indexer_configuration_id (int): tool used to compute
the results		the results

Yields:		Yields:
an iterable of missing id for the tuple (id,		an iterable of missing id for the tuple (id,
indexer_configuration_id)		indexer_configuration_id)

"""		"""
for obj in db.content_metadata_missing_from_list(metadata, cur):		for obj in db.content_metadata_missing_from_list(metadata, cur):
yield obj[0]		yield obj[0]

		@remote_api_endpoint('content_metadata')
@db_transaction_generator()		@db_transaction_generator()
def content_metadata_get(self, ids, db=None, cur=None):		def content_metadata_get(self, ids, db=None, cur=None):
"""Retrieve metadata per id.		"""Retrieve metadata per id.

Args:		Args:
ids (iterable): sha1 checksums		ids (iterable): sha1 checksums

Yields:		Yields:
list: dictionaries with the following keys:		list: dictionaries with the following keys:

id (bytes)		id (bytes)
translated_metadata (str): associated metadata		translated_metadata (str): associated metadata
tool (dict): tool used to compute metadata		tool (dict): tool used to compute metadata

"""		"""
for c in db.content_metadata_get_from_list(ids, cur):		for c in db.content_metadata_get_from_list(ids, cur):
yield converters.db_to_metadata(		yield converters.db_to_metadata(
dict(zip(db.content_metadata_cols, c)))		dict(zip(db.content_metadata_cols, c)))

		@remote_api_endpoint('content_metadata/add')
@db_transaction()		@db_transaction()
def content_metadata_add(self, metadata, conflict_update=False, db=None,		def content_metadata_add(self, metadata, conflict_update=False, db=None,
cur=None):		cur=None):
"""Add metadata not present in storage.		"""Add metadata not present in storage.

Args:		Args:
metadata (iterable): dictionaries with keys:		metadata (iterable): dictionaries with keys:

id: sha1		id: sha1
translated_metadata: bytes / jsonb ?		translated_metadata: bytes / jsonb ?

conflict_update: Flag to determine if we want to overwrite (true)		conflict_update: Flag to determine if we want to overwrite (true)
or skip duplicates (false, the default)		or skip duplicates (false, the default)

"""		"""
db.mktemp_content_metadata(cur)		db.mktemp_content_metadata(cur)
# empty metadata is mapped to 'unknown'		# empty metadata is mapped to 'unknown'

db.copy_to(metadata, 'tmp_content_metadata',		db.copy_to(metadata, 'tmp_content_metadata',
['id', 'translated_metadata', 'indexer_configuration_id'],		['id', 'translated_metadata', 'indexer_configuration_id'],
cur)		cur)
db.content_metadata_add_from_temp(conflict_update, cur)		db.content_metadata_add_from_temp(conflict_update, cur)

		@remote_api_endpoint('revision_metadata/missing')
@db_transaction_generator()		@db_transaction_generator()
def revision_metadata_missing(self, metadata, db=None, cur=None):		def revision_metadata_missing(self, metadata, db=None, cur=None):
"""List metadata missing from storage.		"""List metadata missing from storage.

Args:		Args:
metadata (iterable): dictionaries with keys:		metadata (iterable): dictionaries with keys:

id (bytes): sha1_git revision identifier		id (bytes): sha1_git revision identifier
indexer_configuration_id (int): tool used to compute		indexer_configuration_id (int): tool used to compute
the results		the results

Returns:		Returns:
iterable: missing ids		iterable: missing ids

"""		"""
for obj in db.revision_metadata_missing_from_list(metadata, cur):		for obj in db.revision_metadata_missing_from_list(metadata, cur):
yield obj[0]		yield obj[0]

		@remote_api_endpoint('revision_metadata')
@db_transaction_generator()		@db_transaction_generator()
def revision_metadata_get(self, ids, db=None, cur=None):		def revision_metadata_get(self, ids, db=None, cur=None):
"""Retrieve revision metadata per id.		"""Retrieve revision metadata per id.

Args:		Args:
ids (iterable): sha1 checksums		ids (iterable): sha1 checksums

Yields:		Yields:
list: dictionaries with the following keys:		list: dictionaries with the following keys:

id (bytes)		id (bytes)
translated_metadata (str): associated metadata		translated_metadata (str): associated metadata
tool (dict): tool used to compute metadata		tool (dict): tool used to compute metadata

"""		"""
for c in db.revision_metadata_get_from_list(ids, cur):		for c in db.revision_metadata_get_from_list(ids, cur):
yield converters.db_to_metadata(		yield converters.db_to_metadata(
dict(zip(db.revision_metadata_cols, c)))		dict(zip(db.revision_metadata_cols, c)))

		@remote_api_endpoint('revision_metadata/add')
@db_transaction()		@db_transaction()
def revision_metadata_add(self, metadata, conflict_update=False, db=None,		def revision_metadata_add(self, metadata, conflict_update=False, db=None,
cur=None):		cur=None):
"""Add metadata not present in storage.		"""Add metadata not present in storage.

Args:		Args:
metadata (iterable): dictionaries with keys:		metadata (iterable): dictionaries with keys:

- id: sha1_git of revision		- id: sha1_git of revision
- translated_metadata: bytes / jsonb ?		- translated_metadata: bytes / jsonb ?

conflict_update: Flag to determine if we want to overwrite (true)		conflict_update: Flag to determine if we want to overwrite (true)
or skip duplicates (false, the default)		or skip duplicates (false, the default)

"""		"""
db.mktemp_revision_metadata(cur)		db.mktemp_revision_metadata(cur)
# empty metadata is mapped to 'unknown'		# empty metadata is mapped to 'unknown'

db.copy_to(metadata, 'tmp_revision_metadata',		db.copy_to(metadata, 'tmp_revision_metadata',
['id', 'translated_metadata', 'indexer_configuration_id'],		['id', 'translated_metadata', 'indexer_configuration_id'],
cur)		cur)
db.revision_metadata_add_from_temp(conflict_update, cur)		db.revision_metadata_add_from_temp(conflict_update, cur)

		@remote_api_endpoint('indexer_configuration/add')
@db_transaction_generator()		@db_transaction_generator()
def indexer_configuration_add(self, tools, db=None, cur=None):		def indexer_configuration_add(self, tools, db=None, cur=None):
"""Add new tools to the storage.		"""Add new tools to the storage.

Args:		Args:
tools ([dict]): List of dictionary representing tool to		tools ([dict]): List of dictionary representing tool to
insert in the db. Dictionary with the following keys::		insert in the db. Dictionary with the following keys::

Show All 12 Lines	def indexer_configuration_add(self, tools, db=None, cur=None):
db.copy_to(tools, 'tmp_indexer_configuration',		db.copy_to(tools, 'tmp_indexer_configuration',
['tool_name', 'tool_version', 'tool_configuration'],		['tool_name', 'tool_version', 'tool_configuration'],
cur)		cur)

tools = db.indexer_configuration_add_from_temp(cur)		tools = db.indexer_configuration_add_from_temp(cur)
for line in tools:		for line in tools:
yield dict(zip(db.indexer_configuration_cols, line))		yield dict(zip(db.indexer_configuration_cols, line))

		@remote_api_endpoint('indexer_configuration/data')
@db_transaction()		@db_transaction()
def indexer_configuration_get(self, tool, db=None, cur=None):		def indexer_configuration_get(self, tool, db=None, cur=None):
"""Retrieve tool information.		"""Retrieve tool information.

Args:		Args:
tool (dict): Dictionary representing a tool with the		tool (dict): Dictionary representing a tool with the
following keys::		following keys::

Show All 18 Lines

View Options

swh/indexer/tests/storage/test_in_memory.py

View Options

swh/indexer/tests/test_ctags.py

View Options

swh/indexer/tests/test_fossology_license.py

View Options

swh/indexer/tests/test_mimetype.py

View Options

swh/indexer/tests/test_utils.py

Remove mocks from range tests of mimetype and license indexers.
ClosedPublic
Actions

Details

Diff Detail

Event Timeline

Revision Contents
Changeset List

Diff 2488

swh/indexer/fossology_license.py

swh/indexer/indexer.py

swh/indexer/storage/init.py

swh/indexer/storage/in_memory.py

swh/indexer/tests/storage/test_in_memory.py

swh/indexer/tests/test_ctags.py

swh/indexer/tests/test_fossology_license.py

swh/indexer/tests/test_mimetype.py

swh/indexer/tests/test_utils.py

Remove mocks from range tests of mimetype and license indexers.ClosedPublicActions

Details

Diff Detail

Event Timeline

Revision ContentsChangeset List

Diff 2488

swh/indexer/fossology_license.py

swh/indexer/indexer.py

swh/indexer/storage/__init__.py

swh/indexer/storage/in_memory.py

swh/indexer/tests/storage/test_in_memory.py

swh/indexer/tests/test_ctags.py

swh/indexer/tests/test_fossology_license.py

swh/indexer/tests/test_mimetype.py

swh/indexer/tests/test_utils.py

Remove mocks from range tests of mimetype and license indexers.
ClosedPublic
Actions

Revision Contents
Changeset List

swh/indexer/storage/init.py