No OneTemporary
Actions

Size

98 KB

Subscribers

None

View Options

	diff --git a/swh/indexer/storage/__init__.py b/swh/indexer/storage/__init__.py
	index 7cc498d..b241f86 100644
	--- a/swh/indexer/storage/__init__.py
	+++ b/swh/indexer/storage/__init__.py
	@@ -1,686 +1,715 @@
	# Copyright (C) 2015-2018 The Software Heritage developers
	# See the AUTHORS file at the top-level directory of this distribution
	# License: GNU General Public License version 3, or any later version
	# See top-level LICENSE file for more information


	import json
	import psycopg2

	from collections import defaultdict

	from swh.core.api import remote_api_endpoint
	from swh.storage.common import db_transaction_generator, db_transaction
	from swh.storage.exc import StorageDBError
	from .db import Db

	from . import converters


	INDEXER_CFG_KEY = 'indexer_storage'


	def get_indexer_storage(cls, args):
	"""Get an indexer storage object of class `storage_class` with
	arguments `storage_args`.

	Args:
	cls (str): storage's class, either 'local' or 'remote'
	args (dict): dictionary of arguments passed to the
	storage class constructor

	Returns:
	an instance of swh.indexer's storage (either local or remote)

	Raises:
	ValueError if passed an unknown storage class.

	"""
	if cls == 'remote':
	from .api.client import RemoteStorage as IndexerStorage
	elif cls == 'local':
	from . import IndexerStorage
	else:
	raise ValueError('Unknown indexer storage class `%s`' % cls)

	return IndexerStorage(**args)


	class IndexerStorage:
	"""SWH Indexer Storage

	"""
	def __init__(self, db, min_pool_conns=1, max_pool_conns=10):
	"""
	Args:
	db_conn: either a libpq connection string, or a psycopg2 connection

	"""
	try:
	if isinstance(db, psycopg2.extensions.connection):
	self._pool = None
	self._db = Db(db)
	else:
	self._pool = psycopg2.pool.ThreadedConnectionPool(
	min_pool_conns, max_pool_conns, db
	)
	self._db = None
	except psycopg2.OperationalError as e:
	raise StorageDBError(e)

	def get_db(self):
	if self._db:
	return self._db
	return Db.from_pool(self._pool)

	@remote_api_endpoint('check_config')
	def check_config(self, *, check_write):
	"""Check that the storage is configured and ready to go."""
	# Check permissions on one of the tables
	with self.get_db().transaction() as cur:
	if check_write:
	check = 'INSERT'
	else:
	check = 'SELECT'

	cur.execute(
	"select has_table_privilege(current_user, 'content_mimetype', %s)", # noqa
	(check,)
	)
	return cur.fetchone()[0]

	return True

	@remote_api_endpoint('content_mimetype/missing')
	@db_transaction_generator()
	def content_mimetype_missing(self, mimetypes, db=None, cur=None):
	"""Generate mimetypes missing from storage.

	Args:
	mimetypes (iterable): iterable of dict with keys:

	- id (bytes): sha1 identifier
	- indexer_configuration_id (int): tool used to compute the
	results

	Yields:
	tuple (id, indexer_configuration_id): missing id

	"""
	for obj in db.content_mimetype_missing_from_list(mimetypes, cur):
	yield obj[0]

	def _content_get_range(self, content_type, start, end,
	indexer_configuration_id, limit=1000,
	db=None, cur=None):
	"""Retrieve ids of type content_type within range [start, end] bound
	by limit.

	Args:
	content_type (str): content's type (mimetype, language, etc...)
	start (bytes): Starting identifier range (expected smaller
	than end)
	end (bytes): Ending identifier range (expected larger
	than start)
	indexer_configuration_id (int): The tool used to index data
	limit (int): Limit result (default to 1000)

	Raises:
	ValueError for;
	- limit to None
	- wrong content_type provided

	Returns:
	a dict with keys:
	- ids [bytes]: iterable of content ids within the range.
	- next (Optional[bytes]): The next range of sha1 starts at
	this sha1 if any

	"""
	if limit is None:
	raise ValueError('Development error: limit should not be None')
	if content_type not in db.content_indexer_names:
	err = 'Development error: Wrong type. Should be one of [%s]' % (
	','.join(db.content_indexer_names))
	raise ValueError(err)

	ids = []
	next_id = None
	for counter, obj in enumerate(db.content_get_range(
	content_type, start, end, indexer_configuration_id,
	limit=limit+1, cur=cur)):
	_id = obj[0]
	if counter >= limit:
	next_id = _id
	break

	ids.append(_id)

	return {
	'ids': ids,
	'next': next_id
	}

	@remote_api_endpoint('content_mimetype/range')
	@db_transaction()
	def content_mimetype_get_range(self, start, end, indexer_configuration_id,
	limit=1000, db=None, cur=None):
	"""Retrieve mimetypes within range [start, end] bound by limit.

	Args:
	start (bytes): Starting identifier range (expected smaller
	than end)
	end (bytes): Ending identifier range (expected larger
	than start)
	indexer_configuration_id (int): The tool used to index data
	limit (int): Limit result (default to 1000)

	Raises:
	ValueError for limit to None

	Returns:
	a dict with keys:
	- ids [bytes]: iterable of content ids within the range.
	- next (Optional[bytes]): The next range of sha1 starts at
	this sha1 if any

	"""
	return self._content_get_range('mimetype', start, end,
	indexer_configuration_id, limit=limit,
	db=db, cur=cur)

	@remote_api_endpoint('content_mimetype/add')
	@db_transaction()
	def content_mimetype_add(self, mimetypes, conflict_update=False, db=None,
	cur=None):
	"""Add mimetypes not present in storage.

	Args:
	mimetypes (iterable): dictionaries with keys:

	- id (bytes): sha1 identifier
	- mimetype (bytes): raw content's mimetype
	- encoding (bytes): raw content's encoding
	- indexer_configuration_id (int): tool's id used to
	compute the results
	- conflict_update (bool): Flag to determine if we want to
	overwrite (``True``) or skip duplicates (``False``, the
	default)

	"""
	db.mktemp_content_mimetype(cur)
	db.copy_to(mimetypes, 'tmp_content_mimetype',
	['id', 'mimetype', 'encoding', 'indexer_configuration_id'],
	cur)
	db.content_mimetype_add_from_temp(conflict_update, cur)

	@remote_api_endpoint('content_mimetype')
	@db_transaction_generator()
	def content_mimetype_get(self, ids, db=None, cur=None):
	"""Retrieve full content mimetype per ids.

	Args:
	ids (iterable): sha1 identifier

	Yields:
	mimetypes (iterable): dictionaries with keys:

	- id (bytes): sha1 identifier
	- mimetype (bytes): raw content's mimetype
	- encoding (bytes): raw content's encoding
	- tool (dict): Tool used to compute the language

	"""
	for c in db.content_mimetype_get_from_list(ids, cur):
	yield converters.db_to_mimetype(
	dict(zip(db.content_mimetype_cols, c)))

	@remote_api_endpoint('content_language/missing')
	@db_transaction_generator()
	def content_language_missing(self, languages, db=None, cur=None):
	"""List languages missing from storage.

	Args:
	languages (iterable): dictionaries with keys:

	- id (bytes): sha1 identifier
	- indexer_configuration_id (int): tool used to compute
	the results

	Yields:
	an iterable of missing id for the tuple (id,
	indexer_configuration_id)

	"""
	for obj in db.content_language_missing_from_list(languages, cur):
	yield obj[0]

	@remote_api_endpoint('content_language')
	@db_transaction_generator()
	def content_language_get(self, ids, db=None, cur=None):
	"""Retrieve full content language per ids.

	Args:
	ids (iterable): sha1 identifier

	Yields:
	languages (iterable): dictionaries with keys:

	- id (bytes): sha1 identifier
	- lang (bytes): raw content's language
	- tool (dict): Tool used to compute the language

	"""
	for c in db.content_language_get_from_list(ids, cur):
	yield converters.db_to_language(
	dict(zip(db.content_language_cols, c)))

	@remote_api_endpoint('content_language/add')
	@db_transaction()
	def content_language_add(self, languages, conflict_update=False, db=None,
	cur=None):
	"""Add languages not present in storage.

	Args:
	languages (iterable): dictionaries with keys:

	- id (bytes): sha1
	- lang (bytes): language detected

	conflict_update (bool): Flag to determine if we want to
	overwrite (true) or skip duplicates (false, the
	default)

	"""
	db.mktemp_content_language(cur)
	# empty language is mapped to 'unknown'
	db.copy_to(
	({
	'id': l['id'],
	'lang': 'unknown' if not l['lang'] else l['lang'],
	'indexer_configuration_id': l['indexer_configuration_id'],
	} for l in languages),
	'tmp_content_language',
	['id', 'lang', 'indexer_configuration_id'], cur)

	db.content_language_add_from_temp(conflict_update, cur)

	@remote_api_endpoint('content/ctags/missing')
	@db_transaction_generator()
	def content_ctags_missing(self, ctags, db=None, cur=None):
	"""List ctags missing from storage.

	Args:
	ctags (iterable): dicts with keys:

	- id (bytes): sha1 identifier
	- indexer_configuration_id (int): tool used to compute
	the results

	Yields:
	an iterable of missing id for the tuple (id,
	indexer_configuration_id)

	"""
	for obj in db.content_ctags_missing_from_list(ctags, cur):
	yield obj[0]

	@remote_api_endpoint('content/ctags')
	@db_transaction_generator()
	def content_ctags_get(self, ids, db=None, cur=None):
	"""Retrieve ctags per id.

	Args:
	ids (iterable): sha1 checksums

	Yields:
	Dictionaries with keys:

	- id (bytes): content's identifier
	- name (str): symbol's name
	- kind (str): symbol's kind
	- language (str): language for that content
	- tool (dict): tool used to compute the ctags' info


	"""
	for c in db.content_ctags_get_from_list(ids, cur):
	yield converters.db_to_ctags(dict(zip(db.content_ctags_cols, c)))

	@remote_api_endpoint('content/ctags/add')
	@db_transaction()
	def content_ctags_add(self, ctags, conflict_update=False, db=None,
	cur=None):
	"""Add ctags not present in storage

	Args:
	ctags (iterable): dictionaries with keys:

	- id (bytes): sha1
	- ctags ([list): List of dictionary with keys: name, kind,
	line, language

	"""
	def _convert_ctags(__ctags):
	"""Convert ctags dict to list of ctags.

	"""
	for ctags in __ctags:
	yield from converters.ctags_to_db(ctags)

	db.mktemp_content_ctags(cur)
	db.copy_to(list(_convert_ctags(ctags)),
	tblname='tmp_content_ctags',
	columns=['id', 'name', 'kind', 'line',
	'lang', 'indexer_configuration_id'],
	cur=cur)

	db.content_ctags_add_from_temp(conflict_update, cur)

	@remote_api_endpoint('content/ctags/search')
	@db_transaction_generator()
	def content_ctags_search(self, expression,
	limit=10, last_sha1=None, db=None, cur=None):
	"""Search through content's raw ctags symbols.

	Args:
	expression (str): Expression to search for
	limit (int): Number of rows to return (default to 10).
	last_sha1 (str): Offset from which retrieving data (default to '').

	Yields:
	rows of ctags including id, name, lang, kind, line, etc...

	"""
	for obj in db.content_ctags_search(expression, last_sha1, limit,
	cur=cur):
	yield converters.db_to_ctags(dict(zip(db.content_ctags_cols, obj)))

	@remote_api_endpoint('content/fossology_license')
	@db_transaction_generator()
	def content_fossology_license_get(self, ids, db=None, cur=None):
	"""Retrieve licenses per id.

	Args:
	ids (iterable): sha1 checksums

	Yields:
	list: dictionaries with the following keys:

	- id (bytes)
	- licenses ([str]): associated licenses for that content
	- tool (dict): Tool used to compute the license

	"""
	d = defaultdict(list)
	for c in db.content_fossology_license_get_from_list(ids, cur):
	license = dict(zip(db.content_fossology_license_cols, c))

	id_ = license['id']
	d[id_].append(converters.db_to_fossology_license(license))

	for id_, facts in d.items():
	yield {id_: facts}

	@remote_api_endpoint('content/fossology_license/add')
	@db_transaction()
	def content_fossology_license_add(self, licenses, conflict_update=False,
	db=None, cur=None):
	"""Add licenses not present in storage.

	Args:
	licenses (iterable): dictionaries with keys:

	- id: sha1
	- license ([bytes]): List of licenses associated to sha1
	- tool (str): nomossa

	conflict_update: Flag to determine if we want to overwrite (true)
	or skip duplicates (false, the default)

	Returns:
	list: content_license entries which failed due to unknown licenses

	"""
	# Then, we add the correct ones
	db.mktemp_content_fossology_license(cur)
	db.copy_to(
	({
	'id': sha1['id'],
	'indexer_configuration_id': sha1['indexer_configuration_id'],
	'license': license,
	} for sha1 in licenses
	for license in sha1['licenses']),
	tblname='tmp_content_fossology_license',
	columns=['id', 'license', 'indexer_configuration_id'],
	cur=cur)
	db.content_fossology_license_add_from_temp(conflict_update, cur)

	+ @remote_api_endpoint('content/fossology_license/range')
	+ @db_transaction()
	+ def content_fossology_license_get_range(
	+ self, start, end, indexer_configuration_id,
	+ limit=1000, db=None, cur=None):
	+ """Retrieve licenses within range [start, end] bound by limit.
	+
	+ Args:
	+ start (bytes): Starting identifier range (expected smaller
	+ than end)
	+ end (bytes): Ending identifier range (expected larger
	+ than start)
	+ indexer_configuration_id (int): The tool used to index data
	+ limit (int): Limit result (default to 1000)
	+
	+ Raises:
	+ ValueError for limit to None
	+
	+ Returns:
	+ a dict with keys:
	+ - ids [bytes]: iterable of content ids within the range.
	+ - next (Optional[bytes]): The next range of sha1 starts at
	+ this sha1 if any
	+
	+ """
	+ return self._content_get_range('fossology_license', start, end,
	+ indexer_configuration_id, limit=limit,
	+ db=db, cur=cur)
	+
	@remote_api_endpoint('content_metadata/missing')
	@db_transaction_generator()
	def content_metadata_missing(self, metadata, db=None, cur=None):
	"""List metadata missing from storage.

	Args:
	metadata (iterable): dictionaries with keys:

	- id (bytes): sha1 identifier
	- indexer_configuration_id (int): tool used to compute
	the results

	Yields:
	an iterable of missing id for the tuple (id,
	indexer_configuration_id)

	"""
	for obj in db.content_metadata_missing_from_list(metadata, cur):
	yield obj[0]

	@remote_api_endpoint('content_metadata')
	@db_transaction_generator()
	def content_metadata_get(self, ids, db=None, cur=None):
	"""Retrieve metadata per id.

	Args:
	ids (iterable): sha1 checksums

	Yields:
	list: dictionaries with the following keys:

	id (bytes)
	translated_metadata (str): associated metadata
	tool (dict): tool used to compute metadata

	"""
	for c in db.content_metadata_get_from_list(ids, cur):
	yield converters.db_to_metadata(
	dict(zip(db.content_metadata_cols, c)))

	@remote_api_endpoint('content_metadata/add')
	@db_transaction()
	def content_metadata_add(self, metadata, conflict_update=False, db=None,
	cur=None):
	"""Add metadata not present in storage.

	Args:
	metadata (iterable): dictionaries with keys:

	- id: sha1
	- translated_metadata: arbitrary dict

	conflict_update: Flag to determine if we want to overwrite (true)
	or skip duplicates (false, the default)

	"""
	db.mktemp_content_metadata(cur)

	db.copy_to(metadata, 'tmp_content_metadata',
	['id', 'translated_metadata', 'indexer_configuration_id'],
	cur)
	db.content_metadata_add_from_temp(conflict_update, cur)

	@remote_api_endpoint('revision_metadata/missing')
	@db_transaction_generator()
	def revision_metadata_missing(self, metadata, db=None, cur=None):
	"""List metadata missing from storage.

	Args:
	metadata (iterable): dictionaries with keys:

	- id (bytes): sha1_git revision identifier
	- indexer_configuration_id (int): tool used to compute
	the results

	Returns:
	iterable: missing ids

	"""
	for obj in db.revision_metadata_missing_from_list(metadata, cur):
	yield obj[0]

	@remote_api_endpoint('revision_metadata')
	@db_transaction_generator()
	def revision_metadata_get(self, ids, db=None, cur=None):
	"""Retrieve revision metadata per id.

	Args:
	ids (iterable): sha1 checksums

	Yields:
	list: dictionaries with the following keys:

	- id (bytes)
	- translated_metadata (str): associated metadata
	- tool (dict): tool used to compute metadata

	"""
	for c in db.revision_metadata_get_from_list(ids, cur):
	yield converters.db_to_metadata(
	dict(zip(db.revision_metadata_cols, c)))

	@remote_api_endpoint('revision_metadata/add')
	@db_transaction()
	def revision_metadata_add(self, metadata, conflict_update=False, db=None,
	cur=None):
	"""Add metadata not present in storage.

	Args:
	metadata (iterable): dictionaries with keys:

	- id: sha1_git of revision
	- translated_metadata: arbitrary dict

	conflict_update: Flag to determine if we want to overwrite (true)
	or skip duplicates (false, the default)

	"""
	db.mktemp_revision_metadata(cur)

	db.copy_to(metadata, 'tmp_revision_metadata',
	['id', 'translated_metadata', 'indexer_configuration_id'],
	cur)
	db.revision_metadata_add_from_temp(conflict_update, cur)

	@remote_api_endpoint('origin_intrinsic_metadata')
	@db_transaction_generator()
	def origin_intrinsic_metadata_get(self, ids, db=None, cur=None):
	"""Retrieve origin metadata per id.

	Args:
	ids (iterable): origin identifiers

	Yields:
	list: dictionaries with the following keys:

	- id (int)
	- translated_metadata (str): associated metadata
	- tool (dict): tool used to compute metadata

	"""
	for c in db.origin_intrinsic_metadata_get_from_list(ids, cur):
	yield converters.db_to_metadata(
	dict(zip(db.origin_intrinsic_metadata_cols, c)))

	@remote_api_endpoint('origin_intrinsic_metadata/add')
	@db_transaction()
	def origin_intrinsic_metadata_add(self, metadata,
	conflict_update=False, db=None,
	cur=None):
	"""Add origin metadata not present in storage.

	Args:
	metadata (iterable): dictionaries with keys:

	- origin_id: origin identifier
	- from_revision: sha1 id of the revision used to generate
	these metadata.
	- metadata: arbitrary dict

	conflict_update: Flag to determine if we want to overwrite (true)
	or skip duplicates (false, the default)

	"""
	db.mktemp_origin_intrinsic_metadata(cur)

	db.copy_to(metadata, 'tmp_origin_intrinsic_metadata',
	['origin_id', 'metadata', 'indexer_configuration_id',
	'from_revision'],
	cur)
	db.origin_intrinsic_metadata_add_from_temp(conflict_update, cur)

	@remote_api_endpoint('indexer_configuration/add')
	@db_transaction_generator()
	def indexer_configuration_add(self, tools, db=None, cur=None):
	"""Add new tools to the storage.

	Args:
	tools ([dict]): List of dictionary representing tool to
	insert in the db. Dictionary with the following keys:

	- tool_name (str): tool's name
	- tool_version (str): tool's version
	- tool_configuration (dict): tool's configuration
	(free form dict)

	Returns:
	List of dict inserted in the db (holding the id key as
	well). The order of the list is not guaranteed to match
	the order of the initial list.

	"""
	db.mktemp_indexer_configuration(cur)
	db.copy_to(tools, 'tmp_indexer_configuration',
	['tool_name', 'tool_version', 'tool_configuration'],
	cur)

	tools = db.indexer_configuration_add_from_temp(cur)
	for line in tools:
	yield dict(zip(db.indexer_configuration_cols, line))

	@remote_api_endpoint('indexer_configuration/data')
	@db_transaction()
	def indexer_configuration_get(self, tool, db=None, cur=None):
	"""Retrieve tool information.

	Args:
	tool (dict): Dictionary representing a tool with the
	following keys:

	- tool_name (str): tool's name
	- tool_version (str): tool's version
	- tool_configuration (dict): tool's configuration
	(free form dict)

	Returns:
	The identifier of the tool if it exists, None otherwise.

	"""
	tool_conf = tool['tool_configuration']
	if isinstance(tool_conf, dict):
	tool_conf = json.dumps(tool_conf)
	idx = db.indexer_configuration_get(tool['tool_name'],
	tool['tool_version'],
	tool_conf)
	if not idx:
	return None
	return dict(zip(db.indexer_configuration_cols, idx))
	diff --git a/swh/indexer/storage/db.py b/swh/indexer/storage/db.py
	index 1e1d3fe..c04fbd7 100644
	--- a/swh/indexer/storage/db.py
	+++ b/swh/indexer/storage/db.py
	@@ -1,359 +1,359 @@
	# Copyright (C) 2015-2018 The Software Heritage developers
	# See the AUTHORS file at the top-level directory of this distribution
	# License: GNU General Public License version 3, or any later version
	# See top-level LICENSE file for more information

	from swh.model import hashutil

	from swh.storage.db import BaseDb, stored_procedure, cursor_to_bytes
	from swh.storage.db import line_to_bytes, execute_values_to_bytes


	class Db(BaseDb):
	"""Proxy to the SWH Indexer DB, with wrappers around stored procedures

	"""
	content_mimetype_hash_keys = ['id', 'indexer_configuration_id']

	def _missing_from_list(self, table, data, hash_keys, cur=None):
	"""Read from table the data with hash_keys that are missing.

	Args:
	table (str): Table name (e.g content_mimetype, content_language,
	etc...)
	data (dict): Dict of data to read from
	hash_keys ([str]): List of keys to read in the data dict.

	Yields:
	The data which is missing from the db.

	"""
	cur = self._cursor(cur)
	keys = ', '.join(hash_keys)
	equality = ' AND '.join(
	('t.%s = c.%s' % (key, key)) for key in hash_keys
	)
	yield from execute_values_to_bytes(
	cur, """
	select %s from (values %%s) as t(%s)
	where not exists (
	select 1 from %s c
	where %s
	)
	""" % (keys, keys, table, equality),
	(tuple(m[k] for k in hash_keys) for m in data)
	)

	def content_mimetype_missing_from_list(self, mimetypes, cur=None):
	"""List missing mimetypes.

	"""
	yield from self._missing_from_list(
	'content_mimetype', mimetypes, self.content_mimetype_hash_keys,
	cur=cur)

	content_mimetype_cols = [
	'id', 'mimetype', 'encoding',
	'tool_id', 'tool_name', 'tool_version', 'tool_configuration']

	@stored_procedure('swh_mktemp_content_mimetype')
	def mktemp_content_mimetype(self, cur=None): pass

	def content_mimetype_add_from_temp(self, conflict_update, cur=None):
	self._cursor(cur).execute("SELECT swh_content_mimetype_add(%s)",
	(conflict_update, ))

	def _convert_key(self, key, main_table='c'):
	"""Convert keys according to specific use in the module.
	Args:
	key (str): Key expression to change according to the alias
	used in the query
	main_table (str): Alias to use for the main table. Default
	to c for content_{something}.

	Expected:
	Tables content_{something} being aliased as 'c' (something
	in {language, mimetype, ...}), table indexer_configuration
	being aliased as 'i'.

	"""
	if key == 'id':
	return '%s.id' % main_table
	elif key == 'tool_id':
	return 'i.id as tool_id'
	elif key == 'licenses':
	return '''
	array(select name
	from fossology_license
	where id = ANY(
	array_agg(%s.license_id))) as licenses''' % main_table
	return key

	def _get_from_list(self, table, ids, cols, cur=None, id_col='id'):
	"""Fetches entries from the `table` such that their `id` field
	(or whatever is given to `id_col`) is in `ids`.
	Returns the columns `cols`.
	The `cur`sor is used to connect to the database.
	"""
	cur = self._cursor(cur)
	keys = map(self._convert_key, cols)
	query = """
	select {keys}
	from (values %s) as t(id)
	inner join {table} c
	on c.{id_col}=t.id
	inner join indexer_configuration i
	on c.indexer_configuration_id=i.id;
	""".format(
	keys=', '.join(keys),
	id_col=id_col,
	table=table)
	yield from execute_values_to_bytes(
	cur, query,
	((_id,) for _id in ids)
	)

	content_indexer_names = {
	'mimetype': 'content_mimetype',
	- 'language': 'content_language',
	+ 'fossology_license': 'content_fossology_license',
	}

	def content_get_range(self, content_type, start, end,
	indexer_configuration_id, limit=1000, cur=None):
	"""Retrieve contents with content_type, within range [start, end]
	bound by limit and associated to the given indexer
	configuration id.

	"""
	cur = self._cursor(cur)
	table = self.content_indexer_names[content_type]
	query = """select t.id
	from %s t
	inner join indexer_configuration ic
	on t.indexer_configuration_id=ic.id
	where ic.id=%%s and
	%%s <= t.id and t.id <= %%s
	order by t.indexer_configuration_id, t.id
	limit %%s""" % table
	cur.execute(query, (indexer_configuration_id, start, end, limit))
	yield from cursor_to_bytes(cur)

	def content_mimetype_get_from_list(self, ids, cur=None):
	yield from self._get_from_list(
	'content_mimetype', ids, self.content_mimetype_cols, cur=cur)

	content_language_hash_keys = ['id', 'indexer_configuration_id']

	def content_language_missing_from_list(self, languages, cur=None):
	"""List missing languages.

	"""
	yield from self._missing_from_list(
	'content_language', languages, self.content_language_hash_keys,
	cur=cur)

	content_language_cols = [
	'id', 'lang',
	'tool_id', 'tool_name', 'tool_version', 'tool_configuration']

	@stored_procedure('swh_mktemp_content_language')
	def mktemp_content_language(self, cur=None): pass

	def content_language_add_from_temp(self, conflict_update, cur=None):
	self._cursor(cur).execute("SELECT swh_content_language_add(%s)",
	(conflict_update, ))

	def content_language_get_from_list(self, ids, cur=None):
	yield from self._get_from_list(
	'content_language', ids, self.content_language_cols, cur=cur)

	content_ctags_hash_keys = ['id', 'indexer_configuration_id']

	def content_ctags_missing_from_list(self, ctags, cur=None):
	"""List missing ctags.

	"""
	yield from self._missing_from_list(
	'content_ctags', ctags, self.content_ctags_hash_keys,
	cur=cur)

	content_ctags_cols = [
	'id', 'name', 'kind', 'line', 'lang',
	'tool_id', 'tool_name', 'tool_version', 'tool_configuration']

	@stored_procedure('swh_mktemp_content_ctags')
	def mktemp_content_ctags(self, cur=None): pass

	def content_ctags_add_from_temp(self, conflict_update, cur=None):
	self._cursor(cur).execute("SELECT swh_content_ctags_add(%s)",
	(conflict_update, ))

	def content_ctags_get_from_list(self, ids, cur=None):
	cur = self._cursor(cur)
	keys = map(self._convert_key, self.content_ctags_cols)
	yield from execute_values_to_bytes(
	cur, """
	select %s
	from (values %%s) as t(id)
	inner join content_ctags c
	on c.id=t.id
	inner join indexer_configuration i
	on c.indexer_configuration_id=i.id
	order by line
	""" % ', '.join(keys),
	((_id,) for _id in ids)
	)

	def content_ctags_search(self, expression, last_sha1, limit, cur=None):
	cur = self._cursor(cur)
	if not last_sha1:
	query = """SELECT %s
	FROM swh_content_ctags_search(%%s, %%s)""" % (
	','.join(self.content_ctags_cols))
	cur.execute(query, (expression, limit))
	else:
	if last_sha1 and isinstance(last_sha1, bytes):
	last_sha1 = '\\x%s' % hashutil.hash_to_hex(last_sha1)
	elif last_sha1:
	last_sha1 = '\\x%s' % last_sha1

	query = """SELECT %s
	FROM swh_content_ctags_search(%%s, %%s, %%s)""" % (
	','.join(self.content_ctags_cols))
	cur.execute(query, (expression, limit, last_sha1))

	yield from cursor_to_bytes(cur)

	content_fossology_license_cols = [
	'id', 'tool_id', 'tool_name', 'tool_version', 'tool_configuration',
	'licenses']

	@stored_procedure('swh_mktemp_content_fossology_license')
	def mktemp_content_fossology_license(self, cur=None): pass

	def content_fossology_license_add_from_temp(self, conflict_update,
	cur=None):
	"""Add new licenses per content.

	"""
	self._cursor(cur).execute(
	"SELECT swh_content_fossology_license_add(%s)",
	(conflict_update, ))

	def content_fossology_license_get_from_list(self, ids, cur=None):
	"""Retrieve licenses per id.

	"""
	cur = self._cursor(cur)
	keys = map(self._convert_key, self.content_fossology_license_cols)
	yield from execute_values_to_bytes(
	cur, """
	select %s
	from (values %%s) as t(id)
	inner join content_fossology_license c on t.id=c.id
	inner join indexer_configuration i
	on i.id=c.indexer_configuration_id
	group by c.id, i.id, i.tool_name, i.tool_version,
	i.tool_configuration;
	""" % ', '.join(keys),
	((_id,) for _id in ids)
	)

	content_metadata_hash_keys = ['id', 'indexer_configuration_id']

	def content_metadata_missing_from_list(self, metadata, cur=None):
	"""List missing metadata.

	"""
	yield from self._missing_from_list(
	'content_metadata', metadata, self.content_metadata_hash_keys,
	cur=cur)

	content_metadata_cols = [
	'id', 'translated_metadata',
	'tool_id', 'tool_name', 'tool_version', 'tool_configuration']

	@stored_procedure('swh_mktemp_content_metadata')
	def mktemp_content_metadata(self, cur=None): pass

	def content_metadata_add_from_temp(self, conflict_update, cur=None):
	self._cursor(cur).execute("SELECT swh_content_metadata_add(%s)",
	(conflict_update, ))

	def content_metadata_get_from_list(self, ids, cur=None):
	yield from self._get_from_list(
	'content_metadata', ids, self.content_metadata_cols, cur=cur)

	revision_metadata_hash_keys = ['id', 'indexer_configuration_id']

	def revision_metadata_missing_from_list(self, metadata, cur=None):
	"""List missing metadata.

	"""
	yield from self._missing_from_list(
	'revision_metadata', metadata, self.revision_metadata_hash_keys,
	cur=cur)

	revision_metadata_cols = [
	'id', 'translated_metadata',
	'tool_id', 'tool_name', 'tool_version', 'tool_configuration']

	@stored_procedure('swh_mktemp_revision_metadata')
	def mktemp_revision_metadata(self, cur=None): pass

	def revision_metadata_add_from_temp(self, conflict_update, cur=None):
	self._cursor(cur).execute("SELECT swh_revision_metadata_add(%s)",
	(conflict_update, ))

	def revision_metadata_get_from_list(self, ids, cur=None):
	yield from self._get_from_list(
	'revision_metadata', ids, self.revision_metadata_cols, cur=cur)

	origin_intrinsic_metadata_cols = [
	'origin_id', 'metadata', 'from_revision',
	'tool_id', 'tool_name', 'tool_version', 'tool_configuration']

	@stored_procedure('swh_mktemp_origin_intrinsic_metadata')
	def mktemp_origin_intrinsic_metadata(self, cur=None): pass

	def origin_intrinsic_metadata_add_from_temp(
	self, conflict_update, cur=None):
	cur = self._cursor(cur)
	cur.execute(
	"SELECT swh_origin_intrinsic_metadata_add(%s)",
	(conflict_update, ))

	def origin_intrinsic_metadata_get_from_list(self, orig_ids, cur=None):
	yield from self._get_from_list(
	'origin_intrinsic_metadata', orig_ids,
	self.origin_intrinsic_metadata_cols, cur=cur,
	id_col='origin_id')

	indexer_configuration_cols = ['id', 'tool_name', 'tool_version',
	'tool_configuration']

	@stored_procedure('swh_mktemp_indexer_configuration')
	def mktemp_indexer_configuration(self, cur=None):
	pass

	def indexer_configuration_add_from_temp(self, cur=None):
	cur = self._cursor(cur)
	cur.execute("SELECT %s from swh_indexer_configuration_add()" % (
	','.join(self.indexer_configuration_cols), ))
	yield from cursor_to_bytes(cur)

	def indexer_configuration_get(self, tool_name,
	tool_version, tool_configuration, cur=None):
	cur = self._cursor(cur)
	cur.execute('''select %s
	from indexer_configuration
	where tool_name=%%s and
	tool_version=%%s and
	tool_configuration=%%s''' % (
	','.join(self.indexer_configuration_cols)),
	(tool_name, tool_version, tool_configuration))

	data = cur.fetchone()
	if not data:
	return None
	return line_to_bytes(data)
	diff --git a/swh/indexer/tests/storage/__init__.py b/swh/indexer/tests/storage/__init__.py
	index f2c5f52..a194d3d 100644
	--- a/swh/indexer/tests/storage/__init__.py
	+++ b/swh/indexer/tests/storage/__init__.py
	@@ -1,83 +1,142 @@
	# Copyright (C) 2018 The Software Heritage developers
	# See the AUTHORS file at the top-level directory of this distribution
	# License: GNU General Public License version 3, or any later version
	# See top-level LICENSE file for more information

	from os import path
	import swh.storage

	from swh.model.hashutil import MultiHash
	from hypothesis.strategies import (composite, sets, one_of, uuids,
	tuples, sampled_from)

	SQL_DIR = path.join(path.dirname(swh.indexer.__file__), 'sql')


	MIMETYPES = [
	b'application/json',
	b'application/octet-stream',
	b'application/xml',
	b'text/plain',
	]

	ENCODINGS = [
	b'iso8859-1',
	b'iso8859-15',
	b'latin1',
	b'utf-8',
	]


	def gen_mimetype():
	"""Generate one mimetype strategy.

	"""
	return one_of(sampled_from(MIMETYPES))


	def gen_encoding():
	"""Generate one encoding strategy.

	"""
	return one_of(sampled_from(ENCODINGS))


	+def _init_content(uuid):
	+ """Given a uuid, initialize a content
	+
	+ """
	+ return {
	+ 'id': MultiHash.from_data(uuid.bytes, {'sha1'}).digest()['sha1'],
	+ 'indexer_configuration_id': 1,
	+ }
	+
	+
	@composite
	def gen_content_mimetypes(draw, *, min_size=0, max_size=100):
	"""Generate valid and consistent content_mimetypes.

	Context: Test purposes

	Args:
	draw (callable): Used by hypothesis to generate data
	min_size (int): Minimal number of elements to generate
	(default: 0)
	max_size (int): Maximal number of elements to generate
	(default: 100)

	Returns:
	List of content_mimetypes as expected by the
	content_mimetype_add api endpoint.

	"""
	_ids = draw(
	sets(
	tuples(
	uuids(),
	gen_mimetype(),
	gen_encoding()
	),
	min_size=min_size, max_size=max_size
	)
	)

	content_mimetypes = []
	for uuid, mimetype, encoding in _ids:
	- content_id = MultiHash.from_data(uuid.bytes, {'sha1'}).digest()['sha1']
	content_mimetypes.append({
	- 'id': content_id,
	+ **_init_content(uuid),
	'mimetype': mimetype,
	'encoding': encoding,
	- 'indexer_configuration_id': 1,
	})
	return content_mimetypes
	+
	+
	+FOSSOLOGY_LICENSES = [
	+ b'3DFX',
	+ b'BSD',
	+ b'GPL',
	+ b'Apache2',
	+ b'MIT',
	+]
	+
	+
	+def gen_license():
	+ return one_of(sampled_from(FOSSOLOGY_LICENSES))
	+
	+
	+@composite
	+def gen_content_fossology_licenses(draw, *, min_size=0, max_size=100):
	+ """Generate valid and consistent content_fossology_licenses.
	+
	+ Context: Test purposes
	+
	+ Args:
	+ draw (callable): Used by hypothesis to generate data
	+ min_size (int): Minimal number of elements to generate
	+ (default: 0)
	+ max_size (int): Maximal number of elements to generate
	+ (default: 100)
	+
	+ Returns:
	+ List of content_fossology_licenses as expected by the
	+ content_fossology_license_add api endpoint.
	+
	+ """
	+ _ids = draw(
	+ sets(
	+ tuples(
	+ uuids(),
	+ gen_license(),
	+ ),
	+ min_size=min_size, max_size=max_size
	+ )
	+ )
	+
	+ content_licenses = []
	+ for uuid, license in _ids:
	+ content_licenses.append({
	+ **_init_content(uuid),
	+ 'licenses': [license],
	+ 'indexer_configuration_id': 1,
	+ })
	+ return content_licenses
	diff --git a/swh/indexer/tests/storage/test_storage.py b/swh/indexer/tests/storage/test_storage.py
	index 5ab4001..c2e766b 100644
	--- a/swh/indexer/tests/storage/test_storage.py
	+++ b/swh/indexer/tests/storage/test_storage.py
	@@ -1,1706 +1,1785 @@
	# Copyright (C) 2015-2018 The Software Heritage developers
	# See the AUTHORS file at the top-level directory of this distribution
	# License: GNU General Public License version 3, or any later version
	# See top-level LICENSE file for more information

	import os
	import pytest
	import unittest

	from hypothesis import given

	from swh.model.hashutil import hash_to_bytes

	from swh.indexer.storage import get_indexer_storage
	from swh.core.tests.db_testing import SingleDbTestFixture
	-from swh.indexer.tests.storage import SQL_DIR, gen_content_mimetypes
	+from swh.indexer.tests.storage import (
	+ SQL_DIR, gen_content_mimetypes, gen_content_fossology_licenses
	+)


	@pytest.mark.db
	class BaseTestStorage(SingleDbTestFixture):
	"""Base test class for most indexer tests.

	It adds support for Storage testing to the SingleDbTestFixture class.
	It will also build the database from the swh-indexed/sql/*.sql files.
	"""

	TEST_DB_NAME = 'softwareheritage-test-indexer'
	TEST_DB_DUMP = os.path.join(SQL_DIR, '*.sql')

	def setUp(self):
	super().setUp()
	self.storage_config = {
	'cls': 'local',
	'args': {
	'db': 'dbname=%s' % self.TEST_DB_NAME,
	},
	}
	self.storage = get_indexer_storage(**self.storage_config)

	self.sha1_1 = hash_to_bytes('34973274ccef6ab4dfaaf86599792fa9c3fe4689')
	self.sha1_2 = hash_to_bytes('61c2b3a30496d329e21af70dd2d7e097046d07b7')
	self.revision_id_1 = hash_to_bytes(
	'7026b7c1a2af56521e951c01ed20f255fa054238')
	self.revision_id_2 = hash_to_bytes(
	'7026b7c1a2af56521e9587659012345678904321')
	self.origin_id_1 = 54974445

	cur = self.test_db[self.TEST_DB_NAME].cursor
	tools = {}
	cur.execute('''
	select tool_name, id, tool_version, tool_configuration
	from indexer_configuration
	order by id''')
	for row in cur.fetchall():
	key = row[0]
	while key in tools:
	key = '_' + key
	tools[key] = {
	'id': row[1],
	'name': row[0],
	'version': row[2],
	'configuration': row[3]
	}
	self.tools = tools

	def tearDown(self):
	self.reset_storage_tables()
	self.storage = None
	super().tearDown()

	def reset_storage_tables(self):
	excluded = {'indexer_configuration'}
	self.reset_db_tables(self.TEST_DB_NAME, excluded=excluded)

	db = self.test_db[self.TEST_DB_NAME]
	db.conn.commit()


	@pytest.mark.db
	class CommonTestStorage(BaseTestStorage):
	"""Base class for Indexer Storage testing.

	"""

	def test_check_config(self):
	self.assertTrue(self.storage.check_config(check_write=True))
	self.assertTrue(self.storage.check_config(check_write=False))

	def test_content_mimetype_missing(self):
	# given
	tool_id = self.tools['file']['id']

	mimetypes = [
	{
	'id': self.sha1_1,
	'indexer_configuration_id': tool_id,
	},
	{
	'id': self.sha1_2,
	'indexer_configuration_id': tool_id,
	}]

	# when
	actual_missing = self.storage.content_mimetype_missing(mimetypes)

	# then
	self.assertEqual(list(actual_missing), [
	self.sha1_1,
	self.sha1_2,
	])

	# given
	self.storage.content_mimetype_add([{
	'id': self.sha1_2,
	'mimetype': b'text/plain',
	'encoding': b'utf-8',
	'indexer_configuration_id': tool_id,
	}])

	# when
	actual_missing = self.storage.content_mimetype_missing(mimetypes)

	# then
	self.assertEqual(list(actual_missing), [self.sha1_1])

	def test_content_mimetype_add__drop_duplicate(self):
	# given
	tool_id = self.tools['file']['id']

	mimetype_v1 = {
	'id': self.sha1_2,
	'mimetype': b'text/plain',
	'encoding': b'utf-8',
	'indexer_configuration_id': tool_id,
	}

	# given
	self.storage.content_mimetype_add([mimetype_v1])

	# when
	actual_mimetypes = list(self.storage.content_mimetype_get(
	[self.sha1_2]))

	# then
	expected_mimetypes_v1 = [{
	'id': self.sha1_2,
	'mimetype': b'text/plain',
	'encoding': b'utf-8',
	'tool': self.tools['file'],
	}]
	self.assertEqual(actual_mimetypes, expected_mimetypes_v1)

	# given
	mimetype_v2 = mimetype_v1.copy()
	mimetype_v2.update({
	'mimetype': b'text/html',
	'encoding': b'us-ascii',
	})

	self.storage.content_mimetype_add([mimetype_v2])

	actual_mimetypes = list(self.storage.content_mimetype_get(
	[self.sha1_2]))

	# mimetype did not change as the v2 was dropped.
	self.assertEqual(actual_mimetypes, expected_mimetypes_v1)

	def test_content_mimetype_add__update_in_place_duplicate(self):
	# given
	tool_id = self.tools['file']['id']

	mimetype_v1 = {
	'id': self.sha1_2,
	'mimetype': b'text/plain',
	'encoding': b'utf-8',
	'indexer_configuration_id': tool_id,
	}

	# given
	self.storage.content_mimetype_add([mimetype_v1])

	# when
	actual_mimetypes = list(self.storage.content_mimetype_get(
	[self.sha1_2]))

	expected_mimetypes_v1 = [{
	'id': self.sha1_2,
	'mimetype': b'text/plain',
	'encoding': b'utf-8',
	'tool': self.tools['file'],
	}]

	# then
	self.assertEqual(actual_mimetypes, expected_mimetypes_v1)

	# given
	mimetype_v2 = mimetype_v1.copy()
	mimetype_v2.update({
	'mimetype': b'text/html',
	'encoding': b'us-ascii',
	})

	self.storage.content_mimetype_add([mimetype_v2], conflict_update=True)

	actual_mimetypes = list(self.storage.content_mimetype_get(
	[self.sha1_2]))

	expected_mimetypes_v2 = [{
	'id': self.sha1_2,
	'mimetype': b'text/html',
	'encoding': b'us-ascii',
	'tool': {
	'id': 2,
	'name': 'file',
	'version': '5.22',
	'configuration': {'command_line': 'file --mime <filepath>'}
	}
	}]

	# mimetype did change as the v2 was used to overwrite v1
	self.assertEqual(actual_mimetypes, expected_mimetypes_v2)

	def test_content_mimetype_get(self):
	# given
	tool_id = self.tools['file']['id']

	mimetypes = [self.sha1_2, self.sha1_1]

	mimetype1 = {
	'id': self.sha1_2,
	'mimetype': b'text/plain',
	'encoding': b'utf-8',
	'indexer_configuration_id': tool_id,
	}

	# when
	self.storage.content_mimetype_add([mimetype1])

	# then
	actual_mimetypes = list(self.storage.content_mimetype_get(mimetypes))

	# then
	expected_mimetypes = [{
	'id': self.sha1_2,
	'mimetype': b'text/plain',
	'encoding': b'utf-8',
	'tool': self.tools['file']
	}]

	self.assertEqual(actual_mimetypes, expected_mimetypes)

	def test_content_language_missing(self):
	# given
	tool_id = self.tools['pygments']['id']

	languages = [
	{
	'id': self.sha1_2,
	'indexer_configuration_id': tool_id,
	},
	{
	'id': self.sha1_1,
	'indexer_configuration_id': tool_id,
	}
	]

	# when
	actual_missing = list(self.storage.content_language_missing(languages))

	# then
	self.assertEqual(list(actual_missing), [
	self.sha1_2,
	self.sha1_1,
	])

	# given
	self.storage.content_language_add([{
	'id': self.sha1_2,
	'lang': 'haskell',
	'indexer_configuration_id': tool_id,
	}])

	# when
	actual_missing = list(self.storage.content_language_missing(languages))

	# then
	self.assertEqual(actual_missing, [self.sha1_1])

	def test_content_language_get(self):
	# given
	tool_id = self.tools['pygments']['id']

	language1 = {
	'id': self.sha1_2,
	'lang': 'common-lisp',
	'indexer_configuration_id': tool_id,
	}

	# when
	self.storage.content_language_add([language1])

	# then
	actual_languages = list(self.storage.content_language_get(
	[self.sha1_2, self.sha1_1]))

	# then
	expected_languages = [{
	'id': self.sha1_2,
	'lang': 'common-lisp',
	'tool': self.tools['pygments']
	}]

	self.assertEqual(actual_languages, expected_languages)

	def test_content_language_add__drop_duplicate(self):
	# given
	tool_id = self.tools['pygments']['id']

	language_v1 = {
	'id': self.sha1_2,
	'lang': 'emacslisp',
	'indexer_configuration_id': tool_id,
	}

	# given
	self.storage.content_language_add([language_v1])

	# when
	actual_languages = list(self.storage.content_language_get(
	[self.sha1_2]))

	# then
	expected_languages_v1 = [{
	'id': self.sha1_2,
	'lang': 'emacslisp',
	'tool': self.tools['pygments']
	}]
	self.assertEqual(actual_languages, expected_languages_v1)

	# given
	language_v2 = language_v1.copy()
	language_v2.update({
	'lang': 'common-lisp',
	})

	self.storage.content_language_add([language_v2])

	actual_languages = list(self.storage.content_language_get(
	[self.sha1_2]))

	# language did not change as the v2 was dropped.
	self.assertEqual(actual_languages, expected_languages_v1)

	def test_content_language_add__update_in_place_duplicate(self):
	# given
	tool_id = self.tools['pygments']['id']

	language_v1 = {
	'id': self.sha1_2,
	'lang': 'common-lisp',
	'indexer_configuration_id': tool_id,
	}

	# given
	self.storage.content_language_add([language_v1])

	# when
	actual_languages = list(self.storage.content_language_get(
	[self.sha1_2]))

	# then
	expected_languages_v1 = [{
	'id': self.sha1_2,
	'lang': 'common-lisp',
	'tool': self.tools['pygments']
	}]
	self.assertEqual(actual_languages, expected_languages_v1)

	# given
	language_v2 = language_v1.copy()
	language_v2.update({
	'lang': 'emacslisp',
	})

	self.storage.content_language_add([language_v2], conflict_update=True)

	actual_languages = list(self.storage.content_language_get(
	[self.sha1_2]))

	# language did not change as the v2 was dropped.
	expected_languages_v2 = [{
	'id': self.sha1_2,
	'lang': 'emacslisp',
	'tool': self.tools['pygments']
	}]

	# language did change as the v2 was used to overwrite v1
	self.assertEqual(actual_languages, expected_languages_v2)

	def test_content_ctags_missing(self):
	# given
	tool_id = self.tools['universal-ctags']['id']

	ctags = [
	{
	'id': self.sha1_2,
	'indexer_configuration_id': tool_id,
	},
	{
	'id': self.sha1_1,
	'indexer_configuration_id': tool_id,
	}
	]

	# when
	actual_missing = self.storage.content_ctags_missing(ctags)

	# then
	self.assertEqual(list(actual_missing), [
	self.sha1_2,
	self.sha1_1
	])

	# given
	self.storage.content_ctags_add([
	{
	'id': self.sha1_2,
	'indexer_configuration_id': tool_id,
	'ctags': [{
	'name': 'done',
	'kind': 'variable',
	'line': 119,
	'lang': 'OCaml',
	}]
	},
	])

	# when
	actual_missing = self.storage.content_ctags_missing(ctags)

	# then
	self.assertEqual(list(actual_missing), [self.sha1_1])

	def test_content_ctags_get(self):
	# given
	tool_id = self.tools['universal-ctags']['id']

	ctags = [self.sha1_2, self.sha1_1]

	ctag1 = {
	'id': self.sha1_2,
	'indexer_configuration_id': tool_id,
	'ctags': [
	{
	'name': 'done',
	'kind': 'variable',
	'line': 100,
	'lang': 'Python',
	},
	{
	'name': 'main',
	'kind': 'function',
	'line': 119,
	'lang': 'Python',
	}]
	}

	# when
	self.storage.content_ctags_add([ctag1])

	# then
	actual_ctags = list(self.storage.content_ctags_get(ctags))

	# then

	expected_ctags = [
	{
	'id': self.sha1_2,
	'tool': self.tools['universal-ctags'],
	'name': 'done',
	'kind': 'variable',
	'line': 100,
	'lang': 'Python',
	},
	{
	'id': self.sha1_2,
	'tool': self.tools['universal-ctags'],
	'name': 'main',
	'kind': 'function',
	'line': 119,
	'lang': 'Python',
	}
	]

	self.assertEqual(actual_ctags, expected_ctags)

	def test_content_ctags_search(self):
	# 1. given
	tool = self.tools['universal-ctags']
	tool_id = tool['id']

	ctag1 = {
	'id': self.sha1_1,
	'indexer_configuration_id': tool_id,
	'ctags': [
	{
	'name': 'hello',
	'kind': 'function',
	'line': 133,
	'lang': 'Python',
	},
	{
	'name': 'counter',
	'kind': 'variable',
	'line': 119,
	'lang': 'Python',
	},
	]
	}

	ctag2 = {
	'id': self.sha1_2,
	'indexer_configuration_id': tool_id,
	'ctags': [
	{
	'name': 'hello',
	'kind': 'variable',
	'line': 100,
	'lang': 'C',
	},
	]
	}

	self.storage.content_ctags_add([ctag1, ctag2])

	# 1. when
	actual_ctags = list(self.storage.content_ctags_search('hello',
	limit=1))

	# 1. then
	self.assertEqual(actual_ctags, [
	{
	'id': ctag1['id'],
	'tool': tool,
	'name': 'hello',
	'kind': 'function',
	'line': 133,
	'lang': 'Python',
	}
	])

	# 2. when
	actual_ctags = list(self.storage.content_ctags_search(
	'hello',
	limit=1,
	last_sha1=ctag1['id']))

	# 2. then
	self.assertEqual(actual_ctags, [
	{
	'id': ctag2['id'],
	'tool': tool,
	'name': 'hello',
	'kind': 'variable',
	'line': 100,
	'lang': 'C',
	}
	])

	# 3. when
	actual_ctags = list(self.storage.content_ctags_search('hello'))

	# 3. then
	self.assertEqual(actual_ctags, [
	{
	'id': ctag1['id'],
	'tool': tool,
	'name': 'hello',
	'kind': 'function',
	'line': 133,
	'lang': 'Python',
	},
	{
	'id': ctag2['id'],
	'tool': tool,
	'name': 'hello',
	'kind': 'variable',
	'line': 100,
	'lang': 'C',
	},
	])

	# 4. when
	actual_ctags = list(self.storage.content_ctags_search('counter'))

	# then
	self.assertEqual(actual_ctags, [{
	'id': ctag1['id'],
	'tool': tool,
	'name': 'counter',
	'kind': 'variable',
	'line': 119,
	'lang': 'Python',
	}])

	def test_content_ctags_search_no_result(self):
	actual_ctags = list(self.storage.content_ctags_search('counter'))

	self.assertEqual(actual_ctags, [])

	def test_content_ctags_add__add_new_ctags_added(self):
	# given
	tool = self.tools['universal-ctags']
	tool_id = tool['id']

	ctag_v1 = {
	'id': self.sha1_2,
	'indexer_configuration_id': tool_id,
	'ctags': [{
	'name': 'done',
	'kind': 'variable',
	'line': 100,
	'lang': 'Scheme',
	}]
	}

	# given
	self.storage.content_ctags_add([ctag_v1])
	self.storage.content_ctags_add([ctag_v1]) # conflict does nothing

	# when
	actual_ctags = list(self.storage.content_ctags_get(
	[self.sha1_2]))

	# then
	expected_ctags = [{
	'id': self.sha1_2,
	'name': 'done',
	'kind': 'variable',
	'line': 100,
	'lang': 'Scheme',
	'tool': tool,
	}]

	self.assertEqual(actual_ctags, expected_ctags)

	# given
	ctag_v2 = ctag_v1.copy()
	ctag_v2.update({
	'ctags': [
	{
	'name': 'defn',
	'kind': 'function',
	'line': 120,
	'lang': 'Scheme',
	}
	]
	})

	self.storage.content_ctags_add([ctag_v2])

	expected_ctags = [
	{
	'id': self.sha1_2,
	'name': 'done',
	'kind': 'variable',
	'line': 100,
	'lang': 'Scheme',
	'tool': tool,
	}, {
	'id': self.sha1_2,
	'name': 'defn',
	'kind': 'function',
	'line': 120,
	'lang': 'Scheme',
	'tool': tool,
	}
	]

	actual_ctags = list(self.storage.content_ctags_get(
	[self.sha1_2]))

	self.assertEqual(actual_ctags, expected_ctags)

	def test_content_ctags_add__update_in_place(self):
	# given
	tool = self.tools['universal-ctags']
	tool_id = tool['id']

	ctag_v1 = {
	'id': self.sha1_2,
	'indexer_configuration_id': tool_id,
	'ctags': [{
	'name': 'done',
	'kind': 'variable',
	'line': 100,
	'lang': 'Scheme',
	}]
	}

	# given
	self.storage.content_ctags_add([ctag_v1])

	# when
	actual_ctags = list(self.storage.content_ctags_get(
	[self.sha1_2]))

	# then
	expected_ctags = [
	{
	'id': self.sha1_2,
	'name': 'done',
	'kind': 'variable',
	'line': 100,
	'lang': 'Scheme',
	'tool': tool
	}
	]
	self.assertEqual(actual_ctags, expected_ctags)

	# given
	ctag_v2 = ctag_v1.copy()
	ctag_v2.update({
	'ctags': [
	{
	'name': 'done',
	'kind': 'variable',
	'line': 100,
	'lang': 'Scheme',
	},
	{
	'name': 'defn',
	'kind': 'function',
	'line': 120,
	'lang': 'Scheme',
	}
	]
	})

	self.storage.content_ctags_add([ctag_v2], conflict_update=True)

	actual_ctags = list(self.storage.content_ctags_get(
	[self.sha1_2]))

	# ctag did change as the v2 was used to overwrite v1
	expected_ctags = [
	{
	'id': self.sha1_2,
	'name': 'done',
	'kind': 'variable',
	'line': 100,
	'lang': 'Scheme',
	'tool': tool,
	},
	{
	'id': self.sha1_2,
	'name': 'defn',
	'kind': 'function',
	'line': 120,
	'lang': 'Scheme',
	'tool': tool,
	}
	]
	self.assertEqual(actual_ctags, expected_ctags)

	def test_content_fossology_license_get(self):
	# given
	tool = self.tools['nomos']
	tool_id = tool['id']

	license1 = {
	'id': self.sha1_1,
	'licenses': ['GPL-2.0+'],
	'indexer_configuration_id': tool_id,
	}

	# when
	self.storage.content_fossology_license_add([license1])

	# then
	actual_licenses = list(self.storage.content_fossology_license_get(
	[self.sha1_2, self.sha1_1]))

	expected_license = {
	self.sha1_1: [{
	'licenses': ['GPL-2.0+'],
	'tool': tool,
	}]
	}

	# then
	self.assertEqual(actual_licenses, [expected_license])

	def test_content_fossology_license_add__new_license_added(self):
	# given
	tool = self.tools['nomos']
	tool_id = tool['id']

	license_v1 = {
	'id': self.sha1_1,
	'licenses': ['Apache-2.0'],
	'indexer_configuration_id': tool_id,
	}

	# given
	self.storage.content_fossology_license_add([license_v1])
	# conflict does nothing
	self.storage.content_fossology_license_add([license_v1])

	# when
	actual_licenses = list(self.storage.content_fossology_license_get(
	[self.sha1_1]))

	# then
	expected_license = {
	self.sha1_1: [{
	'licenses': ['Apache-2.0'],
	'tool': tool,
	}]
	}
	self.assertEqual(actual_licenses, [expected_license])

	# given
	license_v2 = license_v1.copy()
	license_v2.update({
	'licenses': ['BSD-2-Clause'],
	})

	self.storage.content_fossology_license_add([license_v2])

	actual_licenses = list(self.storage.content_fossology_license_get(
	[self.sha1_1]))

	expected_license = {
	self.sha1_1: [{
	'licenses': ['Apache-2.0', 'BSD-2-Clause'],
	'tool': tool
	}]
	}

	# license did not change as the v2 was dropped.
	self.assertEqual(actual_licenses, [expected_license])

	def test_content_fossology_license_add__update_in_place_duplicate(self):
	# given
	tool = self.tools['nomos']
	tool_id = tool['id']

	license_v1 = {
	'id': self.sha1_1,
	'licenses': ['CECILL'],
	'indexer_configuration_id': tool_id,
	}

	# given
	self.storage.content_fossology_license_add([license_v1])
	# conflict does nothing
	self.storage.content_fossology_license_add([license_v1])

	# when
	actual_licenses = list(self.storage.content_fossology_license_get(
	[self.sha1_1]))

	# then
	expected_license = {
	self.sha1_1: [{
	'licenses': ['CECILL'],
	'tool': tool,
	}]
	}
	self.assertEqual(actual_licenses, [expected_license])

	# given
	license_v2 = license_v1.copy()
	license_v2.update({
	'licenses': ['CECILL-2.0']
	})

	self.storage.content_fossology_license_add([license_v2],
	conflict_update=True)

	actual_licenses = list(self.storage.content_fossology_license_get(
	[self.sha1_1]))

	# license did change as the v2 was used to overwrite v1
	expected_license = {
	self.sha1_1: [{
	'licenses': ['CECILL-2.0'],
	'tool': tool,
	}]
	}
	self.assertEqual(actual_licenses, [expected_license])

	def test_content_metadata_missing(self):
	# given
	tool_id = self.tools['swh-metadata-translator']['id']

	metadata = [
	{
	'id': self.sha1_2,
	'indexer_configuration_id': tool_id,
	},
	{
	'id': self.sha1_1,
	'indexer_configuration_id': tool_id,
	}
	]

	# when
	actual_missing = list(self.storage.content_metadata_missing(metadata))

	# then
	self.assertEqual(list(actual_missing), [
	self.sha1_2,
	self.sha1_1,
	])

	# given
	self.storage.content_metadata_add([{
	'id': self.sha1_2,
	'translated_metadata': {
	'other': {},
	'codeRepository': {
	'type': 'git',
	'url': 'https://github.com/moranegg/metadata_test'
	},
	'description': 'Simple package.json test for indexer',
	'name': 'test_metadata',
	'version': '0.0.1'
	},
	'indexer_configuration_id': tool_id
	}])

	# when
	actual_missing = list(self.storage.content_metadata_missing(metadata))

	# then
	self.assertEqual(actual_missing, [self.sha1_1])

	def test_content_metadata_get(self):
	# given
	tool_id = self.tools['swh-metadata-translator']['id']

	metadata1 = {
	'id': self.sha1_2,
	'translated_metadata': {
	'other': {},
	'codeRepository': {
	'type': 'git',
	'url': 'https://github.com/moranegg/metadata_test'
	},
	'description': 'Simple package.json test for indexer',
	'name': 'test_metadata',
	'version': '0.0.1'
	},
	'indexer_configuration_id': tool_id,
	}

	# when
	self.storage.content_metadata_add([metadata1])

	# then
	actual_metadata = list(self.storage.content_metadata_get(
	[self.sha1_2, self.sha1_1]))

	expected_metadata = [{
	'id': self.sha1_2,
	'translated_metadata': {
	'other': {},
	'codeRepository': {
	'type': 'git',
	'url': 'https://github.com/moranegg/metadata_test'
	},
	'description': 'Simple package.json test for indexer',
	'name': 'test_metadata',
	'version': '0.0.1'
	},
	'tool': self.tools['swh-metadata-translator']
	}]

	self.assertEqual(actual_metadata, expected_metadata)

	def test_content_metadata_add_drop_duplicate(self):
	# given
	tool_id = self.tools['swh-metadata-translator']['id']

	metadata_v1 = {
	'id': self.sha1_2,
	'translated_metadata': {
	'other': {},
	'name': 'test_metadata',
	'version': '0.0.1'
	},
	'indexer_configuration_id': tool_id,
	}

	# given
	self.storage.content_metadata_add([metadata_v1])

	# when
	actual_metadata = list(self.storage.content_metadata_get(
	[self.sha1_2]))

	expected_metadata_v1 = [{
	'id': self.sha1_2,
	'translated_metadata': {
	'other': {},
	'name': 'test_metadata',
	'version': '0.0.1'
	},
	'tool': self.tools['swh-metadata-translator']
	}]

	self.assertEqual(actual_metadata, expected_metadata_v1)

	# given
	metadata_v2 = metadata_v1.copy()
	metadata_v2.update({
	'translated_metadata': {
	'other': {},
	'name': 'test_drop_duplicated_metadata',
	'version': '0.0.1'
	},
	})

	self.storage.content_metadata_add([metadata_v2])

	# then
	actual_metadata = list(self.storage.content_metadata_get(
	[self.sha1_2]))

	# metadata did not change as the v2 was dropped.
	self.assertEqual(actual_metadata, expected_metadata_v1)

	def test_content_metadata_add_update_in_place_duplicate(self):
	# given
	tool_id = self.tools['swh-metadata-translator']['id']

	metadata_v1 = {
	'id': self.sha1_2,
	'translated_metadata': {
	'other': {},
	'name': 'test_metadata',
	'version': '0.0.1'
	},
	'indexer_configuration_id': tool_id,
	}

	# given
	self.storage.content_metadata_add([metadata_v1])

	# when
	actual_metadata = list(self.storage.content_metadata_get(
	[self.sha1_2]))

	# then
	expected_metadata_v1 = [{
	'id': self.sha1_2,
	'translated_metadata': {
	'other': {},
	'name': 'test_metadata',
	'version': '0.0.1'
	},
	'tool': self.tools['swh-metadata-translator']
	}]
	self.assertEqual(actual_metadata, expected_metadata_v1)

	# given
	metadata_v2 = metadata_v1.copy()
	metadata_v2.update({
	'translated_metadata': {
	'other': {},
	'name': 'test_update_duplicated_metadata',
	'version': '0.0.1'
	},
	})
	self.storage.content_metadata_add([metadata_v2], conflict_update=True)

	actual_metadata = list(self.storage.content_metadata_get(
	[self.sha1_2]))

	# language did not change as the v2 was dropped.
	expected_metadata_v2 = [{
	'id': self.sha1_2,
	'translated_metadata': {
	'other': {},
	'name': 'test_update_duplicated_metadata',
	'version': '0.0.1'
	},
	'tool': self.tools['swh-metadata-translator']
	}]

	# metadata did change as the v2 was used to overwrite v1
	self.assertEqual(actual_metadata, expected_metadata_v2)

	def test_revision_metadata_missing(self):
	# given
	tool_id = self.tools['swh-metadata-detector']['id']

	metadata = [
	{
	'id': self.revision_id_1,
	'indexer_configuration_id': tool_id,
	},
	{
	'id': self.revision_id_2,
	'indexer_configuration_id': tool_id,
	}
	]

	# when
	actual_missing = list(self.storage.revision_metadata_missing(
	metadata))

	# then
	self.assertEqual(list(actual_missing), [
	self.revision_id_1,
	self.revision_id_2,
	])

	# given
	self.storage.revision_metadata_add([{
	'id': self.revision_id_1,
	'translated_metadata': {
	'developmentStatus': None,
	'version': None,
	'operatingSystem': None,
	'description': None,
	'keywords': None,
	'issueTracker': None,
	'name': None,
	'author': None,
	'relatedLink': None,
	'url': None,
	'license': None,
	'maintainer': None,
	'email': None,
	'softwareRequirements': None,
	'identifier': None
	},
	'indexer_configuration_id': tool_id
	}])

	# when
	actual_missing = list(self.storage.revision_metadata_missing(
	metadata))

	# then
	self.assertEqual(actual_missing, [self.revision_id_2])

	def test_revision_metadata_get(self):
	# given
	tool_id = self.tools['swh-metadata-detector']['id']

	metadata_rev = {
	'id': self.revision_id_2,
	'translated_metadata': {
	'developmentStatus': None,
	'version': None,
	'operatingSystem': None,
	'description': None,
	'keywords': None,
	'issueTracker': None,
	'name': None,
	'author': None,
	'relatedLink': None,
	'url': None,
	'license': None,
	'maintainer': None,
	'email': None,
	'softwareRequirements': None,
	'identifier': None
	},
	'indexer_configuration_id': tool_id
	}

	# when
	self.storage.revision_metadata_add([metadata_rev])

	# then
	actual_metadata = list(self.storage.revision_metadata_get(
	[self.revision_id_2, self.revision_id_1]))

	expected_metadata = [{
	'id': self.revision_id_2,
	'translated_metadata': metadata_rev['translated_metadata'],
	'tool': self.tools['swh-metadata-detector']
	}]

	self.assertEqual(actual_metadata, expected_metadata)

	def test_revision_metadata_add_drop_duplicate(self):
	# given
	tool_id = self.tools['swh-metadata-detector']['id']

	metadata_v1 = {
	'id': self.revision_id_1,
	'translated_metadata': {
	'developmentStatus': None,
	'version': None,
	'operatingSystem': None,
	'description': None,
	'keywords': None,
	'issueTracker': None,
	'name': None,
	'author': None,
	'relatedLink': None,
	'url': None,
	'license': None,
	'maintainer': None,
	'email': None,
	'softwareRequirements': None,
	'identifier': None
	},
	'indexer_configuration_id': tool_id,
	}

	# given
	self.storage.revision_metadata_add([metadata_v1])

	# when
	actual_metadata = list(self.storage.revision_metadata_get(
	[self.revision_id_1]))

	expected_metadata_v1 = [{
	'id': self.revision_id_1,
	'translated_metadata': metadata_v1['translated_metadata'],
	'tool': self.tools['swh-metadata-detector']
	}]

	self.assertEqual(actual_metadata, expected_metadata_v1)

	# given
	metadata_v2 = metadata_v1.copy()
	metadata_v2.update({
	'translated_metadata': {
	'name': 'test_metadata',
	'author': 'MG',
	},
	})

	self.storage.revision_metadata_add([metadata_v2])

	# then
	actual_metadata = list(self.storage.revision_metadata_get(
	[self.revision_id_1]))

	# metadata did not change as the v2 was dropped.
	self.assertEqual(actual_metadata, expected_metadata_v1)

	def test_revision_metadata_add_update_in_place_duplicate(self):
	# given
	tool_id = self.tools['swh-metadata-detector']['id']

	metadata_v1 = {
	'id': self.revision_id_2,
	'translated_metadata': {
	'developmentStatus': None,
	'version': None,
	'operatingSystem': None,
	'description': None,
	'keywords': None,
	'issueTracker': None,
	'name': None,
	'author': None,
	'relatedLink': None,
	'url': None,
	'license': None,
	'maintainer': None,
	'email': None,
	'softwareRequirements': None,
	'identifier': None
	},
	'indexer_configuration_id': tool_id,
	}

	# given
	self.storage.revision_metadata_add([metadata_v1])

	# when
	actual_metadata = list(self.storage.revision_metadata_get(
	[self.revision_id_2]))

	# then
	expected_metadata_v1 = [{
	'id': self.revision_id_2,
	'translated_metadata': metadata_v1['translated_metadata'],
	'tool': self.tools['swh-metadata-detector']
	}]
	self.assertEqual(actual_metadata, expected_metadata_v1)

	# given
	metadata_v2 = metadata_v1.copy()
	metadata_v2.update({
	'translated_metadata': {
	'name': 'test_update_duplicated_metadata',
	'author': 'MG'
	},
	})
	self.storage.revision_metadata_add([metadata_v2], conflict_update=True)

	actual_metadata = list(self.storage.revision_metadata_get(
	[self.revision_id_2]))

	expected_metadata_v2 = [{
	'id': self.revision_id_2,
	'translated_metadata': metadata_v2['translated_metadata'],
	'tool': self.tools['swh-metadata-detector']
	}]

	# metadata did change as the v2 was used to overwrite v1
	self.assertEqual(actual_metadata, expected_metadata_v2)

	def test_origin_intrinsic_metadata_get(self):
	# given
	tool_id = self.tools['swh-metadata-detector']['id']

	metadata = {
	'developmentStatus': None,
	'version': None,
	'operatingSystem': None,
	'description': None,
	'keywords': None,
	'issueTracker': None,
	'name': None,
	'author': None,
	'relatedLink': None,
	'url': None,
	'license': None,
	'maintainer': None,
	'email': None,
	'softwareRequirements': None,
	'identifier': None,
	}
	metadata_rev = {
	'id': self.revision_id_2,
	'translated_metadata': metadata,
	'indexer_configuration_id': tool_id,
	}
	metadata_origin = {
	'origin_id': self.origin_id_1,
	'metadata': metadata,
	'indexer_configuration_id': tool_id,
	'from_revision': self.revision_id_2,
	}

	# when
	self.storage.revision_metadata_add([metadata_rev])
	self.storage.origin_intrinsic_metadata_add([metadata_origin])

	# then
	actual_metadata = list(self.storage.origin_intrinsic_metadata_get(
	[self.origin_id_1, 42]))

	expected_metadata = [{
	'origin_id': self.origin_id_1,
	'metadata': metadata,
	'tool': self.tools['swh-metadata-detector'],
	'from_revision': self.revision_id_2,
	}]

	self.assertEqual(actual_metadata, expected_metadata)

	def test_origin_intrinsic_metadata_add_drop_duplicate(self):
	# given
	tool_id = self.tools['swh-metadata-detector']['id']

	metadata_v1 = {
	'developmentStatus': None,
	'version': None,
	'operatingSystem': None,
	'description': None,
	'keywords': None,
	'issueTracker': None,
	'name': None,
	'author': None,
	'relatedLink': None,
	'url': None,
	'license': None,
	'maintainer': None,
	'email': None,
	'softwareRequirements': None,
	'identifier': None
	}
	metadata_rev_v1 = {
	'id': self.revision_id_1,
	'translated_metadata': metadata_v1.copy(),
	'indexer_configuration_id': tool_id,
	}
	metadata_origin_v1 = {
	'origin_id': self.origin_id_1,
	'metadata': metadata_v1.copy(),
	'indexer_configuration_id': tool_id,
	'from_revision': self.revision_id_1,
	}

	# given
	self.storage.revision_metadata_add([metadata_rev_v1])
	self.storage.origin_intrinsic_metadata_add([metadata_origin_v1])

	# when
	actual_metadata = list(self.storage.origin_intrinsic_metadata_get(
	[self.origin_id_1, 42]))

	expected_metadata_v1 = [{
	'origin_id': self.origin_id_1,
	'metadata': metadata_v1,
	'tool': self.tools['swh-metadata-detector'],
	'from_revision': self.revision_id_1,
	}]

	self.assertEqual(actual_metadata, expected_metadata_v1)

	# given
	metadata_v2 = metadata_v1.copy()
	metadata_v2.update({
	'name': 'test_metadata',
	'author': 'MG',
	})
	metadata_rev_v2 = metadata_rev_v1.copy()
	metadata_origin_v2 = metadata_origin_v1.copy()
	metadata_rev_v2['translated_metadata'] = metadata_v2
	metadata_origin_v2['translated_metadata'] = metadata_v2

	self.storage.revision_metadata_add([metadata_rev_v2])
	self.storage.origin_intrinsic_metadata_add([metadata_origin_v2])

	# then
	actual_metadata = list(self.storage.origin_intrinsic_metadata_get(
	[self.origin_id_1]))

	# metadata did not change as the v2 was dropped.
	self.assertEqual(actual_metadata, expected_metadata_v1)

	def test_origin_intrinsic_metadata_add_update_in_place_duplicate(self):
	# given
	tool_id = self.tools['swh-metadata-detector']['id']

	metadata_v1 = {
	'developmentStatus': None,
	'version': None,
	'operatingSystem': None,
	'description': None,
	'keywords': None,
	'issueTracker': None,
	'name': None,
	'author': None,
	'relatedLink': None,
	'url': None,
	'license': None,
	'maintainer': None,
	'email': None,
	'softwareRequirements': None,
	'identifier': None
	}
	metadata_rev_v1 = {
	'id': self.revision_id_2,
	'translated_metadata': metadata_v1,
	'indexer_configuration_id': tool_id,
	}
	metadata_origin_v1 = {
	'origin_id': self.origin_id_1,
	'metadata': metadata_v1.copy(),
	'indexer_configuration_id': tool_id,
	'from_revision': self.revision_id_2,
	}

	# given
	self.storage.revision_metadata_add([metadata_rev_v1])
	self.storage.origin_intrinsic_metadata_add([metadata_origin_v1])

	# when
	actual_metadata = list(self.storage.origin_intrinsic_metadata_get(
	[self.origin_id_1]))

	# then
	expected_metadata_v1 = [{
	'origin_id': self.origin_id_1,
	'metadata': metadata_v1,
	'tool': self.tools['swh-metadata-detector'],
	'from_revision': self.revision_id_2,
	}]
	self.assertEqual(actual_metadata, expected_metadata_v1)

	# given
	metadata_v2 = metadata_v1.copy()
	metadata_v2.update({
	'name': 'test_update_duplicated_metadata',
	'author': 'MG',
	})
	metadata_rev_v2 = metadata_rev_v1.copy()
	metadata_origin_v2 = metadata_origin_v1.copy()
	metadata_rev_v2['translated_metadata'] = metadata_v2
	metadata_origin_v2['metadata'] = metadata_v2

	self.storage.revision_metadata_add([metadata_rev_v2],
	conflict_update=True)
	self.storage.origin_intrinsic_metadata_add([metadata_origin_v2],
	conflict_update=True)

	actual_metadata = list(self.storage.origin_intrinsic_metadata_get(
	[self.origin_id_1]))

	expected_metadata_v2 = [{
	'origin_id': self.origin_id_1,
	'metadata': metadata_v2,
	'tool': self.tools['swh-metadata-detector'],
	'from_revision': self.revision_id_2,
	}]

	# metadata did change as the v2 was used to overwrite v1
	self.assertEqual(actual_metadata, expected_metadata_v2)

	def test_indexer_configuration_add(self):
	tool = {
	'tool_name': 'some-unknown-tool',
	'tool_version': 'some-version',
	'tool_configuration': {"debian-package": "some-package"},
	}

	actual_tool = self.storage.indexer_configuration_get(tool)
	self.assertIsNone(actual_tool) # does not exist

	# add it
	actual_tools = list(self.storage.indexer_configuration_add([tool]))

	self.assertEqual(len(actual_tools), 1)
	actual_tool = actual_tools[0]
	self.assertIsNotNone(actual_tool) # now it exists
	new_id = actual_tool.pop('id')
	self.assertEqual(actual_tool, tool)

	actual_tools2 = list(self.storage.indexer_configuration_add([tool]))
	actual_tool2 = actual_tools2[0]
	self.assertIsNotNone(actual_tool2) # now it exists
	new_id2 = actual_tool2.pop('id')

	self.assertEqual(new_id, new_id2)
	self.assertEqual(actual_tool, actual_tool2)

	def test_indexer_configuration_add_multiple(self):
	tool = {
	'tool_name': 'some-unknown-tool',
	'tool_version': 'some-version',
	'tool_configuration': {"debian-package": "some-package"},
	}

	actual_tools = list(self.storage.indexer_configuration_add([tool]))
	self.assertEqual(len(actual_tools), 1)

	new_tools = [tool, {
	'tool_name': 'yet-another-tool',
	'tool_version': 'version',
	'tool_configuration': {},
	}]

	actual_tools = list(self.storage.indexer_configuration_add(new_tools))
	self.assertEqual(len(actual_tools), 2)

	# order not guaranteed, so we iterate over results to check
	for tool in actual_tools:
	_id = tool.pop('id')
	self.assertIsNotNone(_id)
	self.assertIn(tool, new_tools)

	def test_indexer_configuration_get_missing(self):
	tool = {
	'tool_name': 'unknown-tool',
	'tool_version': '3.1.0rc2-31-ga2cbb8c',
	'tool_configuration': {"command_line": "nomossa <filepath>"},
	}

	actual_tool = self.storage.indexer_configuration_get(tool)

	self.assertIsNone(actual_tool)

	def test_indexer_configuration_get(self):
	tool = {
	'tool_name': 'nomos',
	'tool_version': '3.1.0rc2-31-ga2cbb8c',
	'tool_configuration': {"command_line": "nomossa <filepath>"},
	}

	actual_tool = self.storage.indexer_configuration_get(tool)

	expected_tool = tool.copy()
	expected_tool['id'] = 1

	self.assertEqual(expected_tool, actual_tool)

	def test_indexer_configuration_metadata_get_missing_context(self):
	tool = {
	'tool_name': 'swh-metadata-translator',
	'tool_version': '0.0.1',
	'tool_configuration': {"context": "unknown-context"},
	}

	actual_tool = self.storage.indexer_configuration_get(tool)

	self.assertIsNone(actual_tool)

	def test_indexer_configuration_metadata_get(self):
	tool = {
	'tool_name': 'swh-metadata-translator',
	'tool_version': '0.0.1',
	'tool_configuration': {"type": "local", "context": "NpmMapping"},
	}

	actual_tool = self.storage.indexer_configuration_get(tool)

	expected_tool = tool.copy()
	expected_tool['id'] = actual_tool['id']

	self.assertEqual(expected_tool, actual_tool)


	@pytest.mark.property_based
	class PropBasedTestStorage(BaseTestStorage, unittest.TestCase):
	"""Properties-based tests

	"""
	def test_generate_content_mimetype_get_range_limit_none(self):
	"""mimetype_get_range call with wrong limit input should fail"""
	with self.assertRaises(ValueError) as e:
	self.storage.content_mimetype_get_range(
	start=None, end=None, indexer_configuration_id=None,
	limit=None)

	self.assertEqual(e.exception.args, (
	'Development error: limit should not be None',))

	@given(gen_content_mimetypes(min_size=1, max_size=4))
	def test_generate_content_mimetype_get_range_no_limit(self, mimetypes):
	"""mimetype_get_range returns mimetypes within range provided"""
	self.reset_storage_tables()
	# add mimetypes to storage
	self.storage.content_mimetype_add(mimetypes)

	# All ids from the db
	content_ids = sorted([c['id'] for c in mimetypes])

	start = content_ids[0]
	end = content_ids[-1]

	# retrieve mimetypes
	tool_id = mimetypes[0]['indexer_configuration_id']
	actual_result = self.storage.content_mimetype_get_range(
	start, end, indexer_configuration_id=tool_id)

	actual_ids = actual_result['ids']
	actual_next = actual_result['next']

	self.assertEqual(len(mimetypes), len(actual_ids))
	self.assertIsNone(actual_next)
	self.assertEqual(content_ids, actual_ids)

	@given(gen_content_mimetypes(min_size=4, max_size=4))
	def test_generate_content_mimetype_get_range_limit(self, mimetypes):
	"""mimetype_get_range paginates results if limit exceeded"""
	self.reset_storage_tables()

	# add mimetypes to storage
	self.storage.content_mimetype_add(mimetypes)

	# input the list of sha1s we want from storage
	content_ids = sorted([c['id'] for c in mimetypes])
	start = content_ids[0]
	end = content_ids[-1]

	# retrieve mimetypes limited to 3 results
	limited_results = len(mimetypes) - 1
	tool_id = mimetypes[0]['indexer_configuration_id']
	actual_result = self.storage.content_mimetype_get_range(
	start, end,
	indexer_configuration_id=tool_id, limit=limited_results)

	actual_ids = actual_result['ids']
	actual_next = actual_result['next']

	self.assertEqual(limited_results, len(actual_ids))
	self.assertIsNotNone(actual_next)
	self.assertEqual(actual_next, content_ids[-1])

	expected_mimetypes = content_ids[:-1]
	self.assertEqual(expected_mimetypes, actual_ids)

	# retrieve next part
	actual_results2 = self.storage.content_mimetype_get_range(
	start=end, end=end, indexer_configuration_id=tool_id)
	actual_ids2 = actual_results2['ids']
	actual_next2 = actual_results2['next']

	self.assertIsNone(actual_next2)
	expected_mimetypes2 = [content_ids[-1]]
	self.assertEqual(expected_mimetypes2, actual_ids2)

	+ def test_generate_content_fossology_license_get_range_limit_none(self):
	+ """license_get_range call with wrong limit input should fail"""
	+ with self.assertRaises(ValueError) as e:
	+ self.storage.content_fossology_license_get_range(
	+ start=None, end=None, indexer_configuration_id=None,
	+ limit=None)
	+
	+ self.assertEqual(e.exception.args, (
	+ 'Development error: limit should not be None',))
	+
	+ @given(gen_content_fossology_licenses(min_size=1, max_size=4))
	+ def test_generate_content_fossology_license_get_range_no_limit(
	+ self, fossology_licenses):
	+ """license_get_range returns licenses within range provided"""
	+ self.reset_storage_tables()
	+ # add fossology_licenses to storage
	+ self.storage.content_fossology_license_add(fossology_licenses)
	+
	+ # All ids from the db
	+ content_ids = sorted([c['id'] for c in fossology_licenses])
	+
	+ start = content_ids[0]
	+ end = content_ids[-1]
	+
	+ # retrieve fossology_licenses
	+ tool_id = fossology_licenses[0]['indexer_configuration_id']
	+ actual_result = self.storage.content_fossology_license_get_range(
	+ start, end, indexer_configuration_id=tool_id)
	+
	+ actual_ids = actual_result['ids']
	+ actual_next = actual_result['next']
	+
	+ self.assertEqual(len(fossology_licenses), len(actual_ids))
	+ self.assertIsNone(actual_next)
	+ self.assertEqual(content_ids, actual_ids)
	+
	+ @given(gen_content_fossology_licenses(min_size=4, max_size=4))
	+ def test_generate_fossology_license_get_range_limit(
	+ self, fossology_licenses):
	+ """fossology_license_get_range paginates results if limit exceeded"""
	+ self.reset_storage_tables()
	+
	+ # add fossology_licenses to storage
	+ self.storage.content_fossology_license_add(fossology_licenses)
	+
	+ # input the list of sha1s we want from storage
	+ content_ids = sorted([c['id'] for c in fossology_licenses])
	+ start = content_ids[0]
	+ end = content_ids[-1]
	+
	+ # retrieve fossology_licenses limited to 3 results
	+ limited_results = len(fossology_licenses) - 1
	+ tool_id = fossology_licenses[0]['indexer_configuration_id']
	+ actual_result = self.storage.content_fossology_license_get_range(
	+ start, end,
	+ indexer_configuration_id=tool_id, limit=limited_results)
	+
	+ actual_ids = actual_result['ids']
	+ actual_next = actual_result['next']
	+
	+ self.assertEqual(limited_results, len(actual_ids))
	+ self.assertIsNotNone(actual_next)
	+ self.assertEqual(actual_next, content_ids[-1])
	+
	+ expected_fossology_licenses = content_ids[:-1]
	+ self.assertEqual(expected_fossology_licenses, actual_ids)
	+
	+ # retrieve next part
	+ actual_results2 = self.storage.content_fossology_license_get_range(
	+ start=end, end=end, indexer_configuration_id=tool_id)
	+ actual_ids2 = actual_results2['ids']
	+ actual_next2 = actual_results2['next']
	+
	+ self.assertIsNone(actual_next2)
	+ expected_fossology_licenses2 = [content_ids[-1]]
	+ self.assertEqual(expected_fossology_licenses2, actual_ids2)
	+

	class IndexerTestStorage(CommonTestStorage, unittest.TestCase):
	"""Running the tests locally.

	For the client api tests (remote storage), see
	`class`:swh.indexer.storage.test_api_client:TestRemoteStorage
	class.

	"""
	pass

File Metadata

Mime Type: text/x-diff
Expires: Jul 4 2025, 7:55 AM (10 w, 3 d ago)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 3251531

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions