Differential D654 Diff 2057 swh/indexer/storage/db.py

Changeset View

Standalone View

swh/indexer/storage/db.py

Show First 20 Lines • Show All 107 Lines • ▼ Show 20 Lines	def _get_from_list(self, table, ids, cols, cur=None, id_col='id'):
keys=', '.join(keys),		keys=', '.join(keys),
id_col=id_col,		id_col=id_col,
table=table)		table=table)
yield from execute_values_to_bytes(		yield from execute_values_to_bytes(
cur, query,		cur, query,
((_id,) for _id in ids)		((_id,) for _id in ids)
)		)

		content_tables = {
		'mimetype': 'content_mimetype',
		ardumontAuthorUnsubmitted Done Inline Actions This is not used yet. It shows how i plan to use it when adding new contents range endpoints. ardumont: This is not used yet. It shows how i plan to use it when adding new contents range endpoints.
		'language': 'content_language',
		}

		def content_get_range(self, content_type, start, end,
		indexer_configuration_id, limit=1000, cur=None):
		"""Retrieve contents with content_type, within range [start, end]
		bound by limit and associated to the given indexer
		configuration id.

		"""
		cur = self._cursor(cur)
		table = self.content_tables.get(content_type)
		if not table:
		raise ValueError(
		'Development error: Wrong type. Should be one of [%s]' % (
		','.join(self.content_tables)))
		query = """select %s
		from %s t
		inner join indexer_configuration ic
		on t.indexer_configuration_id=ic.id
		where tc.id=%%s and
		%%s <= t.sha1 and t.sha1 <= %%s
		order by t.indexer_configuration_id, t.sha1
		limit %%s""" % (
		', '.join(self.content_get_metadata_keys), # keys
		table
		)
		cur.execute(query, (start, end, indexer_configuration_id, limit))
		yield from cursor_to_bytes(cur)

def content_mimetype_get_from_list(self, ids, cur=None):		def content_mimetype_get_from_list(self, ids, cur=None):
yield from self._get_from_list(		yield from self._get_from_list(
'content_mimetype', ids, self.content_mimetype_cols, cur=cur)		'content_mimetype', ids, self.content_mimetype_cols, cur=cur)

content_language_hash_keys = ['id', 'indexer_configuration_id']		content_language_hash_keys = ['id', 'indexer_configuration_id']

def content_language_missing_from_list(self, languages, cur=None):		def content_language_missing_from_list(self, languages, cur=None):
"""List missing languages.		"""List missing languages.
▲ Show 20 Lines • Show All 211 Lines • Show Last 20 Lines