Changeset View
Changeset View
Standalone View
Standalone View
swh/indexer/storage/db.py
# Copyright (C) 2015-2022 The Software Heritage developers | # Copyright (C) 2015-2022 The Software Heritage developers | ||||
# See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
# License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
# See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
from typing import Dict, Iterable, Iterator, List | from typing import Dict, Iterable, Iterator, List | ||||
from swh.core.db import BaseDb | from swh.core.db import BaseDb | ||||
from swh.core.db.db_utils import execute_values_generator, stored_procedure | from swh.core.db.db_utils import execute_values_generator, stored_procedure | ||||
from swh.model import hashutil | |||||
from .interface import Sha1 | from .interface import Sha1 | ||||
class Db(BaseDb): | class Db(BaseDb): | ||||
"""Proxy to the SWH Indexer DB, with wrappers around stored procedures""" | """Proxy to the SWH Indexer DB, with wrappers around stored procedures""" | ||||
content_mimetype_hash_keys = ["id", "indexer_configuration_id"] | content_mimetype_hash_keys = ["id", "indexer_configuration_id"] | ||||
▲ Show 20 Lines • Show All 189 Lines • ▼ Show 20 Lines | def content_language_add_from_temp(self, cur=None): | ||||
cur.execute("select * from swh_content_language_add()") | cur.execute("select * from swh_content_language_add()") | ||||
return cur.fetchone()[0] | return cur.fetchone()[0] | ||||
def content_language_get_from_list(self, ids, cur=None): | def content_language_get_from_list(self, ids, cur=None): | ||||
yield from self._get_from_list( | yield from self._get_from_list( | ||||
"content_language", ids, self.content_language_cols, cur=cur | "content_language", ids, self.content_language_cols, cur=cur | ||||
) | ) | ||||
content_ctags_hash_keys = ["id", "indexer_configuration_id"] | |||||
def content_ctags_missing_from_list(self, ctags, cur=None): | |||||
"""List missing ctags.""" | |||||
yield from self._missing_from_list( | |||||
"content_ctags", ctags, self.content_ctags_hash_keys, cur=cur | |||||
) | |||||
content_ctags_cols = [ | |||||
"id", | |||||
"name", | |||||
"kind", | |||||
"line", | |||||
"lang", | |||||
"tool_id", | |||||
"tool_name", | |||||
"tool_version", | |||||
"tool_configuration", | |||||
] | |||||
@stored_procedure("swh_mktemp_content_ctags") | |||||
def mktemp_content_ctags(self, cur=None): | |||||
pass | |||||
def content_ctags_add_from_temp(self, cur=None): | |||||
cur = self._cursor(cur) | |||||
cur.execute("select * from swh_content_ctags_add()") | |||||
return cur.fetchone()[0] | |||||
def content_ctags_get_from_list(self, ids, cur=None): | |||||
cur = self._cursor(cur) | |||||
keys = map(self._convert_key, self.content_ctags_cols) | |||||
yield from execute_values_generator( | |||||
cur, | |||||
""" | |||||
select %s | |||||
from (values %%s) as t(id) | |||||
inner join content_ctags c | |||||
on c.id=t.id | |||||
inner join indexer_configuration i | |||||
on c.indexer_configuration_id=i.id | |||||
order by line | |||||
""" | |||||
% ", ".join(keys), | |||||
((_id,) for _id in ids), | |||||
) | |||||
def content_ctags_search(self, expression, last_sha1, limit, cur=None): | |||||
cur = self._cursor(cur) | |||||
if not last_sha1: | |||||
query = """SELECT %s | |||||
FROM swh_content_ctags_search(%%s, %%s)""" % ( | |||||
",".join(self.content_ctags_cols) | |||||
) | |||||
cur.execute(query, (expression, limit)) | |||||
else: | |||||
if last_sha1 and isinstance(last_sha1, bytes): | |||||
last_sha1 = "\\x%s" % hashutil.hash_to_hex(last_sha1) | |||||
elif last_sha1: | |||||
last_sha1 = "\\x%s" % last_sha1 | |||||
query = """SELECT %s | |||||
FROM swh_content_ctags_search(%%s, %%s, %%s)""" % ( | |||||
",".join(self.content_ctags_cols) | |||||
) | |||||
cur.execute(query, (expression, limit, last_sha1)) | |||||
yield from cur | |||||
content_fossology_license_cols = [ | content_fossology_license_cols = [ | ||||
"id", | "id", | ||||
"tool_id", | "tool_id", | ||||
"tool_name", | "tool_name", | ||||
"tool_version", | "tool_version", | ||||
"tool_configuration", | "tool_configuration", | ||||
"license", | "license", | ||||
] | ] | ||||
▲ Show 20 Lines • Show All 271 Lines • Show Last 20 Lines |