Changeset View
Changeset View
Standalone View
Standalone View
swh/indexer/storage/db.py
| Show All 16 Lines | class Db(BaseDb): | ||||
| content_mimetype_hash_keys = ["id", "indexer_configuration_id"] | content_mimetype_hash_keys = ["id", "indexer_configuration_id"] | ||||
| def _missing_from_list( | def _missing_from_list( | ||||
| self, table: str, data: Iterable[Dict], hash_keys: List[str], cur=None | self, table: str, data: Iterable[Dict], hash_keys: List[str], cur=None | ||||
| ): | ): | ||||
| """Read from table the data with hash_keys that are missing. | """Read from table the data with hash_keys that are missing. | ||||
| Args: | Args: | ||||
| table: Table name (e.g content_mimetype, content_language, | table: Table name (e.g content_mimetype, fossology_license, etc...) | ||||
| etc...) | |||||
| data: Dict of data to read from | data: Dict of data to read from | ||||
| hash_keys: List of keys to read in the data dict. | hash_keys: List of keys to read in the data dict. | ||||
| Yields: | Yields: | ||||
| The data which is missing from the db. | The data which is missing from the db. | ||||
| """ | """ | ||||
| cur = self._cursor(cur) | cur = self._cursor(cur) | ||||
| ▲ Show 20 Lines • Show All 45 Lines • ▼ Show 20 Lines | def _convert_key(self, key, main_table="c"): | ||||
| Args: | Args: | ||||
| key (str): Key expression to change according to the alias | key (str): Key expression to change according to the alias | ||||
| used in the query | used in the query | ||||
| main_table (str): Alias to use for the main table. Default | main_table (str): Alias to use for the main table. Default | ||||
| to c for content_{something}. | to c for content_{something}. | ||||
| Expected: | Expected: | ||||
| Tables content_{something} being aliased as 'c' (something | Tables content_{something} being aliased as 'c' (something | ||||
| in {language, mimetype, ...}), table indexer_configuration | in {mimetype, ...}), table indexer_configuration being aliased as 'i'. | ||||
| being aliased as 'i'. | |||||
| """ | """ | ||||
| if key == "id": | if key == "id": | ||||
| return "%s.id" % main_table | return "%s.id" % main_table | ||||
| elif key == "tool_id": | elif key == "tool_id": | ||||
| return "i.id as tool_id" | return "i.id as tool_id" | ||||
| elif key == "license": | elif key == "license": | ||||
| return ( | return ( | ||||
| ▲ Show 20 Lines • Show All 78 Lines • ▼ Show 20 Lines | ): | ||||
| ) | ) | ||||
| yield from cur | yield from cur | ||||
| def content_mimetype_get_from_list(self, ids, cur=None): | def content_mimetype_get_from_list(self, ids, cur=None): | ||||
| yield from self._get_from_list( | yield from self._get_from_list( | ||||
| "content_mimetype", ids, self.content_mimetype_cols, cur=cur | "content_mimetype", ids, self.content_mimetype_cols, cur=cur | ||||
| ) | ) | ||||
| content_language_hash_keys = ["id", "indexer_configuration_id"] | |||||
| def content_language_missing_from_list(self, languages, cur=None): | |||||
| """List missing languages.""" | |||||
| yield from self._missing_from_list( | |||||
| "content_language", languages, self.content_language_hash_keys, cur=cur | |||||
| ) | |||||
| content_language_cols = [ | |||||
| "id", | |||||
| "lang", | |||||
| "tool_id", | |||||
| "tool_name", | |||||
| "tool_version", | |||||
| "tool_configuration", | |||||
| ] | |||||
| @stored_procedure("swh_mktemp_content_language") | |||||
| def mktemp_content_language(self, cur=None): | |||||
| pass | |||||
| def content_language_add_from_temp(self, cur=None): | |||||
| cur = self._cursor(cur) | |||||
| cur.execute("select * from swh_content_language_add()") | |||||
| return cur.fetchone()[0] | |||||
| def content_language_get_from_list(self, ids, cur=None): | |||||
| yield from self._get_from_list( | |||||
| "content_language", ids, self.content_language_cols, cur=cur | |||||
| ) | |||||
| content_fossology_license_cols = [ | content_fossology_license_cols = [ | ||||
| "id", | "id", | ||||
| "tool_id", | "tool_id", | ||||
| "tool_name", | "tool_name", | ||||
| "tool_version", | "tool_version", | ||||
| "tool_configuration", | "tool_configuration", | ||||
| "license", | "license", | ||||
| ] | ] | ||||
| ▲ Show 20 Lines • Show All 271 Lines • Show Last 20 Lines | |||||