Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/cassandra/cql.py
| # Copyright (C) 2019-2020 The Software Heritage developers | # Copyright (C) 2019-2020 The Software Heritage developers | ||||
| # See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
| # License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
| # See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
| import datetime | |||||
| import functools | import functools | ||||
| import json | import json | ||||
| import logging | import logging | ||||
| import random | import random | ||||
| from typing import ( | from typing import ( | ||||
| Any, | Any, | ||||
| Callable, | Callable, | ||||
| Dict, | Dict, | ||||
| ▲ Show 20 Lines • Show All 691 Lines • ▼ Show 20 Lines | ) -> None: | ||||
| assert self._origin_visit_status_keys[-1] == "metadata" | assert self._origin_visit_status_keys[-1] == "metadata" | ||||
| keys = self._origin_visit_status_keys | keys = self._origin_visit_status_keys | ||||
| metadata = json.dumps(visit_update.metadata) | metadata = json.dumps(visit_update.metadata) | ||||
| self._execute_with_retries( | self._execute_with_retries( | ||||
| statement, [getattr(visit_update, key) for key in keys[:-1]] + [metadata] | statement, [getattr(visit_update, key) for key in keys[:-1]] + [metadata] | ||||
| ) | ) | ||||
| def _format_origin_visit_status_row( | def origin_visit_status_get_latest(self, origin: str, visit: int,) -> Optional[Row]: | ||||
| self, visit_status: ResultSet | """Given an origin visit id, return its latest origin_visit_status | ||||
| ) -> Dict[str, Any]: | |||||
| """Format a row visit_status into an origin_visit_status dict | |||||
| """ | """ | ||||
| return { | rows = self.origin_visit_status_get(origin, visit) | ||||
| **visit_status._asdict(), | return rows[0] if rows else None | ||||
| "origin": visit_status.origin, | |||||
| "date": visit_status.date.replace(tzinfo=datetime.timezone.utc), | |||||
| "metadata": ( | |||||
| json.loads(visit_status.metadata) if visit_status.metadata else None | |||||
| ), | |||||
| } | |||||
| @_prepared_statement( | @_prepared_statement( | ||||
| "SELECT * FROM origin_visit_status " | "SELECT * FROM origin_visit_status " | ||||
| "WHERE origin = ? AND visit = ? " | "WHERE origin = ? AND visit = ? " | ||||
| "ORDER BY date DESC " | "ORDER BY date DESC" | ||||
| "LIMIT 1" | |||||
| ) | ) | ||||
| def origin_visit_status_get_latest( | def origin_visit_status_get( | ||||
| self, origin: str, visit: int, *, statement | self, | ||||
| ) -> Optional[Dict[str, Any]]: | origin: str, | ||||
| """Given an origin visit id, return its latest origin_visit_status | visit: int, | ||||
| allowed_statuses: Optional[List[str]] = None, | |||||
| require_snapshot: bool = False, | |||||
| *, | |||||
| statement, | |||||
| ) -> List[Row]: | |||||
| """Return all origin visit status for a given visit | |||||
vlorentz: statuses | |||||
| """ | """ | ||||
| rows = list(self._execute_with_retries(statement, [origin, visit])) | return list(self._execute_with_retries(statement, [origin, visit])) | ||||
Done Inline Actionsmake this a prepared statement vlorentz: make this a prepared statement | |||||
Done Inline Actionsah yeah, indeed, we can now. I iterated a bit prior to find something that works. ardumont: ah yeah, indeed, we can now. I iterated a bit prior to find something that works. | |||||
| if rows: | |||||
| return self._format_origin_visit_status_row(rows[0]) | |||||
| else: | |||||
| return None | |||||
| @_prepared_statement("SELECT * FROM origin_visit WHERE origin = ? AND visit = ?") | @_prepared_statement("SELECT * FROM origin_visit WHERE origin = ? AND visit = ?") | ||||
| def origin_visit_get_one( | def origin_visit_get_one( | ||||
| self, origin_url: str, visit_id: int, *, statement | self, origin_url: str, visit_id: int, *, statement | ||||
| ) -> Optional[Row]: | ) -> Optional[Row]: | ||||
| # TODO: error handling | # TODO: error handling | ||||
| rows = list(self._execute_with_retries(statement, [origin_url, visit_id])) | rows = list(self._execute_with_retries(statement, [origin_url, visit_id])) | ||||
| if rows: | if rows: | ||||
| return rows[0] | return rows[0] | ||||
Done Inline Actionsmove this to cassandra/storage.py; cassandra/cql.py should only be a thin abstraction over the queries. vlorentz: move this to cassandra/storage.py; cassandra/cql.py should only be a thin abstraction over the… | |||||
Done Inline ActionsI keep it here so the implementation and name are in adequation. ardumont: I keep it here so the implementation and name are in adequation.
Doing what you ask, I need to… | |||||
| else: | else: | ||||
| return None | return None | ||||
| @_prepared_statement("SELECT * FROM origin_visit WHERE origin = ?") | @_prepared_statement("SELECT * FROM origin_visit WHERE origin = ?") | ||||
| def origin_visit_get_all(self, origin_url: str, *, statement) -> ResultSet: | def origin_visit_get_all(self, origin_url: str, *, statement) -> ResultSet: | ||||
| return self._execute_with_retries(statement, [origin_url]) | return self._execute_with_retries(statement, [origin_url]) | ||||
| @_prepared_statement("SELECT * FROM origin_visit WHERE token(origin) >= ?") | @_prepared_statement("SELECT * FROM origin_visit WHERE token(origin) >= ?") | ||||
| ▲ Show 20 Lines • Show All 150 Lines • Show Last 20 Lines | |||||
statuses