Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/cassandra/cql.py
# Copyright (C) 2019-2020 The Software Heritage developers | # Copyright (C) 2019-2020 The Software Heritage developers | ||||
# See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
# License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
# See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
import datetime | |||||
import functools | import functools | ||||
import json | import json | ||||
import logging | import logging | ||||
import random | import random | ||||
from typing import ( | from typing import ( | ||||
Any, | Any, | ||||
Callable, | Callable, | ||||
Dict, | Dict, | ||||
▲ Show 20 Lines • Show All 691 Lines • ▼ Show 20 Lines | ) -> None: | ||||
assert self._origin_visit_status_keys[-1] == "metadata" | assert self._origin_visit_status_keys[-1] == "metadata" | ||||
keys = self._origin_visit_status_keys | keys = self._origin_visit_status_keys | ||||
metadata = json.dumps(visit_update.metadata) | metadata = json.dumps(visit_update.metadata) | ||||
self._execute_with_retries( | self._execute_with_retries( | ||||
statement, [getattr(visit_update, key) for key in keys[:-1]] + [metadata] | statement, [getattr(visit_update, key) for key in keys[:-1]] + [metadata] | ||||
) | ) | ||||
def _format_origin_visit_status_row( | def origin_visit_status_get_latest(self, origin: str, visit: int,) -> Optional[Row]: | ||||
self, visit_status: ResultSet | """Given an origin visit id, return its latest origin_visit_status | ||||
) -> Dict[str, Any]: | |||||
"""Format a row visit_status into an origin_visit_status dict | |||||
""" | """ | ||||
return { | rows = self.origin_visit_status_get(origin, visit) | ||||
**visit_status._asdict(), | return rows[0] if rows else None | ||||
"origin": visit_status.origin, | |||||
"date": visit_status.date.replace(tzinfo=datetime.timezone.utc), | |||||
"metadata": ( | |||||
json.loads(visit_status.metadata) if visit_status.metadata else None | |||||
), | |||||
} | |||||
@_prepared_statement( | @_prepared_statement( | ||||
"SELECT * FROM origin_visit_status " | "SELECT * FROM origin_visit_status " | ||||
"WHERE origin = ? AND visit = ? " | "WHERE origin = ? AND visit = ? " | ||||
"ORDER BY date DESC " | "ORDER BY date DESC" | ||||
"LIMIT 1" | |||||
) | ) | ||||
def origin_visit_status_get_latest( | def origin_visit_status_get( | ||||
self, origin: str, visit: int, *, statement | self, | ||||
) -> Optional[Dict[str, Any]]: | origin: str, | ||||
"""Given an origin visit id, return its latest origin_visit_status | visit: int, | ||||
allowed_statuses: Optional[List[str]] = None, | |||||
require_snapshot: bool = False, | |||||
*, | |||||
statement, | |||||
) -> List[Row]: | |||||
vlorentz: statuses | |||||
"""Return all origin visit statuses for a given visit | |||||
""" | """ | ||||
rows = list(self._execute_with_retries(statement, [origin, visit])) | return list(self._execute_with_retries(statement, [origin, visit])) | ||||
Done Inline Actionsmake this a prepared statement vlorentz: make this a prepared statement | |||||
Done Inline Actionsah yeah, indeed, we can now. I iterated a bit prior to find something that works. ardumont: ah yeah, indeed, we can now. I iterated a bit prior to find something that works. | |||||
if rows: | |||||
return self._format_origin_visit_status_row(rows[0]) | |||||
else: | |||||
return None | |||||
@_prepared_statement("SELECT * FROM origin_visit WHERE origin = ? AND visit = ?") | @_prepared_statement("SELECT * FROM origin_visit WHERE origin = ? AND visit = ?") | ||||
def origin_visit_get_one( | def origin_visit_get_one( | ||||
self, origin_url: str, visit_id: int, *, statement | self, origin_url: str, visit_id: int, *, statement | ||||
) -> Optional[Row]: | ) -> Optional[Row]: | ||||
# TODO: error handling | # TODO: error handling | ||||
rows = list(self._execute_with_retries(statement, [origin_url, visit_id])) | rows = list(self._execute_with_retries(statement, [origin_url, visit_id])) | ||||
if rows: | if rows: | ||||
return rows[0] | return rows[0] | ||||
Done Inline Actionsmove this to cassandra/storage.py; cassandra/cql.py should only be a thin abstraction over the queries. vlorentz: move this to cassandra/storage.py; cassandra/cql.py should only be a thin abstraction over the… | |||||
Done Inline ActionsI keep it here so the implementation and name are in adequation. ardumont: I keep it here so the implementation and name are in adequation.
Doing what you ask, I need to… | |||||
else: | else: | ||||
return None | return None | ||||
@_prepared_statement("SELECT * FROM origin_visit WHERE origin = ?") | @_prepared_statement("SELECT * FROM origin_visit WHERE origin = ?") | ||||
def origin_visit_get_all(self, origin_url: str, *, statement) -> ResultSet: | def origin_visit_get_all(self, origin_url: str, *, statement) -> ResultSet: | ||||
return self._execute_with_retries(statement, [origin_url]) | return self._execute_with_retries(statement, [origin_url]) | ||||
@_prepared_statement("SELECT * FROM origin_visit WHERE token(origin) >= ?") | @_prepared_statement("SELECT * FROM origin_visit WHERE token(origin) >= ?") | ||||
▲ Show 20 Lines • Show All 150 Lines • Show Last 20 Lines |
statuses