Differential D3355 Diff 11984 swh/storage/cassandra/cql.py

Changeset View

Standalone View

swh/storage/cassandra/cql.py

# Copyright (C) 2019-2020 The Software Heritage developers		# Copyright (C) 2019-2020 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution		# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version		# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information		# See top-level LICENSE file for more information

		import datetime
import functools		import functools
import json		import json
import logging		import logging
import random		import random
from typing import (		from typing import (
Any,		Any,
Callable,		Callable,
Dict,		Dict,
▲ Show 20 Lines • Show All 846 Lines • ▼ Show 20 Lines	def metadata_fetcher_add(self, name, version, metadata, *, statement):
return self._execute_with_retries(statement, [name, version, metadata])		return self._execute_with_retries(statement, [name, version, metadata])

@_prepared_statement(		@_prepared_statement(
"SELECT * from metadata_fetcher WHERE name = ? AND version = ?"		"SELECT * from metadata_fetcher WHERE name = ? AND version = ?"
)		)
def metadata_fetcher_get(self, name, version, *, statement) -> Optional[Row]:		def metadata_fetcher_get(self, name, version, *, statement) -> Optional[Row]:
return next(iter(self._execute_with_retries(statement, [name, version])), None)		return next(iter(self._execute_with_retries(statement, [name, version])), None)

##########################		#########################
# 'origin_metadata' table		# 'object_metadata' table
##########################		#########################

_origin_metadata_keys = [		_object_metadata_keys = [
"origin",		"type",
		"id",
"authority_type",		"authority_type",
"authority_url",		"authority_url",
"discovery_date",		"discovery_date",
"fetcher_name",		"fetcher_name",
"fetcher_version",		"fetcher_version",
"format",		"format",
"metadata",		"metadata",
]		]

@_prepared_insert_statement("origin_metadata", _origin_metadata_keys)		@_prepared_statement(
def origin_metadata_add(		f"INSERT INTO object_metadata ({', '.join(_object_metadata_keys)}) "
		f"VALUES ({', '.join('?' for _ in _object_metadata_keys)})"
		ardumontUnsubmitted Not Done Inline Actions i just realized we could do that heh ;) (build stuff within the decorator) ardumont: i just realized we could do that heh ;) (build stuff within the decorator)
		)
		def object_metadata_add(
self,		self,
origin,		object_type: str,
		id: str,
authority_type,		authority_type,
authority_url,		authority_url,
discovery_date,		discovery_date,
fetcher_name,		fetcher_name,
fetcher_version,		fetcher_version,
format,		format,
metadata,		metadata,
*,		*,
statement,		statement,
):		):
return self._execute_with_retries(		params = [
statement,		object_type,
[		id,
origin,
authority_type,		authority_type,
authority_url,		authority_url,
discovery_date,		discovery_date,
fetcher_name,		fetcher_name,
fetcher_version,		fetcher_version,
format,		format,
metadata,		metadata,
],		]
)
		return self._execute_with_retries(statement, params,)

@_prepared_statement(		@_prepared_statement(
"SELECT * from origin_metadata "		"SELECT * from object_metadata "
"WHERE origin=? AND authority_url=? AND discovery_date>? "		"WHERE id=? AND authority_url=? AND discovery_date>? AND authority_type=?"
"AND authority_type=?"
)		)
def origin_metadata_get_after_date(		def object_metadata_get_after_date(
self, origin, authority_type, authority_url, after, *, statement		self,
		id: str,
		authority_type: str,
		authority_url: str,
		after: datetime.datetime,
		*,
		statement,
):		):
return self._execute_with_retries(		return self._execute_with_retries(
statement, [origin, authority_url, after, authority_type]		statement, [id, authority_url, after, authority_type]
)		)

@_prepared_statement(		@_prepared_statement(
"SELECT * from origin_metadata "		"SELECT * from object_metadata "
"WHERE origin=? AND authority_type=? AND authority_url=? "		"WHERE id=? AND authority_type=? AND authority_url=? "
"AND (discovery_date, fetcher_name, fetcher_version) > (?, ?, ?)"		"AND (discovery_date, fetcher_name, fetcher_version) > (?, ?, ?)"
)		)
def origin_metadata_get_after_date_and_fetcher(		def object_metadata_get_after_date_and_fetcher(
self,		self,
origin,		id: str,
authority_type,		authority_type: str,
authority_url,		authority_url: str,
after_date,		after_date: datetime.datetime,
after_fetcher_name,		after_fetcher_name: str,
after_fetcher_version,		after_fetcher_version: str,
*,		*,
statement,		statement,
):		):
return self._execute_with_retries(		return self._execute_with_retries(
statement,		statement,
[		[
origin,		id,
authority_type,		authority_type,
authority_url,		authority_url,
after_date,		after_date,
after_fetcher_name,		after_fetcher_name,
after_fetcher_version,		after_fetcher_version,
],		],
)		)

@_prepared_statement(		@_prepared_statement(
"SELECT * from origin_metadata "		"SELECT * from object_metadata "
"WHERE origin=? AND authority_url=? AND authority_type=?"		"WHERE id=? AND authority_url=? AND authority_type=?"
)		)
def origin_metadata_get(		def object_metadata_get(
		ardumontUnsubmitted Not Done Inline Actions types? ardumont: types?
self, origin, authority_type, authority_url, *, statement		self, id: str, authority_type: str, authority_url: str, *, statement
) -> Iterable[Row]:		) -> Iterable[Row]:
return self._execute_with_retries(		return self._execute_with_retries(
statement, [origin, authority_url, authority_type]		statement, [id, authority_url, authority_type]
)		)

##########################		##########################
# Miscellaneous		# Miscellaneous
##########################		##########################

@_prepared_statement("SELECT uuid() FROM revision LIMIT 1;")		@_prepared_statement("SELECT uuid() FROM revision LIMIT 1;")
def check_read(self, *, statement):		def check_read(self, *, statement):
self._execute_with_retries(statement, [])		self._execute_with_retries(statement, [])

@_prepared_statement(		@_prepared_statement(
"SELECT object_type, count FROM object_count WHERE partition_key=0"		"SELECT object_type, count FROM object_count WHERE partition_key=0"
)		)
def stat_counters(self, *, statement) -> ResultSet:		def stat_counters(self, *, statement) -> ResultSet:
return self._execute_with_retries(statement, [])		return self._execute_with_retries(statement, [])