Differential D6249 Diff 22666 swh/storage/in_memory.py

Changeset View

Standalone View

swh/storage/in_memory.py

# Copyright (C) 2015-2020 The Software Heritage developers		# Copyright (C) 2015-2021 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution		# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version		# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information		# See top-level LICENSE file for more information

from collections import defaultdict		from collections import defaultdict
import datetime		import datetime
import functools		import functools
import itertools		import itertools
Show All 16 Lines
from swh.model.model import Content, Sha1Git, SkippedContent		from swh.model.model import Content, Sha1Git, SkippedContent
from swh.storage.cassandra import CassandraStorage		from swh.storage.cassandra import CassandraStorage
from swh.storage.cassandra.model import (		from swh.storage.cassandra.model import (
BaseRow,		BaseRow,
ContentRow,		ContentRow,
DirectoryEntryRow,		DirectoryEntryRow,
DirectoryRow,		DirectoryRow,
ExtIDByTargetRow,		ExtIDByTargetRow,
		ExtIDByTargetTypeVersionRow,
ExtIDRow,		ExtIDRow,
MetadataAuthorityRow,		MetadataAuthorityRow,
MetadataFetcherRow,		MetadataFetcherRow,
ObjectCountRow,		ObjectCountRow,
OriginRow,		OriginRow,
OriginVisitRow,		OriginVisitRow,
OriginVisitStatusRow,		OriginVisitStatusRow,
RawExtrinsicMetadataByIdRow,		RawExtrinsicMetadataByIdRow,
▲ Show 20 Lines • Show All 645 Lines • ▼ Show 20 Lines	class InMemoryCqlRunner:

def extid_add_prepare(self, extid: ExtIDRow):		def extid_add_prepare(self, extid: ExtIDRow):
finalizer = functools.partial(self._extid_add_finalize, extid)		finalizer = functools.partial(self._extid_add_finalize, extid)
return (self._extid.token(self._extid.partition_key(extid)), finalizer)		return (self._extid.token(self._extid.partition_key(extid)), finalizer)

def extid_index_add_one(self, row: ExtIDByTargetRow) -> None:		def extid_index_add_one(self, row: ExtIDByTargetRow) -> None:
pass		pass

		def extid_index_2_add_one(self, row: ExtIDByTargetTypeVersionRow,) -> None:
		pass

def extid_get_from_pk(		def extid_get_from_pk(
self, extid_type: str, extid: bytes, extid_version: int, target: ExtendedSWHID,		self, extid_type: str, extid: bytes, extid_version: int, target: ExtendedSWHID,
) -> Optional[ExtIDRow]:		) -> Optional[ExtIDRow]:
primary_key = self._extid.primary_key_from_dict(		primary_key = self._extid.primary_key_from_dict(
dict(		dict(
extid_type=extid_type,		extid_type=extid_type,
extid=extid,		extid=extid,
extid_version=extid_version,		extid_version=extid_version,
target_type=target.object_type.value,		target_type=target.object_type.value,
target=target.object_id,		target=target.object_id,
)		)
)		)
return self._extid.get_from_primary_key(primary_key)		return self._extid.get_from_primary_key(primary_key)

def extid_get_from_extid(self, extid_type: str, extid: bytes) -> Iterable[ExtIDRow]:		def extid_get_from_extid(
		self, extid_type: str, extid: bytes,
		) -> Iterable[ExtIDRow]:
return (		return (
row		row
for pk, row in self._extid.iter_all()		for pk, row in self._extid.iter_all()
if row.extid_type == extid_type and row.extid == extid		if row.extid_type == extid_type and row.extid == extid
)		)

		def extid_get_from_extid_and_version(
		self, extid_type: str, extid: bytes, extid_version: int,
		) -> Iterable[ExtIDRow]:
		return (
		row
		for pk, row in self._extid.iter_all()
		if row.extid_type == extid_type
		and row.extid == extid
		and (extid_version is None or row.extid_version == extid_version)
		)

def extid_get_from_target(		def extid_get_from_target(
self, target_type: str, target: bytes		self,
		target_type: str,
		target: bytes,
		extid_type: Optional[str] = None,
		extid_version: Optional[int] = None,
) -> Iterable[ExtIDRow]:		) -> Iterable[ExtIDRow]:
		if extid_version is not None and not isinstance(extid_version, int):
		raise ValueError("ExtID version should be an integer")
		if (extid_version is not None and extid_type is None) or (
		extid_version is None and extid_type is not None
		):
		raise ValueError("You must provide both extid_type and extid_version")

return (		return (
row		row
for pk, row in self._extid.iter_all()		for pk, row in self._extid.iter_all()
if row.target_type == target_type and row.target == target		if row.target_type == target_type
		and row.target == target
		and (
		(extid_version is None and extid_type is None)
		or (
		extid_version is not None
		and row.extid_version == extid_version
		and extid_type is not None
		and row.extid_type == extid_type
		)
		)
)		)
		vlorentzUnsubmitted Done Inline Actions what about splitting this into two functions (one for each case), to make it more readable? vlorentz: what about splitting this into two functions (one for each case), to make it more readable?
		ardumontAuthorUnsubmitted Done Inline Actions sure ardumont: sure


class InMemoryStorage(CassandraStorage):		class InMemoryStorage(CassandraStorage):
_cql_runner: InMemoryCqlRunner # type: ignore		_cql_runner: InMemoryCqlRunner # type: ignore

def __init__(self, journal_writer=None):		def __init__(self, journal_writer=None):
self.reset()		self.reset()
self.journal_writer = JournalWriter(journal_writer)		self.journal_writer = JournalWriter(journal_writer)
Show All 9 Lines