diff --git a/swh/storage/in_memory.py b/swh/storage/in_memory.py --- a/swh/storage/in_memory.py +++ b/swh/storage/in_memory.py @@ -3,17 +3,13 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -import bisect -import collections import datetime import functools -import itertools import random from collections import defaultdict from typing import ( Any, - Callable, Dict, Generic, Iterable, @@ -58,65 +54,6 @@ from .converters import origin_url_to_sha1 from .writer import JournalWriter -# Max block size of contents to return -BULK_BLOCK_CONTENT_LEN_MAX = 10000 - - -SortedListItem = TypeVar("SortedListItem") -SortedListKey = TypeVar("SortedListKey") - -FetcherKey = Tuple[str, str] - - -class SortedList(collections.UserList, Generic[SortedListKey, SortedListItem]): - data: List[Tuple[SortedListKey, SortedListItem]] - - # https://github.com/python/mypy/issues/708 - # key: Callable[[SortedListItem], SortedListKey] - - def __init__( - self, - data: List[SortedListItem] = None, - key: Optional[Callable[[SortedListItem], SortedListKey]] = None, - ): - if key is None: - - def key(item): - return item - - assert key is not None # for mypy - super().__init__(sorted((key(x), x) for x in data or [])) - - self.key: Callable[[SortedListItem], SortedListKey] = key - - def add(self, item: SortedListItem): - k = self.key(item) - bisect.insort(self.data, (k, item)) - - def __iter__(self) -> Iterator[SortedListItem]: - for (k, item) in self.data: - yield item - - def iter_from(self, start_key: Any) -> Iterator[SortedListItem]: - """Returns an iterator over all the elements whose key is greater - or equal to `start_key`. - (This is an efficient equivalent to: - `(x for x in L if key(x) >= start_key)`) - """ - from_index = bisect.bisect_left(self.data, (start_key,)) - for (k, item) in itertools.islice(self.data, from_index, None): - yield item - - def iter_after(self, start_key: Any) -> Iterator[SortedListItem]: - """Same as iter_from, but using a strict inequality.""" - it = self.iter_from(start_key) - for item in it: - if self.key(item) > start_key: # type: ignore - yield item - break - - yield from it - TRow = TypeVar("TRow", bound=BaseRow) @@ -687,30 +624,7 @@ def reset(self): self._cql_runner = InMemoryCqlRunner() - self._persons = {} - - self._objects = defaultdict(list) - self._sorted_sha1s = SortedList[bytes, bytes]() - self.objstorage = ObjStorage({"cls": "memory", "args": {}}) def check_config(self, *, check_write: bool) -> bool: return True - - def diff_directories(self, from_dir, to_dir, track_renaming=False): - raise NotImplementedError("InMemoryStorage.diff_directories") - - def diff_revisions(self, from_rev, to_rev, track_renaming=False): - raise NotImplementedError("InMemoryStorage.diff_revisions") - - def diff_revision(self, revision, track_renaming=False): - raise NotImplementedError("InMemoryStorage.diff_revision") - - def clear_buffers(self, object_types: Optional[List[str]] = None) -> None: - """Do nothing - - """ - return None - - def flush(self, object_types: Optional[List[str]] = None) -> Dict: - return {} diff --git a/swh/storage/tests/test_in_memory.py b/swh/storage/tests/test_in_memory.py --- a/swh/storage/tests/test_in_memory.py +++ b/swh/storage/tests/test_in_memory.py @@ -8,7 +8,7 @@ import pytest from swh.storage.cassandra.model import BaseRow -from swh.storage.in_memory import SortedList, Table +from swh.storage.in_memory import Table from swh.storage.tests.test_storage import TestStorage as _TestStorage from swh.storage.tests.test_storage import ( TestStorageGeneratedData as _TestStorageGeneratedData, @@ -28,70 +28,6 @@ } -parametrize = pytest.mark.parametrize( - "items", - [ - [1, 2, 3, 4, 5, 6, 10, 100], - [10, 100, 6, 5, 4, 3, 2, 1], - [10, 4, 5, 6, 1, 2, 3, 100], - ], -) - - -@parametrize -def test_sorted_list_iter(items): - list1 = SortedList() - for item in items: - list1.add(item) - assert list(list1) == sorted(items) - - list2 = SortedList(items) - assert list(list2) == sorted(items) - - -@parametrize -def test_sorted_list_iter__key(items): - list1 = SortedList(key=lambda item: -item) - for item in items: - list1.add(item) - assert list(list1) == list(reversed(sorted(items))) - - list2 = SortedList(items, key=lambda item: -item) - assert list(list2) == list(reversed(sorted(items))) - - -@parametrize -def test_sorted_list_iter_from(items): - list_ = SortedList(items) - for split in items: - expected = sorted(item for item in items if item >= split) - assert list(list_.iter_from(split)) == expected, f"split: {split}" - - -@parametrize -def test_sorted_list_iter_from__key(items): - list_ = SortedList(items, key=lambda item: -item) - for split in items: - expected = reversed(sorted(item for item in items if item <= split)) - assert list(list_.iter_from(-split)) == list(expected), f"split: {split}" - - -@parametrize -def test_sorted_list_iter_after(items): - list_ = SortedList(items) - for split in items: - expected = sorted(item for item in items if item > split) - assert list(list_.iter_after(split)) == expected, f"split: {split}" - - -@parametrize -def test_sorted_list_iter_after__key(items): - list_ = SortedList(items, key=lambda item: -item) - for split in items: - expected = reversed(sorted(item for item in items if item < split)) - assert list(list_.iter_after(-split)) == list(expected), f"split: {split}" - - @dataclasses.dataclass class Row(BaseRow): PARTITION_KEY = ("col1", "col2") diff --git a/swh/storage/tests/test_replay.py b/swh/storage/tests/test_replay.py --- a/swh/storage/tests/test_replay.py +++ b/swh/storage/tests/test_replay.py @@ -201,10 +201,6 @@ """Simple utility function to compare the content of 2 in_memory storages """ - expected_persons = set(src._persons.values()) - got_persons = set(dst._persons.values()) - assert got_persons == expected_persons - for attr_ in ( "contents", "skipped_contents", @@ -362,12 +358,6 @@ ) return row - expected_persons = { - maybe_anonymize("persons", person) for person in src._persons.values() - } - got_persons = set(dst._persons.values()) - assert got_persons == expected_persons - for attr_ in ( "contents", "skipped_contents",