Differential D3356 Diff 11985 swh/storage/cassandra/storage.py

Changeset View

Standalone View

swh/storage/cassandra/storage.py

# Copyright (C) 2019-2020 The Software Heritage developers		# Copyright (C) 2019-2020 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution		# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version		# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information		# See top-level LICENSE file for more information

import datetime		import datetime
import itertools		import itertools
import json		import json
import random		import random
import re		import re
from typing import Any, Dict, List, Iterable, Optional		from typing import Any, Dict, List, Iterable, Optional, Union

import attr		import attr
from deprecated import deprecated		from deprecated import deprecated

from swh.core.api.serializers import msgpack_loads, msgpack_dumps		from swh.core.api.serializers import msgpack_loads, msgpack_dumps
from swh.model.model import (		from swh.model.model import (
Revision,		Revision,
Release,		Release,
Directory,		Directory,
DirectoryEntry,		DirectoryEntry,
Content,		Content,
SkippedContent,		SkippedContent,
OriginVisit,		OriginVisit,
OriginVisitStatus,		OriginVisitStatus,
Snapshot,		Snapshot,
Origin,		Origin,
)		)
from swh.model.hashutil import DEFAULT_ALGORITHMS		from swh.model.hashutil import DEFAULT_ALGORITHMS
from swh.storage.objstorage import ObjStorage		from swh.storage.objstorage import ObjStorage
from swh.storage.writer import JournalWriter		from swh.storage.writer import JournalWriter
from swh.storage.utils import now		from swh.storage.utils import now

from ..exc import StorageArgumentException, HashCollision		from ..exc import StorageArgumentException, HashCollision
		from ..extrinsic_metadata import check_extrinsic_metadata_context, CONTEXT_KEYS
from .common import TOKEN_BEGIN, TOKEN_END		from .common import TOKEN_BEGIN, TOKEN_END
from .converters import (		from .converters import (
revision_to_db,		revision_to_db,
revision_from_db,		revision_from_db,
release_to_db,		release_to_db,
release_from_db,		release_from_db,
row_to_visit_status,		row_to_visit_status,
)		)
▲ Show 20 Lines • Show All 982 Lines • ▼ Show 20 Lines	def origin_metadata_add(
fetcher: Dict[str, Any],		fetcher: Dict[str, Any],
format: str,		format: str,
metadata: bytes,		metadata: bytes,
) -> None:		) -> None:
if not isinstance(origin_url, str):		if not isinstance(origin_url, str):
raise StorageArgumentException(		raise StorageArgumentException(
"origin_url must be str, not %r" % (origin_url,)		"origin_url must be str, not %r" % (origin_url,)
)		)

		context: Dict[str, Union[str, bytes, int]] = {} # origins have no context

self._object_metadata_add(		self._object_metadata_add(
"origin", origin_url, discovery_date, authority, fetcher, format, metadata,		"origin",
		origin_url,
		discovery_date,
		authority,
		fetcher,
		format,
		metadata,
		context,
)		)

def origin_metadata_get(		def origin_metadata_get(
self,		self,
origin_url: str,		origin_url: str,
authority: Dict[str, str],		authority: Dict[str, str],
after: Optional[datetime.datetime] = None,		after: Optional[datetime.datetime] = None,
page_token: Optional[bytes] = None,		page_token: Optional[bytes] = None,
Show All 14 Lines	def _object_metadata_add(
self,		self,
object_type: str,		object_type: str,
id: str,		id: str,
discovery_date: datetime.datetime,		discovery_date: datetime.datetime,
authority: Dict[str, Any],		authority: Dict[str, Any],
fetcher: Dict[str, Any],		fetcher: Dict[str, Any],
format: str,		format: str,
metadata: bytes,		metadata: bytes,
		context: Dict[str, Union[str, bytes, int]],
) -> None:		) -> None:
		check_extrinsic_metadata_context(object_type, context)

if not self._cql_runner.metadata_authority_get(**authority):		if not self._cql_runner.metadata_authority_get(**authority):
raise StorageArgumentException(f"Unknown authority {authority}")		raise StorageArgumentException(f"Unknown authority {authority}")
if not self._cql_runner.metadata_fetcher_get(**fetcher):		if not self._cql_runner.metadata_fetcher_get(**fetcher):
raise StorageArgumentException(f"Unknown fetcher {fetcher}")		raise StorageArgumentException(f"Unknown fetcher {fetcher}")

try:		try:
self._cql_runner.object_metadata_add(		self._cql_runner.object_metadata_add(
object_type,		object_type,
id,		id,
authority["type"],		authority["type"],
authority["url"],		authority["url"],
discovery_date,		discovery_date,
fetcher["name"],		fetcher["name"],
fetcher["version"],		fetcher["version"],
format,		format,
metadata,		metadata,
		context,
)		)
except TypeError as e:		except TypeError as e:
raise StorageArgumentException(*e.args)		raise StorageArgumentException(*e.args)

def _object_metadata_get(		def _object_metadata_get(
self,		self,
object_type: str,		object_type: str,
id: str,		id: str,
▲ Show 20 Lines • Show All 44 Lines • ▼ Show 20 Lines	) -> Dict[str, Any]:
"name": entry.fetcher_name,		"name": entry.fetcher_name,
"version": entry.fetcher_version,		"version": entry.fetcher_version,
},		},
"discovery_date": discovery_date,		"discovery_date": discovery_date,
"format": entry.format,		"format": entry.format,
"metadata": entry.metadata,		"metadata": entry.metadata,
}		}

		if CONTEXT_KEYS[object_type]:
		context = {}
		for key in CONTEXT_KEYS[object_type]:
		value = getattr(entry, key)
		if value is not None:
		context[key] = value
		result["context"] = context

results.append(result)		results.append(result)

if len(results) > limit:		if len(results) > limit:
results.pop()		results.pop()
assert len(results) == limit		assert len(results) == limit
last_result = results[-1]		last_result = results[-1]
next_page_token: Optional[bytes] = msgpack_dumps(		next_page_token: Optional[bytes] = msgpack_dumps(
(		(
▲ Show 20 Lines • Show All 53 Lines • Show Last 20 Lines