Differential D2821 Diff 10014 swh/storage/cassandra/storage.py

Changeset View

Standalone View

swh/storage/cassandra/storage.py

# Copyright (C) 2019-2020 The Software Heritage developers		# Copyright (C) 2019-2020 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution		# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version		# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information		# See top-level LICENSE file for more information

import datetime		import datetime
import json		import json
import random		import random
import re		import re
from typing import Any, Dict, List, Iterable, Optional, Union		from typing import Any, Dict, List, Iterable, Optional, Union
import uuid		import uuid

import attr		import attr
import dateutil		import dateutil

from swh.model.model import (		from swh.model.model import (
BaseModel, Revision, Release, Directory, DirectoryEntry, Content,		Revision, Release, Directory, DirectoryEntry, Content,
SkippedContent, OriginVisit, Snapshot, Origin		SkippedContent, OriginVisit, Snapshot, Origin
)		)
from swh.model.hashutil import DEFAULT_ALGORITHMS		from swh.model.hashutil import DEFAULT_ALGORITHMS
from swh.storage.objstorage import ObjStorage		from swh.storage.objstorage import ObjStorage
from swh.storage.writer import JournalWriter		from swh.storage.writer import JournalWriter

from .. import HashCollision		from .. import HashCollision
from ..exc import StorageArgumentException		from ..exc import StorageArgumentException
▲ Show 20 Lines • Show All 637 Lines • ▼ Show 20 Lines	def origin_get(self, origins):
results = [self.origin_get_one(origin) for origin in origins]		results = [self.origin_get_one(origin) for origin in origins]

if return_single:		if return_single:
assert len(results) == 1		assert len(results) == 1
return results[0]		return results[0]
else:		else:
return results		return results

def origin_get_one(self, origin):		def origin_get_one(self, origin: Dict[str, Any]) -> Optional[
		Dict[str, Any]]:
		vlorentzUnsubmitted Not Done Inline Actions you should break line earlier, so the return type isn't split in half. (and why isn't there a corresponding change in `interface.py`?) vlorentz: you should break line earlier, so the return type isn't split in half. (and why isn't there a…
		ardumontAuthorUnsubmitted Done Inline Actions i think it's because it's not exposed, it's a cassandra specific one. ardumont: i think it's because it's not exposed, it's a cassandra specific one.
		ardumontAuthorUnsubmitted Done Inline Actions Yes, that's the reason. ardumont: Yes, that's the reason.
if 'id' in origin:		if 'id' in origin:
raise StorageArgumentException('Origin ids are not supported.')		raise StorageArgumentException('Origin ids are not supported.')
if 'url' not in origin:		if 'url' not in origin:
raise StorageArgumentException('Missing origin url')		raise StorageArgumentException('Missing origin url')
rows = self._cql_runner.origin_get_by_url(origin['url'])		rows = self._cql_runner.origin_get_by_url(origin['url'])

rows = list(rows)		rows = list(rows)
if rows:		if rows:
▲ Show 20 Lines • Show All 74 Lines • ▼ Show 20 Lines	def origin_add_one(self, origin: Origin) -> str:

self._cql_runner.origin_add_one(origin)		self._cql_runner.origin_add_one(origin)
origin_url = origin.url		origin_url = origin.url

return origin_url		return origin_url

def origin_visit_add(self, origin: Origin,		def origin_visit_add(self, origin: Origin,
date: Union[str, datetime.datetime],		date: Union[str, datetime.datetime],
type: str) -> Dict[str, BaseModel]:		type: str) -> OriginVisit:
origin_url = origin.url		origin_url = origin.url

if isinstance(date, str):		if isinstance(date, str):
# FIXME: Converge on iso8601 at some point		# FIXME: Converge on iso8601 at some point
date = dateutil.parser.parse(date)		date = dateutil.parser.parse(date)
elif not isinstance(date, datetime.datetime):		elif not isinstance(date, datetime.datetime):
raise StorageArgumentException(		raise StorageArgumentException(
'Date must be a datetime or a string')		'Date must be a datetime or a string')

origin = self.origin_get_one({'url': origin_url})		if not self.origin_get_one({'url': origin.url}):
if not origin: # Cannot add a visit without an origin
raise StorageArgumentException(		raise StorageArgumentException(
'Unknown origin %s', origin_url)		'Unknown origin %s', origin_url)

visit_id = self._cql_runner.origin_generate_unique_visit_id(origin_url)		visit_id = self._cql_runner.origin_generate_unique_visit_id(origin_url)

try:		try:
visit = OriginVisit.from_dict({		visit = OriginVisit.from_dict({
'origin': origin_url,		'origin': origin_url,
'date': date,		'date': date,
'type': type,		'type': type,
'status': 'ongoing',		'status': 'ongoing',
'snapshot': None,		'snapshot': None,
'metadata': None,		'metadata': None,
'visit': visit_id		'visit': visit_id
})		})
except (KeyError, TypeError, ValueError) as e:		except (KeyError, TypeError, ValueError) as e:
raise StorageArgumentException(*e.args)		raise StorageArgumentException(*e.args)
self.journal_writer.origin_visit_add(visit)		self.journal_writer.origin_visit_add(visit)

self._cql_runner.origin_visit_add_one(visit)		self._cql_runner.origin_visit_add_one(visit)
		return visit
return {
'origin': origin,
'visit': visit,
}

def origin_visit_update(		def origin_visit_update(
self, origin: Origin, visit: OriginVisit,		self, origin: Origin, visit: OriginVisit,
status: Optional[str] = None, metadata: Optional[Dict] = None,		status: Optional[str] = None, metadata: Optional[Dict] = None,
snapshot: Optional[bytes] = None) -> None:		snapshot: Optional[bytes] = None) -> None:
# Get the existing data of the visit		# Get the existing data of the visit
row = self._cql_runner.origin_visit_get_one(origin.url, visit.visit)		row = self._cql_runner.origin_visit_get_one(origin.url, visit.visit)
if not row:		if not row:
▲ Show 20 Lines • Show All 153 Lines • Show Last 20 Lines