Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/in_memory.py
| Show All 13 Lines | |||||
| from collections import defaultdict | from collections import defaultdict | ||||
| from datetime import timedelta | from datetime import timedelta | ||||
| from typing import Any, Dict, Iterable, List, Optional, Union | from typing import Any, Dict, Iterable, List, Optional, Union | ||||
| import attr | import attr | ||||
| from swh.model.model import ( | from swh.model.model import ( | ||||
| BaseContent, Content, SkippedContent, Directory, Revision, Release, | BaseContent, Content, SkippedContent, Directory, Revision, | ||||
| Snapshot, OriginVisit, Origin, SHA1_SIZE) | Release, Snapshot, OriginVisit, Origin, SHA1_SIZE | ||||
| ) | |||||
| from swh.model.hashutil import DEFAULT_ALGORITHMS, hash_to_bytes, hash_to_hex | from swh.model.hashutil import DEFAULT_ALGORITHMS, hash_to_bytes, hash_to_hex | ||||
| from swh.storage.objstorage import ObjStorage | from swh.storage.objstorage import ObjStorage | ||||
| from . import HashCollision | from . import HashCollision | ||||
| from .exc import StorageArgumentException | from .exc import StorageArgumentException | ||||
| from .converters import origin_url_to_sha1 | from .converters import origin_url_to_sha1 | ||||
| from .utils import get_partition_bounds_bytes | from .utils import get_partition_bounds_bytes | ||||
| ▲ Show 20 Lines • Show All 665 Lines • ▼ Show 20 Lines | def origin_add_one(self, origin: Origin) -> str: | ||||
| self._origins[origin.url] = origin | self._origins[origin.url] = origin | ||||
| self._origins_by_sha1[origin_url_to_sha1(origin.url)] = origin | self._origins_by_sha1[origin_url_to_sha1(origin.url)] = origin | ||||
| self._origin_visits[origin.url] = [] | self._origin_visits[origin.url] = [] | ||||
| self._objects[origin.url].append(('origin', origin.url)) | self._objects[origin.url].append(('origin', origin.url)) | ||||
| return origin.url | return origin.url | ||||
| def origin_visit_add( | def origin_visit_add(self, origin: Origin, | ||||
| self, origin, date, type) -> Optional[Dict[str, Union[str, int]]]: | date: Union[str, datetime.datetime], | ||||
| origin_url = origin | type: str) -> OriginVisit: | ||||
| if origin_url is None: | origin_url = origin.url | ||||
| raise StorageArgumentException('Unknown origin.') | |||||
| if isinstance(date, str): | if isinstance(date, str): | ||||
| # FIXME: Converge on iso8601 at some point | # FIXME: Converge on iso8601 at some point | ||||
| date = dateutil.parser.parse(date) | date = dateutil.parser.parse(date) | ||||
| elif not isinstance(date, datetime.datetime): | elif not isinstance(date, datetime.datetime): | ||||
| raise StorageArgumentException( | raise StorageArgumentException( | ||||
| 'date must be a datetime or a string.') | 'Date must be a datetime or a string') | ||||
| origin = self.origin_get({'url': origin_url}) | |||||
| if not origin: # Cannot add a visit without an origin | |||||
| raise StorageArgumentException( | |||||
| 'Unknown origin %s', origin_url) | |||||
| visit_ret = None | |||||
| if origin_url in self._origins: | if origin_url in self._origins: | ||||
| origin = self._origins[origin_url] | origin = self._origins[origin_url] | ||||
| # visit ids are in the range [1, +inf[ | # visit ids are in the range [1, +inf[ | ||||
| visit_id = len(self._origin_visits[origin_url]) + 1 | visit_id = len(self._origin_visits[origin_url]) + 1 | ||||
| status = 'ongoing' | status = 'ongoing' | ||||
| visit = OriginVisit( | visit = OriginVisit( | ||||
| origin=origin.url, | origin=origin.url, | ||||
| date=date, | date=date, | ||||
| type=type, | type=type, | ||||
| status=status, | status=status, | ||||
| snapshot=None, | snapshot=None, | ||||
| metadata=None, | metadata=None, | ||||
| visit=visit_id, | visit=visit_id, | ||||
| ) | ) | ||||
| self._origin_visits[origin_url].append(visit) | self._origin_visits[origin_url].append(visit) | ||||
| visit_ret = { | visit = visit | ||||
| 'origin': origin.url, | |||||
| 'visit': visit_id, | |||||
| } | |||||
| self._objects[(origin_url, visit_id)].append( | self._objects[(origin.url, visit.visit)].append( | ||||
| ('origin_visit', None)) | ('origin_visit', None)) | ||||
| self.journal_writer.origin_visit_add(visit) | self.journal_writer.origin_visit_add(visit) | ||||
| return visit_ret | # return last visit | ||||
| return visit | |||||
| def origin_visit_update( | def origin_visit_update( | ||||
| self, origin: str, visit_id: int, status: Optional[str] = None, | self, origin: str, visit_id: int, status: Optional[str] = None, | ||||
| metadata: Optional[Dict] = None, snapshot: Optional[bytes] = None): | metadata: Optional[Dict] = None, snapshot: Optional[bytes] = None): | ||||
| origin_url = self._get_origin_url(origin) | origin_url = self._get_origin_url(origin) | ||||
| if origin_url is None: | if origin_url is None: | ||||
| raise StorageArgumentException('Unknown origin.') | raise StorageArgumentException('Unknown origin.') | ||||
| ▲ Show 20 Lines • Show All 250 Lines • Show Last 20 Lines | |||||