Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/in_memory.py
| Show First 20 Lines • Show All 834 Lines • ▼ Show 20 Lines | def origin_visit_upsert(self, visits: Iterable[OriginVisit]) -> None: | ||||
| self._origin_visits[origin_url].append(None) | self._origin_visits[origin_url].append(None) | ||||
| self._origin_visits[origin_url][visit.visit-1] = visit | self._origin_visits[origin_url][visit.visit-1] = visit | ||||
| self._origin_visit_updates[visit_key].append(visit_update) | self._origin_visit_updates[visit_key].append(visit_update) | ||||
| self._objects[visit_key].append( | self._objects[visit_key].append( | ||||
| ('origin_visit', None)) | ('origin_visit', None)) | ||||
| def _origin_visit_get_updated(self, origin: str, visit_id: int): | def _origin_visit_get_updated( | ||||
| self, origin: str, visit_id: int) -> Optional[OriginVisit]: | |||||
| """Merge origin visit and latest origin visit update | |||||
| """ | |||||
| assert visit_id >= 1 | assert visit_id >= 1 | ||||
| visit = self._origin_visits[origin][visit_id-1] | visit = self._origin_visits[origin][visit_id-1] | ||||
| if visit is None: | if visit is None: | ||||
| return None | return None | ||||
| visit_key = (origin, visit_id) | visit_key = (origin, visit_id) | ||||
| visit_update = max( | visit_update = max( | ||||
| self._origin_visit_updates[visit_key], key=lambda v: v.date) | self._origin_visit_updates[visit_key], key=lambda v: v.date) | ||||
| ▲ Show 20 Lines • Show All 88 Lines • ▼ Show 20 Lines | def origin_visit_get_random(self, type: str) -> Optional[Dict[str, Any]]: | ||||
| url = self._select_random_origin_visit_by_type(type) | url = self._select_random_origin_visit_by_type(type) | ||||
| random_origin_visits = copy.deepcopy(self._origin_visits[url]) | random_origin_visits = copy.deepcopy(self._origin_visits[url]) | ||||
| random_origin_visits.reverse() | random_origin_visits.reverse() | ||||
| back_in_the_day = now() - timedelta(weeks=12) # 3 months back | back_in_the_day = now() - timedelta(weeks=12) # 3 months back | ||||
| # This should be enough for tests | # This should be enough for tests | ||||
| for visit in random_origin_visits: | for visit in random_origin_visits: | ||||
| updated_visit = self._origin_visit_get_updated( | updated_visit = self._origin_visit_get_updated( | ||||
| url, visit.visit) | url, visit.visit) | ||||
| assert updated_visit is not None | |||||
| if updated_visit.date > back_in_the_day \ | if updated_visit.date > back_in_the_day \ | ||||
| and updated_visit.status == 'full': | and updated_visit.status == 'full': | ||||
| return updated_visit.to_dict() | return updated_visit.to_dict() | ||||
| else: | else: | ||||
| return None | return None | ||||
| def stat_counters(self): | def stat_counters(self): | ||||
| keys = ( | keys = ( | ||||
| ▲ Show 20 Lines • Show All 128 Lines • Show Last 20 Lines | |||||