Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/in_memory.py
| Show First 20 Lines • Show All 414 Lines • ▼ Show 20 Lines | def skipped_content_missing(self, contents): | ||||
| if algorithm == 'blake2s256': | if algorithm == 'blake2s256': | ||||
| continue | continue | ||||
| if key not in self._skipped_content_indexes[algorithm]: | if key not in self._skipped_content_indexes[algorithm]: | ||||
| # index must contain hashes of algos except blake2s256 | # index must contain hashes of algos except blake2s256 | ||||
| # else the content is considered skipped | # else the content is considered skipped | ||||
| yield content | yield content | ||||
| break | break | ||||
| def content_get_random(self): | |||||
| """Finds a random content id. | |||||
| Returns: | |||||
| a sha1_git | |||||
| """ | |||||
| return random.choice(list(self._content_indexes['sha1_git'])) | |||||
| def directory_add(self, directories): | def directory_add(self, directories): | ||||
| """Add directories to the storage | """Add directories to the storage | ||||
| Args: | Args: | ||||
| directories (iterable): iterable of dictionaries representing the | directories (iterable): iterable of dictionaries representing the | ||||
| individual directories to add. Each dict has the following | individual directories to add. Each dict has the following | ||||
| keys: | keys: | ||||
| ▲ Show 20 Lines • Show All 99 Lines • ▼ Show 20 Lines | def directory_entry_get_by_path(self, directory, paths): | ||||
| (top) to right (bottom). | (top) to right (bottom). | ||||
| Returns: | Returns: | ||||
| The corresponding directory entry if found, None otherwise. | The corresponding directory entry if found, None otherwise. | ||||
| """ | """ | ||||
| return self._directory_entry_get_by_path(directory, paths, b'') | return self._directory_entry_get_by_path(directory, paths, b'') | ||||
| def directory_get_random(self): | |||||
| """Finds a random directory id. | |||||
| Returns: | |||||
| a sha1_git | |||||
| """ | |||||
| return random.choice(list(self._directories)) | |||||
| def _directory_entry_get_by_path(self, directory, paths, prefix): | def _directory_entry_get_by_path(self, directory, paths, prefix): | ||||
| if not paths: | if not paths: | ||||
| return | return | ||||
| contents = list(self.directory_ls(directory)) | contents = list(self.directory_ls(directory)) | ||||
| if not contents: | if not contents: | ||||
| return | return | ||||
| ▲ Show 20 Lines • Show All 130 Lines • ▼ Show 20 Lines | def revision_shortlog(self, revisions, limit=None): | ||||
| Yields: | Yields: | ||||
| a list of (id, parents) tuples. | a list of (id, parents) tuples. | ||||
| """ | """ | ||||
| yield from ((rev['id'], rev['parents']) | yield from ((rev['id'], rev['parents']) | ||||
| for rev in self.revision_log(revisions, limit)) | for rev in self.revision_log(revisions, limit)) | ||||
| def revision_get_random(self): | |||||
| """Finds a random revision id. | |||||
| Returns: | |||||
| a sha1_git | |||||
| """ | |||||
| return random.choice(list(self._revisions)) | |||||
| def release_add(self, releases): | def release_add(self, releases): | ||||
| """Add releases to the storage | """Add releases to the storage | ||||
| Args: | Args: | ||||
| releases (Iterable[dict]): iterable of dictionaries representing | releases (Iterable[dict]): iterable of dictionaries representing | ||||
| the individual releases to add. Each dict has the following | the individual releases to add. Each dict has the following | ||||
| keys: | keys: | ||||
| ▲ Show 20 Lines • Show All 59 Lines • ▼ Show 20 Lines | def release_get(self, releases): | ||||
| """ | """ | ||||
| for rel_id in releases: | for rel_id in releases: | ||||
| if rel_id in self._releases: | if rel_id in self._releases: | ||||
| yield self._releases[rel_id].to_dict() | yield self._releases[rel_id].to_dict() | ||||
| else: | else: | ||||
| yield None | yield None | ||||
| def release_get_random(self): | |||||
| """Finds a random release id. | |||||
| Returns: | |||||
| a sha1_git | |||||
| """ | |||||
| return random.choice(list(self._releases)) | |||||
| def snapshot_add(self, snapshots): | def snapshot_add(self, snapshots): | ||||
| """Add a snapshot to the storage | """Add a snapshot to the storage | ||||
| Args: | Args: | ||||
| snapshot ([dict]): the snapshots to add, containing the | snapshot ([dict]): the snapshots to add, containing the | ||||
| following keys: | following keys: | ||||
| - **id** (:class:`bytes`): id of the snapshot | - **id** (:class:`bytes`): id of the snapshot | ||||
| ▲ Show 20 Lines • Show All 211 Lines • ▼ Show 20 Lines | def snapshot_get_branches(self, snapshot_id, branches_from=b'', | ||||
| for (name, branch) in branches.items()} | for (name, branch) in branches.items()} | ||||
| return { | return { | ||||
| 'id': snapshot_id, | 'id': snapshot_id, | ||||
| 'branches': branches, | 'branches': branches, | ||||
| 'next_branch': next_branch, | 'next_branch': next_branch, | ||||
| } | } | ||||
| def snapshot_get_random(self): | |||||
| """Finds a random snapshot id. | |||||
| Returns: | |||||
| a sha1_git | |||||
| """ | |||||
| return random.choice(list(self._snapshots)) | |||||
| def object_find_by_sha1_git(self, ids, db=None, cur=None): | def object_find_by_sha1_git(self, ids, db=None, cur=None): | ||||
| """Return the objects found with the given ids. | """Return the objects found with the given ids. | ||||
| Args: | Args: | ||||
| ids: a generator of sha1_gits | ids: a generator of sha1_gits | ||||
| Returns: | Returns: | ||||
| dict: a mapping from id to the list of objects found. Each object | dict: a mapping from id to the list of objects found. Each object | ||||
| ▲ Show 20 Lines • Show All 93 Lines • ▼ Show 20 Lines | def origin_get_by_sha1(self, sha1s): | ||||
| by :meth:`swh.storage.in_memory.Storage.origin_get`, or None if an | by :meth:`swh.storage.in_memory.Storage.origin_get`, or None if an | ||||
| origin matching the sha1 is not found. | origin matching the sha1 is not found. | ||||
| """ | """ | ||||
| return [ | return [ | ||||
| self._convert_origin(self._origins_by_sha1.get(sha1)) | self._convert_origin(self._origins_by_sha1.get(sha1)) | ||||
| for sha1 in sha1s | for sha1 in sha1s | ||||
| ] | ] | ||||
| def _select_random_origin_by_type(self, type: str) -> str: | |||||
| """Select randomly an origin visit """ | |||||
| while True: | |||||
| url = random.choice(list(self._origin_visits.keys())) | |||||
| random_origin_visits = self._origin_visits[url] | |||||
| if random_origin_visits[0].type == type: | |||||
| return url | |||||
| def origin_visit_get_random(self, type: str) -> Mapping[str, Any]: | |||||
| """Randomly select one origin with <type> whose visit was successful | |||||
| in the last 3 months. | |||||
| Returns: | |||||
| origin dict selected randomly on the dataset | |||||
| """ | |||||
| random_visit: Dict[str, Any] = {} | |||||
| if not self._origin_visits: # empty dataset | |||||
| return random_visit | |||||
| url = self._select_random_origin_by_type(type) | |||||
| random_origin_visits = copy.deepcopy(self._origin_visits[url]) | |||||
| random_origin_visits.reverse() | |||||
| back_in_the_day = now() - timedelta(weeks=12) # 3 months back | |||||
| # This should be enough for tests | |||||
| for visit in random_origin_visits: | |||||
| if visit.date > back_in_the_day and visit.status == 'full': | |||||
| random_visit = visit.to_dict() | |||||
| break | |||||
| return random_visit | |||||
| def origin_get_range(self, origin_from=1, origin_count=100): | def origin_get_range(self, origin_from=1, origin_count=100): | ||||
| """Retrieve ``origin_count`` origins whose ids are greater | """Retrieve ``origin_count`` origins whose ids are greater | ||||
| or equal than ``origin_from``. | or equal than ``origin_from``. | ||||
| Origins are sorted by id before retrieving them. | Origins are sorted by id before retrieving them. | ||||
| Args: | Args: | ||||
| origin_from (int): the minimum id of origins to retrieve | origin_from (int): the minimum id of origins to retrieve | ||||
| ▲ Show 20 Lines • Show All 369 Lines • ▼ Show 20 Lines | def origin_visit_get_latest( | ||||
| if require_snapshot: | if require_snapshot: | ||||
| visits = [visit for visit in visits | visits = [visit for visit in visits | ||||
| if visit.snapshot] | if visit.snapshot] | ||||
| visit = max( | visit = max( | ||||
| visits, key=lambda v: (v.date, v.visit), default=None) | visits, key=lambda v: (v.date, v.visit), default=None) | ||||
| return self._convert_visit(visit) | return self._convert_visit(visit) | ||||
| def _select_random_origin_visit_by_type(self, type: str) -> str: | |||||
| """Select randomly an origin visit """ | |||||
| while True: | |||||
| url = random.choice(list(self._origin_visits.keys())) | |||||
| random_origin_visits = self._origin_visits[url] | |||||
| if random_origin_visits[0].type == type: | |||||
| return url | |||||
| def origin_visit_get_random(self, type: str) -> Mapping[str, Any]: | |||||
| """Randomly select one successful origin visit with <type> | |||||
| made in the last 3 months. | |||||
| Returns: | |||||
| dict representing an origin visit, in the same format as | |||||
| `origin_visit_get`. | |||||
| """ | |||||
| random_visit: Dict[str, Any] = {} | |||||
| if not self._origin_visits: # empty dataset | |||||
| return random_visit | |||||
| url = self._select_random_origin_visit_by_type(type) | |||||
| random_origin_visits = copy.deepcopy(self._origin_visits[url]) | |||||
| random_origin_visits.reverse() | |||||
| back_in_the_day = now() - timedelta(weeks=12) # 3 months back | |||||
| # This should be enough for tests | |||||
| for visit in random_origin_visits: | |||||
| if visit.date > back_in_the_day and visit.status == 'full': | |||||
| random_visit = visit.to_dict() | |||||
| break | |||||
| return random_visit | |||||
| def stat_counters(self): | def stat_counters(self): | ||||
| """compute statistics about the number of tuples in various tables | """compute statistics about the number of tuples in various tables | ||||
| Returns: | Returns: | ||||
| dict: a dictionary mapping textual labels (e.g., content) to | dict: a dictionary mapping textual labels (e.g., content) to | ||||
| integer values (e.g., the number of tuples in table content) | integer values (e.g., the number of tuples in table content) | ||||
| """ | """ | ||||
| ▲ Show 20 Lines • Show All 224 Lines • Show Last 20 Lines | |||||