Page MenuHomeSoftware Heritage
Differential D4798

Don't log throttle config passthrough to Sentry.
ClosedPublic
Actions

Authored by vlorentz on Dec 30 2020, 11:39 AM.

Details

Reviewers
olasd
Group Reviewers
Reviewers
Commits
rDWAPPSe131fa7ae236: Don't log throttle config passthrough to Sentry.
Summary

It's not an actual error, it doesn't deserve to be logged.

Replaces D4794.

Diff Detail

Repository
rDWAPPS Web applications
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

vlorentz created this revision.Dec 30 2020, 11:39 AM
Herald added a reviewer: Reviewers. · View Herald TranscriptDec 30 2020, 11:39 AM
swh-public-ci added a comment.Dec 30 2020, 11:46 AM

Build is green

Patch application report for D4798 (id=16978)

Rebasing onto 2c9c8527be...

Current branch diff-target is up to date.
Changes applied before test
commit e131fa7ae2364aa3f5bc4138b411324c61b7d006
Author: Valentin Lorentz <vlorentz@softwareheritage.org>
Date:   Wed Dec 30 11:39:10 2020 +0100

    Don't log throttle config passthrough to Sentry.
    
    It's not an actual error, it doesn't deserve to be logged.

See https://jenkins.softwareheritage.org/job/DWAPPS/job/tests-on-diff/538/ for more details.

Harbormaster completed remote builds in B18115: Diff 16978.Dec 30 2020, 11:46 AM
vlorentz requested review of this revision.Dec 30 2020, 11:46 AM
olasd accepted this revision.Dec 30 2020, 11:51 AM
olasd added a subscriber: olasd.

Thanks!

This revision is now accepted and ready to land.Dec 30 2020, 11:51 AM
Closed by commit rDWAPPSe131fa7ae236: Don't log throttle config passthrough to Sentry. (authored by vlorentz). · Explain WhyDec 30 2020, 1:56 PM
This revision was automatically updated to reflect the committed changes.
vlorentz added a commit: rDWAPPSe131fa7ae236: Don't log throttle config passthrough to Sentry..

Revision Contents
Changeset List

PathSize
swh/
web/
api/
3 lines

Diff 16980

View Options

swh/web/api/throttling.py

# Copyright (C) 2018-2021 The Software Heritage developers # Copyright (C) 2018-2022 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution # See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version # License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information # See top-level LICENSE file for more information
from __future__ import annotations
from abc import ABCMeta, abstractmethod from abc import ABCMeta, abstractmethod
from collections import deque from collections import deque
import dataclasses
import heapq import heapq
from typing import TYPE_CHECKING, Dict, List, Optional, Set, TypeVar
from swh.model.model import Sha1Git
if TYPE_CHECKING:
from swh.storage.interface import StorageInterface
@dataclasses.dataclass
class State:
done: Set[Sha1Git] = dataclasses.field(default_factory=set)
revs_to_visit: List = dataclasses.field(default_factory=list)
last_rev: Optional[Dict] = None
num_revs: int = 0
missing_revs: Set[Sha1Git] = dataclasses.field(default_factory=set)
_revs_walker_classes = {} _revs_walker_classes = {}
class _RevisionsWalkerMetaClass(ABCMeta): class _RevisionsWalkerMetaClass(ABCMeta):
def __new__(cls, clsname, bases, attrs): def __new__(cls, clsname, bases, attrs):
newclass = super().__new__(cls, clsname, bases, attrs) newclass = super().__new__(cls, clsname, bases, attrs)
if "rw_type" in attrs: if "rw_type" in attrs:
_revs_walker_classes[attrs["rw_type"]] = newclass _revs_walker_classes[attrs["rw_type"]] = newclass
return newclass return newclass
TWalker = TypeVar("TWalker", bound="RevisionsWalker")
class RevisionsWalker(metaclass=_RevisionsWalkerMetaClass): class RevisionsWalker(metaclass=_RevisionsWalkerMetaClass):
""" """
Abstract base class encapsulating the logic to walk across Abstract base class encapsulating the logic to walk across
a revisions history starting from a given one. a revisions history starting from a given one.
It defines an iterator returning the revisions according It defines an iterator returning the revisions according
to a specific ordering implemented in derived classes. to a specific ordering implemented in derived classes.
Show All 14 Lines The iteration step performs the following operations:
method :meth:`should_return` and returns it if method :meth:`should_return` and returns it if
it is the case it is the case
In order to easily instantiate a specific type of revisions In order to easily instantiate a specific type of revisions
walker, it is recommended to use the factory function walker, it is recommended to use the factory function
:func:`get_revisions_walker`. :func:`get_revisions_walker`.
Args: Args:
storage (swh.storage.interface.StorageInterface): instance of swh storage storage: instance of swh storage (either local or remote)
(either local or remote) rev_start: a revision identifier
rev_start (bytes): a revision identifier max_revs: maximum number of revisions to return
max_revs (Optional[int]): maximum number of revisions to return state: previous state of that revisions walker
state (Optional[dict]): previous state of that revisions walker
""" """
def __init__(self, storage, rev_start, max_revs=None, state=None): def __init__(
self._revs_to_visit = [] self,
self._done = set() storage: StorageInterface,
self._revs = {} rev_start: Sha1Git,
self._last_rev = None max_revs: Optional[int] = None,
self._num_revs = 0 state: Optional[State] = None,
):
self._revs: Dict[Sha1Git, Dict] = {}
self._max_revs = max_revs self._max_revs = max_revs
self._missing_revs = set() self._state = state or State()
if state:
self._revs_to_visit = state["revs_to_visit"]
self._done = state["done"]
self._last_rev = state["last_rev"]
self._num_revs = state["num_revs"]
self._missing_revs = state["missing_revs"]
self.storage = storage self.storage = storage
self.process_rev(rev_start) self.process_rev(rev_start)
@abstractmethod @abstractmethod
def process_rev(self, rev_id): def process_rev(self, rev_id: Sha1Git) -> None:
""" """
Abstract method whose purpose is to process a newly visited Abstract method whose purpose is to process a newly visited
revision during the walk. revision during the walk.
Derived classes must implement it according to the desired Derived classes must implement it according to the desired
method to walk across the revisions history (for instance method to walk across the revisions history (for instance
through a dfs on the revisions DAG). through a dfs on the revisions DAG).
Args: Args:
rev_id (bytes): the newly visited revision identifier rev_id: the newly visited revision identifier
""" """
pass pass
@abstractmethod @abstractmethod
def get_next_rev_id(self): def get_next_rev_id(self) -> Sha1Git:
""" """
Abstract method whose purpose is to return the next revision Abstract method whose purpose is to return the next revision
during the iteration. during the iteration.
Derived classes must implement it according to the desired Derived classes must implement it according to the desired
method to walk across the revisions history. method to walk across the revisions history.
Returns:
dict: A dict describing a revision as returned by
:meth:`swh.storage.interface.StorageInterface.revision_get`
""" """
pass pass
def process_parent_revs(self, rev): def process_parent_revs(self, rev: Dict) -> None:
""" """
Process the parents of a revision when it is iterated. Process the parents of a revision when it is iterated.
The default implementation simply calls :meth:`process_rev` The default implementation simply calls :meth:`process_rev`
for each parent revision in the order they are declared. for each parent revision in the order they are declared.
Args: Args:
rev (dict): A dict describing a revision as returned by rev (dict): A dict describing a revision as returned by
:meth:`swh.storage.interface.StorageInterface.revision_get` :meth:`swh.storage.interface.StorageInterface.revision_get`
""" """
for parent_id in rev["parents"]: for parent_id in rev["parents"]:
self.process_rev(parent_id) self.process_rev(parent_id)
def should_return(self, rev): def should_return(self, rev: Dict) -> bool:
""" """
Filter out a revision to return if needed. Filter out a revision to return if needed.
Default implementation returns all iterated revisions. Default implementation returns all iterated revisions.
Args: Args:
rev (dict): A dict describing a revision as returned by rev (dict): A dict describing a revision as returned by
:meth:`swh.storage.interface.StorageInterface.revision_get` :meth:`swh.storage.interface.StorageInterface.revision_get`
Returns: Returns:
bool: Whether to return the revision in the iteration bool: Whether to return the revision in the iteration
""" """
return True return True
def is_finished(self): def is_finished(self) -> bool:
""" """
Determine if the iteration is finished. Determine if the iteration is finished.
This method is called at the beginning of each iteration loop. This method is called at the beginning of each iteration loop.
Returns: Returns:
bool: Whether the iteration is finished bool: Whether the iteration is finished
""" """
if self._max_revs is not None and self._num_revs >= self._max_revs: if self._max_revs is not None and self._state.num_revs >= self._max_revs:
return True return True
if not self._revs_to_visit: if not self._state.revs_to_visit:
return True return True
return False return False
def _get_rev(self, rev_id): def _get_rev(self, rev_id: Sha1Git) -> Optional[Dict]:
rev = self._revs.get(rev_id) rev = self._revs.get(rev_id)
if rev is None: if rev is None:
# cache some revisions in advance to avoid sending too much # cache some revisions in advance to avoid sending too much
# requests to storage and thus speedup the revisions walk # requests to storage and thus speedup the revisions walk
for rev in self.storage.revision_log([rev_id], limit=100): for rev in self.storage.revision_log([rev_id], limit=100):
# revision data is missing, returned history will be truncated # revision data is missing, returned history will be truncated
if rev is None: if rev is None:
continue continue
self._revs[rev["id"]] = rev self._revs[rev["id"]] = rev
return self._revs.get(rev_id) return self._revs.get(rev_id)
def missing_revisions(self): def missing_revisions(self) -> Set[Sha1Git]:
""" """
Return a set of revision identifiers whose associated data were Return a set of revision identifiers whose associated data were
found missing into the archive content while walking on the found missing into the archive content while walking on the
revisions graph. revisions graph.
Returns: Returns:
Set[bytes]: a set of revision identifiers Set[bytes]: a set of revision identifiers
""" """
return self._missing_revs return self._state.missing_revs
def is_history_truncated(self): def is_history_truncated(self) -> bool:
""" """
Return if the revision history generated so far has been truncated Return if the revision history generated so far has been truncated
of not. A revision history might end up truncated if some revision of not. A revision history might end up truncated if some revision
data were found missing into the archive content. data were found missing into the archive content.
Returns: Returns:
bool: Whether the history got truncated or not bool: Whether the history got truncated or not
""" """
return len(self.missing_revisions()) > 0 return len(self.missing_revisions()) > 0
def export_state(self): def export_state(self) -> State:
""" """
Export the internal state of that revision walker to a dict. Export the internal state of that revision walker to a dict.
Its purpose is to continue the iteration in a pagination context. Its purpose is to continue the iteration in a pagination context.
Returns: Returns:
dict: A dict containing the internal state of that revisions walker dict: A dict containing the internal state of that revisions walker
douarddaUnsubmitted
Not Done
Inline Actions

shouldn't this be updated?

douardda: shouldn't this be updated?
""" """
return { return self._state
"revs_to_visit": self._revs_to_visit,
"done": self._done,
"last_rev": self._last_rev,
"num_revs": self._num_revs,
"missing_revs": self._missing_revs,
}
def __next__(self): def __next__(self) -> Dict:
if self.is_finished(): if self.is_finished():
raise StopIteration raise StopIteration
while self._revs_to_visit: while self._state.revs_to_visit:
rev_id = self.get_next_rev_id() rev_id = self.get_next_rev_id()
if rev_id in self._done: if rev_id in self._state.done:
continue continue
self._done.add(rev_id) self._state.done.add(rev_id)
rev = self._get_rev(rev_id) rev = self._get_rev(rev_id)
# revision data is missing, returned history will be truncated # revision data is missing, returned history will be truncated
if rev is None: if rev is None:
self._missing_revs.add(rev_id) self._state.missing_revs.add(rev_id)
continue continue
self.process_parent_revs(rev) self.process_parent_revs(rev)
if self.should_return(rev): if self.should_return(rev):
self._num_revs += 1 self._state.num_revs += 1
self._last_rev = rev self._state.last_rev = rev
return rev return rev
raise StopIteration raise StopIteration
def __iter__(self): def __iter__(self: TWalker) -> TWalker:
return self return self
class CommitterDateRevisionsWalker(RevisionsWalker): class CommitterDateRevisionsWalker(RevisionsWalker):
""" """
Revisions walker that returns revisions in reverse chronological Revisions walker that returns revisions in reverse chronological
order according to committer date (same behaviour as ``git log``) order according to committer date (same behaviour as ``git log``)
""" """
rw_type = "committer_date" rw_type = "committer_date"
def process_rev(self, rev_id): def process_rev(self, rev_id: Sha1Git) -> None:
""" """
Add the revision to a priority queue according to the committer date. Add the revision to a priority queue according to the committer date.
Args: Args:
rev_id (bytes): the newly visited revision identifier rev_id (bytes): the newly visited revision identifier
""" """
if rev_id not in self._done: if rev_id not in self._state.done:
rev = self._get_rev(rev_id) rev = self._get_rev(rev_id)
if rev is not None: if rev is not None:
commit_time = ( commit_time = (
rev["committer_date"]["timestamp"]["seconds"] rev["committer_date"]["timestamp"]["seconds"]
if rev["committer_date"] if rev["committer_date"]
# allows to avoid failure with a revision without commit date # allows to avoid failure with a revision without commit date
# and iterate on such revision before its parents # and iterate on such revision before its parents
else len(self._revs_to_visit) else len(self._state.revs_to_visit)
) )
heapq.heappush(self._revs_to_visit, (-commit_time, rev_id)) heapq.heappush(
self._state.revs_to_visit, (-commit_time, rev_id)
) # type: ignore
else: else:
self._missing_revs.add(rev_id) self._state.missing_revs.add(rev_id)
def get_next_rev_id(self): def get_next_rev_id(self) -> Sha1Git:
""" """
Return the smallest revision from the priority queue, i.e. Return the smallest revision from the priority queue, i.e.
the one with highest committer date. the one with highest committer date.
Returns: Returns:
dict: A dict describing a revision as returned by dict: A dict describing a revision as returned by
:meth:`swh.storage.interface.StorageInterface.revision_get` :meth:`swh.storage.interface.StorageInterface.revision_get`
""" """
_, rev_id = heapq.heappop(self._revs_to_visit) _, rev_id = heapq.heappop(self._state.revs_to_visit)
return rev_id return rev_id
class BFSRevisionsWalker(RevisionsWalker): class BFSRevisionsWalker(RevisionsWalker):
""" """
Revisions walker that returns revisions in the same order Revisions walker that returns revisions in the same order
as when performing a breadth-first search on the revisions as when performing a breadth-first search on the revisions
DAG. DAG.
""" """
rw_type = "bfs" rw_type = "bfs"
def __init__(self, *args, **kwargs): def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs) super().__init__(*args, **kwargs)
self._revs_to_visit = deque(self._revs_to_visit) self._state.revs_to_visit = deque(self._state.revs_to_visit) # type: ignore
def process_rev(self, rev_id): def process_rev(self, rev_id: Sha1Git) -> None:
""" """
Append the revision to a queue. Append the revision to a queue.
Args: Args:
rev_id (bytes): the newly visited revision identifier rev_id (bytes): the newly visited revision identifier
""" """
if rev_id not in self._done: if rev_id not in self._state.done:
self._revs_to_visit.append(rev_id) self._state.revs_to_visit.append(rev_id)
def get_next_rev_id(self): def get_next_rev_id(self) -> Sha1Git:
""" """
Return the next revision from the queue. Return the next revision from the queue.
Returns: Returns:
dict: A dict describing a revision as returned by dict: A dict describing a revision as returned by
:meth:`swh.storage.interface.StorageInterface.revision_get` :meth:`swh.storage.interface.StorageInterface.revision_get`
""" """
return self._revs_to_visit.popleft() return self._state.revs_to_visit.popleft() # type: ignore # type: ignore
olasdUnsubmitted
Not Done
Inline Actions

doubleplustype: ignore for more effect

olasd: doubleplustype: ignore for more effect
class DFSPostRevisionsWalker(RevisionsWalker): class DFSPostRevisionsWalker(RevisionsWalker):
""" """
Revisions walker that returns revisions in the same order Revisions walker that returns revisions in the same order
as when performing a depth-first search in post-order on the as when performing a depth-first search in post-order on the
revisions DAG (i.e. after visiting a merge commit, revisions DAG (i.e. after visiting a merge commit,
the merged commit will be visited before the base it was the merged commit will be visited before the base it was
merged on). merged on).
""" """
rw_type = "dfs_post" rw_type = "dfs_post"
def process_rev(self, rev_id): def process_rev(self, rev_id: Sha1Git) -> None:
""" """
Append the revision to a stack. Append the revision to a stack.
Args: Args:
rev_id (bytes): the newly visited revision identifier rev_id (bytes): the newly visited revision identifier
""" """
if rev_id not in self._done: if rev_id not in self._state.done:
self._revs_to_visit.append(rev_id) self._state.revs_to_visit.append(rev_id)
def get_next_rev_id(self): def get_next_rev_id(self) -> Sha1Git:
""" """
Return the next revision from the stack. Return the next revision from the stack.
Returns: Returns:
dict: A dict describing a revision as returned by dict: A dict describing a revision as returned by
:meth:`swh.storage.interface.StorageInterface.revision_get` :meth:`swh.storage.interface.StorageInterface.revision_get`
""" """
return self._revs_to_visit.pop() return self._state.revs_to_visit.pop()
class DFSRevisionsWalker(DFSPostRevisionsWalker): class DFSRevisionsWalker(DFSPostRevisionsWalker):
""" """
Revisions walker that returns revisions in the same order Revisions walker that returns revisions in the same order
as when performing a depth-first search in pre-order on the as when performing a depth-first search in pre-order on the
revisions DAG (i.e. after visiting a merge commit, revisions DAG (i.e. after visiting a merge commit,
the base commit it was merged on will be visited before the base commit it was merged on will be visited before
the merged commit). the merged commit).
""" """
rw_type = "dfs" rw_type = "dfs"
def process_parent_revs(self, rev): def process_parent_revs(self, rev: Dict) -> None:
""" """
Process the parents of a revision when it is iterated in Process the parents of a revision when it is iterated in
the reversed order they are declared. the reversed order they are declared.
Args: Args:
rev (dict): A dict describing a revision as returned by rev (dict): A dict describing a revision as returned by
:meth:`swh.storage.interface.StorageInterface.revision_get` :meth:`swh.storage.interface.StorageInterface.revision_get`
""" """
▲ Show 20 Lines • Show All 208 Lines • Show Last 20 Lines
About · Developer information · Legal