Changeset View
Changeset View
Standalone View
Standalone View
swh/scheduler/backend.py
# Copyright (C) 2015-2020 The Software Heritage developers | # Copyright (C) 2015-2021 The Software Heritage developers | ||||
# See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
# License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
# See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
import json | import json | ||||
import logging | import logging | ||||
from typing import Any, Dict, Iterable, List, Optional, Tuple, Union | from typing import Any, Dict, Iterable, List, Optional, Tuple, Union | ||||
from uuid import UUID | from uuid import UUID | ||||
import attr | import attr | ||||
import psycopg2.extras | import psycopg2.extras | ||||
import psycopg2.pool | import psycopg2.pool | ||||
from swh.core.db import BaseDb | from swh.core.db import BaseDb | ||||
from swh.core.db.common import db_transaction | from swh.core.db.common import db_transaction | ||||
from swh.scheduler.utils import utcnow | from swh.scheduler.utils import utcnow | ||||
from .exc import StaleData, UnknownPolicy | from .exc import StaleData, UnknownPolicy | ||||
from .model import ( | from .model import ( | ||||
ListedOrigin, | ListedOrigin, | ||||
ListedOriginPageToken, | ListedOriginPageToken, | ||||
Lister, | Lister, | ||||
OriginVisitStats, | |||||
PaginatedListedOriginList, | PaginatedListedOriginList, | ||||
) | ) | ||||
logger = logging.getLogger(__name__) | logger = logging.getLogger(__name__) | ||||
psycopg2.extensions.register_adapter(dict, psycopg2.extras.Json) | psycopg2.extensions.register_adapter(dict, psycopg2.extras.Json) | ||||
psycopg2.extras.register_uuid() | psycopg2.extras.register_uuid() | ||||
▲ Show 20 Lines • Show All 720 Lines • ▼ Show 20 Lines | def get_task_runs(self, task_ids, limit=None, db=None, cur=None): | ||||
args.append(limit) | args.append(limit) | ||||
cur.execute(query, args) | cur.execute(query, args) | ||||
return cur.fetchall() | return cur.fetchall() | ||||
@db_transaction() | @db_transaction() | ||||
def get_priority_ratios(self, db=None, cur=None): | def get_priority_ratios(self, db=None, cur=None): | ||||
cur.execute("select id, ratio from priority_ratio") | cur.execute("select id, ratio from priority_ratio") | ||||
return {row["id"]: row["ratio"] for row in cur.fetchall()} | return {row["id"]: row["ratio"] for row in cur.fetchall()} | ||||
@db_transaction() | |||||
def origin_visit_stats_upsert( | |||||
self, visit_stats: OriginVisitStats, db=None, cur=None | |||||
) -> None: | |||||
query = """ | |||||
INSERT into origin_visit_stats AS ovi ( | |||||
url, | |||||
visit_type, | |||||
last_eventful, | |||||
last_uneventful, | |||||
vlorentz: these lines are too long, could you dedent them?
(I also remove duplicate spaces and upper… | |||||
last_failed | |||||
) | |||||
VALUES (%s, %s, %s, %s, %s) ON CONFLICT (url, visit_type) DO | |||||
UPDATE | |||||
SET last_eventful = coalesce( | |||||
excluded.last_eventful, | |||||
ovi.last_eventful | |||||
), | |||||
last_uneventful = coalesce( | |||||
excluded.last_uneventful, | |||||
ovi.last_uneventful | |||||
), | |||||
last_failed = coalesce( | |||||
excluded.last_failed, | |||||
ovi.last_failed | |||||
) | |||||
""" | |||||
cur.execute( | |||||
query, | |||||
( | |||||
visit_stats.url, | |||||
visit_stats.visit_type, | |||||
visit_stats.last_eventful, | |||||
visit_stats.last_uneventful, | |||||
visit_stats.last_failed, | |||||
), | |||||
) | |||||
@db_transaction() | |||||
def origin_visit_stats_get( | |||||
self, url: str, visit_type: str, db=None, cur=None | |||||
) -> Optional[OriginVisitStats]: | |||||
query = format_query( | |||||
"SELECT {keys} FROM origin_visit_stats WHERE url=%s AND visit_type=%s", | |||||
OriginVisitStats.select_columns(), | |||||
) | |||||
cur.execute(query, (url, visit_type)) | |||||
row = cur.fetchone() | |||||
if row: | |||||
return OriginVisitStats(**row) | |||||
else: | |||||
return None |
these lines are too long, could you dedent them?
(I also remove duplicate spaces and upper-cased as in the suggested change)