Changeset View
Changeset View
Standalone View
Standalone View
swh/scheduler/backend.py
| # Copyright (C) 2015-2020 The Software Heritage developers | # Copyright (C) 2015-2021 The Software Heritage developers | ||||
| # See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
| # License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
| # See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
| import json | import json | ||||
| import logging | import logging | ||||
| from typing import Any, Dict, Iterable, List, Optional, Tuple, Union | from typing import Any, Dict, Iterable, List, Optional, Tuple, Union | ||||
| from uuid import UUID | from uuid import UUID | ||||
| import attr | import attr | ||||
| import psycopg2.extras | import psycopg2.extras | ||||
| import psycopg2.pool | import psycopg2.pool | ||||
| from swh.core.db import BaseDb | from swh.core.db import BaseDb | ||||
| from swh.core.db.common import db_transaction | from swh.core.db.common import db_transaction | ||||
| from swh.scheduler.utils import utcnow | from swh.scheduler.utils import utcnow | ||||
| from .exc import StaleData, UnknownPolicy | from .exc import StaleData, UnknownPolicy | ||||
| from .model import ( | from .model import ( | ||||
| ListedOrigin, | ListedOrigin, | ||||
| ListedOriginPageToken, | ListedOriginPageToken, | ||||
| Lister, | Lister, | ||||
| OriginVisitStats, | |||||
| PaginatedListedOriginList, | PaginatedListedOriginList, | ||||
| ) | ) | ||||
| logger = logging.getLogger(__name__) | logger = logging.getLogger(__name__) | ||||
| psycopg2.extensions.register_adapter(dict, psycopg2.extras.Json) | psycopg2.extensions.register_adapter(dict, psycopg2.extras.Json) | ||||
| psycopg2.extras.register_uuid() | psycopg2.extras.register_uuid() | ||||
| ▲ Show 20 Lines • Show All 720 Lines • ▼ Show 20 Lines | def get_task_runs(self, task_ids, limit=None, db=None, cur=None): | ||||
| args.append(limit) | args.append(limit) | ||||
| cur.execute(query, args) | cur.execute(query, args) | ||||
| return cur.fetchall() | return cur.fetchall() | ||||
| @db_transaction() | @db_transaction() | ||||
| def get_priority_ratios(self, db=None, cur=None): | def get_priority_ratios(self, db=None, cur=None): | ||||
| cur.execute("select id, ratio from priority_ratio") | cur.execute("select id, ratio from priority_ratio") | ||||
| return {row["id"]: row["ratio"] for row in cur.fetchall()} | return {row["id"]: row["ratio"] for row in cur.fetchall()} | ||||
| @db_transaction() | |||||
| def origin_visit_stats_upsert( | |||||
| self, visit_stats: OriginVisitStats, db=None, cur=None | |||||
| ) -> None: | |||||
| query = """ | |||||
| INSERT into origin_visit_stats AS ovi ( | |||||
| url, | |||||
| visit_type, | |||||
| last_eventful, | |||||
| last_uneventful, | |||||
vlorentz: these lines are too long, could you dedent them?
(I also remove duplicate spaces and upper… | |||||
| last_failed | |||||
| ) | |||||
| VALUES (%s, %s, %s, %s, %s) ON CONFLICT (url, visit_type) DO | |||||
| UPDATE | |||||
| SET last_eventful = coalesce( | |||||
| excluded.last_eventful, | |||||
| ovi.last_eventful | |||||
| ), | |||||
| last_uneventful = coalesce( | |||||
| excluded.last_uneventful, | |||||
| ovi.last_uneventful | |||||
| ), | |||||
| last_failed = coalesce( | |||||
| excluded.last_failed, | |||||
| ovi.last_failed | |||||
| ) | |||||
| """ | |||||
| cur.execute( | |||||
| query, | |||||
| ( | |||||
| visit_stats.url, | |||||
| visit_stats.visit_type, | |||||
| visit_stats.last_eventful, | |||||
| visit_stats.last_uneventful, | |||||
| visit_stats.last_failed, | |||||
| ), | |||||
| ) | |||||
| @db_transaction() | |||||
| def origin_visit_stats_get( | |||||
| self, url: str, visit_type: str, db=None, cur=None | |||||
| ) -> Optional[OriginVisitStats]: | |||||
| query = format_query( | |||||
| "SELECT {keys} FROM origin_visit_stats WHERE url=%s AND visit_type=%s", | |||||
| OriginVisitStats.select_columns(), | |||||
| ) | |||||
| cur.execute(query, (url, visit_type)) | |||||
| row = cur.fetchone() | |||||
| if row: | |||||
| return OriginVisitStats(**row) | |||||
| else: | |||||
| return None | |||||
these lines are too long, could you dedent them?
(I also remove duplicate spaces and upper-cased as in the suggested change)