Changeset View
Changeset View
Standalone View
Standalone View
swh/scheduler/backend.py
| # Copyright (C) 2015-2020 The Software Heritage developers | # Copyright (C) 2015-2021 The Software Heritage developers | ||||
| # See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
| # License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
| # See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
| import json | import json | ||||
| import logging | import logging | ||||
| from typing import Any, Dict, Iterable, List, Optional, Tuple, Union | from typing import Any, Dict, Iterable, List, Optional, Tuple, Union | ||||
| from uuid import UUID | from uuid import UUID | ||||
| import attr | import attr | ||||
| import psycopg2.extras | import psycopg2.extras | ||||
| import psycopg2.pool | import psycopg2.pool | ||||
| from swh.core.db import BaseDb | from swh.core.db import BaseDb | ||||
| from swh.core.db.common import db_transaction | from swh.core.db.common import db_transaction | ||||
| from swh.scheduler.utils import utcnow | from swh.scheduler.utils import utcnow | ||||
| from .exc import StaleData | from .exc import StaleData | ||||
| from .model import ( | from .model import ( | ||||
| ListedOrigin, | ListedOrigin, | ||||
| ListedOriginPageToken, | ListedOriginPageToken, | ||||
| Lister, | Lister, | ||||
| OriginVisitInfo, | |||||
| PaginatedListedOriginList, | PaginatedListedOriginList, | ||||
| ) | ) | ||||
| logger = logging.getLogger(__name__) | logger = logging.getLogger(__name__) | ||||
| psycopg2.extensions.register_adapter(dict, psycopg2.extras.Json) | psycopg2.extensions.register_adapter(dict, psycopg2.extras.Json) | ||||
| psycopg2.extras.register_uuid() | psycopg2.extras.register_uuid() | ||||
| ▲ Show 20 Lines • Show All 688 Lines • ▼ Show 20 Lines | def get_task_runs(self, task_ids, limit=None, db=None, cur=None): | ||||
| args.append(limit) | args.append(limit) | ||||
| cur.execute(query, args) | cur.execute(query, args) | ||||
| return cur.fetchall() | return cur.fetchall() | ||||
| @db_transaction() | @db_transaction() | ||||
| def get_priority_ratios(self, db=None, cur=None): | def get_priority_ratios(self, db=None, cur=None): | ||||
| cur.execute("select id, ratio from priority_ratio") | cur.execute("select id, ratio from priority_ratio") | ||||
| return {row["id"]: row["ratio"] for row in cur.fetchall()} | return {row["id"]: row["ratio"] for row in cur.fetchall()} | ||||
| @db_transaction() | |||||
| def origin_visit_info_upsert( | |||||
| self, visit_info: OriginVisitInfo, db=None, cur=None | |||||
| ) -> None: | |||||
| query = """INSERT into origin_visit_info as ovi (url, visit_type, eventful, uneventful, failed) | |||||
| VALUES (%s, %s, %s, %s, %s) | |||||
| ON CONFLICT (url, visit_type) DO UPDATE | |||||
| SET eventful = coalesce(excluded.eventful, ovi.eventful), | |||||
| uneventful = coalesce(excluded.uneventful, ovi.uneventful), | |||||
| failed = coalesce(excluded.failed, ovi.failed) | |||||
| """ | |||||
vlorentz: these lines are too long, could you dedent them?
(I also remove duplicate spaces and upper… | |||||
| cur.execute( | |||||
| query, | |||||
| ( | |||||
| visit_info.url, | |||||
| visit_info.visit_type, | |||||
| visit_info.eventful, | |||||
| visit_info.uneventful, | |||||
| visit_info.failed, | |||||
| ), | |||||
| ) | |||||
| @db_transaction() | |||||
| def origin_visit_info_get( | |||||
| self, url: str, visit_type: str, db=None, cur=None | |||||
| ) -> Optional[OriginVisitInfo]: | |||||
| query = format_query( | |||||
| "select {keys} from origin_visit_info where url=%s and visit_type=%s", | |||||
| OriginVisitInfo.select_columns(), | |||||
| ) | |||||
| cur.execute(query, (url, visit_type)) | |||||
| row = cur.fetchone() | |||||
| if row: | |||||
| return OriginVisitInfo(**row) | |||||
| else: | |||||
| return None | |||||
these lines are too long, could you dedent them?
(I also remove duplicate spaces and upper-cased as in the suggested change)