Changeset View
Changeset View
Standalone View
Standalone View
swh/vault/backend.py
# Copyright (C) 2017-2018 The Software Heritage developers | # Copyright (C) 2017-2020 The Software Heritage developers | ||||
# See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
# License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
# See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
from email.mime.text import MIMEText | from email.mime.text import MIMEText | ||||
import smtplib | import smtplib | ||||
import psycopg2.extras | import psycopg2.extras | ||||
import psycopg2.pool | import psycopg2.pool | ||||
from swh.core.db import BaseDb | from swh.core.db import BaseDb | ||||
from swh.core.db.common import db_transaction | from swh.core.db.common import db_transaction | ||||
from swh.model import hashutil | from swh.model import hashutil | ||||
from swh.scheduler import get_scheduler | |||||
from swh.scheduler.utils import create_oneshot_task_dict | from swh.scheduler.utils import create_oneshot_task_dict | ||||
from swh.storage import get_storage | |||||
from swh.vault.cache import VaultCache | |||||
from swh.vault.cookers import get_cooker_cls | from swh.vault.cookers import get_cooker_cls | ||||
from swh.vault.exc import NotFoundExc | from swh.vault.exc import NotFoundExc | ||||
cooking_task_name = "swh.vault.cooking_tasks.SWHCookingTask" | cooking_task_name = "swh.vault.cooking_tasks.SWHCookingTask" | ||||
NOTIF_EMAIL_FROM = '"Software Heritage Vault" ' "<bot@softwareheritage.org>" | NOTIF_EMAIL_FROM = '"Software Heritage Vault" ' "<bot@softwareheritage.org>" | ||||
NOTIF_EMAIL_SUBJECT_SUCCESS = "Bundle ready: {obj_type} {short_id}" | NOTIF_EMAIL_SUBJECT_SUCCESS = "Bundle ready: {obj_type} {short_id}" | ||||
NOTIF_EMAIL_SUBJECT_FAILURE = "Bundle failed: {obj_type} {short_id}" | NOTIF_EMAIL_SUBJECT_FAILURE = "Bundle failed: {obj_type} {short_id}" | ||||
Show All 38 Lines | def batch_to_bytes(batch): | ||||
return [(obj_type, hashutil.hash_to_bytes(obj_id)) for obj_type, obj_id in batch] | return [(obj_type, hashutil.hash_to_bytes(obj_id)) for obj_type, obj_id in batch] | ||||
class VaultBackend: | class VaultBackend: | ||||
""" | """ | ||||
Backend for the Software Heritage vault. | Backend for the Software Heritage vault. | ||||
""" | """ | ||||
def __init__(self, db, cache, scheduler, storage=None, **config): | def __init__(self, db, **config): | ||||
self.config = config | self.config = config | ||||
self.cache = cache | self.cache = VaultCache(**config["cache"]) | ||||
self.scheduler = scheduler | self.scheduler = get_scheduler(**config["scheduler"]) | ||||
self.storage = storage | self.storage = get_storage(**config["storage"]) | ||||
self.smtp_server = smtplib.SMTP() | self.smtp_server = smtplib.SMTP() | ||||
self._pool = psycopg2.pool.ThreadedConnectionPool( | self._pool = psycopg2.pool.ThreadedConnectionPool( | ||||
config.get("min_pool_conns", 1), | config.get("min_pool_conns", 1), | ||||
config.get("max_pool_conns", 10), | config.get("max_pool_conns", 10), | ||||
db, | db, | ||||
cursor_factory=psycopg2.extras.RealDictCursor, | cursor_factory=psycopg2.extras.RealDictCursor, | ||||
) | ) | ||||
▲ Show 20 Lines • Show All 405 Lines • Show Last 20 Lines |