Changeset View
Changeset View
Standalone View
Standalone View
swh/vault/cookers/base.py
| # Copyright (C) 2016-2018 The Software Heritage developers | # Copyright (C) 2016-2018 The Software Heritage developers | ||||
| # See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
| # License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
| # See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
| import abc | import abc | ||||
| import io | import io | ||||
| import logging | import logging | ||||
| import traceback | |||||
| from typing import ClassVar, Set | from typing import ClassVar, Set | ||||
| from psycopg2.extensions import QueryCanceledError | from psycopg2.extensions import QueryCanceledError | ||||
| from swh.model.identifiers import CoreSWHID, ObjectType | from swh.model.identifiers import CoreSWHID, ObjectType | ||||
| from swh.storage.interface import StorageInterface | from swh.storage.interface import StorageInterface | ||||
| MAX_BUNDLE_SIZE = 2 ** 29 # 512 MiB | MAX_BUNDLE_SIZE = 2 ** 29 # 512 MiB | ||||
| ▲ Show 20 Lines • Show All 119 Lines • ▼ Show 20 Lines | def cook(self): | ||||
| # TODO: use proper content streaming instead of put_bundle() | # TODO: use proper content streaming instead of put_bundle() | ||||
| self.backend.put_bundle(self.cache_type_key(), self.swhid, bundle) | self.backend.put_bundle(self.cache_type_key(), self.swhid, bundle) | ||||
| except PolicyError as e: | except PolicyError as e: | ||||
| logging.info("Bundle cooking violated policy: %s", e) | logging.info("Bundle cooking violated policy: %s", e) | ||||
| self.backend.set_status(self.BUNDLE_TYPE, self.swhid, "failed") | self.backend.set_status(self.BUNDLE_TYPE, self.swhid, "failed") | ||||
| self.backend.set_progress(self.BUNDLE_TYPE, self.swhid, str(e)) | self.backend.set_progress(self.BUNDLE_TYPE, self.swhid, str(e)) | ||||
| except Exception: | except Exception: | ||||
| self.backend.set_status(self.BUNDLE_TYPE, self.swhid, "failed") | self.backend.set_status(self.BUNDLE_TYPE, self.swhid, "failed") | ||||
| tb = traceback.format_exc() | |||||
| self.backend.set_progress( | self.backend.set_progress( | ||||
| self.BUNDLE_TYPE, | self.BUNDLE_TYPE, | ||||
| self.swhid, | self.swhid, | ||||
| "Internal Server Error. This incident will be reported.", | f"Internal Server Error. This incident will be reported.\n" | ||||
| f"The full error was: \n\n{tb}", | |||||
| ) | ) | ||||
| logging.exception("Bundle cooking failed.") | logging.exception("Bundle cooking failed.") | ||||
| else: | else: | ||||
| self.backend.set_status(self.BUNDLE_TYPE, self.swhid, "done") | self.backend.set_status(self.BUNDLE_TYPE, self.swhid, "done") | ||||
| self.backend.set_progress(self.BUNDLE_TYPE, self.swhid, None) | self.backend.set_progress(self.BUNDLE_TYPE, self.swhid, None) | ||||
| finally: | finally: | ||||
| self.backend.send_notif(self.BUNDLE_TYPE, self.swhid) | self.backend.send_notif(self.BUNDLE_TYPE, self.swhid) | ||||