Changeset View
Changeset View
Standalone View
Standalone View
swh/vault/cookers/base.py
| # Copyright (C) 2016-2018 The Software Heritage developers | # Copyright (C) 2016-2018 The Software Heritage developers | ||||
| # See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
| # License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
| # See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
| import abc | import abc | ||||
| import io | import io | ||||
| import logging | import logging | ||||
| from psycopg2.extensions import QueryCanceledError | |||||
| from swh.model import hashutil | from swh.model import hashutil | ||||
| MAX_BUNDLE_SIZE = 2 ** 29 # 512 MiB | MAX_BUNDLE_SIZE = 2 ** 29 # 512 MiB | ||||
| DEFAULT_CONFIG_PATH = 'vault/cooker' | DEFAULT_CONFIG_PATH = 'vault/cooker' | ||||
| DEFAULT_CONFIG = { | DEFAULT_CONFIG = { | ||||
| 'storage': ('dict', { | 'storage': ('dict', { | ||||
| 'cls': 'remote', | 'cls': 'remote', | ||||
| ▲ Show 20 Lines • Show All 89 Lines • ▼ Show 20 Lines | class BaseVaultCooker(metaclass=abc.ABCMeta): | ||||
| def cook(self): | def cook(self): | ||||
| """Cook the requested object into a bundle | """Cook the requested object into a bundle | ||||
| """ | """ | ||||
| self.backend.set_status(self.obj_type, self.obj_id, 'pending') | self.backend.set_status(self.obj_type, self.obj_id, 'pending') | ||||
| self.backend.set_progress(self.obj_type, self.obj_id, 'Processing...') | self.backend.set_progress(self.obj_type, self.obj_id, 'Processing...') | ||||
| self.fileobj = BytesIOBundleSizeLimit(size_limit=self.max_bundle_size) | self.fileobj = BytesIOBundleSizeLimit(size_limit=self.max_bundle_size) | ||||
| try: | try: | ||||
| try: | |||||
| self.prepare_bundle() | self.prepare_bundle() | ||||
| except QueryCanceledError: | |||||
| raise PolicyError( | |||||
| "Timeout reached while assembling the requested bundle") | |||||
| bundle = self.fileobj.getvalue() | bundle = self.fileobj.getvalue() | ||||
| # TODO: use proper content streaming instead of put_bundle() | # TODO: use proper content streaming instead of put_bundle() | ||||
| self.backend.put_bundle(self.CACHE_TYPE_KEY, self.obj_id, bundle) | self.backend.put_bundle(self.CACHE_TYPE_KEY, self.obj_id, bundle) | ||||
| except PolicyError as e: | except PolicyError as e: | ||||
| self.backend.set_status(self.obj_type, self.obj_id, 'failed') | self.backend.set_status(self.obj_type, self.obj_id, 'failed') | ||||
| self.backend.set_progress(self.obj_type, self.obj_id, str(e)) | self.backend.set_progress(self.obj_type, self.obj_id, str(e)) | ||||
| except Exception: | except Exception: | ||||
| self.backend.set_status(self.obj_type, self.obj_id, 'failed') | self.backend.set_status(self.obj_type, self.obj_id, 'failed') | ||||
| Show All 9 Lines | |||||