Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/archiver/director.py
| # Copyright (C) 2015 The Software Heritage developers | # Copyright (C) 2015 The Software Heritage developers | ||||
| # See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
| # License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
| # See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
| import swh | import swh | ||||
| import logging | import logging | ||||
| import click | |||||
| from datetime import datetime | from datetime import datetime | ||||
| from swh.core import hashutil | from swh.core import hashutil, config | ||||
| from swh.scheduler.celery_backend.config import app | from swh.scheduler.celery_backend.config import app | ||||
| from . import tasks # NOQA | from . import tasks # NOQA | ||||
| DEFAULT_CONFIG = { | DEFAULT_CONFIG = { | ||||
| 'objstorage_path': '/tmp/swh-storage/objects', | 'objstorage_path': ('str', '/tmp/swh-storage/objects'), | ||||
| 'batch_max_size': 50, | 'batch_max_size': ('int', 50), | ||||
| 'archival_max_age': 3600, | 'archival_max_age': ('int', 3600), | ||||
| 'retention_policy': 2, | 'retention_policy': ('int', 2), | ||||
| 'asynchronous': True, | 'asynchronous': ('bool', True), | ||||
| 'dbname': 'softwareheritage', | 'dbconn': ('str', 'dbname=softwareheritage-dev user=guest') | ||||
| 'user': 'root' | |||||
| } | } | ||||
| task_name = 'swh.storage.archiver.tasks.SWHArchiverTask' | task_name = 'swh.storage.archiver.tasks.SWHArchiverTask' | ||||
| logger = logging.getLogger() | logger = logging.getLogger() | ||||
| class ArchiverDirector(): | class ArchiverDirector(): | ||||
| ▲ Show 20 Lines • Show All 178 Lines • ▼ Show 20 Lines | def get_virtual_status(self, status, mtime): | ||||
| elapsed = (datetime.now() - mtime).total_seconds() | elapsed = (datetime.now() - mtime).total_seconds() | ||||
| if elapsed <= self.config['archival_max_age']: | if elapsed <= self.config['archival_max_age']: | ||||
| return 'present' | return 'present' | ||||
| else: | else: | ||||
| return 'missing' | return 'missing' | ||||
| else: | else: | ||||
| raise ValueError("status must be either 'present', 'missing' " | raise ValueError("status must be either 'present', 'missing' " | ||||
| "or 'ongoing'") | "or 'ongoing'") | ||||
| @click.command() | |||||
| @click.argument('config-path', required=1) | |||||
| @click.option('--dbconn', default=DEFAULT_CONFIG['dbconn'][1], | |||||
| help="Connection string for the database") | |||||
| @click.option('--async/--sync', default=DEFAULT_CONFIG['asynchronous'][1], | |||||
| help="Indicates if the archiver should run asynchronously") | |||||
| def launch(config_path, dbconn, async): | |||||
| # The configuration have following priority : | |||||
| # command line > file config > default config | |||||
| cl_config = { | |||||
| 'dbconn': ('str', dbconn), | |||||
| 'asynchronous': ('bool', async) | |||||
| } | |||||
| DEFAULT_CONFIG.update(cl_config) | |||||
| conf = config.read(config_path, DEFAULT_CONFIG) | |||||
| print(conf) | |||||
| # Create connection data and run the archiver. | |||||
| archiver = ArchiverDirector(conf['dbconn'], conf) | |||||
| logger.info("Starting an archival at", datetime.now()) | |||||
| archiver.run() | |||||
| if __name__ == '__main__': | |||||
| launch() | |||||