diff --git a/data/defaults.yaml b/data/defaults.yaml --- a/data/defaults.yaml +++ b/data/defaults.yaml @@ -1459,7 +1459,7 @@ - origin - origin_visit -swh::deploy::archiver_content_updater::conf_file: "%{hiera('swh::deploy::worker::swh_storage_archiver::conf_directory')}/content_updater.yml" +swh::deploy::archiver_content_updater::conf_file: "%{hiera('swh::deploy::worker::storage_archiver::conf_directory')}/content_updater.yml" swh::deploy::archiver_content_updater::user: swhstorage swh::deploy::archiver_content_updater::group: swhstorage swh::deploy::archiver_content_updater::config: @@ -1473,7 +1473,7 @@ archiver_storage: cls: db args: - dbconn: "host=%{hiera('swh::deploy::worker::swh_storage_archiver::db::host')} user=%{hiera('swh::deploy::worker::swh_storage_archiver::db::user')} dbname=%{hiera('swh::deploy::worker::swh_storage_archiver::db::dbname')} password=%{hiera('swh::deploy::worker::swh_storage_archiver::db::password')}" + dbconn: "host=%{hiera('swh::deploy::worker::storage_archiver::db::host')} user=%{hiera('swh::deploy::worker::storage_archiver::db::user')} dbname=%{hiera('swh::deploy::worker::storage_archiver::db::dbname')} password=%{hiera('swh::deploy::worker::storage_archiver::db::password')}" sources_present: - uffizi @@ -1543,12 +1543,12 @@ password: "%{hiera('swh::deploy::storage::db::password')}" media_root: "%{hiera('swh::deploy::deposit::media_root_directory')}" -swh::deploy::worker::swh_loader_deposit::config_file: "%{hiera('swh::conf_directory')}/loader_deposit.yml" -swh::deploy::worker::swh_loader_deposit::concurrency: 2 -swh::deploy::worker::swh_loader_deposit::private_tmp: true -swh::deploy::worker::swh_loader_deposit::loglevel: info +swh::deploy::worker::loader_deposit::config_file: "%{hiera('swh::conf_directory')}/loader_deposit.yml" +swh::deploy::worker::loader_deposit::concurrency: 2 +swh::deploy::worker::loader_deposit::private_tmp: true +swh::deploy::worker::loader_deposit::loglevel: info # deposit_basic_auth_swhworker_{username|password} in private_data -swh::deploy::worker::swh_loader_deposit::config: +swh::deploy::worker::loader_deposit::config: storage: "%{alias('swh::remote_service::storage::config')}" extraction_dir: /tmp/swh.loader.deposit/ celery: @@ -1607,10 +1607,10 @@ swh::deploy::objstorage_archive_notifier_checker::group: "%{hiera('swh::deploy::objstorage::group')}" swh::deploy::objstorage_archive_notifier_checker::log_tag: objstorage.checker.archive_notifier swh::deploy::objstorage_archive_notifier_checker::storage_name: this-must-be-overriden-on-a-host-basis -swh::deploy::objstorage_archive_notifier_checker::db::host: "%{hiera('swh::deploy::worker::swh_storage_archiver::db::host')}" -swh::deploy::objstorage_archive_notifier_checker::db::dbname: "%{hiera('swh::deploy::worker::swh_storage_archiver::db::dbname')}" -swh::deploy::objstorage_archive_notifier_checker::db::user: "%{hiera('swh::deploy::worker::swh_storage_archiver::db::user')}" -swh::deploy::objstorage_archive_notifier_checker::db::password: "%{hiera('swh::deploy::worker::swh_storage_archiver::db::password')}" +swh::deploy::objstorage_archive_notifier_checker::db::host: "%{hiera('swh::deploy::worker::storage_archiver::db::host')}" +swh::deploy::objstorage_archive_notifier_checker::db::dbname: "%{hiera('swh::deploy::worker::storage_archiver::db::dbname')}" +swh::deploy::objstorage_archive_notifier_checker::db::user: "%{hiera('swh::deploy::worker::storage_archiver::db::user')}" +swh::deploy::objstorage_archive_notifier_checker::db::password: "%{hiera('swh::deploy::worker::storage_archiver::db::password')}" swh::deploy::objstorage_archive_notifier_checker::config: storage: cls: pathslicing @@ -1623,7 +1623,7 @@ archiver_storage: cls: db args: - dbconn: "host=%{hiera('swh::deploy::worker::swh_storage_archiver::db::host')} user=%{hiera('swh::deploy::worker::swh_storage_archiver::db::user')} dbname=%{hiera('swh::deploy::worker::swh_storage_archiver::db::dbname')} password=%{hiera('swh::deploy::worker::swh_storage_archiver::db::password')}" + dbconn: "host=%{hiera('swh::deploy::worker::storage_archiver::db::host')} user=%{hiera('swh::deploy::worker::storage_archiver::db::user')} dbname=%{hiera('swh::deploy::worker::storage_archiver::db::dbname')} password=%{hiera('swh::deploy::worker::storage_archiver::db::password')}" swh::deploy::webapp::backported_packages: stretch: @@ -1859,15 +1859,15 @@ # swh::deploy::worker::task_broker::password in private data swh::deploy::worker::task_broker: "amqp://swhconsumer:%{hiera('swh::deploy::worker::task_broker::password')}@rabbitmq:5672//" swh::deploy::worker::instances: - - swh_loader_debian - - swh_loader_git - - swh_lister_debian - - swh_lister_github - -swh::deploy::worker::swh_loader_git::config_file: "%{hiera('swh::conf_directory')}/loader_git.yml" -swh::deploy::worker::swh_loader_git::concurrency: 2 -swh::deploy::worker::swh_loader_git::loglevel: info -swh::deploy::worker::swh_loader_git::config: + - loader_debian + - loader_git + - lister_debian + - lister_github + +swh::deploy::worker::loader_git::config_file: "%{hiera('swh::conf_directory')}/loader_git.yml" +swh::deploy::worker::loader_git::concurrency: 2 +swh::deploy::worker::loader_git::loglevel: info +swh::deploy::worker::loader_git::config: storage: "%{alias('swh::remote_service::storage::config')}" save_data: true save_data_path: /srv/storage/space/data/sharded_packfiles @@ -1883,10 +1883,10 @@ - swh.loader.git.tasks.LoadDiskGitRepository - swh.loader.git.tasks.UncompressAndLoadDiskGitRepository -swh::deploy::worker::swh_loader_debian::config_file: "%{hiera('swh::conf_directory')}/loader_debian.yml" -swh::deploy::worker::swh_loader_debian::concurrency: 1 -swh::deploy::worker::swh_loader_debian::loglevel: info -swh::deploy::worker::swh_loader_debian::config: +swh::deploy::worker::loader_debian::config_file: "%{hiera('swh::conf_directory')}/loader_debian.yml" +swh::deploy::worker::loader_debian::concurrency: 1 +swh::deploy::worker::loader_debian::loglevel: info +swh::deploy::worker::loader_debian::config: storage: "%{alias('swh::remote_service::storage::config')}" directory_packet_size: 100 lister_db_url: "postgresql://%{hiera('swh::deploy::loader_debian::db::user')}:%{hiera('swh::deploy::loader_debian::db::password')}@%{hiera('swh::deploy::loader_debian::db::host')}:%{hiera('swh::deploy::loader_debian::db::port')}/%{hiera('swh::deploy::loader_debian::db::dbname')}" @@ -1899,10 +1899,10 @@ - swh_loader_debian -swh::deploy::worker::swh_lister_debian::config_file: "%{hiera('swh::conf_directory')}/lister_debian.yml" -swh::deploy::worker::swh_lister_debian::concurrency: 1 -swh::deploy::worker::swh_lister_debian::loglevel: info -swh::deploy::worker::swh_lister_debian::config: +swh::deploy::worker::lister_debian::config_file: "%{hiera('swh::conf_directory')}/lister_debian.yml" +swh::deploy::worker::lister_debian::concurrency: 1 +swh::deploy::worker::lister_debian::loglevel: info +swh::deploy::worker::lister_debian::config: storage: "%{alias('swh::remote_service::storage::config')}" scheduler: "%{alias('swh::remote_service::scheduler::config::saatchi')}" lister: @@ -1917,11 +1917,11 @@ - swh.lister.debian.tasks.DebianListerTask - swh_lister_debian -swh::deploy::worker::swh_loader_mercurial::config_file: "%{hiera('swh::conf_directory')}/loader_mercurial.yml" -swh::deploy::worker::swh_loader_mercurial::concurrency: 1 -swh::deploy::worker::swh_loader_mercurial::private_tmp: true -swh::deploy::worker::swh_loader_mercurial::loglevel: info -swh::deploy::worker::swh_loader_mercurial::config: +swh::deploy::worker::loader_mercurial::config_file: "%{hiera('swh::conf_directory')}/loader_mercurial.yml" +swh::deploy::worker::loader_mercurial::concurrency: 1 +swh::deploy::worker::loader_mercurial::private_tmp: true +swh::deploy::worker::loader_mercurial::loglevel: info +swh::deploy::worker::loader_mercurial::config: storage: "%{alias('swh::remote_service::storage::config')}" reduce_effort: False celery: @@ -1934,11 +1934,11 @@ - swh_loader_mercurial - swh_loader_mercurial_archive -swh::deploy::worker::swh_loader_pypi::config_file: "%{hiera('swh::conf_directory')}/loader_pypi.yml" -swh::deploy::worker::swh_loader_pypi::concurrency: 1 -swh::deploy::worker::swh_loader_pypi::private_tmp: true -swh::deploy::worker::swh_loader_pypi::loglevel: info -swh::deploy::worker::swh_loader_pypi::config: +swh::deploy::worker::loader_pypi::config_file: "%{hiera('swh::conf_directory')}/loader_pypi.yml" +swh::deploy::worker::loader_pypi::concurrency: 1 +swh::deploy::worker::loader_pypi::private_tmp: true +swh::deploy::worker::loader_pypi::loglevel: info +swh::deploy::worker::loader_pypi::config: storage: "%{alias('swh::remote_service::storage::config')}" celery: task_broker: "%{alias('swh::deploy::worker::task_broker')}" @@ -1948,13 +1948,13 @@ - swh.loader.pypi.tasks.LoadPyPI - swh_loader_pypi -swh::deploy::worker::swh_loader_svn::config_file: "%{hiera('swh::conf_directory')}/loader_svn.yml" -swh::deploy::worker::swh_loader_svn::concurrency: 1 -swh::deploy::worker::swh_loader_svn::private_tmp: true -swh::deploy::worker::swh_loader_svn::limit_no_file: 8192 -swh::deploy::worker::swh_loader_svn::loglevel: info +swh::deploy::worker::loader_svn::config_file: "%{hiera('swh::conf_directory')}/loader_svn.yml" +swh::deploy::worker::loader_svn::concurrency: 1 +swh::deploy::worker::loader_svn::private_tmp: true +swh::deploy::worker::loader_svn::limit_no_file: 8192 +swh::deploy::worker::loader_svn::loglevel: info # Contains a password: in private data -swh::deploy::worker::swh_loader_svn::config: +swh::deploy::worker::loader_svn::config: storage: "%{alias('swh::remote_service::storage::config')}" celery: task_broker: "%{alias('swh::deploy::worker::task_broker')}" @@ -1967,21 +1967,21 @@ - swh_loader_svn - swh_loader_svn_mount_and_load -swh::deploy::worker::swh_lister_github::config_file: "%{hiera('swh::conf_directory')}/lister_github.yml" -swh::deploy::worker::swh_lister_github::concurrency: 1 -swh::deploy::worker::swh_lister_github::loglevel: warning +swh::deploy::worker::lister_github::config_file: "%{hiera('swh::conf_directory')}/lister_github.yml" +swh::deploy::worker::lister_github::concurrency: 1 +swh::deploy::worker::lister_github::loglevel: warning # Contains a password: in private data -swh::deploy::worker::swh_lister_github::db::host: db -swh::deploy::worker::swh_lister_github::db::dbname: lister-github -swh::deploy::worker::swh_lister_github::db::user: lister-github -#swh::deploy::worker::swh_lister_github::db::password in private data -swh::deploy::worker::swh_lister_github::config: +swh::deploy::worker::lister_github::db::host: db +swh::deploy::worker::lister_github::db::dbname: lister-github +swh::deploy::worker::lister_github::db::user: lister-github +#swh::deploy::worker::lister_github::db::password in private data +swh::deploy::worker::lister_github::config: storage: "%{alias('swh::remote_service::storage::config')}" scheduler: "%{alias('swh::remote_service::scheduler::config::saatchi')}" lister: cls: local args: - db: "postgresql://%{hiera('swh::deploy::worker::swh_lister_github::db::user')}:%{hiera('swh::deploy::worker::swh_lister_github::db::password')}@%{hiera('swh::deploy::worker::swh_lister_github::db::host')}/%{hiera('swh::deploy::worker::swh_lister_github::db::dbname')}" + db: "postgresql://%{hiera('swh::deploy::worker::lister_github::db::user')}:%{hiera('swh::deploy::worker::lister_github::db::password')}@%{hiera('swh::deploy::worker::lister_github::db::host')}/%{hiera('swh::deploy::worker::lister_github::db::dbname')}" # credentials in private data celery: task_broker: "%{alias('swh::deploy::worker::task_broker')}" @@ -1994,21 +1994,21 @@ - swh_lister_github_discover - swh_lister_github_refresh -swh::deploy::worker::swh_lister_gitlab::config_file: "%{hiera('swh::conf_directory')}/lister_gitlab.yml" -swh::deploy::worker::swh_lister_gitlab::concurrency: 1 -swh::deploy::worker::swh_lister_gitlab::loglevel: warning +swh::deploy::worker::lister_gitlab::config_file: "%{hiera('swh::conf_directory')}/lister_gitlab.yml" +swh::deploy::worker::lister_gitlab::concurrency: 1 +swh::deploy::worker::lister_gitlab::loglevel: warning # Contains a password: in private data -swh::deploy::worker::swh_lister_gitlab::db::host: db -swh::deploy::worker::swh_lister_gitlab::db::dbname: lister-gitlab -swh::deploy::worker::swh_lister_gitlab::db::user: lister-gitlab -#swh::deploy::worker::swh_lister_gitlab::db::password in private data -swh::deploy::worker::swh_lister_gitlab::config: +swh::deploy::worker::lister_gitlab::db::host: db +swh::deploy::worker::lister_gitlab::db::dbname: lister-gitlab +swh::deploy::worker::lister_gitlab::db::user: lister-gitlab +#swh::deploy::worker::lister_gitlab::db::password in private data +swh::deploy::worker::lister_gitlab::config: storage: "%{alias('swh::remote_service::storage::config')}" scheduler: "%{alias('swh::remote_service::scheduler::config::saatchi')}" lister: cls: local args: - db: "postgresql://%{hiera('swh::deploy::worker::swh_lister_gitlab::db::user')}:%{hiera('swh::deploy::worker::swh_lister_gitlab::db::password')}@%{hiera('swh::deploy::worker::swh_lister_gitlab::db::host')}/%{hiera('swh::deploy::worker::swh_lister_gitlab::db::dbname')}" + db: "postgresql://%{hiera('swh::deploy::worker::lister_gitlab::db::user')}:%{hiera('swh::deploy::worker::lister_gitlab::db::password')}@%{hiera('swh::deploy::worker::lister_gitlab::db::host')}/%{hiera('swh::deploy::worker::lister_gitlab::db::dbname')}" # credentials in private data celery: task_broker: "%{alias('swh::deploy::worker::task_broker')}" @@ -2021,21 +2021,21 @@ - swh_lister_gitlab_discover - swh_lister_gitlab_refresh -swh::deploy::worker::swh_lister_pypi::config_file: "%{hiera('swh::conf_directory')}/lister_pypi.yml" -swh::deploy::worker::swh_lister_pypi::concurrency: 1 -swh::deploy::worker::swh_lister_pypi::loglevel: warning +swh::deploy::worker::lister_pypi::config_file: "%{hiera('swh::conf_directory')}/lister_pypi.yml" +swh::deploy::worker::lister_pypi::concurrency: 1 +swh::deploy::worker::lister_pypi::loglevel: warning # Contains a password: in private data -swh::deploy::worker::swh_lister_pypi::db::host: db -swh::deploy::worker::swh_lister_pypi::db::dbname: lister-pypi -swh::deploy::worker::swh_lister_pypi::db::user: lister-pypi -#swh::deploy::worker::swh_lister_pypi::db::password in private data -swh::deploy::worker::swh_lister_pypi::config: +swh::deploy::worker::lister_pypi::db::host: db +swh::deploy::worker::lister_pypi::db::dbname: lister-pypi +swh::deploy::worker::lister_pypi::db::user: lister-pypi +#swh::deploy::worker::lister_pypi::db::password in private data +swh::deploy::worker::lister_pypi::config: storage: "%{alias('swh::remote_service::storage::config')}" scheduler: "%{alias('swh::remote_service::scheduler::config::saatchi')}" lister: cls: local args: - db: "postgresql://%{hiera('swh::deploy::worker::swh_lister_pypi::db::user')}:%{hiera('swh::deploy::worker::swh_lister_pypi::db::password')}@%{hiera('swh::deploy::worker::swh_lister_pypi::db::host')}/%{hiera('swh::deploy::worker::swh_lister_pypi::db::dbname')}" + db: "postgresql://%{hiera('swh::deploy::worker::lister_pypi::db::user')}:%{hiera('swh::deploy::worker::lister_pypi::db::password')}@%{hiera('swh::deploy::worker::lister_pypi::db::host')}/%{hiera('swh::deploy::worker::lister_pypi::db::dbname')}" # credentials in private data celery: task_broker: "%{alias('swh::deploy::worker::task_broker')}" @@ -2045,28 +2045,27 @@ - swh.lister.pypi.tasks.PyPIListerTask - swh_lister_pypi_refresh -swh::deploy::worker::swh_storage_archiver::conf_directory: "%{hiera('swh::conf_directory')}/archiver" -swh::deploy::worker::swh_storage_archiver::conf_file: "%{hiera('swh::deploy::worker::swh_storage_archiver::conf_directory')}/worker.yml" -swh::deploy::worker::swh_storage_archiver::archival_max_age: 3600 -swh::deploy::worker::swh_storage_archiver::max_queue_length: 100000 -swh::deploy::worker::swh_storage_archiver::retention_policy: 3 -swh::deploy::worker::swh_storage_archiver::batch_max_size: 5 -swh::deploy::worker::swh_storage_archiver::db::host: db -swh::deploy::worker::swh_storage_archiver::db::dbname: softwareheritage-archiver -swh::deploy::worker::swh_storage_archiver::db::user: "%{hiera('swh::deploy::storage::db::user')}" -swh::deploy::worker::swh_storage_archiver::db::password: "%{hiera('swh::deploy::storage::db::password')}" -swh::deploy::worker::swh_storage_archiver::concurrency: 10 -swh::deploy::worker::swh_storage_archiver::max_tasks_per_child: 500 -swh::deploy::worker::swh_storage_archiver::loglevel: info +swh::deploy::worker::storage_archiver::config_file: "%{hiera('swh::conf_directory')}/storage_archiver.yml" +swh::deploy::worker::storage_archiver::archival_max_age: 3600 +swh::deploy::worker::storage_archiver::max_queue_length: 100000 +swh::deploy::worker::storage_archiver::retention_policy: 3 +swh::deploy::worker::storage_archiver::batch_max_size: 5 +swh::deploy::worker::storage_archiver::db::host: db +swh::deploy::worker::storage_archiver::db::dbname: softwareheritage-archiver +swh::deploy::worker::storage_archiver::db::user: "%{hiera('swh::deploy::storage::db::user')}" +swh::deploy::worker::storage_archiver::db::password: "%{hiera('swh::deploy::storage::db::password')}" +swh::deploy::worker::storage_archiver::concurrency: 10 +swh::deploy::worker::storage_archiver::max_tasks_per_child: 500 +swh::deploy::worker::storage_archiver::loglevel: info # Contains a password: in private data -swh::deploy::worker::swh_storage_archiver::config: - archival_max_age: "%{hiera('swh::deploy::worker::swh_storage_archiver::archival_max_age')}" - retention_policy: "%{hiera('swh::deploy::worker::swh_storage_archiver::retention_policy')}" - batch_max_size: "%{hiera('swh::deploy::worker::swh_storage_archiver::batch_max_size')}" +swh::deploy::worker::storage_archiver::config: + archival_max_age: "%{hiera('swh::deploy::worker::storage_archiver::archival_max_age')}" + retention_policy: "%{hiera('swh::deploy::worker::storage_archiver::retention_policy')}" + batch_max_size: "%{hiera('swh::deploy::worker::storage_archiver::batch_max_size')}" archiver_storage: cls: db args: - dbconn: "host=%{hiera('swh::deploy::worker::swh_storage_archiver::db::host')} user=%{hiera('swh::deploy::worker::swh_storage_archiver::db::user')} dbname=%{hiera('swh::deploy::worker::swh_storage_archiver::db::dbname')} password=%{hiera('swh::deploy::worker::swh_storage_archiver::db::password')}" + dbconn: "host=%{hiera('swh::deploy::worker::storage_archiver::db::host')} user=%{hiera('swh::deploy::worker::storage_archiver::db::user')} dbname=%{hiera('swh::deploy::worker::storage_archiver::db::dbname')} password=%{hiera('swh::deploy::worker::storage_archiver::db::password')}" storages: - host: uffizi <<: *swh_objstorage_config_uffizi @@ -2077,7 +2076,13 @@ sources: - uffizi - banco - max_queue_length: "%{hiera('swh::deploy::worker::swh_storage_archiver::max_queue_length')}" + max_queue_length: "%{hiera('swh::deploy::worker::storage_archiver::max_queue_length')}" + celery: + task_modules: + - swh.archiver.tasks + task_queues: + - swh.archiver.tasks.SWHArchiverWithRetentionPolicyTask + - swh.archiver.tasks.SWHArchiverToBackendTask swh::deploy::base_indexer::config_directory: "%{hiera('swh::conf_directory')}/indexer" @@ -2090,11 +2095,11 @@ consumer_id: swh.indexer.journal_client scheduler: "%{alias('swh::remote_service::scheduler::config::saatchi')}" -swh::deploy::worker::swh_indexer::mimetype::config_file: "%{hiera('swh::conf_directory')}/indexer_content_mimetype.yml" -swh::deploy::worker::swh_indexer::mimetype::concurrency: 1 -swh::deploy::worker::swh_indexer::mimetype::loglevel: info +swh::deploy::worker::indexer_mimetype::config_file: "%{hiera('swh::conf_directory')}/indexer_content_mimetype.yml" +swh::deploy::worker::indexer_mimetype::concurrency: 1 +swh::deploy::worker::indexer_mimetype::loglevel: info # Contains a password: in private data -swh::deploy::worker::swh_indexer::mimetype::config: +swh::deploy::worker::indexer_mimetype::config: scheduler: "%{alias('swh::remote_service::scheduler::config::saatchi')}" indexer_storage: "%{alias('swh::indexer::storage::config::uffizi')}" objstorage: "%{alias('swh::remote_service::objstorage::config')}" @@ -2108,11 +2113,11 @@ - swh.indexer.tasks.ContentRangeMimetype - swh_indexer_content_mimetype_range -swh::deploy::worker::swh_indexer::origin_intrinsic_metadata::config_file: "%{hiera('swh::conf_directory')}/indexer_origin_intrinsic_metadata.yml" -swh::deploy::worker::swh_indexer::origin_intrinsic_metadata::concurrency: 2 -swh::deploy::worker::swh_indexer::origin_intrinsic_metadata::loglevel: info +swh::deploy::worker::indexer_origin_intrinsic_metadata::config_file: "%{hiera('swh::conf_directory')}/indexer_origin_intrinsic_metadata.yml" +swh::deploy::worker::indexer_origin_intrinsic_metadata::concurrency: 2 +swh::deploy::worker::indexer_origin_intrinsic_metadata::loglevel: info # Contains a password: in private data -swh::deploy::worker::swh_indexer::origin_intrinsic_metadata::config: +swh::deploy::worker::indexer_origin_intrinsic_metadata::config: scheduler: "%{alias('swh::remote_service::scheduler::config::saatchi')}" indexer_storage: "%{alias('swh::indexer::storage::config::uffizi')}" objstorage: "%{alias('swh::remote_service::objstorage::config')}" @@ -2124,11 +2129,11 @@ task_queues: - swh.indexer.tasks.OriginMetadata -swh::deploy::worker::swh_indexer::rehash::config_file: "rehash.yml" -swh::deploy::worker::swh_indexer::rehash::concurrency: 5 -swh::deploy::worker::swh_indexer::rehash::loglevel: info +swh::deploy::worker::indexer_rehash::config_file: "rehash.yml" +swh::deploy::worker::indexer_rehash::concurrency: 5 +swh::deploy::worker::indexer_rehash::loglevel: info # Contains a password: in private data -swh::deploy::worker::swh_indexer::rehash::config: +swh::deploy::worker::indexer_rehash::config: # Needs write access: explicitly configured to use uffizi storage: "%{alias('swh::remote_service::storage::config::uffizi')}" objstorage: "%{alias('swh::remote_service::objstorage::config')}" @@ -2137,11 +2142,11 @@ batch_size_retrieve_content: 10000 batch_size_update: 5000 -swh::deploy::worker::swh_indexer::fossology_license::config_file: "%{hiera('swh::conf_directory')}/indexer_fossology_license.yml" -swh::deploy::worker::swh_indexer::fossology_license::concurrency: 1 -swh::deploy::worker::swh_indexer::fossology_license::loglevel: info +swh::deploy::worker::indexer_fossology_license::config_file: "%{hiera('swh::conf_directory')}/indexer_fossology_license.yml" +swh::deploy::worker::indexer_fossology_license::concurrency: 1 +swh::deploy::worker::indexer_fossology_license::loglevel: info # Contains a password: in private data -swh::deploy::worker::swh_indexer::fossology_license::config: +swh::deploy::worker::indexer_fossology_license::config: indexer_storage: "%{alias('swh::indexer::storage::config::uffizi')}" objstorage: "%{alias('swh::remote_service::objstorage::config')}" storage: "%{alias('swh::remote_service::storage::config::azure')}" @@ -2160,11 +2165,11 @@ - swh.indexer.tasks.ContentRangeFossologyLicense - swh_indexer_content_fossology_license_range -swh::deploy::worker::swh_indexer::language::config_file: "%{hiera('swh::conf_directory')}/indexer_content_language.yml" -swh::deploy::worker::swh_indexer::language::concurrency: 3 -swh::deploy::worker::swh_indexer::language::loglevel: info +swh::deploy::worker::indexer_language::config_file: "%{hiera('swh::conf_directory')}/indexer_content_language.yml" +swh::deploy::worker::indexer_language::concurrency: 3 +swh::deploy::worker::indexer_language::loglevel: info # Contains a password: in private data -swh::deploy::worker::swh_indexer::language::config: +swh::deploy::worker::indexer_language::config: indexer_storage: "%{alias('swh::indexer::storage::config::uffizi')}" objstorage: "%{alias('swh::remote_service::objstorage::config')}" tools: @@ -2181,12 +2186,12 @@ task_queues: - swh.indexer.tasks.ContentLanguage -swh::deploy::worker::swh_indexer::ctags::config_file: "%{hiera('swh::conf_directory')}/indexer_content_ctags.yml" -swh::deploy::worker::swh_indexer::ctags::concurrency: 2 -swh::deploy::worker::swh_indexer::ctags::loglevel: info +swh::deploy::worker::indexer_ctags::config_file: "%{hiera('swh::conf_directory')}/indexer_content_ctags.yml" +swh::deploy::worker::indexer_ctags::concurrency: 2 +swh::deploy::worker::indexer_ctags::loglevel: info # Contains a password: in private data # objstorage configuration from swh::azure_objstorage::config is merged in the manifest -swh::deploy::worker::swh_indexer::ctags::config: +swh::deploy::worker::indexer_ctags::config: indexer_storage: "%{alias('swh::indexer::storage::config::uffizi')}" objstorage: "%{alias('swh::remote_service::objstorage::config')}" workdir: /tmp/swh/indexer.ctags/ @@ -2604,11 +2609,11 @@ zephir: Zephir unknown: '' -swh::deploy::worker::swh_vault_cooker::config_file: "%{hiera('swh::conf_directory')}/vault_cooker.yml" -swh::deploy::worker::swh_vault_cooker::concurrency: 20 -swh::deploy::worker::swh_vault_cooker::loglevel: info -swh::deploy::worker::swh_vault_cooker::conf_file: "%{hiera('swh::conf_directory')}/vault/cooker.yml" -swh::deploy::worker::swh_vault_cooker::config: +swh::deploy::worker::vault_cooker::config_file: "%{hiera('swh::conf_directory')}/vault_cooker.yml" +swh::deploy::worker::vault_cooker::concurrency: 20 +swh::deploy::worker::vault_cooker::loglevel: info +swh::deploy::worker::vault_cooker::conf_file: "%{hiera('swh::conf_directory')}/vault/cooker.yml" +swh::deploy::worker::vault_cooker::config: storage: "%{alias('swh::remote_service::storage::config')}" vault: cls: remote diff --git a/data/hostname/orangeriedev.internal.softwareheritage.org.yaml b/data/hostname/orangeriedev.internal.softwareheritage.org.yaml --- a/data/hostname/orangeriedev.internal.softwareheritage.org.yaml +++ b/data/hostname/orangeriedev.internal.softwareheritage.org.yaml @@ -12,6 +12,6 @@ # because they have the same queue names. # # swh::deploy::worker::instances: -# - swh_vault_cooker -# swh::deploy::worker::swh_vault_cooker::vault_url: "http://orangeriedev.internal.softwareheritage.org:5005/" +# - vault_cooker +# swh::deploy::worker::vault_cooker::vault_url: "http://orangeriedev.internal.softwareheritage.org:5005/" swh::deploy::worker::instances: [] diff --git a/data/hostname/uffizi.softwareheritage.org.yaml b/data/hostname/uffizi.softwareheritage.org.yaml --- a/data/hostname/uffizi.softwareheritage.org.yaml +++ b/data/hostname/uffizi.softwareheritage.org.yaml @@ -42,7 +42,7 @@ - "%{alias('swh::deploy::objstorage::ceph::rados_id')}" swh::deploy::worker::instances: - - swh_storage_archiver + - storage_archiver icinga2::host::vars: load: high diff --git a/data/hostname/worker11.euwest.azure.internal.softwareheritage.org.yaml b/data/hostname/worker11.euwest.azure.internal.softwareheritage.org.yaml --- a/data/hostname/worker11.euwest.azure.internal.softwareheritage.org.yaml +++ b/data/hostname/worker11.euwest.azure.internal.softwareheritage.org.yaml @@ -1,3 +1,3 @@ --- swh::deploy::worker::instances: - - swh_vault_cooker + - vault_cooker diff --git a/data/hostname/worker12.euwest.azure.internal.softwareheritage.org.yaml b/data/hostname/worker12.euwest.azure.internal.softwareheritage.org.yaml --- a/data/hostname/worker12.euwest.azure.internal.softwareheritage.org.yaml +++ b/data/hostname/worker12.euwest.azure.internal.softwareheritage.org.yaml @@ -1,3 +1,3 @@ --- swh::deploy::worker::instances: - - swh_vault_cooker + - vault_cooker diff --git a/data/location/azure_euwest.yaml b/data/location/azure_euwest.yaml --- a/data/location/azure_euwest.yaml +++ b/data/location/azure_euwest.yaml @@ -13,9 +13,9 @@ internal_network: 192.168.200.0/21 swh::deploy::worker::instances: - - swh_indexer_mimetype - - swh_indexer_fossology_license - - swh_indexer_origin_intrinsic_metadata + - indexer_mimetype + - indexer_fossology_license + - indexer_origin_intrinsic_metadata swh::remote_service::storage::config: "%{alias('swh::remote_service::storage::config::azure')}" swh::remote_service::objstorage::config: "%{alias('swh::remote_service::objstorage::config::azure_readonly_with_fallback')}" diff --git a/data/location/sesi_rocquencourt.yaml b/data/location/sesi_rocquencourt.yaml --- a/data/location/sesi_rocquencourt.yaml +++ b/data/location/sesi_rocquencourt.yaml @@ -21,13 +21,13 @@ smtp::relayhost: '[smtp.inria.fr]' swh::deploy::worker::instances: - - swh_lister_debian - - swh_lister_github - - swh_lister_gitlab - - swh_lister_pypi - - swh_loader_debian - - swh_loader_deposit - - swh_loader_git - - swh_loader_mercurial - - swh_loader_pypi - - swh_loader_svn + - lister_debian + - lister_github + - lister_gitlab + - lister_pypi + - loader_debian + - loader_deposit + - loader_git + - loader_mercurial + - loader_pypi + - loader_svn diff --git a/site-modules/profile/manifests/swh/deploy/archiver.pp b/site-modules/profile/manifests/swh/deploy/archiver.pp --- a/site-modules/profile/manifests/swh/deploy/archiver.pp +++ b/site-modules/profile/manifests/swh/deploy/archiver.pp @@ -3,15 +3,6 @@ class profile::swh::deploy::archiver { include ::profile::swh::deploy::objstorage_cloud - $config_dir = lookup('swh::deploy::worker::swh_storage_archiver::conf_directory') - - file {$config_dir: - ensure => 'directory', - owner => 'swhworker', - group => 'swhworker', - mode => '0644', - } - $packages = ['python3-swh.archiver'] package {$packages: diff --git a/site-modules/profile/manifests/swh/deploy/worker/swh_indexer_ctags.pp b/site-modules/profile/manifests/swh/deploy/worker/indexer_ctags.pp rename from site-modules/profile/manifests/swh/deploy/worker/swh_indexer_ctags.pp rename to site-modules/profile/manifests/swh/deploy/worker/indexer_ctags.pp --- a/site-modules/profile/manifests/swh/deploy/worker/swh_indexer_ctags.pp +++ b/site-modules/profile/manifests/swh/deploy/worker/indexer_ctags.pp @@ -1,13 +1,13 @@ # Deployment for swh-indexer-ctags -class profile::swh::deploy::worker::swh_indexer_ctags { +class profile::swh::deploy::worker::indexer_ctags { include ::profile::swh::deploy::indexer - $concurrency = lookup('swh::deploy::worker::swh_indexer::ctags::concurrency') - $loglevel = lookup('swh::deploy::worker::swh_indexer::ctags::loglevel') + $concurrency = lookup('swh::deploy::worker::indexer_ctags::concurrency') + $loglevel = lookup('swh::deploy::worker::indexer_ctags::loglevel') - $config_file = lookup('swh::deploy::worker::swh_indexer::ctags::config_file') - $config = lookup('swh::deploy::worker::swh_indexer::ctags::config') + $config_file = lookup('swh::deploy::worker::indexer_ctags::config_file') + $config = lookup('swh::deploy::worker::indexer_ctags::config') $packages = ['universal-ctags'] package {$packages: diff --git a/site-modules/profile/manifests/swh/deploy/worker/swh_indexer_fossology_license.pp b/site-modules/profile/manifests/swh/deploy/worker/indexer_fossology_license.pp rename from site-modules/profile/manifests/swh/deploy/worker/swh_indexer_fossology_license.pp rename to site-modules/profile/manifests/swh/deploy/worker/indexer_fossology_license.pp --- a/site-modules/profile/manifests/swh/deploy/worker/swh_indexer_fossology_license.pp +++ b/site-modules/profile/manifests/swh/deploy/worker/indexer_fossology_license.pp @@ -1,13 +1,13 @@ # Deployment for swh-indexer-fossology-license -class profile::swh::deploy::worker::swh_indexer_fossology_license { +class profile::swh::deploy::worker::indexer_fossology_license { include ::profile::swh::deploy::indexer - $concurrency = lookup('swh::deploy::worker::swh_indexer::fossology_license::concurrency') - $loglevel = lookup('swh::deploy::worker::swh_indexer::fossology_license::loglevel') + $concurrency = lookup('swh::deploy::worker::indexer_fossology_license::concurrency') + $loglevel = lookup('swh::deploy::worker::indexer_fossology_license::loglevel') - $config_file = lookup('swh::deploy::worker::swh_indexer::fossology_license::config_file') - $config = lookup('swh::deploy::worker::swh_indexer::fossology_license::config') + $config_file = lookup('swh::deploy::worker::indexer_fossology_license::config_file') + $config = lookup('swh::deploy::worker::indexer_fossology_license::config') $packages = ['fossology-nomossa'] package {$packages: diff --git a/site-modules/profile/manifests/swh/deploy/worker/swh_indexer_language.pp b/site-modules/profile/manifests/swh/deploy/worker/indexer_language.pp rename from site-modules/profile/manifests/swh/deploy/worker/swh_indexer_language.pp rename to site-modules/profile/manifests/swh/deploy/worker/indexer_language.pp --- a/site-modules/profile/manifests/swh/deploy/worker/swh_indexer_language.pp +++ b/site-modules/profile/manifests/swh/deploy/worker/indexer_language.pp @@ -1,16 +1,13 @@ # Deployment for swh-indexer-language -class profile::swh::deploy::worker::swh_indexer_language { +class profile::swh::deploy::worker::indexer_language { include ::profile::swh::deploy::indexer - $concurrency = lookup('swh::deploy::worker::swh_indexer::language::concurrency') - $loglevel = lookup('swh::deploy::worker::swh_indexer::language::loglevel') + $concurrency = lookup('swh::deploy::worker::indexer_language::concurrency') + $loglevel = lookup('swh::deploy::worker::indexer_language::loglevel') - $config_file = lookup('swh::deploy::worker::swh_indexer::language::config_file') - $config = lookup('swh::deploy::worker::swh_indexer::language::config') - - $task_modules = ['swh.indexer.tasks'] - $task_queues = ['swh_indexer_content_language'] + $config_file = lookup('swh::deploy::worker::indexer_language::config_file') + $config = lookup('swh::deploy::worker::indexer_language::config') Package[$::profile::swh::deploy::base_indexer::packages] ~> ::profile::swh::deploy::worker::instance {'indexer_content_language': ensure => 'stopped', diff --git a/site-modules/profile/manifests/swh/deploy/worker/swh_indexer_mimetype.pp b/site-modules/profile/manifests/swh/deploy/worker/indexer_mimetype.pp rename from site-modules/profile/manifests/swh/deploy/worker/swh_indexer_mimetype.pp rename to site-modules/profile/manifests/swh/deploy/worker/indexer_mimetype.pp --- a/site-modules/profile/manifests/swh/deploy/worker/swh_indexer_mimetype.pp +++ b/site-modules/profile/manifests/swh/deploy/worker/indexer_mimetype.pp @@ -1,13 +1,13 @@ # Deployment for swh-indexer-mimetype -class profile::swh::deploy::worker::swh_indexer_mimetype { +class profile::swh::deploy::worker::indexer_mimetype { include ::profile::swh::deploy::indexer - $concurrency = lookup('swh::deploy::worker::swh_indexer::mimetype::concurrency') - $loglevel = lookup('swh::deploy::worker::swh_indexer::mimetype::loglevel') + $concurrency = lookup('swh::deploy::worker::indexer_mimetype::concurrency') + $loglevel = lookup('swh::deploy::worker::indexer_mimetype::loglevel') - $config_file = lookup('swh::deploy::worker::swh_indexer::mimetype::config_file') - $config = lookup('swh::deploy::worker::swh_indexer::mimetype::config') + $config_file = lookup('swh::deploy::worker::indexer_mimetype::config_file') + $config = lookup('swh::deploy::worker::indexer_mimetype::config') Package[$::profile::swh::deploy::base_indexer::packages] ~> ::profile::swh::deploy::worker::instance {'indexer_content_mimetype': ensure => present, diff --git a/site-modules/profile/manifests/swh/deploy/worker/swh_indexer_origin_intrinsic_metadata.pp b/site-modules/profile/manifests/swh/deploy/worker/indexer_origin_intrinsic_metadata.pp rename from site-modules/profile/manifests/swh/deploy/worker/swh_indexer_origin_intrinsic_metadata.pp rename to site-modules/profile/manifests/swh/deploy/worker/indexer_origin_intrinsic_metadata.pp --- a/site-modules/profile/manifests/swh/deploy/worker/swh_indexer_origin_intrinsic_metadata.pp +++ b/site-modules/profile/manifests/swh/deploy/worker/indexer_origin_intrinsic_metadata.pp @@ -1,13 +1,13 @@ # Deployment for swh-indexer-origin-intrinsic-metadata -class profile::swh::deploy::worker::swh_indexer_origin_intrinsic_metadata { +class profile::swh::deploy::worker::indexer_origin_intrinsic_metadata { include ::profile::swh::deploy::indexer - $concurrency = lookup('swh::deploy::worker::swh_indexer::origin_intrinsic_metadata::concurrency') - $loglevel = lookup('swh::deploy::worker::swh_indexer::origin_intrinsic_metadata::loglevel') + $concurrency = lookup('swh::deploy::worker::indexer_origin_intrinsic_metadata::concurrency') + $loglevel = lookup('swh::deploy::worker::indexer_origin_intrinsic_metadata::loglevel') - $config_file = lookup('swh::deploy::worker::swh_indexer::origin_intrinsic_metadata::config_file') - $config = lookup('swh::deploy::worker::swh_indexer::origin_intrinsic_metadata::config') + $config_file = lookup('swh::deploy::worker::indexer_origin_intrinsic_metadata::config_file') + $config = lookup('swh::deploy::worker::indexer_origin_intrinsic_metadata::config') Package[$::profile::swh::deploy::base_indexer::packages] ~> ::profile::swh::deploy::worker::instance {'indexer_origin_intrinsic_metadata': ensure => present, diff --git a/site-modules/profile/manifests/swh/deploy/worker/swh_indexer_mimetype.pp b/site-modules/profile/manifests/swh/deploy/worker/indexer_rehash.pp rename from site-modules/profile/manifests/swh/deploy/worker/swh_indexer_mimetype.pp rename to site-modules/profile/manifests/swh/deploy/worker/indexer_rehash.pp --- a/site-modules/profile/manifests/swh/deploy/worker/swh_indexer_mimetype.pp +++ b/site-modules/profile/manifests/swh/deploy/worker/indexer_rehash.pp @@ -1,16 +1,16 @@ -# Deployment for swh-indexer-mimetype +# Deployment for swh-indexer-rehash -class profile::swh::deploy::worker::swh_indexer_mimetype { +class profile::swh::deploy::worker::indexer_rehash { include ::profile::swh::deploy::indexer - $concurrency = lookup('swh::deploy::worker::swh_indexer::mimetype::concurrency') - $loglevel = lookup('swh::deploy::worker::swh_indexer::mimetype::loglevel') + $concurrency = lookup('swh::deploy::worker::indexer_rehash::concurrency') + $loglevel = lookup('swh::deploy::worker::indexer_rehash::loglevel') - $config_file = lookup('swh::deploy::worker::swh_indexer::mimetype::config_file') - $config = lookup('swh::deploy::worker::swh_indexer::mimetype::config') + $config_file = lookup('swh::deploy::worker::indexer_rehash::config_file') + $config = lookup('swh::deploy::worker::indexer_rehash::config') - Package[$::profile::swh::deploy::base_indexer::packages] ~> ::profile::swh::deploy::worker::instance {'indexer_content_mimetype': - ensure => present, + Package[$::profile::swh::deploy::base_indexer::packages] ~> ::profile::swh::deploy::worker::instance {'indexer_rehash': + ensure => 'stopped', concurrency => $concurrency, loglevel => $loglevel, require => [ diff --git a/site-modules/profile/manifests/swh/deploy/worker/swh_lister_debian.pp b/site-modules/profile/manifests/swh/deploy/worker/lister_debian.pp rename from site-modules/profile/manifests/swh/deploy/worker/swh_lister_debian.pp rename to site-modules/profile/manifests/swh/deploy/worker/lister_debian.pp --- a/site-modules/profile/manifests/swh/deploy/worker/swh_lister_debian.pp +++ b/site-modules/profile/manifests/swh/deploy/worker/lister_debian.pp @@ -1,10 +1,10 @@ # Deployment for swh-lister-debian -class profile::swh::deploy::worker::swh_lister_debian { - $concurrency = lookup('swh::deploy::worker::swh_lister_debian::concurrency') - $loglevel = lookup('swh::deploy::worker::swh_lister_debian::loglevel') +class profile::swh::deploy::worker::lister_debian { + $concurrency = lookup('swh::deploy::worker::lister_debian::concurrency') + $loglevel = lookup('swh::deploy::worker::lister_debian::loglevel') - $config_file = lookup('swh::deploy::worker::swh_lister_debian::config_file') - $config = lookup('swh::deploy::worker::swh_lister_debian::config', Hash, 'deep') + $config_file = lookup('swh::deploy::worker::lister_debian::config_file') + $config = lookup('swh::deploy::worker::lister_debian::config', Hash, 'deep') include ::profile::swh::deploy::base_lister diff --git a/site-modules/profile/manifests/swh/deploy/worker/swh_lister_github.pp b/site-modules/profile/manifests/swh/deploy/worker/lister_github.pp rename from site-modules/profile/manifests/swh/deploy/worker/swh_lister_github.pp rename to site-modules/profile/manifests/swh/deploy/worker/lister_github.pp --- a/site-modules/profile/manifests/swh/deploy/worker/swh_lister_github.pp +++ b/site-modules/profile/manifests/swh/deploy/worker/lister_github.pp @@ -1,10 +1,10 @@ # Deployment for swh-lister-github -class profile::swh::deploy::worker::swh_lister_github { - $concurrency = lookup('swh::deploy::worker::swh_lister_github::concurrency') - $loglevel = lookup('swh::deploy::worker::swh_lister_github::loglevel') +class profile::swh::deploy::worker::lister_github { + $concurrency = lookup('swh::deploy::worker::lister_github::concurrency') + $loglevel = lookup('swh::deploy::worker::lister_github::loglevel') - $config_file = lookup('swh::deploy::worker::swh_lister_github::config_file') - $config = lookup('swh::deploy::worker::swh_lister_github::config', Hash, 'deep') + $config_file = lookup('swh::deploy::worker::lister_github::config_file') + $config = lookup('swh::deploy::worker::lister_github::config', Hash, 'deep') include ::profile::swh::deploy::base_lister diff --git a/site-modules/profile/manifests/swh/deploy/worker/swh_lister_gitlab.pp b/site-modules/profile/manifests/swh/deploy/worker/lister_gitlab.pp rename from site-modules/profile/manifests/swh/deploy/worker/swh_lister_gitlab.pp rename to site-modules/profile/manifests/swh/deploy/worker/lister_gitlab.pp --- a/site-modules/profile/manifests/swh/deploy/worker/swh_lister_gitlab.pp +++ b/site-modules/profile/manifests/swh/deploy/worker/lister_gitlab.pp @@ -1,10 +1,10 @@ # Deployment for swh-lister-gitlab -class profile::swh::deploy::worker::swh_lister_gitlab { - $concurrency = lookup('swh::deploy::worker::swh_lister_gitlab::concurrency') - $loglevel = lookup('swh::deploy::worker::swh_lister_gitlab::loglevel') +class profile::swh::deploy::worker::lister_gitlab { + $concurrency = lookup('swh::deploy::worker::lister_gitlab::concurrency') + $loglevel = lookup('swh::deploy::worker::lister_gitlab::loglevel') - $config_file = lookup('swh::deploy::worker::swh_lister_gitlab::config_file') - $config = lookup('swh::deploy::worker::swh_lister_gitlab::config', Hash, 'deep') + $config_file = lookup('swh::deploy::worker::lister_gitlab::config_file') + $config = lookup('swh::deploy::worker::lister_gitlab::config', Hash, 'deep') include ::profile::swh::deploy::base_lister diff --git a/site-modules/profile/manifests/swh/deploy/worker/swh_lister_pypi.pp b/site-modules/profile/manifests/swh/deploy/worker/lister_pypi.pp rename from site-modules/profile/manifests/swh/deploy/worker/swh_lister_pypi.pp rename to site-modules/profile/manifests/swh/deploy/worker/lister_pypi.pp --- a/site-modules/profile/manifests/swh/deploy/worker/swh_lister_pypi.pp +++ b/site-modules/profile/manifests/swh/deploy/worker/lister_pypi.pp @@ -1,10 +1,10 @@ # Deployment for swh-lister-pypi -class profile::swh::deploy::worker::swh_lister_pypi { - $concurrency = lookup('swh::deploy::worker::swh_lister_pypi::concurrency') - $loglevel = lookup('swh::deploy::worker::swh_lister_pypi::loglevel') +class profile::swh::deploy::worker::lister_pypi { + $concurrency = lookup('swh::deploy::worker::lister_pypi::concurrency') + $loglevel = lookup('swh::deploy::worker::lister_pypi::loglevel') - $config_file = lookup('swh::deploy::worker::swh_lister_pypi::config_file') - $config = lookup('swh::deploy::worker::swh_lister_pypi::config', Hash, 'deep') + $config_file = lookup('swh::deploy::worker::lister_pypi::config_file') + $config = lookup('swh::deploy::worker::lister_pypi::config', Hash, 'deep') include ::profile::swh::deploy::base_lister diff --git a/site-modules/profile/manifests/swh/deploy/worker/swh_loader_debian.pp b/site-modules/profile/manifests/swh/deploy/worker/loader_debian.pp rename from site-modules/profile/manifests/swh/deploy/worker/swh_loader_debian.pp rename to site-modules/profile/manifests/swh/deploy/worker/loader_debian.pp --- a/site-modules/profile/manifests/swh/deploy/worker/swh_loader_debian.pp +++ b/site-modules/profile/manifests/swh/deploy/worker/loader_debian.pp @@ -1,10 +1,10 @@ # Deployment for swh-loader-debian -class profile::swh::deploy::worker::swh_loader_debian { - $concurrency = lookup('swh::deploy::worker::swh_loader_debian::concurrency') - $loglevel = lookup('swh::deploy::worker::swh_loader_debian::loglevel') +class profile::swh::deploy::worker::loader_debian { + $concurrency = lookup('swh::deploy::worker::loader_debian::concurrency') + $loglevel = lookup('swh::deploy::worker::loader_debian::loglevel') - $config_file = lookup('swh::deploy::worker::swh_loader_debian::config_file') - $config = lookup('swh::deploy::worker::swh_loader_debian::config') + $config_file = lookup('swh::deploy::worker::loader_debian::config_file') + $config = lookup('swh::deploy::worker::loader_debian::config') $packages = ['python3-swh.loader.debian'] diff --git a/site-modules/profile/manifests/swh/deploy/worker/swh_loader_deposit.pp b/site-modules/profile/manifests/swh/deploy/worker/loader_deposit.pp rename from site-modules/profile/manifests/swh/deploy/worker/swh_loader_deposit.pp rename to site-modules/profile/manifests/swh/deploy/worker/loader_deposit.pp --- a/site-modules/profile/manifests/swh/deploy/worker/swh_loader_deposit.pp +++ b/site-modules/profile/manifests/swh/deploy/worker/loader_deposit.pp @@ -1,13 +1,13 @@ # Deployment for swh-loader-deposit -class profile::swh::deploy::worker::swh_loader_deposit { - $concurrency = lookup('swh::deploy::worker::swh_loader_deposit::concurrency') - $loglevel = lookup('swh::deploy::worker::swh_loader_deposit::loglevel') +class profile::swh::deploy::worker::loader_deposit { + $concurrency = lookup('swh::deploy::worker::loader_deposit::concurrency') + $loglevel = lookup('swh::deploy::worker::loader_deposit::loglevel') - $config_file = lookup('swh::deploy::worker::swh_loader_deposit::config_file') - $config = lookup('swh::deploy::worker::swh_loader_deposit::config') + $config_file = lookup('swh::deploy::worker::loader_deposit::config_file') + $config = lookup('swh::deploy::worker::loader_deposit::config') $packages = ['python3-swh.deposit.loader'] - $private_tmp = lookup('swh::deploy::worker::swh_loader_deposit::private_tmp') + $private_tmp = lookup('swh::deploy::worker::loader_deposit::private_tmp') $service_name = 'loader_deposit' diff --git a/site-modules/profile/manifests/swh/deploy/worker/swh_loader_git.pp b/site-modules/profile/manifests/swh/deploy/worker/loader_git.pp rename from site-modules/profile/manifests/swh/deploy/worker/swh_loader_git.pp rename to site-modules/profile/manifests/swh/deploy/worker/loader_git.pp --- a/site-modules/profile/manifests/swh/deploy/worker/swh_loader_git.pp +++ b/site-modules/profile/manifests/swh/deploy/worker/loader_git.pp @@ -1,12 +1,12 @@ # Deployment for swh-loader-git (remote) -class profile::swh::deploy::worker::swh_loader_git { +class profile::swh::deploy::worker::loader_git { include ::profile::swh::deploy::base_loader_git - $concurrency = lookup('swh::deploy::worker::swh_loader_git::concurrency') - $loglevel = lookup('swh::deploy::worker::swh_loader_git::loglevel') + $concurrency = lookup('swh::deploy::worker::loader_git::concurrency') + $loglevel = lookup('swh::deploy::worker::loader_git::loglevel') - $config_file = lookup('swh::deploy::worker::swh_loader_git::config_file') - $config = lookup('swh::deploy::worker::swh_loader_git::config') + $config_file = lookup('swh::deploy::worker::loader_git::config_file') + $config = lookup('swh::deploy::worker::loader_git::config') ::profile::swh::deploy::worker::instance {'loader_git': ensure => present, diff --git a/site-modules/profile/manifests/swh/deploy/worker/swh_loader_mercurial.pp b/site-modules/profile/manifests/swh/deploy/worker/loader_mercurial.pp rename from site-modules/profile/manifests/swh/deploy/worker/swh_loader_mercurial.pp rename to site-modules/profile/manifests/swh/deploy/worker/loader_mercurial.pp --- a/site-modules/profile/manifests/swh/deploy/worker/swh_loader_mercurial.pp +++ b/site-modules/profile/manifests/swh/deploy/worker/loader_mercurial.pp @@ -1,15 +1,15 @@ # Deployment for swh-loader-mercurial (disk) -class profile::swh::deploy::worker::swh_loader_mercurial { +class profile::swh::deploy::worker::loader_mercurial { include ::profile::swh::deploy::base_loader_git - $concurrency = lookup('swh::deploy::worker::swh_loader_mercurial::concurrency') - $loglevel = lookup('swh::deploy::worker::swh_loader_mercurial::loglevel') + $concurrency = lookup('swh::deploy::worker::loader_mercurial::concurrency') + $loglevel = lookup('swh::deploy::worker::loader_mercurial::loglevel') - $config_file = lookup('swh::deploy::worker::swh_loader_mercurial::config_file') - $config = lookup('swh::deploy::worker::swh_loader_mercurial::config') + $config_file = lookup('swh::deploy::worker::loader_mercurial::config_file') + $config = lookup('swh::deploy::worker::loader_mercurial::config') $service_name = 'loader_mercurial' - $private_tmp = lookup('swh::deploy::worker::swh_loader_mercurial::private_tmp') + $private_tmp = lookup('swh::deploy::worker::loader_mercurial::private_tmp') $packages = ['python3-swh.loader.mercurial'] diff --git a/site-modules/profile/manifests/swh/deploy/worker/swh_loader_pypi.pp b/site-modules/profile/manifests/swh/deploy/worker/loader_pypi.pp rename from site-modules/profile/manifests/swh/deploy/worker/swh_loader_pypi.pp rename to site-modules/profile/manifests/swh/deploy/worker/loader_pypi.pp --- a/site-modules/profile/manifests/swh/deploy/worker/swh_loader_pypi.pp +++ b/site-modules/profile/manifests/swh/deploy/worker/loader_pypi.pp @@ -1,12 +1,12 @@ # Deployment for swh-loader-pypi -class profile::swh::deploy::worker::swh_loader_pypi { - $concurrency = lookup('swh::deploy::worker::swh_loader_pypi::concurrency') - $loglevel = lookup('swh::deploy::worker::swh_loader_pypi::loglevel') +class profile::swh::deploy::worker::loader_pypi { + $concurrency = lookup('swh::deploy::worker::loader_pypi::concurrency') + $loglevel = lookup('swh::deploy::worker::loader_pypi::loglevel') - $config_file = lookup('swh::deploy::worker::swh_loader_pypi::config_file') - $config = lookup('swh::deploy::worker::swh_loader_pypi::config') + $config_file = lookup('swh::deploy::worker::loader_pypi::config_file') + $config = lookup('swh::deploy::worker::loader_pypi::config') - $private_tmp = lookup('swh::deploy::worker::swh_loader_pypi::private_tmp') + $private_tmp = lookup('swh::deploy::worker::loader_pypi::private_tmp') $packages = ['python3-swh.loader.pypi'] diff --git a/site-modules/profile/manifests/swh/deploy/worker/swh_loader_svn.pp b/site-modules/profile/manifests/swh/deploy/worker/loader_svn.pp rename from site-modules/profile/manifests/swh/deploy/worker/swh_loader_svn.pp rename to site-modules/profile/manifests/swh/deploy/worker/loader_svn.pp --- a/site-modules/profile/manifests/swh/deploy/worker/swh_loader_svn.pp +++ b/site-modules/profile/manifests/swh/deploy/worker/loader_svn.pp @@ -1,14 +1,14 @@ # Deployment for swh-loader-svn -class profile::swh::deploy::worker::swh_loader_svn { - $concurrency = lookup('swh::deploy::worker::swh_loader_svn::concurrency') - $loglevel = lookup('swh::deploy::worker::swh_loader_svn::loglevel') +class profile::swh::deploy::worker::loader_svn { + $concurrency = lookup('swh::deploy::worker::loader_svn::concurrency') + $loglevel = lookup('swh::deploy::worker::loader_svn::loglevel') $config_file = '/etc/softwareheritage/loader/svn.yml' - $config = lookup('swh::deploy::worker::swh_loader_svn::config') + $config = lookup('swh::deploy::worker::loader_svn::config') $packages = ['python3-swh.loader.svn'] - $limit_no_file = lookup('swh::deploy::worker::swh_loader_svn::limit_no_file') - $private_tmp = lookup('swh::deploy::worker::swh_loader_svn::private_tmp') + $limit_no_file = lookup('swh::deploy::worker::loader_svn::limit_no_file') + $private_tmp = lookup('swh::deploy::worker::loader_svn::private_tmp') package {$packages: ensure => 'latest', diff --git a/site-modules/profile/manifests/swh/deploy/worker/storage_archiver.pp b/site-modules/profile/manifests/swh/deploy/worker/storage_archiver.pp new file mode 100644 --- /dev/null +++ b/site-modules/profile/manifests/swh/deploy/worker/storage_archiver.pp @@ -0,0 +1,30 @@ +# Deployment for swh-storage-archiver +class profile::swh::deploy::worker::storage_archiver { + include ::profile::swh::deploy::archiver + + $concurrency = lookup('swh::deploy::worker::storage_archiver::concurrency') + $max_tasks_per_child = lookup('swh::deploy::worker::storage_archiver::max_tasks_per_child') + $loglevel = lookup('swh::deploy::worker::storage_archiver::loglevel') + + $config_file = lookup('swh::deploy::worker::storage_archiver::config_file') + $config = lookup('swh::deploy::worker::storage_archiver::config') + + ::profile::swh::deploy::worker::instance {'storage_archiver': + ensure => present, + concurrency => $concurrency, + loglevel => $loglevel, + max_tasks_per_child => $max_tasks_per_child, + require => [ + File[$config_file], + ], + } + + file {$config_file: + ensure => 'present', + owner => 'swhworker', + group => 'swhdev', + # Contains passwords + mode => '0640', + content => inline_template("<%= @config.to_yaml %>\n"), + } +} diff --git a/site-modules/profile/manifests/swh/deploy/worker/swh_indexer_rehash.pp b/site-modules/profile/manifests/swh/deploy/worker/swh_indexer_rehash.pp deleted file mode 100644 --- a/site-modules/profile/manifests/swh/deploy/worker/swh_indexer_rehash.pp +++ /dev/null @@ -1,33 +0,0 @@ -# Deployment for swh-indexer-rehash - -class profile::swh::deploy::worker::swh_indexer_rehash { - include ::profile::swh::deploy::indexer - - $concurrency = lookup('swh::deploy::worker::swh_indexer::rehash::concurrency') - $loglevel = lookup('swh::deploy::worker::swh_indexer::rehash::loglevel') - - $config_file = lookup('swh::deploy::worker::swh_indexer::rehash::config_file') - $config_directory = lookup('swh::deploy::base_indexer::config_directory') - $config_path = "${config_directory}/${config_file}" - $config = lookup('swh::deploy::worker::swh_indexer::rehash::config') - - Package[$::profile::swh::deploy::base_indexer::packages] ~> ::profile::swh::deploy::worker::instance {'swh_indexer_rehash': - ensure => 'stopped', - concurrency => $concurrency, - loglevel => $loglevel, - require => [ - Class['profile::swh::deploy::indexer'], - Class['profile::swh::deploy::objstorage_cloud'], - File[$config_path], - ], - } - - file {$config_path: - ensure => 'present', - owner => 'swhworker', - group => 'swhdev', - # Contains passwords - mode => '0640', - content => inline_template("<%= @config.to_yaml %>\n"), - } -} diff --git a/site-modules/profile/manifests/swh/deploy/worker/swh_storage_archiver.pp b/site-modules/profile/manifests/swh/deploy/worker/swh_storage_archiver.pp deleted file mode 100644 --- a/site-modules/profile/manifests/swh/deploy/worker/swh_storage_archiver.pp +++ /dev/null @@ -1,30 +0,0 @@ -# Deployment for swh-storage-archiver -class profile::swh::deploy::worker::swh_storage_archiver { - include ::profile::swh::deploy::archiver - - $concurrency = lookup('swh::deploy::worker::swh_storage_archiver::concurrency') - $max_tasks_per_child = lookup('swh::deploy::worker::swh_storage_archiver::max_tasks_per_child') - $loglevel = lookup('swh::deploy::worker::swh_storage_archiver::loglevel') - - $config_file = lookup('swh::deploy::worker::swh_storage_archiver::conf_file') - $config = lookup('swh::deploy::worker::swh_storage_archiver::config') - - ::profile::swh::deploy::worker::instance {'swh_storage_archiver': - ensure => present, - concurrency => $concurrency, - loglevel => $loglevel, - max_tasks_per_child => $max_tasks_per_child, - require => [ - File[$config_file], - ], - } - - file {$config_file: - ensure => 'present', - owner => 'swhworker', - group => 'swhdev', - # Contains passwords - mode => '0640', - content => inline_template("<%= @config.to_yaml %>\n"), - } -} diff --git a/site-modules/profile/manifests/swh/deploy/worker/swh_vault_cooker.pp b/site-modules/profile/manifests/swh/deploy/worker/vault_cooker.pp rename from site-modules/profile/manifests/swh/deploy/worker/swh_vault_cooker.pp rename to site-modules/profile/manifests/swh/deploy/worker/vault_cooker.pp --- a/site-modules/profile/manifests/swh/deploy/worker/swh_vault_cooker.pp +++ b/site-modules/profile/manifests/swh/deploy/worker/vault_cooker.pp @@ -1,13 +1,13 @@ # Deployment of a vault cooker -class profile::swh::deploy::worker::swh_vault_cooker { +class profile::swh::deploy::worker::vault_cooker { include ::profile::swh::deploy::base_vault - $concurrency = lookup('swh::deploy::worker::swh_vault_cooker::concurrency') - $loglevel = lookup('swh::deploy::worker::swh_vault_cooker::loglevel') + $concurrency = lookup('swh::deploy::worker::vault_cooker::concurrency') + $loglevel = lookup('swh::deploy::worker::vault_cooker::loglevel') - $conf_file = lookup('swh::deploy::worker::swh_vault_cooker::config_file') - $config = lookup('swh::deploy::worker::swh_vault_cooker::config') + $conf_file = lookup('swh::deploy::worker::vault_cooker::config_file') + $config = lookup('swh::deploy::worker::vault_cooker::config') ::profile::swh::deploy::worker::instance {'vault_cooker': ensure => present,