diff --git a/data/defaults.yaml b/data/defaults.yaml --- a/data/defaults.yaml +++ b/data/defaults.yaml @@ -1459,7 +1459,7 @@ - origin - origin_visit -swh::deploy::archiver_content_updater::conf_file: "%{hiera('swh::deploy::worker::storage_archiver::conf_directory')}/content_updater.yml" +swh::deploy::archiver_content_updater::conf_file: "%{hiera('swh::deploy::worker::archiver::conf_directory')}/content_updater.yml" swh::deploy::archiver_content_updater::user: swhstorage swh::deploy::archiver_content_updater::group: swhstorage swh::deploy::archiver_content_updater::config: @@ -1473,7 +1473,7 @@ archiver_storage: cls: db args: - dbconn: "host=%{hiera('swh::deploy::worker::storage_archiver::db::host')} user=%{hiera('swh::deploy::worker::storage_archiver::db::user')} dbname=%{hiera('swh::deploy::worker::storage_archiver::db::dbname')} password=%{hiera('swh::deploy::worker::storage_archiver::db::password')}" + dbconn: "host=%{hiera('swh::deploy::worker::archiver::db::host')} user=%{hiera('swh::deploy::worker::archiver::db::user')} dbname=%{hiera('swh::deploy::worker::archiver::db::dbname')} password=%{hiera('swh::deploy::worker::archiver::db::password')}" sources_present: - uffizi @@ -1607,10 +1607,10 @@ swh::deploy::objstorage_archive_notifier_checker::group: "%{hiera('swh::deploy::objstorage::group')}" swh::deploy::objstorage_archive_notifier_checker::log_tag: objstorage.checker.archive_notifier swh::deploy::objstorage_archive_notifier_checker::storage_name: this-must-be-overriden-on-a-host-basis -swh::deploy::objstorage_archive_notifier_checker::db::host: "%{hiera('swh::deploy::worker::storage_archiver::db::host')}" -swh::deploy::objstorage_archive_notifier_checker::db::dbname: "%{hiera('swh::deploy::worker::storage_archiver::db::dbname')}" -swh::deploy::objstorage_archive_notifier_checker::db::user: "%{hiera('swh::deploy::worker::storage_archiver::db::user')}" -swh::deploy::objstorage_archive_notifier_checker::db::password: "%{hiera('swh::deploy::worker::storage_archiver::db::password')}" +swh::deploy::objstorage_archive_notifier_checker::db::host: "%{hiera('swh::deploy::worker::archiver::db::host')}" +swh::deploy::objstorage_archive_notifier_checker::db::dbname: "%{hiera('swh::deploy::worker::archiver::db::dbname')}" +swh::deploy::objstorage_archive_notifier_checker::db::user: "%{hiera('swh::deploy::worker::archiver::db::user')}" +swh::deploy::objstorage_archive_notifier_checker::db::password: "%{hiera('swh::deploy::worker::archiver::db::password')}" swh::deploy::objstorage_archive_notifier_checker::config: storage: cls: pathslicing @@ -1623,7 +1623,7 @@ archiver_storage: cls: db args: - dbconn: "host=%{hiera('swh::deploy::worker::storage_archiver::db::host')} user=%{hiera('swh::deploy::worker::storage_archiver::db::user')} dbname=%{hiera('swh::deploy::worker::storage_archiver::db::dbname')} password=%{hiera('swh::deploy::worker::storage_archiver::db::password')}" + dbconn: "host=%{hiera('swh::deploy::worker::archiver::db::host')} user=%{hiera('swh::deploy::worker::archiver::db::user')} dbname=%{hiera('swh::deploy::worker::archiver::db::dbname')} password=%{hiera('swh::deploy::worker::archiver::db::password')}" swh::deploy::webapp::backported_packages: stretch: @@ -2045,27 +2045,27 @@ - swh.lister.pypi.tasks.PyPIListerTask - swh_lister_pypi_refresh -swh::deploy::worker::storage_archiver::config_file: "%{hiera('swh::conf_directory')}/storage_archiver.yml" -swh::deploy::worker::storage_archiver::archival_max_age: 3600 -swh::deploy::worker::storage_archiver::max_queue_length: 100000 -swh::deploy::worker::storage_archiver::retention_policy: 3 -swh::deploy::worker::storage_archiver::batch_max_size: 5 -swh::deploy::worker::storage_archiver::db::host: db -swh::deploy::worker::storage_archiver::db::dbname: softwareheritage-archiver -swh::deploy::worker::storage_archiver::db::user: "%{hiera('swh::deploy::storage::db::user')}" -swh::deploy::worker::storage_archiver::db::password: "%{hiera('swh::deploy::storage::db::password')}" -swh::deploy::worker::storage_archiver::concurrency: 10 -swh::deploy::worker::storage_archiver::max_tasks_per_child: 500 -swh::deploy::worker::storage_archiver::loglevel: info +swh::deploy::worker::archiver::config_file: "%{hiera('swh::conf_directory')}/archiver.yml" +swh::deploy::worker::archiver::archival_max_age: 3600 +swh::deploy::worker::archiver::max_queue_length: 100000 +swh::deploy::worker::archiver::retention_policy: 3 +swh::deploy::worker::archiver::batch_max_size: 5 +swh::deploy::worker::archiver::db::host: db +swh::deploy::worker::archiver::db::dbname: softwareheritage-archiver +swh::deploy::worker::archiver::db::user: "%{hiera('swh::deploy::storage::db::user')}" +swh::deploy::worker::archiver::db::password: "%{hiera('swh::deploy::storage::db::password')}" +swh::deploy::worker::archiver::concurrency: 10 +swh::deploy::worker::archiver::max_tasks_per_child: 500 +swh::deploy::worker::archiver::loglevel: info # Contains a password: in private data -swh::deploy::worker::storage_archiver::config: - archival_max_age: "%{hiera('swh::deploy::worker::storage_archiver::archival_max_age')}" - retention_policy: "%{hiera('swh::deploy::worker::storage_archiver::retention_policy')}" - batch_max_size: "%{hiera('swh::deploy::worker::storage_archiver::batch_max_size')}" +swh::deploy::worker::archiver::config: + archival_max_age: "%{hiera('swh::deploy::worker::archiver::archival_max_age')}" + retention_policy: "%{hiera('swh::deploy::worker::archiver::retention_policy')}" + batch_max_size: "%{hiera('swh::deploy::worker::archiver::batch_max_size')}" archiver_storage: cls: db args: - dbconn: "host=%{hiera('swh::deploy::worker::storage_archiver::db::host')} user=%{hiera('swh::deploy::worker::storage_archiver::db::user')} dbname=%{hiera('swh::deploy::worker::storage_archiver::db::dbname')} password=%{hiera('swh::deploy::worker::storage_archiver::db::password')}" + dbconn: "host=%{hiera('swh::deploy::worker::archiver::db::host')} user=%{hiera('swh::deploy::worker::archiver::db::user')} dbname=%{hiera('swh::deploy::worker::archiver::db::dbname')} password=%{hiera('swh::deploy::worker::archiver::db::password')}" storages: - host: uffizi <<: *swh_objstorage_config_uffizi @@ -2076,7 +2076,7 @@ sources: - uffizi - banco - max_queue_length: "%{hiera('swh::deploy::worker::storage_archiver::max_queue_length')}" + max_queue_length: "%{hiera('swh::deploy::worker::archiver::max_queue_length')}" celery: task_modules: - swh.archiver.tasks diff --git a/data/hostname/uffizi.softwareheritage.org.yaml b/data/hostname/uffizi.softwareheritage.org.yaml --- a/data/hostname/uffizi.softwareheritage.org.yaml +++ b/data/hostname/uffizi.softwareheritage.org.yaml @@ -42,7 +42,7 @@ - "%{alias('swh::deploy::objstorage::ceph::rados_id')}" swh::deploy::worker::instances: - - storage_archiver + - archiver icinga2::host::vars: load: high diff --git a/site-modules/profile/manifests/swh/deploy/archiver_content_updater.pp b/site-modules/profile/manifests/swh/deploy/archiver_content_updater.pp --- a/site-modules/profile/manifests/swh/deploy/archiver_content_updater.pp +++ b/site-modules/profile/manifests/swh/deploy/archiver_content_updater.pp @@ -1,7 +1,7 @@ # Deployment of the swh.storage.archiver.updater class profile::swh::deploy::archiver_content_updater { - include profile::swh::deploy::archiver + include profile::swh::deploy::base_archiver $conf_file = lookup('swh::deploy::archiver_content_updater::conf_file') $user = lookup('swh::deploy::archiver_content_updater::user') diff --git a/site-modules/profile/manifests/swh/deploy/archiver.pp b/site-modules/profile/manifests/swh/deploy/base_archiver.pp rename from site-modules/profile/manifests/swh/deploy/archiver.pp rename to site-modules/profile/manifests/swh/deploy/base_archiver.pp --- a/site-modules/profile/manifests/swh/deploy/archiver.pp +++ b/site-modules/profile/manifests/swh/deploy/base_archiver.pp @@ -1,6 +1,6 @@ # Archiver base configuration -class profile::swh::deploy::archiver { +class profile::swh::deploy::base_archiver { include ::profile::swh::deploy::objstorage_cloud $packages = ['python3-swh.archiver'] diff --git a/site-modules/profile/manifests/swh/deploy/base_loader_git.pp b/site-modules/profile/manifests/swh/deploy/base_loader_git.pp --- a/site-modules/profile/manifests/swh/deploy/base_loader_git.pp +++ b/site-modules/profile/manifests/swh/deploy/base_loader_git.pp @@ -6,7 +6,7 @@ $packages = ['python3-swh.loader.git'] package {$packages: - ensure => 'latest', + ensure => 'present', } } diff --git a/site-modules/profile/manifests/swh/deploy/indexer_storage.pp b/site-modules/profile/manifests/swh/deploy/indexer_storage.pp --- a/site-modules/profile/manifests/swh/deploy/indexer_storage.pp +++ b/site-modules/profile/manifests/swh/deploy/indexer_storage.pp @@ -4,7 +4,7 @@ include ::profile::swh::deploy::base_storage package {'python3-swh.indexer.storage': - ensure => 'latest', + ensure => 'present', } ~> ::profile::swh::deploy::rpc_server {'indexer-storage': config_key => 'indexer::storage', executable => 'swh.indexer.storage.api.server:run_from_webserver', diff --git a/site-modules/profile/manifests/swh/deploy/storage.pp b/site-modules/profile/manifests/swh/deploy/storage.pp --- a/site-modules/profile/manifests/swh/deploy/storage.pp +++ b/site-modules/profile/manifests/swh/deploy/storage.pp @@ -4,7 +4,7 @@ include ::profile::swh::deploy::base_storage package {'python3-swh.storage': - ensure => 'latest', + ensure => 'present', } ~> ::profile::swh::deploy::rpc_server {'storage': executable => 'swh.storage.api.server:run_from_webserver', worker => 'sync', diff --git a/site-modules/profile/manifests/swh/deploy/worker/archiver.pp b/site-modules/profile/manifests/swh/deploy/worker/archiver.pp new file mode 100644 --- /dev/null +++ b/site-modules/profile/manifests/swh/deploy/worker/archiver.pp @@ -0,0 +1,14 @@ +# Deployment for archiver (content) +class profile::swh::deploy::worker::archiver { + include ::profile::swh::deploy::base_archiver + + $max_tasks_per_child = lookup('swh::deploy::worker::archiver::max_tasks_per_child') + + ::profile::swh::deploy::worker::instance {'archiver': + ensure => present, + max_tasks_per_child => $max_tasks_per_child, + require => [ + Package[$packages], + ], + } +} diff --git a/site-modules/profile/manifests/swh/deploy/worker/base.pp b/site-modules/profile/manifests/swh/deploy/worker/base.pp --- a/site-modules/profile/manifests/swh/deploy/worker/base.pp +++ b/site-modules/profile/manifests/swh/deploy/worker/base.pp @@ -7,7 +7,6 @@ $systemd_unit_name = 'swh-worker.service' $systemd_slice_name = 'system-swh\x2dworker.slice' $systemd_generator = '/lib/systemd/system-generators/swh-worker-generator' - $config_directory = '/etc/softwareheritage/worker' package {'python3-swh.scheduler': ensure => installed, @@ -40,13 +39,4 @@ notify => Class['systemd::systemctl::daemon_reload'], } - file {$config_directory: - ensure => 'directory', - owner => 'swhworker', - group => 'swhdev', - mode => '0644', - purge => true, - recurse => true, - } - } diff --git a/site-modules/profile/manifests/swh/deploy/worker/loader_deposit.pp b/site-modules/profile/manifests/swh/deploy/worker/loader_deposit.pp --- a/site-modules/profile/manifests/swh/deploy/worker/loader_deposit.pp +++ b/site-modules/profile/manifests/swh/deploy/worker/loader_deposit.pp @@ -4,7 +4,7 @@ $private_tmp = lookup('swh::deploy::worker::loader_deposit::private_tmp') package {$packages: - ensure => 'latest', + ensure => 'present', } # This installs the swh-worker@$service_name service diff --git a/site-modules/profile/manifests/swh/deploy/worker/loader_mercurial.pp b/site-modules/profile/manifests/swh/deploy/worker/loader_mercurial.pp --- a/site-modules/profile/manifests/swh/deploy/worker/loader_mercurial.pp +++ b/site-modules/profile/manifests/swh/deploy/worker/loader_mercurial.pp @@ -4,7 +4,7 @@ $packages = ['python3-swh.loader.mercurial'] package {$packages: - ensure => 'latest', + ensure => 'present', } ::profile::swh::deploy::worker::instance {'loader_mercurial': diff --git a/site-modules/profile/manifests/swh/deploy/worker/loader_pypi.pp b/site-modules/profile/manifests/swh/deploy/worker/loader_pypi.pp --- a/site-modules/profile/manifests/swh/deploy/worker/loader_pypi.pp +++ b/site-modules/profile/manifests/swh/deploy/worker/loader_pypi.pp @@ -5,7 +5,7 @@ $packages = ['python3-swh.loader.pypi'] package {$packages: - ensure => 'latest', + ensure => 'present', } ::profile::swh::deploy::worker::instance {'loader_pypi': diff --git a/site-modules/profile/manifests/swh/deploy/worker/loader_svn.pp b/site-modules/profile/manifests/swh/deploy/worker/loader_svn.pp --- a/site-modules/profile/manifests/swh/deploy/worker/loader_svn.pp +++ b/site-modules/profile/manifests/swh/deploy/worker/loader_svn.pp @@ -5,7 +5,7 @@ $private_tmp = lookup('swh::deploy::worker::loader_svn::private_tmp') package {$packages: - ensure => 'latest', + ensure => 'present', } ::profile::swh::deploy::worker::instance {'loader_svn': diff --git a/site-modules/profile/manifests/swh/deploy/worker/storage_archiver.pp b/site-modules/profile/manifests/swh/deploy/worker/storage_archiver.pp deleted file mode 100644 --- a/site-modules/profile/manifests/swh/deploy/worker/storage_archiver.pp +++ /dev/null @@ -1,14 +0,0 @@ -# Deployment for swh-storage-archiver -class profile::swh::deploy::worker::storage_archiver { - include ::profile::swh::deploy::archiver - - $max_tasks_per_child = lookup('swh::deploy::worker::storage_archiver::max_tasks_per_child') - - ::profile::swh::deploy::worker::instance {'storage_archiver': - ensure => present, - max_tasks_per_child => $max_tasks_per_child, - require => [ - Package[$packages], - ], - } -}