diff --git a/data/defaults.yaml b/data/defaults.yaml --- a/data/defaults.yaml +++ b/data/defaults.yaml @@ -1498,24 +1498,6 @@ - origin - origin_visit -swh::deploy::archiver_content_updater::conf_file: "%{hiera('swh::deploy::worker::archiver::conf_directory')}/content_updater.yml" -swh::deploy::archiver_content_updater::user: swhstorage -swh::deploy::archiver_content_updater::group: swhstorage -swh::deploy::archiver_content_updater::config: - brokers: - - getty.internal.softwareheritage.org - auto_offset_reset: earliest - topic_prefix: swh.journal.objects - consumer_id: swh.journal.archiver.content.updater - object_types: - - content - archiver_storage: - cls: db - args: - dbconn: "host=%{hiera('swh::deploy::worker::archiver::db::host')} user=%{hiera('swh::deploy::worker::archiver::db::user')} dbname=%{hiera('swh::deploy::worker::archiver::db::dbname')} password=%{hiera('swh::deploy::worker::archiver::db::password')}" - sources_present: - - uffizi - swh::deploy::objstorage::conf_directory: "%{hiera('swh::conf_directory')}/objstorage" swh::deploy::objstorage::conf_file: "%{hiera('swh::deploy::objstorage::conf_directory')}/server.yml" swh::deploy::objstorage::user: "%{hiera('swh::deploy::storage::user')}" @@ -1642,30 +1624,6 @@ log_tag: objstorage.checker.repair backup_storages: "%{alias('swh::remote_service::objstorage::config_as_dict')}" -swh::deploy::objstorage_archive_notifier_checker::conf_directory: "%{hiera('swh::deploy::objstorage::conf_directory')}" -swh::deploy::objstorage_archive_notifier_checker::conf_file: "%{hiera('swh::deploy::objstorage_archive_notifier_checker::conf_directory')}/archive_notifier_checker.yml" -swh::deploy::objstorage_archive_notifier_checker::user: "%{hiera('swh::deploy::objstorage::user')}" -swh::deploy::objstorage_archive_notifier_checker::group: "%{hiera('swh::deploy::objstorage::group')}" -swh::deploy::objstorage_archive_notifier_checker::log_tag: objstorage.checker.archive_notifier -swh::deploy::objstorage_archive_notifier_checker::storage_name: this-must-be-overriden-on-a-host-basis -swh::deploy::objstorage_archive_notifier_checker::db::host: "%{hiera('swh::deploy::worker::archiver::db::host')}" -swh::deploy::objstorage_archive_notifier_checker::db::dbname: "%{hiera('swh::deploy::worker::archiver::db::dbname')}" -swh::deploy::objstorage_archive_notifier_checker::db::user: "%{hiera('swh::deploy::worker::archiver::db::user')}" -swh::deploy::objstorage_archive_notifier_checker::db::password: "%{hiera('swh::deploy::worker::archiver::db::password')}" -swh::deploy::objstorage_archive_notifier_checker::config: - storage: - cls: pathslicing - args: - root: "%{hiera('swh::deploy::objstorage::directory')}" - slicing: "%{hiera('swh::deploy::objstorage::slicing')}" - batch_size: 1000 - log_tag: objstorage.checker.archive_notifier - storage_name: this-must-be-overriden-on-a-per-host-basis - archiver_storage: - cls: db - args: - dbconn: "host=%{hiera('swh::deploy::worker::archiver::db::host')} user=%{hiera('swh::deploy::worker::archiver::db::user')} dbname=%{hiera('swh::deploy::worker::archiver::db::dbname')} password=%{hiera('swh::deploy::worker::archiver::db::password')}" - swh::deploy::webapp::backported_packages: stretch: - python3-django @@ -2046,46 +2004,6 @@ - swh_loader_svn_mount_and_load -swh::deploy::worker::archiver::config_file: "%{hiera('swh::conf_directory')}/archiver.yml" -swh::deploy::worker::archiver::archival_max_age: 3600 -swh::deploy::worker::archiver::max_queue_length: 100000 -swh::deploy::worker::archiver::retention_policy: 3 -swh::deploy::worker::archiver::batch_max_size: 5 -swh::deploy::worker::archiver::db::host: belvedere.internal.softwareheritage.org -swh::deploy::worker::archiver::db::port: "%{alias('swh::deploy::db::secondary::port')}" -swh::deploy::worker::archiver::db::dbname: softwareheritage-archiver -swh::deploy::worker::archiver::db::user: "%{hiera('swh::deploy::storage::db::user')}" -swh::deploy::worker::archiver::db::password: "%{hiera('swh::deploy::storage::db::password')}" -swh::deploy::worker::archiver::concurrency: 10 -swh::deploy::worker::archiver::max_tasks_per_child: 500 -swh::deploy::worker::archiver::loglevel: info -# Contains a password: in private data -swh::deploy::worker::archiver::config: - archival_max_age: "%{hiera('swh::deploy::worker::archiver::archival_max_age')}" - retention_policy: "%{hiera('swh::deploy::worker::archiver::retention_policy')}" - batch_max_size: "%{hiera('swh::deploy::worker::archiver::batch_max_size')}" - archiver_storage: - cls: db - args: - dbconn: "host=%{hiera('swh::deploy::worker::archiver::db::host')} port=%{hiera('swh::deploy::worker::archiver::db::port')} user=%{hiera('swh::deploy::worker::archiver::db::user')} dbname=%{hiera('swh::deploy::worker::archiver::db::dbname')} password=%{hiera('swh::deploy::worker::archiver::db::password')}" - storages: - - host: uffizi - <<: *swh_objstorage_config_uffizi - - host: banco - <<: *swh_objstorage_config_banco - - host: azure - <<: *swh_objstorage_config_azure - sources: - - uffizi - - banco - max_queue_length: "%{hiera('swh::deploy::worker::archiver::max_queue_length')}" - celery: - task_modules: - - swh.archiver.tasks - task_queues: - - swh.archiver.tasks.SWHArchiverWithRetentionPolicyTask - - swh.archiver.tasks.SWHArchiverToBackendTask - swh::deploy::base_indexer::config_directory: "%{hiera('swh::conf_directory')}/indexer" swh::deploy::indexer_journal_client::config_file: "journal_client.yml" diff --git a/data/hostname/uffizi.softwareheritage.org.yaml b/data/hostname/uffizi.softwareheritage.org.yaml --- a/data/hostname/uffizi.softwareheritage.org.yaml +++ b/data/hostname/uffizi.softwareheritage.org.yaml @@ -45,9 +45,6 @@ keys: - "%{alias('swh::deploy::objstorage::ceph::rados_id')}" -swh::deploy::worker::instances: - - archiver - icinga2::host::vars: load: high disks: diff --git a/site-modules/profile/manifests/swh/deploy/archiver_content_updater.pp b/site-modules/profile/manifests/swh/deploy/archiver_content_updater.pp deleted file mode 100644 --- a/site-modules/profile/manifests/swh/deploy/archiver_content_updater.pp +++ /dev/null @@ -1,36 +0,0 @@ -# Deployment of the swh.storage.archiver.updater - -class profile::swh::deploy::archiver_content_updater { - include profile::swh::deploy::base_archiver - - $conf_file = lookup('swh::deploy::archiver_content_updater::conf_file') - $user = lookup('swh::deploy::archiver_content_updater::user') - $group = lookup('swh::deploy::archiver_content_updater::group') - - $content_updater_config = lookup('swh::deploy::archiver_content_updater::config') - - $service_name = 'swh-archiver-content-updater' - $unit_name = "${service_name}.service" - - file {$conf_file: - ensure => present, - owner => 'root', - group => $group, - mode => '0640', - content => inline_template("<%= @content_updater_config.to_yaml %>\n"), - notify => Service[$service_name], - } - - # Template uses variables - # - $user - # - $group - # - ::systemd::unit_file {$unit_name: - ensure => present, - content => template('profile/swh/deploy/archiver/swh-content-updater.service.erb'), - } ~> service {$service_name: - ensure => running, - enable => false, - require => File[$conf_file], - } -} diff --git a/site-modules/profile/manifests/swh/deploy/base_archiver.pp b/site-modules/profile/manifests/swh/deploy/base_archiver.pp deleted file mode 100644 --- a/site-modules/profile/manifests/swh/deploy/base_archiver.pp +++ /dev/null @@ -1,10 +0,0 @@ -# Archiver base configuration - -class profile::swh::deploy::base_archiver { - $packages = ['python3-swh.archiver'] - - package {$packages: - ensure => 'installed', - } - -} diff --git a/site-modules/profile/manifests/swh/deploy/objstorage_archive_notifier_checker.pp b/site-modules/profile/manifests/swh/deploy/objstorage_archive_notifier_checker.pp deleted file mode 100644 --- a/site-modules/profile/manifests/swh/deploy/objstorage_archive_notifier_checker.pp +++ /dev/null @@ -1,42 +0,0 @@ -# Deployment of the swh.objstorage.checker.ArchiveNotifierContentChecker - -class profile::swh::deploy::objstorage_archive_notifier_checker { - $conf_directory = lookup('swh::deploy::objstorage_archive_notifier_checker::conf_directory') - $conf_file = lookup('swh::deploy::objstorage_archive_notifier_checker::conf_file') - $user = lookup('swh::deploy::objstorage_archive_notifier_checker::user') - $group = lookup('swh::deploy::objstorage_archive_notifier_checker::group') - - # configuration file - $archive_notifier_config = lookup('swh::deploy::objstorage_archive_notifier_checker::config') - - $swh_packages = ['python3-swh.objstorage.checker'] - - package {$swh_packages: - ensure => latest, - require => Apt::Source['softwareheritage'], - } - - file {$conf_directory: - ensure => directory, - owner => 'root', - group => $group, - mode => '0750', - } - - file {$conf_file: - ensure => present, - owner => 'root', - group => $group, - mode => '0640', - content => inline_template("<%= @archive_notifier_config.to_yaml %>\n"), - } - - ::systemd::unit_file {'objstorage_archive_notifier_checker.service': - ensure => present, - content => template('profile/swh/deploy/storage/objstorage_archive_notifier_checker.service.erb'), - require => [ - File[$conf_file], - Package[$swh_packages], - ] - } -} diff --git a/site-modules/profile/manifests/swh/deploy/objstorage_log_checker.pp b/site-modules/profile/manifests/swh/deploy/objstorage_log_checker.pp deleted file mode 100644 --- a/site-modules/profile/manifests/swh/deploy/objstorage_log_checker.pp +++ /dev/null @@ -1,43 +0,0 @@ -# Deployment of the swh.objstorage.checker.LogContentChecker - -class profile::swh::deploy::objstorage_log_checker { - $conf_directory = lookup('swh::deploy::objstorage_log_checker::conf_directory') - $conf_file = lookup('swh::deploy::objstorage_log_checker::conf_file') - $user = lookup('swh::deploy::objstorage_log_checker::user') - $group = lookup('swh::deploy::objstorage_log_checker::group') - - # configuration file - $log_checker_config = lookup('swh::deploy::objstorage_log_checker::config') - - $swh_packages = ['python3-swh.objstorage.checker'] - - package {$swh_packages: - ensure => latest, - require => Apt::Source['softwareheritage'], - } - - file {$conf_directory: - ensure => directory, - owner => 'root', - group => $group, - mode => '0750', - } - - file {$conf_file: - ensure => present, - owner => 'root', - group => $group, - mode => '0640', - content => inline_template("<%= @log_checker_config.to_yaml %>\n"), - } - - ::systemd::unit_file {'objstorage_log_checker.service': - ensure => present, - content => template('profile/swh/deploy/storage/objstorage_log_checker.service.erb'), - require => [ - File[$conf_file], - Package[$swh_packages], - ] - } - -} diff --git a/site-modules/profile/manifests/swh/deploy/objstorage_repair_checker.pp b/site-modules/profile/manifests/swh/deploy/objstorage_repair_checker.pp deleted file mode 100644 --- a/site-modules/profile/manifests/swh/deploy/objstorage_repair_checker.pp +++ /dev/null @@ -1,42 +0,0 @@ -# Deployment of the swh.objstorage.checker.RepairContentChecker - -class profile::swh::deploy::objstorage_repair_checker { - $conf_directory = lookup('swh::deploy::objstorage_repair_checker::conf_directory') - $conf_file = lookup('swh::deploy::objstorage_repair_checker::conf_file') - $user = lookup('swh::deploy::objstorage_repair_checker::user') - $group = lookup('swh::deploy::objstorage_repair_checker::group') - - $repair_checker_config = lookup('swh::deploy::objstorage_repair_checker::config') - - $swh_packages = ['python3-swh.objstorage.checker'] - - package {$swh_packages: - ensure => latest, - require => Apt::Source['softwareheritage'], - } - - file {$conf_directory: - ensure => directory, - owner => 'root', - group => $group, - mode => '0750', - } - - file {$conf_file: - ensure => present, - owner => 'root', - group => $group, - mode => '0640', - content => inline_template("<%= @repair_checker_config.to_yaml %>\n"), - } - - ::systemd::unit_file {'objstorage_repair_checker.service': - ensure => present, - content => template('profile/swh/deploy/storage/objstorage_repair_checker.service.erb'), - require => [ - File[$conf_file], - Package[$swh_packages], - ] - } - -} diff --git a/site-modules/profile/manifests/swh/deploy/worker/archiver.pp b/site-modules/profile/manifests/swh/deploy/worker/archiver.pp deleted file mode 100644 --- a/site-modules/profile/manifests/swh/deploy/worker/archiver.pp +++ /dev/null @@ -1,14 +0,0 @@ -# Deployment for archiver (content) -class profile::swh::deploy::worker::archiver { - include ::profile::swh::deploy::base_archiver - - $max_tasks_per_child = lookup('swh::deploy::worker::archiver::max_tasks_per_child') - - ::profile::swh::deploy::worker::instance {'archiver': - ensure => present, - max_tasks_per_child => $max_tasks_per_child, - require => [ - Package[$packages], - ], - } -} diff --git a/site-modules/role/manifests/swh_storage.pp b/site-modules/role/manifests/swh_storage.pp --- a/site-modules/role/manifests/swh_storage.pp +++ b/site-modules/role/manifests/swh_storage.pp @@ -3,6 +3,5 @@ include profile::swh::deploy::storage include profile::swh::deploy::indexer_storage include profile::swh::deploy::objstorage - include profile::swh::deploy::worker include profile::swh::deploy::objstorage_ceph }