diff --git a/data/common/common.yaml b/data/common/common.yaml --- a/data/common/common.yaml +++ b/data/common/common.yaml @@ -2766,40 +2766,32 @@ swh::deploy::base_indexer::config_directory: "%{hiera('swh::conf_directory')}/indexer" -swh::deploy::indexer_journal_client::config_file: "journal_client.yml" -swh::deploy::indexer_journal_client::user: swhworker -swh::deploy::indexer_journal_client::group: swhworker -swh::deploy::indexer_journal_client::config: - journal: - brokers: "%{alias('swh::deploy::journal::brokers')}" - group_id: swh.indexer.journal_client - prefix: "%{alias('swh::deploy::journal::prefix')}" - scheduler: "%{alias('swh::remote_service::scheduler::config::writable')}" - tools: - name: swh-metadata-detector - version: 0.0.2 - configuration: {} - # for all indexers swh::deploy::indexer::sentry_swh_package: swh.indexer swh::deploy::indexer::sentry_environment: "%{alias('swh::deploy::environment')}" swh::deploy::indexer::sentry_dsn: "https://%{lookup('swh::deploy::indexer::sentry_token')}@sentry.softwareheritage.org/5" +swh::deploy::indexer::user: swhworker +swh::deploy::indexer::group: swhworker +# To remove once previous implementations services are deployed swh::deploy::worker::indexer_content_mimetype::config_file: "%{hiera('swh::conf_directory')}/indexer_content_mimetype.yml" swh::deploy::worker::indexer_content_mimetype::concurrency: 1 swh::deploy::worker::indexer_content_mimetype::loglevel: info +swh::deploy::worker::indexer_content_mimetype::config: {} + +swh::deploy::worker::indexer_fossology_license::config_file: "%{hiera('swh::conf_directory')}/indexer_fossology_license.yml" +swh::deploy::worker::indexer_fossology_license::concurrency: 1 +swh::deploy::worker::indexer_fossology_license::loglevel: info +swh::deploy::worker::indexer_fossology_license::config: {} + + +swh::deploy::indexer_journal_client::mimetype::config_file: "mimetype.yml" +swh::deploy::indexer_journal_client::mimetype::loglevel: INFO # Contains a password: in private data -swh::deploy::worker::indexer_content_mimetype::config: - scheduler: "%{alias('swh::remote_service::scheduler::config::writable')}" +swh::deploy::indexer_journal_client::mimetype::config: indexer_storage: "%{alias('swh::remote_service::indexer::config::writable')}" objstorage: "%{alias('swh::remote_service::objstorage::config')}" storage: "%{alias('swh::remote_service::storage::config')}" - celery: - task_broker: "%{alias('swh::deploy::worker::task_broker')}" - task_modules: - - swh.indexer.tasks - task_queues: - - swh.indexer.tasks.ContentMimetypePartition tools: name: file version: 2:0.4.15-2 @@ -2808,21 +2800,48 @@ debian-package: python3-magic write_batch_size: 1000 -swh::deploy::worker::indexer_origin_intrinsic_metadata::config_file: "%{hiera('swh::conf_directory')}/indexer_origin_intrinsic_metadata.yml" -swh::deploy::worker::indexer_origin_intrinsic_metadata::concurrency: 1 -swh::deploy::worker::indexer_origin_intrinsic_metadata::loglevel: info +swh::deploy::indexer_journal_client::fossology_license::config_file: "fossology_license.yml" +swh::deploy::indexer_journal_client::fossology_license::loglevel: INFO # Contains a password: in private data -swh::deploy::worker::indexer_origin_intrinsic_metadata::config: +swh::deploy::indexer_journal_client::fossology_license::config: + indexer_storage: "%{alias('swh::remote_service::indexer::config::writable')}" + objstorage: "%{alias('swh::remote_service::objstorage::config')}" + storage: "%{alias('swh::remote_service::storage::config')}" + workdir: /tmp/swh/indexer.fossology.license/ + tools: + name: 'nomos' + version: '3.1-1~bpo9~swh+1' + configuration: + command_line: "nomossa " + write_batch_size: 1000 + +swh::deploy::indexer_journal_client::extrinsic_metadata::config_file: "extrinsic_metadata.yml" +swh::deploy::indexer_journal_client::extrinsic_metadata::loglevel: INFO +swh::deploy::indexer_journal_client::extrinsic_metadata::config: + scheduler: "%{alias('swh::remote_service::scheduler::config::writable')}" + indexer_storage: "%{alias('swh::remote_service::indexer::config::writable')}" + objstorage: "%{alias('swh::remote_service::objstorage::config')}" + journal: + brokers: "%{alias('swh::deploy::journal::brokers')}" + group_id: swh.indexer.journal_client + prefix: "%{alias('swh::deploy::journal::prefix')}" scheduler: "%{alias('swh::remote_service::scheduler::config::writable')}" + tools: + name: swh-metadata-detector + version: 0.0.2 + configuration: {} + +swh::deploy::indexer_journal_client::intrinsic_metadata::config_file: "intrinsic_metadata.yml" +swh::deploy::indexer_journal_client::intrinsic_metadata::loglevel: INFO +swh::deploy::indexer_journal_client::intrinsic_metadata::config: indexer_storage: "%{alias('swh::remote_service::indexer::config::writable')}" objstorage: "%{alias('swh::remote_service::objstorage::config')}" storage: "%{alias('swh::remote_service::storage::config')}" - celery: - task_broker: "%{alias('swh::deploy::worker::task_broker')}" - task_modules: - - swh.indexer.tasks - task_queues: - - swh.indexer.tasks.OriginMetadata + journal: + brokers: "%{alias('swh::deploy::journal::brokers')}" + group_id: swh.indexer.journal_client + prefix: "%{alias('swh::deploy::journal::prefix')}" + scheduler: "%{alias('swh::remote_service::scheduler::config::writable')}" tools: name: swh-metadata-detector version: 0.0.2 @@ -2840,28 +2859,6 @@ batch_size_retrieve_content: 10000 batch_size_update: 5000 -swh::deploy::worker::indexer_fossology_license::config_file: "%{hiera('swh::conf_directory')}/indexer_fossology_license.yml" -swh::deploy::worker::indexer_fossology_license::concurrency: 1 -swh::deploy::worker::indexer_fossology_license::loglevel: info -# Contains a password: in private data -swh::deploy::worker::indexer_fossology_license::config: - indexer_storage: "%{alias('swh::remote_service::indexer::config::writable')}" - objstorage: "%{alias('swh::remote_service::objstorage::config')}" - storage: "%{alias('swh::remote_service::storage::config')}" - workdir: /tmp/swh/indexer.fossology.license/ - tools: - name: 'nomos' - version: '3.1-1~bpo9~swh+1' - configuration: - command_line: "nomossa " - celery: - task_broker: "%{alias('swh::deploy::worker::task_broker')}" - task_modules: - - swh.indexer.tasks - task_queues: - - swh.indexer.tasks.ContentFossologyLicensePartition - write_batch_size: 1000 - swh::deploy::worker::vault_cooker::config_file: "%{hiera('swh::conf_directory')}/vault_cooker.yml" swh::deploy::worker::vault_cooker::concurrency: 20 swh::deploy::worker::vault_cooker::loglevel: info diff --git a/data/deployments/production/azure_euwest.yaml b/data/deployments/production/azure_euwest.yaml --- a/data/deployments/production/azure_euwest.yaml +++ b/data/deployments/production/azure_euwest.yaml @@ -3,6 +3,7 @@ - indexer_content_mimetype - indexer_fossology_license - indexer_origin_intrinsic_metadata + - indexer_origin_extrinsic_metadata swh::remote_service::storage::config: "%{alias('swh::remote_service::storage::config::azure')}" swh::remote_service::indexer::config: "%{alias('swh::remote_service::indexer::config::azure')}" diff --git a/data/deployments/staging/common.yaml b/data/deployments/staging/common.yaml --- a/data/deployments/staging/common.yaml +++ b/data/deployments/staging/common.yaml @@ -37,6 +37,7 @@ - indexer_content_mimetype - indexer_fossology_license - indexer_origin_intrinsic_metadata + - indexer_origin_extrinsic_metadata - checker_deposit - loader_archive - loader_bzr diff --git a/site-modules/profile/manifests/swh/deploy/base_indexer.pp b/site-modules/profile/manifests/swh/deploy/base_indexer.pp --- a/site-modules/profile/manifests/swh/deploy/base_indexer.pp +++ b/site-modules/profile/manifests/swh/deploy/base_indexer.pp @@ -1,6 +1,12 @@ # Base class for indexer related manifests class profile::swh::deploy::base_indexer { $config_directory = lookup('swh::deploy::base_indexer::config_directory') + $service_basename = "swh-indexer-journal-client" + $unit_name = "${service_basename}@.service" + + $user = lookup("swh::deploy::indexer::user") + $group = lookup("swh::deploy::indexer::group") + $sentry_name = 'indexer' file {$config_directory: ensure => 'directory', @@ -13,4 +19,15 @@ package {$packages: ensure => 'present', } + + # Template uses variables + # - $user + # - $group + # - $sentry_dsn + # - $sentry_environment + # - $sentry_swh_package + ::systemd::unit_file {$unit_name: + ensure => present, + content => template("profile/swh/deploy/journal/${unit_name}.erb"), + } } diff --git a/site-modules/profile/manifests/swh/deploy/indexer_journal_client.pp b/site-modules/profile/manifests/swh/deploy/indexer_journal_client.pp --- a/site-modules/profile/manifests/swh/deploy/indexer_journal_client.pp +++ b/site-modules/profile/manifests/swh/deploy/indexer_journal_client.pp @@ -1,43 +1,77 @@ # Deployment of the swh.indexer.journal_client -class profile::swh::deploy::indexer_journal_client { +define profile::swh::deploy::indexer_journal_client ( + $ensure = present, + $instance_name = $title, + $sentry_name = $title, +) +{ include ::profile::swh::deploy::base_indexer include ::profile::swh::deploy::journal - $config_file = lookup('swh::deploy::indexer_journal_client::config_file') - $config_directory = lookup('swh::deploy::base_indexer::config_directory') + $config_file = lookup("swh::deploy::indexer_journal_client::${instance_name}::config_file") + $config_directory = $::profile::swh::deploy::base_indexer::config_directory $config_path = "${config_directory}/${config_file}" - $config = lookup('swh::deploy::indexer_journal_client::config') - $user = lookup('swh::deploy::indexer_journal_client::user') - $group = lookup('swh::deploy::indexer_journal_client::group') + $service_basename = $::profile::swh::deploy::base_indexer::service_basename + $service_name = "${service_basename}@${instance_name}.service" + $parameters_conf_path = "${service_name}/parameters.conf" - $service_name = 'swh-indexer-journal-client' - $unit_name = "${service_name}.service" + case $ensure { + 'present', 'running': { + $config = lookup("swh::deploy::indexer_journal_client::${instance_name}::config") + $loglevel = lookup("swh::deploy::indexer_journal_client::${instance_name}::loglevel") - $sentry_dsn = lookup("swh::deploy::indexer::sentry_dsn", Optional[String], 'first', undef) - $sentry_environment = lookup("swh::deploy::indexer::sentry_environment", Optional[String], 'first', undef) - $sentry_swh_package = lookup("swh::deploy::indexer::sentry_swh_package", Optional[String], 'first', undef) + $sentry_dsn = lookup("swh::deploy::indexer::sentry_dsn", Optional[String], "first", undef) + $sentry_environment = lookup("swh::deploy::indexer::sentry_environment", Optional[String], "first", undef) + $sentry_swh_package = lookup("swh::deploy::indexer::sentry_swh_package", Optional[String], "first", undef) - file {$config_path: - ensure => present, - owner => 'root', - group => $group, - mode => '0640', - content => inline_template("<%= @config.to_yaml %>\n"), - notify => Service[$service_name], - } + file {$config_path: + ensure => present, + owner => "root", + group => $::profile::swh::deploy::base_indexer::group, + mode => "0640", + content => inline_template("<%= @config.to_yaml %>\n"), + notify => Service[$service_name], + require => File[$config_directory], + } + + # Template uses variables + # - $config_path + # - $sentry_dsn + # - $sentry_environment + # - $sentry_swh_package + # - $loglevel + ::systemd::dropin_file {$parameters_conf_path: + ensure => present, + unit => $service_name, + filename => 'parameters.conf', + content => template('profile/swh/deploy/journal/parameters.conf.erb'), + } + + service {$service_name: + ensure => running, + enable => true, + require => [ + File[$config_path], + ], + } + } + + # Otherwise, clean up everything + default: { + ::systemd::dropin_file {$parameters_conf_path: + ensure => absent, + unit => $service_name, + filename => 'parameters.conf', + } + + service {$service_basename: + ensure => stopped, + } - # Template uses variables - # - $user - # - $group - # - $sentry_dsn - # - $sentry_environment - # - $sentry_swh_package - ::systemd::unit_file {$unit_name: - ensure => present, - content => template("profile/swh/deploy/journal/${unit_name}.erb"), - } ~> service {$service_name: - ensure => running, - enable => true, + file {$config_path: + ensure => absent, + } + } } } diff --git a/site-modules/profile/manifests/swh/deploy/worker/indexer_content_mimetype.pp b/site-modules/profile/manifests/swh/deploy/worker/indexer_content_mimetype.pp --- a/site-modules/profile/manifests/swh/deploy/worker/indexer_content_mimetype.pp +++ b/site-modules/profile/manifests/swh/deploy/worker/indexer_content_mimetype.pp @@ -1,14 +1,19 @@ -# Deployment for swh-indexer-mimetype - +# Deployment for indexer mimetype class profile::swh::deploy::worker::indexer_content_mimetype { include ::profile::swh::deploy::indexer - Package[$::profile::swh::deploy::base_indexer::packages] ~> ::profile::swh::deploy::worker::instance {'indexer_content_mimetype': - ensure => present, - sentry_name => 'indexer', - send_task_events => true, - require => [ + ::profile::swh::deploy::worker::instance {'indexer_content_mimetype': + ensure => absent, + } + + Package[$::profile::swh::deploy::base_indexer::packages] + ~> ::profile::swh::deploy::indexer_journal_client {'mimetype': + ensure => present, + sentry_name => $::profile::swh::deploy::base_indexer::sentry_name, + require => [ + Package[$::profile::swh::deploy::base_indexer::packages], Class['profile::swh::deploy::indexer'] ], } + } diff --git a/site-modules/profile/manifests/swh/deploy/worker/indexer_fossology_license.pp b/site-modules/profile/manifests/swh/deploy/worker/indexer_fossology_license.pp --- a/site-modules/profile/manifests/swh/deploy/worker/indexer_fossology_license.pp +++ b/site-modules/profile/manifests/swh/deploy/worker/indexer_fossology_license.pp @@ -1,21 +1,26 @@ -# Deployment for swh-indexer-fossology-license - +# Deployment for indexer fossology-license class profile::swh::deploy::worker::indexer_fossology_license { include ::profile::swh::deploy::indexer $packages = ['fossology-nomossa'] - package {$packages: ensure => 'present', require => Apt::Source['softwareheritage'], } - Package[$::profile::swh::deploy::base_indexer::packages] ~> ::profile::swh::deploy::worker::instance {'indexer_fossology_license': - ensure => present, - sentry_name => 'indexer', - send_task_events => true, - require => [ - Class['profile::swh::deploy::indexer'], - Package[$packages], + # Clean up previous indexer service implementation + ::profile::swh::deploy::worker::instance {'indexer_fossology_license': + ensure => absent, + } + + Package[$::profile::swh::deploy::base_indexer::packages] + ~> ::profile::swh::deploy::indexer_journal_client {'fossology_license': + ensure => present, + sentry_name => $::profile::swh::deploy::base_indexer::sentry_name, + require => [ + Package[$::profile::swh::deploy::base_indexer::packages], + Package[$packages::profile::swh::deploy::base_indexer::packages], + Class['profile::swh::deploy::indexer'] ], } + } diff --git a/site-modules/profile/manifests/swh/deploy/worker/indexer_origin_extrinsic_metadata.pp b/site-modules/profile/manifests/swh/deploy/worker/indexer_origin_extrinsic_metadata.pp new file mode 100644 --- /dev/null +++ b/site-modules/profile/manifests/swh/deploy/worker/indexer_origin_extrinsic_metadata.pp @@ -0,0 +1,13 @@ +# Deployment for indexer origin-extrinsic-metadata +class profile::swh::deploy::worker::indexer_origin_extrinsic_metadata { + include ::profile::swh::deploy::indexer + + ::profile::swh::deploy::indexer_journal_client {'extrinsic_metadata': + ensure => present, + sentry_name => $::profile::swh::deploy::base_indexer::sentry_name, + require => [ + Package[$::profile::swh::deploy::base_indexer::packages], + Class['profile::swh::deploy::indexer'] + ], + } +} diff --git a/site-modules/profile/manifests/swh/deploy/worker/indexer_origin_intrinsic_metadata.pp b/site-modules/profile/manifests/swh/deploy/worker/indexer_origin_intrinsic_metadata.pp --- a/site-modules/profile/manifests/swh/deploy/worker/indexer_origin_intrinsic_metadata.pp +++ b/site-modules/profile/manifests/swh/deploy/worker/indexer_origin_intrinsic_metadata.pp @@ -1,11 +1,13 @@ -# Deployment for swh-indexer-origin-intrinsic-metadata +# Deployment for indexer origin-intrinsic-metadata class profile::swh::deploy::worker::indexer_origin_intrinsic_metadata { include ::profile::swh::deploy::indexer - # Remove deprecated service - ::profile::swh::deploy::worker::instance {'indexer_origin_intrinsic_metadata': - ensure => absent, + ::profile::swh::deploy::indexer_journal_client {'intrinsic_metadata': + ensure => present, + sentry_name => $::profile::swh::deploy::base_indexer::sentry_name, + require => [ + Package[$::profile::swh::deploy::base_indexer::packages], + Class['profile::swh::deploy::indexer'] + ], } - - include ::profile::swh::deploy::indexer_journal_client } diff --git a/site-modules/profile/templates/swh/deploy/journal/parameters.conf.erb b/site-modules/profile/templates/swh/deploy/journal/parameters.conf.erb new file mode 100644 --- /dev/null +++ b/site-modules/profile/templates/swh/deploy/journal/parameters.conf.erb @@ -0,0 +1,9 @@ +# Managed by puppet - modifications will be overwritten +# In defined class profile::swh::deploy::indexer_journal_client + +[Service] +Environment=SWH_CONFIG_FILENAME=<%= @config_path %> +Environment=SWH_SENTRY_DSN=<%= @sentry_dsn %> +Environment=SWH_SENTRY_ENVIRONMENT=<%= @sentry_environment %> +Environment=SWH_MAIN_PACKAGE=<%= @sentry_swh_package %> +Environment=LOGLEVEL=<%= @loglevel %> diff --git a/site-modules/profile/templates/swh/deploy/journal/swh-indexer-journal-client.service.erb b/site-modules/profile/templates/swh/deploy/journal/swh-indexer-journal-client.service.erb deleted file mode 100644 --- a/site-modules/profile/templates/swh/deploy/journal/swh-indexer-journal-client.service.erb +++ /dev/null @@ -1,28 +0,0 @@ -# Indexer Journal Client unit file -# Managed by puppet class profile::swh::deploy::indexer_journal_client -# Changes will be overwritten - -[Unit] -Description=Software Heritage Indexer Journal Client -After=network.target - -[Service] -<%- if !@sentry_dsn.nil? and !@sentry_dsn.empty? -%> -Environment=SWH_SENTRY_DSN=<%= @sentry_dsn %> -<%- end -%> -<%- if !@sentry_environment.nil? and !@sentry_environment.empty? -%> -Environment=SWH_SENTRY_ENVIRONMENT=<%= @sentry_environment %> -<%- end -%> -<%- if !@sentry_swh_package.nil? and !@sentry_swh_package.empty? -%> -Environment=SWH_MAIN_PACKAGE=<%= @sentry_swh_package %> -<%- end -%> -Environment=SWH_CONFIG_FILENAME=<%= @config_path %> -User=<%= @user %> -Group=<%= @group %> -Type=simple -ExecStart=/usr/bin/swh indexer --config-file <%= @config_path %> journal-client '*' -Restart=always -RestartSec=10 - -[Install] -WantedBy=multi-user.target diff --git a/site-modules/profile/templates/swh/deploy/journal/swh-indexer-journal-client@.service.erb b/site-modules/profile/templates/swh/deploy/journal/swh-indexer-journal-client@.service.erb new file mode 100644 --- /dev/null +++ b/site-modules/profile/templates/swh/deploy/journal/swh-indexer-journal-client@.service.erb @@ -0,0 +1,30 @@ +# Indexer Journal Client unit file +# Managed by puppet class profile::swh::deploy::base_indexer +# Changes will be overwritten + +[Unit] +Description=Software Heritage Indexer Journal Client %i +After=network.target + +[Service] +User=<%= @user %> +Group=<%= @group %> + +Type=simple +# Mandatory environment variables (check associated parameters.conf for override) +Environment=SWH_CONFIG_FILENAME=<%= @config_directory %>/%i.yml +Environment=SWH_SENTRY_DSN=<%= @sentry_dsn %> +Environment=SWH_SENTRY_ENVIRONMENT=<%= @sentry_environment %> +Environment=SWH_MAIN_PACKAGE=<%= @sentry_swh_package %> +Environment=LOGLEVEL=INFO + +ExecStart=/usr/bin/swh \ + --log-level $LOGLEVEL \ + indexer \ + --config-file $SWH_CONFIG_FILENAME \ + journal-client %i +Restart=always +RestartSec=10 + +[Install] +WantedBy=multi-user.target