D8161.id29475.diff
No OneTemporary
Actions

Size

20 KB

Subscribers

None

D8161.id29475.diff
View Options

	diff --git a/data/common/common.yaml b/data/common/common.yaml
	--- a/data/common/common.yaml
	+++ b/data/common/common.yaml
	@@ -2766,40 +2766,32 @@

	swh::deploy::base_indexer::config_directory: "%{hiera('swh::conf_directory')}/indexer"

	-swh::deploy::indexer_journal_client::config_file: "journal_client.yml"
	-swh::deploy::indexer_journal_client::user: swhworker
	-swh::deploy::indexer_journal_client::group: swhworker
	-swh::deploy::indexer_journal_client::config:
	- journal:
	- brokers: "%{alias('swh::deploy::journal::brokers')}"
	- group_id: swh.indexer.journal_client
	- prefix: "%{alias('swh::deploy::journal::prefix')}"
	- scheduler: "%{alias('swh::remote_service::scheduler::config::writable')}"
	- tools:
	- name: swh-metadata-detector
	- version: 0.0.2
	- configuration: {}
	-
	# for all indexers
	swh::deploy::indexer::sentry_swh_package: swh.indexer
	swh::deploy::indexer::sentry_environment: "%{alias('swh::deploy::environment')}"
	swh::deploy::indexer::sentry_dsn: "https://%{lookup('swh::deploy::indexer::sentry_token')}@sentry.softwareheritage.org/5"
	+swh::deploy::indexer::user: swhworker
	+swh::deploy::indexer::group: swhworker

	+# To remove once previous implementations services are deployed
	swh::deploy::worker::indexer_content_mimetype::config_file: "%{hiera('swh::conf_directory')}/indexer_content_mimetype.yml"
	swh::deploy::worker::indexer_content_mimetype::concurrency: 1
	swh::deploy::worker::indexer_content_mimetype::loglevel: info
	+swh::deploy::worker::indexer_content_mimetype::config: {}
	+
	+swh::deploy::worker::indexer_fossology_license::config_file: "%{hiera('swh::conf_directory')}/indexer_fossology_license.yml"
	+swh::deploy::worker::indexer_fossology_license::concurrency: 1
	+swh::deploy::worker::indexer_fossology_license::loglevel: info
	+swh::deploy::worker::indexer_fossology_license::config: {}
	+
	+
	+swh::deploy::indexer_journal_client::mimetype::config_file: "mimetype.yml"
	+swh::deploy::indexer_journal_client::mimetype::loglevel: INFO
	# Contains a password: in private data
	-swh::deploy::worker::indexer_content_mimetype::config:
	- scheduler: "%{alias('swh::remote_service::scheduler::config::writable')}"
	+swh::deploy::indexer_journal_client::mimetype::config:
	indexer_storage: "%{alias('swh::remote_service::indexer::config::writable')}"
	objstorage: "%{alias('swh::remote_service::objstorage::config')}"
	storage: "%{alias('swh::remote_service::storage::config')}"
	- celery:
	- task_broker: "%{alias('swh::deploy::worker::task_broker')}"
	- task_modules:
	- - swh.indexer.tasks
	- task_queues:
	- - swh.indexer.tasks.ContentMimetypePartition
	tools:
	name: file
	version: 2:0.4.15-2
	@@ -2808,21 +2800,48 @@
	debian-package: python3-magic
	write_batch_size: 1000

	-swh::deploy::worker::indexer_origin_intrinsic_metadata::config_file: "%{hiera('swh::conf_directory')}/indexer_origin_intrinsic_metadata.yml"
	-swh::deploy::worker::indexer_origin_intrinsic_metadata::concurrency: 1
	-swh::deploy::worker::indexer_origin_intrinsic_metadata::loglevel: info
	+swh::deploy::indexer_journal_client::fossology_license::config_file: "fossology_license.yml"
	+swh::deploy::indexer_journal_client::fossology_license::loglevel: INFO
	# Contains a password: in private data
	-swh::deploy::worker::indexer_origin_intrinsic_metadata::config:
	+swh::deploy::indexer_journal_client::fossology_license::config:
	+ indexer_storage: "%{alias('swh::remote_service::indexer::config::writable')}"
	+ objstorage: "%{alias('swh::remote_service::objstorage::config')}"
	+ storage: "%{alias('swh::remote_service::storage::config')}"
	+ workdir: /tmp/swh/indexer.fossology.license/
	+ tools:
	+ name: 'nomos'
	+ version: '3.1-1~bpo9~swh+1'
	+ configuration:
	+ command_line: "nomossa <filepath>"
	+ write_batch_size: 1000
	+
	+swh::deploy::indexer_journal_client::extrinsic_metadata::config_file: "extrinsic_metadata.yml"
	+swh::deploy::indexer_journal_client::extrinsic_metadata::loglevel: INFO
	+swh::deploy::indexer_journal_client::extrinsic_metadata::config:
	+ scheduler: "%{alias('swh::remote_service::scheduler::config::writable')}"
	+ indexer_storage: "%{alias('swh::remote_service::indexer::config::writable')}"
	+ objstorage: "%{alias('swh::remote_service::objstorage::config')}"
	+ journal:
	+ brokers: "%{alias('swh::deploy::journal::brokers')}"
	+ group_id: swh.indexer.journal_client
	+ prefix: "%{alias('swh::deploy::journal::prefix')}"
	scheduler: "%{alias('swh::remote_service::scheduler::config::writable')}"
	+ tools:
	+ name: swh-metadata-detector
	+ version: 0.0.2
	+ configuration: {}
	+
	+swh::deploy::indexer_journal_client::intrinsic_metadata::config_file: "intrinsic_metadata.yml"
	+swh::deploy::indexer_journal_client::intrinsic_metadata::loglevel: INFO
	+swh::deploy::indexer_journal_client::intrinsic_metadata::config:
	indexer_storage: "%{alias('swh::remote_service::indexer::config::writable')}"
	objstorage: "%{alias('swh::remote_service::objstorage::config')}"
	storage: "%{alias('swh::remote_service::storage::config')}"
	- celery:
	- task_broker: "%{alias('swh::deploy::worker::task_broker')}"
	- task_modules:
	- - swh.indexer.tasks
	- task_queues:
	- - swh.indexer.tasks.OriginMetadata
	+ journal:
	+ brokers: "%{alias('swh::deploy::journal::brokers')}"
	+ group_id: swh.indexer.journal_client
	+ prefix: "%{alias('swh::deploy::journal::prefix')}"
	+ scheduler: "%{alias('swh::remote_service::scheduler::config::writable')}"
	tools:
	name: swh-metadata-detector
	version: 0.0.2
	@@ -2840,28 +2859,6 @@
	batch_size_retrieve_content: 10000
	batch_size_update: 5000

	-swh::deploy::worker::indexer_fossology_license::config_file: "%{hiera('swh::conf_directory')}/indexer_fossology_license.yml"
	-swh::deploy::worker::indexer_fossology_license::concurrency: 1
	-swh::deploy::worker::indexer_fossology_license::loglevel: info
	-# Contains a password: in private data
	-swh::deploy::worker::indexer_fossology_license::config:
	- indexer_storage: "%{alias('swh::remote_service::indexer::config::writable')}"
	- objstorage: "%{alias('swh::remote_service::objstorage::config')}"
	- storage: "%{alias('swh::remote_service::storage::config')}"
	- workdir: /tmp/swh/indexer.fossology.license/
	- tools:
	- name: 'nomos'
	- version: '3.1-1~bpo9~swh+1'
	- configuration:
	- command_line: "nomossa <filepath>"
	- celery:
	- task_broker: "%{alias('swh::deploy::worker::task_broker')}"
	- task_modules:
	- - swh.indexer.tasks
	- task_queues:
	- - swh.indexer.tasks.ContentFossologyLicensePartition
	- write_batch_size: 1000
	-
	swh::deploy::worker::vault_cooker::config_file: "%{hiera('swh::conf_directory')}/vault_cooker.yml"
	swh::deploy::worker::vault_cooker::concurrency: 20
	swh::deploy::worker::vault_cooker::loglevel: info
	diff --git a/data/deployments/production/azure_euwest.yaml b/data/deployments/production/azure_euwest.yaml
	--- a/data/deployments/production/azure_euwest.yaml
	+++ b/data/deployments/production/azure_euwest.yaml
	@@ -3,6 +3,7 @@
	- indexer_content_mimetype
	- indexer_fossology_license
	- indexer_origin_intrinsic_metadata
	+ - indexer_origin_extrinsic_metadata

	swh::remote_service::storage::config: "%{alias('swh::remote_service::storage::config::azure')}"
	swh::remote_service::indexer::config: "%{alias('swh::remote_service::indexer::config::azure')}"
	diff --git a/data/deployments/staging/common.yaml b/data/deployments/staging/common.yaml
	--- a/data/deployments/staging/common.yaml
	+++ b/data/deployments/staging/common.yaml
	@@ -37,6 +37,7 @@
	- indexer_content_mimetype
	- indexer_fossology_license
	- indexer_origin_intrinsic_metadata
	+ - indexer_origin_extrinsic_metadata
	- checker_deposit
	- loader_archive
	- loader_bzr
	diff --git a/site-modules/profile/manifests/swh/deploy/base_indexer.pp b/site-modules/profile/manifests/swh/deploy/base_indexer.pp
	--- a/site-modules/profile/manifests/swh/deploy/base_indexer.pp
	+++ b/site-modules/profile/manifests/swh/deploy/base_indexer.pp
	@@ -1,6 +1,12 @@
	# Base class for indexer related manifests
	class profile::swh::deploy::base_indexer {
	$config_directory = lookup('swh::deploy::base_indexer::config_directory')
	+ $service_basename = "swh-indexer-journal-client"
	+ $unit_name = "${service_basename}@.service"
	+
	+ $user = lookup("swh::deploy::indexer::user")
	+ $group = lookup("swh::deploy::indexer::group")
	+ $sentry_name = 'indexer'

	file {$config_directory:
	ensure => 'directory',
	@@ -13,4 +19,15 @@
	package {$packages:
	ensure => 'present',
	}
	+
	+ # Template uses variables
	+ # - $user
	+ # - $group
	+ # - $sentry_dsn
	+ # - $sentry_environment
	+ # - $sentry_swh_package
	+ ::systemd::unit_file {$unit_name:
	+ ensure => present,
	+ content => template("profile/swh/deploy/journal/${unit_name}.erb"),
	+ }
	}
	diff --git a/site-modules/profile/manifests/swh/deploy/indexer_journal_client.pp b/site-modules/profile/manifests/swh/deploy/indexer_journal_client.pp
	--- a/site-modules/profile/manifests/swh/deploy/indexer_journal_client.pp
	+++ b/site-modules/profile/manifests/swh/deploy/indexer_journal_client.pp
	@@ -1,43 +1,77 @@
	# Deployment of the swh.indexer.journal_client
	-class profile::swh::deploy::indexer_journal_client {
	+define profile::swh::deploy::indexer_journal_client (
	+ $ensure = present,
	+ $instance_name = $title,
	+ $sentry_name = $title,
	+)
	+{
	include ::profile::swh::deploy::base_indexer
	include ::profile::swh::deploy::journal

	- $config_file = lookup('swh::deploy::indexer_journal_client::config_file')
	- $config_directory = lookup('swh::deploy::base_indexer::config_directory')
	+ $config_file = lookup("swh::deploy::indexer_journal_client::${instance_name}::config_file")
	+ $config_directory = $::profile::swh::deploy::base_indexer::config_directory
	$config_path = "${config_directory}/${config_file}"
	- $config = lookup('swh::deploy::indexer_journal_client::config')

	- $user = lookup('swh::deploy::indexer_journal_client::user')
	- $group = lookup('swh::deploy::indexer_journal_client::group')
	+ $service_basename = $::profile::swh::deploy::base_indexer::service_basename
	+ $service_name = "${service_basename}@${instance_name}.service"
	+ $parameters_conf_path = "${service_name}/parameters.conf"

	- $service_name = 'swh-indexer-journal-client'
	- $unit_name = "${service_name}.service"
	+ case $ensure {
	+ 'present', 'running': {
	+ $config = lookup("swh::deploy::indexer_journal_client::${instance_name}::config")
	+ $loglevel = lookup("swh::deploy::indexer_journal_client::${instance_name}::loglevel")

	- $sentry_dsn = lookup("swh::deploy::indexer::sentry_dsn", Optional[String], 'first', undef)
	- $sentry_environment = lookup("swh::deploy::indexer::sentry_environment", Optional[String], 'first', undef)
	- $sentry_swh_package = lookup("swh::deploy::indexer::sentry_swh_package", Optional[String], 'first', undef)
	+ $sentry_dsn = lookup("swh::deploy::indexer::sentry_dsn", Optional[String], "first", undef)
	+ $sentry_environment = lookup("swh::deploy::indexer::sentry_environment", Optional[String], "first", undef)
	+ $sentry_swh_package = lookup("swh::deploy::indexer::sentry_swh_package", Optional[String], "first", undef)

	- file {$config_path:
	- ensure => present,
	- owner => 'root',
	- group => $group,
	- mode => '0640',
	- content => inline_template("<%= @config.to_yaml %>\n"),
	- notify => Service[$service_name],
	- }
	+ file {$config_path:
	+ ensure => present,
	+ owner => "root",
	+ group => $::profile::swh::deploy::base_indexer::group,
	+ mode => "0640",
	+ content => inline_template("<%= @config.to_yaml %>\n"),
	+ notify => Service[$service_name],
	+ require => File[$config_directory],
	+ }
	+
	+ # Template uses variables
	+ # - $config_path
	+ # - $sentry_dsn
	+ # - $sentry_environment
	+ # - $sentry_swh_package
	+ # - $loglevel
	+ ::systemd::dropin_file {$parameters_conf_path:
	+ ensure => present,
	+ unit => $service_name,
	+ filename => 'parameters.conf',
	+ content => template('profile/swh/deploy/journal/parameters.conf.erb'),
	+ }
	+
	+ service {$service_name:
	+ ensure => running,
	+ enable => true,
	+ require => [
	+ File[$config_path],
	+ ],
	+ }
	+ }
	+
	+ # Otherwise, clean up everything
	+ default: {
	+ ::systemd::dropin_file {$parameters_conf_path:
	+ ensure => absent,
	+ unit => $service_name,
	+ filename => 'parameters.conf',
	+ }
	+
	+ service {$service_basename:
	+ ensure => stopped,
	+ }

	- # Template uses variables
	- # - $user
	- # - $group
	- # - $sentry_dsn
	- # - $sentry_environment
	- # - $sentry_swh_package
	- ::systemd::unit_file {$unit_name:
	- ensure => present,
	- content => template("profile/swh/deploy/journal/${unit_name}.erb"),
	- } ~> service {$service_name:
	- ensure => running,
	- enable => true,
	+ file {$config_path:
	+ ensure => absent,
	+ }
	+ }
	}
	}
	diff --git a/site-modules/profile/manifests/swh/deploy/worker/indexer_content_mimetype.pp b/site-modules/profile/manifests/swh/deploy/worker/indexer_content_mimetype.pp
	--- a/site-modules/profile/manifests/swh/deploy/worker/indexer_content_mimetype.pp
	+++ b/site-modules/profile/manifests/swh/deploy/worker/indexer_content_mimetype.pp
	@@ -1,14 +1,19 @@
	-# Deployment for swh-indexer-mimetype
	-
	+# Deployment for indexer mimetype
	class profile::swh::deploy::worker::indexer_content_mimetype {
	include ::profile::swh::deploy::indexer

	- Package[$::profile::swh::deploy::base_indexer::packages] ~> ::profile::swh::deploy::worker::instance {'indexer_content_mimetype':
	- ensure => present,
	- sentry_name => 'indexer',
	- send_task_events => true,
	- require => [
	+ ::profile::swh::deploy::worker::instance {'indexer_content_mimetype':
	+ ensure => absent,
	+ }
	+
	+ Package[$::profile::swh::deploy::base_indexer::packages]
	+ ~> ::profile::swh::deploy::indexer_journal_client {'mimetype':
	+ ensure => present,
	+ sentry_name => $::profile::swh::deploy::base_indexer::sentry_name,
	+ require => [
	+ Package[$::profile::swh::deploy::base_indexer::packages],
	Class['profile::swh::deploy::indexer']
	],
	}
	+
	}
	diff --git a/site-modules/profile/manifests/swh/deploy/worker/indexer_fossology_license.pp b/site-modules/profile/manifests/swh/deploy/worker/indexer_fossology_license.pp
	--- a/site-modules/profile/manifests/swh/deploy/worker/indexer_fossology_license.pp
	+++ b/site-modules/profile/manifests/swh/deploy/worker/indexer_fossology_license.pp
	@@ -1,21 +1,26 @@
	-# Deployment for swh-indexer-fossology-license
	-
	+# Deployment for indexer fossology-license
	class profile::swh::deploy::worker::indexer_fossology_license {
	include ::profile::swh::deploy::indexer
	$packages = ['fossology-nomossa']
	-
	package {$packages:
	ensure => 'present',
	require => Apt::Source['softwareheritage'],
	}

	- Package[$::profile::swh::deploy::base_indexer::packages] ~> ::profile::swh::deploy::worker::instance {'indexer_fossology_license':
	- ensure => present,
	- sentry_name => 'indexer',
	- send_task_events => true,
	- require => [
	- Class['profile::swh::deploy::indexer'],
	- Package[$packages],
	+ # Clean up previous indexer service implementation
	+ ::profile::swh::deploy::worker::instance {'indexer_fossology_license':
	+ ensure => absent,
	+ }
	+
	+ Package[$::profile::swh::deploy::base_indexer::packages]
	+ ~> ::profile::swh::deploy::indexer_journal_client {'fossology_license':
	+ ensure => present,
	+ sentry_name => $::profile::swh::deploy::base_indexer::sentry_name,
	+ require => [
	+ Package[$::profile::swh::deploy::base_indexer::packages],
	+ Package[$packages::profile::swh::deploy::base_indexer::packages],
	+ Class['profile::swh::deploy::indexer']
	],
	}
	+
	}
	diff --git a/site-modules/profile/manifests/swh/deploy/worker/indexer_origin_extrinsic_metadata.pp b/site-modules/profile/manifests/swh/deploy/worker/indexer_origin_extrinsic_metadata.pp
	new file mode 100644
	--- /dev/null
	+++ b/site-modules/profile/manifests/swh/deploy/worker/indexer_origin_extrinsic_metadata.pp
	@@ -0,0 +1,13 @@
	+# Deployment for indexer origin-extrinsic-metadata
	+class profile::swh::deploy::worker::indexer_origin_extrinsic_metadata {
	+ include ::profile::swh::deploy::indexer
	+
	+ ::profile::swh::deploy::indexer_journal_client {'extrinsic_metadata':
	+ ensure => present,
	+ sentry_name => $::profile::swh::deploy::base_indexer::sentry_name,
	+ require => [
	+ Package[$::profile::swh::deploy::base_indexer::packages],
	+ Class['profile::swh::deploy::indexer']
	+ ],
	+ }
	+}
	diff --git a/site-modules/profile/manifests/swh/deploy/worker/indexer_origin_intrinsic_metadata.pp b/site-modules/profile/manifests/swh/deploy/worker/indexer_origin_intrinsic_metadata.pp
	--- a/site-modules/profile/manifests/swh/deploy/worker/indexer_origin_intrinsic_metadata.pp
	+++ b/site-modules/profile/manifests/swh/deploy/worker/indexer_origin_intrinsic_metadata.pp
	@@ -1,11 +1,13 @@
	-# Deployment for swh-indexer-origin-intrinsic-metadata
	+# Deployment for indexer origin-intrinsic-metadata
	class profile::swh::deploy::worker::indexer_origin_intrinsic_metadata {
	include ::profile::swh::deploy::indexer

	- # Remove deprecated service
	- ::profile::swh::deploy::worker::instance {'indexer_origin_intrinsic_metadata':
	- ensure => absent,
	+ ::profile::swh::deploy::indexer_journal_client {'intrinsic_metadata':
	+ ensure => present,
	+ sentry_name => $::profile::swh::deploy::base_indexer::sentry_name,
	+ require => [
	+ Package[$::profile::swh::deploy::base_indexer::packages],
	+ Class['profile::swh::deploy::indexer']
	+ ],
	}
	-
	- include ::profile::swh::deploy::indexer_journal_client
	}
	diff --git a/site-modules/profile/templates/swh/deploy/journal/parameters.conf.erb b/site-modules/profile/templates/swh/deploy/journal/parameters.conf.erb
	new file mode 100644
	--- /dev/null
	+++ b/site-modules/profile/templates/swh/deploy/journal/parameters.conf.erb
	@@ -0,0 +1,9 @@
	+# Managed by puppet - modifications will be overwritten
	+# In defined class profile::swh::deploy::indexer_journal_client
	+
	+[Service]
	+Environment=SWH_CONFIG_FILENAME=<%= @config_path %>
	+Environment=SWH_SENTRY_DSN=<%= @sentry_dsn %>
	+Environment=SWH_SENTRY_ENVIRONMENT=<%= @sentry_environment %>
	+Environment=SWH_MAIN_PACKAGE=<%= @sentry_swh_package %>
	+Environment=LOGLEVEL=<%= @loglevel %>
	diff --git a/site-modules/profile/templates/swh/deploy/journal/swh-indexer-journal-client.service.erb b/site-modules/profile/templates/swh/deploy/journal/swh-indexer-journal-client.service.erb
	deleted file mode 100644
	--- a/site-modules/profile/templates/swh/deploy/journal/swh-indexer-journal-client.service.erb
	+++ /dev/null
	@@ -1,28 +0,0 @@
	-# Indexer Journal Client unit file
	-# Managed by puppet class profile::swh::deploy::indexer_journal_client
	-# Changes will be overwritten
	-
	-[Unit]
	-Description=Software Heritage Indexer Journal Client
	-After=network.target
	-
	-[Service]
	-<%- if !@sentry_dsn.nil? and !@sentry_dsn.empty? -%>
	-Environment=SWH_SENTRY_DSN=<%= @sentry_dsn %>
	-<%- end -%>
	-<%- if !@sentry_environment.nil? and !@sentry_environment.empty? -%>
	-Environment=SWH_SENTRY_ENVIRONMENT=<%= @sentry_environment %>
	-<%- end -%>
	-<%- if !@sentry_swh_package.nil? and !@sentry_swh_package.empty? -%>
	-Environment=SWH_MAIN_PACKAGE=<%= @sentry_swh_package %>
	-<%- end -%>
	-Environment=SWH_CONFIG_FILENAME=<%= @config_path %>
	-User=<%= @user %>
	-Group=<%= @group %>
	-Type=simple
	-ExecStart=/usr/bin/swh indexer --config-file <%= @config_path %> journal-client '*'
	-Restart=always
	-RestartSec=10
	-
	-[Install]
	-WantedBy=multi-user.target
	diff --git a/site-modules/profile/templates/swh/deploy/journal/swh-indexer-journal-client@.service.erb b/site-modules/profile/templates/swh/deploy/journal/swh-indexer-journal-client@.service.erb
	new file mode 100644
	--- /dev/null
	+++ b/site-modules/profile/templates/swh/deploy/journal/swh-indexer-journal-client@.service.erb
	@@ -0,0 +1,30 @@
	+# Indexer Journal Client unit file
	+# Managed by puppet class profile::swh::deploy::base_indexer
	+# Changes will be overwritten
	+
	+[Unit]
	+Description=Software Heritage Indexer Journal Client %i
	+After=network.target
	+
	+[Service]
	+User=<%= @user %>
	+Group=<%= @group %>
	+
	+Type=simple
	+# Mandatory environment variables (check associated parameters.conf for override)
	+Environment=SWH_CONFIG_FILENAME=<%= @config_directory %>/%i.yml
	+Environment=SWH_SENTRY_DSN=<%= @sentry_dsn %>
	+Environment=SWH_SENTRY_ENVIRONMENT=<%= @sentry_environment %>
	+Environment=SWH_MAIN_PACKAGE=<%= @sentry_swh_package %>
	+Environment=LOGLEVEL=INFO
	+
	+ExecStart=/usr/bin/swh \
	+ --log-level $LOGLEVEL \
	+ indexer \
	+ --config-file $SWH_CONFIG_FILENAME \
	+ journal-client %i
	+Restart=always
	+RestartSec=10
	+
	+[Install]
	+WantedBy=multi-user.target

File Metadata

Mime Type: text/plain
Expires: Tue, Dec 17, 11:44 PM (2 d, 13 h ago)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 3231425

D8161.id29475.diffNo OneTemporaryActions

D8161.id29475.diffView Options

File Metadata

Event Timeline

D8161.id29475.diff
No OneTemporary
Actions

D8161.id29475.diff
View Options