Page MenuHomeSoftware Heritage

Migrate indexer services to indexer journal client services
ClosedPublic

Authored by ardumont on Wed, Jul 27, 12:01 PM.

Details

Summary

This cleans up old indexer services not based on journal client
(swh-worker@${old}.servicd for old in {indexer_content_mimetype,
indexer_fossology_license}).

It installs new services swh-indexer-journal-client@${new}.service for new in
{mimetype,fossology_license,intrinsic_metadata,extrinsic_metadata).

Related to T4406

Test Plan

vagrant:

root@worker0:~# puppet agent --test
...
Notice: /Stage[main]/Icinga2::Service/Service[icinga2]: Triggered 'refresh' from 1 event
Notice: /Stage[main]/Profile::Swh::Deploy::Worker::Base/Profile::Cron::D[cleanup-workers-tmp]/Profile::Cron::File[swh-worker]/File[/etc/puppet-cron.d/swh-worker]/ensure: defined content as '{md5}bfdd5f183ad8b0874184d8c58d094e7c'
Notice: /Stage[main]/Profile::Swh::Deploy::Worker::Base/Profile::Cron::D[cleanup-workers-tmp]/Profile::Cron::File[swh-worker]/File[/etc/cron.d/puppet-swh-worker]/ensure: created
Notice: Applied catalog in 4.97 seconds

root@worker0:~# systemctl list-units | grep journal-client
  swh-indexer-journal-client.service                                                        loaded active     running      Software Heritage Indexer Journal Client
  swh-indexer-journal-client@extrinsic_metadata.service                                     loaded activating auto-restart Software Heritage Indexer Journal Client extrinsic_metadata
  swh-indexer-journal-client@fossology_license.service                                      loaded activating auto-restart Software Heritage Indexer Journal Client fossology_license
  swh-indexer-journal-client@intrinsic_metadata.service                                     loaded activating auto-restart Software Heritage Indexer Journal Client intrinsic_metadata
  swh-indexer-journal-client@mimetype.service                                               loaded activating auto-restart Software Heritage Indexer Journal Client mimetype
# for each service ^, parameters.conf defines the necessary override, for example for mimetype
root@worker0:~# cat /etc/systemd/system/swh-indexer-journal-client@mimetype.service.d/parameters.conf
# Managed by puppet - modifications will be overwritten
# In defined class profile::swh::deploy::indexer_journal_client

[Service]
Environment=SWH_CONFIG_FILENAME=/etc/softwareheritage/indexer/mimetype.yml
Environment=SWH_SENTRY_DSN=https://swh::deploy::indexer::sentry_token@sentry.softwareheritage.org/5
Environment=SWH_SENTRY_ENVIRONMENT=staging
Environment=SWH_MAIN_PACKAGE=swh.indexer
Environment=LOGLEVEL=INFO

octo-diff:

$ $SWH_PUPPET_ENVIRONMENT_HOME/bin/octocatalog-diff worker0.internal.staging.swh.network
Found host worker0.internal.staging.swh.network
...
*******************************************
- Concat_fragment[profile::cron::swh-worker-indexer_content_mimetype-autorestart]
*******************************************
- Concat_fragment[profile::cron::swh-worker-indexer_fossology_license-autorestart]
*******************************************
+ File[/etc/softwareheritage/indexer/extrinsic_metadata.yml] =>
   parameters =>
     "content": "---\nscheduler:\n  cls: remote\n  url: http://scheduler0.interna...
     "ensure": "present",
     "group": "swhworker",
     "mode": "0640",
     "notify": "Service[swh-indexer-journal-client@extrinsic_metadata.service]",
     "owner": "root"
*******************************************
+ File[/etc/softwareheritage/indexer/fossology_license.yml] =>
   parameters =>
     "content": "---\nindexer_storage:\n  cls: remote\n  url: http://storage1.int...
     "ensure": "present",
     "group": "swhworker",
     "mode": "0640",
     "notify": "Service[swh-indexer-journal-client@fossology_license.service]",
     "owner": "root"
*******************************************
+ File[/etc/softwareheritage/indexer/intrinsic_metadata.yml] =>
   parameters =>
     "content": "---\nindexer_storage:\n  cls: remote\n  url: http://storage1.int...
     "ensure": "present",
     "group": "swhworker",
     "mode": "0640",
     "notify": "Service[swh-indexer-journal-client@intrinsic_metadata.service]",
     "owner": "root"
*******************************************
- File[/etc/softwareheritage/indexer/journal_client.yml]
*******************************************
+ File[/etc/softwareheritage/indexer/mimetype.yml] =>
   parameters =>
     "content": "---\nindexer_storage:\n  cls: remote\n  url: http://storage1.int...
     "ensure": "present",
     "group": "swhworker",
     "mode": "0640",
     "notify": "Service[swh-indexer-journal-client@mimetype.service]",
     "owner": "root"
*******************************************
  File[/etc/softwareheritage/indexer_content_mimetype.yml] =>
   parameters =>
     ensure =>
      - present
      + absent
*******************************************
  File[/etc/softwareheritage/indexer_fossology_license.yml] =>
   parameters =>
     ensure =>
      - present
      + absent
*******************************************
- File[/etc/softwareheritage/indexer_origin_intrinsic_metadata.yml]
*******************************************
- File[/etc/systemd/system/swh-indexer-journal-client.service]
*******************************************
+ File[/etc/systemd/system/swh-indexer-journal-client@.service] =>
   parameters =>
     "content": "# Indexer Journal Client unit file\n# Managed by puppet class pr...
     "ensure": "file",
     "group": "root",
     "mode": "0444",
     "notify": "Class[Systemd::Systemctl::Daemon_reload]",
     "owner": "root",
     "show_diff": true
*******************************************
+ File[/etc/systemd/system/swh-indexer-journal-client@extrinsic_metadata.service.d/parameters.conf] =>
   parameters =>
     "content": "# Managed by puppet - modifications will be overwritten\n# In de...
     "ensure": "file",
     "group": "root",
     "mode": "0444",
     "notify": [
       "Class[Systemd::Systemctl::Daemon_reload]"
     ],
     "owner": "root",
     "selinux_ignore_defaults": false,
     "show_diff": true
*******************************************
+ File[/etc/systemd/system/swh-indexer-journal-client@extrinsic_metadata.service.d] =>
   parameters =>
     "ensure": "directory",
     "group": "root",
     "owner": "root",
     "purge": true,
     "recurse": true,
     "selinux_ignore_defaults": false
*******************************************
+ File[/etc/systemd/system/swh-indexer-journal-client@fossology_license.service.d/parameters.conf] =>
   parameters =>
     "content": "# Managed by puppet - modifications will be overwritten\n# In de...
     "ensure": "file",
     "group": "root",
     "mode": "0444",
     "notify": [
       "Class[Systemd::Systemctl::Daemon_reload]"
     ],
     "owner": "root",
     "selinux_ignore_defaults": false,
     "show_diff": true
*******************************************
+ File[/etc/systemd/system/swh-indexer-journal-client@fossology_license.service.d] =>
   parameters =>
     "ensure": "directory",
     "group": "root",
     "owner": "root",
     "purge": true,
     "recurse": true,
     "selinux_ignore_defaults": false
*******************************************
+ File[/etc/systemd/system/swh-indexer-journal-client@intrinsic_metadata.service.d/parameters.conf] =>
   parameters =>
     "content": "# Managed by puppet - modifications will be overwritten\n# In de...
     "ensure": "file",
     "group": "root",
     "mode": "0444",
     "notify": [
       "Class[Systemd::Systemctl::Daemon_reload]"
     ],
     "owner": "root",
     "selinux_ignore_defaults": false,
     "show_diff": true
*******************************************
+ File[/etc/systemd/system/swh-indexer-journal-client@intrinsic_metadata.service.d] =>
   parameters =>
     "ensure": "directory",
     "group": "root",
     "owner": "root",
     "purge": true,
     "recurse": true,
     "selinux_ignore_defaults": false
*******************************************
+ File[/etc/systemd/system/swh-indexer-journal-client@mimetype.service.d/parameters.conf] =>
   parameters =>
     "content": "# Managed by puppet - modifications will be overwritten\n# In de...
     "ensure": "file",
     "group": "root",
     "mode": "0444",
     "notify": [
       "Class[Systemd::Systemctl::Daemon_reload]"
     ],
     "owner": "root",
     "selinux_ignore_defaults": false,
     "show_diff": true
*******************************************
+ File[/etc/systemd/system/swh-indexer-journal-client@mimetype.service.d] =>
   parameters =>
     "ensure": "directory",
     "group": "root",
     "owner": "root",
     "purge": true,
     "recurse": true,
     "selinux_ignore_defaults": false
*******************************************
  File[/etc/systemd/system/swh-worker@indexer_content_mimetype.service.d/parameters.conf] =>
   parameters =>
     ensure =>
      - file
      + absent
*******************************************
- File[/etc/systemd/system/swh-worker@indexer_content_mimetype.service.d]
*******************************************
  File[/etc/systemd/system/swh-worker@indexer_fossology_license.service.d/parameters.conf] =>
   parameters =>
     ensure =>
      - file
      + absent
*******************************************
- File[/etc/systemd/system/swh-worker@indexer_fossology_license.service.d]
*******************************************
- File[/etc/systemd/system/swh-worker@indexer_origin_intrinsic_metadata.service.d/parameters.conf]
*******************************************
  Package[python3-swh.indexer] =>
   parameters =>
     notify =>
      - ["Profile::Swh::Deploy::Worker::Instance[indexer_content_mimetype]", "Profile::Swh::Deploy::Worker::Instance[indexer_fossology_license]"]
      + ["Profile::Swh::Deploy::Indexer_journal_client[fossology_license]", "Profile::Swh::Deploy::Indexer_journal_client[mimetype]"]
*******************************************
- Profile::Cron::D[swh-worker-indexer_content_mimetype-autorestart]
*******************************************
- Profile::Cron::D[swh-worker-indexer_fossology_license-autorestart]
*******************************************
+ Profile::Swh::Deploy::Indexer_journal_client[extrinsic_metadata] =>
   parameters =>
     "ensure": "present",
     "instance_name": "extrinsic_metadata",
     "sentry_name": "indexer"
*******************************************
+ Profile::Swh::Deploy::Indexer_journal_client[fossology_license] =>
   parameters =>
     "ensure": "present",
     "instance_name": "fossology_license",
     "sentry_name": "indexer"
*******************************************
+ Profile::Swh::Deploy::Indexer_journal_client[intrinsic_metadata] =>
   parameters =>
     "ensure": "present",
     "instance_name": "intrinsic_metadata",
     "sentry_name": "indexer"
*******************************************
+ Profile::Swh::Deploy::Indexer_journal_client[mimetype] =>
   parameters =>
     "ensure": "present",
     "instance_name": "mimetype",
     "sentry_name": "indexer"
*******************************************
  Profile::Swh::Deploy::Worker::Instance[indexer_content_mimetype] =>
   parameters =>
     ensure =>
      - present
      + absent
     send_task_events =>
      - true
      + false
     sentry_name =>
      - indexer
      + indexer_content_mimetype
*******************************************
  Profile::Swh::Deploy::Worker::Instance[indexer_fossology_license] =>
   parameters =>
     ensure =>
      - present
      + absent
     send_task_events =>
      - true
      + false
     sentry_name =>
      - indexer
      + indexer_fossology_license
*******************************************
- Profile::Swh::Deploy::Worker::Instance[indexer_origin_intrinsic_metadata]
*******************************************
+ Service[swh-indexer-journal-client@extrinsic_metadata.service] =>
   parameters =>
     "enable": true,
     "ensure": "running"
*******************************************
+ Service[swh-indexer-journal-client@fossology_license.service] =>
   parameters =>
     "enable": true,
     "ensure": "running"
*******************************************
+ Service[swh-indexer-journal-client@intrinsic_metadata.service] =>
   parameters =>
     "enable": true,
     "ensure": "running"
*******************************************
+ Service[swh-indexer-journal-client@mimetype.service] =>
   parameters =>
     "enable": true,
     "ensure": "running"
*******************************************
- Service[swh-indexer-journal-client]
*******************************************
  Service[swh-worker@indexer_content_mimetype] =>
   parameters =>
     enable =>
      - true
     ensure =>
      + stopped
*******************************************
  Service[swh-worker@indexer_fossology_license] =>
   parameters =>
     enable =>
      - true
     ensure =>
      + stopped
*******************************************
- Service[swh-worker@indexer_origin_intrinsic_metadata]
*******************************************
+ Systemd::Dropin_file[swh-indexer-journal-client@extrinsic_metadata.service/parameters.conf] =>
   parameters =>
     "content": "# Managed by puppet - modifications will be overwritten\n# In de...
     "daemon_reload": "lazy",
     "ensure": "present",
     "filename": "parameters.conf",
     "group": "root",
     "mode": "0444",
     "owner": "root",
     "path": "/etc/systemd/system",
     "selinux_ignore_defaults": false,
     "show_diff": true,
     "unit": "swh-indexer-journal-client@extrinsic_metadata.service"
*******************************************
+ Systemd::Dropin_file[swh-indexer-journal-client@fossology_license.service/parameters.conf] =>
   parameters =>
     "content": "# Managed by puppet - modifications will be overwritten\n# In de...
     "daemon_reload": "lazy",
     "ensure": "present",
     "filename": "parameters.conf",
     "group": "root",
     "mode": "0444",
     "owner": "root",
     "path": "/etc/systemd/system",
     "selinux_ignore_defaults": false,
     "show_diff": true,
     "unit": "swh-indexer-journal-client@fossology_license.service"
*******************************************
+ Systemd::Dropin_file[swh-indexer-journal-client@intrinsic_metadata.service/parameters.conf] =>
   parameters =>
     "content": "# Managed by puppet - modifications will be overwritten\n# In de...
     "daemon_reload": "lazy",
     "ensure": "present",
     "filename": "parameters.conf",
     "group": "root",
     "mode": "0444",
     "owner": "root",
     "path": "/etc/systemd/system",
     "selinux_ignore_defaults": false,
     "show_diff": true,
     "unit": "swh-indexer-journal-client@intrinsic_metadata.service"
*******************************************
+ Systemd::Dropin_file[swh-indexer-journal-client@mimetype.service/parameters.conf] =>
   parameters =>
     "content": "# Managed by puppet - modifications will be overwritten\n# In de...
     "daemon_reload": "lazy",
     "ensure": "present",
     "filename": "parameters.conf",
     "group": "root",
     "mode": "0444",
     "owner": "root",
     "path": "/etc/systemd/system",
     "selinux_ignore_defaults": false,
     "show_diff": true,
     "unit": "swh-indexer-journal-client@mimetype.service"
*******************************************
  Systemd::Dropin_file[swh-worker@indexer_content_mimetype/parameters.conf] =>
   parameters =>
     content =>
      - # Managed by puppet - modifications will be overwritten
# In defined class profile::swh::deploy::worker::instance

[Service]
Environment=CONCURRENCY=1
Environment=MAX_TASKS_PER_CHILD=5
Environment=LOGLEVEL=info
Environment=SWH_SENTRY_DSN=https://swh::deploy::indexer::sentry_token@sentry.softwareheritage.org/5
Environment=SWH_SENTRY_ENVIRONMENT=staging
Environment=SWH_MAIN_PACKAGE=swh.indexer


Environment=SWH_WORKER_CLI_EXTRA_ARGS=--events

     ensure =>
      - present
      + absent
*******************************************
  Systemd::Dropin_file[swh-worker@indexer_fossology_license/parameters.conf] =>
   parameters =>
     content =>
      - # Managed by puppet - modifications will be overwritten
# In defined class profile::swh::deploy::worker::instance

[Service]
Environment=CONCURRENCY=1
Environment=MAX_TASKS_PER_CHILD=5
Environment=LOGLEVEL=info
Environment=SWH_SENTRY_DSN=https://swh::deploy::indexer::sentry_token@sentry.softwareheritage.org/5
Environment=SWH_SENTRY_ENVIRONMENT=staging
Environment=SWH_MAIN_PACKAGE=swh.indexer


Environment=SWH_WORKER_CLI_EXTRA_ARGS=--events

     ensure =>
      - present
      + absent
*******************************************
- Systemd::Dropin_file[swh-worker@indexer_origin_intrinsic_metadata/parameters.conf]
*******************************************
- Systemd::Unit_file[swh-indexer-journal-client.service]
*******************************************
+ Systemd::Unit_file[swh-indexer-journal-client@.service] =>
   parameters =>
     "content": "# Indexer Journal Client unit file\n# Managed by puppet class pr...
     "ensure": "present",
     "group": "root",
     "mode": "0444",
     "owner": "root",
     "path": "/etc/systemd/system",
     "show_diff": true
*******************************************
*** End octocatalog-diff on worker0.internal.staging.swh.network

Diff Detail

Repository
rSPSITE puppet-swh-site
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

ardumont edited the test plan for this revision. (Show Details)
ardumont edited the summary of this revision. (Show Details)
ardumont edited the test plan for this revision. (Show Details)

Fix typo in actual service running the journal client

ardumont edited the test plan for this revision. (Show Details)
This revision was not accepted when it landed; it landed in state Needs Review.Fri, Jul 29, 10:27 AM
This revision was automatically updated to reflect the committed changes.