diff --git a/manifests/site.pp b/manifests/site.pp index 9edbfd40..5846058e 100644 --- a/manifests/site.pp +++ b/manifests/site.pp @@ -1,206 +1,204 @@ node 'louvre.internal.softwareheritage.org' { include role::swh_server } node /^(orsay|beaubourg|hypervisor\d+|branly|pompidou|uffizi)\.(internal\.)?softwareheritage\.org$/ { include role::swh_hypervisor } node 'pergamon.softwareheritage.org' { include role::swh_sysadmin include profile::export_archive_counters } node 'tate.softwareheritage.org' { include role::swh_forge } node 'moma.softwareheritage.org' { include role::swh_rp_webapps } node 'webapp1.internal.softwareheritage.org' { include role::swh_rp_webapp_with_swh_search_and_storage } node /^search-esnode\d\.internal\.softwareheritage\.org$/ { include role::swh_elasticsearch } node /^search\d\.internal\.softwareheritage\.org$/ { include role::swh_search_with_journal_client } node 'saatchi.internal.softwareheritage.org' { include role::swh_scheduler_with_journal_client } node /^(belvedere|somerset).(internal.)?softwareheritage.org$/ { include role::swh_database include profile::pgbouncer } node 'banco.softwareheritage.org' { include role::swh_backup include role::postgresql_backup } node /^esnode\d+.(internal.)?softwareheritage.org$/ { include role::swh_elasticsearch } node /^kafka\d+\./ { include role::swh_kafka_broker } node /^cassandra\d+\./ { include role::swh_cassandra_node } node 'granet.internal.softwareheritage.org' { include role::swh_graph_backend } node /^(unibo-prod|vangogh).(euwest.azure.)?(internal.)?softwareheritage.org$/ { include role::swh_vault } node /^saam\.(internal\.)?softwareheritage\.org$/ { include role::swh_storage_baremetal } node 'storage01.euwest.azure.internal.softwareheritage.org' { include role::swh_storage_cloud } node /^getty.(internal.)?softwareheritage.org$/ { - include role::swh_journal_orchestrator + include role::swh_journal_orchestrator_with_backfill_config } node /^worker\d+\.(internal\.)?softwareheritage\.org$/ { include role::swh_worker_inria } node /^worker\d+\..*\.azure\.internal\.softwareheritage\.org$/ { include role::swh_worker_azure } node /^dbreplica(0|1)\.euwest\.azure\.internal\.softwareheritage\.org$/ { include role::swh_database } node /^ceph-osd\d+\.internal\.softwareheritage\.org$/ { include role::swh_ceph_osd } node /^ceph-mon\d+\.internal\.softwareheritage\.org$/ { include role::swh_ceph_mon } node /^ns\d+\.(.*\.azure\.)?internal\.softwareheritage\.org/ { include role::swh_nameserver_secondary } node 'thyssen.internal.softwareheritage.org' { include role::swh_ci_server } node 'riverside.internal.softwareheritage.org' { include role::swh_sentry } node /^jenkins-debian\d+\.internal\.softwareheritage\.org$/ { include role::swh_ci_agent_debian } node 'logstash0.internal.softwareheritage.org' { include role::swh_logstash_instance } node 'kibana0.internal.softwareheritage.org' { include role::swh_kibana_instance } node 'kelvingrove.internal.softwareheritage.org' { include role::swh_idp_primary } node 'giverny.softwareheritage.org' { include role::swh_desktop } node /^db\d\.internal\.staging\.swh\.network$/ { include role::swh_database include profile::postgresql::server include profile::pgbouncer include profile::postgresql::client } node "bardo.internal.admin.swh.network" { include role::swh_hedgedoc } node 'scheduler0.internal.staging.swh.network' { include role::swh_scheduler_with_journal_client include profile::postgresql::client } node 'gateway.internal.staging.swh.network' { include role::swh_gateway } node /^storage\d\.internal\.staging\.swh\.network$/ { - include role::swh_base_storage - include profile::postgresql::client - include profile::swh::deploy::journal::backfill + include role::swh_storage_with_backfill_config } node /^worker\d\.internal\.staging\.swh\.network$/ { include role::swh_worker_inria } node /^search-esnode\d\.internal\.staging\.swh\.network$/ { include role::swh_elasticsearch } node /^search\d\.internal\.staging\.swh\.network$/ { include role::swh_search_with_journal_client } node 'webapp.internal.staging.swh.network' { include role::swh_webapp } node 'deposit.internal.staging.swh.network' { include role::swh_deposit } node 'vault.internal.staging.swh.network' { include role::swh_vault } node /^rp\d\.internal\.(staging|admin)\.swh\.network$/ { include role::swh_reverse_proxy } node 'journal0.internal.staging.swh.network' { include role::swh_journal_allinone } # Read-only storage for mirrors node 'objstorage0.internal.staging.swh.network' { include role::swh_remote_objstorage } node 'bojimans.internal.softwareheritage.org' { include role::swh_netbox } node 'clearly-defined.internal.staging.swh.network' { include role::swh_db_client } node default { include role::swh_base } diff --git a/site-modules/profile/manifests/swh/deploy/base_storage.pp b/site-modules/profile/manifests/swh/deploy/base_storage.pp index 93401e27..43a9770b 100644 --- a/site-modules/profile/manifests/swh/deploy/base_storage.pp +++ b/site-modules/profile/manifests/swh/deploy/base_storage.pp @@ -1,11 +1,15 @@ class profile::swh::deploy::base_storage { $swh_conf_storage_directory = lookup('swh::deploy::storage::conf_directory') file {$swh_conf_storage_directory: ensure => directory, owner => 'root', group => 'root', mode => '0755', } + $package = 'python3-swh.storage' + package {$package: + ensure => 'present', + } } diff --git a/site-modules/profile/manifests/swh/deploy/journal/backfill.pp b/site-modules/profile/manifests/swh/deploy/journal/backfill.pp index a8797ee8..6e57f91a 100644 --- a/site-modules/profile/manifests/swh/deploy/journal/backfill.pp +++ b/site-modules/profile/manifests/swh/deploy/journal/backfill.pp @@ -1,28 +1,31 @@ # Deployment of journal backfill configuration class profile::swh::deploy::journal::backfill { + include profile::swh::deploy::base_storage + include profile::swh::deploy::journal + $config_path = lookup('swh::deploy::journal::backfill::config_file') $config = lookup('swh::deploy::journal::backfill::config') $config_logging_path = lookup('swh::deploy::journal::backfill::config_logging_file') $config_logging = lookup('swh::deploy::journal::backfill::config_logging') $user = lookup('swh::deploy::journal::backfill::user') $group = lookup('swh::deploy::journal::backfill::group') file {$config_path: ensure => present, owner => $user, group => $group, mode => '0644', content => inline_template("<%= @config.to_yaml %>\n") } file {$config_logging_path: ensure => present, owner => $user, group => $group, mode => '0644', content => inline_template("<%= @config_logging.to_yaml %>\n") } } diff --git a/site-modules/profile/manifests/swh/deploy/storage.pp b/site-modules/profile/manifests/swh/deploy/storage.pp index 1d84e1a0..a8cec8ba 100644 --- a/site-modules/profile/manifests/swh/deploy/storage.pp +++ b/site-modules/profile/manifests/swh/deploy/storage.pp @@ -1,25 +1,26 @@ # Deployment of the swh.storage.api server class profile::swh::deploy::storage { include ::profile::swh::deploy::base_storage - package {'python3-swh.storage': - ensure => 'present', - } ~> ::profile::swh::deploy::rpc_server {'storage': + $package = $::profile::swh::deploy::base_storage::package + + ::profile::swh::deploy::rpc_server {'storage': executable => 'swh.storage.api.server:make_app_from_configfile()', worker => 'sync', http_check_string => 'Software Heritage storage server', + subscribe => Package[$package] } $storage_config = lookup('swh::deploy::storage::config')['storage'] if ($storage_config['cls'] == 'local' and $storage_config['args']['journal_writer'] and $storage_config['args']['journal_writer']['cls'] == 'kafka') { include ::profile::swh::deploy::journal } if $storage_config['cls'] == 'cassandra' { include ::profile::swh::deploy::storage_cassandra } } diff --git a/site-modules/role/manifests/swh_journal_orchestrator_with_backfill_config.pp b/site-modules/role/manifests/swh_journal_orchestrator_with_backfill_config.pp new file mode 100644 index 00000000..5191aca0 --- /dev/null +++ b/site-modules/role/manifests/swh_journal_orchestrator_with_backfill_config.pp @@ -0,0 +1,3 @@ +class role::swh_journal_orchestrator_with_backfill_config inherits role::swh_journal_orchestrator { + include profile::swh::deploy::journal::backfill +} diff --git a/site-modules/role/manifests/swh_storage_with_backfill_config.pp b/site-modules/role/manifests/swh_storage_with_backfill_config.pp new file mode 100644 index 00000000..1f6a7ea6 --- /dev/null +++ b/site-modules/role/manifests/swh_storage_with_backfill_config.pp @@ -0,0 +1,4 @@ +class role::swh_storage_with_backfill_config inherits role::swh_base_storage { + include profile::postgresql::client + include profile::swh::deploy::journal::backfill +}