diff --git a/Puppetfile b/Puppetfile index d9707367..15eab0e3 100644 --- a/Puppetfile +++ b/Puppetfile @@ -1,212 +1,216 @@ mod 'dar', :git => 'https://forge.softwareheritage.org/source/puppet-swh-dar', :branch => :control_branch, :default_branch => 'master' mod 'gunicorn', :git => 'https://forge.softwareheritage.org/source/puppet-swh-gunicorn', :branch => :control_branch, :default_branch => 'master' mod 'mediawiki', :git => 'https://forge.softwareheritage.org/source/puppet-swh-mediawiki', :branch => :control_branch, :default_branch => 'master' mod 'postfix', :git => 'https://forge.softwareheritage.org/source/puppet-swh-postfix', :branch => :control_branch, :default_branch => 'master' mod 'uwsgi', :git => 'https://forge.softwareheritage.org/source/puppet-swh-uwsgi', :branch => :control_branch, :default_branch => 'master' mod 'apache', :git => 'https://forge.softwareheritage.org/source/puppet-puppetlabs-apache', :tag => 'v7.0.0' mod 'apt', :git => 'https://forge.softwareheritage.org/source/puppet-puppetlabs-apt', :tag => 'v8.3.0' mod 'archive', :git => 'https://forge.softwareheritage.org/source/puppet-puppet-archive', :tag => 'v6.0.2' mod 'bind', :git => 'https://forge.softwareheritage.org/source/puppet-inkblot-bind', :ref => '7.4.0' mod 'cassandra', :git => 'https://forge.softwareheritage.org/source/puppet-puppet-cassandra', :ref => 'master' mod 'ceph', :git => 'https://forge.softwareheritage.org/source/puppet-openstack-ceph', :ref => 'master' mod 'concat', :git => 'https://forge.softwareheritage.org/source/puppet-puppetlabs-concat', :tag => 'v7.1.1' mod 'cups', :git => 'https://forge.softwareheritage.org/source/puppet-mosen-cups', :ref => 'master' mod 'datacat', :git => 'https://forge.softwareheritage.org/source/puppet-richardc-datacat', :ref => '0.6.2' mod 'debconf', :git => 'https://forge.softwareheritage.org/source/puppet-stm-debconf', :ref => 'v4.1.0' mod 'debnet', :git => 'https://forge.softwareheritage.org/source/puppet-trepasi-debnet', :ref => '8d856df078352a8848a43ca0ee9f2ef9086b343a' mod 'docker', :git => 'https://forge.softwareheritage.org/source/puppet-puppetlabs-docker', :ref => 'v4.1.2' mod 'elasticsearch', :git => 'https://forge.softwareheritage.org/source/puppet-elastic-elasticsearch', :ref => '6.4.0' mod 'extlib', :git => 'https://forge.softwareheritage.org/source/puppet-puppet-extlib', :tag => 'v5.3.0' mod 'grafana', :git => 'https://forge.softwareheritage.org/source/puppet-puppet-grafana', :ref => 'feature/puppet5-compat' mod 'hitch', :git => 'https://forge.softwareheritage.org/source/puppet-ssm-hitch', :ref => 'feature/additional-config-0.1.5' mod 'icinga2', :git => 'https://forge.softwareheritage.org/source/puppet-icinga-icinga2', :tag => 'v3.2.1' mod 'icingaweb2', :git => 'https://forge.softwareheritage.org/source/puppet-icinga-icingaweb2', :tag => 'v3.2.1' mod 'inifile', :git => 'https://forge.softwareheritage.org/source/puppet-puppetlabs-inifile', :ref => 'v5.2.0' mod 'java', :git => 'https://forge.softwareheritage.org/source/puppet-puppetlabs-java', :tag => 'v7.3.0' mod 'java_ks', :git => 'https://forge.softwareheritage.org/source/puppet-puppetlabs-java_ks', :tag => 'v4.2.0' mod 'kafka', :git => 'https://forge.softwareheritage.org/source/puppet-puppet-kafka', :ref => 'v8.0.0' mod 'keycloak', :git => 'https://forge.softwareheritage.org/source/puppet-treydock-keycloak', # Need to upgrade to keycloak 12 before bumping to 7.x :ref => 'v6.26.0' mod 'letsencrypt', :git => 'https://forge.softwareheritage.org/source/puppet-puppet-letsencrypt', :ref => 'v7.0.0' mod 'locales', :git => 'https://forge.softwareheritage.org/source/puppet-saz-locales', :ref => 'v3.1.0' mod 'mysql', :git => 'https://forge.softwareheritage.org/source/puppet-puppetlabs-mysql', :ref => 'v12.0.1' mod 'nginx', :git => 'https://forge.softwareheritage.org/source/puppet-puppet-nginx', :ref => 'v3.3.0' mod 'ntp', :git => 'https://forge.softwareheritage.org/source/puppet-puppetlabs-ntp', :ref => 'v9.1.0' mod 'php', :git => 'https://forge.softwareheritage.org/source/puppet-puppet-php', :ref => 'v8.0.2' mod 'postgresql', :git => 'https://forge.softwareheritage.org/source/puppet-puppetlabs-postgresql', :ref => 'v7.5.0' mod 'pgbouncer', :git => 'https://forge.softwareheritage.org/source/puppet-covermymeds-pgbouncer', :ref => '9ec0d8a1255bbb309c2ff38f229167209cad496b' mod 'puppet', :git => 'https://forge.softwareheritage.org/source/puppet-theforeman-puppet', :ref => 'latest_passenger' mod 'puppetdb', :git => 'https://forge.softwareheritage.org/source/puppet-puppetlabs-puppetdb', :ref => '7.10.0' mod 'memcached', :git => 'https://forge.softwareheritage.org/source/puppet-saz-memcached', :ref => 'v7.0.0' mod 'rabbitmq', :git => 'https://forge.softwareheritage.org/source/puppet-puppet-rabbitmq', :ref => 'v11.1.0' mod 'redis', :git => 'https://forge.softwareheritage.org/source/puppet-puppet-redis', :ref => 'v8.2.0' mod 'resolv_conf', :git => 'https://forge.softwareheritage.org/source/puppet-saz-resolv_conf', :ref => 'v5.0.0' mod 'ssh', :git => 'https://forge.softwareheritage.org/source/puppet-saz-ssh', :ref => 'v8.0.0' mod 'stdlib', :git => 'https://forge.softwareheritage.org/source/puppet-puppetlabs-stdlib', :ref => 'v8.1.0' mod 'sudo', :git => 'https://forge.softwareheritage.org/source/puppet-saz-sudo', :ref => 'v7.0.2' mod 'systemd', :git => 'https://forge.softwareheritage.org/source/puppet-camptocamp-systemd', # camptocamp/puppet-systemd 3.x dropped explicit systemctl daemon-reload # calls in favor of the built-in support in Puppet 6.1. We use Puppet 5.5.x, # so we can't upgrade this module. :ref => '2.12.0' mod 'timezone', :git => 'https://forge.softwareheritage.org/source/puppet-saz-timezone', :ref => 'v6.1.0' mod 'unattended_upgrades', :git => 'https://forge.softwareheritage.org/source/puppet-puppet-unattended_upgrades', :ref => 'v6.0.0' mod 'varnish', :git => 'https://forge.softwareheritage.org/source/puppet-claranet-varnish', :ref => 'bugfix/systemd-unit' mod 'vcsrepo', :git => 'https://forge.softwareheritage.org/source/puppet-puppetlabs-vcsrepo', :ref => 'v5.0.0' +mod 'zfs_core', + :git => 'https://forge.softwareheritage.org/source/puppet-puppetlabs-zfs_core', + :ref => '1.0.0' # Higher versions not compatible with puppet < 6.0 + mod 'zookeeper', :git => 'https://forge.softwareheritage.org/source/puppet-deric-zookeeper', :ref => 'v1.2.1' diff --git a/data/hostname/elastic-worker0.internal.staging.swh.network.yaml b/data/hostname/elastic-worker0.internal.staging.swh.network.yaml new file mode 100644 index 00000000..2ad29586 --- /dev/null +++ b/data/hostname/elastic-worker0.internal.staging.swh.network.yaml @@ -0,0 +1,6 @@ +# enable contrib and non-free apt sources +swh::apt_config::enable_non_free: true +# which allows the following dependency to be installed +packages: + # install zfs + - zfs-dkms diff --git a/manifests/site.pp b/manifests/site.pp index a3cf60f8..129af858 100644 --- a/manifests/site.pp +++ b/manifests/site.pp @@ -1,233 +1,238 @@ node /^(pompidou|uffizi)\.(internal\.)?softwareheritage\.org$/ { include role::swh_hypervisor } node /^(beaubourg|hypervisor\d+|branly)\.(internal\.)?softwareheritage\.org$/ { include role::swh_hypervisor_with_ceph } node 'pergamon.softwareheritage.org' { include role::swh_sysadmin } node 'grafana0.internal.admin.swh.network' { include role::swh_grafana } node 'tate.softwareheritage.org' { include role::swh_forge } node 'moma.softwareheritage.org' { include role::swh_rp_webapps } node 'webapp1.internal.softwareheritage.org' { include role::swh_rp_webapp } node /^search-esnode\d\.internal\.softwareheritage\.org$/ { include role::swh_elasticsearch } node /^search\d\.internal\.softwareheritage\.org$/ { include role::swh_search_with_journal_client } node /^counters\d\.internal\.softwareheritage\.org$/ { include role::swh_counters_with_journal_client } node 'saatchi.internal.softwareheritage.org' { include role::swh_scheduler_with_journal_client } node /^(belvedere|somerset).(internal.)?softwareheritage.org$/ { include role::swh_database include profile::pgbouncer } node 'banco.softwareheritage.org' { include role::swh_backup include role::postgresql_backup } node /^esnode\d+.(internal.)?softwareheritage.org$/ { include role::swh_elasticsearch } node /^kafka\d+\./ { include role::swh_kafka_broker } node /^cassandra\d+\./ { include role::swh_cassandra_node } node 'granet.internal.softwareheritage.org' { include role::swh_graph_backend } node 'met.internal.softwareheritage.org' { include role::swh_provenance } node /^(unibo-prod|vangogh).(euwest.azure.)?(internal.)?softwareheritage.org$/ { include role::swh_vault } node /^saam\.(internal\.)?softwareheritage\.org$/ { include role::swh_storage_baremetal } node 'storage01.euwest.azure.internal.softwareheritage.org' { include role::swh_storage_cloud } node /^getty.(internal.)?softwareheritage.org$/ { include role::swh_journal_orchestrator_with_backfill_config } node /^worker\d+\.(internal\.)?softwareheritage\.org$/ { include role::swh_worker_inria } node /^worker\d+\..*\.azure\.internal\.softwareheritage\.org$/ { include role::swh_worker_azure } node /^dbreplica(0|1)\.euwest\.azure\.internal\.softwareheritage\.org$/ { include role::swh_database } node /^ceph-osd\d+\.internal\.softwareheritage\.org$/ { include role::swh_ceph_osd } node /^ceph-mon\d+\.internal\.softwareheritage\.org$/ { include role::swh_ceph_mon } node /^ns\d+\.(.*\.azure\.)?internal\.softwareheritage\.org/ { include role::swh_nameserver_secondary } node 'thyssen.internal.softwareheritage.org' { include role::swh_ci_server } node 'riverside.internal.admin.swh.network' { include role::swh_sentry } node /^jenkins-debian\d+\.internal\.softwareheritage\.org$/ { include role::swh_ci_agent_debian } node 'logstash0.internal.softwareheritage.org' { include role::swh_logstash_instance } node 'kibana0.internal.softwareheritage.org' { include role::swh_kibana_instance } node 'kelvingrove.internal.softwareheritage.org' { include role::swh_idp_primary } node 'giverny.softwareheritage.org' { include role::swh_desktop } node /^db\d\.internal\.staging\.swh\.network$/ { include role::swh_database_staging } node 'dali.internal.admin.swh.network' { include role::swh_admin_database } node "bardo.internal.admin.swh.network" { include role::swh_hedgedoc } node 'scheduler0.internal.staging.swh.network' { include role::swh_scheduler_with_journal_client include profile::postgresql::client } node 'gateway.internal.staging.swh.network' { include role::swh_gateway } node /^storage\d\.internal\.staging\.swh\.network$/ { include role::swh_storage_with_journal } node /^worker\d\.internal\.staging\.swh\.network$/ { include role::swh_worker_inria } node /^search-esnode\d\.internal\.staging\.swh\.network$/ { include role::swh_elasticsearch } node /^search\d\.internal\.staging\.swh\.network$/ { include role::swh_search_with_journal_client } node /^counters\d\.internal\.staging\.swh\.network$/ { include role::swh_counters_with_journal_client } node /^scrubber\d+\.internal\.staging\.swh\.network/ { include role::swh_scrubber_checkers } node 'webapp.internal.staging.swh.network' { include role::swh_webapp } node 'deposit.internal.staging.swh.network' { include role::swh_deposit } node 'vault.internal.staging.swh.network' { include role::swh_vault } node /^rp\d\.internal\.(staging|admin)\.swh\.network$/ { include role::swh_reverse_proxy } # Read-only storage for mirrors node 'objstorage0.internal.staging.swh.network' { include role::swh_remote_objstorage } node 'bojimans.internal.admin.swh.network' { include role::swh_netbox } node /^mirror-test\.internal\.staging\.swh\.network$/ { include profile::postgresql::client } node 'backup01.euwest.azure.internal.softwareheritage.org' { include role::zfs_snapshots_storage } node 'money.internal.admin.swh.network' { include role::swh_azure_billing_report } node 'maven-exporter0.internal.staging.swh.network' { include role::swh_maven_index_exporter } +node /^elastic-worker\d+\.internal\.staging\.swh\.network$/ { + include role::swh_worker_elastic +} + + node default { include role::swh_base } diff --git a/site-modules/profile/manifests/zfs/docker.pp b/site-modules/profile/manifests/zfs/docker.pp new file mode 100644 index 00000000..22b72c23 --- /dev/null +++ b/site-modules/profile/manifests/zfs/docker.pp @@ -0,0 +1,25 @@ +# Handle /var/lib/docker partition as zfs mountpoint +# To reduce the disk usage +class profile::zfs::docker { + # zpool create -f data /dev/vdb + zpool {'data': + ensure => 'present', + disk => 'vdb', + } + + # zfs create -o mountpoint=/var/lib/docker \ + # -o atime=off \ + # -o relatime=on \ # not supported by the following + # -o compression=zstd \ + # data/docker + + zfs { 'data/docker': + ensure => present, + atime => 'off', + compression => 'zstd', + mountpoint => '/var/lib/docker', + require => Zpool['data'], + notify => Service['docker'], + } + -> Package['docker'] +} diff --git a/site-modules/role/manifests/swh_worker_elastic.pp b/site-modules/role/manifests/swh_worker_elastic.pp new file mode 100644 index 00000000..6cf86a94 --- /dev/null +++ b/site-modules/role/manifests/swh_worker_elastic.pp @@ -0,0 +1,4 @@ +class role::swh_worker_elastic inherits role::swh_base { + include profile::docker + include profile::zfs::docker +}