diff --git a/Puppetfile b/Puppetfile index 64810a93..f9a59a8e 100644 --- a/Puppetfile +++ b/Puppetfile @@ -1,208 +1,208 @@ mod 'dar', :git => 'https://forge.softwareheritage.org/source/puppet-swh-dar', :branch => :control_branch, :default_branch => 'master' mod 'gunicorn', :git => 'https://forge.softwareheritage.org/source/puppet-swh-gunicorn', :branch => :control_branch, :default_branch => 'master' mod 'mediawiki', :git => 'https://forge.softwareheritage.org/source/puppet-swh-mediawiki', :branch => :control_branch, :default_branch => 'master' mod 'postfix', :git => 'https://forge.softwareheritage.org/source/puppet-swh-postfix', :branch => :control_branch, :default_branch => 'master' mod 'uwsgi', :git => 'https://forge.softwareheritage.org/source/puppet-swh-uwsgi', :branch => :control_branch, :default_branch => 'master' mod 'apache', :git => 'https://forge.softwareheritage.org/source/puppet-puppetlabs-apache', :tag => 'v5.3.0' mod 'apt', :git => 'https://forge.softwareheritage.org/source/puppet-puppetlabs-apt', :tag => 'v7.3.0' mod 'archive', :git => 'https://forge.softwareheritage.org/source/puppet-puppet-archive', :tag => 'v4.4.0' mod 'bind', :git => 'https://forge.softwareheritage.org/source/puppet-inkblot-bind', :ref => '7.4.0' mod 'cassandra', :git => 'https://forge.softwareheritage.org/source/puppet-puppet-cassandra', :ref => 'master' mod 'ceph', :git => 'https://forge.softwareheritage.org/source/puppet-openstack-ceph', :ref => 'master' mod 'concat', :git => 'https://forge.softwareheritage.org/source/puppet-puppetlabs-concat', :tag => 'v6.1.0' mod 'cups', :git => 'https://forge.softwareheritage.org/source/puppet-mosen-cups', :ref => 'master' mod 'datacat', :git => 'https://forge.softwareheritage.org/source/puppet-richardc-datacat', :ref => '0.6.2' mod 'debconf', :git => 'https://forge.softwareheritage.org/source/puppet-stm-debconf', :ref => 'v3.0.0' mod 'debnet', :git => 'https://forge.softwareheritage.org/source/puppet-trepasi-debnet', :ref => '8d856df078352a8848a43ca0ee9f2ef9086b343a' mod 'docker', :git => 'https://forge.softwareheritage.org/source/puppet-puppetlabs-docker', :ref => 'v3.9.0' mod 'elasticsearch', :git => 'https://forge.softwareheritage.org/source/puppet-elastic-elasticsearch', :ref => '6.4.0' mod 'extlib', :git => 'https://forge.softwareheritage.org/source/puppet-puppet-extlib', :tag => 'v4.1.0' mod 'grafana', :git => 'https://forge.softwareheritage.org/source/puppet-puppet-grafana', :tag => 'v6.1.0' mod 'hitch', :git => 'https://forge.softwareheritage.org/source/puppet-ssm-hitch', :ref => 'feature/additional-config-0.1.5' mod 'icinga2', :git => 'https://forge.softwareheritage.org/source/puppet-icinga-icinga2', :tag => 'v2.3.2' mod 'icingaweb2', :git => 'https://forge.softwareheritage.org/source/puppet-icinga-icingaweb2', :tag => 'v2.3.1' mod 'inifile', :git => 'https://forge.softwareheritage.org/source/puppet-puppetlabs-inifile', :ref => 'v4.0.0' mod 'java', :git => 'https://forge.softwareheritage.org/source/puppet-puppetlabs-java', - :tag => 'v6.0.0' + :tag => 'v7.3.0' mod 'java_ks', :git => 'https://forge.softwareheritage.org/source/puppet-puppetlabs-java_ks', :tag => 'v3.1.0' mod 'kafka', :git => 'https://forge.softwareheritage.org/source/puppet-puppet-kafka', :ref => 'v5.3.0' mod 'keycloak', :git => 'https://forge.softwareheritage.org/source/puppet-treydock-keycloak', :ref => 'v6.19.0' mod 'letsencrypt', :git => 'https://forge.softwareheritage.org/source/puppet-puppet-letsencrypt', :ref => 'v5.0.0' mod 'locales', :git => 'https://forge.softwareheritage.org/source/puppet-saz-locales', :ref => 'v2.5.1' mod 'mysql', :git => 'https://forge.softwareheritage.org/source/puppet-puppetlabs-mysql', :ref => 'v10.3.0' mod 'nginx', :git => 'https://forge.softwareheritage.org/source/puppet-puppet-nginx', :ref => 'v1.0.0' mod 'ntp', :git => 'https://forge.softwareheritage.org/source/puppet-puppetlabs-ntp', :ref => 'v8.2.0' mod 'php', :git => 'https://forge.softwareheritage.org/source/puppet-puppet-php', :ref => 'v7.0.0' mod 'postgresql', :git => 'https://forge.softwareheritage.org/source/puppet-puppetlabs-postgresql', :ref => 'v6.4.0' mod 'pgbouncer', :git => 'https://forge.softwareheritage.org/source/puppet-covermymeds-pgbouncer', :ref => '9ec0d8a1255bbb309c2ff38f229167209cad496b' mod 'puppet', :git => 'https://forge.softwareheritage.org/source/puppet-theforeman-puppet', :ref => 'latest_passenger' mod 'puppetdb', :git => 'https://forge.softwareheritage.org/source/puppet-puppetlabs-puppetdb', :ref => '7.4.0' mod 'memcached', :git => 'https://forge.softwareheritage.org/source/puppet-saz-memcached', :ref => 'v3.4.0' mod 'rabbitmq', :git => 'https://forge.softwareheritage.org/source/puppet-puppet-rabbitmq', :ref => 'v10.0.0' mod 'redis', :git => 'https://forge.softwareheritage.org/source/puppet-puppet-redis', :ref => 'v6.1.0' mod 'resolv_conf', :git => 'https://forge.softwareheritage.org/source/puppet-saz-resolv_conf', :ref => 'v4.1.0' mod 'ssh', :git => 'https://forge.softwareheritage.org/source/puppet-saz-ssh', :ref => 'v6.0.0' mod 'stdlib', :git => 'https://forge.softwareheritage.org/source/puppet-puppetlabs-stdlib', :ref => 'v6.2.0' mod 'sudo', :git => 'https://forge.softwareheritage.org/source/puppet-saz-sudo', :ref => 'v6.0.0' mod 'systemd', :git => 'https://forge.softwareheritage.org/source/puppet-camptocamp-systemd', :ref => '2.12.0' mod 'timezone', :git => 'https://forge.softwareheritage.org/source/puppet-saz-timezone', :ref => 'v5.1.1' mod 'unattended_upgrades', :git => 'https://forge.softwareheritage.org/source/puppet-puppet-unattended_upgrades', :ref => 'v4.0.0' mod 'varnish', :git => 'https://forge.softwareheritage.org/source/puppet-claranet-varnish', :ref => 'bugfix/systemd-unit' mod 'vcsrepo', :git => 'https://forge.softwareheritage.org/source/puppet-puppetlabs-vcsrepo', :ref => 'v3.1.0' mod 'zookeeper', :git => 'https://forge.softwareheritage.org/source/puppet-deric-zookeeper', :ref => 'v0.8.7' diff --git a/data/common/kafka.yaml b/data/common/kafka.yaml index 9320e881..4594afa6 100644 --- a/data/common/kafka.yaml +++ b/data/common/kafka.yaml @@ -1,71 +1,72 @@ --- zookeeper::clusters: rocquencourt: '1': kafka1.internal.softwareheritage.org '2': kafka2.internal.softwareheritage.org '3': kafka3.internal.softwareheritage.org '4': kafka4.internal.softwareheritage.org zookeeper::datastore: /var/lib/zookeeper zookeeper::client_port: 2181 zookeeper::election_port: 2888 zookeeper::leader_port: 3888 kafka::version: '2.6.0' kafka::scala_version: '2.13' kafka::mirror_url: https://archive.apache.org/dist/ +kafka::cluster::heap_ops: "-Xmx6G -Xms6G" kafka::logdirs: - /srv/kafka/logdir kafka::broker_config: log.dirs: "%{alias('kafka::logdirs')}" num.recovery.threads.per.data.dir: 10 # Increase zookeeper and replication timeouts # https://cwiki.apache.org/confluence/display/KAFKA/KIP-537%3A+Increase+default+zookeeper+session+timeout will be default in 2.5.0 zookeeper.session.timeout.ms: 18000 replica.lag.time.max.ms: 30000 # Bump consumer offset retention to 30 days instead of the default of 7 days offsets.retention.minutes: 43200 # Increase the socket request max size to 200 MB socket.request.max.bytes: 209715200 # And the max message size to 100 MB message.max.bytes: 104857600 # For upgrades after 2.6 inter.broker.protocol.version: "2.6" # kafka::broker::password in private-data kafka::clusters: rocquencourt: zookeeper::chroot: '/kafka/softwareheritage' zookeeper::servers: - kafka1.internal.softwareheritage.org - kafka2.internal.softwareheritage.org - kafka3.internal.softwareheritage.org - kafka4.internal.softwareheritage.org brokers: kafka1.internal.softwareheritage.org: id: 1 public_hostname: broker1.journal.softwareheritage.org kafka2.internal.softwareheritage.org: id: 2 public_hostname: broker2.journal.softwareheritage.org kafka3.internal.softwareheritage.org: id: 3 public_hostname: broker3.journal.softwareheritage.org kafka4.internal.softwareheritage.org: id: 4 public_hostname: broker4.journal.softwareheritage.org superusers: - User:swh-admin-olasd # Users connecting in the plaintext endpoint are ANONYMOUS # TODO: remove when explicit ACLs are given to producers - User:ANONYMOUS - broker::heap_opts: "-Xmx6G -Xms6G" + broker::heap_opts: "%{lookup('kafka::cluster::heap_ops')}" tls: true plaintext_port: 9092 public_tls_port: 9093 internal_tls_port: 9094 public_listener_network: 128.93.166.0/26 diff --git a/data/deployments/production/vagrant.yaml b/data/deployments/production/vagrant.yaml index 7a44d387..62a69cbf 100644 --- a/data/deployments/production/vagrant.yaml +++ b/data/deployments/production/vagrant.yaml @@ -1,4 +1,4 @@ elasticsearch::jvm_options::heap_size: 512m swh::postgresql::shared_buffers: 128MB -swh::provenance::db::shared_buffers: 128MB \ No newline at end of file +swh::provenance::db::shared_buffers: 128MB diff --git a/data/subnets/vagrant.yaml b/data/subnets/vagrant.yaml index ce0e5cff..332fa27f 100644 --- a/data/subnets/vagrant.yaml +++ b/data/subnets/vagrant.yaml @@ -1,215 +1,224 @@ --- dns::local_nameservers: - 192.168.100.29 - 192.168.200.22 # forwarder for : # - swh network # - Inria network # - external network dns::forwarders: - 192.168.100.29 - 192.168.200.22 - 128.93.77.234 - 1.1.1.1 dns::forwarder_insecure: true ntp::servers: - sesi-ntp1.inria.fr - sesi-ntp2.inria.fr networks::manage_interfaces: false internal_network: 10.168.128.0/16 puppet::master::codedir: /tmp/puppet networks::private_routes: {} smtp::relay_hostname: 'none' swh::postgresql::network_accesses: - 10.168.100.0/24 swh::deploy::worker::instances: - checker_deposit - lister - loader_archive - loader_cran - loader_debian - loader_deposit - loader_git - loader_mercurial - loader_nixguix - loader_npm - loader_pypi - loader_svn - loader_high_priority dns::forward_zones: { } netbox::vhost::letsencrypt_cert: inventory-vagrant netbox::vhost::name: inventory-vagrant.internal.softwareheritage.org netbox::mail::from: inventory+vagrant@softwareheritage.org netbox::admin::email: sysop+vagrant@softwareheritage.org kafka::cluster::public_network: 10.168.130.0/24 +kafka::cluster::heap_ops: "-Xmx512m -Xms512m" puppet::master::manage_puppetdb: true puppetdb::listen_address: 0.0.0.0 swh::puppetdb::etcdir: /etc/puppetdb puppetdb::confdir: "%{lookup('swh::puppetdb::etcdir')}/conf.d" puppetdb::ssl_dir: "%{lookup('swh::puppetdb::etcdir')}/ssl" swh::puppetdb::ssl_key_path: "%{lookup('puppetdb::ssl_dir')}/key.pem" swh::puppetdb::ssl_key: "%{::puppet_vardir}/ssl/private_keys/pergamon.softwareheritage.org.pem" swh::puppetdb::ssl_cert: "%{::puppet_vardir}/ssl/certs/pergamon.softwareheritage.org.pem" swh::puppetdb::ssl_cert_path: "%{lookup('puppetdb::ssl_dir')}/cert.pem" swh::puppetdb::ssl_ca_cert: "%{::puppet_vardir}/ssl/ca/ca_crt.pem" swh::puppetdb::ssl_ca_cert_path: "%{lookup('puppetdb::ssl_dir')}/ca_crt.pem" puppet::autosign_entries: - '*.softwareheritage.org' - '*.staging.swh.network' - '*.admin.swh.network' static_hostnames: 10.168.50.10: host: bardo.internal.admin.swh.network 10.168.50.20: host: rp1.internal.admin.swh.network aliases: - hedgedoc.softwareheritage.org 10.168.100.18: host: banco.internal.softwareheritage.org aliases: - backup.internal.softwareheritage.org - kibana.internal.softwareheritage.org 10.168.100.19: host: logstash0.internal.softwareheritage.org aliases: - logstash.internal.softwareheritage.org 10.168.100.29: host: pergamon.internal.softwareheritage.org aliases: - pergamon.softwareheritage.org - icinga.internal.softwareheritage.org - grafana.softwareheritage.org - stats.export.softwareheritage 10.168.100.30: host: jenkins.softwareheritage.org 10.168.100.31: host: moma.internal.softwareheritage.org aliases: - archive.internal.softwareheritage.org - deposit.internal.softwareheritage.org - objstorage.softwareheritage.org - objstorage.internal.softwareheritage.org 10.168.100.32: host: beaubourg.internal.softwareheritage.org 10.168.100.34: host: hypervisor3.internal.softwareheritage.org 10.168.100.52: host: riverside.internal.softwareheritage.org aliases: - sentry.softwareheritage.org 10.168.100.61: host: esnode1.internal.softwareheritage.org 10.168.100.62: host: esnode2.internal.softwareheritage.org 10.168.100.63: host: esnode3.internal.softwareheritage.org 10.168.100.71: host: webapp1.internal.softwareheritage.org 10.168.100.81: host: search-esnode1.internal.softwareheritage.org 10.168.100.82: host: search-esnode2.internal.softwareheritage.org 10.168.100.83: host: search-esnode3.internal.softwareheritage.org 10.168.100.85: host: search1.internal.softwareheritage.org 10.168.100.86: host: search-esnode4.internal.softwareheritage.org 10.168.100.87: host: search-esnode5.internal.softwareheritage.org 10.168.100.88: host: search-esnode6.internal.softwareheritage.org 10.168.100.95: host: counters1.internal.softwareheritage.org 10.168.100.101: host: uffizi.internal.softwareheritage.org 10.168.100.104: host: saatchi.internal.softwareheritage.org aliases: - rabbitmq.internal.softwareheritage.org 10.168.100.106: host: kelvingrove.internal.softwareheritage.org aliases: - auth.softwareheritage.org 10.168.100.108: host: branly.internal.softwareheritage.org 10.168.100.109: host: saam.internal.softwareheritage.org 10.168.100.110: host: met.internal.softwareheritage.org 10.168.100.131: host: zookeeper1.internal.softwareheritage.org 10.168.100.132: host: zookeeper2.internal.softwareheritage.org 10.168.100.133: host: zookeeper3.internal.softwareheritage.org 10.168.100.170: host: pompidou.internal.softwareheritage.org 10.168.100.210: host: belvedere.internal.softwareheritage.org aliases: - db.internal.softwareheritage.org 10.168.100.199: host: bojimans.internal.softwareheritage.org aliases: - inventory.internal.softwareheritage.org 10.168.100.201: host: kafka1.internal.softwareheritage.org + aliases: + - broker1.journal.softwareheritage.org 10.168.100.202: host: kafka2.internal.softwareheritage.org + aliases: + - broker2.journal.softwareheritage.org 10.168.100.203: host: kafka3.internal.softwareheritage.org + aliases: + - broker3.journal.softwareheritage.org 10.168.100.204: host: kafka4.internal.softwareheritage.org + aliases: + - broker4.journal.softwareheritage.org 10.168.130.11: host: db1.internal.staging.swh.network 10.168.130.20: host: rp1.internal.staging.swh.network aliases: - webapp.staging.swh.network - deposit.staging.swh.network - objstorage.staging.swh.network 10.168.130.30: host: webapp.internal.staging.swh.network 10.168.130.31: host: deposit.internal.staging.swh.network 10.168.130.41: host: storage1.internal.staging.swh.network aliases: - journal1.internal.staging.swh.network 10.168.130.50: host: scheduler0.internal.staging.swh.network 10.168.130.60: host: vault.internal.staging.swh.network 10.168.130.80: host: search-esnode0.internal.staging.swh.network 10.168.130.90: host: search0.internal.staging.swh.network 10.168.130.95: host: counters0.internal.staging.swh.network 10.168.130.100: host: worker0.internal.staging.swh.network 10.168.130.101: host: worker1.internal.staging.swh.network 10.168.130.102: host: worker2.internal.staging.swh.network 10.168.130.103: host: worker3.internal.staging.swh.network 10.168.130.110: host: objstorage0.internal.staging.swh.network 10.168.130.160: host: mirror-test.internal.staging.swh.network