diff --git a/data/common/kafka.yaml b/data/common/kafka.yaml index 7ad2a269..09b136fe 100644 --- a/data/common/kafka.yaml +++ b/data/common/kafka.yaml @@ -1,123 +1,103 @@ --- zookeeper::clusters: - rocquencourt_legacy: - '1': zookeeper1.internal.softwareheritage.org - '2': zookeeper2.internal.softwareheritage.org - '3': zookeeper3.internal.softwareheritage.org rocquencourt: '1': kafka1.internal.softwareheritage.org '2': kafka2.internal.softwareheritage.org '3': kafka3.internal.softwareheritage.org '4': kafka4.internal.softwareheritage.org azure: '1': kafka01.euwest.azure.internal.softwareheritage.org '2': kafka02.euwest.azure.internal.softwareheritage.org '3': kafka03.euwest.azure.internal.softwareheritage.org '4': kafka04.euwest.azure.internal.softwareheritage.org '5': kafka05.euwest.azure.internal.softwareheritage.org '6': kafka06.euwest.azure.internal.softwareheritage.org zookeeper::datastore: /var/lib/zookeeper zookeeper::client_port: 2181 zookeeper::election_port: 2888 zookeeper::leader_port: 3888 kafka::version: '2.6.0' kafka::scala_version: '2.13' kafka::mirror_url: https://mirrors.ircam.fr/pub/apache/ kafka::logdirs: - /srv/kafka/logdir kafka::broker_config: log.dirs: "%{alias('kafka::logdirs')}" num.recovery.threads.per.data.dir: 10 # Increase zookeeper and replication timeouts # https://cwiki.apache.org/confluence/display/KAFKA/KIP-537%3A+Increase+default+zookeeper+session+timeout will be default in 2.5.0 zookeeper.session.timeout.ms: 18000 replica.lag.time.max.ms: 30000 # Increase the socket request max size to 200 MB socket.request.max.bytes: 209715200 # And the max message size to 100 MB message.max.bytes: 104857600 # For upgrades after 2.6 inter.broker.protocol.version: "2.6" # kafka::broker::password in private-data kafka::clusters: - rocquencourt_legacy: - zookeeper::chroot: '/kafka/softwareheritage' - zookeeper::servers: - - zookeeper1.internal.softwareheritage.org - - zookeeper2.internal.softwareheritage.org - - zookeeper3.internal.softwareheritage.org - brokers: - esnode1.internal.softwareheritage.org: - id: 11 - esnode2.internal.softwareheritage.org: - id: 12 - esnode3.internal.softwareheritage.org: - id: 13 - broker::heap_opts: "-Xmx6G -Xms6G" - tls: false - plaintext_port: 9092 rocquencourt: zookeeper::chroot: '/kafka/softwareheritage' zookeeper::servers: - kafka1.internal.softwareheritage.org - kafka2.internal.softwareheritage.org - kafka3.internal.softwareheritage.org - kafka4.internal.softwareheritage.org brokers: kafka1.internal.softwareheritage.org: id: 1 public_hostname: broker1.journal.softwareheritage.org kafka2.internal.softwareheritage.org: id: 2 public_hostname: broker2.journal.softwareheritage.org kafka3.internal.softwareheritage.org: id: 3 public_hostname: broker3.journal.softwareheritage.org kafka4.internal.softwareheritage.org: id: 4 public_hostname: broker4.journal.softwareheritage.org superusers: - User:swh-admin-olasd # Users connecting in the plaintext endpoint are ANONYMOUS # TODO: remove when explicit ACLs are given to producers - User:ANONYMOUS broker::heap_opts: "-Xmx6G -Xms6G" tls: true plaintext_port: 9092 public_tls_port: 9093 internal_tls_port: 9094 public_listener_network: 128.93.166.0/26 azure: zookeeper::chroot: '/kafka/softwareheritage' zookeeper::servers: - kafka01.euwest.azure.internal.softwareheritage.org - kafka02.euwest.azure.internal.softwareheritage.org - kafka03.euwest.azure.internal.softwareheritage.org - kafka04.euwest.azure.internal.softwareheritage.org - kafka05.euwest.azure.internal.softwareheritage.org - kafka06.euwest.azure.internal.softwareheritage.org brokers: kafka01.euwest.azure.internal.softwareheritage.org: id: 1 kafka02.euwest.azure.internal.softwareheritage.org: id: 2 kafka03.euwest.azure.internal.softwareheritage.org: id: 3 kafka04.euwest.azure.internal.softwareheritage.org: id: 4 kafka05.euwest.azure.internal.softwareheritage.org: id: 5 kafka06.euwest.azure.internal.softwareheritage.org: id: 6 broker::heap_opts: "-Xmx1G -Xms1G" tls: true plaintext_port: 9092 public_tls_port: 9093 internal_tls_port: 9094 diff --git a/manifests/site.pp b/manifests/site.pp index 004803ce..66148bfd 100644 --- a/manifests/site.pp +++ b/manifests/site.pp @@ -1,197 +1,193 @@ node 'louvre.internal.softwareheritage.org' { include role::swh_server } node /^(orsay|beaubourg|hypervisor\d+|branly|pompidou|uffizi)\.(internal\.)?softwareheritage\.org$/ { include role::swh_hypervisor } node 'pergamon.softwareheritage.org' { include role::swh_sysadmin include profile::export_archive_counters } node 'tate.softwareheritage.org' { include role::swh_forge } node 'moma.softwareheritage.org' { include role::swh_rp_webapps } node 'webapp0.softwareheritage.org' { include role::swh_rp_webapp } node 'saatchi.internal.softwareheritage.org' { include role::swh_scheduler } node /^(belvedere|somerset).(internal.)?softwareheritage.org$/ { include role::swh_database include profile::pgbouncer } node 'banco.softwareheritage.org' { include role::swh_backup include role::postgresql_backup } node /^esnode\d+.(internal.)?softwareheritage.org$/ { include role::swh_elasticsearch } -node /^zookeeper\d+.(internal.)?softwareheritage.org$/ { - include role::swh_zookeeper -} - node /^kafka\d+\./ { include role::swh_kafka_broker } node /^cassandra\d+\./ { include role::swh_cassandra_node } node 'granet.internal.softwareheritage.org' { include role::swh_graph_backend } node /^(unibo-prod|vangogh).(euwest.azure.)?(internal.)?softwareheritage.org$/ { include role::swh_vault } node /^saam\.(internal\.)?softwareheritage\.org$/ { include role::swh_storage_baremetal } node 'storage01.euwest.azure.internal.softwareheritage.org' { include role::swh_storage_cloud } node 'storage02.euwest.azure.internal.softwareheritage.org' { include role::swh_storage_cassandra } node /^getty.(internal.)?softwareheritage.org$/ { include role::swh_journal_orchestrator } node /^worker\d+\.(internal\.)?softwareheritage\.org$/ { include role::swh_worker_inria } node /^worker\d+\..*\.azure\.internal\.softwareheritage\.org$/ { include role::swh_worker_azure } node /^dbreplica(0|1)\.euwest\.azure\.internal\.softwareheritage\.org$/ { include role::swh_database } node /^ceph-osd\d+\.internal\.softwareheritage\.org$/ { include role::swh_ceph_osd } node /^ceph-mon\d+\.internal\.softwareheritage\.org$/ { include role::swh_ceph_mon } node /^ns\d+\.(.*\.azure\.)?internal\.softwareheritage\.org/ { include role::swh_nameserver_secondary } node 'thyssen.internal.softwareheritage.org' { include role::swh_ci_server } node 'riverside.internal.softwareheritage.org' { include role::swh_sentry } node /^jenkins-debian\d+\.internal\.softwareheritage\.org$/ { include role::swh_ci_agent_debian } node 'logstash0.internal.softwareheritage.org' { include role::swh_logstash_instance } node 'kibana0.internal.softwareheritage.org' { include role::swh_kibana_instance } node 'kelvingrove.internal.softwareheritage.org' { include role::swh_idp_primary } node 'giverny.softwareheritage.org' { include role::swh_desktop } node /^db\d\.internal\.staging\.swh\.network$/ { include role::swh_database include profile::postgresql::server include profile::pgbouncer include profile::postgresql::client } node 'scheduler0.internal.staging.swh.network' { include role::swh_scheduler include profile::postgresql::client } node 'gateway.internal.staging.swh.network' { include role::swh_gateway } node /^storage\d\.internal\.staging\.swh\.network$/ { include role::swh_base_storage include profile::postgresql::client include profile::swh::deploy::journal::backfill } node /^worker\d\.internal\.staging\.swh\.network$/ { include role::swh_worker_inria } node /^search-esnode\d\.internal\.staging\.swh\.network$/ { include role::swh_elasticsearch } node /^search\d\.internal\.staging\.swh\.network$/ { include role::swh_search_with_journal_client } node 'webapp.internal.staging.swh.network' { include role::swh_webapp } node 'deposit.internal.staging.swh.network' { include role::swh_deposit } node 'vault.internal.staging.swh.network' { include role::swh_vault } node /^rp\d\.internal\.staging\.swh\.network$/ { include role::swh_reverse_proxy } node 'journal0.internal.staging.swh.network' { include role::swh_journal_allinone } node 'bojimans.internal.softwareheritage.org' { include role::swh_netbox } node 'clearly-defined.internal.staging.swh.network' { include role::swh_db_client } node default { include role::swh_base }