diff --git a/data/common/kafka.yaml b/data/common/kafka.yaml index 0f47d8f1..cb6533ee 100644 --- a/data/common/kafka.yaml +++ b/data/common/kafka.yaml @@ -1,116 +1,116 @@ --- zookeeper::clusters: rocquencourt_legacy: '1': zookeeper1.internal.softwareheritage.org '2': zookeeper2.internal.softwareheritage.org '3': zookeeper3.internal.softwareheritage.org rocquencourt: '1': kafka1.internal.softwareheritage.org '2': kafka2.internal.softwareheritage.org '3': kafka3.internal.softwareheritage.org '4': kafka4.internal.softwareheritage.org azure: '1': kafka01.euwest.azure.internal.softwareheritage.org '2': kafka02.euwest.azure.internal.softwareheritage.org '3': kafka03.euwest.azure.internal.softwareheritage.org '4': kafka04.euwest.azure.internal.softwareheritage.org '5': kafka05.euwest.azure.internal.softwareheritage.org '6': kafka06.euwest.azure.internal.softwareheritage.org zookeeper::datastore: /var/lib/zookeeper zookeeper::client_port: 2181 zookeeper::election_port: 2888 zookeeper::leader_port: 3888 kafka::version: '2.6.0' kafka::scala_version: '2.13' kafka::mirror_url: https://mirrors.ircam.fr/pub/apache/ kafka::logdirs: - /srv/kafka/logdir kafka::broker_config: log.dirs: "%{alias('kafka::logdirs')}" num.recovery.threads.per.data.dir: 10 # Increase zookeeper and replication timeouts # https://cwiki.apache.org/confluence/display/KAFKA/KIP-537%3A+Increase+default+zookeeper+session+timeout will be default in 2.5.0 zookeeper.session.timeout.ms: 18000 replica.lag.time.max.ms: 30000 # Increase the socket request max size to 200 MB socket.request.max.bytes: 209715200 # And the max message size to 100 MB message.max.bytes: 104857600 - # upgrading from 2.4 - inter.broker.protocol.version: "2.4" + # For upgrades after 2.6 + inter.broker.protocol.version: "2.6" kafka::clusters: rocquencourt_legacy: zookeeper::chroot: '/kafka/softwareheritage' zookeeper::servers: - zookeeper1.internal.softwareheritage.org - zookeeper2.internal.softwareheritage.org - zookeeper3.internal.softwareheritage.org brokers: esnode1.internal.softwareheritage.org: id: 11 esnode2.internal.softwareheritage.org: id: 12 esnode3.internal.softwareheritage.org: id: 13 broker::heap_opts: "-Xmx6G -Xms6G" tls: false plaintext_port: 9092 rocquencourt: zookeeper::chroot: '/kafka/softwareheritage' zookeeper::servers: - kafka1.internal.softwareheritage.org - kafka2.internal.softwareheritage.org - kafka3.internal.softwareheritage.org - kafka4.internal.softwareheritage.org brokers: kafka1.internal.softwareheritage.org: id: 1 public_hostname: broker1.journal.softwareheritage.org kafka2.internal.softwareheritage.org: id: 2 public_hostname: broker2.journal.softwareheritage.org kafka3.internal.softwareheritage.org: id: 3 public_hostname: broker3.journal.softwareheritage.org kafka4.internal.softwareheritage.org: id: 4 public_hostname: broker4.journal.softwareheritage.org broker::heap_opts: "-Xmx6G -Xms6G" tls: true plaintext_port: 9092 public_tls_port: 9093 internal_tls_port: 9094 public_listener_network: 128.93.166.0/26 azure: zookeeper::chroot: '/kafka/softwareheritage' zookeeper::servers: - kafka01.euwest.azure.internal.softwareheritage.org - kafka02.euwest.azure.internal.softwareheritage.org - kafka03.euwest.azure.internal.softwareheritage.org - kafka04.euwest.azure.internal.softwareheritage.org - kafka05.euwest.azure.internal.softwareheritage.org - kafka06.euwest.azure.internal.softwareheritage.org brokers: kafka01.euwest.azure.internal.softwareheritage.org: id: 1 kafka02.euwest.azure.internal.softwareheritage.org: id: 2 kafka03.euwest.azure.internal.softwareheritage.org: id: 3 kafka04.euwest.azure.internal.softwareheritage.org: id: 4 kafka05.euwest.azure.internal.softwareheritage.org: id: 5 kafka06.euwest.azure.internal.softwareheritage.org: id: 6 broker::heap_opts: "-Xmx1G -Xms1G" tls: true plaintext_port: 9092 public_tls_port: 9093 internal_tls_port: 9094