diff --git a/data/common/kafka.yaml b/data/common/kafka.yaml index 9afd7633..5f2cd044 100644 --- a/data/common/kafka.yaml +++ b/data/common/kafka.yaml @@ -1,69 +1,71 @@ --- zookeeper::clusters: rocquencourt: '1': kafka1.internal.softwareheritage.org '2': kafka2.internal.softwareheritage.org '3': kafka3.internal.softwareheritage.org '4': kafka4.internal.softwareheritage.org zookeeper::datastore: /var/lib/zookeeper zookeeper::client_port: 2181 zookeeper::election_port: 2888 zookeeper::leader_port: 3888 kafka::version: '2.6.0' kafka::scala_version: '2.13' kafka::mirror_url: https://mirrors.ircam.fr/pub/apache/ kafka::logdirs: - /srv/kafka/logdir kafka::broker_config: log.dirs: "%{alias('kafka::logdirs')}" num.recovery.threads.per.data.dir: 10 # Increase zookeeper and replication timeouts # https://cwiki.apache.org/confluence/display/KAFKA/KIP-537%3A+Increase+default+zookeeper+session+timeout will be default in 2.5.0 zookeeper.session.timeout.ms: 18000 replica.lag.time.max.ms: 30000 + # Bump consumer offset retention to 30 days instead of the default of 7 days + offsets.retention.minutes: 43200 # Increase the socket request max size to 200 MB socket.request.max.bytes: 209715200 # And the max message size to 100 MB message.max.bytes: 104857600 # For upgrades after 2.6 inter.broker.protocol.version: "2.6" # kafka::broker::password in private-data kafka::clusters: rocquencourt: zookeeper::chroot: '/kafka/softwareheritage' zookeeper::servers: - kafka1.internal.softwareheritage.org - kafka2.internal.softwareheritage.org - kafka3.internal.softwareheritage.org - kafka4.internal.softwareheritage.org brokers: kafka1.internal.softwareheritage.org: id: 1 public_hostname: broker1.journal.softwareheritage.org kafka2.internal.softwareheritage.org: id: 2 public_hostname: broker2.journal.softwareheritage.org kafka3.internal.softwareheritage.org: id: 3 public_hostname: broker3.journal.softwareheritage.org kafka4.internal.softwareheritage.org: id: 4 public_hostname: broker4.journal.softwareheritage.org superusers: - User:swh-admin-olasd # Users connecting in the plaintext endpoint are ANONYMOUS # TODO: remove when explicit ACLs are given to producers - User:ANONYMOUS broker::heap_opts: "-Xmx6G -Xms6G" tls: true plaintext_port: 9092 public_tls_port: 9093 internal_tls_port: 9094 public_listener_network: 128.93.166.0/26