diff --git a/data/common/kafka.yaml b/data/common/kafka.yaml index 09b136fe..9afd7633 100644 --- a/data/common/kafka.yaml +++ b/data/common/kafka.yaml @@ -1,103 +1,69 @@ --- zookeeper::clusters: rocquencourt: '1': kafka1.internal.softwareheritage.org '2': kafka2.internal.softwareheritage.org '3': kafka3.internal.softwareheritage.org '4': kafka4.internal.softwareheritage.org - azure: - '1': kafka01.euwest.azure.internal.softwareheritage.org - '2': kafka02.euwest.azure.internal.softwareheritage.org - '3': kafka03.euwest.azure.internal.softwareheritage.org - '4': kafka04.euwest.azure.internal.softwareheritage.org - '5': kafka05.euwest.azure.internal.softwareheritage.org - '6': kafka06.euwest.azure.internal.softwareheritage.org zookeeper::datastore: /var/lib/zookeeper zookeeper::client_port: 2181 zookeeper::election_port: 2888 zookeeper::leader_port: 3888 kafka::version: '2.6.0' kafka::scala_version: '2.13' kafka::mirror_url: https://mirrors.ircam.fr/pub/apache/ kafka::logdirs: - /srv/kafka/logdir kafka::broker_config: log.dirs: "%{alias('kafka::logdirs')}" num.recovery.threads.per.data.dir: 10 # Increase zookeeper and replication timeouts # https://cwiki.apache.org/confluence/display/KAFKA/KIP-537%3A+Increase+default+zookeeper+session+timeout will be default in 2.5.0 zookeeper.session.timeout.ms: 18000 replica.lag.time.max.ms: 30000 # Increase the socket request max size to 200 MB socket.request.max.bytes: 209715200 # And the max message size to 100 MB message.max.bytes: 104857600 # For upgrades after 2.6 inter.broker.protocol.version: "2.6" # kafka::broker::password in private-data kafka::clusters: rocquencourt: zookeeper::chroot: '/kafka/softwareheritage' zookeeper::servers: - kafka1.internal.softwareheritage.org - kafka2.internal.softwareheritage.org - kafka3.internal.softwareheritage.org - kafka4.internal.softwareheritage.org brokers: kafka1.internal.softwareheritage.org: id: 1 public_hostname: broker1.journal.softwareheritage.org kafka2.internal.softwareheritage.org: id: 2 public_hostname: broker2.journal.softwareheritage.org kafka3.internal.softwareheritage.org: id: 3 public_hostname: broker3.journal.softwareheritage.org kafka4.internal.softwareheritage.org: id: 4 public_hostname: broker4.journal.softwareheritage.org superusers: - User:swh-admin-olasd # Users connecting in the plaintext endpoint are ANONYMOUS # TODO: remove when explicit ACLs are given to producers - User:ANONYMOUS broker::heap_opts: "-Xmx6G -Xms6G" tls: true plaintext_port: 9092 public_tls_port: 9093 internal_tls_port: 9094 public_listener_network: 128.93.166.0/26 - azure: - zookeeper::chroot: '/kafka/softwareheritage' - zookeeper::servers: - - kafka01.euwest.azure.internal.softwareheritage.org - - kafka02.euwest.azure.internal.softwareheritage.org - - kafka03.euwest.azure.internal.softwareheritage.org - - kafka04.euwest.azure.internal.softwareheritage.org - - kafka05.euwest.azure.internal.softwareheritage.org - - kafka06.euwest.azure.internal.softwareheritage.org - brokers: - kafka01.euwest.azure.internal.softwareheritage.org: - id: 1 - kafka02.euwest.azure.internal.softwareheritage.org: - id: 2 - kafka03.euwest.azure.internal.softwareheritage.org: - id: 3 - kafka04.euwest.azure.internal.softwareheritage.org: - id: 4 - kafka05.euwest.azure.internal.softwareheritage.org: - id: 5 - kafka06.euwest.azure.internal.softwareheritage.org: - id: 6 - broker::heap_opts: "-Xmx1G -Xms1G" - tls: true - plaintext_port: 9092 - public_tls_port: 9093 - internal_tls_port: 9094