diff --git a/data/common/kafka.yaml b/data/common/kafka.yaml index cf49f8b4..4d38c523 100644 --- a/data/common/kafka.yaml +++ b/data/common/kafka.yaml @@ -1,124 +1,133 @@ --- zookeeper::clusters: rocquencourt: '1': kafka1.internal.softwareheritage.org '2': kafka2.internal.softwareheritage.org '3': kafka3.internal.softwareheritage.org '4': kafka4.internal.softwareheritage.org zookeeper::datastore: /var/lib/zookeeper zookeeper::client_port: 2181 zookeeper::election_port: 2888 zookeeper::leader_port: 3888 kafka::version: "3.3.1" kafka::scala_version: '2.13' kafka::mirror_url: https://dlcdn.apache.org/ kafka::cluster::heap_ops: "-Xmx6G -Xms6G" kafka::inter_broker_protocol_version: "3.3" kafka::log_message_format_version: "3.3" kafka::log_cleaner::threads: 6 kafka::log_cleaner::dedupe_buffer_size: 536870912 # 512 MiB kafka::log_cleaner::max_compaction_lag_ms: 2678400000 # 31 * 24 * 60 * 60 * 1000 = 31 days kafka::log_cleaner::min_compaction_lag_ms: 3600000 # 60 * 60 * 1000 = 1 hour kafka::log_cleaner::min_cleanable_dirty_ratio: 0.1 +kafka::threads::network: 16 +kafka::threads::io: 32 +kafka::threads::background: 16 + kafka::logdirs: - /srv/kafka/logdir kafka::broker_config: log.dirs: "%{alias('kafka::logdirs')}" num.recovery.threads.per.data.dir: 10 # Increase zookeeper and replication timeouts # https://cwiki.apache.org/confluence/display/KAFKA/KIP-537%3A+Increase+default+zookeeper+session+timeout will be default in 2.5.0 zookeeper.session.timeout.ms: 18000 replica.lag.time.max.ms: 30000 # Bump consumer offset retention to 30 days instead of the default of 7 days offsets.retention.minutes: 43200 # Increase the socket request max size to 200 MB socket.request.max.bytes: 209715200 # And the max message size to 100 MB message.max.bytes: 104857600 inter.broker.protocol.version: "%{alias('kafka::inter_broker_protocol_version')}" log.message.format.version: "%{alias('kafka::log_message_format_version')}" # Disable automatic topic creation auto.create.topics.enable: false # Log cleaner settings # number of parallel threads log.cleaner.threads: "%{alias('kafka::log_cleaner::threads')}" # buffer size for the deduplication keys; The cleaner will do multiple passes if # the number of messages in the dirty section of the log overflows this # buffer. log.cleaner.dedupe.buffer.size: "%{alias('kafka::log_cleaner::dedupe_buffer_size')}" # the cleaner will not trigger faster than this threshold log.cleaner.min.compaction.lag.ms: "%{alias('kafka::log_cleaner::min_compaction_lag_ms')}" # the cleaner will trigger at least at this interval log.cleaner.max.compaction.lag.ms: "%{alias('kafka::log_cleaner::max_compaction_lag_ms')}" # the cleaner will trigger when the number of messages in the dirty section # reaches this ratio log.cleaner.min.cleanable.dirty.ratio: "%{alias('kafka::log_cleaner::min_cleanable_dirty_ratio')}" + # Threads + num.network.threads: "%{alias('kafka::threads::network')}" + num.io.threads: "%{alias('kafka::threads::io')}" + background.threads: "%{alias('kafka::threads::background')}" + # kafka::broker::password in private-data kafka::clusters: rocquencourt: zookeeper::chroot: '/kafka/softwareheritage' zookeeper::servers: - kafka1.internal.softwareheritage.org - kafka2.internal.softwareheritage.org - kafka3.internal.softwareheritage.org - kafka4.internal.softwareheritage.org brokers: kafka1.internal.softwareheritage.org: id: 1 public_hostname: broker1.journal.softwareheritage.org kafka2.internal.softwareheritage.org: id: 2 public_hostname: broker2.journal.softwareheritage.org kafka3.internal.softwareheritage.org: id: 3 public_hostname: broker3.journal.softwareheritage.org kafka4.internal.softwareheritage.org: id: 4 public_hostname: broker4.journal.softwareheritage.org superusers: - User:swh-admin-olasd # Users connecting in the plaintext endpoint are ANONYMOUS # TODO: remove when explicit ACLs are given to producers - User:ANONYMOUS broker::heap_opts: "%{lookup('kafka::cluster::heap_ops')}" tls: true plaintext_port: 9092 public_tls_port: 9093 internal_tls_port: 9094 public_listener_network: 128.93.166.0/26 # to label the prometheus exporter metrics environment: production rocquencourt_staging: zookeeper::chroot: '/kafka/softwareheritage' zookeeper::servers: - journal1.internal.staging.swh.network brokers: storage1.internal.staging.swh.network: id: 2 public_hostname: broker1.journal.staging.swh.network broker::heap_opts: "%{alias('kafka::broker::heap_opts')}" superusers: - User:swh-admin-olasd # Users connecting in the plaintext endpoint are ANONYMOUS # TODO: remove when explicit ACLs are given to producers - User:ANONYMOUS tls: true plaintext_port: 9092 public_tls_port: 9093 internal_tls_port: 9094 cluster_config_overrides: offsets.topic.replication.factor: 1 # this is mandatory with only one node public_listener_network: "%{alias('kafka::cluster::public_network')}" # to label the prometheus exporter metrics environment: staging