Page Menu
Home
Software Heritage
Search
Configure Global Search
Log In
Files
F9340099
No One
Temporary
Actions
View File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Award Token
Flag For Later
Size
5 KB
Subscribers
None
View Options
diff --git a/data/common/kafka.yaml b/data/common/kafka.yaml
index db0b310b..cf49f8b4 100644
--- a/data/common/kafka.yaml
+++ b/data/common/kafka.yaml
@@ -1,124 +1,124 @@
---
zookeeper::clusters:
rocquencourt:
'1': kafka1.internal.softwareheritage.org
'2': kafka2.internal.softwareheritage.org
'3': kafka3.internal.softwareheritage.org
'4': kafka4.internal.softwareheritage.org
zookeeper::datastore: /var/lib/zookeeper
zookeeper::client_port: 2181
zookeeper::election_port: 2888
zookeeper::leader_port: 3888
kafka::version: "3.3.1"
kafka::scala_version: '2.13'
kafka::mirror_url: https://dlcdn.apache.org/
kafka::cluster::heap_ops: "-Xmx6G -Xms6G"
kafka::inter_broker_protocol_version: "3.3"
kafka::log_message_format_version: "3.3"
kafka::log_cleaner::threads: 6
-kafka::log_cleaner::dedupe_buffer_size: 2147483648 # 2 GiB
+kafka::log_cleaner::dedupe_buffer_size: 536870912 # 512 MiB
kafka::log_cleaner::max_compaction_lag_ms: 2678400000 # 31 * 24 * 60 * 60 * 1000 = 31 days
kafka::log_cleaner::min_compaction_lag_ms: 3600000 # 60 * 60 * 1000 = 1 hour
kafka::log_cleaner::min_cleanable_dirty_ratio: 0.1
kafka::logdirs:
- /srv/kafka/logdir
kafka::broker_config:
log.dirs: "%{alias('kafka::logdirs')}"
num.recovery.threads.per.data.dir: 10
# Increase zookeeper and replication timeouts
# https://cwiki.apache.org/confluence/display/KAFKA/KIP-537%3A+Increase+default+zookeeper+session+timeout will be default in 2.5.0
zookeeper.session.timeout.ms: 18000
replica.lag.time.max.ms: 30000
# Bump consumer offset retention to 30 days instead of the default of 7 days
offsets.retention.minutes: 43200
# Increase the socket request max size to 200 MB
socket.request.max.bytes: 209715200
# And the max message size to 100 MB
message.max.bytes: 104857600
inter.broker.protocol.version: "%{alias('kafka::inter_broker_protocol_version')}"
log.message.format.version: "%{alias('kafka::log_message_format_version')}"
# Disable automatic topic creation
auto.create.topics.enable: false
# Log cleaner settings
# number of parallel threads
log.cleaner.threads: "%{alias('kafka::log_cleaner::threads')}"
# buffer size for the deduplication keys; The cleaner will do multiple passes if
# the number of messages in the dirty section of the log overflows this
# buffer.
log.cleaner.dedupe.buffer.size: "%{alias('kafka::log_cleaner::dedupe_buffer_size')}"
# the cleaner will not trigger faster than this threshold
log.cleaner.min.compaction.lag.ms: "%{alias('kafka::log_cleaner::min_compaction_lag_ms')}"
# the cleaner will trigger at least at this interval
log.cleaner.max.compaction.lag.ms: "%{alias('kafka::log_cleaner::max_compaction_lag_ms')}"
# the cleaner will trigger when the number of messages in the dirty section
# reaches this ratio
log.cleaner.min.cleanable.dirty.ratio: "%{alias('kafka::log_cleaner::min_cleanable_dirty_ratio')}"
# kafka::broker::password in private-data
kafka::clusters:
rocquencourt:
zookeeper::chroot: '/kafka/softwareheritage'
zookeeper::servers:
- kafka1.internal.softwareheritage.org
- kafka2.internal.softwareheritage.org
- kafka3.internal.softwareheritage.org
- kafka4.internal.softwareheritage.org
brokers:
kafka1.internal.softwareheritage.org:
id: 1
public_hostname: broker1.journal.softwareheritage.org
kafka2.internal.softwareheritage.org:
id: 2
public_hostname: broker2.journal.softwareheritage.org
kafka3.internal.softwareheritage.org:
id: 3
public_hostname: broker3.journal.softwareheritage.org
kafka4.internal.softwareheritage.org:
id: 4
public_hostname: broker4.journal.softwareheritage.org
superusers:
- User:swh-admin-olasd
# Users connecting in the plaintext endpoint are ANONYMOUS
# TODO: remove when explicit ACLs are given to producers
- User:ANONYMOUS
broker::heap_opts: "%{lookup('kafka::cluster::heap_ops')}"
tls: true
plaintext_port: 9092
public_tls_port: 9093
internal_tls_port: 9094
public_listener_network: 128.93.166.0/26
# to label the prometheus exporter metrics
environment: production
rocquencourt_staging:
zookeeper::chroot: '/kafka/softwareheritage'
zookeeper::servers:
- journal1.internal.staging.swh.network
brokers:
storage1.internal.staging.swh.network:
id: 2
public_hostname: broker1.journal.staging.swh.network
broker::heap_opts: "%{alias('kafka::broker::heap_opts')}"
superusers:
- User:swh-admin-olasd
# Users connecting in the plaintext endpoint are ANONYMOUS
# TODO: remove when explicit ACLs are given to producers
- User:ANONYMOUS
tls: true
plaintext_port: 9092
public_tls_port: 9093
internal_tls_port: 9094
cluster_config_overrides:
offsets.topic.replication.factor: 1 # this is mandatory with only one node
public_listener_network: "%{alias('kafka::cluster::public_network')}"
# to label the prometheus exporter metrics
environment: staging
File Metadata
Details
Attached
Mime Type
text/x-diff
Expires
Jul 4 2025, 10:14 AM (5 w, 5 h ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3285770
Attached To
rSPSITE puppet-swh-site
Event Timeline
Log In to Comment