diff --git a/data/common/kafka.yaml b/data/common/kafka.yaml --- a/data/common/kafka.yaml +++ b/data/common/kafka.yaml @@ -68,5 +68,26 @@ public_tls_port: 9093 internal_tls_port: 9094 public_listener_network: 128.93.166.0/26 + rocquencourt_staging: + zookeeper::chroot: '/kafka/softwareheritage' + zookeeper::servers: + - journal1.internal.staging.swh.network + brokers: + storage1.internal.staging.swh.network: + id: 2 + public_hostname: broker1.journal.staging.swh.network + broker::heap_opts: "%{alias('kafka::broker::heap_opts')}" + superusers: + - User:swh-admin-olasd + # Users connecting in the plaintext endpoint are ANONYMOUS + # TODO: remove when explicit ACLs are given to producers + - User:ANONYMOUS + tls: true + plaintext_port: 9092 + public_tls_port: 9093 + internal_tls_port: 9094 + cluster_config_overrides: + offsets.topic.replication.factor: 1 # this is mandatory with only one node + public_listener_network: "%{alias('kafka::cluster::public_network')}" diff --git a/data/deployments/staging/common.yaml b/data/deployments/staging/common.yaml --- a/data/deployments/staging/common.yaml +++ b/data/deployments/staging/common.yaml @@ -98,29 +98,6 @@ kafka::broker::heap_opts: "-Xmx3G -Xms3G" -kafka::clusters: - rocquencourt_staging: - zookeeper::chroot: '/kafka/softwareheritage' - zookeeper::servers: - - journal1.internal.staging.swh.network - brokers: - storage1.internal.staging.swh.network: - id: 2 - public_hostname: broker1.journal.staging.swh.network - broker::heap_opts: "%{alias('kafka::broker::heap_opts')}" - superusers: - - User:swh-admin-olasd - # Users connecting in the plaintext endpoint are ANONYMOUS - # TODO: remove when explicit ACLs are given to producers - - User:ANONYMOUS - tls: true - plaintext_port: 9092 - public_tls_port: 9093 - internal_tls_port: 9094 - cluster_config_overrides: - offsets.topic.replication.factor: 1 # this is mandatory with only one node - public_listener_network: "%{alias('kafka::cluster::public_network')}" - swh::deploy::journal::brokers: - journal1.internal.staging.swh.network diff --git a/data/subnets/vagrant.yaml b/data/subnets/vagrant.yaml --- a/data/subnets/vagrant.yaml +++ b/data/subnets/vagrant.yaml @@ -147,6 +147,8 @@ host: counters1.internal.softwareheritage.org 10.168.100.101: host: uffizi.internal.softwareheritage.org + 10.168.100.102: + host: gettys.internal.softwareheritage.org 10.168.100.103: host: somerset.internal.softwareheritage.org 10.168.100.104: diff --git a/site-modules/profile/manifests/kafka/management_scripts.pp b/site-modules/profile/manifests/kafka/management_scripts.pp new file mode 100644 --- /dev/null +++ b/site-modules/profile/manifests/kafka/management_scripts.pp @@ -0,0 +1,40 @@ +# Journal management scripts +class profile::kafka::management_scripts { + $clusters = lookup('kafka::clusters') + + $zookeeper_port = lookup('zookeeper::client_port', Integer) + + $clusters.each | $cluster, $config | { + + $script_name = "/usr/local/sbin/manage_kafka_user_${cluster}.sh" + $kafka_plaintext_port = $config['plaintext_port'] + $zookeeper_chroot = $config['zookeeper::chroot'] + $zookeeper_servers = $config['zookeeper::servers'] + + $zookeeper_server_string = join( + $zookeeper_servers.map |$server| {"${server}:${zookeeper_port}"}, + ',' + ) + + $zookeeper_connection_string = "${zookeeper_server_string}${zookeeper_chroot}" + + $brokers_connection_string = join($config['brokers'].map | $broker, $broker_config | { + "${broker}:${kafka_plaintext_port}" }, ',' + ) + + # the template uses + # - zookeeper_connection_string + # - brokers_connection_string + # using an indirection to avoid a parsing bug + $filename = "/usr/local/sbin/create_kafka_users_${cluster}.sh" + file { $filename: + ensure => 'present', + content => template('profile/kafka/create_kafka_users.sh.erb'), + owner => 'root', + group => 'root', + mode => '0700', + } + + } + +} diff --git a/site-modules/profile/templates/kafka/create_kafka_users.sh.erb b/site-modules/profile/templates/kafka/create_kafka_users.sh.erb new file mode 100644 --- /dev/null +++ b/site-modules/profile/templates/kafka/create_kafka_users.sh.erb @@ -0,0 +1,73 @@ +#!/bin/bash +# +# Managed by Puppet (class profile::kafka::management_scripts), changes will be lost. +# +set -e + +zookeepers=<%= @zookeeper_connection_string %> +brokers=<%= @brokers_connection_string %> + +usage () { + echo "$0 [--privileged] [--consumer-group-prefix prefix] username" +} + +if (( $# < 1 )) || (( $# > 4 )); then + usage + exit 1 +fi + +privileged="unprivileged" +cgrp_prefix="" + +while (( $# )); do + if [ $1 = "--privileged" ]; then + privileged="privileged" + shift + elif [ $1 = "--consumer-group-prefix" ]; then + cgrp_prefix=$2 + shift + shift + else + username=$1 + break + fi +done + +if [ -z "$username" ]; then + usage + exit 1 +fi + +if [ -z "$cgrp_prefix" ]; then + cgrp_prefix="$username-" +fi + +echo "Creating user $username, with $privileged access to consumer group prefix $cgrp_prefix" + +read -s -p "Password for user $username: " password +echo + +echo "Setting user credentials" + +/opt/kafka/bin/kafka-configs.sh \ + --zookeeper "$zookeepers" \ + --alter \ + --add-config "SCRAM-SHA-256=[iterations=8192,password=$password],SCRAM-SHA-512=[password=$password]" \ + --entity-type users \ + --entity-name $username + +topic_prefixes="swh.journal.objects. swh.journal.indexed." + +if [ $privileged = "privileged" ]; then + topic_prefixes="$topic_prefixes swh.journal.objects_privileged." +fi + +for topic_prefix in $topic_prefixes; do + echo "Granting access to topics $topic_prefix to $username" + for op in READ DESCRIBE; do + /opt/kafka/bin/kafka-acls.sh --bootstrap-server $brokers --add --resource-pattern-type PREFIXED --topic $topic_prefix --allow-principal User:$username --operation $op + done +done + +echo "Granting access to consumer group prefix $cgrp_prefix to $username" +/opt/kafka/bin/kafka-acls.sh --bootstrap-server $brokers --add --resource-pattern-type PREFIXED --group ${cgrp_prefix} --allow-principal User:$username --operation READ diff --git a/site-modules/role/manifests/swh_journal_orchestrator.pp b/site-modules/role/manifests/swh_journal_orchestrator.pp --- a/site-modules/role/manifests/swh_journal_orchestrator.pp +++ b/site-modules/role/manifests/swh_journal_orchestrator.pp @@ -1,4 +1,5 @@ class role::swh_journal_orchestrator inherits role::swh_base { include profile::kafka include profile::kafka::prometheus_consumer_group_exporter + include profile::kafka::management_scripts } diff --git a/site-modules/role/manifests/swh_storage_with_journal.pp b/site-modules/role/manifests/swh_storage_with_journal.pp --- a/site-modules/role/manifests/swh_storage_with_journal.pp +++ b/site-modules/role/manifests/swh_storage_with_journal.pp @@ -5,5 +5,4 @@ # journal include profile::zookeeper include profile::kafka::broker - include profile::kafka::prometheus_consumer_group_exporter }