diff --git a/data/common/cassandra.yaml b/data/common/cassandra.yaml index 508a2fcc..1a65e597 100644 --- a/data/common/cassandra.yaml +++ b/data/common/cassandra.yaml @@ -1,78 +1,84 @@ cassandra::base_data_directory: /srv/cassandra cassandra::base_config_directory: /etc/cassandra cassandra::base_log_directory: /var/log/cassandra cassandra::version: 4.0.5 cassandra::listen_network: "%{lookup('internal_network')}" cassandra::nodes: cassandra01.internal.softwareheritage.org: instances: instance1: cluster_name: archive_production cassandra02.internal.softwareheritage.org: instances: instance1: cluster_name: archive_production cassandra03.internal.softwareheritage.org: instances: instance1: cluster_name: archive_production cassandra04.internal.softwareheritage.org: instances: instance1: cluster_name: archive_production cassandra05.internal.softwareheritage.org: instances: instance1: cluster_name: archive_production cassandra06.internal.softwareheritage.org: instances: instance1: cluster_name: archive_production cassandra::clusters: archive_production: seed_provider: - class_name: org.apache.cassandra.locator.SimpleSeedProvider parameters: - seeds: "cassandra01.internal.softwareheritage.org:7000" cassandra::default_instance_configuration: cluster_name: "%{lookup('cassandra::default_cluster_name')}" datacenter: "%{::subnet}" # rack: "%{::hostname}" rack: rack1 native_transport_port: 9042 storage_port: 7000 jmx_port: 7199 jmx_exporter_port: 7070 + jmx_remote: true + jmx_user: "%{lookup('cassandra::jmx::user')}" + jmx_password: "%{lookup('cassandra::jmx::password')}" + +cassandra::jmx::user: cassandra +# cassandra::jmx::password in private data # Reflect the base of the cassandra.yaml content # These list is completed and/or overridden in cassandra::instance cassandra::base_instance_configuration: num_tokens: 16 allocate_tokens_for_local_replication_factor: 3 disk_optimization_strategy: ssd concurrent_compactors: 4 # should be min(nb core, nb disks) internode_compression: dc # default dc possible all|dc|none concurrent_reads: 64 # 16 x number of drives concurrent_writes: 128 # 8 x number of cores concurrent_counter_writes: 48 commitlog_sync: periodic # default periodic commitlog_sync_period_in_ms: 10000 # default 10000 commitlog_total_space_in_mb: 16384 # default 8192 commitlog_segment_size_in_mb: 256 # default 32 (due to oversize mutation on revision table) partitioner: org.apache.cassandra.dht.Murmur3Partitioner endpoint_snitch: GossipingPropertyFileSnitch enable_user_defined_functions: true # needed by swh-storage compaction_throughput_mb_per_sec: 160 read_request_timeout_in_ms: 5000 range_request_timeout_in_ms: 10000 write_request_timeout_in_ms: 2000 counter_write_request_timeout_in_ms: 5000 cas_contention_timeout_in_ms: 1000 truncate_request_timeout_in_ms: 60000 request_timeout_in_ms: 10000 slow_query_log_timeout_in_ms: 1000 diff --git a/site-modules/profile/manifests/cassandra.pp b/site-modules/profile/manifests/cassandra.pp index 9d3c3faa..8bb137f1 100644 --- a/site-modules/profile/manifests/cassandra.pp +++ b/site-modules/profile/manifests/cassandra.pp @@ -1,134 +1,165 @@ # Install the base components of a 4.x cassandra server # Configure all the instances declared in cassandra::instances property # # Look at profile::cassandra:node for more information about the # instance(s) configuration class profile::cassandra { include profile::prometheus::jmx $jmx_exporter_version = lookup('prometheus::jmx::version') $cassandra_user = 'cassandra' $cassandra_group = 'cassandra' $cassandra_home = '/home/cassandra' $cassandra_version = lookup('cassandra::version') $cassandra_archive_name = "apache-cassandra-${cassandra_version}-bin.tar.gz" $cassandra_bin_url = "https://dlcdn.apache.org/cassandra/${cassandra_version}/${cassandra_archive_name}" $cassandra_bin_checksum_type = 'sha512' $cassandra_bin_checksum = '188e131392ea0e48b46f24b1be297ef6335197f4480c9421328006507e069dce659ce3ce473906398273a5926e331960cbf824362e40cb4c74670cde95458349' $systemd_service = 'cassandra@.service' $download_path = "/opt/${cassandra_archive_name}" $cassandra_install_directory = "/opt/cassandra-${cassandra_version}" $cassandra_base_data_directory = lookup('cassandra::base_data_directory') $cassandra_config_directory = lookup('cassandra::base_config_directory') $cassandra_log_directory = lookup('cassandra::base_log_directory') $cassandra_nodes = lookup('cassandra::nodes') $node_definition = $cassandra_nodes["$::fqdn"] $instances = $node_definition['instances'] $default_instance_config = lookup('cassandra::default_instance_configuration') $clusters_config = lookup('cassandra::clusters') + $jmx_remote = $default_instance_config['jmx_remote'] + $jmx_access_file = "${cassandra_config_directory}/jmxremote.access" + $jmx_password_file = "${cassandra_config_directory}/jmxremote.password" + group {$cassandra_group: system => true, } user {$cassandra_user: system => true, gid => $cassandra_group, shell => '/usr/sbin/nologin', home => $cassandra_home, } file { [ $cassandra_install_directory, $cassandra_config_directory, ]: ensure => directory, owner => 'root', group => 'root', mode => '0755' } file { [ $cassandra_base_data_directory, $cassandra_log_directory, ]: ensure => directory, owner => $cassandra_user, group => $cassandra_group, mode => '0755' } ensure_packages(['openjdk-11-jdk', 'libnetty-java']) archive { 'cassandra': path => $download_path, extract => true, extract_command => 'tar xzf %s --strip-components=1 --no-same-owner --no-same-permissions', source => $cassandra_bin_url, extract_path => $cassandra_install_directory, checksum_type => $cassandra_bin_checksum_type, checksum => $cassandra_bin_checksum, creates => "${cassandra_install_directory}/bin/cassandra", cleanup => true, user => 'root', group => 'root', require => File[$cassandra_install_directory], } -> file {'/opt/cassandra': ensure => link, force => true, target => $cassandra_install_directory } ::systemd::unit_file {$systemd_service: ensure => present, content => template('profile/cassandra/cassandra.service.erb'), } $config_files_to_copy = [ 'jvm11-clients.options', 'jvm-clients.options', 'logback-tools.xml', ] $config_files_to_copy.each | $file_name | { file { "${cassandra_config_directory}/${file_name}": ensure => present, owner => 'root', group => 'root', mode => '0644', source => "/opt/cassandra/conf/${file_name}", require => [File[$cassandra_config_directory], Archive['cassandra']], } } file {"${cassandra_config_directory}/jmx_exporter.yml": ensure => present, owner => 'root', group => 'root', mode => '0644', source => "https://raw.githubusercontent.com/prometheus/jmx_exporter/parent-${jmx_exporter_version}/example_configs/cassandra.yml", require => [File[$cassandra_config_directory]], } + if $jmx_remote == true { + $jmx_user = $default_instance_config['jmx_user'] + $jmx_password = $default_instance_config['jmx_password'] + + file {$jmx_access_file: + ensure => present, + owner => 'root', + group => $cassandra_group, + mode => '0540', + content => template('profile/cassandra/jmxremote.access.erb'), + require => [File[$cassandra_config_directory]] + } + file {$jmx_password_file: + ensure => present, + owner => 'root', + group => $cassandra_group, + mode => '0540', + content => template('profile/cassandra/jmxremote.password.erb'), + require => [File[$cassandra_config_directory]] + } + } else { + file {[ $jmx_access_file, + $jmx_password_file ]: + ensure => absent, + } + } + $instances.each | $instance_name, $instance_config | { $merged_instance_config = $default_instance_config + $instance_config $cluster_config = $clusters_config[$merged_instance_config["cluster_name"]] $merged_config = $cluster_config + $merged_instance_config profile::cassandra::instance{$instance_name: config => $merged_config } } } diff --git a/site-modules/profile/manifests/cassandra/instance.pp b/site-modules/profile/manifests/cassandra/instance.pp index 56027ad2..b4dc99be 100644 --- a/site-modules/profile/manifests/cassandra/instance.pp +++ b/site-modules/profile/manifests/cassandra/instance.pp @@ -1,112 +1,122 @@ # Configure a cassandra node on a server # Several nodes can coexist on a same server # # It supposes the profile::cassandra class # was installed before this define profile::cassandra::instance ( $instance_name = $name, $config = {} ) { $service_name = "cassandra@${instance_name}.service" $listen_network = lookup('internal_network') $listen_address = ip_for_network($listen_network) $cassandra_base_data_dir = lookup('cassandra::base_data_directory') $instance_base_data_dir = "${cassandra_base_data_dir}/${instance_name}" $cassandra_config_dir = lookup('cassandra::base_config_directory') $cassandra_log_dir = lookup('cassandra::base_log_directory') $base_data_dir = "${instance_base_data_dir}/data" $commitlog_dir = "${instance_base_data_dir}/commitlog" $data_dir = "${base_data_dir}/data" $hints_dir = "${data_dir}/hints" $saved_caches_dir = "${data_dir}/saved_caches" $config_dir = "${cassandra_config_dir}/${instance_name}" $log_dir = "${cassandra_log_dir}/${instance_name}" $jmx_exporter_path = $::profile::prometheus::jmx::jar_path + $jmx_remote = $config['jmx_remote'] + $jmx_port = $config['jmx_port'] $base_configuration = lookup('cassandra::base_instance_configuration') $instance_configuration = { cluster_name => $config["cluster_name"], data_file_directories => [ $base_data_dir, ], commitlog_directory => $commitlog_dir, hints_directory => $hints_dir, saved_caches_directory => $saved_caches_dir, listen_address => $listen_address, native_transport_port => $config['native_transport_port'], storage_port => $config['storage_port'], seed_provider => $config['seed_provider'] } $computed_configuration = $base_configuration + $instance_configuration + # jmx port is hardcoded in the cassandra-env.sh file so it needs to be overriden in the + # service configuration + if $jmx_remote { + $extra_jmx_option = "-Dcassandra.jmx.remote.port=${jmx_port} -Dcom.sun.management.jmxremote.access.file=${cassandra_config_dir}/jmxremote.access" + } else { + $extra_jmx_option = "-Dcassandra.jmx.local.port=${jmx_port}" + } + file {[ $instance_base_data_dir, $base_data_dir, # $commitlog_dir, $config_dir, $log_dir, ] : ensure => directory, owner => $::profile::cassandra::cassandra_user, group => $::profile::cassandra::cassandra_group, require => [ # File[$::profile::cassandra::cassandra_base_data_directory], # File[$::profile::cassandra::cassandra_config_directory], # File[$::profile::cassandra::cassandra_log_directory], ] } ::systemd::dropin_file { "${service_name}.d/parameters.conf": ensure => present, unit => "cassandra@${instance_name}.service", filename => 'parameters.conf', content => template('profile/cassandra/instance-parameters.conf.erb'), } service {$service_name: enable => true, } $config_files_to_copy = [ 'jvm11-server.options', 'jvm-server.options', 'logback.xml', 'cassandra-env.sh', ] $config_files_to_copy.each | $file_name | { file { "${config_dir}/${file_name}": ensure => present, owner => 'root', group => 'root', mode => '0644', source => "/opt/cassandra/conf/${file_name}", require => [File[$config_dir]], } } file { "${config_dir}/cassandra.yaml": ensure => present, owner => 'root', group => 'root', mode => '0644', content => inline_yaml($computed_configuration), require => [File[$config_dir]], } file { "${config_dir}/cassandra-rackdc.properties": ensure => present, owner => 'root', group => 'root', mode => '0644', content => template('profile/cassandra/cassandra-rackdc.properties.erb'), require => [File[$config_dir]], } } diff --git a/site-modules/profile/templates/cassandra/instance-parameters.conf.erb b/site-modules/profile/templates/cassandra/instance-parameters.conf.erb index 3b41df23..0c040a7b 100644 --- a/site-modules/profile/templates/cassandra/instance-parameters.conf.erb +++ b/site-modules/profile/templates/cassandra/instance-parameters.conf.erb @@ -1,13 +1,16 @@ # Managed by puppet (class profile::cassandra::node); Changes will be lost. [Unit] Description=Cassandra <%= @instance_name %> instance After=network.target [Service] -Environment=JVM_EXTRA_OPTS="-javaagent:<%= @jmx_exporter_path %>=<%= @config["jmx_exporter_port"] %>:/etc/cassandra/jmx_exporter.yml -Dcassandra.jmx.local.port=<%= @config["jmx_port"] %> -Dcom.sun.management.jmxremote.authenticate=false" +Environment=JVM_EXTRA_OPTS="-javaagent:<%= @jmx_exporter_path %>=<%= @config["jmx_exporter_port"] %>:/etc/cassandra/jmx_exporter.yml <%= @extra_jmx_option %>" Environment=CASSANDRA_CONF=<%= @config_dir %> Environment=CASSANDRA_LOG_DIR=<%= @log_dir %> +<%- if @jmx_remote -%> +Environment=LOCAL_JMX=no +<%- end -%> [Install] WantedBy=multi-user.target diff --git a/site-modules/profile/templates/cassandra/jmxremote.access.erb b/site-modules/profile/templates/cassandra/jmxremote.access.erb new file mode 100644 index 00000000..b826f024 --- /dev/null +++ b/site-modules/profile/templates/cassandra/jmxremote.access.erb @@ -0,0 +1,5 @@ +monitorRole readonly +controlRole readwrite \ + create javax.management.monitor.*,javax.management.timer.* \ + unregister +<%= @jmx_user %> readwrite diff --git a/site-modules/profile/templates/cassandra/jmxremote.password.erb b/site-modules/profile/templates/cassandra/jmxremote.password.erb new file mode 100644 index 00000000..243688ba --- /dev/null +++ b/site-modules/profile/templates/cassandra/jmxremote.password.erb @@ -0,0 +1 @@ +<%= @jmx_user %> <%= @jmx_password %>