diff --git a/data/common/cassandra.yaml b/data/common/cassandra.yaml index 1a65e597..d472423f 100644 --- a/data/common/cassandra.yaml +++ b/data/common/cassandra.yaml @@ -1,84 +1,87 @@ cassandra::base_data_directory: /srv/cassandra cassandra::base_config_directory: /etc/cassandra cassandra::base_log_directory: /var/log/cassandra cassandra::version: 4.0.5 cassandra::listen_network: "%{lookup('internal_network')}" cassandra::nodes: cassandra01.internal.softwareheritage.org: instances: instance1: cluster_name: archive_production cassandra02.internal.softwareheritage.org: instances: instance1: cluster_name: archive_production cassandra03.internal.softwareheritage.org: instances: instance1: cluster_name: archive_production cassandra04.internal.softwareheritage.org: instances: instance1: cluster_name: archive_production cassandra05.internal.softwareheritage.org: instances: instance1: cluster_name: archive_production cassandra06.internal.softwareheritage.org: instances: instance1: cluster_name: archive_production cassandra::clusters: archive_production: seed_provider: - class_name: org.apache.cassandra.locator.SimpleSeedProvider parameters: - seeds: "cassandra01.internal.softwareheritage.org:7000" cassandra::default_instance_configuration: cluster_name: "%{lookup('cassandra::default_cluster_name')}" datacenter: "%{::subnet}" # rack: "%{::hostname}" rack: rack1 native_transport_port: 9042 storage_port: 7000 jmx_port: 7199 jmx_exporter_port: 7070 jmx_remote: true jmx_user: "%{lookup('cassandra::jmx::user')}" jmx_password: "%{lookup('cassandra::jmx::password')}" + heap: "%{lookup('cassandra::heap')}" cassandra::jmx::user: cassandra +cassandra::heap: 16G + # cassandra::jmx::password in private data # Reflect the base of the cassandra.yaml content # These list is completed and/or overridden in cassandra::instance cassandra::base_instance_configuration: num_tokens: 16 allocate_tokens_for_local_replication_factor: 3 disk_optimization_strategy: ssd concurrent_compactors: 4 # should be min(nb core, nb disks) internode_compression: dc # default dc possible all|dc|none concurrent_reads: 64 # 16 x number of drives concurrent_writes: 128 # 8 x number of cores concurrent_counter_writes: 48 commitlog_sync: periodic # default periodic commitlog_sync_period_in_ms: 10000 # default 10000 commitlog_total_space_in_mb: 16384 # default 8192 commitlog_segment_size_in_mb: 256 # default 32 (due to oversize mutation on revision table) partitioner: org.apache.cassandra.dht.Murmur3Partitioner endpoint_snitch: GossipingPropertyFileSnitch enable_user_defined_functions: true # needed by swh-storage compaction_throughput_mb_per_sec: 160 read_request_timeout_in_ms: 5000 range_request_timeout_in_ms: 10000 write_request_timeout_in_ms: 2000 counter_write_request_timeout_in_ms: 5000 cas_contention_timeout_in_ms: 1000 truncate_request_timeout_in_ms: 60000 request_timeout_in_ms: 10000 slow_query_log_timeout_in_ms: 1000 diff --git a/data/deployments/production/vagrant.yaml b/data/deployments/production/vagrant.yaml index 62a69cbf..ad5ca5b0 100644 --- a/data/deployments/production/vagrant.yaml +++ b/data/deployments/production/vagrant.yaml @@ -1,4 +1,6 @@ elasticsearch::jvm_options::heap_size: 512m swh::postgresql::shared_buffers: 128MB swh::provenance::db::shared_buffers: 128MB + +cassandra::heap: 1G diff --git a/site-modules/profile/manifests/cassandra/instance.pp b/site-modules/profile/manifests/cassandra/instance.pp index b4dc99be..fe130f74 100644 --- a/site-modules/profile/manifests/cassandra/instance.pp +++ b/site-modules/profile/manifests/cassandra/instance.pp @@ -1,122 +1,124 @@ # Configure a cassandra node on a server # Several nodes can coexist on a same server # # It supposes the profile::cassandra class # was installed before this define profile::cassandra::instance ( $instance_name = $name, $config = {} ) { $service_name = "cassandra@${instance_name}.service" $listen_network = lookup('internal_network') $listen_address = ip_for_network($listen_network) $cassandra_base_data_dir = lookup('cassandra::base_data_directory') $instance_base_data_dir = "${cassandra_base_data_dir}/${instance_name}" $cassandra_config_dir = lookup('cassandra::base_config_directory') $cassandra_log_dir = lookup('cassandra::base_log_directory') $base_data_dir = "${instance_base_data_dir}/data" $commitlog_dir = "${instance_base_data_dir}/commitlog" $data_dir = "${base_data_dir}/data" $hints_dir = "${data_dir}/hints" $saved_caches_dir = "${data_dir}/saved_caches" $config_dir = "${cassandra_config_dir}/${instance_name}" $log_dir = "${cassandra_log_dir}/${instance_name}" $jmx_exporter_path = $::profile::prometheus::jmx::jar_path $jmx_remote = $config['jmx_remote'] $jmx_port = $config['jmx_port'] + $heap = $config['heap'] + $base_configuration = lookup('cassandra::base_instance_configuration') $instance_configuration = { cluster_name => $config["cluster_name"], data_file_directories => [ $base_data_dir, ], commitlog_directory => $commitlog_dir, hints_directory => $hints_dir, saved_caches_directory => $saved_caches_dir, listen_address => $listen_address, native_transport_port => $config['native_transport_port'], storage_port => $config['storage_port'], - seed_provider => $config['seed_provider'] + seed_provider => $config['seed_provider'], } $computed_configuration = $base_configuration + $instance_configuration # jmx port is hardcoded in the cassandra-env.sh file so it needs to be overriden in the # service configuration if $jmx_remote { $extra_jmx_option = "-Dcassandra.jmx.remote.port=${jmx_port} -Dcom.sun.management.jmxremote.access.file=${cassandra_config_dir}/jmxremote.access" } else { $extra_jmx_option = "-Dcassandra.jmx.local.port=${jmx_port}" } file {[ $instance_base_data_dir, $base_data_dir, # $commitlog_dir, $config_dir, $log_dir, ] : ensure => directory, owner => $::profile::cassandra::cassandra_user, group => $::profile::cassandra::cassandra_group, require => [ # File[$::profile::cassandra::cassandra_base_data_directory], # File[$::profile::cassandra::cassandra_config_directory], # File[$::profile::cassandra::cassandra_log_directory], ] } ::systemd::dropin_file { "${service_name}.d/parameters.conf": ensure => present, unit => "cassandra@${instance_name}.service", filename => 'parameters.conf', content => template('profile/cassandra/instance-parameters.conf.erb'), } service {$service_name: enable => true, } $config_files_to_copy = [ 'jvm11-server.options', 'jvm-server.options', 'logback.xml', 'cassandra-env.sh', ] $config_files_to_copy.each | $file_name | { file { "${config_dir}/${file_name}": ensure => present, owner => 'root', group => 'root', mode => '0644', source => "/opt/cassandra/conf/${file_name}", require => [File[$config_dir]], } } file { "${config_dir}/cassandra.yaml": ensure => present, owner => 'root', group => 'root', mode => '0644', content => inline_yaml($computed_configuration), require => [File[$config_dir]], } file { "${config_dir}/cassandra-rackdc.properties": ensure => present, owner => 'root', group => 'root', mode => '0644', content => template('profile/cassandra/cassandra-rackdc.properties.erb'), require => [File[$config_dir]], } } diff --git a/site-modules/profile/templates/cassandra/instance-parameters.conf.erb b/site-modules/profile/templates/cassandra/instance-parameters.conf.erb index 0c040a7b..c6916828 100644 --- a/site-modules/profile/templates/cassandra/instance-parameters.conf.erb +++ b/site-modules/profile/templates/cassandra/instance-parameters.conf.erb @@ -1,16 +1,16 @@ # Managed by puppet (class profile::cassandra::node); Changes will be lost. [Unit] Description=Cassandra <%= @instance_name %> instance After=network.target [Service] -Environment=JVM_EXTRA_OPTS="-javaagent:<%= @jmx_exporter_path %>=<%= @config["jmx_exporter_port"] %>:/etc/cassandra/jmx_exporter.yml <%= @extra_jmx_option %>" +Environment=JVM_EXTRA_OPTS="-javaagent:<%= @jmx_exporter_path %>=<%= @config["jmx_exporter_port"] %>:/etc/cassandra/jmx_exporter.yml <%= @extra_jmx_option %> -Xmx<%= @heap %> -Xms<%= @heap %>" Environment=CASSANDRA_CONF=<%= @config_dir %> Environment=CASSANDRA_LOG_DIR=<%= @log_dir %> <%- if @jmx_remote -%> Environment=LOCAL_JMX=no <%- end -%> [Install] WantedBy=multi-user.target