diff --git a/sysadmin/grid5000/cassandra/ansible/templates/cassandra.yaml b/sysadmin/grid5000/cassandra/ansible/templates/cassandra.yaml index 9a7b1f0..0100663 100644 --- a/sysadmin/grid5000/cassandra/ansible/templates/cassandra.yaml +++ b/sysadmin/grid5000/cassandra/ansible/templates/cassandra.yaml @@ -1,43 +1,72 @@ cluster_name: swh-storage # default 'Test Cluster' num_tokens: 256 # default 256 allocate_tokens_for_local_replication_factor: 3 data_file_directories: - {{ cassandra_data_dir }} # TODO use several disks # local_system_data_file_directory: {{ cassandra_data_dir_system }} commitlog_directory: {{ cassandra_commitlogs_dir }} disk_optimization_strategy: spinning # spinning | ssd # listen_address: 0.0.0.0 # always wrong according to the documentation listen_interface: {{ cassandra_listen_interface }} # always wrong according to the documentation -concurrent_compactors: 1 # should be min(nb core, nb disks) +concurrent_compactors: 2 # should be min(nb core, nb disks) internode_compression: dc # default dc possible all|dc|none -concurrent_reads: 16 # 16 x number of drives -concurrent_writes: 32 # 8 x number of cores +concurrent_reads: 64 # 16 x number of drives +concurrent_writes: 128 # 8 x number of cores +concurrent_counter_writes: 48 commitlog_sync: periodic # default periodic commitlog_sync_period_in_ms: 10000 # default 10000 partitioner: org.apache.cassandra.dht.Murmur3Partitioner endpoint_snitch: SimpleSnitch seed_provider: - class_name: org.apache.cassandra.locator.SimpleSeedProvider parameters: # seeds is actually a comma-delimited list of addresses. # Ex: ",," - seeds: "{{ cassandra_seed_ips }}" # needed by swh-storage enable_user_defined_functions: true # TODO Test this options effects # disk_failure_policy: # cdc_enabled #end # Trying to reduce cassandra_compaction_pendingtasks -compaction_throughput_mb_per_sec: 80 +compaction_throughput_mb_per_sec: 160 + +# https://forge.softwareheritage.org/source/cassandra-replayer-deployment/browse/master/playbooks/templates/cassandra.yaml$854 +# How long the coordinator should wait for read operations to complete. +# Lowest acceptable value is 10 ms. +read_request_timeout_in_ms: 5000 +# How long the coordinator should wait for seq or index scans to complete. +# Lowest acceptable value is 10 ms. +range_request_timeout_in_ms: 10000 +# How long the coordinator should wait for writes to complete. +# Lowest acceptable value is 10 ms. +write_request_timeout_in_ms: 2000 +# How long the coordinator should wait for counter writes to complete. +# Lowest acceptable value is 10 ms. +counter_write_request_timeout_in_ms: 5000 +# How long a coordinator should continue to retry a CAS operation +# that contends with other proposals for the same row. +# Lowest acceptable value is 10 ms. +cas_contention_timeout_in_ms: 1000 +# How long the coordinator should wait for truncates to complete +# (This can be much longer, because unless auto_snapshot is disabled +# we need to flush first so we can snapshot before removing the data.) +# Lowest acceptable value is 10 ms. +truncate_request_timeout_in_ms: 60000 +# The default timeout for other, miscellaneous operations. +# Lowest acceptable value is 10 ms. +request_timeout_in_ms: 10000 + +slow_query_log_timeout_in_ms: 1000 diff --git a/sysadmin/grid5000/cassandra/ansible/templates/jvm11-server.options b/sysadmin/grid5000/cassandra/ansible/templates/jvm11-server.options index 3680e3a..16aa304 100644 --- a/sysadmin/grid5000/cassandra/ansible/templates/jvm11-server.options +++ b/sysadmin/grid5000/cassandra/ansible/templates/jvm11-server.options @@ -1,105 +1,106 @@ ########################################################################### # jvm11-server.options # # # # See jvm-server.options. This file is specific for Java 11 and newer. # ########################################################################### ################# # GC SETTINGS # ################# - +-Xmx16G +-Xms16G ### CMS Settings -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:SurvivorRatio=8 -XX:MaxTenuringThreshold=1 -XX:CMSInitiatingOccupancyFraction=75 -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSWaitDuration=10000 -XX:+CMSParallelInitialMarkEnabled -XX:+CMSEdenChunksRecordAlways ## some JVMs will fill up their heap when accessed via JMX, see CASSANDRA-6541 -XX:+CMSClassUnloadingEnabled ### G1 Settings ## Use the Hotspot garbage-first collector. #-XX:+UseG1GC #-XX:+ParallelRefProcEnabled # ## Have the JVM do less remembered set work during STW, instead ## preferring concurrent GC. Reduces p99.9 latency. #-XX:G1RSetUpdatingPauseTimePercent=5 # ## Main G1GC tunable: lowering the pause target will lower throughput and vise versa. ## 200ms is the JVM default and lowest viable setting ## 1000ms increases throughput. Keep it smaller than the timeouts in cassandra.yaml. #-XX:MaxGCPauseMillis=500 ## Optional G1 Settings # Save CPU time on large (>= 16GB) heaps by delaying region scanning # until the heap is 70% full. The default in Hotspot 8u40 is 40%. #-XX:InitiatingHeapOccupancyPercent=70 # For systems with > 8 cores, the default ParallelGCThreads is 5/8 the number of logical cores. # Otherwise equal to the number of cores when 8 or less. # Machines with > 10 cores should try setting these to <= full cores. #-XX:ParallelGCThreads=16 # By default, ConcGCThreads is 1/4 of ParallelGCThreads. # Setting both to the same value can reduce STW durations. #-XX:ConcGCThreads=16 ### JPMS -Djdk.attach.allowAttachSelf=true --add-exports java.base/jdk.internal.misc=ALL-UNNAMED --add-exports java.base/jdk.internal.ref=ALL-UNNAMED --add-exports java.base/sun.nio.ch=ALL-UNNAMED --add-exports java.management.rmi/com.sun.jmx.remote.internal.rmi=ALL-UNNAMED --add-exports java.rmi/sun.rmi.registry=ALL-UNNAMED --add-exports java.rmi/sun.rmi.server=ALL-UNNAMED --add-exports java.sql/java.sql=ALL-UNNAMED --add-opens java.base/java.lang.module=ALL-UNNAMED --add-opens java.base/jdk.internal.loader=ALL-UNNAMED --add-opens java.base/jdk.internal.ref=ALL-UNNAMED --add-opens java.base/jdk.internal.reflect=ALL-UNNAMED --add-opens java.base/jdk.internal.math=ALL-UNNAMED --add-opens java.base/jdk.internal.module=ALL-UNNAMED --add-opens java.base/jdk.internal.util.jar=ALL-UNNAMED --add-opens jdk.management/com.sun.management.internal=ALL-UNNAMED ### GC logging options -- uncomment to enable # Java 11 (and newer) GC logging options: # See description of https://bugs.openjdk.java.net/browse/JDK-8046148 for details about the syntax # The following is the equivalent to -XX:+PrintGCDetails -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=10M #-Xlog:gc=info,heap*=trace,age*=debug,safepoint=info,promotion*=trace:file=/var/log/cassandra/gc.log:time,uptime,pid,tid,level:filecount=10,filesize=10485760 # Notes for Java 8 migration: # # -XX:+PrintGCDetails maps to -Xlog:gc*:... - i.e. add a '*' after "gc" # -XX:+PrintGCDateStamps maps to decorator 'time' # # -XX:+PrintHeapAtGC maps to 'heap' with level 'trace' # -XX:+PrintTenuringDistribution maps to 'age' with level 'debug' # -XX:+PrintGCApplicationStoppedTime maps to 'safepoint' with level 'info' # -XX:+PrintPromotionFailure maps to 'promotion' with level 'trace' # -XX:PrintFLSStatistics=1 maps to 'freelist' with level 'trace' ### Netty Options # On Java >= 9 Netty requires the io.netty.tryReflectionSetAccessible system property to be set to true to enable # creation of direct buffers using Unsafe. Without it, this falls back to ByteBuffer.allocateDirect which has # inferior performance and risks exceeding MaxDirectMemory -Dio.netty.tryReflectionSetAccessible=true -javaagent:/opt/jmx_prometheus_javaagent-0.15.0.jar=7070:/opt/jmx_exporter.yml # The newline in the end of file is intentional diff --git a/sysadmin/grid5000/cassandra/ansible/templates/swhstorage/replayer.yml b/sysadmin/grid5000/cassandra/ansible/templates/swhstorage/replayer.yml index c567b43..605b8a5 100644 --- a/sysadmin/grid5000/cassandra/ansible/templates/swhstorage/replayer.yml +++ b/sysadmin/grid5000/cassandra/ansible/templates/swhstorage/replayer.yml @@ -1,24 +1,24 @@ storage: cls: cassandra args: hosts: {{ cassandra_seed_ips.split(',') }} keyspace: swh objstorage: cls: memory args: {} journal_client: cls: kafka brokers: {% for broker in journal.brokers %} - {{ broker }} {% endfor %} sasl.username: {{ journal.consumer.user }} sasl.password: {{ journal_consumer_password }} security.protocol: sasl_ssl sasl.mechanism: SCRAM-SHA-512 # The prefix must match the username group_id: {{ journal.consumer.group }}-{{ item }} - batch_size: 500 + batch_size: 250 object_types: - {{ item }}