diff --git a/sysadmin/grid5000/cassandra/monitoring/grafana/dashboards/scylla/alternator.4.4.json b/sysadmin/grid5000/cassandra/monitoring/grafana/dashboards/scylla/alternator.4.4.json new file mode 100644 index 0000000..e60fba9 --- /dev/null +++ b/sysadmin/grid5000/cassandra/monitoring/grafana/dashboards/scylla/alternator.4.4.json @@ -0,0 +1,4587 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "class": "annotation_restart", + "datasource": "Prometheus", + "enable": true, + "expr": "resets(scylla_gossip_heart_beat[1m])>0", + "hide": false, + "iconColor": "rgba(255, 96, 96, 1)", + "limit": 100, + "name": "node_restart", + "showIn": 0, + "tagKeys": "instance,dc,cluster", + "tags": [], + "titleFormat": "restart", + "type": "tags" + }, + { + "class": "annotation_stall", + "datasource": "Prometheus", + "enable": false, + "expr": "changes(scylla_stall_detector_reported[1m])>0", + "hide": false, + "iconColor": "rgba(255, 96, 96, 1)", + "limit": 100, + "name": "stall detector", + "showIn": 0, + "tagKeys": "dc,instance,shard", + "tags": [], + "titleFormat": "Stall found", + "type": "tags" + }, + { + "class": "annotation_schema_changed", + "datasource": "Prometheus", + "enable": false, + "expr": "changes(scylla_database_schema_changed[30s])>0", + "hide": false, + "iconColor": "rgba(255, 96, 96, 1)", + "limit": 100, + "name": "Schema Changed", + "showIn": 0, + "tagKeys": "instance,dc,cluster", + "tags": [], + "titleFormat": "schema changed", + "type": "tags" + } + ] + }, + "class": "dashboard", + "editable": true, + "gnetId": null, + "graphTooltip": 1, + "hideControls": true, + "id": null, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [], + "type": "dashboards" + } + ], + "originalTitle": "Scylla Cluster Metrics", + "overwrite": true, + "panels": [ + { + "class": "text_panel", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "isNew": true, + "links": [], + "mode": "html", + "options": { + "content": "
\n [[cluster]]
", + "mode": "html" + }, + "span": 12, + "title": "", + "transparent": true, + "type": "text" + }, + { + "cacheTimeout": null, + "class": "single_stat_panel", + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "Prometheus", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 6, + "w": 2, + "x": 0, + "y": 4 + }, + "id": 2, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 1, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "count(up{job=\"scylla\", cluster=~\"$cluster|$^\"})", + "intervalFactor": 1, + "legendFormat": "Total Nodes", + "refId": "A", + "step": 40 + } + ], + "thresholds": "", + "title": "Total Nodes", + "transparent": true, + "type": "singlestat", + "valueFontSize": "150%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "class": "single_stat_panel_fail", + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(250, 113, 0, 0.89)", + "rgba(255, 0, 0, 0.9)" + ], + "datasource": "Prometheus", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": false + }, + "gridPos": { + "h": 6, + "w": 2, + "x": 2, + "y": 4 + }, + "id": 3, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 1, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "count(scrape_samples_scraped{job=\"scylla\", cluster=~\"$cluster|$^\"}==0) OR vector(0)", + "intervalFactor": 1, + "legendFormat": "Unreachable", + "refId": "A", + "step": 20 + } + ], + "thresholds": "1,2", + "title": "Unreachable", + "transparent": true, + "type": "singlestat", + "valueFontSize": "150%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "class": "single_stat_panel_fail", + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(250, 113, 0, 0.89)", + "rgba(255, 0, 0, 0.9)" + ], + "datasource": "Prometheus", + "description": "Number of nodes that reported their status as Starting or Joining", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": false + }, + "gridPos": { + "h": 6, + "w": 2, + "x": 4, + "y": 4 + }, + "id": 4, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 1, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "count(scylla_node_operation_mode<=2)OR vector(0)", + "intervalFactor": 1, + "legendFormat": "Joining", + "refId": "A", + "step": 20 + } + ], + "thresholds": "1,2", + "title": "Joining", + "transparent": true, + "type": "singlestat", + "valueFontSize": "150%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "class": "single_stat_panel_fail", + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(250, 113, 0, 0.89)", + "rgba(255, 0, 0, 0.9)" + ], + "datasource": "Prometheus", + "description": "Number of nodes that reported their status as Leaving, Decommissioned, Draining or Drained", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": false + }, + "gridPos": { + "h": 6, + "w": 2, + "x": 6, + "y": 4 + }, + "id": 5, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 1, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "count(scylla_node_operation_mode>3)OR vector(0)", + "intervalFactor": 1, + "legendFormat": "Leaving", + "refId": "A", + "step": 20 + } + ], + "thresholds": "1,2", + "title": "Leaving", + "transparent": true, + "type": "singlestat", + "valueFontSize": "150%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "class": "percent_panel", + "datasource": "Prometheus", + "description": "The percentage of the time during which Scylla utilized the CPU. Note that because Scylla does busy polling for some time before going idle, CPU utilization as seen by the operating system may be much higher. Your system is not yet CPU-bottlenecked until this metric is high.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 4 + }, + "id": 6, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 4, + "targets": [ + { + "expr": "avg(scylla_reactor_utilization{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"} ) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 4 + } + ], + "title": "Load", + "type": "timeseries" + }, + { + "class": "small_nodes_table", + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": null, + "filterable": true + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 85 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Value #B" + }, + "properties": [ + { + "id": "custom.displayMode", + "value": "lcd-gauge" + }, + { + "id": "min", + "value": 0 + }, + { + "id": "max", + "value": 101 + }, + { + "id": "displayName", + "value": "Load" + }, + { + "id": "custom.width", + "value": 120 + }, + { + "id": "custom.filterable", + "value": false + }, + { + "id": "decimals", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "svr" + }, + "properties": [ + { + "id": "custom.width", + "value": 85 + }, + { + "id": "displayName", + "value": "Version" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "instance" + }, + "properties": [ + { + "id": "custom.width", + "value": 105 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #A" + }, + "properties": [ + { + "id": "custom.width", + "value": 105 + }, + { + "id": "displayName", + "value": "Status" + }, + { + "id": "custom.displayMode", + "value": "color-background" + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "yellow", + "value": null + }, + { + "color": "rgba(0, 0, 0, 0)", + "value": 3 + }, + { + "color": "red", + "value": 4 + } + ] + } + }, + { + "id": "custom.align", + "value": "center" + }, + { + "id": "mappings", + "value": [ + { + "from": "", + "id": 1, + "text": "Starting", + "to": "", + "type": 1, + "value": "1" + }, + { + "from": "", + "id": 2, + "text": "Joining", + "to": "", + "type": 1, + "value": "2" + }, + { + "from": "", + "id": 3, + "text": "Normal", + "to": "", + "type": 1, + "value": "3" + }, + { + "from": "", + "id": 4, + "text": "Leaving", + "to": "", + "type": 1, + "value": "4" + }, + { + "from": "", + "id": 5, + "text": "Decommissioned", + "to": "", + "type": 1, + "value": "5" + }, + { + "from": "", + "id": 6, + "text": "Draining", + "to": "", + "type": 1, + "value": "6" + }, + { + "from": "", + "id": 7, + "text": "Drained", + "to": "", + "type": 1, + "value": "7" + }, + { + "from": "", + "id": 8, + "text": "Moving", + "to": "", + "type": 1, + "value": "8" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #E" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "CQL Information Dashboard. Warning indicates that there are potential issues in the CQL Optimization", + "url": "./d/cql-[[dash_version]]/scylla-cql?refresh=30s&orgId=1&var-by=instance&var-node=${__data.fields[0]}" + } + ] + }, + { + "id": "custom.width", + "value": 90 + }, + { + "id": "custom.filterable", + "value": false + }, + { + "id": "mappings", + "value": [ + { + "from": "", + "id": 1, + "text": "CQL Info", + "to": "", + "type": 1, + "value": "0" + }, + { + "from": "1", + "id": 1, + "text": "CQL Warnings", + "to": "20", + "type": 2, + "value": "" + } + ] + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(0, 0, 0,0)", + "value": null + }, + { + "color": "dark-orange", + "value": 1 + } + ] + } + }, + { + "id": "custom.displayMode", + "value": "color-background" + }, + { + "id": "custom.align", + "value": "left" + }, + { + "id": "displayName", + "value": "CQL" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #C" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "OS Information Dashboard, an Error indicates there are OS related errors", + "url": "./d/OS-[[dash_version]]/OS-metrics?refresh=30s&orgId=1&var-by=instance&var-node=${__data.fields[0]}" + } + ] + }, + { + "id": "custom.width", + "value": 90 + }, + { + "id": "custom.filterable", + "value": true + }, + { + "id": "mappings", + "value": [ + { + "from": "", + "id": 1, + "text": "OS Info", + "to": "", + "type": 1, + "value": "0" + }, + { + "from": "0", + "id": 2, + "text": "OS Errors", + "to": "100000", + "type": 2, + "value": "0" + } + ] + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(0, 0, 0,0)", + "value": null + }, + { + "color": "dark-orange", + "value": 0.001 + } + ] + } + }, + { + "id": "custom.displayMode", + "value": "color-background" + }, + { + "id": "custom.align", + "value": "left" + }, + { + "id": "displayName", + "value": "OS" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #D" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "Cordinator and Replica node errors", + "url": "./d/detailed-[[dash_version]]/Detailed?refresh=30s&orgId=1&var-by=instance&var-node=${__data.fields[0]}" + } + ] + }, + { + "id": "custom.width", + "value": 80 + }, + { + "id": "custom.filterable", + "value": true + }, + { + "id": "mappings", + "value": [ + { + "from": "", + "id": 1, + "text": "", + "to": "", + "type": 1, + "value": "0" + }, + { + "from": "", + "id": 2, + "text": "Errors", + "to": "", + "type": 1, + "value": "1" + } + ] + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(0, 0, 0,0)", + "value": null + }, + { + "color": "dark-orange", + "value": 0.001 + } + ] + } + }, + { + "id": "custom.displayMode", + "value": "color-background" + }, + { + "id": "custom.align", + "value": "left" + }, + { + "id": "displayName", + "value": " " + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "instance" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "Detailed view", + "url": "./d/detailed-[[dash_version]]/Detailed?refresh=30s&orgId=1&var-by=instance&var-node=${__data.fields[0]}" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 4 + }, + "id": 7, + "scopedVars": { + "dc": { + "selected": false, + "text": "datacenter1", + "value": "datacenter1" + } + }, + "span": 4, + "targets": [ + { + "expr": "0*scylla_scylladb_current_version{cluster=~\"$cluster|$^\", dc=~\"$dc\"} + on (instance) group_left() scylla_node_operation_mode{cluster=~\"$cluster|$^\", dc=~\"$dc\"}", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + }, + { + "expr": "avg(scylla_reactor_utilization{cluster=~\"$cluster\", dc=~\"$dc\"} ) by (instance)", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "B" + }, + { + "expr": "sum(rate(scylla_reactor_aio_errors{cluster=~\"$cluster\", dc=~\"$dc\"}[1m])) by (instance)", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "C" + }, + { + "expr": "sum(errors:nodes_total{cluster=~\"$cluster\", dc=~\"$dc\"}) by (instance) >bool 0", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "D" + }, + { + "expr": "(sum(cql:non_system_prepared1m{cluster=~\"$cluster\", dc=~\"$dc\"}) by (instance) >bool 1) + (sum(rate(scylla_cql_reverse_queries{cluster=~\"$cluster\", dc=~\"$dc\"}[60s])) by(instance) >bool 1) + (sum(cql:non_paged_no_system1m{cluster=~\"$cluster\", dc=~\"$dc\"}) by (instance) >bool 1)", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "E" + } + ], + "title": "Nodes", + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "instance", + "svr", + "Value #A", + "Value #B", + "Value #C", + "Value #D" + ] + } + } + }, + { + "id": "seriesToColumns", + "options": { + "byField": "instance" + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": { + "Value #A": 4, + "Value #B": 5, + "Value #C": 2, + "Value #D": 1, + "instance": 0, + "svr": 3 + }, + "renameByName": {} + } + } + ], + "type": "table" + }, + { + "class": "bytes_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 10 + }, + "id": 8, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 4, + "targets": [ + { + "expr": "sum(node_filesystem_size_bytes{mountpoint=\"$mount_point\", instance=~\"$node\"}) by ([[by]])-sum(node_filesystem_avail_bytes{mountpoint=\"$mount_point\", instance=~\"$node\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + }, + { + "expr": "sum(node_filesystem_size{mountpoint=\"$mount_point\", instance=~\"$node\"}) by ([[by]])-sum(node_filesystem_avail{mountpoint=\"$mount_point\", instance=~\"$node\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "B", + "step": 1 + } + ], + "title": "Disk Size by $by", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "Prometheus", + "description": "Number of Alternator Actions", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 10 + }, + "id": 9, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 4, + "targets": [ + { + "expr": "$func(rate(scylla_alternator_total_operations{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 4 + } + ], + "title": "Total Actions", + "type": "timeseries" + }, + { + "class": "alert_table", + "columns": [], + "datasource": "alertmanager", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": null, + "filterable": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute" + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.width", + "value": 150 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "instance" + }, + "properties": [ + { + "id": "custom.width", + "value": 100 + } + ] + } + ] + }, + "fontSize": "100%", + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 10 + }, + "id": 10, + "links": [], + "options": { + "showHeader": true + }, + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "span": 4, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "link": true, + "linkTooltip": "Jump to the see the node", + "linkUrl": "/d/detailed-[[dash_version]]/detailed?refresh=30s&orgId=1&var-by=instance&var-node=${__cell_4}&from=${__cell_0}", + "pattern": "Time", + "type": "date" + }, + { + "class": "hidden_column", + "pattern": "severity", + "type": "hidden" + }, + { + "class": "hidden_column", + "pattern": "alertname", + "type": "hidden" + }, + { + "class": "hidden_column", + "pattern": "cluster", + "type": "hidden" + }, + { + "class": "hidden_column", + "pattern": "monitor", + "type": "hidden" + }, + { + "class": "hidden_column", + "pattern": "summary", + "type": "hidden" + }, + { + "alias": "Instance", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "link": true, + "linkTooltip": "Jump to the see the node", + "linkUrl": "/d/detailed-[[dash_version]]/detailed?refresh=30s&orgId=1&var-by=instance&var-node=${__cell}", + "mappingType": 1, + "pattern": "instance", + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "active": true, + "annotations": true, + "filters": "job!=\"scylla_manager\",advisor=\"\"", + "legendFormat": "{{description}}", + "refId": "A", + "target": "Query" + } + ], + "title": "Active Alerts", + "transform": "table", + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "Time", + "summary", + "instance" + ] + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": { + "Time": 0, + "instance": 1, + "summary": 2 + }, + "renameByName": {} + } + } + ], + "type": "table" + }, + { + "class": "collapsible_row_panel", + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 11, + "panels": [], + "repeat": "", + "title": "Data Plane Actions", + "type": "row" + }, + { + "class": "plain_text", + "content": "

Data Plane Actions

", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 12, + "isNew": true, + "links": [], + "mode": "html", + "options": {}, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "ops_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 19 + }, + "id": 13, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 4, + "targets": [ + { + "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"GetItem\"}[60s])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "GetItem by [[by]]", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 19 + }, + "id": 14, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 4, + "targets": [ + { + "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"PutItem\"}[60s])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "PutItem by [[by]]", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 19 + }, + "id": 15, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 4, + "targets": [ + { + "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"UpdateItem\"}[60s])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "UpdateItem by [[by]]", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 25 + }, + "id": 16, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 4, + "targets": [ + { + "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"DeleteItem\"}[60s])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "DeleteItem by [[by]]", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 25 + }, + "id": 17, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 4, + "targets": [ + { + "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"BatchWriteItem\"}[60s])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "BatchWriteItem by [[by]]", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 25 + }, + "id": 18, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 4, + "targets": [ + { + "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"Query\"}[60s])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Query by [[by]]", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 31 + }, + "id": 19, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 4, + "targets": [ + { + "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"Scan\"}[60s])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Scan by [[by]]", + "type": "timeseries" + }, + { + "class": "plain_text", + "content": "

Data Plane Latencies

", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 37 + }, + "id": 20, + "isNew": true, + "links": [], + "mode": "html", + "options": {}, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 39 + }, + "id": 21, + "panels": [], + "repeat": "alternator_latency_ops", + "title": "$alternator_latency_ops", + "type": "row" + }, + { + "class": "ops_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 40 + }, + "id": 22, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_alternator_op_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"$alternator_latency_ops\"}[60s])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Completed $alternator_latency_ops", + "type": "timeseries" + }, + { + "class": "us_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 40 + }, + "id": 23, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_alternator_op_latency_sum{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"$alternator_latency_ops\"}[60s])) by ([[by]])/($func(rate(scylla_alternator_op_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"$alternator_latency_ops\"}[60s])) by ([[by]]) + 1)", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Average $alternator_latency_ops latency by [[by]]", + "type": "timeseries" + }, + { + "class": "us_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 40 + }, + "id": 24, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum(rate(scylla_alternator_op_latency_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"$alternator_latency_ops\"}[60s])) by ([[by]], le))", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "95th percentile $alternator_latency_ops latency by [[by]]", + "type": "timeseries" + }, + { + "class": "us_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 40 + }, + "id": 25, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(scylla_alternator_op_latency_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"$alternator_latency_ops\"}[60s])) by ([[by]], le))", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "99th percentile $alternator_latency_ops latency by [[by]]", + "type": "timeseries" + }, + { + "class": "collapsible_row_panel", + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 46 + }, + "id": 26, + "panels": [], + "repeat": "", + "title": "Streams", + "type": "row" + }, + { + "class": "plain_text", + "content": "

Streams Actions

", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 47 + }, + "id": 27, + "isNew": true, + "links": [], + "mode": "html", + "options": {}, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "ops_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 49 + }, + "id": 28, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 4, + "targets": [ + { + "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"ListStreams\"}[60s])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "ListStreams by [[by]]", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 49 + }, + "id": 29, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 4, + "targets": [ + { + "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"DescribeStream\"}[60s])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "DescribeStream by [[by]]", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 49 + }, + "id": 30, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 4, + "targets": [ + { + "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"GetShardIterator\"}[60s])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "GetShardIterator by [[by]]", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 55 + }, + "id": 31, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 4, + "targets": [ + { + "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"GetRecords\"}[60s])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "GetRecords by [[by]]", + "type": "timeseries" + }, + { + "class": "plain_text", + "content": "

Streams Latencies

", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 61 + }, + "id": 32, + "isNew": true, + "links": [], + "mode": "html", + "options": {}, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 63 + }, + "id": 33, + "panels": [], + "repeat": "alternator_streams_latency_ops", + "title": "$alternator_streams_latency_ops", + "type": "row" + }, + { + "class": "ops_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 64 + }, + "id": 34, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_alternator_op_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"$alternator_streams_latency_ops\"}[60s])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Completed $alternator_streams_latency_ops", + "type": "timeseries" + }, + { + "class": "us_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 64 + }, + "id": 35, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_alternator_op_latency_sum{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"$alternator_streams_latency_ops\"}[60s])) by ([[by]])/($func(rate(scylla_alternator_op_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"$alternator_streams_latency_ops\"}[60s])) by ([[by]]) + 1)", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Average $alternator_streams_latency_ops latency by [[by]]", + "type": "timeseries" + }, + { + "class": "us_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 64 + }, + "id": 36, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "histogram_quantile(0.95, sum(rate(scylla_alternator_op_latency_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"$alternator_streams_latency_ops\"}[60s])) by ([[by]], le))", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "95th percentile $alternator_streams_latency_ops latency by [[by]]", + "type": "timeseries" + }, + { + "class": "us_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 64 + }, + "id": 37, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(scylla_alternator_op_latency_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"$alternator_streams_latency_ops\"}[60s])) by ([[by]], le))", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "99th percentile $alternator_streams_latency_ops latency by [[by]]", + "type": "timeseries" + }, + { + "class": "collapsible_row_panel", + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 70 + }, + "id": 38, + "panels": [], + "repeat": "", + "title": "Control plane", + "type": "row" + }, + { + "class": "plain_text", + "content": "

Control Plane Actions

", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 71 + }, + "id": 39, + "isNew": true, + "links": [], + "mode": "html", + "options": {}, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "ops_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 73 + }, + "id": 40, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 4, + "targets": [ + { + "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"CreateTable\"}[60s])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "CreateTable by [[by]]", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 73 + }, + "id": 41, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 4, + "targets": [ + { + "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"DeleteTable\"}[60s])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "DeleteTable by [[by]]", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 73 + }, + "id": 42, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 4, + "targets": [ + { + "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"DescribeTable\"}[60s])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "DescribeTable by [[by]]", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 79 + }, + "id": 43, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 4, + "targets": [ + { + "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"ListTables\"}[60s])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "ListTables by [[by]]", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 79 + }, + "id": 44, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 4, + "targets": [ + { + "expr": "$func(rate(scylla_alternator_operation{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", op=\"DescribeEndpoints\"}[60s])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "DescribeEndpoints by [[by]]", + "type": "timeseries" + }, + { + "class": "text_panel", + "content": "## ", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 79 + }, + "id": 45, + "isNew": true, + "links": [], + "mode": "markdown", + "options": {}, + "span": 4, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "plain_text", + "content": "

Cache

", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 12, + "x": 0, + "y": 85 + }, + "id": 46, + "isNew": true, + "links": [], + "mode": "html", + "options": {}, + "span": 6, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "plain_text", + "content": "

Timeouts

", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 12, + "x": 12, + "y": 85 + }, + "id": 47, + "isNew": true, + "links": [], + "mode": "html", + "options": {}, + "span": 6, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "ops_panel", + "datasource": "Prometheus", + "description": "Number of rows that were read from the cache, without needing to be fetched from storage.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 87 + }, + "id": 48, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_cache_row_hits{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "Cache Hits", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "Prometheus", + "description": "Number of rows that were not present in the cache, and had to be fetched from storage.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 87 + }, + "id": 49, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_cache_row_misses{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "Cache Misses", + "type": "timeseries" + }, + { + "class": "wpm_panel", + "datasource": "Prometheus", + "description": "Requests that Scylla tried to write but timed out. Timeouts are counted in the node that received the request (the coordinator), not at the replicas.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:writes/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 87 + }, + "id": 50, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(delta(scylla_storage_proxy_coordinator_write_timeouts{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "Write Timeouts/Minutes by [[by]]", + "type": "timeseries" + }, + { + "class": "rpm_panel", + "datasource": "Prometheus", + "description": "Requests that Scylla tried to read but timed out. Timeouts are counted in the node that received the request (the coordinator), not at the replicas.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:reads/m" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 87 + }, + "id": 51, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(delta(scylla_storage_proxy_coordinator_read_timeouts{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "Read Timeouts/Minutes by [[by]]", + "type": "timeseries" + }, + { + "class": "plain_text", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 93 + }, + "id": 52, + "isNew": true, + "links": [], + "mode": "html", + "options": { + "content": "

Your Panels

", + "mode": "html" + }, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "user_panel", + "datasource": "Prometheus", + "description": "This graph panel was left empty on purpose for ad-hoc usage. Change it when needed. Pay attention that changes to the panel will not be saved.\n\nIf you do need a panel that can be saved, create a new dashboard, or edit the panel inside the json file", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 95 + }, + "id": 53, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 6, + "title": "Your Graph here", + "type": "timeseries" + }, + { + "class": "user_panel", + "datasource": "Prometheus", + "description": "This graph panel was left empty on purpose for ad-hoc usage. Change it when needed. Pay attention that changes to the panel will not be saved.\n\nIf you do need a panel that can be saved, create a new dashboard, or edit the panel inside the json file", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 95 + }, + "id": 54, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 6, + "title": "Your Graph here", + "type": "timeseries" + }, + { + "class": "plain_text", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 101 + }, + "id": 55, + "isNew": true, + "links": [], + "mode": "html", + "options": { + "content": "
Scylla Monitoring version - master
\n    
", + "mode": "html" + }, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + } + ], + "refresh": "30s", + "schemaVersion": 26, + "style": "dark", + "tags": [ + "4.4" + ], + "templating": { + "list": [ + { + "allValue": null, + "class": "by_template_var", + "current": { + "tags": [], + "text": "Instance", + "value": "instance" + }, + "error": null, + "hide": 0, + "includeAll": false, + "label": "by", + "multi": false, + "name": "by", + "options": [ + { + "selected": false, + "text": "Cluster", + "value": "cluster" + }, + { + "selected": false, + "text": "DC", + "value": "dc" + }, + { + "selected": true, + "text": "Instance", + "value": "instance" + }, + { + "selected": false, + "text": "instance,shard", + "value": "instance,shard" + } + ], + "query": "Cluster,DC,Instance,Shard", + "skipUrlSync": false, + "type": "custom" + }, + { + "allValue": null, + "class": "template_variable_single", + "current": { + "isNone": true, + "selected": false, + "text": "None", + "value": "" + }, + "datasource": "Prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [], + "query": "label_values(scylla_reactor_utilization, cluster)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "class": "template_variable_all", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "Prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": true, + "label": "dc", + "multi": true, + "name": "dc", + "options": [], + "query": "label_values(scylla_reactor_utilization{cluster=~\"$cluster\"}, dc)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "class": "template_variable_all", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "Prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": true, + "label": "node", + "multi": true, + "name": "node", + "options": [], + "query": "label_values(scylla_reactor_utilization{cluster=~\"$cluster|$^\", dc=~\"$dc\"}, instance)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "class": "template_variable_all", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "Prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": true, + "label": "shard", + "multi": true, + "name": "shard", + "options": [], + "query": "label_values(scylla_reactor_utilization,shard)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 3, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "class": "template_variable_single", + "current": { + "text": "/var/lib/scylla", + "value": "/var/lib/scylla" + }, + "datasource": "Prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": false, + "label": "Mount path", + "multi": false, + "name": "mount_point", + "options": [], + "query": "node_filesystem_avail_bytes", + "refresh": 2, + "regex": "/mountpoint=\"([^\"]*)\".*/", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "class": "aggregation_function", + "current": { + "tags": [], + "text": "sum", + "value": "sum" + }, + "hide": 0, + "includeAll": false, + "label": "Function", + "multi": false, + "name": "func", + "options": [ + { + "selected": true, + "text": "sum", + "value": "sum" + }, + { + "selected": false, + "text": "avg", + "value": "avg" + }, + { + "selected": false, + "text": "max", + "value": "max" + }, + { + "selected": false, + "text": "min", + "value": "min" + }, + { + "selected": false, + "text": "stddev", + "value": "stddev" + }, + { + "selected": false, + "text": "stdvar", + "value": "stdvar" + } + ], + "query": "sum,avg,max,min,stddev,stdvar", + "skipUrlSync": false, + "type": "custom" + }, + { + "allValue": null, + "class": "template_variable_custom", + "current": { + "text": "4-4", + "value": "4-4" + }, + "error": null, + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "dash_version", + "options": [ + { + "selected": true, + "text": "4-4", + "value": "4-4" + } + ], + "query": "4-4", + "skipUrlSync": false, + "type": "custom" + }, + { + "allValue": null, + "class": "template_variable_custom", + "current": { + "text": "4.4", + "value": "4.4" + }, + "error": null, + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "scylla_version", + "options": [ + { + "selected": true, + "text": "4.4", + "value": "4.4" + } + ], + "query": "4.4", + "skipUrlSync": false, + "type": "custom" + }, + { + "allValue": null, + "class": "monitor_version_var", + "current": { + "text": "master", + "value": "master" + }, + "error": null, + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "monitoring_version", + "options": [ + { + "selected": true, + "text": "master", + "value": "master" + } + ], + "query": "master", + "skipUrlSync": false, + "type": "custom" + }, + { + "allValue": null, + "class": "template_variable_custom", + "current": { + "text": "All", + "value": "$__all" + }, + "error": null, + "hide": 2, + "includeAll": true, + "label": null, + "multi": true, + "name": "alternator_latency_ops", + "options": [ + { + "selected": true, + "text": "All", + "value": "$__all" + }, + { + "selected": false, + "text": "GetItem", + "value": "GetItem" + }, + { + "selected": false, + "text": "PutItem", + "value": "PutItem" + }, + { + "selected": false, + "text": "UpdateItem", + "value": "UpdateItem" + }, + { + "selected": false, + "text": "DeleteItem", + "value": "DeleteItem" + } + ], + "query": "GetItem,PutItem,UpdateItem,DeleteItem", + "skipUrlSync": false, + "type": "custom" + }, + { + "allValue": null, + "class": "template_variable_custom", + "current": { + "text": "All", + "value": "$__all" + }, + "error": null, + "hide": 2, + "includeAll": true, + "label": null, + "multi": true, + "name": "alternator_streams_latency_ops", + "options": [ + { + "selected": true, + "text": "All", + "value": "$__all" + }, + { + "selected": false, + "text": "GetRecords", + "value": "GetRecords" + } + ], + "query": "GetRecords", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": { + "now": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "utc", + "title": "Alternator", + "uid": "alternator-4-4", + "version": 1 +} diff --git a/sysadmin/grid5000/cassandra/monitoring/grafana/dashboards/scylla/scylla-advanced.4.4.json b/sysadmin/grid5000/cassandra/monitoring/grafana/dashboards/scylla/scylla-advanced.4.4.json new file mode 100644 index 0000000..e23a7c1 --- /dev/null +++ b/sysadmin/grid5000/cassandra/monitoring/grafana/dashboards/scylla/scylla-advanced.4.4.json @@ -0,0 +1,2851 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "class": "annotation_restart", + "datasource": "Prometheus", + "enable": true, + "expr": "resets(scylla_gossip_heart_beat[1m])>0", + "hide": false, + "iconColor": "rgba(255, 96, 96, 1)", + "limit": 100, + "name": "node_restart", + "showIn": 0, + "tagKeys": "instance,dc,cluster", + "tags": [], + "titleFormat": "restart", + "type": "tags" + }, + { + "class": "annotation_stall", + "datasource": "Prometheus", + "enable": false, + "expr": "changes(scylla_stall_detector_reported[1m])>0", + "hide": false, + "iconColor": "rgba(255, 96, 96, 1)", + "limit": 100, + "name": "stall detector", + "showIn": 0, + "tagKeys": "dc,instance,shard", + "tags": [], + "titleFormat": "Stall found", + "type": "tags" + }, + { + "class": "annotation_schema_changed", + "datasource": "Prometheus", + "enable": false, + "expr": "changes(scylla_database_schema_changed[30s])>0", + "hide": false, + "iconColor": "rgba(255, 96, 96, 1)", + "limit": 100, + "name": "Schema Changed", + "showIn": 0, + "tagKeys": "instance,dc,cluster", + "tags": [], + "titleFormat": "schema changed", + "type": "tags" + } + ] + }, + "class": "dashboard", + "editable": true, + "gnetId": null, + "graphTooltip": 1, + "hideControls": true, + "id": null, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [], + "type": "dashboards" + } + ], + "overwrite": true, + "panels": [ + { + "class": "text_panel", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 1, + "w": 9, + "x": 0, + "y": 0 + }, + "id": 1, + "isNew": true, + "links": [], + "mode": "html", + "options": { + "content": "# " + }, + "span": 12, + "title": "", + "transparent": false, + "type": "text" + }, + { + "class": "text_panel", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 1, + "w": 4, + "x": 9, + "y": 0 + }, + "id": 2, + "isNew": true, + "links": [], + "mode": "html", + "options": { + "content": "# " + }, + "span": 12, + "title": "", + "transparent": false, + "type": "text" + }, + { + "class": "text_panel", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 1, + "w": 4, + "x": 13, + "y": 0 + }, + "id": 3, + "isNew": true, + "links": [], + "mode": "html", + "options": { + "content": "# " + }, + "span": 12, + "title": "", + "transparent": false, + "type": "text" + }, + { + "class": "text_panel", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 1, + "w": 7, + "x": 17, + "y": 0 + }, + "id": 4, + "isNew": true, + "links": [], + "mode": "html", + "options": { + "content": "# " + }, + "span": 12, + "title": "", + "transparent": false, + "type": "text" + }, + { + "class": "small_stat", + "datasource": "Prometheus", + "description": "The number of nodes configured in the cluster.", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 0, + "y": 1 + }, + "id": 5, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "count(up{job=\"scylla\", cluster=~\"$cluster|$^\"})", + "intervalFactor": 1, + "legendFormat": "Total Nodes", + "refId": "A", + "step": 40 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "# Nodes", + "type": "stat" + }, + { + "class": "small_stat", + "datasource": "Prometheus", + "description": "The number of unreachable nodes.\nUsually because a machine is down or unreachable.", + "fieldConfig": { + "defaults": { + "custom": {}, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 1 + } + ] + } + } + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 2, + "y": 1 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "(count(scrape_samples_scraped{job=\"scylla\", cluster=~\"$cluster|$^\"}==0) OR vector(0))", + "intervalFactor": 1, + "legendFormat": "Offline ", + "refId": "A", + "step": 20 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Unreachable", + "type": "stat" + }, + { + "class": "small_stat", + "datasource": "Prometheus", + "description": "The number of joining and leaving nodes.\nThe number of nodes that are up but not actively part of the cluster, either because they are still joining or because they are leaving.", + "fieldConfig": { + "defaults": { + "custom": {}, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 1 + } + ] + } + } + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 4, + "y": 1 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "count(scylla_node_operation_mode!=3)OR vector(0)", + "intervalFactor": 1, + "legendFormat": "Offline ", + "refId": "A", + "step": 20 + } + ], + "thresholds": "1,2", + "timeFrom": null, + "timeShift": null, + "title": "Inactive", + "type": "stat" + }, + { + "class": "small_stat", + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "mappings": [ + { + "from": "", + "id": 0, + "text": "Backup", + "to": "", + "type": 1, + "value": "2" + }, + { + "from": "", + "id": 1, + "text": "Repair", + "to": "", + "type": 1, + "value": "1" + }, + { + "from": "", + "id": 2, + "text": "Online", + "to": "", + "type": 1, + "value": "0" + }, + { + "from": "", + "id": 2, + "text": "Offline", + "to": "", + "type": 1, + "value": "-1" + } + ], + "noValue": " Offline", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 0 + } + ] + } + } + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 6, + "y": 1 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "(sum(scylla_manager_task_active_count{type=~\"repair\",cluster=~\"$cluster|$^\"}) or on() vector(0)) + (sum(scylla_manager_task_active_count{type=~\"backup\",cluster=~\"$cluster|$^\"})*2 or on() vector(0)) + (sum(scylla_manager_server_current_version{}) or on() vector(-1))", + "intervalFactor": 1, + "refId": "A", + "step": 40 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Manager", + "type": "stat" + }, + { + "class": "vertical_lcd", + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 1, + "x": 8, + "y": 1 + }, + "id": 9, + "options": { + "displayMode": "lcd", + "orientation": "vertical", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "(sum(avg(scylla_manager_repair_progress{cluster=~\"$cluster|$^\"}) by (task) * sum(scylla_manager_task_active_count{type=\"repair\",cluster=~\"$cluster|$^\"}) by (task)) or on () vector(0)) + (sum(avg(scylla_manager_backup_percent_progress{cluster=~\"$cluster|$^\"}) by (task) * sum(scylla_manager_task_active_count{type=\"backup\",cluster=~\"$cluster|$^\"}) by (task)) or on () vector(0))", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "", + "type": "bargauge" + }, + { + "class": "small_stat", + "datasource": "Prometheus", + "description": "Average Write Latency", + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 50000 + } + ] + }, + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 9, + "y": 1 + }, + "id": 10, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "wlatencya{by=\"cluster\", cluster=~\"$cluster|^$\",scheduling_group_name=~\"$scheduling_group\"} or on() (sum(rate(scylla_storage_proxy_coordinator_write_latency_sum{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[60s]))/(sum(rate(scylla_storage_proxy_coordinator_write_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[60s])) + 1))", + "instant": true, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 4 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Avg Write", + "type": "stat" + }, + { + "class": "small_stat", + "datasource": "Prometheus", + "description": "99% write Latency", + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 100000 + } + ] + }, + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 11, + "y": 1 + }, + "id": 11, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "wlatencyp99{by=\"cluster\", cluster=~\"$cluster|^$\",scheduling_group_name=~\"$scheduling_group\"} or on() (histogram_quantile(0.99, sum(rate(scylla_storage_proxy_coordinator_write_latency_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$scheduling_group\"}[60s])) by (le)))", + "instant": true, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 4 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "99% Write", + "type": "stat" + }, + { + "class": "small_stat", + "datasource": "Prometheus", + "description": "Average Read Latency", + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 50000 + } + ] + }, + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 13, + "y": 1 + }, + "id": 12, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "rlatencya{by=\"cluster\", cluster=~\"$cluster|^$\",scheduling_group_name=~\"$scheduling_group\"} or on() (sum(rate(scylla_storage_proxy_coordinator_read_latency_sum{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[60s]))/(sum(rate(scylla_storage_proxy_coordinator_read_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[60s])) + 1))", + "instant": true, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 4 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Avg Read", + "type": "stat" + }, + { + "class": "small_stat", + "datasource": "Prometheus", + "description": "99% read Latency", + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 100000 + } + ] + }, + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 15, + "y": 1 + }, + "id": 13, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "rlatencyp99{by=\"cluster\", cluster=~\"$cluster|^$\",scheduling_group_name=~\"$scheduling_group\"} or on() (histogram_quantile(0.99, sum(rate(scylla_storage_proxy_coordinator_read_latency_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$scheduling_group\"}[60s])) by (le)))", + "instant": true, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 4 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "99% Read", + "type": "stat" + }, + { + "class": "small_stat", + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "si:" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 17, + "y": 1 + }, + "id": 14, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "sum(rate(scylla_transport_requests_served{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) + (sum(rate(scylla_thrift_served{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) or on() vector(0))", + "instant": true, + "intervalFactor": 1, + "refId": "A", + "step": 40 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Requests/s", + "type": "stat" + }, + { + "class": "small_stat", + "datasource": "Prometheus", + "description": "The percentage of the time during which Scylla utilized the CPU. Note that because Scylla does busy polling for some time before going idle, CPU utilization as seen by the operating system may be much higher. Your system is not yet CPU-bottlenecked until this metric is high.", + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 20, + "y": 1 + }, + "id": 15, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "avg(scylla_reactor_utilization{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"} )", + "instant": true, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 4 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Load", + "type": "stat" + }, + { + "class": "small_stat", + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 22, + "y": 1 + }, + "id": 16, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "sum(rate(scylla_storage_proxy_coordinator_write_timeouts{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m]))", + "instant": true, + "intervalFactor": 1, + "refId": "A", + "step": 40 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Timeouts", + "type": "stat" + }, + { + "class": "collapsible_row_panel", + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 17, + "panels": [], + "repeat": "", + "title": "", + "type": "row" + }, + { + "class": "plain_text", + "content": "

IO Queue Information

", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 18, + "isNew": true, + "links": [], + "mode": "html", + "options": {}, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 19, + "panels": [], + "repeat": "classes", + "title": "$classes", + "type": "row" + }, + { + "class": "us_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 9 + }, + "id": 20, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 4, + "targets": [ + { + "expr": "1000000*max(scylla_io_queue_delay{class=\"$classes\", instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"} ) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "seastar_io_queue_delay", + "refId": "A", + "step": 30 + } + ], + "title": "$classes I/O Queue delay by [[by]]", + "type": "timeseries" + }, + { + "class": "bps_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 9 + }, + "id": 21, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 4, + "targets": [ + { + "expr": "sum(rate(scylla_io_queue_total_bytes{class=\"$classes\", instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "seastar_io_queue_delay", + "refId": "A", + "step": 30 + } + ], + "title": "$classes I/O Queue bandwidth by [[by]]", + "type": "timeseries" + }, + { + "class": "iops_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "iops" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 9 + }, + "id": 22, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 4, + "targets": [ + { + "expr": "sum(rate(scylla_io_queue_total_operations{class=\"$classes\", instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "seastar_io_queue_delay", + "refId": "A", + "step": 30 + } + ], + "title": "$classes I/O Queue IOPS by [[by]]", + "type": "timeseries" + }, + { + "class": "collapsible_row_panel", + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 15 + }, + "id": 23, + "panels": [], + "repeat": "", + "title": "", + "type": "row" + }, + { + "class": "plain_text", + "content": "

Information by Task Group

", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 24, + "isNew": true, + "links": [], + "mode": "html", + "options": {}, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 25, + "panels": [], + "repeat": "group", + "title": "$group", + "type": "row" + }, + { + "class": "ms_panel", + "datasource": "Prometheus", + "description": "Scylla employs an event-loop like reactor that alternates between the execution of different groups of tasks periodically. This graph shows how much time was spent in $group group", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 19 + }, + "id": 26, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pointradius": 1, + "span": 4, + "targets": [ + { + "expr": "$func(rate(scylla_scheduler_runtime_ms{instance=~\"[[node]]\" ,cluster=~\"$cluster|$^\", dc=~\"$dc\",group=~\"$group\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "title": "Time used by [[by]] - $group", + "type": "timeseries" + }, + { + "class": "ms_panel", + "datasource": "Prometheus", + "description": "Scylla employs an event-loop like reactor that alternates between the execution of different groups of tasks periodically. The maximum amount of time during which a task group can run is called the \"task quota\". Some task groups may disrespect that and run for longer. This may cause latency issues", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 19 + }, + "id": 27, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pointradius": 1, + "span": 4, + "targets": [ + { + "expr": "$func(rate(scylla_scheduler_time_spent_on_task_quota_violations_ms{instance=~\"[[node]]\" ,cluster=~\"$cluster|$^\", dc=~\"$dc\",group=~\"$group\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "title": "Time spent in task quota violations by [[by]] - $group", + "type": "timeseries" + }, + { + "class": "graph_panel", + "datasource": "Prometheus", + "description": "Shares assigned to the $group. Shares determine how Scylla reactor distributes the task quotas between groups (Higher share gets more quotas)", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 19 + }, + "id": 28, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pointradius": 1, + "span": 4, + "targets": [ + { + "expr": "$func(scylla_scheduler_shares{instance=~\"[[node]]\" ,cluster=~\"$cluster|$^\", dc=~\"$dc\",group=~\"$group\", shard=~\"[[shard]]\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "title": "Scheduler shares [[by]] - $group", + "type": "timeseries" + }, + { + "class": "collapsible_row_panel", + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 29, + "panels": [], + "repeat": "", + "title": "Internal node errors", + "type": "row" + }, + { + "class": "plain_text", + "content": "

Internal node Errors - $cluster

", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 26 + }, + "id": 30, + "isNew": true, + "links": [], + "mode": "html", + "options": {}, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "rps_panel", + "datasource": "Prometheus", + "description": "Number of Read requests that failed due to an 'unavailable' error", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:reads/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 28 + }, + "id": 31, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 4, + "targets": [ + { + "expr": "avg(rate(scylla_storage_proxy_coordinator_read_errors_local_node{instance=~\"[[node]]\" ,cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "title": "Local Reads Error by [[by]]", + "type": "timeseries" + }, + { + "class": "wps_panel", + "datasource": "Prometheus", + "description": "Number of write requests that failed due to an 'unavailable' error", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:writes/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 28 + }, + "id": 32, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 4, + "targets": [ + { + "expr": "avg(rate(scylla_storage_proxy_coordinator_write_errors_local_node{instance=~\"[[node]]\" ,cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "title": "Local Write Error by [[by]]", + "type": "timeseries" + }, + { + "class": "text_panel", + "content": "## ", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 28 + }, + "id": 33, + "isNew": true, + "links": [], + "mode": "markdown", + "options": {}, + "span": 4, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "rps_panel", + "datasource": "Prometheus", + "description": "Number of Read requests that failed due to an 'unavailable' error", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:reads/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 34 + }, + "id": 34, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 4, + "targets": [ + { + "expr": "avg(rate(scylla_storage_proxy_coordinator_read_unavailable{instance=~\"[[node]]\" ,cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "title": "Reads Unavailable Error by [[by]]", + "type": "timeseries" + }, + { + "class": "wps_panel", + "datasource": "Prometheus", + "description": "Number of write requests that failed on a local Node", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:writes/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 34 + }, + "id": 35, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 4, + "targets": [ + { + "expr": "avg(rate(scylla_storage_proxy_coordinator_write_unavailable{instance=~\"[[node]]\" ,cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "title": "Write Unavailable Error by [[by]]", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "Prometheus", + "description": "Number of write requests that failed on a local Node", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 34 + }, + "id": 36, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 4, + "targets": [ + { + "expr": "avg(rate(scylla_storage_proxy_coordinator_range_unavailable{instance=~\"[[node]]\" ,cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "title": "Range Unavailable Error by [[by]]", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "Prometheus", + "description": "Number of AIO Errors", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 40 + }, + "id": 37, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 4, + "targets": [ + { + "expr": "avg(rate(scylla_reactor_aio_errors{instance=~\"[[node]]\" ,cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "title": "AIO Error by [[by]]", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "Prometheus", + "description": "Total number of abandoned failed futures, futures destroyed while still containing an exception.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 40 + }, + "id": 38, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 4, + "targets": [ + { + "expr": "sum(rate(scylla_reactor_abandoned_failed_futures{instance=~\"[[node]]\" ,cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "title": "Ignored Future By [[by]]", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "Prometheus", + "description": "Number of C++ exceptions thrown.\n\n An exception by itself does not indicate a problem", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 40 + }, + "id": 39, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 4, + "targets": [ + { + "expr": "sum(rate(scylla_reactor_cpp_exceptions{instance=~\"[[node]]\" ,cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "title": "C++ Exceptions [[by]]", + "type": "timeseries" + }, + { + "class": "collapsible_row_panel", + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 46 + }, + "id": 40, + "panels": [], + "repeat": "", + "title": "Your panels", + "type": "row" + }, + { + "class": "plain_text", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 47 + }, + "id": 41, + "isNew": true, + "links": [], + "mode": "html", + "options": { + "content": "

Your Panels

", + "mode": "html" + }, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "user_panel", + "datasource": "Prometheus", + "description": "This graph panel was left empty on purpose for ad-hoc usage. Change it when needed. Pay attention that changes to the panel will not be saved.\n\nIf you do need a panel that can be saved, create a new dashboard, or edit the panel inside the json file", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 49 + }, + "id": 42, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 6, + "title": "Your Graph here", + "type": "timeseries" + }, + { + "class": "user_panel", + "datasource": "Prometheus", + "description": "This graph panel was left empty on purpose for ad-hoc usage. Change it when needed. Pay attention that changes to the panel will not be saved.\n\nIf you do need a panel that can be saved, create a new dashboard, or edit the panel inside the json file", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 49 + }, + "id": 43, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 6, + "title": "Your Graph here", + "type": "timeseries" + }, + { + "class": "plain_text", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 55 + }, + "id": 44, + "isNew": true, + "links": [], + "mode": "html", + "options": { + "content": "
Scylla Monitoring version - master
\n    
", + "mode": "html" + }, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + } + ], + "refresh": "30s", + "schemaVersion": 26, + "style": "dark", + "tags": [ + "4.4" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { + "tags": [], + "text": "Instance", + "value": "instance" + }, + "hide": 0, + "includeAll": false, + "label": "by", + "multi": false, + "name": "by", + "options": [ + { + "selected": false, + "text": "Cluster", + "value": "cluster" + }, + { + "selected": false, + "text": "DC", + "value": "dc" + }, + { + "selected": true, + "text": "Instance", + "value": "instance" + }, + { + "selected": false, + "text": "Shard", + "value": "instance,shard" + } + ], + "query": "Cluster,DC,Instance,Shard", + "type": "custom" + }, + { + "allValue": null, + "class": "template_variable_single", + "current": { + "isNone": true, + "selected": false, + "text": "None", + "value": "" + }, + "datasource": "Prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [], + "query": "label_values(scylla_reactor_utilization, cluster)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "class": "template_variable_all", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "Prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": true, + "label": "dc", + "multi": true, + "name": "dc", + "options": [], + "query": "label_values(scylla_reactor_utilization{cluster=~\"$cluster\"}, dc)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "class": "template_variable_all", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "Prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": true, + "label": "node", + "multi": true, + "name": "node", + "options": [], + "query": "label_values(scylla_reactor_utilization{cluster=~\"$cluster|$^\", dc=~\"$dc\"}, instance)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "class": "template_variable_all", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "Prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": true, + "label": "shard", + "multi": true, + "name": "shard", + "options": [], + "query": "label_values(scylla_reactor_utilization,shard)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 3, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "class": "template_variable_all", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "Prometheus", + "definition": "", + "error": null, + "hide": 2, + "includeAll": true, + "label": "classes", + "multi": true, + "name": "classes", + "options": [], + "query": "label_values(scylla_io_queue_delay,class)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 3, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "class": "template_variable_all", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "Prometheus", + "definition": "", + "error": null, + "hide": 2, + "includeAll": true, + "label": "group", + "multi": true, + "name": "group", + "options": [], + "query": "label_values(scylla_scheduler_time_spent_on_task_quota_violations_ms,group)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 3, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "class": "adhoc_filter", + "datasource": "Prometheus", + "error": null, + "filters": [], + "hide": 0, + "label": "ad hoc", + "name": "adhoc", + "skipUrlSync": false, + "type": "adhoc" + }, + { + "allValue": null, + "class": "aggregation_function", + "current": { + "tags": [], + "text": "avg", + "value": "avg" + }, + "hide": 0, + "includeAll": false, + "label": "Function", + "multi": false, + "name": "func", + "options": [ + { + "selected": true, + "text": "sum", + "value": "sum" + }, + { + "selected": false, + "text": "avg", + "value": "avg" + }, + { + "selected": false, + "text": "max", + "value": "max" + }, + { + "selected": false, + "text": "min", + "value": "min" + }, + { + "selected": false, + "text": "stddev", + "value": "stddev" + }, + { + "selected": false, + "text": "stdvar", + "value": "stdvar" + } + ], + "query": "sum,avg,max,min,stddev,stdvar", + "skipUrlSync": false, + "type": "custom" + }, + { + "allValue": null, + "class": "template_variable_custom", + "current": { + "text": "4.4", + "value": "4.4" + }, + "error": null, + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "scylla_version", + "options": [ + { + "selected": true, + "text": "4.4", + "value": "4.4" + } + ], + "query": "4.4", + "skipUrlSync": false, + "type": "custom" + }, + { + "allValue": null, + "class": "monitor_version_var", + "current": { + "text": "master", + "value": "master" + }, + "error": null, + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "monitoring_version", + "options": [ + { + "selected": true, + "text": "master", + "value": "master" + } + ], + "query": "master", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": { + "now": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "utc", + "title": "Advanced", + "uid": "advanced-4-4", + "version": 5 +} diff --git a/sysadmin/grid5000/cassandra/monitoring/grafana/dashboards/scylla/scylla-cql.4.4.json b/sysadmin/grid5000/cassandra/monitoring/grafana/dashboards/scylla/scylla-cql.4.4.json new file mode 100644 index 0000000..cd9ef22 --- /dev/null +++ b/sysadmin/grid5000/cassandra/monitoring/grafana/dashboards/scylla/scylla-cql.4.4.json @@ -0,0 +1,4263 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "class": "annotation_restart", + "datasource": "Prometheus", + "enable": true, + "expr": "resets(scylla_gossip_heart_beat[1m])>0", + "hide": false, + "iconColor": "rgba(255, 96, 96, 1)", + "limit": 100, + "name": "node_restart", + "showIn": 0, + "tagKeys": "instance,dc,cluster", + "tags": [], + "titleFormat": "restart", + "type": "tags" + }, + { + "class": "annotation_stall", + "datasource": "Prometheus", + "enable": false, + "expr": "changes(scylla_stall_detector_reported[1m])>0", + "hide": false, + "iconColor": "rgba(255, 96, 96, 1)", + "limit": 100, + "name": "stall detector", + "showIn": 0, + "tagKeys": "dc,instance,shard", + "tags": [], + "titleFormat": "Stall found", + "type": "tags" + }, + { + "class": "annotation_schema_changed", + "datasource": "Prometheus", + "enable": false, + "expr": "changes(scylla_database_schema_changed[30s])>0", + "hide": false, + "iconColor": "rgba(255, 96, 96, 1)", + "limit": 100, + "name": "Schema Changed", + "showIn": 0, + "tagKeys": "instance,dc,cluster", + "tags": [], + "titleFormat": "schema changed", + "type": "tags" + } + ] + }, + "class": "dashboard", + "editable": true, + "gnetId": null, + "graphTooltip": 1, + "hideControls": true, + "id": null, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [], + "type": "dashboards" + } + ], + "originalTitle": "CQL", + "overwrite": true, + "panels": [ + { + "class": "collapsible_row_panel", + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "repeat": "", + "title": "", + "type": "row" + }, + { + "class": "text_panel", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 2, + "isNew": true, + "links": [], + "mode": "html", + "options": { + "content": "
\n\n [[cluster]]
", + "mode": "html" + }, + "span": 12, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "collapsible_row_panel", + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 4 + }, + "id": 3, + "panels": [], + "repeat": "", + "title": "CQL By User", + "type": "row" + }, + { + "class": "text_panel", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 4, + "isNew": true, + "links": [], + "mode": "html", + "options": { + "content": "

CQL By User - Coordinator

", + "mode": "html" + }, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "ops_panel", + "datasource": "Prometheus", + "description": "Number of CQL INSERT commands generated by the user", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 7 + }, + "id": 5, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "sum(rate(scylla_cql_inserts{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]) - sum(rate(scylla_cql_inserts_per_ks{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\", who=\"internal\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "CQL Insert", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "Prometheus", + "description": "Number of CQL SELECT commands generated by the user", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 7 + }, + "id": 6, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "sum(rate(scylla_cql_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]) - sum(rate(scylla_cql_reads_per_ks{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\", who=\"internal\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "CQL Reads", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "Prometheus", + "description": "Number of CQL DELETE commands generated by the user", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 7 + }, + "id": 7, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "sum(rate(scylla_cql_deletes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])-sum(rate(scylla_cql_deletes_per_ks{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\", who=\"internal\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "CQL Deletes", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "Prometheus", + "description": "Number of CQL UPDATE commands generated by the user", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 7 + }, + "id": 8, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "sum(rate(scylla_cql_updates{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])-sum(rate(scylla_cql_updates_per_ks{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\", who=\"internal\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "CQL Updates", + "type": "timeseries" + }, + { + "class": "graph_panel", + "datasource": "Prometheus", + "description": "amount of CQL connections currently established", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 13 + }, + "id": 9, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pointradius": 1, + "span": 3, + "targets": [ + { + "expr": "sum(scylla_transport_current_connections{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "title": "Client CQL connections by [[by]]", + "type": "timeseries" + }, + { + "class": "graph_panel", + "datasource": "Prometheus", + "description": "Number of CQL batches command, each batched command is counted once", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 13 + }, + "id": 10, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pointradius": 1, + "span": 3, + "targets": [ + { + "expr": "sum(rate(scylla_cql_batches{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "title": "CQL Batches by [[by]]", + "type": "timeseries" + }, + { + "class": "graph_panel", + "datasource": "Prometheus", + "description": "Number of CQL command batched. Each batch would add the number of commands inside the batch", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 13 + }, + "id": 11, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pointradius": 1, + "span": 3, + "targets": [ + { + "expr": "sum(rate(scylla_cql_statements_in_batches{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "title": "CQL Command In Batches by [[by]]", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "Prometheus", + "description": "Counts the number of SELECT statements with BYPASS CACHE option", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 13 + }, + "id": 12, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "sum(rate(scylla_cql_select_bypass_caches{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "refId": "A" + } + ], + "title": "BYPASS CACHE", + "type": "timeseries" + }, + { + "class": "graph_panel", + "dashversion": [ + ">4.4", + ">2021.1" + ], + "datasource": "Prometheus", + "description": "CQL errors by type, only active errors are shown", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 19 + }, + "id": 13, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pointradius": 1, + "span": 3, + "targets": [ + { + "expr": "sum(rate(scylla_transport_cql_errors_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]],type) >0", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "title": "CQL Errors [[by]]", + "type": "timeseries" + }, + { + "class": "graph_panel", + "datasource": "Prometheus", + "description": "Number of CQL row reads", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 19 + }, + "id": 14, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pointradius": 1, + "span": 3, + "targets": [ + { + "expr": "sum(rate(scylla_cql_rows_read{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "title": "CQL Row Reads [[by]]", + "type": "timeseries" + }, + { + "class": "graph_panel", + "datasource": "Prometheus", + "description": "Number of reads using secondary indexes", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 19 + }, + "id": 15, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pointradius": 1, + "span": 3, + "targets": [ + { + "expr": "sum(rate(scylla_cql_secondary_index_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "title": "Secondary indexes Reads [[by]]", + "type": "timeseries" + }, + { + "class": "collapsible_row_panel", + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 16, + "panels": [ + { + "class": "single_value_table", + "columns": [ + { + "text": "Avg", + "value": "avg" + } + ], + "datasource": "scylla-datasource", + "fieldConfig": { + "defaults": { + "custom": { + "align": null, + "filterable": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "fontSize": "100%", + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 26 + }, + "id": 17, + "links": [], + "options": { + "showHeader": true + }, + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "span": 12, + "targets": [ + { + "dashversion": [ + ">4.4", + ">2020.1" + ], + "queryHost": "$node", + "queryText": "select address, port, shard_id, connection_stage, client_type, ssl_enabled, username, driver_name, driver_version, protocol_version from system.clients", + "refId": "A" + } + ], + "title": "Connection Table", + "transform": "table", + "type": "table" + } + ], + "repeat": "", + "title": "Connection", + "type": "row" + }, + { + "class": "collapsible_row_panel", + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 32 + }, + "id": 18, + "panels": [ + { + "class": "single_value_table", + "columns": [ + { + "text": "Avg", + "value": "avg" + } + ], + "datasource": "scylla-datasource", + "fieldConfig": { + "defaults": { + "custom": { + "align": null, + "filterable": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "row_size" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + } + ] + } + ] + }, + "fontSize": "100%", + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 33 + }, + "id": 19, + "links": [], + "options": { + "showHeader": true + }, + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "span": 12, + "targets": [ + { + "queryHost": "$node", + "queryText": "select keyspace_name, table_name,partition_key, clustering_key, row_size from system.large_rows", + "refId": "A" + } + ], + "title": "Large Rows", + "transform": "table", + "type": "table" + } + ], + "repeat": "", + "title": "Large Rows", + "type": "row" + }, + { + "class": "collapsible_row_panel", + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 39 + }, + "id": 20, + "panels": [ + { + "class": "single_value_table", + "columns": [ + { + "text": "Avg", + "value": "avg" + } + ], + "datasource": "scylla-datasource", + "fieldConfig": { + "defaults": { + "custom": { + "align": null, + "filterable": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "cell_size" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + } + ] + } + ] + }, + "fontSize": "100%", + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 21, + "links": [], + "options": { + "showHeader": true + }, + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "span": 12, + "targets": [ + { + "queryHost": "$node", + "queryText": "select keyspace_name, table_name,partition_key, clustering_key, column_name,cell_size from system.large_cells", + "refId": "A" + } + ], + "title": "Large Cells", + "transform": "table", + "type": "table" + } + ], + "repeat": "", + "title": "Large Cells", + "type": "row" + }, + { + "class": "collapsible_row_panel", + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 46 + }, + "id": 22, + "panels": [ + { + "class": "single_value_table", + "columns": [ + { + "text": "Avg", + "value": "avg" + } + ], + "datasource": "scylla-datasource", + "fieldConfig": { + "defaults": { + "custom": { + "align": null, + "filterable": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "partition_size" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + } + ] + } + ] + }, + "fontSize": "100%", + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 47 + }, + "id": 23, + "links": [], + "options": { + "showHeader": true + }, + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "span": 12, + "targets": [ + { + "queryHost": "$node", + "queryText": "select keyspace_name, table_name,partition_key, partition_size from system.large_partitions", + "refId": "A" + } + ], + "title": "Large Partitions", + "transform": "table", + "type": "table" + } + ], + "repeat": "", + "title": "Large Partitions", + "type": "row" + }, + { + "class": "collapsible_row_panel", + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 53 + }, + "id": 24, + "panels": [ + { + "class": "text_panel", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 54 + }, + "id": 25, + "isNew": true, + "links": [], + "mode": "html", + "options": { + "content": "

CQL Internal - Coordinator

", + "mode": "html" + }, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "ops_panel", + "datasource": "Prometheus", + "description": "Number of CQL INSERT commands generated by intenal operations", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 56 + }, + "id": 26, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "sum(rate(scylla_cql_inserts_per_ks{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\", who=\"internal\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "CQL Internal Insert", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "Prometheus", + "description": "Number of CQL SELECT commands generated by intenal operations", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 56 + }, + "id": 27, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "sum(rate(scylla_cql_reads_per_ks{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\", who=\"internal\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "CQL Internal Reads", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "Prometheus", + "description": "Number of CQL DELETE commands generated by intenal operations", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 56 + }, + "id": 28, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "sum(rate(scylla_cql_deletes_per_ks{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\", who=\"internal\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "CQL Deletes", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "Prometheus", + "description": "Number of CQL UPDATE commands generated by intenal operations", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 56 + }, + "id": 29, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "sum(rate(scylla_cql_updates_per_ks{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\", who=\"internal\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "CQL Updates", + "type": "timeseries" + } + ], + "repeat": "", + "title": "CQL Internal", + "type": "row" + }, + { + "class": "collapsible_row_panel", + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 62 + }, + "id": 30, + "panels": [], + "repeat": "", + "title": "LWT", + "type": "row" + }, + { + "class": "text_panel", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 63 + }, + "id": 31, + "isNew": true, + "links": [], + "mode": "html", + "options": { + "content": "

LWT

", + "mode": "html" + }, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "ops_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 65 + }, + "id": 32, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "sum(rate(scylla_cql_inserts{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\", conditional=\"yes\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "CQL Insert", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 65 + }, + "id": 33, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "sum(rate(scylla_cql_deletes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\", conditional=\"yes\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "CQL Deletes", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 65 + }, + "id": 34, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "sum(rate(scylla_cql_updates{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\", conditional=\"yes\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "CQL Updates", + "type": "timeseries" + }, + { + "class": "graph_panel", + "datasource": "Prometheus", + "description": "Number of CQL batches command, each batched command is counted once", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 65 + }, + "id": 35, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pointradius": 1, + "span": 3, + "targets": [ + { + "expr": "sum(rate(scylla_cql_batches{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\", conditional=\"yes\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 30 + } + ], + "title": "CQL Batches by [[by]]", + "type": "timeseries" + }, + { + "class": "collapsible_row_panel", + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 71 + }, + "id": 36, + "panels": [], + "repeat": "", + "title": "Optimization", + "type": "row" + }, + { + "class": "text_panel", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 72 + }, + "id": 37, + "isNew": true, + "links": [], + "mode": "html", + "options": { + "content": "

Optimization

", + "mode": "html" + }, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "cacheTimeout": null, + "class": "gauge_errors_panel", + "datasource": "Prometheus", + "description": "All of the requests should be prepared\n\nPrepared statements remove the overhead of parsing the query every time and allow optimal routing of requests from client to server", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": {}, + "mappings": [ + { + "id": 0, + "op": "=", + "text": "N/A", + "type": 1, + "value": "null" + } + ], + "max": 100, + "min": 0, + "nullValueMode": "connected", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 4 + }, + { + "color": "#d44a3a", + "value": 10 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 2, + "x": 0, + "y": 74 + }, + "id": 38, + "links": [], + "options": { + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "7.3.5", + "span": 1, + "targets": [ + { + "expr": "floor(100 *sum(cql:non_system_prepared1m)/ (sum(cql:all_shardrate1m{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) - sum(cql:all_system_shardrate1m{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}))) OR vector(0)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "CQL Non-Prepared Statements", + "transparent": true, + "type": "gauge" + }, + { + "class": "ops_panel", + "datasource": "Prometheus", + "description": "All of the requests should be prepared\n", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 2, + "y": 74 + }, + "id": 39, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "sum(cql:non_system_prepared1m{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "refId": "A" + } + ], + "title": "CQL Non-Prepared Statements", + "type": "timeseries" + }, + { + "cacheTimeout": null, + "class": "gauge_errors_panel", + "datasource": "Prometheus", + "description": "All requests should be paged\n\nNon Paged request sources:\n- Client modifying the fetch size\n\nNon Paged requests require reading all the results and returning them in a single request.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": {}, + "mappings": [ + { + "id": 0, + "op": "=", + "text": "N/A", + "type": 1, + "value": "null" + } + ], + "max": 100, + "min": 0, + "nullValueMode": "connected", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 4 + }, + { + "color": "#d44a3a", + "value": 10 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 2, + "x": 8, + "y": 74 + }, + "id": 40, + "links": [], + "options": { + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "7.3.5", + "span": 1, + "targets": [ + { + "expr": "100 * ((sum(rate(scylla_cql_unpaged_select_queries{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval]))-sum(rate(scylla_cql_unpaged_select_queries_per_ks{ks=\"system\",instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]))/sum(rate(scylla_cql_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval]))) OR vector(0)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Non-Paged CQL Reads", + "transparent": true, + "type": "gauge" + }, + { + "class": "ops_panel", + "datasource": "Prometheus", + "description": "Non-Paged requests require reading all the results and returning them in a single request", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 10, + "y": 74 + }, + "id": 41, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "sum(rate(scylla_cql_unpaged_select_queries{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])-sum(rate(scylla_cql_unpaged_select_queries_per_ks{ks=\"system\",instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "refId": "A" + } + ], + "title": "Non-Paged CQL Reads", + "type": "timeseries" + }, + { + "cacheTimeout": null, + "class": "gauge_errors_panel", + "datasource": "Prometheus", + "description": "All of the requests should be Token Aware\n\nNon Token Aware requests sources:\n* Non-Prepared Stamements\n* Client not using a Token Aware load balancing policy\n\nTokenAware requests are sent to a Scylla node that is also a replica. Token Un-Aware requests require extra hop and additional processing.\n\nNote that the metric shows incorrect values when batches are used.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": {}, + "mappings": [ + { + "id": 0, + "op": "=", + "text": "N/A", + "type": 1, + "value": "null" + } + ], + "max": 100, + "min": 0, + "nullValueMode": "connected", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 4 + }, + { + "color": "#d44a3a", + "value": 10 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 2, + "x": 16, + "y": 74 + }, + "id": 42, + "links": [], + "options": { + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "7.3.5", + "span": 1, + "targets": [ + { + "expr": "scalar(sum(rate(scylla_storage_proxy_coordinator_cas_total_operations{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) <=bool 0)*scalar(sum(rate(scylla_storage_proxy_replica_received_counter_updates{cluster=~\"$cluster|$^\"}[1m]))<=bool 0) *(100 - clamp_max(100*(sum(rate(scylla_storage_proxy_coordinator_reads_local_node{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) + sum(rate(scylla_storage_proxy_coordinator_total_write_attempts_local_node{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])))/(sum(rate(scylla_cql_inserts{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) +sum(rate(scylla_cql_updates{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) +sum(rate(scylla_cql_deletes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m])) +sum(rate(scylla_cql_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[1m]))) + 1,100)) OR vector(0)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Non-Token Aware", + "transparent": true, + "type": "gauge" + }, + { + "class": "ops_panel", + "datasource": "Prometheus", + "description": "Requests that are not token aware indicates that requests are not routed to the right node, which require extra hop and additional processing.\n\nNote that the metric shows incorrect values when batches are used.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 74 + }, + "id": 43, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "scalar(sum(rate(scylla_storage_proxy_coordinator_cas_total_operations{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) <=bool 0)*scalar(sum(rate(scylla_storage_proxy_replica_received_counter_updates{cluster=~\"$cluster|$^\"}[1m]))<=bool 0) *clamp_min(sum(rate(scylla_cql_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]) - sum(rate(scylla_storage_proxy_coordinator_reads_local_node{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]) + sum(rate(scylla_cql_inserts{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval]) + rate(scylla_cql_updates{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval]) + rate(scylla_cql_deletes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]) - sum(rate(scylla_storage_proxy_coordinator_total_write_attempts_local_node{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]),0)", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "refId": "A" + } + ], + "title": "Non-Token Aware Queries", + "type": "timeseries" + }, + { + "cacheTimeout": null, + "class": "gauge_errors_panel", + "datasource": "Prometheus", + "description": "Reversed CQL Reads entail additional processing on server side\n\nSources: CQL Read requests with ORDER BY that is different from the \"CLUSTERING ORDER BY\" of the table\nAlternatives:\n\n* Denormalize your data (use a Materialized View)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": {}, + "mappings": [ + { + "id": 0, + "op": "=", + "text": "N/A", + "type": 1, + "value": "null" + } + ], + "max": 100, + "min": 0, + "nullValueMode": "connected", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 4 + }, + { + "color": "#d44a3a", + "value": 10 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 2, + "x": 0, + "y": 80 + }, + "id": 44, + "links": [], + "options": { + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "7.3.5", + "span": 1, + "targets": [ + { + "expr": "100 * sum(rate(scylla_cql_reverse_queries{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) / sum(rate(scylla_cql_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) OR vector(0)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Reversed CQL Reads", + "transparent": true, + "type": "gauge" + }, + { + "class": "ops_panel", + "datasource": "Prometheus", + "description": "Reversed CQL Reads entail additional processing on server side and should be avoided", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 2, + "y": 80 + }, + "id": 45, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "sum(rate(scylla_cql_reverse_queries{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "refId": "A" + } + ], + "title": "Reversed CQL Reads", + "type": "timeseries" + }, + { + "cacheTimeout": null, + "class": "gauge_errors_panel", + "datasource": "Prometheus", + "description": "ALLOW FILTERING CQL Reads, the percentage of read requests with 'ALLOW FILTERING'\n\nALLOW FILTERING CQL Reads entail additional processing on server side\n\nSources: CQL Read requests with \"ALLOW FILTERING\"\n\nALLOW FILTERING should be used when large parts of the filtered data is returned - check \n\"ALLOW FILTERING CQL Read Filtered Rows to check what percentage of the data is used\"\n\nAlternatives:\n- Use a Secondary Index\n- Denormalize your data (use a Materialized View)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": {}, + "mappings": [ + { + "id": 0, + "op": "=", + "text": "N/A", + "type": 1, + "value": "null" + } + ], + "max": 100, + "min": 0, + "nullValueMode": "connected", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 4 + }, + { + "color": "#d44a3a", + "value": 10 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 2, + "x": 8, + "y": 80 + }, + "id": 46, + "links": [], + "options": { + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "7.3.5", + "span": 1, + "targets": [ + { + "expr": "100 * sum(rate(scylla_cql_filtered_read_requests{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) / sum(rate(scylla_cql_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) OR vector(0)", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "ALLOW FILTERING CQL Reads", + "transparent": true, + "type": "gauge" + }, + { + "class": "rps_panel", + "datasource": "Prometheus", + "description": "Read requests with ALLOW FILTERING\n\nALLOW FILTERING CQL Reads entail additional processing on server side and should be avoided", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:reads/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 10, + "y": 80 + }, + "id": 47, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "sum(rate(scylla_cql_filtered_read_requests{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "refId": "A" + } + ], + "title": "ALLOW FILTERING CQL Reads", + "type": "timeseries" + }, + { + "cacheTimeout": null, + "class": "gauge_errors_panel", + "datasource": "Prometheus", + "description": "ALLOW FILTERING Filtered rows, the percentage of rows that were read and then filtered.\n\nALLOW FILTERING CQL Reads entail additional processing on server side. \nReading a row and then filter it is a waste of resources.\n\nSources: CQL Read requests with \"ALLOW FILTERING\"\n\nALLOW FILTERING should be used when large parts of the filtered data is returned\n\nAlternatives:\n- Use a Secondary Index\n- Denormalize your data (use a Materialized View)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": {}, + "mappings": [ + { + "id": 0, + "op": "=", + "text": "N/A", + "type": 1, + "value": "null" + } + ], + "max": 100, + "min": 0, + "nullValueMode": "connected", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 4 + }, + { + "color": "#d44a3a", + "value": 10 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 2, + "x": 16, + "y": 80 + }, + "id": 48, + "links": [], + "options": { + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "7.3.5", + "span": 1, + "targets": [ + { + "expr": "100 * sum(rate(scylla_cql_filtered_rows_dropped_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) /sum(rate(scylla_cql_filtered_rows_read_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) OR vector(0)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "ALLOW FILTERING Filtered Rows", + "transparent": true, + "type": "gauge" + }, + { + "class": "rps_panel", + "datasource": "Prometheus", + "description": "CQL Queries with ALLOW FILTERING should be avoided.\nDropped rows are rows that were read but were filtered by the server.\nWhen dropped rows is relatively high you should consider the alternatives", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:reads/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 80 + }, + "id": 49, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "sum(rate(scylla_cql_filtered_rows_read_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "rows read $node $shard", + "refId": "A" + }, + { + "expr": "sum(rate(scylla_cql_filtered_rows_matched_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "rows matched $node $shard", + "refId": "B" + }, + { + "expr": "sum(rate(scylla_cql_filtered_rows_dropped_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "rows dropped $node $shard", + "refId": "C" + } + ], + "title": "ALLOW FILTERING CQL Read Filtering", + "type": "timeseries" + }, + { + "cacheTimeout": null, + "class": "gauge_errors_panel", + "datasource": "Prometheus", + "description": "Using consistency level ANY in a query may hurt persistency, if the node receiving the request will fail the data may be lost", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": {}, + "mappings": [ + { + "id": 0, + "op": "=", + "text": "N/A", + "type": 1, + "value": "null" + } + ], + "max": 100, + "min": 0, + "nullValueMode": "connected", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 4 + }, + { + "color": "#d44a3a", + "value": 10 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 2, + "x": 0, + "y": 86 + }, + "id": 50, + "links": [], + "options": { + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "7.3.5", + "span": 1, + "targets": [ + { + "expr": "floor(100 *sum(rate(scylla_query_processor_queries{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\", consistency_level=\"ANY\"}[$__rate_interval]))/sum(rate(scylla_query_processor_queries{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval]))) OR vector(0)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "CQL ANY Queries", + "transparent": true, + "type": "gauge" + }, + { + "class": "ops_panel", + "datasource": "Prometheus", + "description": "Using consistency level ANY in a query may hurt persistency, if the node receiving the request will fail the data may be lost", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 2, + "y": 86 + }, + "id": 51, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "sum(rate(scylla_query_processor_queries{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\", consistency_level=\"ANY\"}[$__rate_interval])) by ([[by]])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "refId": "A" + } + ], + "title": "CQL ANY CL Queries", + "type": "timeseries" + }, + { + "cacheTimeout": null, + "class": "gauge_errors_panel", + "datasource": "Prometheus", + "description": "Using consistency level ALL in a query may hurt availability, if a node is unavailable operations will fail", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": {}, + "mappings": [ + { + "id": 0, + "op": "=", + "text": "N/A", + "type": 1, + "value": "null" + } + ], + "max": 100, + "min": 0, + "nullValueMode": "connected", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 4 + }, + { + "color": "#d44a3a", + "value": 10 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 2, + "x": 8, + "y": 86 + }, + "id": 52, + "links": [], + "options": { + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "7.3.5", + "span": 1, + "targets": [ + { + "expr": "floor(100 *sum(rate(scylla_query_processor_queries{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\", consistency_level=\"ALL\"}[$__rate_interval]))/sum(rate(scylla_query_processor_queries{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval]))) OR vector(0)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "CQL ALL CL Queries", + "transparent": true, + "type": "gauge" + }, + { + "class": "ops_panel", + "datasource": "Prometheus", + "description": "Using consistency level ALL in a query may hurt availability, if a node is unavailable operations will fail", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 10, + "y": 86 + }, + "id": 53, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "sum(rate(scylla_query_processor_queries{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\", consistency_level=\"ALL\"}[$__rate_interval])) by ([[by]])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "refId": "A" + } + ], + "title": "CQL ALL CL Queries", + "type": "timeseries" + }, + { + "class": "text_panel", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 92 + }, + "id": 54, + "isNew": true, + "links": [], + "mode": "html", + "options": { + "content": "

Cross DC Information

", + "mode": "html" + }, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "cacheTimeout": null, + "class": "gauge_errors_panel", + "datasource": "Prometheus", + "description": "Using consistency level ONE in a query when there is more than one DC may hurt performance, queries may end in the non-local DC. Use LOCAL_ONE instead", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": {}, + "mappings": [ + { + "id": 0, + "op": "=", + "text": "N/A", + "type": 1, + "value": "null" + } + ], + "max": 100, + "min": 0, + "nullValueMode": "connected", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 4 + }, + { + "color": "#d44a3a", + "value": 10 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 2, + "x": 0, + "y": 94 + }, + "id": 55, + "links": [], + "options": { + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "7.3.5", + "span": 1, + "targets": [ + { + "expr": "floor(100 *sum(rate(scylla_query_processor_queries{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\", consistency_level=\"ONE\"}[$__rate_interval]))/sum(rate(scylla_query_processor_queries{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval]))) OR vector(0)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "CQL ONE Queries", + "transparent": true, + "type": "gauge" + }, + { + "class": "ops_panel", + "datasource": "Prometheus", + "description": "Using consistency level ONE in a query when there is more than one DC may hurt performance, queries may end in the non-local DC. Use LOCAL_ONE instead", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 2, + "y": 94 + }, + "id": 56, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "sum(rate(scylla_query_processor_queries{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\", consistency_level=\"ONE\"}[$__rate_interval])) by ([[by]])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "refId": "A" + } + ], + "title": "CQL ONE CL Queries", + "type": "timeseries" + }, + { + "cacheTimeout": null, + "class": "gauge_errors_panel", + "datasource": "Prometheus", + "description": "Using consistency level QUORUM in a query when there is more than one DC may hurt performance, queries may end in the non-local DC. Use LOCAL_QUORUM instead", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": {}, + "mappings": [ + { + "id": 0, + "op": "=", + "text": "N/A", + "type": 1, + "value": "null" + } + ], + "max": 100, + "min": 0, + "nullValueMode": "connected", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 4 + }, + { + "color": "#d44a3a", + "value": 10 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 2, + "x": 8, + "y": 94 + }, + "id": 57, + "links": [], + "options": { + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "7.3.5", + "span": 1, + "targets": [ + { + "expr": "floor(100 *sum(rate(scylla_query_processor_queries{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\", consistency_level=\"QUORUM\"}[$__rate_interval]))/sum(rate(scylla_query_processor_queries{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval]))) OR vector(0)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "CQL QUORUM CL Queries", + "transparent": true, + "type": "gauge" + }, + { + "class": "ops_panel", + "datasource": "Prometheus", + "description": "Using consistency level QUORUM in a query when there is more than one DC may hurt performance, queries may end in the non-local DC. Use LOCAL_QUORUM instead", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 10, + "y": 94 + }, + "id": 58, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "sum(rate(scylla_query_processor_queries{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\", consistency_level=\"QUORUM\"}[$__rate_interval])) by ([[by]])", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "refId": "A" + } + ], + "title": "CQL QUORUM CL Queries", + "type": "timeseries" + }, + { + "cacheTimeout": null, + "class": "gauge_errors_panel", + "datasource": "Prometheus", + "description": "Cross DC traffic may cause additional latencies and network loads and in most cases, should be avoided.\n\nCross DC Read requests sources:\n- Consistency Level that is not LOCAL_XXX\n- Tables with read_repair_chance > 0\n\nNote:\n- If requests are supposed to be DC local - verify client is using a DCAware policy and a LOCAL_XX consistency level", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": {}, + "mappings": [ + { + "id": 0, + "op": "=", + "text": "N/A", + "type": 1, + "value": "null" + } + ], + "max": 100, + "min": 0, + "nullValueMode": "connected", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "#299c46", + "value": null + }, + { + "color": "rgba(237, 129, 40, 0.89)", + "value": 4 + }, + { + "color": "#d44a3a", + "value": 10 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 100 + }, + "id": 59, + "links": [], + "options": { + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "7.3.5", + "repeat": "dc", + "span": 2, + "targets": [ + { + "expr": "100*(sum(rate(scylla_storage_proxy_coordinator_reads_remote_node{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) - sum(rate(scylla_storage_proxy_coordinator_reads_remote_node{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", datacenter=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])))/sum(rate(scylla_storage_proxy_coordinator_reads_remote_node{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) OR vector(0)", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Cross DC read requests $dc", + "transparent": true, + "type": "gauge" + }, + { + "class": "plain_text", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 106 + }, + "id": 60, + "isNew": true, + "links": [], + "mode": "html", + "options": { + "content": "
Scylla Monitoring version - master
\n    
", + "mode": "html" + }, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + } + ], + "refresh": "30s", + "schemaVersion": 26, + "style": "dark", + "tags": [ + "4.4" + ], + "templating": { + "list": [ + { + "allValue": null, + "class": "by_template_var", + "current": { + "tags": [], + "text": "Instance", + "value": "instance" + }, + "error": null, + "hide": 0, + "includeAll": false, + "label": "by", + "multi": false, + "name": "by", + "options": [ + { + "selected": false, + "text": "Cluster", + "value": "cluster" + }, + { + "selected": false, + "text": "DC", + "value": "dc" + }, + { + "selected": true, + "text": "Instance", + "value": "instance" + }, + { + "selected": false, + "text": "instance,shard", + "value": "instance,shard" + } + ], + "query": "Cluster,DC,Instance,Shard", + "skipUrlSync": false, + "type": "custom" + }, + { + "allValue": null, + "class": "template_variable_single", + "current": { + "isNone": true, + "selected": false, + "text": "None", + "value": "" + }, + "datasource": "Prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [], + "query": "label_values(scylla_reactor_utilization, cluster)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "class": "template_variable_all", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "Prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": true, + "label": "dc", + "multi": true, + "name": "dc", + "options": [], + "query": "label_values(scylla_reactor_utilization{cluster=~\"$cluster\"}, dc)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "class": "template_variable_all", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "Prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": true, + "label": "node", + "multi": true, + "name": "node", + "options": [], + "query": "label_values(scylla_reactor_utilization{cluster=~\"$cluster|$^\", dc=~\"$dc\"}, instance)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "class": "template_variable_all", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "Prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": true, + "label": "shard", + "multi": true, + "name": "shard", + "options": [], + "query": "label_values(scylla_reactor_utilization,shard)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 3, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "class": "adhoc_filter", + "datasource": "Prometheus", + "error": null, + "filters": [], + "hide": 0, + "label": "ad hoc", + "name": "adhoc", + "skipUrlSync": false, + "type": "adhoc" + }, + { + "allValue": null, + "class": "template_variable_custom", + "current": { + "text": "4-4", + "value": "4-4" + }, + "error": null, + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "dash_version", + "options": [ + { + "selected": true, + "text": "4-4", + "value": "4-4" + } + ], + "query": "4-4", + "skipUrlSync": false, + "type": "custom" + }, + { + "allValue": null, + "class": "template_variable_custom", + "current": { + "text": "4.4", + "value": "4.4" + }, + "error": null, + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "scylla_version", + "options": [ + { + "selected": true, + "text": "4.4", + "value": "4.4" + } + ], + "query": "4.4", + "skipUrlSync": false, + "type": "custom" + }, + { + "allValue": null, + "class": "monitor_version_var", + "current": { + "text": "master", + "value": "master" + }, + "error": null, + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "monitoring_version", + "options": [ + { + "selected": true, + "text": "master", + "value": "master" + } + ], + "query": "master", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": { + "now": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "utc", + "title": "Scylla CQL", + "uid": "cql-4-4", + "version": 1 +} diff --git a/sysadmin/grid5000/cassandra/monitoring/grafana/dashboards/scylla/scylla-detailed.4.4.json b/sysadmin/grid5000/cassandra/monitoring/grafana/dashboards/scylla/scylla-detailed.4.4.json new file mode 100644 index 0000000..8d3c8e3 --- /dev/null +++ b/sysadmin/grid5000/cassandra/monitoring/grafana/dashboards/scylla/scylla-detailed.4.4.json @@ -0,0 +1,8001 @@ +{ + "annotations": { + "class": "default_annotations", + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "class": "annotation_restart", + "datasource": "Prometheus", + "enable": true, + "expr": "resets(scylla_gossip_heart_beat[1m])>0", + "hide": false, + "iconColor": "rgba(255, 96, 96, 1)", + "limit": 100, + "name": "node_restart", + "showIn": 0, + "tagKeys": "instance,dc,cluster", + "tags": [], + "titleFormat": "restart", + "type": "tags" + }, + { + "class": "annotation_stall", + "datasource": "Prometheus", + "enable": false, + "expr": "changes(scylla_stall_detector_reported[1m])>0", + "hide": false, + "iconColor": "rgba(255, 96, 96, 1)", + "limit": 100, + "name": "stall detector", + "showIn": 0, + "tagKeys": "dc,instance,shard", + "tags": [], + "titleFormat": "Stall found", + "type": "tags" + }, + { + "class": "annotation_schema_changed", + "datasource": "Prometheus", + "enable": false, + "expr": "changes(scylla_database_schema_changed[30s])>0", + "hide": false, + "iconColor": "rgba(255, 96, 96, 1)", + "limit": 100, + "name": "Schema Changed", + "showIn": 0, + "tagKeys": "instance,dc,cluster", + "tags": [], + "titleFormat": "schema changed", + "type": "tags" + }, + { + "class": "annotation_manager_task", + "datasource": "Prometheus", + "enable": true, + "expr": "scylla_manager_task_active_count{type=~\"repair|backup\",cluster=~\"$cluster|$^\"}>0", + "hide": false, + "iconColor": "#73BF69", + "limit": 100, + "name": "Task", + "showIn": 0, + "tagKeys": "type", + "tags": [], + "titleFormat": "Running", + "type": "tags" + }, + { + "class": "annotation_hints_writes", + "datasource": "Prometheus", + "enable": false, + "expr": "changes(scylla_hints_manager_written[60s])>0", + "hide": false, + "iconColor": "rgb(255, 176, 0, 128)", + "limit": 100, + "name": "Hints Write", + "showIn": 0, + "tagKeys": "instance,dc,cluster", + "tags": [], + "titleFormat": "Hints write", + "type": "tags" + }, + { + "class": "annotation_hints_sent", + "datasource": "Prometheus", + "enable": false, + "expr": "changes(scylla_hints_manager_written[60s])>0", + "hide": false, + "iconColor": "rgb(50, 176, 0, 128)", + "limit": 100, + "name": "Hints Sent", + "showIn": 0, + "tagKeys": "instance,dc,cluster", + "tags": [], + "titleFormat": "Hints Sent", + "type": "tags" + }, + { + "class": "mv_building", + "datasource": "Prometheus", + "enable": true, + "expr": "sum(scylla_view_builder_builds_in_progress)>0", + "hide": false, + "iconColor": "rgb(50, 176, 0, 128)", + "limit": 100, + "name": "MV", + "showIn": 0, + "tagKeys": "instance,dc,cluster", + "tags": [], + "titleFormat": "Materialized View built", + "type": "tags" + } + ] + }, + "class": "dashboard", + "editable": true, + "gnetId": null, + "graphTooltip": 1, + "hideControls": true, + "id": null, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [], + "type": "dashboards" + } + ], + "overwrite": true, + "panels": [ + { + "class": "text_panel", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 1, + "isNew": true, + "links": [], + "mode": "html", + "options": { + "content": "
\n\n [[cluster]]
", + "mode": "html" + }, + "span": 12, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "percent_panel", + "datasource": "Prometheus", + "description": "The percentage of the time during which Scylla utilized the CPU. Note that because Scylla does busy polling for some time before going idle, CPU utilization as seen by the operating system may be much higher. Your system is not yet CPU-bottlenecked until this metric is high.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 3 + }, + "id": 2, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "avg(scylla_reactor_utilization{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"} ) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 4 + } + ], + "title": "Load", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "Prometheus", + "description": "Amount of requests served as the coordinator. Imbalances here represent dispersion at the connection level, not your data model.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 3 + }, + "id": 3, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_transport_requests_served{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]) + ($func(rate(scylla_thrift_served{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]]) or max(scylla_transport_requests_served{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by([[by]])*0)", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "Requests Served per [[by]] - Coordinator", + "type": "timeseries" + }, + { + "class": "rps_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:reads/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 3 + }, + "id": 4, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_database_total_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Reads per [[by]] - Replica", + "type": "timeseries" + }, + { + "class": "wps_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:writes/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 3 + }, + "id": 5, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_database_total_writes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "Writes per [[by]] - Replica", + "type": "timeseries" + }, + { + "class": "collapsible_row_panel", + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 6, + "panels": [], + "repeat": "", + "title": "reads and writes", + "type": "row" + }, + { + "class": "plain_text", + "content": "

Reads and Writes - Coordinator

", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 10 + }, + "id": 7, + "isNew": true, + "links": [], + "mode": "html", + "options": {}, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "wps_panel", + "datasource": "Prometheus", + "description": "Foreground writes are writes that weren't acknowledged yet to the application. For instance, if a single replica responded and two are needed due to the consistency level. This metric represents a queue size, not a rate. High values here correlate with increased write latencies.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:writes/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 12 + }, + "id": 8, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(scylla_storage_proxy_coordinator_foreground_writes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Foreground Writes per [[by]]", + "type": "timeseries" + }, + { + "class": "wps_panel", + "datasource": "Prometheus", + "description": "Background writes are writes that are already acknowledged to the application but have additional work to be done. For instance, if a replica responded and only one is needed, this request is still listed as a background request until all replicas respond. This metric represents a queue size, not a rate. High values here correlate with increased write latencies.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:writes/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 12 + }, + "id": 9, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(scylla_storage_proxy_coordinator_background_writes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Background Writes per [[by]]", + "type": "timeseries" + }, + { + "class": "rps_panel", + "datasource": "Prometheus", + "description": "Foreground reads are reads that weren't acknowledged yet to the application. For instance, if a single replica responded and two are needed due to the consistency level. This metric represents a queue size, not a rate. High values here correlate with increased read latencies.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:reads/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 12 + }, + "id": 10, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(scylla_storage_proxy_coordinator_foreground_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "Foreground Reads per [[by]]", + "type": "timeseries" + }, + { + "class": "rps_panel", + "datasource": "Prometheus", + "description": "Background reads are reads that are already acknowledged to the application but have additional work to be done. For instance, if a replica responded and only one is needed, this request is still listed as a background request until all replicas respond. This metric represents a queue size, not a rate. High values here correlate with increased read latencies.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:reads/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 12 + }, + "id": 11, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(scylla_storage_proxy_coordinator_background_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 4 + } + ], + "title": "Background Reads per [[by]]", + "type": "timeseries" + }, + { + "class": "wps_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:writes/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 18 + }, + "id": 12, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_hints_manager_written{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Hints Written per [[by]]", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 18 + }, + "id": 13, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_hints_manager_sent{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 4 + } + ], + "title": "Hints sent per [[by]]", + "type": "timeseries" + }, + { + "class": "rps_panel", + "datasource": "Prometheus", + "description": "Number of times a digest read was done on behalf of a speculative retry.\n\nSpeculative retry is a mechanism that causes the client or server to speculate that a request may fail, and send a new request.\n\nspeculative retry may reduce latency in exchange for system load, but only if there is little activity.\n\nA lot of speculative retries increases load and can harm latency more than helping.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:reads/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 18 + }, + "id": 14, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_speculative_digest_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 4 + } + ], + "title": "Speculative Digest Reads By [[by]]", + "type": "timeseries" + }, + { + "class": "rps_panel", + "datasource": "Prometheus", + "description": "Number of times a read was done on behalf of a speculative retry.\n\nSpeculative retry is a mechanism that causes the client or server to speculate that a request may fail, and send a new request.\n\nspeculative retry may reduce latency in exchange for system load, but only if there is little activity.\n\nA lot of speculative retries increases load and can harm latency more than helping.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:reads/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 18 + }, + "id": 15, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_speculative_data_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 4 + } + ], + "title": "Speculative Data Reads By [[by]]", + "type": "timeseries" + }, + { + "class": "collapsible_row_panel", + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 16, + "panels": [], + "repeat": "", + "title": "Timeouts and Errors", + "type": "row" + }, + { + "class": "plain_text", + "content": "

Timeouts and Errors - Coordinator

", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 17, + "isNew": true, + "links": [], + "mode": "html", + "options": {}, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "wps_panel", + "datasource": "Prometheus", + "description": "Requests that Scylla tried to write but timed out. Timeouts are counted in the node that received the request (the coordinator), not at the replicas.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:writes/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 27 + }, + "id": 18, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_write_timeouts{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Write Timeouts/Seconds per [[by]]", + "type": "timeseries" + }, + { + "class": "wps_panel", + "datasource": "Prometheus", + "description": "Requests that Scylla did not even try to write because replicas that were needed to execute this write were unavailable. Unavailable writes are counted in the node that received the request (the coordinator), not at the replicas.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:writes/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 27 + }, + "id": 19, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_write_unavailable{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Write Unavailable/Seconds per [[by]]", + "type": "timeseries" + }, + { + "class": "rps_panel", + "datasource": "Prometheus", + "description": "Requests that Scylla tried to read but timed out. Timeouts are counted in the node that received the request (the coordinator), not at the replicas.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:reads/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 27 + }, + "id": 20, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_read_timeouts{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval]) + rate(scylla_storage_proxy_coordinator_cas_read_timeouts{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])+rate(scylla_storage_proxy_coordinator_range_timeouts{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Read Timeouts/Seconds per [[by]]", + "type": "timeseries" + }, + { + "class": "rps_panel", + "datasource": "Prometheus", + "description": "Requests that Scylla did not even try to read because replicas that were needed to execute this write were unavailable. Unavailable reads are counted in the node that received the request (the coordinator), not at the replicas.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:reads/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 27 + }, + "id": 21, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_read_unavailable{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 4 + } + ], + "title": "Read Unavailable/Seconds per [[by]]", + "type": "timeseries" + }, + { + "class": "collapsible_row_panel", + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 33 + }, + "id": 22, + "panels": [], + "repeat": "", + "title": "Replica", + "type": "row" + }, + { + "class": "plain_text", + "content": "

Replica

", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 23, + "isNew": true, + "links": [], + "mode": "html", + "options": {}, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "reads_panel", + "datasource": "Prometheus", + "description": "The number of currently active read operations", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:reads" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 36 + }, + "id": 24, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(scylla_database_active_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Active sstable reads", + "type": "timeseries" + }, + { + "class": "reads_panel", + "datasource": "Prometheus", + "description": "number of currently queued read operations", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:reads" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 36 + }, + "id": 25, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(scylla_database_queued_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Queued sstable reads", + "type": "timeseries" + }, + { + "class": "writes_panel", + "datasource": "Prometheus", + "description": "The current number of requests blocked due to reaching the memory quota. Non-zero value indicates that our bottleneck is memory", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:writes/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 36 + }, + "id": 26, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(scylla_database_requests_blocked_memory_current{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Writes currently blocked on dirty", + "type": "timeseries" + }, + { + "class": "writes_panel", + "datasource": "Prometheus", + "description": "number of currently pending allocations. A non-zero value indicates that we have a bottleneck in the disk write flow.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:writes/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 36 + }, + "id": 27, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(scylla_commitlog_pending_allocations{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Writes currently blocked on commitlog", + "type": "timeseries" + }, + { + "class": "graph_panel", + "datasource": "Prometheus", + "description": "Reciprocal Miss Rate is a score in the range of 1 to 100 that is used to decide the fraction of read requests to send to each replica - a replica with twice the RMR value of another replica will serve twice the number of read requests.\n\nRMR is calculated on a table level, this is an aggregate estimation of that score.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 42 + }, + "id": 28, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "clamp_max(1 + sum(rate(scylla_cache_row_hits[1m])) by ([[by]])/(sum(rate(scylla_cache_row_misses[1m])) by ([[by]]) + 0.00001),100)", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Reciprocal Miss Rate (HWLB)", + "type": "timeseries" + }, + { + "class": "rps_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:reads/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 42 + }, + "id": 29, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_database_total_reads_failed{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Reads failed", + "type": "timeseries" + }, + { + "class": "wps_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:writes/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 42 + }, + "id": 30, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_database_requests_blocked_memory{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Writes blocked on dirty", + "type": "timeseries" + }, + { + "class": "wps_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:writes/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 42 + }, + "id": 31, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_commitlog_requests_blocked_memory{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Writes blocked on commitlog", + "type": "timeseries" + }, + { + "class": "text_panel", + "content": "", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 48 + }, + "id": 32, + "isNew": true, + "links": [], + "mode": "markdown", + "options": {}, + "span": 3, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "text_panel", + "content": "", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 48 + }, + "id": 33, + "isNew": true, + "links": [], + "mode": "markdown", + "options": {}, + "span": 3, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "wps_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:writes/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 48 + }, + "id": 34, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_database_total_writes_failed{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Writes failed", + "type": "timeseries" + }, + { + "class": "wps_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:writes/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 48 + }, + "id": 35, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_database_total_writes_timedout{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Writes timed out", + "type": "timeseries" + }, + { + "class": "collapsible_row_panel", + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 54 + }, + "id": 36, + "panels": [], + "repeat": "", + "title": "Cache", + "type": "row" + }, + { + "class": "text_panel", + "content": "

Cache - Replica

", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 55 + }, + "id": 37, + "isNew": true, + "links": [], + "mode": "html", + "options": {}, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "rps_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:reads/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 57 + }, + "id": 38, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 6, + "targets": [ + { + "expr": "$func(rate(scylla_cache_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval]) - rate(scylla_cache_reads_with_misses{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Reads with no misses", + "type": "timeseries" + }, + { + "class": "rps_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:reads/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 57 + }, + "id": 39, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 6, + "targets": [ + { + "expr": "$func(rate(scylla_cache_reads_with_misses{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Reads with misses", + "type": "timeseries" + }, + { + "class": "rps_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:reads/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 63 + }, + "id": 40, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_cache_row_hits{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Row Hits", + "type": "timeseries" + }, + { + "class": "rps_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:reads/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 63 + }, + "id": 41, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_cache_partition_hits{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Partition Hits", + "type": "timeseries" + }, + { + "class": "rps_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:reads/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 63 + }, + "id": 42, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_cache_row_misses{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Row Misses", + "type": "timeseries" + }, + { + "class": "rps_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:reads/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 63 + }, + "id": 43, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_cache_partition_misses{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Partition Misses", + "type": "timeseries" + }, + { + "class": "rps_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:reads/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 69 + }, + "id": 44, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_cache_row_insertions{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Row Insertions", + "type": "timeseries" + }, + { + "class": "rps_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:reads/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 69 + }, + "id": 45, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_cache_partition_insertions{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Partition Insertions", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 69 + }, + "id": 46, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_cache_row_evictions{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Row Evictions", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 69 + }, + "id": 47, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_cache_partition_evictions{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Partition Evictions", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 75 + }, + "id": 48, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_cache_rows_merged_from_memtable{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Row Merges", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 75 + }, + "id": 49, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_cache_partition_merges{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Partition Merges", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 75 + }, + "id": 50, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_cache_row_removals{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Row Removals", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 75 + }, + "id": 51, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_cache_partition_removals{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Partition Removals", + "type": "timeseries" + }, + { + "class": "graph_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 81 + }, + "id": 52, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(scylla_cache_rows{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Rows", + "type": "timeseries" + }, + { + "class": "graph_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 81 + }, + "id": 53, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(scylla_cache_partitions{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Partitions", + "type": "timeseries" + }, + { + "class": "bytes_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 81 + }, + "id": 54, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(scylla_cache_bytes_used{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Used Bytes", + "type": "timeseries" + }, + { + "class": "bytes_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 81 + }, + "id": 55, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(scylla_cache_bytes_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Total Bytes", + "type": "timeseries" + }, + { + "class": "collapsible_row_panel", + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 87 + }, + "id": 56, + "panels": [ + { + "class": "text_panel", + "content": "

Materialized Views - Replica

", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 88 + }, + "id": 57, + "isNew": true, + "links": [], + "mode": "html", + "options": {}, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "wps_panel", + "datasource": "Prometheus", + "description": "Number of view update locally", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:writes/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 90 + }, + "id": 58, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_database_total_view_updates_pushed_local{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "View Local Update", + "type": "timeseries" + }, + { + "class": "wps_panel", + "datasource": "Prometheus", + "description": "Number of view update remotely", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:writes/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 90 + }, + "id": 59, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_database_total_view_updates_pushed_remote{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "View Remote Update", + "type": "timeseries" + }, + { + "class": "bytes_panel", + "datasource": "Prometheus", + "description": "Size in bytes of the view update backlog at each base replica.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 90 + }, + "id": 60, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(scylla_database_view_update_backlog{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "View Update Backlog", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "Prometheus", + "description": "Number of dropped view updates due to an excessive view update backlog.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 90 + }, + "id": 61, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_database_dropped_view_updates{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "Dropped View Updates", + "type": "timeseries" + }, + { + "class": "wps_panel", + "datasource": "Prometheus", + "description": "Number of hints sent for view.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:writes/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 96 + }, + "id": 62, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_hints_for_views_manager_sent{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "Hints for view", + "type": "timeseries" + }, + { + "class": "wps_panel", + "datasource": "Prometheus", + "description": "Currently throttled base writes, as a consequence of the respective view update backlog.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:writes/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 96 + }, + "id": 63, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(scylla_storage_proxy_coordinator_current_throttled_base_writes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "Throttled Base Writes", + "type": "timeseries" + } + ], + "repeat": "", + "title": "Materialized Views", + "type": "row" + }, + { + "class": "collapsible_row_panel", + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 102 + }, + "id": 64, + "panels": [ + { + "class": "text_panel", + "content": "

LWT - Coordinator

", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 103 + }, + "id": 65, + "isNew": true, + "links": [], + "mode": "html", + "options": {}, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "rps_panel", + "datasource": "Prometheus", + "description": "LWT read rate.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:reads/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 105 + }, + "id": 66, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_cas_read_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "LWT Reads", + "type": "timeseries" + }, + { + "class": "us_panel", + "datasource": "Prometheus", + "description": "LWT Avrage Read latency.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 105 + }, + "id": 67, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_cas_read_latency_sum{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]])/($func(rate(scylla_storage_proxy_coordinator_cas_read_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]]) + 1)", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "LWT Avrage Read latency", + "type": "timeseries" + }, + { + "class": "us_panel", + "datasource": "Prometheus", + "description": "LWT 95% Read latency.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 105 + }, + "id": 68, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "casrlatencyp95{by=\"[[by]]\", instance=~\"[[node]]|^$\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"} or on() (histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_cas_read_latency_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"statement|$\"}[$__rate_interval])) by ([[by]], le)))", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "95% latency", + "type": "timeseries" + }, + { + "class": "rps_panel", + "datasource": "Prometheus", + "description": "LWT Read Timeouts", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:reads/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 105 + }, + "id": 69, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_cas_read_timeouts{instance=~\"[[node]]|^$\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"statement|$\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "LWT Read Timeouts", + "type": "timeseries" + }, + { + "class": "wps_panel", + "datasource": "Prometheus", + "description": "LWT write rate.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:writes/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 111 + }, + "id": 70, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_cas_write_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "LWT Writes", + "type": "timeseries" + }, + { + "class": "us_panel", + "datasource": "Prometheus", + "description": "LWT Avrage Write latency.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 111 + }, + "id": 71, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_cas_write_latency_sum{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]])/($func(rate(scylla_storage_proxy_coordinator_cas_write_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]]) + 1)", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "LWT Avrage Write latency", + "type": "timeseries" + }, + { + "class": "us_panel", + "datasource": "Prometheus", + "description": "LWT 95% write latency.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 111 + }, + "id": 72, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "caswlatencyp95{by=\"[[by]]\", instance=~\"[[node]]|^$\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"} or on() (histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_cas_write_latency_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"statement|$\"}[$__rate_interval])) by ([[by]], le)))", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "95% latency", + "type": "timeseries" + }, + { + "class": "wps_panel", + "datasource": "Prometheus", + "description": "LWT Write Timeouts", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:writes/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 111 + }, + "id": 73, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_cas_write_timeouts{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "LWT Write Timeouts", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "Prometheus", + "description": "A single Read/Write LWT will result in multiple paxos operations", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 117 + }, + "id": 74, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_cas_total_operations{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "Paxos operations", + "type": "timeseries" + }, + { + "class": "graph_panel", + "datasource": "Prometheus", + "description": "How many paxos operations that did not yet produce a result are running", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 117 + }, + "id": 75, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(scylla_storage_proxy_coordinator_cas_foreground{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "Paxos Foreground operations", + "type": "timeseries" + }, + { + "class": "graph_panel", + "datasource": "Prometheus", + "description": "How many paxos operations are still running after a result was alredy returned", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 117 + }, + "id": 76, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(scylla_storage_proxy_coordinator_cas_background{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "Paxos Background operations", + "type": "timeseries" + }, + { + "class": "text_panel", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 117 + }, + "id": 77, + "isNew": true, + "links": [], + "mode": "html", + "options": { + "content": "# " + }, + "span": 3, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "wps_panel", + "datasource": "Prometheus", + "description": "An LWT INSERT, UPDATE or DELETE command that involves a condition will be rejected if the condition is not met.\n\nWhile it is ok, a high value may indicate that there is a potential problem with data distribution", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:writes/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 123 + }, + "id": 78, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_cas_write_condition_not_met{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "LWT Condition-Not-Met", + "type": "timeseries" + }, + { + "class": "wps_panel", + "datasource": "Prometheus", + "description": "Number of times some INSERT, UPDATE or DELETE request with conditions had to retry because there was a concurrent conditional statement against the same key. Each retry is performed after a randomized sleep interval, so it can lead to statement timing out completely.\n\nIt can indicates contention over a hot row or key", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:writes/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 123 + }, + "id": 79, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_cas_write_contention_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "LWT Write Contention", + "type": "timeseries" + }, + { + "class": "rps_panel", + "datasource": "Prometheus", + "description": "Number of times some SELECT with SERIAL consistency had to retry because there was a concurrent conditional statement against the same key. Each retry is performed after a randomized sleep interval, so it can lead to statement timing out completely.\n\nIt can indicates contention over a hot row or key", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:reads/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 123 + }, + "id": 80, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_cas_read_contention_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]]) - $func(rate(scylla_storage_proxy_coordinator_cas_read_contention_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\", le=\"1.000000\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "LWT Read Contention", + "type": "timeseries" + }, + { + "class": "wps_panel", + "datasource": "Prometheus", + "description": "Number of partially succeeded conditional statements. These statements were not committed by the coordinator, due to some replicas responding with errors or timing out. The coordinator had to propagate the error to the client. However, the statement succeeded on a minority of replicas, so may later be propagated to the rest during repair.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:writes/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 123 + }, + "id": 81, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_cas_write_timeout_due_to_uncertainty{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "LWT Write Timeout Due to Uncertainty", + "type": "timeseries" + }, + { + "class": "wps_panel", + "datasource": "Prometheus", + "description": "Number of times a INSERT, UPDATE, or DELETE with conditions failed after being unable to contact enough replicas to match the consistency level", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:writes/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 129 + }, + "id": 82, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_cas_write_unavailable{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "LWT Write Unavailable", + "type": "timeseries" + }, + { + "class": "rps_panel", + "datasource": "Prometheus", + "description": "Number of times a SELECT with SERIAL consistency failed after being unable to contact enough replicas to match the consistency level", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:reads/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 129 + }, + "id": 83, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_cas_read_unavailable{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "LWT Read Unavailable", + "type": "timeseries" + }, + { + "class": "wps_panel", + "datasource": "Prometheus", + "description": "Number of Paxos-repairs of INSERT, UPDATE, or DELETE with conditions.\n\nA repair is necessary when a previous Paxos statement was partialy successful. A subsequent statement then may not proceed before completing the work of its predecessor. A repair is not guaranteed to succeed, the metric indicates the number of repair attempts made", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:writes/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 129 + }, + "id": 84, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_cas_write_unfinished_commit{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "LWT Write Unfinished - Repair Attempts", + "type": "timeseries" + }, + { + "class": "rps_panel", + "datasource": "Prometheus", + "description": "Number of Paxos-repairs of SELECT statement with SERIAL consistency.\n\nA repair is necessary when a previous Paxos statement was partialy successful. A subsequent statement then may not proceed before completing the work of its predecessor. A repair is not guaranteed to succeed, the metric indicates the number of repair attempts made", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:reads/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 129 + }, + "id": 85, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_cas_read_unfinished_commit{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "LWT Read Unfinished - Repair Attempts", + "type": "timeseries" + }, + { + "class": "rps_panel", + "datasource": "Prometheus", + "description": "Normally, a PREPARE Paxos-round piggy-backs the previous value along with the PREPARE response. When the coordinator is unable to obtain the previous value (or its digest) from some of the participants, or when the digests did not match, a separate repair round has to be performed.\n\nThis indicates that some Paxos queries did not run successfully to completion, e.g. because some node is overloaded, down, or there was contention around a key.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:reads/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 135 + }, + "id": 86, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_cas_failed_read_round_optimization{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "Failed Read-Round Optimization", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "Prometheus", + "description": "Number of pruning requests.\n\nA successful conditional statement deletes the intermediate state from system.paxos table using PRUNE command.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 135 + }, + "id": 87, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_cas_prune{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "LWT Prune", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "Prometheus", + "description": "Number of Dropped pruning requests.\n\nA successful conditional statement deletes the intermediate state from system.paxos table using PRUNE command. If the system is busy it may not keep up with the PRUNE requests, so such requests are dropped.\n\nHigh value suggests the system is overloaded and also that system.paxos table is taking up space. If a prune is dropped, system.paxos table key and value for respective LWT transaction will stay around until next transaction against the same key or until the gc_grace_period, when it's removed by compaction.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 135 + }, + "id": 88, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_cas_dropped_prune{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "LWT Dropped Prune", + "type": "timeseries" + } + ], + "repeat": "", + "title": "LWT", + "type": "row" + }, + { + "class": "collapsible_row_panel", + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 141 + }, + "id": 89, + "panels": [ + { + "class": "text_panel", + "content": "

CDC - Replica

", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 142 + }, + "id": 90, + "isNew": true, + "links": [], + "mode": "html", + "options": {}, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "ops_panel", + "datasource": "Prometheus", + "description": "The rate of CDC operations.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 144 + }, + "id": 91, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 4, + "targets": [ + { + "expr": "$func(rate(scylla_cdc_operations_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "CDC Operations", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "Prometheus", + "description": "The rate of failed CDC operations.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 144 + }, + "id": 92, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 4, + "targets": [ + { + "expr": "$func(rate(scylla_cdc_operations_failed{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]])/($func(rate(scylla_storage_proxy_coordinator_cas_read_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]]) + 1)", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 10 + } + ], + "title": "Failed CDC operations", + "type": "timeseries" + } + ], + "repeat": "", + "title": "CDC", + "type": "row" + }, + { + "class": "collapsible_row_panel", + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 150 + }, + "id": 93, + "panels": [], + "repeat": "", + "title": "Memory", + "type": "row" + }, + { + "class": "text_panel", + "content": "

Memory - Replica

", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 151 + }, + "id": 94, + "isNew": true, + "links": [], + "mode": "html", + "options": {}, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "bytes_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 153 + }, + "id": 95, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 6, + "targets": [ + { + "expr": "$func(scylla_lsa_total_space_bytes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "LSA total memory", + "type": "timeseries" + }, + { + "class": "bytes_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 153 + }, + "id": 96, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 6, + "targets": [ + { + "expr": "$func(scylla_lsa_non_lsa_used_space_bytes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Non-LSA used memory", + "type": "timeseries" + }, + { + "class": "collapsible_row_panel", + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 159 + }, + "id": 97, + "panels": [], + "repeat": "", + "title": "Compaction", + "type": "row" + }, + { + "class": "text_panel", + "content": "

Compaction - Replica

", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 160 + }, + "id": 98, + "isNew": true, + "links": [], + "mode": "html", + "options": {}, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "graph_panel_int", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 162 + }, + "id": 99, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 4, + "targets": [ + { + "expr": "$func(scylla_compaction_manager_compactions{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "Running Compactions", + "type": "timeseries" + }, + { + "class": "percent_panel", + "datasource": "Prometheus", + "description": "Percentage of CPU time used by compaction", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 162 + }, + "id": 100, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 4, + "targets": [ + { + "expr": "($func(rate(scylla_scheduler_runtime_ms{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\", group=\"compaction\"}[$__rate_interval])) by ([[by]]))/10", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "Compactions CPU Runtime", + "type": "timeseries" + }, + { + "class": "graph_panel", + "datasource": "Prometheus", + "description": "Shares assigned to the compaction", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 162 + }, + "id": 101, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 4, + "targets": [ + { + "expr": "avg(scylla_scheduler_shares{group=\"compaction\", instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "title": "Compactions Shares", + "type": "timeseries" + }, + { + "class": "collapsible_row_panel", + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 168 + }, + "id": 102, + "panels": [ + { + "class": "wps_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:writes/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 169 + }, + "id": 103, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_write_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[60s])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Writes by [[by]]", + "type": "timeseries" + }, + { + "class": "us_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 169 + }, + "id": 104, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "wlatencya{by=\"[[by]]\", instance=~\"[[node]]|$^\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$scheduling_group\"} or on() ($func(rate(scylla_storage_proxy_coordinator_write_latency_sum{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[60s])) by ([[by]])/($func(rate(scylla_storage_proxy_coordinator_write_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[60s])) by ([[by]]) + 1))", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Average write latency by [[by]]", + "type": "timeseries" + }, + { + "class": "us_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 169 + }, + "id": 105, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "wlatencyp95{by=\"[[by]]\", instance=~\"[[node]]|^$\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$scheduling_group|$\"} or on() (histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_write_latency_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$scheduling_group\"}[60s])) by ([[by]], le)))", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "95th percentile write latency by [[by]]", + "type": "timeseries" + }, + { + "class": "us_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 169 + }, + "id": 106, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "wlatencyp99{by=\"[[by]]\", instance=~\"[[node]]|^$\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$scheduling_group\"} or on() (histogram_quantile(0.99, sum(rate(scylla_storage_proxy_coordinator_write_latency_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$scheduling_group\"}[60s])) by ([[by]], le)))", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "99th percentile write latency by [[by]]", + "type": "timeseries" + }, + { + "class": "rps_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:reads/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 175 + }, + "id": 107, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_read_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[60s])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Reads by [[by]]", + "type": "timeseries" + }, + { + "class": "us_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 175 + }, + "id": 108, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "rlatencya{by=\"[[by]]\", instance=~\"[[node]]|$^\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$scheduling_group\"} or on() ($func(rate(scylla_storage_proxy_coordinator_read_latency_sum{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$scheduling_group\"}[60s])) by ([[by]])/($func(rate(scylla_storage_proxy_coordinator_read_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[60s])) by ([[by]]) + 1))", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Average read latency by [[by]]", + "type": "timeseries" + }, + { + "class": "us_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 175 + }, + "id": 109, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "rlatencyp95{by=\"[[by]]\", instance=~\"[[node]]|^$\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$scheduling_group\"} or on() (histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_read_latency_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$scheduling_group\"}[60s])) by ([[by]], le)))", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "95th percentile read latency by [[by]]", + "type": "timeseries" + }, + { + "class": "us_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 175 + }, + "id": 110, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "rlatencyp99{by=\"[[by]]\", instance=~\"[[node]]|^$\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$scheduling_group\"} or on() (histogram_quantile(0.99, sum(rate(scylla_storage_proxy_coordinator_read_latency_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$scheduling_group\"}[60s])) by ([[by]], le)))", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "99th percentile read latency by [[by]]", + "type": "timeseries" + } + ], + "repeat": "scheduling_group", + "title": "Latencies - $scheduling_group", + "type": "row" + }, + { + "class": "collapsible_row_panel", + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 181 + }, + "id": 111, + "panels": [], + "repeat": "", + "title": "", + "type": "row" + }, + { + "class": "plain_text", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 182 + }, + "id": 112, + "isNew": true, + "links": [], + "mode": "html", + "options": { + "content": "

Your Panels

", + "mode": "html" + }, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "user_panel", + "datasource": "Prometheus", + "description": "This graph panel was left empty on purpose for ad-hoc usage. Change it when needed. Pay attention that changes to the panel will not be saved.\n\nIf you do need a panel that can be saved, create a new dashboard, or edit the panel inside the json file", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 184 + }, + "id": 113, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 6, + "title": "Your Graph here", + "type": "timeseries" + }, + { + "class": "user_panel", + "datasource": "Prometheus", + "description": "This graph panel was left empty on purpose for ad-hoc usage. Change it when needed. Pay attention that changes to the panel will not be saved.\n\nIf you do need a panel that can be saved, create a new dashboard, or edit the panel inside the json file", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 184 + }, + "id": 114, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 6, + "title": "Your Graph here", + "type": "timeseries" + }, + { + "class": "plain_text", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 190 + }, + "id": 115, + "isNew": true, + "links": [], + "mode": "html", + "options": { + "content": "
Scylla Monitoring version - master
\n    
", + "mode": "html" + }, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + } + ], + "refresh": "30s", + "schemaVersion": 26, + "style": "dark", + "tags": [ + "4.4" + ], + "templating": { + "list": [ + { + "allValue": null, + "class": "by_template_var", + "current": { + "tags": [], + "text": "Instance", + "value": "instance" + }, + "error": null, + "hide": 0, + "includeAll": false, + "label": "by", + "multi": false, + "name": "by", + "options": [ + { + "selected": false, + "text": "Cluster", + "value": "cluster" + }, + { + "selected": false, + "text": "DC", + "value": "dc" + }, + { + "selected": true, + "text": "Instance", + "value": "instance" + }, + { + "selected": false, + "text": "instance,shard", + "value": "instance,shard" + } + ], + "query": "Cluster,DC,Instance,Shard", + "skipUrlSync": false, + "type": "custom" + }, + { + "allValue": null, + "class": "template_variable_single", + "current": { + "isNone": true, + "selected": false, + "text": "None", + "value": "" + }, + "datasource": "Prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [], + "query": "label_values(scylla_reactor_utilization, cluster)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "class": "template_variable_all", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "Prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": true, + "label": "dc", + "multi": true, + "name": "dc", + "options": [], + "query": "label_values(scylla_reactor_utilization{cluster=~\"$cluster\"}, dc)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "class": "template_variable_all", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "Prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": true, + "label": "node", + "multi": true, + "name": "node", + "options": [], + "query": "label_values(scylla_reactor_utilization{cluster=~\"$cluster|$^\", dc=~\"$dc\"}, instance)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "class": "template_variable_all", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "Prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": true, + "label": "shard", + "multi": true, + "name": "shard", + "options": [], + "query": "label_values(scylla_reactor_utilization,shard)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 3, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "class": "template_variable_all", + "current": { + "selected": true, + "tags": [], + "text": [ + "statement" + ], + "value": [ + "statement" + ] + }, + "dashversion": [ + ">4.3" + ], + "datasource": "Prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": true, + "label": "SG", + "multi": true, + "name": "scheduling_group", + "options": [], + "query": "label_values(scylla_storage_proxy_coordinator_read_latency_bucket,scheduling_group_name)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 3, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "class": "aggregation_function", + "current": { + "tags": [], + "text": "sum", + "value": "sum" + }, + "hide": 0, + "includeAll": false, + "label": "Function", + "multi": false, + "name": "func", + "options": [ + { + "selected": true, + "text": "sum", + "value": "sum" + }, + { + "selected": false, + "text": "avg", + "value": "avg" + }, + { + "selected": false, + "text": "max", + "value": "max" + }, + { + "selected": false, + "text": "min", + "value": "min" + }, + { + "selected": false, + "text": "stddev", + "value": "stddev" + }, + { + "selected": false, + "text": "stdvar", + "value": "stdvar" + } + ], + "query": "sum,avg,max,min,stddev,stdvar", + "skipUrlSync": false, + "type": "custom" + }, + { + "class": "adhoc_filter", + "datasource": "Prometheus", + "error": null, + "filters": [], + "hide": 0, + "label": "ad hoc", + "name": "adhoc", + "skipUrlSync": false, + "type": "adhoc" + }, + { + "allValue": null, + "class": "template_variable_custom", + "current": { + "text": "4.4", + "value": "4.4" + }, + "error": null, + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "scylla_version", + "options": [ + { + "selected": true, + "text": "4.4", + "value": "4.4" + } + ], + "query": "4.4", + "skipUrlSync": false, + "type": "custom" + }, + { + "allValue": null, + "class": "monitor_version_var", + "current": { + "text": "master", + "value": "master" + }, + "error": null, + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "monitoring_version", + "options": [ + { + "selected": true, + "text": "master", + "value": "master" + } + ], + "query": "master", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": { + "now": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "utc", + "title": "Detailed", + "uid": "detailed-4-4", + "version": 5 +} diff --git a/sysadmin/grid5000/cassandra/monitoring/grafana/dashboards/scylla/scylla-ks.4.4.json b/sysadmin/grid5000/cassandra/monitoring/grafana/dashboards/scylla/scylla-ks.4.4.json new file mode 100644 index 0000000..8c5acb5 --- /dev/null +++ b/sysadmin/grid5000/cassandra/monitoring/grafana/dashboards/scylla/scylla-ks.4.4.json @@ -0,0 +1,1404 @@ +{ + "annotations": { + "class": "default_annotations", + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "class": "annotation_restart", + "datasource": "Prometheus", + "enable": true, + "expr": "resets(scylla_gossip_heart_beat[1m])>0", + "hide": false, + "iconColor": "rgba(255, 96, 96, 1)", + "limit": 100, + "name": "node_restart", + "showIn": 0, + "tagKeys": "instance,dc,cluster", + "tags": [], + "titleFormat": "restart", + "type": "tags" + }, + { + "class": "annotation_stall", + "datasource": "Prometheus", + "enable": false, + "expr": "changes(scylla_stall_detector_reported[1m])>0", + "hide": false, + "iconColor": "rgba(255, 96, 96, 1)", + "limit": 100, + "name": "stall detector", + "showIn": 0, + "tagKeys": "dc,instance,shard", + "tags": [], + "titleFormat": "Stall found", + "type": "tags" + }, + { + "class": "annotation_schema_changed", + "datasource": "Prometheus", + "enable": false, + "expr": "changes(scylla_database_schema_changed[30s])>0", + "hide": false, + "iconColor": "rgba(255, 96, 96, 1)", + "limit": 100, + "name": "Schema Changed", + "showIn": 0, + "tagKeys": "instance,dc,cluster", + "tags": [], + "titleFormat": "schema changed", + "type": "tags" + }, + { + "class": "annotation_manager_task", + "datasource": "Prometheus", + "enable": true, + "expr": "scylla_manager_task_active_count{type=~\"repair|backup\",cluster=~\"$cluster|$^\"}>0", + "hide": false, + "iconColor": "#73BF69", + "limit": 100, + "name": "Task", + "showIn": 0, + "tagKeys": "type", + "tags": [], + "titleFormat": "Running", + "type": "tags" + }, + { + "class": "annotation_hints_writes", + "datasource": "Prometheus", + "enable": false, + "expr": "changes(scylla_hints_manager_written[60s])>0", + "hide": false, + "iconColor": "rgb(255, 176, 0, 128)", + "limit": 100, + "name": "Hints Write", + "showIn": 0, + "tagKeys": "instance,dc,cluster", + "tags": [], + "titleFormat": "Hints write", + "type": "tags" + }, + { + "class": "annotation_hints_sent", + "datasource": "Prometheus", + "enable": false, + "expr": "changes(scylla_hints_manager_written[60s])>0", + "hide": false, + "iconColor": "rgb(50, 176, 0, 128)", + "limit": 100, + "name": "Hints Sent", + "showIn": 0, + "tagKeys": "instance,dc,cluster", + "tags": [], + "titleFormat": "Hints Sent", + "type": "tags" + }, + { + "class": "mv_building", + "datasource": "Prometheus", + "enable": true, + "expr": "sum(scylla_view_builder_builds_in_progress)>0", + "hide": false, + "iconColor": "rgb(50, 176, 0, 128)", + "limit": 100, + "name": "MV", + "showIn": 0, + "tagKeys": "instance,dc,cluster", + "tags": [], + "titleFormat": "Materialized View built", + "type": "tags" + } + ] + }, + "class": "dashboard", + "editable": true, + "gnetId": null, + "graphTooltip": 1, + "hideControls": true, + "id": null, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [], + "type": "dashboards" + } + ], + "overwrite": true, + "panels": [ + { + "class": "text_panel", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 1, + "isNew": true, + "links": [], + "mode": "html", + "options": { + "content": "
\n\n [[cluster]]
", + "mode": "html" + }, + "span": 12, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "collapsible_row_panel", + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 2, + "panels": [ + { + "class": "wps_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:writes/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 4 + }, + "id": 3, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_column_family_write_latency_count{ks=\"$ks\", cf=\"$table\", instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[60s])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Writes by [[by]]", + "type": "timeseries" + }, + { + "class": "us_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 4 + }, + "id": 4, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "wlatencyaks{ks=\"$ks\", cf=\"$table\", by=\"[[by]]\", instance=~\"[[node]]|$^\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Average write latency by [[by]]", + "type": "timeseries" + }, + { + "class": "us_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 4 + }, + "id": 5, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "wlatencyp95ks{ks=\"$ks\", cf=\"$table\", by=\"[[by]]\", instance=~\"[[node]]|^$\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$scheduling_group|$\"} ", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "95th percentile write latency by [[by]]", + "type": "timeseries" + }, + { + "class": "us_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 4 + }, + "id": 6, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "wlatencyp99ks{ks=\"$ks\", cf=\"$table\", by=\"[[by]]\", instance=~\"[[node]]|^$\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"} ", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "99th percentile write latency by [[by]]", + "type": "timeseries" + }, + { + "class": "rps_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:reads/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 10 + }, + "id": 7, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_column_family_read_latency_count{ks=\"$ks\", cf=\"$table\", instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[60s])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Reads by [[by]]", + "type": "timeseries" + }, + { + "class": "us_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 10 + }, + "id": 8, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "rlatencyaks{ks=\"$ks\", cf=\"$table\", by=\"[[by]]\", instance=~\"[[node]]|$^\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Average read latency by [[by]]", + "type": "timeseries" + }, + { + "class": "us_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 10 + }, + "id": 9, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "rlatencyp95ks{ks=\"$ks\", cf=\"$table\", by=\"[[by]]\", instance=~\"[[node]]|^$\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"} ", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "95th percentile read latency by [[by]]", + "type": "timeseries" + }, + { + "class": "us_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 10 + }, + "id": 10, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "rlatencyp99ks{ks=\"$ks\", cf=\"$table\", by=\"[[by]]\", instance=~\"[[node]]|^$\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"} ", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "99th percentile read latency by [[by]]", + "type": "timeseries" + } + ], + "repeat": "table", + "title": "Latencies - $ks:$table", + "type": "row" + }, + { + "class": "collapsible_row_panel", + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 11, + "panels": [], + "repeat": "", + "title": "", + "type": "row" + }, + { + "class": "plain_text", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 12, + "isNew": true, + "links": [], + "mode": "html", + "options": { + "content": "

Your Panels

", + "mode": "html" + }, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "user_panel", + "datasource": "Prometheus", + "description": "This graph panel was left empty on purpose for ad-hoc usage. Change it when needed. Pay attention that changes to the panel will not be saved.\n\nIf you do need a panel that can be saved, create a new dashboard, or edit the panel inside the json file", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 19 + }, + "id": 13, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 6, + "title": "Your Graph here", + "type": "timeseries" + }, + { + "class": "user_panel", + "datasource": "Prometheus", + "description": "This graph panel was left empty on purpose for ad-hoc usage. Change it when needed. Pay attention that changes to the panel will not be saved.\n\nIf you do need a panel that can be saved, create a new dashboard, or edit the panel inside the json file", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 19 + }, + "id": 14, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 6, + "title": "Your Graph here", + "type": "timeseries" + }, + { + "class": "plain_text", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 15, + "isNew": true, + "links": [], + "mode": "html", + "options": { + "content": "
Scylla Monitoring version - master
\n    
", + "mode": "html" + }, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + } + ], + "refresh": "30s", + "schemaVersion": 26, + "style": "dark", + "tags": [ + "4.4" + ], + "templating": { + "list": [ + { + "allValue": null, + "class": "by_template_var", + "current": { + "tags": [], + "text": "Instance", + "value": "instance" + }, + "error": null, + "hide": 0, + "includeAll": false, + "label": "by", + "multi": false, + "name": "by", + "options": [ + { + "selected": false, + "text": "Cluster", + "value": "cluster" + }, + { + "selected": false, + "text": "DC", + "value": "dc" + }, + { + "selected": true, + "text": "Instance", + "value": "instance" + }, + { + "selected": false, + "text": "instance,shard", + "value": "instance,shard" + } + ], + "query": "Cluster,DC,Instance,Shard", + "skipUrlSync": false, + "type": "custom" + }, + { + "allValue": null, + "class": "template_variable_single", + "current": { + "isNone": true, + "selected": false, + "text": "None", + "value": "" + }, + "datasource": "Prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [], + "query": "label_values(scylla_reactor_utilization, cluster)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "class": "template_variable_all", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "Prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": true, + "label": "dc", + "multi": true, + "name": "dc", + "options": [], + "query": "label_values(scylla_reactor_utilization{cluster=~\"$cluster\"}, dc)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "class": "template_variable_all", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "Prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": true, + "label": "node", + "multi": true, + "name": "node", + "options": [], + "query": "label_values(scylla_reactor_utilization{cluster=~\"$cluster|$^\", dc=~\"$dc\"}, instance)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "class": "template_variable_all", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "Prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": true, + "label": "shard", + "multi": true, + "name": "shard", + "options": [], + "query": "label_values(scylla_reactor_utilization,shard)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 3, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "class": "template_variable_single", + "current": { + "isNone": true, + "selected": false, + "text": "None", + "value": "" + }, + "datasource": "Prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": false, + "label": "ks", + "multi": false, + "name": "ks", + "options": [], + "query": "label_values(scylla_column_family_write_latency_count{cluster=~\"$cluster|$^\"},ks)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 3, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "class": "template_variable_all", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "Prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": true, + "label": "table", + "multi": true, + "name": "table", + "options": [], + "query": "label_values(scylla_column_family_write_latency_count{cluster=~\"$cluster|$^\", ks=\"$ks\"},cf)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 3, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "class": "aggregation_function", + "current": { + "tags": [], + "text": "sum", + "value": "sum" + }, + "hide": 0, + "includeAll": false, + "label": "Function", + "multi": false, + "name": "func", + "options": [ + { + "selected": true, + "text": "sum", + "value": "sum" + }, + { + "selected": false, + "text": "avg", + "value": "avg" + }, + { + "selected": false, + "text": "max", + "value": "max" + }, + { + "selected": false, + "text": "min", + "value": "min" + }, + { + "selected": false, + "text": "stddev", + "value": "stddev" + }, + { + "selected": false, + "text": "stdvar", + "value": "stdvar" + } + ], + "query": "sum,avg,max,min,stddev,stdvar", + "skipUrlSync": false, + "type": "custom" + }, + { + "allValue": null, + "class": "template_variable_custom", + "current": { + "text": "4.4", + "value": "4.4" + }, + "error": null, + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "scylla_version", + "options": [ + { + "selected": true, + "text": "4.4", + "value": "4.4" + } + ], + "query": "4.4", + "skipUrlSync": false, + "type": "custom" + }, + { + "allValue": null, + "class": "monitor_version_var", + "current": { + "text": "master", + "value": "master" + }, + "error": null, + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "monitoring_version", + "options": [ + { + "selected": true, + "text": "master", + "value": "master" + } + ], + "query": "master", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": { + "now": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "utc", + "title": "Keyspace", + "uid": "ks-4-4", + "version": 5 +} diff --git a/sysadmin/grid5000/cassandra/monitoring/grafana/dashboards/scylla/scylla-os.4.4.json b/sysadmin/grid5000/cassandra/monitoring/grafana/dashboards/scylla/scylla-os.4.4.json new file mode 100644 index 0000000..1f2718a --- /dev/null +++ b/sysadmin/grid5000/cassandra/monitoring/grafana/dashboards/scylla/scylla-os.4.4.json @@ -0,0 +1,2091 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "class": "annotation_restart", + "datasource": "Prometheus", + "enable": true, + "expr": "resets(scylla_gossip_heart_beat[1m])>0", + "hide": false, + "iconColor": "rgba(255, 96, 96, 1)", + "limit": 100, + "name": "node_restart", + "showIn": 0, + "tagKeys": "instance,dc,cluster", + "tags": [], + "titleFormat": "restart", + "type": "tags" + }, + { + "class": "annotation_stall", + "datasource": "Prometheus", + "enable": false, + "expr": "changes(scylla_stall_detector_reported[1m])>0", + "hide": false, + "iconColor": "rgba(255, 96, 96, 1)", + "limit": 100, + "name": "stall detector", + "showIn": 0, + "tagKeys": "dc,instance,shard", + "tags": [], + "titleFormat": "Stall found", + "type": "tags" + }, + { + "class": "annotation_schema_changed", + "datasource": "Prometheus", + "enable": false, + "expr": "changes(scylla_database_schema_changed[30s])>0", + "hide": false, + "iconColor": "rgba(255, 96, 96, 1)", + "limit": 100, + "name": "Schema Changed", + "showIn": 0, + "tagKeys": "instance,dc,cluster", + "tags": [], + "titleFormat": "schema changed", + "type": "tags" + } + ] + }, + "class": "dashboard", + "editable": true, + "gnetId": null, + "graphTooltip": 1, + "hideControls": true, + "id": null, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [], + "type": "dashboards" + } + ], + "overwrite": true, + "panels": [ + { + "class": "collapsible_row_panel", + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "repeat": "", + "title": "", + "type": "row" + }, + { + "class": "text_panel", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 2, + "isNew": true, + "links": [], + "mode": "html", + "options": { + "content": "
\n\n [[cluster]]
", + "mode": "html" + }, + "span": 12, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "collapsible_row_panel", + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 4 + }, + "id": 3, + "panels": [], + "repeat": "", + "title": "", + "type": "row" + }, + { + "aliasColors": {}, + "cacheTimeout": null, + "class": "pie_chart_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fontSize": "80%", + "format": "bytes", + "gridPos": { + "h": 8, + "w": 4, + "x": 0, + "y": 5 + }, + "height": "250px", + "id": 4, + "interval": null, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "legendType": "On graph", + "links": [], + "maxDataPoints": 3, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "pieType": "pie", + "repeat": "node", + "span": 2, + "strokeWidth": 1, + "targets": [ + { + "expr": "sum(node_filesystem_avail_bytes{mountpoint=\"$mount_point\", instance=~\"$node\"})", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "Free", + "metric": "", + "refId": "A", + "step": 7200 + }, + { + "expr": "(sum(node_filesystem_size_bytes{mountpoint=\"$mount_point\", instance=~\"$node\"})-sum(node_filesystem_avail_bytes{mountpoint=\"$mount_point\", instance=~\"$node\"}))", + "instant": true, + "intervalFactor": 1, + "legendFormat": "Used", + "refId": "B", + "step": 7200 + } + ], + "title": "Total Storage $node", + "type": "grafana-piechart-panel", + "valueName": "current" + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 13 + }, + "id": 5, + "panels": [], + "repeat": "mount_point", + "title": "Partition $mount_point", + "type": "row" + }, + { + "class": "percentunit_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 14 + }, + "id": 6, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "1-sum(node_filesystem_avail_bytes{mountpoint=\"$mount_point\", instance=~\"$node\"}) by ([[by]])/sum(node_filesystem_size_bytes{mountpoint=\"$mount_point\", instance=~\"$node\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "Used disk by $by", + "type": "timeseries" + }, + { + "class": "bytes_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 14 + }, + "id": 7, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "sum(node_filesystem_size_bytes{mountpoint=\"$mount_point\", instance=~\"$node\"}) by ([[by]])-sum(node_filesystem_avail_bytes{mountpoint=\"$mount_point\", instance=~\"$node\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + }, + { + "expr": "sum(node_filesystem_size{mountpoint=\"$mount_point\", instance=~\"$node\"}) by ([[by]])-sum(node_filesystem_avail{mountpoint=\"$mount_point\", instance=~\"$node\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "B", + "step": 1 + } + ], + "title": "Used Bytes by $by", + "type": "timeseries" + }, + { + "class": "bytes_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 14 + }, + "id": 8, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "sum(node_filesystem_free_bytes{mountpoint=\"$mount_point\", instance=~\"$node\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "Free Bytes by $by", + "type": "timeseries" + }, + { + "class": "graph_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 14 + }, + "id": 9, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "sum(node_filesystem_files{mountpoint=\"$mount_point\", instance=~\"$node\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "Number of files by $by", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 10, + "panels": [], + "repeat": "monitor_disk", + "title": "Disk $monitor_disk", + "type": "row" + }, + { + "class": "text_panel", + "content": "

Disk $monitor_disk

", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 11, + "isNew": true, + "links": [], + "mode": "html", + "options": {}, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "wps_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:writes/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 23 + }, + "id": 12, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "sum(rate(node_disk_writes_completed_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_disk\"}[4m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + }, + { + "expr": "sum(rate(node_disk_writes_completed{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_disk\"}[4m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "B", + "step": 1 + } + ], + "title": "Disk Writes per $by", + "type": "timeseries" + }, + { + "class": "rps_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:reads/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 23 + }, + "id": 13, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "sum(rate(node_disk_reads_completed_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_disk\"}[4m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + }, + { + "expr": "sum(rate(node_disk_reads_completed{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_disk\"}[4m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "B", + "step": 1 + } + ], + "title": "Disk Reads per $by", + "type": "timeseries" + }, + { + "class": "bps_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 23 + }, + "id": 14, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "sum(rate(node_disk_written_bytes_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_disk\"}[4m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + }, + { + "expr": "sum(rate(node_disk_bytes_written{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_disk\"}[4m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "B", + "step": 1 + } + ], + "title": "Disk Writes Bps per $by", + "type": "timeseries" + }, + { + "class": "bps_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 23 + }, + "id": 15, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "sum(rate(node_disk_read_bytes_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_disk\"}[4m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + }, + { + "expr": "sum(rate(node_disk_bytes_read{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_disk\"}[4m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "B", + "step": 1 + } + ], + "title": "Disk Read Bps per $by", + "type": "timeseries" + }, + { + "class": "seconds_panel", + "datasource": "Prometheus", + "description": "The average read time from disk", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 29 + }, + "id": 16, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "rate(node_disk_read_time_seconds_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_disk\"}[4m])/rate(node_disk_reads_completed_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_disk\"}[4m])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Read AWait per instance", + "type": "timeseries" + }, + { + "class": "seconds_panel", + "datasource": "Prometheus", + "description": "The average write time to disk", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 29 + }, + "id": 17, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "rate(node_disk_write_time_seconds_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_disk\"}[4m])/rate(node_disk_writes_completed_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_disk\"}[4m])", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Write AWait per instance", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 35 + }, + "id": 18, + "panels": [], + "repeat": "monitor_network_interface", + "title": "Network Interface $monitor_network_interface", + "type": "row" + }, + { + "class": "text_panel", + "content": "

Network $monitor_network_interface

", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 36 + }, + "id": 19, + "isNew": true, + "links": [], + "mode": "html", + "options": {}, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "pps_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "pps" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 38 + }, + "id": 20, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 6, + "targets": [ + { + "expr": "sum(rate(node_network_receive_packets_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_network_interface\"}[4m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + }, + { + "expr": "sum(rate(node_network_receive_packets{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_network_interface\"}[4m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "B", + "step": 1 + } + ], + "title": "Interface Rx Packets", + "type": "timeseries" + }, + { + "class": "pps_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "pps" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 38 + }, + "id": 21, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 6, + "targets": [ + { + "expr": "sum(rate(node_network_transmit_packets_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_network_interface\"}[4m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + }, + { + "expr": "sum(rate(node_network_transmit_packets{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_network_interface\"}[4m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "B", + "step": 1 + } + ], + "title": "Interface Tx Packets", + "type": "timeseries" + }, + { + "class": "bps_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 44 + }, + "id": 22, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 6, + "targets": [ + { + "expr": "sum(rate(node_network_receive_bytes_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_network_interface\"}[4m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + }, + { + "expr": "sum(rate(node_network_receive_bytes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_network_interface\"}[4m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "B", + "step": 1 + } + ], + "title": "Interface Rx Bps", + "type": "timeseries" + }, + { + "class": "bps_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 44 + }, + "id": 23, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 6, + "targets": [ + { + "expr": "sum(rate(node_network_transmit_bytes_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_network_interface\"}[4m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + }, + { + "expr": "sum(rate(node_network_transmit_bytes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", device=\"$monitor_network_interface\"}[4m])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "B", + "step": 1 + } + ], + "title": "Interface Tx Bps", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 50 + }, + "id": 24, + "panels": [], + "title": "CPU and Memory", + "type": "row" + }, + { + "class": "bps_panel", + "datasource": "Prometheus", + "description": "The available memory, note that in a production environment we expect this to be low, Scylla would use most of the available memory when possible", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "Bps" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 51 + }, + "id": 25, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "sum(node_memory_MemAvailable_bytes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "Available memory", + "type": "timeseries" + }, + { + "class": "percentunit_panel", + "datasource": "Prometheus", + "description": "Percent of available memory, note that in a production environment we expect this to be low, Scylla would use most of the available memory when possible", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 51 + }, + "id": 26, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "sum(node_memory_MemAvailable_bytes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\"}) by ([[by]])/sum(node_memory_MemTotal_bytes{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "Available memory", + "type": "timeseries" + }, + { + "class": "percentunit_panel", + "datasource": "Prometheus", + "description": "Percent of CPU used, note that in production Scylla would try to use most of the CPU and this is not a problem", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 51 + }, + "id": 27, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 3, + "targets": [ + { + "expr": "1-sum(rate(node_cpu_seconds_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", mode=\"idle\"}[3m])) by ([[by]])/count(node_cpu_seconds_total{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", mode=\"idle\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "CPU used", + "type": "timeseries" + }, + { + "class": "graph_panel", + "datasource": "Prometheus", + "description": "CPU frequency should be set for performance.\n\n The current frequency should match the max frequency. If that is not the case, check your host configuration.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "links": [], + "unit": "hertz" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 51 + }, + "id": 28, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + { + "$$hashKey": "object:211", + "alias": "Max", + "color": "#F2495C" + } + ], + "span": 3, + "targets": [ + { + "expr": "max(node_cpu_scaling_frequency_max_hertz{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\"}) or on() max(node_cpu_frequency_max_hertz{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\"})", + "intervalFactor": 1, + "legendFormat": "Max", + "metric": "", + "refId": "A", + "step": 1 + }, + { + "expr": "min(node_cpu_scaling_frequency_hertz{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\"}) by ([[by]]) or on() min(node_cpu_frequency_hertz{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "B", + "step": 1 + } + ], + "title": "CPU Frequency", + "type": "timeseries" + }, + { + "class": "plain_text", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 57 + }, + "id": 29, + "isNew": true, + "links": [], + "mode": "html", + "options": { + "content": "
Scylla Monitoring version - master
\n    
", + "mode": "html" + }, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + } + ], + "refresh": "30s", + "schemaVersion": 26, + "style": "dark", + "tags": [ + "4.4" + ], + "templating": { + "list": [ + { + "allValue": null, + "class": "template_variable_single", + "current": { + "isNone": true, + "selected": false, + "text": "None", + "value": "" + }, + "datasource": "Prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [], + "query": "label_values(node_filesystem_avail_bytes, cluster)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "class": "template_variable_all", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "Prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": true, + "label": "dc", + "multi": true, + "name": "dc", + "options": [], + "query": "label_values(node_filesystem_avail_bytes{cluster=~\"$cluster\"}, dc)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "class": "template_variable_all", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "Prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": true, + "label": "node", + "multi": true, + "name": "node", + "options": [], + "query": "label_values(node_filesystem_avail_bytes{cluster=~\"$cluster|$^\", dc=~\"$dc\"}, instance)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "tags": [], + "text": "Instance", + "value": "instance" + }, + "hide": 0, + "includeAll": false, + "label": "by", + "multi": false, + "name": "by", + "options": [ + { + "selected": false, + "text": "Cluster", + "value": "cluster" + }, + { + "selected": false, + "text": "DC", + "value": "dc" + }, + { + "selected": true, + "text": "Instance", + "value": "instance" + } + ], + "query": "Cluster,DC,Instance", + "type": "custom" + }, + { + "allValue": null, + "current": { + "isNone": true, + "text": "None", + "value": "" + }, + "datasource": "Prometheus", + "hide": 0, + "includeAll": false, + "label": null, + "multi": true, + "name": "monitor_disk", + "options": [], + "query": "node_disk_read_bytes_total", + "refresh": 2, + "regex": "/.*device=\"([^\\\"]*)\".*/", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "isNone": true, + "text": "None", + "value": "" + }, + "datasource": "Prometheus", + "hide": 0, + "includeAll": false, + "label": null, + "multi": true, + "name": "monitor_network_interface", + "options": [], + "query": "node_network_receive_packets_total", + "refresh": 2, + "regex": "/.*device=\"([^\\\"]*)\".*/", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "/var/lib/scylla", + "value": "/var/lib/scylla" + }, + "datasource": "Prometheus", + "hide": 0, + "includeAll": false, + "label": "Mount path", + "multi": true, + "name": "mount_point", + "options": [], + "query": "node_filesystem_avail_bytes", + "refresh": 2, + "regex": "/mountpoint=\"([^\"]*)\".*/", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "class": "adhoc_filter", + "datasource": "Prometheus", + "error": null, + "filters": [], + "hide": 0, + "label": "ad hoc", + "name": "adhoc", + "skipUrlSync": false, + "type": "adhoc" + }, + { + "allValue": null, + "class": "template_variable_custom", + "current": { + "text": "4.4", + "value": "4.4" + }, + "error": null, + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "scylla_version", + "options": [ + { + "selected": true, + "text": "4.4", + "value": "4.4" + } + ], + "query": "4.4", + "skipUrlSync": false, + "type": "custom" + }, + { + "allValue": null, + "class": "monitor_version_var", + "current": { + "text": "master", + "value": "master" + }, + "error": null, + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "monitoring_version", + "options": [ + { + "selected": true, + "text": "master", + "value": "master" + } + ], + "query": "master", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": { + "now": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "utc", + "title": "OS Metrics", + "uid": "OS-4-4", + "version": 5 +} diff --git a/sysadmin/grid5000/cassandra/monitoring/grafana/dashboards/scylla/scylla-overview.4.4.json b/sysadmin/grid5000/cassandra/monitoring/grafana/dashboards/scylla/scylla-overview.4.4.json new file mode 100644 index 0000000..e44c3d2 --- /dev/null +++ b/sysadmin/grid5000/cassandra/monitoring/grafana/dashboards/scylla/scylla-overview.4.4.json @@ -0,0 +1,4491 @@ +{ + "annotations": { + "class": "default_annotations", + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "class": "annotation_restart", + "datasource": "Prometheus", + "enable": true, + "expr": "resets(scylla_gossip_heart_beat[1m])>0", + "hide": false, + "iconColor": "rgba(255, 96, 96, 1)", + "limit": 100, + "name": "node_restart", + "showIn": 0, + "tagKeys": "instance,dc,cluster", + "tags": [], + "titleFormat": "restart", + "type": "tags" + }, + { + "class": "annotation_stall", + "datasource": "Prometheus", + "enable": false, + "expr": "changes(scylla_stall_detector_reported[1m])>0", + "hide": false, + "iconColor": "rgba(255, 96, 96, 1)", + "limit": 100, + "name": "stall detector", + "showIn": 0, + "tagKeys": "dc,instance,shard", + "tags": [], + "titleFormat": "Stall found", + "type": "tags" + }, + { + "class": "annotation_schema_changed", + "datasource": "Prometheus", + "enable": false, + "expr": "changes(scylla_database_schema_changed[30s])>0", + "hide": false, + "iconColor": "rgba(255, 96, 96, 1)", + "limit": 100, + "name": "Schema Changed", + "showIn": 0, + "tagKeys": "instance,dc,cluster", + "tags": [], + "titleFormat": "schema changed", + "type": "tags" + }, + { + "class": "annotation_manager_task", + "datasource": "Prometheus", + "enable": true, + "expr": "scylla_manager_task_active_count{type=~\"repair|backup\",cluster=~\"$cluster|$^\"}>0", + "hide": false, + "iconColor": "#73BF69", + "limit": 100, + "name": "Task", + "showIn": 0, + "tagKeys": "type", + "tags": [], + "titleFormat": "Running", + "type": "tags" + }, + { + "class": "annotation_hints_writes", + "datasource": "Prometheus", + "enable": false, + "expr": "changes(scylla_hints_manager_written[60s])>0", + "hide": false, + "iconColor": "rgb(255, 176, 0, 128)", + "limit": 100, + "name": "Hints Write", + "showIn": 0, + "tagKeys": "instance,dc,cluster", + "tags": [], + "titleFormat": "Hints write", + "type": "tags" + }, + { + "class": "annotation_hints_sent", + "datasource": "Prometheus", + "enable": false, + "expr": "changes(scylla_hints_manager_written[60s])>0", + "hide": false, + "iconColor": "rgb(50, 176, 0, 128)", + "limit": 100, + "name": "Hints Sent", + "showIn": 0, + "tagKeys": "instance,dc,cluster", + "tags": [], + "titleFormat": "Hints Sent", + "type": "tags" + }, + { + "class": "mv_building", + "datasource": "Prometheus", + "enable": true, + "expr": "sum(scylla_view_builder_builds_in_progress)>0", + "hide": false, + "iconColor": "rgb(50, 176, 0, 128)", + "limit": 100, + "name": "MV", + "showIn": 0, + "tagKeys": "instance,dc,cluster", + "tags": [], + "titleFormat": "Materialized View built", + "type": "tags" + } + ] + }, + "class": "dashboard", + "editable": true, + "gnetId": null, + "graphTooltip": 1, + "hideControls": true, + "id": null, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [], + "type": "dashboards" + } + ], + "originalTitle": "Scylla Cluster Metrics", + "overwrite": true, + "panels": [ + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "panels": [], + "title": "Cluster overview $cluster", + "type": "row" + }, + { + "class": "text_panel", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 1, + "w": 9, + "x": 0, + "y": 1 + }, + "id": 2, + "isNew": true, + "links": [], + "mode": "html", + "options": { + "content": "# " + }, + "span": 12, + "title": "", + "transparent": false, + "type": "text" + }, + { + "class": "text_panel", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 1, + "w": 4, + "x": 9, + "y": 1 + }, + "id": 3, + "isNew": true, + "links": [], + "mode": "html", + "options": { + "content": "# " + }, + "span": 12, + "title": "", + "transparent": false, + "type": "text" + }, + { + "class": "text_panel", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 1, + "w": 4, + "x": 13, + "y": 1 + }, + "id": 4, + "isNew": true, + "links": [], + "mode": "html", + "options": { + "content": "# " + }, + "span": 12, + "title": "", + "transparent": false, + "type": "text" + }, + { + "class": "text_panel", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 1, + "w": 7, + "x": 17, + "y": 1 + }, + "id": 5, + "isNew": true, + "links": [], + "mode": "html", + "options": { + "content": "# " + }, + "span": 12, + "title": "", + "transparent": false, + "type": "text" + }, + { + "class": "small_stat", + "datasource": "Prometheus", + "description": "The number of nodes configured in the cluster.", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 0, + "y": 2 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "count(up{job=\"scylla\", cluster=~\"$cluster|$^\"})", + "intervalFactor": 1, + "legendFormat": "Total Nodes", + "refId": "A", + "step": 40 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "# Nodes", + "type": "stat" + }, + { + "class": "small_stat", + "datasource": "Prometheus", + "description": "The number of unreachable nodes.\nUsually because a machine is down or unreachable.", + "fieldConfig": { + "defaults": { + "custom": {}, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 1 + } + ] + } + } + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 2, + "y": 2 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "(count(scrape_samples_scraped{job=\"scylla\", cluster=~\"$cluster|$^\"}==0) OR vector(0))", + "intervalFactor": 1, + "legendFormat": "Offline ", + "refId": "A", + "step": 20 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Unreachable", + "type": "stat" + }, + { + "class": "small_stat", + "datasource": "Prometheus", + "description": "The number of joining and leaving nodes.\nThe number of nodes that are up but not actively part of the cluster, either because they are still joining or because they are leaving.", + "fieldConfig": { + "defaults": { + "custom": {}, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 1 + } + ] + } + } + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 4, + "y": 2 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "count(scylla_node_operation_mode!=3)OR vector(0)", + "intervalFactor": 1, + "legendFormat": "Offline ", + "refId": "A", + "step": 20 + } + ], + "thresholds": "1,2", + "timeFrom": null, + "timeShift": null, + "title": "Inactive", + "type": "stat" + }, + { + "class": "small_stat", + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "mappings": [ + { + "from": "", + "id": 0, + "text": "Backup", + "to": "", + "type": 1, + "value": "2" + }, + { + "from": "", + "id": 1, + "text": "Repair", + "to": "", + "type": 1, + "value": "1" + }, + { + "from": "", + "id": 2, + "text": "Online", + "to": "", + "type": 1, + "value": "0" + }, + { + "from": "", + "id": 2, + "text": "Offline", + "to": "", + "type": 1, + "value": "-1" + } + ], + "noValue": " Offline", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 0 + } + ] + } + } + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 6, + "y": 2 + }, + "id": 9, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "(sum(scylla_manager_task_active_count{type=~\"repair\",cluster=~\"$cluster|$^\"}) or on() vector(0)) + (sum(scylla_manager_task_active_count{type=~\"backup\",cluster=~\"$cluster|$^\"})*2 or on() vector(0)) + (sum(scylla_manager_server_current_version{}) or on() vector(-1))", + "intervalFactor": 1, + "refId": "A", + "step": 40 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Manager", + "type": "stat" + }, + { + "class": "vertical_lcd", + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 1, + "x": 8, + "y": 2 + }, + "id": 10, + "options": { + "displayMode": "lcd", + "orientation": "vertical", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "(sum(avg(scylla_manager_repair_progress{cluster=~\"$cluster|$^\"}) by (task) * sum(scylla_manager_task_active_count{type=\"repair\",cluster=~\"$cluster|$^\"}) by (task)) or on () vector(0)) + (sum(avg(scylla_manager_backup_percent_progress{cluster=~\"$cluster|$^\"}) by (task) * sum(scylla_manager_task_active_count{type=\"backup\",cluster=~\"$cluster|$^\"}) by (task)) or on () vector(0))", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "", + "type": "bargauge" + }, + { + "class": "small_stat", + "datasource": "Prometheus", + "description": "Average Write Latency", + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 50000 + } + ] + }, + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 9, + "y": 2 + }, + "id": 11, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "wlatencya{by=\"cluster\", cluster=~\"$cluster|^$\",scheduling_group_name=~\"$scheduling_group\"} or on() (sum(rate(scylla_storage_proxy_coordinator_write_latency_sum{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[60s]))/(sum(rate(scylla_storage_proxy_coordinator_write_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[60s])) + 1))", + "instant": true, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 4 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Avg Write", + "type": "stat" + }, + { + "class": "small_stat", + "datasource": "Prometheus", + "description": "99% write Latency", + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 100000 + } + ] + }, + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 11, + "y": 2 + }, + "id": 12, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "wlatencyp99{by=\"cluster\", cluster=~\"$cluster|^$\",scheduling_group_name=~\"$scheduling_group\"} or on() (histogram_quantile(0.99, sum(rate(scylla_storage_proxy_coordinator_write_latency_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$scheduling_group\"}[60s])) by (le)))", + "instant": true, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 4 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "99% Write", + "type": "stat" + }, + { + "class": "small_stat", + "datasource": "Prometheus", + "description": "Average Read Latency", + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 50000 + } + ] + }, + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 13, + "y": 2 + }, + "id": 13, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "rlatencya{by=\"cluster\", cluster=~\"$cluster|^$\",scheduling_group_name=~\"$scheduling_group\"} or on() (sum(rate(scylla_storage_proxy_coordinator_read_latency_sum{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[60s]))/(sum(rate(scylla_storage_proxy_coordinator_read_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[60s])) + 1))", + "instant": true, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 4 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Avg Read", + "type": "stat" + }, + { + "class": "small_stat", + "datasource": "Prometheus", + "description": "99% read Latency", + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 100000 + } + ] + }, + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 15, + "y": 2 + }, + "id": 14, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "rlatencyp99{by=\"cluster\", cluster=~\"$cluster|^$\",scheduling_group_name=~\"$scheduling_group\"} or on() (histogram_quantile(0.99, sum(rate(scylla_storage_proxy_coordinator_read_latency_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$scheduling_group\"}[60s])) by (le)))", + "instant": true, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 4 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "99% Read", + "type": "stat" + }, + { + "class": "small_stat", + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "si:" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 17, + "y": 2 + }, + "id": 15, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "sum(rate(scylla_transport_requests_served{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) + (sum(rate(scylla_thrift_served{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[60s])) or on() vector(0))", + "instant": true, + "intervalFactor": 1, + "refId": "A", + "step": 40 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Requests/s", + "type": "stat" + }, + { + "class": "small_stat", + "datasource": "Prometheus", + "description": "The percentage of the time during which Scylla utilized the CPU. Note that because Scylla does busy polling for some time before going idle, CPU utilization as seen by the operating system may be much higher. Your system is not yet CPU-bottlenecked until this metric is high.", + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 20, + "y": 2 + }, + "id": 16, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "avg(scylla_reactor_utilization{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"} )", + "instant": true, + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 4 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Load", + "type": "stat" + }, + { + "class": "small_stat", + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 22, + "y": 2 + }, + "id": 17, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "sum(rate(scylla_storage_proxy_coordinator_write_timeouts{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m]))", + "instant": true, + "intervalFactor": 1, + "refId": "A", + "step": 40 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Timeouts", + "type": "stat" + }, + { + "class": "alert_table", + "columns": [], + "datasource": "alertmanager", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": null, + "filterable": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute" + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.width", + "value": 150 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "instance" + }, + "properties": [ + { + "id": "custom.width", + "value": 100 + } + ] + } + ] + }, + "fontSize": "100%", + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 6 + }, + "id": 18, + "links": [], + "options": { + "showHeader": true + }, + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "span": 4, + "targets": [ + { + "active": true, + "annotations": true, + "filters": "job!=\"scylla_manager\",advisor=\"\"", + "legendFormat": "{{description}}", + "refId": "A", + "target": "Query" + } + ], + "title": "Active Alerts", + "transform": "table", + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "Time", + "summary", + "instance" + ] + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": { + "Time": 0, + "instance": 1, + "summary": 2 + }, + "renameByName": {} + } + } + ], + "type": "table" + }, + { + "class": "ops_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 8, + "y": 6 + }, + "id": 19, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 2, + "targets": [ + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_write_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval]))", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "title": "Writes", + "type": "timeseries" + }, + { + "class": "us_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 12, + "y": 6 + }, + "id": 20, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "class": "show_legend", + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 2, + "targets": [ + { + "expr": "wlatencyp95{by=\"cluster\", cluster=~\"$cluster|$^\",scheduling_group_name=~\"$scheduling_group\"} or on() histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_write_latency_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$scheduling_group\"}[$__rate_interval])) by (le))", + "intervalFactor": 1, + "legendFormat": "95%", + "refId": "A", + "step": 1 + }, + { + "expr": "wlatencyp99{by=\"cluster\", cluster=~\"$cluster|$^\",scheduling_group_name=~\"$scheduling_group\"} or on() histogram_quantile(0.99, sum(rate(scylla_storage_proxy_coordinator_write_latency_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$scheduling_group\"}[$__rate_interval])) by (le))", + "intervalFactor": 1, + "legendFormat": "99%", + "refId": "B", + "step": 1 + } + ], + "title": "Write Latencies", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 16, + "y": 6 + }, + "id": 21, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 2, + "targets": [ + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_read_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval]))", + "intervalFactor": 1, + "legendFormat": "Reads", + "refId": "A", + "step": 1 + } + ], + "title": "Reads", + "type": "timeseries" + }, + { + "class": "us_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 20, + "y": 6 + }, + "id": 22, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "class": "show_legend", + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 2, + "targets": [ + { + "expr": "rlatencyp95{by=\"cluster\", cluster=~\"$cluster|$^\",scheduling_group_name=~\"$scheduling_group\"} or on() histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_read_latency_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$scheduling_group\"}[$__rate_interval])) by (le))", + "intervalFactor": 1, + "legendFormat": "95%", + "refId": "A", + "step": 1 + }, + { + "expr": "rlatencyp99{by=\"cluster\", cluster=~\"$cluster|$^\",scheduling_group_name=~\"$scheduling_group\"} or on() histogram_quantile(0.99, sum(rate(scylla_storage_proxy_coordinator_read_latency_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$scheduling_group\"}[$__rate_interval])) by (le))", + "intervalFactor": 1, + "legendFormat": "99%", + "refId": "B", + "step": 1 + } + ], + "title": "Read Latencies", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 12 + }, + "id": 23, + "panels": [], + "title": "", + "type": "row" + }, + { + "class": "plain_text", + "content": "

Advisor

", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 13 + }, + "id": 24, + "isNew": true, + "links": [], + "mode": "html", + "options": {}, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 12, + "x": 12, + "y": 14 + }, + "id": 25, + "options": { + "content": "### Balance\nAn Imbalance between shards or nodes may indicates a potential problem", + "mode": "markdown" + }, + "pluginVersion": "7.3.4", + "targets": [ + { + "queryType": "randomWalk", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "", + "type": "text" + }, + { + "class": "advisor_table", + "dashversion": ">4.1", + "datasource": "alertmanager", + "fieldConfig": { + "defaults": { + "custom": { + "align": null, + "filterable": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "dashboard" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "", + "url": "./d/${__data.fields.dashboard}-[[dash_version]]/?refresh=30s&orgId=1&var-by=instance" + } + ] + }, + { + "id": "custom.width", + "value": 100 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "advisor" + }, + "properties": [ + { + "id": "custom.width", + "value": 120 + }, + { + "id": "displayName", + "value": "Category" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "severity" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "Open an issue", + "url": "https://github.com/scylladb/scylla/issues/new?body=description%3D${__data.fields[4]}%0ASource%3DAdvisor%0AScylla-versions%3D${all_scyllas_versions}%0Ascylla-monitoring%3D${monitoring_version}%0Acluster%3D${count_dc}%0Aname%3D${cluster}%0A%0A" + } + ] + }, + { + "id": "mappings", + "value": [ + { + "from": "0", + "id": 1, + "text": "\ud83d\udd14", + "to": "10", + "type": 2, + "value": "" + } + ] + }, + { + "id": "displayName", + "value": "Report" + }, + { + "id": "custom.width", + "value": 65 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "summary" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "${__data.fields.description}\n\n click for more information", + "url": "https://monitoring.docs.scylladb.com/branch-master/use-monitoring/advisor/${__data.fields.alertname}" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "alertname" + }, + "properties": [ + { + "id": "displayName", + "value": "." + }, + { + "id": "custom.width", + "value": 1 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "description" + }, + "properties": [ + { + "id": "displayName", + "value": "." + }, + { + "id": "custom.width", + "value": 1 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.width", + "value": 150 + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 17 + }, + "id": 26, + "links": [], + "options": { + "showHeader": true + }, + "pluginVersion": "7.3.4", + "targets": [ + { + "active": true, + "annotations": true, + "filters": "advisor!=\"\"", + "legendFormat": "{{description}}", + "refId": "A", + "target": "Query" + } + ], + "title": "", + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "advisor", + "dashboard", + "description", + "severity", + "alertname", + "summary", + "Time" + ] + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": { + "Time": 1, + "advisor": 2, + "dashboard": 3, + "severity": 0, + "summary": 4 + }, + "renameByName": {} + } + } + ], + "type": "table" + }, + { + "class": "small_stat_error", + "datasource": "Prometheus", + "description": "", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [ + { + "from": "-1000", + "id": 1, + "text": "\u2713", + "to": "0.001", + "type": 2, + "value": "" + }, + { + "from": "0.001", + "id": 2, + "text": "\u26a0", + "to": "10000", + "type": 2, + "value": "0.001" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.001 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 12, + "y": 17 + }, + "id": 27, + "links": [ + { + "title": "The number of connections per shard should be balanced", + "url": "./d/cql-[[dash_version]]/scylla-cql?orgId=1&refresh=30s&var-by=instance,shard&from=$__from&to=$__to" + } + ], + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "max(abs(sum(scylla_transport_current_connections{instance=~\"[[node]]\" ,cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by (instance,shard)-scalar(avg(scylla_transport_current_connections{instance=~\"[[node]]\" ,cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"})))) - 8", + "hide": false, + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Connections", + "transformations": [ + { + "id": "calculateField", + "options": { + "mode": "reduceRow", + "reduce": { + "reducer": "max" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, + { + "class": "small_stat_error", + "datasource": "Prometheus", + "description": "", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [ + { + "from": "-1000", + "id": 1, + "text": "\u2713", + "to": "0.001", + "type": 2, + "value": "" + }, + { + "from": "0.001", + "id": 2, + "text": "\u26a0", + "to": "10000", + "type": 2, + "value": "0.001" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.001 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 14, + "y": 17 + }, + "id": 28, + "links": [ + { + "title": "Indicates that the number of CQL operations (inserts, updates, deletes, reads) is not balance between shards in one of the nodes", + "url": "./d/cql-[[dash_version]]/scylla-cql?orgId=1&refresh=30s&var-by=instance,shard&from=$__from&to=$__to" + } + ], + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "max(abs(rate(scylla_cql_updates{conditional=\"no\", instance=~\"[[node]]\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m]) - on(dc) group_left avg(rate(scylla_cql_updates{conditional=\"no\", instance=~\"[[node]]\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by (dc))/on(dc) group_left sum(stddev(rate(scylla_cql_updates{conditional=\"no\", instance=~\"[[node]]\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by(dc)+100) by(dc))-3", + "format": "time_series", + "hide": false, + "interval": "", + "legendFormat": "", + "refId": "A" + }, + { + "expr": "max(abs(rate(scylla_cql_inserts{conditional=\"no\", instance=~\"[[node]]\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m]) - on(dc) group_left avg(rate(scylla_cql_inserts{conditional=\"no\", instance=~\"[[node]]\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by (dc))/on(dc) group_left sum(stddev(rate(scylla_cql_inserts{conditional=\"no\", instance=~\"[[node]]\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by(dc)+100) by(dc))-3", + "hide": false, + "interval": "", + "legendFormat": "", + "refId": "B" + }, + { + "expr": "max(abs(rate(scylla_cql_reads{instance=~\"[[node]]\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m]) - on(dc) group_left avg(rate(scylla_cql_reads{instance=~\"[[node]]\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by (dc))/on(dc) group_left sum(stddev(rate(scylla_cql_reads{instance=~\"[[node]]\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by(dc)+100) by(dc))-3", + "hide": false, + "interval": "", + "legendFormat": "", + "refId": "C" + }, + { + "expr": "max(abs(rate(scylla_cql_deletes{conditional=\"no\", instance=~\"[[node]]\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m]) - on(dc) group_left avg(rate(scylla_cql_deletes{conditional=\"no\", instance=~\"[[node]]\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by (dc))/on(dc) group_left sum(stddev(rate(scylla_cql_deletes{conditional=\"no\", instance=~\"[[node]]\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[1m])) by(dc)+100) by(dc))-3", + "hide": false, + "interval": "", + "legendFormat": "", + "refId": "D" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "CQL OPs", + "transformations": [ + { + "id": "calculateField", + "options": { + "mode": "reduceRow", + "reduce": { + "reducer": "max" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, + { + "class": "small_stat_error", + "datasource": "Prometheus", + "description": "", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [ + { + "from": "-1000", + "id": 1, + "text": "\u2713", + "to": "0.001", + "type": 2, + "value": "" + }, + { + "from": "0.001", + "id": 2, + "text": "\u26a0", + "to": "10000", + "type": 2, + "value": "0.001" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.001 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 16, + "y": 17 + }, + "id": 29, + "links": [ + { + "title": "A single node with higher latency is an indication for a node related issue", + "url": "./d/cql-[[dash_version]]/scylla-cql?orgId=1&refresh=30s&var-by=instance,shard&from=$__from&to=$__to" + } + ], + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "((max(wlatencyp99{by=\"instance\", cluster=~\"$cluster|$^\",scheduling_group_name=~\"$scheduling_group\"})-scalar(avg(wlatencyp99{by=\"instance\", cluster=~\"$cluster|$^\",scheduling_group_name=~\"$scheduling_group\"}>0)))/(scalar(stddev(wlatencyp99{by=\"instance\", cluster=~\"$cluster|$^\",scheduling_group_name=~\"$scheduling_group\"}>0))+100)-3) or on() (max(histogram_quantile(0.99, sum(rate(scylla_storage_proxy_coordinator_write_latency_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\",scheduling_group_name=~\"$scheduling_group\"}[$__rate_interval])) by (instance, le)) -scalar(avg(histogram_quantile(0.99, sum(rate(scylla_storage_proxy_coordinator_write_latency_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\",scheduling_group_name=~\"$scheduling_group\"}[$__rate_interval])) by (instance,le))>0)))/ scalar(stddev(histogram_quantile(0.99, sum(rate(scylla_storage_proxy_coordinator_write_latency_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\",scheduling_group_name=~\"$scheduling_group\"}[$__rate_interval])) by (instance,le))>0)+100)-3)", + "legendFormat": "", + "refId": "A" + }, + { + "expr": "((max(rlatencyp99{by=\"instance\", cluster=~\"$cluster|$^\",scheduling_group_name=~\"$scheduling_group\"})-scalar(avg(rlatencyp99{by=\"instance\", cluster=~\"$cluster|$^\",scheduling_group_name=~\"$scheduling_group\"}>0)))/(scalar(stddev(rlatencyp99{by=\"instance\", cluster=~\"$cluster|$^\",scheduling_group_name=~\"$scheduling_group\"}>0))+100)-3) or on() (max(histogram_quantile(0.99, sum(rate(scylla_storage_proxy_coordinator_read_latency_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\",scheduling_group_name=~\"$scheduling_group\"}[$__rate_interval])) by (instance, le)) -scalar(avg(histogram_quantile(0.99, sum(rate(scylla_storage_proxy_coordinator_read_latency_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\",scheduling_group_name=~\"$scheduling_group\"}[$__rate_interval])) by (instance,le))>0)))/ scalar(stddev(histogram_quantile(0.99, sum(rate(scylla_storage_proxy_coordinator_read_latency_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\",scheduling_group_name=~\"$scheduling_group\"}[$__rate_interval])) by (instance,le))>0)+100)-3)", + "legendFormat": "", + "refId": "B" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Node Latency", + "transformations": [ + { + "id": "calculateField", + "options": { + "mode": "reduceRow", + "reduce": { + "reducer": "max" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, + { + "class": "small_stat_error", + "datasource": "Prometheus", + "description": "", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [ + { + "from": "-1000", + "id": 1, + "text": "\u2713", + "to": "0.001", + "type": 2, + "value": "" + }, + { + "from": "0.001", + "id": 2, + "text": "\u26a0", + "to": "10000", + "type": 2, + "value": "0.001" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.001 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 18, + "y": 17 + }, + "id": 30, + "links": [ + { + "title": "A single shard with high latency is an indication of a hot-partition, or a large row/cell/partition", + "url": "./d/cql-[[dash_version]]/scylla-cql?orgId=1&refresh=30s&var-by=instance,shard&from=$__from&to=$__to" + } + ], + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "((max(wlatencyp99{by=\"instance,shard\", cluster=~\"$cluster|$^\",scheduling_group_name=~\"$scheduling_group\"})-scalar(avg(wlatencyp99{by=\"instance,shard\", cluster=~\"$cluster|$^\",scheduling_group_name=~\"$scheduling_group\"}>0)))/(scalar(stddev(wlatencyp99{by=\"instance,shard\", cluster=~\"$cluster|$^\",scheduling_group_name=~\"$scheduling_group\"}>0))+100)-3) or on() (max(histogram_quantile(0.99, sum(rate(scylla_storage_proxy_coordinator_write_latency_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$scheduling_group\"}[$__rate_interval])) by (instance,shard, le)) -scalar(avg(histogram_quantile(0.99, sum(rate(scylla_storage_proxy_coordinator_write_latency_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$scheduling_group\"}[$__rate_interval])) by (instance,shard,le))>0)))/ scalar(stddev(histogram_quantile(0.99, sum(rate(scylla_storage_proxy_coordinator_write_latency_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$scheduling_group\"}[$__rate_interval])) by (instance,shard, le))>0)+100)-3)", + "legendFormat": "", + "refId": "A" + }, + { + "expr": "((max(rlatencyp99{by=\"instance,shard\", cluster=~\"$cluster|$^\",scheduling_group_name=~\"$scheduling_group\"})-scalar(avg(rlatencyp99{by=\"instance,shard\", cluster=~\"$cluster|$^\",scheduling_group_name=~\"$scheduling_group\"}>0)))/(scalar(stddev(rlatencyp99{by=\"instance,shard\", cluster=~\"$cluster|$^\",scheduling_group_name=~\"$scheduling_group\"}>0))+100)-3) or on() (max(histogram_quantile(0.99, sum(rate(scylla_storage_proxy_coordinator_read_latency_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$scheduling_group\"}[$__rate_interval])) by (instance,shard, le)) -scalar(avg(histogram_quantile(0.99, sum(rate(scylla_storage_proxy_coordinator_read_latency_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$scheduling_group\"}[$__rate_interval])) by (instance,shard,le))>0)))/ scalar(stddev(histogram_quantile(0.99, sum(rate(scylla_storage_proxy_coordinator_read_latency_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$scheduling_group\"}[$__rate_interval])) by (instance,shard, le))>0)+100)-3)", + "legendFormat": "", + "refId": "B" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Shard Latency", + "transformations": [ + { + "id": "calculateField", + "options": { + "mode": "reduceRow", + "reduce": { + "reducer": "max" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, + { + "class": "small_stat_error", + "datasource": "Prometheus", + "description": "A shard that reads more from the cache could be an indication for hot partition", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [ + { + "from": "-1000", + "id": 1, + "text": "\u2713", + "to": "0.001", + "type": 2, + "value": "" + }, + { + "from": "0.001", + "id": 2, + "text": "\u26a0", + "to": "10000", + "type": 2, + "value": "0.001" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.001 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 20, + "y": 17 + }, + "id": 31, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "((rate(scylla_cache_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval]) - rate(scylla_cache_reads_with_misses{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval]))- scalar(avg(rate(scylla_cache_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval]) - rate(scylla_cache_reads_with_misses{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval]))))/scalar(stddev(rate(scylla_cache_reads{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval]) - rate(scylla_cache_reads_with_misses{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval]))+100)-3", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Cache", + "transformations": [ + { + "id": "calculateField", + "options": { + "mode": "reduceRow", + "reduce": { + "reducer": "max" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, + { + "class": "small_stat_error", + "datasource": "Prometheus", + "description": "A single shard that reads more from sstables, could be that a node is slow", + "fieldConfig": { + "defaults": { + "custom": {}, + "mappings": [ + { + "from": "-1000", + "id": 1, + "text": "\u2713", + "to": "0.001", + "type": 2, + "value": "" + }, + { + "from": "0.001", + "id": 2, + "text": "\u26a0", + "to": "10000", + "type": 2, + "value": "0.001" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 0.001 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2, + "x": 22, + "y": 17 + }, + "id": 32, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "max(abs(scylla_database_active_reads{instance=~\"[[node]]\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"} - scalar(avg(scylla_database_active_reads{instance=~\"[[node]]\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"})))/scalar(stddev(scylla_database_active_reads{instance=~\"[[node]]\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"})+0.001))-3", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 1 + } + ], + "timeFrom": null, + "timeShift": null, + "title": "SSTable", + "transformations": [ + { + "id": "calculateField", + "options": { + "mode": "reduceRow", + "reduce": { + "reducer": "max" + }, + "replaceFields": true + } + } + ], + "type": "stat" + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 33, + "panels": [], + "repeat": "dc", + "title": "", + "type": "row" + }, + { + "class": "plain_text", + "content": "

Information for $dc

", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 34, + "isNew": true, + "links": [], + "mode": "html", + "options": {}, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "vertical_lcd", + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "custom": {}, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 85 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 1, + "x": 0, + "y": 26 + }, + "id": 35, + "options": { + "displayMode": "lcd", + "orientation": "vertical", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "pluginVersion": "7.1.3", + "targets": [ + { + "expr": "avg(scylla_reactor_utilization{instance=~\"[[node]]\",cluster=~\"$cluster\", dc=~\"$dc\", shard=~\"[[shard]]\"} )", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Load", + "type": "bargauge" + }, + { + "class": "bytes_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 3, + "x": 1, + "y": 26 + }, + "id": 36, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 5, + "targets": [ + { + "expr": "Avg(node_filesystem_size_bytes{mountpoint=\"$mount_point\", dc=~\"$dc\", instance=~\"$node\"}) by ([[by]])-avg(node_filesystem_avail_bytes{mountpoint=\"$mount_point\", dc=~\"$dc\", instance=~\"$node\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "Avg Usage {{[[by]]}}", + "metric": "", + "refId": "A", + "step": 1 + }, + { + "expr": "avg(node_filesystem_size_bytes{mountpoint=\"$mount_point\", dc=~\"$dc\", instance=~\"$node\"}) by ([[by]])", + "interval": "", + "legendFormat": "Size {{[[by]]}}", + "refId": "B" + } + ], + "title": "Disk Size by $by", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 4, + "y": 26 + }, + "id": 37, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "class": "show_legend", + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + { + "alias": "1 Day Ago", + "dashLength": 4, + "dashes": true + }, + { + "alias": "1 Week Ago", + "dashLength": 2, + "dashes": true + } + ], + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_write_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "Writes", + "refId": "A", + "step": 1 + }, + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_write_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval] offset 1d))", + "interval": "", + "intervalFactor": 1, + "legendFormat": "1 Day Ago", + "refId": "B", + "step": 1 + }, + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_write_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval] offset 1w))", + "interval": "", + "intervalFactor": 1, + "legendFormat": "1 Week Ago", + "refId": "C", + "step": 1 + } + ], + "title": "Writes", + "type": "timeseries" + }, + { + "class": "us_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 10, + "y": 26 + }, + "id": 38, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "class": "show_legend", + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 2, + "targets": [ + { + "expr": "wlatencyp95{by=\"[[by]]\", instance=~\"[[node]]|^$\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$scheduling_group\"} or on() (histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_write_latency_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$scheduling_group\"}[$__rate_interval])) by ([[by]], le)))", + "intervalFactor": 1, + "legendFormat": "95% {{[[by]]}}", + "refId": "A", + "step": 1 + }, + { + "expr": "wlatencyp99{by=\"[[by]]\", instance=~\"[[node]]|^$\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$scheduling_group\"} or on() (histogram_quantile(0.99, sum(rate(scylla_storage_proxy_coordinator_write_latency_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$scheduling_group\"}[$__rate_interval])) by ([[by]], le)))", + "intervalFactor": 1, + "legendFormat": "99% {{[[by]]}}", + "refId": "B", + "step": 1 + } + ], + "title": "Write Latencies", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 14, + "y": 26 + }, + "id": 39, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "class": "show_legend", + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + { + "alias": "1 Day Ago", + "dashLength": 4, + "dashes": true + }, + { + "alias": "1 Week Ago", + "dashLength": 2, + "dashes": true + } + ], + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_read_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$scheduling_group\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "Reads", + "refId": "A", + "step": 1 + }, + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_read_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$scheduling_group\"}[$__rate_interval] offset 1d))", + "intervalFactor": 1, + "legendFormat": "1 Day Ago", + "refId": "B", + "step": 1 + }, + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_read_latency_count{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$scheduling_group\"}[$__rate_interval] offset 1w))", + "intervalFactor": 1, + "legendFormat": "1 Week Ago", + "refId": "C", + "step": 1 + } + ], + "title": "Reads", + "type": "timeseries" + }, + { + "class": "us_panel", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "\u00b5s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 20, + "y": 26 + }, + "id": 40, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "class": "show_legend", + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 2, + "targets": [ + { + "expr": "rlatencyp95{by=\"[[by]]\", instance=~\"[[node]]|^$\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$scheduling_group\"} or on() (histogram_quantile(0.95, sum(rate(scylla_storage_proxy_coordinator_read_latency_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$scheduling_group\"}[$__rate_interval])) by ([[by]], le)))", + "intervalFactor": 1, + "legendFormat": "95% {{[[by]]}}", + "refId": "A", + "step": 1 + }, + { + "expr": "rlatencyp99{by=\"[[by]]\", instance=~\"[[node]]|^$\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$scheduling_group\"} or on() (histogram_quantile(0.99, sum(rate(scylla_storage_proxy_coordinator_read_latency_bucket{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\",scheduling_group_name=~\"$scheduling_group\"}[$__rate_interval])) by ([[by]], le)))", + "intervalFactor": 1, + "legendFormat": "99% {{[[by]]}}", + "refId": "B", + "step": 1 + } + ], + "title": "Read Latencies", + "type": "timeseries" + }, + { + "class": "small_nodes_table", + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": null, + "filterable": true + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 85 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Value #B" + }, + "properties": [ + { + "id": "custom.displayMode", + "value": "lcd-gauge" + }, + { + "id": "min", + "value": 0 + }, + { + "id": "max", + "value": 101 + }, + { + "id": "displayName", + "value": "Load" + }, + { + "id": "custom.width", + "value": 120 + }, + { + "id": "custom.filterable", + "value": false + }, + { + "id": "decimals", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "svr" + }, + "properties": [ + { + "id": "custom.width", + "value": 85 + }, + { + "id": "displayName", + "value": "Version" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "instance" + }, + "properties": [ + { + "id": "custom.width", + "value": 105 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #A" + }, + "properties": [ + { + "id": "custom.width", + "value": 105 + }, + { + "id": "displayName", + "value": "Status" + }, + { + "id": "custom.displayMode", + "value": "color-background" + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "yellow", + "value": null + }, + { + "color": "rgba(0, 0, 0, 0)", + "value": 3 + }, + { + "color": "red", + "value": 4 + } + ] + } + }, + { + "id": "custom.align", + "value": "center" + }, + { + "id": "mappings", + "value": [ + { + "from": "", + "id": 1, + "text": "Starting", + "to": "", + "type": 1, + "value": "1" + }, + { + "from": "", + "id": 2, + "text": "Joining", + "to": "", + "type": 1, + "value": "2" + }, + { + "from": "", + "id": 3, + "text": "Normal", + "to": "", + "type": 1, + "value": "3" + }, + { + "from": "", + "id": 4, + "text": "Leaving", + "to": "", + "type": 1, + "value": "4" + }, + { + "from": "", + "id": 5, + "text": "Decommissioned", + "to": "", + "type": 1, + "value": "5" + }, + { + "from": "", + "id": 6, + "text": "Draining", + "to": "", + "type": 1, + "value": "6" + }, + { + "from": "", + "id": 7, + "text": "Drained", + "to": "", + "type": 1, + "value": "7" + }, + { + "from": "", + "id": 8, + "text": "Moving", + "to": "", + "type": 1, + "value": "8" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #E" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "CQL Information Dashboard. Warning indicates that there are potential issues in the CQL Optimization", + "url": "./d/cql-[[dash_version]]/scylla-cql?refresh=30s&orgId=1&var-by=instance&var-node=${__data.fields[0]}" + } + ] + }, + { + "id": "custom.width", + "value": 90 + }, + { + "id": "custom.filterable", + "value": false + }, + { + "id": "mappings", + "value": [ + { + "from": "", + "id": 1, + "text": "CQL Info", + "to": "", + "type": 1, + "value": "0" + }, + { + "from": "1", + "id": 1, + "text": "CQL Warnings", + "to": "20", + "type": 2, + "value": "" + } + ] + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(0, 0, 0,0)", + "value": null + }, + { + "color": "dark-orange", + "value": 1 + } + ] + } + }, + { + "id": "custom.displayMode", + "value": "color-background" + }, + { + "id": "custom.align", + "value": "left" + }, + { + "id": "displayName", + "value": "CQL" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #C" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "OS Information Dashboard, an Error indicates there are OS related errors", + "url": "./d/OS-[[dash_version]]/OS-metrics?refresh=30s&orgId=1&var-by=instance&var-node=${__data.fields[0]}" + } + ] + }, + { + "id": "custom.width", + "value": 90 + }, + { + "id": "custom.filterable", + "value": true + }, + { + "id": "mappings", + "value": [ + { + "from": "", + "id": 1, + "text": "OS Info", + "to": "", + "type": 1, + "value": "0" + }, + { + "from": "0", + "id": 2, + "text": "OS Errors", + "to": "100000", + "type": 2, + "value": "0" + } + ] + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(0, 0, 0,0)", + "value": null + }, + { + "color": "dark-orange", + "value": 0.001 + } + ] + } + }, + { + "id": "custom.displayMode", + "value": "color-background" + }, + { + "id": "custom.align", + "value": "left" + }, + { + "id": "displayName", + "value": "OS" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #D" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "Cordinator and Replica node errors", + "url": "./d/detailed-[[dash_version]]/Detailed?refresh=30s&orgId=1&var-by=instance&var-node=${__data.fields[0]}" + } + ] + }, + { + "id": "custom.width", + "value": 80 + }, + { + "id": "custom.filterable", + "value": true + }, + { + "id": "mappings", + "value": [ + { + "from": "", + "id": 1, + "text": "", + "to": "", + "type": 1, + "value": "0" + }, + { + "from": "", + "id": 2, + "text": "Errors", + "to": "", + "type": 1, + "value": "1" + } + ] + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(0, 0, 0,0)", + "value": null + }, + { + "color": "dark-orange", + "value": 0.001 + } + ] + } + }, + { + "id": "custom.displayMode", + "value": "color-background" + }, + { + "id": "custom.align", + "value": "left" + }, + { + "id": "displayName", + "value": " " + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "instance" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "title": "Detailed view", + "url": "./d/detailed-[[dash_version]]/Detailed?refresh=30s&orgId=1&var-by=instance&var-node=${__data.fields[0]}" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 10, + "x": 0, + "y": 32 + }, + "id": 41, + "scopedVars": { + "dc": { + "selected": false, + "text": "datacenter1", + "value": "datacenter1" + } + }, + "span": 5, + "targets": [ + { + "expr": "0*scylla_scylladb_current_version{cluster=~\"$cluster|$^\", dc=~\"$dc\"} + on (instance) group_left() scylla_node_operation_mode{cluster=~\"$cluster|$^\", dc=~\"$dc\"}", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + }, + { + "expr": "avg(scylla_reactor_utilization{cluster=~\"$cluster\", dc=~\"$dc\"} ) by (instance)", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "B" + }, + { + "expr": "sum(rate(scylla_reactor_aio_errors{cluster=~\"$cluster\", dc=~\"$dc\"}[1m])) by (instance)", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "C" + }, + { + "expr": "sum(errors:nodes_total{cluster=~\"$cluster\", dc=~\"$dc\"}) by (instance) >bool 0", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "D" + }, + { + "expr": "(sum(cql:non_system_prepared1m{cluster=~\"$cluster\", dc=~\"$dc\"}) by (instance) >bool 1) + (sum(rate(scylla_cql_reverse_queries{cluster=~\"$cluster\", dc=~\"$dc\"}[60s])) by(instance) >bool 1) + (sum(cql:non_paged_no_system1m{cluster=~\"$cluster\", dc=~\"$dc\"}) by (instance) >bool 1)", + "format": "table", + "instant": true, + "interval": "", + "legendFormat": "", + "refId": "E" + } + ], + "title": "Nodes", + "transformations": [ + { + "id": "filterFieldsByName", + "options": { + "include": { + "names": [ + "instance", + "svr", + "Value #A", + "Value #B", + "Value #C", + "Value #D", + "Value #E" + ] + } + } + }, + { + "id": "seriesToColumns", + "options": { + "byField": "instance" + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": { + "Value #A": 5, + "Value #B": 6, + "Value #C": 3, + "Value #D": 1, + "Value #E": 2, + "instance": 0, + "svr": 4 + }, + "renameByName": {} + } + } + ], + "type": "table" + }, + { + "class": "graph_panel_int", + "datasource": "Prometheus", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 10, + "y": 32 + }, + "id": 42, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "span": 2, + "targets": [ + { + "expr": "$func(scylla_compaction_manager_compactions{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 1 + } + ], + "title": "Running Compactions", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "Prometheus", + "description": "The Hits and Misses", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 14, + "y": 32 + }, + "id": 43, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "class": "show_legend", + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 3, + "targets": [ + { + "expr": "$func(rate(scylla_cache_row_hits{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "Hit {{[[by]]}}", + "refId": "A", + "step": 10 + }, + { + "expr": "$func(rate(scylla_cache_row_misses{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "Misses {{[[by]]}}", + "refId": "B", + "step": 10 + } + ], + "title": "Cache Hits/Misses", + "type": "timeseries" + }, + { + "class": "ops_panel", + "datasource": "Prometheus", + "description": "Requests that Scylla tried to write but timed out. Timeouts are counted in the node that received the request (the coordinator), not at the replicas.", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 20, + "y": 32 + }, + "id": 44, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": false, + "class": "show_legend", + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 2, + "targets": [ + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_write_timeouts{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]|$^\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "Writes {{[[by]]}}", + "refId": "A", + "step": 10 + }, + { + "expr": "$func(rate(scylla_storage_proxy_coordinator_read_timeouts{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])+rate(scylla_storage_proxy_coordinator_cas_read_timeouts{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])+rate(scylla_storage_proxy_coordinator_range_timeouts{instance=~\"[[node]]\",cluster=~\"$cluster|$^\", dc=~\"$dc\", shard=~\"[[shard]]\"}[$__rate_interval])) by ([[by]])", + "intervalFactor": 1, + "legendFormat": "Read {{[[by]]}}", + "refId": "B", + "step": 10 + } + ], + "title": "Read/Write Timeouts by [[by]]", + "type": "timeseries" + }, + { + "class": "collapsible_row_panel", + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 38 + }, + "id": 45, + "panels": [], + "repeat": "", + "title": "Your panels", + "type": "row" + }, + { + "class": "plain_text", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 39 + }, + "id": 46, + "isNew": true, + "links": [], + "mode": "html", + "options": { + "content": "

Your Panels

", + "mode": "html" + }, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "class": "user_panel", + "datasource": "Prometheus", + "description": "This graph panel was left empty on purpose for ad-hoc usage. Change it when needed. Pay attention that changes to the panel will not be saved.\n\nIf you do need a panel that can be saved, create a new dashboard, or edit the panel inside the json file", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 41 + }, + "id": 47, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 6, + "title": "Your Graph here", + "type": "timeseries" + }, + { + "class": "user_panel", + "datasource": "Prometheus", + "description": "This graph panel was left empty on purpose for ad-hoc usage. Change it when needed. Pay attention that changes to the panel will not be saved.\n\nIf you do need a panel that can be saved, create a new dashboard, or edit the panel inside the json file", + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "class": "fieldConfig_defaults", + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "si:ops/s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 41 + }, + "id": 48, + "isNew": true, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "seriesOverrides": [ + {} + ], + "span": 6, + "title": "Your Graph here", + "type": "timeseries" + }, + { + "class": "plain_text", + "datasource": null, + "editable": true, + "error": false, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 47 + }, + "id": 49, + "isNew": true, + "links": [], + "mode": "html", + "options": { + "content": "
Scylla Monitoring version - master
\n    
", + "mode": "html" + }, + "span": 12, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + } + ], + "refresh": "30s", + "schemaVersion": 26, + "style": "dark", + "tags": [ + "4.4" + ], + "templating": { + "list": [ + { + "allValue": null, + "class": "by_template_var", + "current": { + "tags": [], + "text": "DC", + "value": "dc" + }, + "error": null, + "hide": 0, + "includeAll": false, + "label": "by", + "multi": false, + "name": "by", + "options": [ + { + "selected": false, + "text": "Cluster", + "value": "cluster" + }, + { + "selected": true, + "text": "DC", + "value": "dc" + }, + { + "selected": false, + "text": "Instance", + "value": "instance" + }, + { + "selected": false, + "text": "instance,shard", + "value": "instance,shard" + } + ], + "query": "Cluster,DC,Instance,Shard", + "skipUrlSync": false, + "type": "custom" + }, + { + "allValue": null, + "class": "template_variable_single", + "current": { + "isNone": true, + "selected": false, + "text": "None", + "value": "" + }, + "datasource": "Prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [], + "query": "label_values(scylla_reactor_utilization, cluster)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "class": "template_variable_all", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "Prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": true, + "label": "dc", + "multi": true, + "name": "dc", + "options": [], + "query": "label_values(scylla_reactor_utilization{cluster=~\"$cluster\"}, dc)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "class": "template_variable_all", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "Prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": true, + "label": "node", + "multi": true, + "name": "node", + "options": [], + "query": "label_values(scylla_reactor_utilization{cluster=~\"$cluster|$^\", dc=~\"$dc\"}, instance)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "class": "template_variable_all", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "Prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": true, + "label": "shard", + "multi": true, + "name": "shard", + "options": [], + "query": "label_values(scylla_reactor_utilization,shard)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 3, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "class": "template_variable_single", + "current": { + "text": "/var/lib/scylla", + "value": "/var/lib/scylla" + }, + "datasource": "Prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": false, + "label": "Mount path", + "multi": false, + "name": "mount_point", + "options": [], + "query": "node_filesystem_avail_bytes", + "refresh": 2, + "regex": "/mountpoint=\"([^\"]*)\".*/", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "class": "template_variable_all", + "current": { + "selected": true, + "tags": [], + "text": [ + "statement" + ], + "value": [ + "statement" + ] + }, + "dashversion": [ + ">4.3" + ], + "datasource": "Prometheus", + "definition": "", + "error": null, + "hide": 0, + "includeAll": true, + "label": "SG", + "multi": true, + "name": "scheduling_group", + "options": [], + "query": "label_values(scylla_storage_proxy_coordinator_read_latency_bucket,scheduling_group_name)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 3, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "class": "aggregation_function", + "current": { + "tags": [], + "text": "sum", + "value": "sum" + }, + "hide": 0, + "includeAll": false, + "label": "Function", + "multi": false, + "name": "func", + "options": [ + { + "selected": true, + "text": "sum", + "value": "sum" + }, + { + "selected": false, + "text": "avg", + "value": "avg" + }, + { + "selected": false, + "text": "max", + "value": "max" + }, + { + "selected": false, + "text": "min", + "value": "min" + }, + { + "selected": false, + "text": "stddev", + "value": "stddev" + }, + { + "selected": false, + "text": "stdvar", + "value": "stdvar" + } + ], + "query": "sum,avg,max,min,stddev,stdvar", + "skipUrlSync": false, + "type": "custom" + }, + { + "class": "adhoc_filter", + "datasource": "Prometheus", + "error": null, + "filters": [], + "hide": 0, + "label": "ad hoc", + "name": "adhoc", + "skipUrlSync": false, + "type": "adhoc" + }, + { + "allValue": null, + "class": "template_variable_custom", + "current": { + "text": "4-4", + "value": "4-4" + }, + "error": null, + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "dash_version", + "options": [ + { + "selected": true, + "text": "4-4", + "value": "4-4" + } + ], + "query": "4-4", + "skipUrlSync": false, + "type": "custom" + }, + { + "allValue": null, + "class": "template_variable_all", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "Prometheus", + "definition": "", + "error": null, + "hide": 2, + "includeAll": true, + "multi": true, + "name": "all_scyllas_versions", + "options": [], + "query": "label_values(scylla_scylladb_current_version{cluster=~\"$cluster|$^\"}, version)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "class": "template_variable_all", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "Prometheus", + "definition": "query_result(count(up{job=\"scylla\"}) by (dc))", + "error": null, + "hide": 2, + "includeAll": true, + "multi": true, + "name": "count_dc", + "options": [], + "query": { + "query": "query_result(count(up{job=\"scylla\"}) by (dc))", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "/(?\\{dc=\"[^\"]+\".* \\d+) .*/", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "class": "template_variable_custom", + "current": { + "text": "4.4", + "value": "4.4" + }, + "error": null, + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "scylla_version", + "options": [ + { + "selected": true, + "text": "4.4", + "value": "4.4" + } + ], + "query": "4.4", + "skipUrlSync": false, + "type": "custom" + }, + { + "allValue": null, + "class": "monitor_version_var", + "current": { + "text": "master", + "value": "master" + }, + "error": null, + "hide": 2, + "includeAll": false, + "label": null, + "multi": false, + "name": "monitoring_version", + "options": [ + { + "selected": true, + "text": "master", + "value": "master" + } + ], + "query": "master", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": { + "now": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "utc", + "title": "Overview", + "uid": "overview-4-4", + "version": 1 +}