{ "__inputs": [ { "name": "DS_PROMETHEUS", "label": "Prometheus", "description": "", "type": "datasource", "pluginId": "prometheus", "pluginName": "Prometheus" } ], "__elements": {}, "__requires": [ { "type": "panel", "id": "bargauge", "name": "Bar gauge", "version": "" }, { "type": "grafana", "id": "grafana", "name": "Grafana", "version": "10.4.5" }, { "type": "panel", "id": "piechart", "name": "Pie chart", "version": "" }, { "type": "datasource", "id": "prometheus", "name": "Prometheus", "version": "1.0.0" }, { "type": "panel", "id": "stat", "name": "Stat", "version": "" }, { "type": "panel", "id": "timeseries", "name": "Time series", "version": "" } ], "annotations": { "list": [ { "builtIn": 1, "datasource": { "type": "grafana", "uid": "-- Grafana --" }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard" } ] }, "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, "id": 4, "links": [], "panels": [ { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 1, "panels": [], "title": "Cluster", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the total capacity, used storage, and usage percentage of the Alluxio cache. This panel is critical for monitoring overall cache utilization and preventing out-of-storage issues.\t", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "bytes" }, "overrides": [ { "matcher": { "id": "byName", "options": "Used (%)" }, "properties": [ { "id": "unit", "value": "percentunit" }, { "id": "thresholds", "value": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 0.9 }, { "color": "red", "value": 0.95 } ] } } ] } ] }, "gridPos": { "h": 5, "w": 8, "x": 0, "y": 1 }, "id": 3, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "text": { "titleSize": 24, "valueSize": 36 }, "textMode": "value_and_name", "wideLayout": true }, "pluginVersion": "10.4.5", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum(alluxio_cached_capacity_bytes{job=\"worker\",cluster_name=~\"$cluster\"})", "hide": false, "instant": false, "legendFormat": "Capacity", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum(alluxio_cached_storage_bytes{job=\"worker\",cluster_name=~\"$cluster\"})", "instant": false, "legendFormat": "Used", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum(alluxio_cached_storage_bytes{job=\"worker\",cluster_name=~\"$cluster\"}) / sum(alluxio_cached_capacity_bytes{job=\"worker\",cluster_name=~\"$cluster\"})", "hide": false, "instant": false, "legendFormat": "Used (%)", "range": true, "refId": "C" } ], "title": "Storage", "transparent": true, "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Shows the read throughput from different sources: Fuse, S3 API, and direct Worker access. This helps identify the primary data access interfaces for read operations.\t", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 1, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "binBps" }, "overrides": [ { "matcher": { "id": "byName", "options": "Waiting" }, "properties": [ { "id": "unit", "value": "reqps" }, { "id": "thresholds", "value": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 100 }, { "color": "red", "value": 2000 } ] } } ] } ] }, "gridPos": { "h": 5, "w": 7, "x": 9, "y": 1 }, "id": 48, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "text": {}, "textMode": "auto", "wideLayout": true }, "pluginVersion": "10.4.5", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum(irate(alluxio_data_access_bytes_sum{job=\"fuse\",cluster_name=~\"$cluster\"}[5m])) or on() vector(0)", "hide": false, "instant": false, "legendFormat": "Fuse", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum(irate(alluxio_s3_api_throughput_bytes_sum{job=\"worker\",method=\"read\",cluster_name=~\"$cluster\"}[5m])) or on() vector(0)", "hide": false, "instant": false, "legendFormat": "S3", "range": true, "refId": "D" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum(irate(alluxio_data_throughput_bytes_total{job=\"worker\",method=\"read\",cluster_name=~\"$cluster\"}[5m])) or on() vector(0)", "hide": false, "instant": false, "legendFormat": "Worker", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "avg(avg_over_time(alluxio_rpc_executor_current_queue_length{job=\"worker\",executor_name=\"netty-reader\"}[5m])) or on() vector(0)", "hide": false, "instant": false, "legendFormat": "Waiting", "range": true, "refId": "C" } ], "title": "Read - Throughput", "transparent": true, "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the average read load (thread utilization) across all workers over the last 5 minutes. A high average load may indicate a system-wide performance bottleneck.\t", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 2, "mappings": [], "max": 1, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 0.8 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 5, "w": 3, "x": 16, "y": 1 }, "id": 73, "options": { "minVizHeight": 75, "minVizWidth": 75, "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": false, "sizing": "auto" }, "pluginVersion": "10.4.5", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "avg(avg_over_time(alluxio_rpc_executor_active_threads{job=\"worker\", executor_name=\"netty-reader\"}[5m])/avg_over_time(alluxio_rpc_executor_max_threads{job=\"worker\", executor_name=\"netty-reader\"}[5m]))", "instant": false, "legendFormat": "{{label_name}}", "range": true, "refId": "A" } ], "title": "Read - Load (5m)", "transparent": true, "type": "gauge" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays per-worker read load (for workers exceeding 50% utilization), and highlights the most heavily used workers. This helps identify uneven I/O distribution and potential hotspots within the cluster.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 2, "mappings": [], "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 0.8 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 5, "w": 5, "x": 19, "y": 1 }, "id": 71, "options": { "displayMode": "basic", "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false }, "maxVizHeight": 300, "minVizHeight": 10, "minVizWidth": 0, "namePlacement": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showUnfilled": true, "sizing": "auto", "text": {}, "valueMode": "color" }, "pluginVersion": "10.4.5", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "avg_over_time(alluxio_rpc_executor_active_threads{job=\"worker\", executor_name=\"netty-reader\"}[5m])/avg_over_time(alluxio_rpc_executor_max_threads{job=\"worker\", executor_name=\"netty-reader\"}[5m]) > 0.5", "hide": false, "instant": false, "legendFormat": "{{instance}}", "range": true, "refId": "netty-reader" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "avg_over_time(alluxio_rpc_executor_active_threads{job=\"worker\", executor_name=\"netty-writer\"}[5m])/avg_over_time(alluxio_rpc_executor_max_threads{job=\"worker\", executor_name=\"netty-writer\"}[5m]) > 0.5", "hide": true, "instant": false, "legendFormat": "{{instance}}", "range": true, "refId": "netty-writer" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "avg_over_time(alluxio_rpc_executor_active_threads{job=\"worker\", executor_name=\"grpc-metadata\"}[5m])/avg_over_time(alluxio_rpc_executor_max_threads{job=\"worker\", executor_name=\"grpc-metadata\"}[5m]) > 0.5", "hide": true, "instant": false, "legendFormat": "{{instance}}", "range": true, "refId": "grpc-metadata" } ], "title": "Read - Hotspot (load > 50%)", "transparent": true, "type": "bargauge" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the total number of files and pages currently stored in the Alluxio cache. This helps in understanding the composition and granularity of the cached data.\t", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "fieldMinMax": false, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 5, "w": 8, "x": 0, "y": 6 }, "id": 45, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "text": { "titleSize": 24, "valueSize": 36 }, "textMode": "value_and_name", "wideLayout": true }, "pluginVersion": "10.4.5", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum(alluxio_data_cached_files{job=\"worker\",cluster_name=~\"$cluster\"})", "instant": false, "legendFormat": "Files", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum(alluxio_data_cached_pages{job=\"worker\",cluster_name=~\"$cluster\"})", "format": "time_series", "hide": false, "instant": false, "legendFormat": "Pages", "range": true, "refId": "B" } ], "title": "Cache File", "transparent": true, "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Shows the write throughput from different sources: Fuse, S3 API, and direct Worker access. This helps identify the primary data access interfaces for write operations.\t", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 1, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "binBps" }, "overrides": [ { "matcher": { "id": "byName", "options": "Waiting" }, "properties": [ { "id": "unit", "value": "reqps" }, { "id": "thresholds", "value": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 100 }, { "color": "red", "value": 2000 } ] } } ] } ] }, "gridPos": { "h": 5, "w": 7, "x": 9, "y": 6 }, "id": 84, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "text": {}, "textMode": "auto", "wideLayout": true }, "pluginVersion": "10.4.5", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum(irate(alluxio_ufs_data_access_bytes_total{job=\"fuse\",method=\"write\",cluster_name=~\"$cluster\"}[5m])) or on() vector(0)", "hide": false, "instant": false, "legendFormat": "Fuse", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum(irate(alluxio_s3_api_throughput_bytes_sum{job=\"worker\",method=\"write\",cluster_name=~\"$cluster\"}[5m])) or on() vector(0)", "hide": false, "instant": false, "legendFormat": "S3", "range": true, "refId": "D" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum(irate(alluxio_data_throughput_bytes_total{job=\"worker\",destination=\"write_buffer\",method=\"write\"}[5m])) or on() vector(0)", "hide": false, "instant": false, "legendFormat": "Worker", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "avg(avg_over_time(alluxio_rpc_executor_current_queue_length{job=\"worker\",executor_name=\"grpc-metadata\"}[5m]))", "hide": false, "instant": false, "legendFormat": "Waiting", "range": true, "refId": "C" } ], "title": "Write - Throughput", "transparent": true, "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the average write load (thread utilization) across all workers over the last 5 minutes. A high average load may indicate a system-wide performance bottleneck.\t", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 2, "mappings": [], "max": 1, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 0.8 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 5, "w": 3, "x": 16, "y": 6 }, "id": 82, "options": { "minVizHeight": 75, "minVizWidth": 75, "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": false, "sizing": "auto" }, "pluginVersion": "10.4.5", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "avg(avg_over_time(alluxio_rpc_executor_active_threads{job=\"worker\", executor_name=\"netty-writer\"}[5m])/avg_over_time(alluxio_rpc_executor_max_threads{job=\"worker\", executor_name=\"netty-writer\"}[5m]))", "instant": false, "legendFormat": "{{label_name}}", "range": true, "refId": "A" } ], "title": "Write - Load (5m)", "transparent": true, "type": "gauge" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays per-worker write load (for workers exceeding 50% utilization), and highlights the most heavily used workers. This helps identify uneven I/O distribution and potential hotspots within the cluster.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 2, "mappings": [], "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 0.8 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 5, "w": 5, "x": 19, "y": 6 }, "id": 132, "options": { "displayMode": "basic", "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false }, "maxVizHeight": 300, "minVizHeight": 10, "minVizWidth": 0, "namePlacement": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showUnfilled": true, "sizing": "auto", "text": {}, "valueMode": "color" }, "pluginVersion": "10.4.5", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "avg_over_time(alluxio_rpc_executor_active_threads{job=\"worker\", executor_name=\"netty-reader\"}[5m])/avg_over_time(alluxio_rpc_executor_max_threads{job=\"worker\", executor_name=\"netty-reader\"}[5m]) > 0.5", "hide": true, "instant": false, "legendFormat": "{{instance}}", "range": true, "refId": "netty-reader" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "avg_over_time(alluxio_rpc_executor_active_threads{job=\"worker\", executor_name=\"netty-writer\"}[5m])/avg_over_time(alluxio_rpc_executor_max_threads{job=\"worker\", executor_name=\"netty-writer\"}[5m]) > 0.5", "hide": false, "instant": false, "legendFormat": "{{instance}}", "range": true, "refId": "netty-writer" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "avg_over_time(alluxio_rpc_executor_active_threads{job=\"worker\", executor_name=\"grpc-metadata\"}[5m])/avg_over_time(alluxio_rpc_executor_max_threads{job=\"worker\", executor_name=\"grpc-metadata\"}[5m]) > 0.5", "hide": true, "instant": false, "legendFormat": "{{instance}}", "range": true, "refId": "grpc-metadata" } ], "title": "Write - Hotspot (load > 50%)", "transparent": true, "type": "bargauge" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the number of live (healthy) and lost (unhealthy) workers in the cluster. This is a key indicator of cluster health and stability.\t", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "Lost" }, "properties": [ { "id": "thresholds", "value": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 1 } ] } } ] } ] }, "gridPos": { "h": 5, "w": 4, "x": 0, "y": 11 }, "id": 5, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "text": { "titleSize": 24, "valueSize": 36 }, "textMode": "value_and_name", "wideLayout": true }, "pluginVersion": "10.4.5", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum(up{job=\"worker\",cluster_name=~\"$cluster\"})", "instant": false, "legendFormat": "Live", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum by () (prometheus_target_scrape_pool_targets{scrape_job=\"worker\"}) - sum by () (up{job=\"worker\",cluster_name=~\"$cluster\"})", "hide": false, "instant": false, "legendFormat": "Lost", "range": true, "refId": "B" } ], "title": "Workers", "transparent": true, "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the number of currently running and waiting jobs in the job service queue. This helps monitor the status of asynchronous operations like data loading.\t", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short" }, "overrides": [ { "matcher": { "id": "byName", "options": "Waiting Jobs" }, "properties": [ { "id": "thresholds", "value": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 100 } ] } } ] } ] }, "gridPos": { "h": 5, "w": 4, "x": 4, "y": 11 }, "id": 74, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "text": { "titleSize": 24, "valueSize": 36 }, "textMode": "value_and_name", "wideLayout": true }, "pluginVersion": "10.4.5", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum(alluxio_active_job_count{job=\"coordinator\",type=\"running\"})", "instant": false, "legendFormat": "Running", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum(alluxio_active_job_count{job=\"coordinator\",type=\"waiting\"})", "hide": false, "instant": false, "legendFormat": "Waiting", "range": true, "refId": "B" } ], "title": "Jobs", "transparent": true, "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the metadata Requests Per Second (RPS) from different sources: Fuse, S3 API, and direct Worker access. This is important for monitoring metadata workload intensity.\t", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 1, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "reqps" }, "overrides": [ { "matcher": { "id": "byName", "options": "Waiting" }, "properties": [ { "id": "unit", "value": "reqps" }, { "id": "thresholds", "value": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 100 }, { "color": "red", "value": 2000 } ] } } ] } ] }, "gridPos": { "h": 5, "w": 7, "x": 9, "y": 11 }, "id": 81, "options": { "colorMode": "background", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "text": {}, "textMode": "auto", "wideLayout": true }, "pluginVersion": "10.4.5", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum(alluxio_fuse_concurrency{job=\"fuse\",method=~\"Fuse\\\\.Create|Fuse\\\\.Getattr|Fuse\\\\.Readdir|Fuse\\\\.Statfs|Fuse\\\\.Unlink\",cluster_name=~\"$cluster\"}) or on() vector(0)", "hide": false, "instant": false, "legendFormat": "Fuse", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum(irate(alluxio_s3_api_call_latency_ms_count{job=\"worker\", method!~\"GetObject|PutObject\",cluster_name=~\"$cluster\"}[5m])) or on() vector(0)", "hide": false, "instant": false, "legendFormat": "S3", "range": true, "refId": "S3" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum(irate(alluxio_meta_operation_total{job=\"worker\",cluster_name=~\"$cluster\"}[5m])) or on() vector(0)", "hide": false, "instant": false, "legendFormat": "Worker", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "avg(avg_over_time(alluxio_rpc_executor_current_queue_length{job=\"worker\",executor_name=\"grpc-metadata\"}[5m]))", "hide": false, "instant": false, "legendFormat": "Waiting", "range": true, "refId": "C" } ], "title": "Meta - RPS", "transparent": true, "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the average metadata operation load (thread utilization) across all workers over the last 5 minutes. High load can indicate a metadata-intensive workload.\t", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 2, "mappings": [], "max": 1, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 0.8 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 5, "w": 3, "x": 16, "y": 11 }, "id": 85, "options": { "minVizHeight": 75, "minVizWidth": 75, "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": false, "sizing": "auto" }, "pluginVersion": "10.4.5", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "avg(avg_over_time(alluxio_rpc_executor_active_threads{job=\"worker\", executor_name=\"grpc-metadata\"}[5m])/avg_over_time(alluxio_rpc_executor_max_threads{job=\"worker\", executor_name=\"grpc-metadata\"}[5m]))", "instant": false, "legendFormat": "{{label_name}}", "range": true, "refId": "A" } ], "title": "Meta - Load (5m)", "transparent": true, "type": "gauge" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays per-worker metadata operation load (for workers exceeding 50% utilization), and highlights the most heavily used workers. This helps identify uneven I/O distribution and potential hotspots within the cluster.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 2, "mappings": [], "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 0.8 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 5, "w": 5, "x": 19, "y": 11 }, "id": 133, "options": { "displayMode": "basic", "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false }, "maxVizHeight": 300, "minVizHeight": 10, "minVizWidth": 0, "namePlacement": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showUnfilled": true, "sizing": "auto", "text": {}, "valueMode": "color" }, "pluginVersion": "10.4.5", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "avg_over_time(alluxio_rpc_executor_active_threads{job=\"worker\", executor_name=\"netty-reader\"}[5m])/avg_over_time(alluxio_rpc_executor_max_threads{job=\"worker\", executor_name=\"netty-reader\"}[5m]) > 0.5", "hide": true, "instant": false, "legendFormat": "{{instance}}", "range": true, "refId": "netty-reader" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "avg_over_time(alluxio_rpc_executor_active_threads{job=\"worker\", executor_name=\"netty-writer\"}[5m])/avg_over_time(alluxio_rpc_executor_max_threads{job=\"worker\", executor_name=\"netty-writer\"}[5m]) > 0.5", "hide": true, "instant": false, "legendFormat": "{{instance}}", "range": true, "refId": "netty-writer" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "avg_over_time(alluxio_rpc_executor_active_threads{job=\"worker\", executor_name=\"grpc-metadata\"}[5m])/avg_over_time(alluxio_rpc_executor_max_threads{job=\"worker\", executor_name=\"grpc-metadata\"}[5m]) > 0.5", "hide": false, "instant": false, "legendFormat": "{{instance}}", "range": true, "refId": "grpc-metadata" } ], "title": "Meta - Hotspot (load > 50%)", "transparent": true, "type": "bargauge" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the expiration date of the Alluxio enterprise license. This is important for ensuring uninterrupted access to enterprise features.\t", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "fieldMinMax": false, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "dateTimeFromNow" }, "overrides": [] }, "gridPos": { "h": 5, "w": 4, "x": 0, "y": 16 }, "id": 46, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "percentChangeColorMode": "standard", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "text": {}, "textMode": "value_and_name", "wideLayout": true }, "pluginVersion": "10.4.5", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "min(alluxio_license_expiration_date{job=~\"coordinator|worker\"}) * 1000", "format": "time_series", "instant": false, "legendFormat": " ", "range": true, "refId": "A" } ], "title": "License (Valid To)", "transparent": true, "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the distribution of Alluxio versions running across the cluster. This is useful for verifying version consistency during upgrades or troubleshooting.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "hideFrom": { "legend": false, "tooltip": false, "viz": false } }, "mappings": [] }, "overrides": [] }, "gridPos": { "h": 7, "w": 4, "x": 4, "y": 16 }, "id": 72, "options": { "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "pieType": "pie", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "tooltip": { "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "count (alluxio_version_info{cluster_name=~\"$cluster\"}) by (version)", "instant": false, "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Component Version", "transparent": true, "type": "piechart" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the uptime of each Alluxio component (coordinator, worker, fuse). This helps track service stability and identify recent restarts.\t", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 7, "w": 15, "x": 9, "y": 16 }, "id": 2, "options": { "displayMode": "basic", "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false }, "maxVizHeight": 300, "minVizHeight": 10, "minVizWidth": 0, "namePlacement": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showUnfilled": true, "sizing": "auto", "valueMode": "color" }, "pluginVersion": "10.4.5", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "timestamp(process_start_time_seconds{job=~\"coordinator|worker|fuse\",cluster_name=~\"$cluster\"}) - process_start_time_seconds{job=~\"coordinator|worker|fuse\",cluster_name=~\"$cluster\"}", "instant": false, "legendFormat": "{{job}} - {{instance}}", "range": true, "refId": "A" } ], "title": "Component Uptime", "transparent": true, "type": "bargauge" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the percentage of data/metadata read requests served from the Alluxio cache. A higher value indicates better cache efficiency and reduced load on the underlying storage.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "yellow", "value": null }, { "color": "green", "value": 0.8 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 23 }, "id": 20, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum(irate(alluxio_cached_data_read_bytes_total{job=\"worker\",cluster_name=~\"$cluster\"}[5m])) / (sum(irate(alluxio_cached_data_read_bytes_total{job=\"worker\",cluster_name=~\"$cluster\"}[5m])) + (sum(irate(alluxio_missed_data_read_bytes_total{job=\"worker\",cluster_name=~\"$cluster\"}[5m])) or on() vector(0)))", "hide": false, "instant": false, "legendFormat": "Data", "range": true, "refId": "Data" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum(irate(alluxio_metadata_cache_hit_calls_total{job=\"worker\",cluster_name=~\"$cluster\"}[5m])) / (sum(irate(alluxio_metadata_cache_hit_calls_total{job=\"worker\",cluster_name=~\"$cluster\"}[5m])) + sum(irate(alluxio_metadata_cache_miss_calls_total{job=\"worker\",cluster_name=~\"$cluster\"}[5m])) or on() vector(0))", "hide": false, "instant": false, "legendFormat": "Metadata", "range": true, "refId": "Metadata" } ], "title": "Cache Hit(%)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the rate at which data is being evicted from the cache. A high eviction rate may indicate insufficient cache capacity for the workload.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "displayName": "Cache Eviction", "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "binBps" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 23 }, "id": 27, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": false }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum(irate(alluxio_cached_evicted_data_bytes_total{job=\"worker\",cluster_name=~\"$cluster\"}[5m]))", "instant": false, "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Cache Eviction", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Provides a detailed breakdown of read throughput across different data paths (FUSE, S3, Worker, UFS). Essential for understanding data flow and identifying bottlenecks.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "binBps" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 30 }, "id": 19, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum(irate(alluxio_data_access_bytes_sum{job=\"fuse\",cluster_name=~\"$cluster\"}[5m])) or on() vector(0)", "instant": false, "legendFormat": "FUSE", "range": true, "refId": "FUSE" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum(irate(alluxio_ufs_data_access_bytes_total{job=\"fuse\",method=\"read\",cluster_name=~\"$cluster\"}[5m])) or on() vector(0)", "hide": false, "instant": false, "legendFormat": "FUSE fallback", "range": true, "refId": "FUSE fallback" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum(irate(alluxio_s3_api_throughput_bytes_sum{job=\"worker\",method=\"read\",cluster_name=~\"$cluster\"}[5m])) or on() vector", "hide": false, "instant": false, "legendFormat": "S3", "range": true, "refId": "S3" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum(irate(alluxio_data_throughput_bytes_total{job=\"worker\",method=\"read\",cluster_name=~\"$cluster\"}[5m])) or on() vector(0)", "hide": false, "instant": false, "legendFormat": "Worker", "range": true, "refId": "worker" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum(irate(alluxio_ufs_data_access_bytes_total{job=\"worker\",method=\"read\",cluster_name=~\"$cluster\"}[5m])) or on() vector(0)", "hide": false, "instant": false, "legendFormat": "UFS", "range": true, "refId": "UFS" } ], "title": "Throughput - Read", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Provides a detailed breakdown of write throughput across different data paths (FUSE, S3, Worker, UFS). Critical for understanding write efficiency and patterns.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "binBps" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 30 }, "id": 18, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum(irate(alluxio_ufs_data_access_bytes_total{job=\"fuse\",method=\"write\",cluster_name=~\"$cluster\"}[5m])) or on() vector(0)", "instant": false, "legendFormat": "FUSE", "range": true, "refId": "FUSE" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum(irate(alluxio_s3_api_throughput_bytes_sum{job=\"worker\",method=\"write\",cluster_name=~\"$cluster\"}[5m])) or on() vector", "hide": false, "instant": false, "legendFormat": "S3", "range": true, "refId": "S3" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum(irate(alluxio_data_throughput_bytes_total{job=\"worker\",destination=\"write_buffer\",method=\"write\"}[5m])) or on() vector(0)", "hide": false, "instant": false, "legendFormat": "Worker", "range": true, "refId": "Worker" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum(irate(alluxio_write_buffer_async_persist_throughput_bytes_total{job=\"worker\"}[5m])) or on() vector(0)", "hide": false, "instant": false, "legendFormat": "UFS (Write Back)", "range": true, "refId": "UFS" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum(irate(alluxio_ufs_data_access_bytes_total{job=\"worker\",method=\"write\",cluster_name=~\"$cluster\"}[5m])) or on() vector(0)", "hide": false, "instant": false, "legendFormat": "UFS (All)", "range": true, "refId": "UFS All" } ], "title": "Throughput - Write", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the total read request rate across all interfaces (FUSE, S3, Worker). Important for understanding read workload intensity and traffic patterns.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "reqps" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 37 }, "id": 34, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum(irate(alluxio_data_access_bytes_count{job=\"fuse\",cluster_name=~\"$cluster\"}[5m])) or on() vector(0)", "instant": false, "legendFormat": "FUSE", "range": true, "refId": "FUSE" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum(irate(alluxio_s3_api_throughput_bytes_count{method=\"read\",job=\"worker\",instance=~\"$instance\",cluster_name=~\"$cluster\"}[5m])) or on() vector(0)", "hide": false, "instant": false, "legendFormat": "S3", "range": true, "refId": "S3" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum(irate(alluxio_data_access_bytes_count{job=\"worker\",method=\"read\",cluster_name=~\"$cluster\"}[5m])) or on() vector(0)", "hide": false, "instant": false, "legendFormat": "Worker", "range": true, "refId": "worker" } ], "title": "Request/s - Read", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the total write request rate across all interfaces. Critical for understanding write workload patterns and identifying write-heavy phases.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "reqps" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 37 }, "id": 76, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum(irate(alluxio_data_access_bytes_count{job=\"fuse\",method=\"write\",cluster_name=~\"$cluster\"}[5m])) or on() vector(0)", "hide": false, "instant": false, "legendFormat": "FUSE", "range": true, "refId": "FUSE" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum(sum(irate(alluxio_s3_api_throughput_bytes_count{method=\"write\",job=\"worker\",instance=~\"$instance\",cluster_name=~\"$cluster\"}[5m])) or on() vector(0)) or on() vector(0)", "hide": false, "instant": false, "legendFormat": "S3", "range": true, "refId": "S3" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum(irate(alluxio_data_access_bytes_count{job=\"worker\",method=\"write\",cluster_name=~\"$cluster\"}[5m])) or on() vector(0)", "hide": false, "instant": false, "legendFormat": "Worker", "range": true, "refId": "worker" } ], "title": "Request/s - Write", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the metadata operation request rate across all interfaces. Important for understanding metadata workload intensity, which can impact overall cluster performance.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "reqps" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 44 }, "id": 35, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum(alluxio_fuse_concurrency{job=\"fuse\",method=~\"Fuse\\\\.Create|Fuse\\\\.Getattr|Fuse\\\\.Readdir|Fuse\\\\.Statfs|Fuse\\\\.Unlink\",cluster_name=~\"$cluster\"}) or on() vector(0)", "instant": false, "legendFormat": "FUSE", "range": true, "refId": "FUSE" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum(irate(alluxio_s3_api_call_latency_ms_count{job=\"worker\", method!~\"GetObject|PutObject\",cluster_name=~\"$cluster\"}[5m])) or on() vector(0)", "hide": false, "instant": false, "legendFormat": "S3", "range": true, "refId": "S3" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum(irate(alluxio_meta_operation_total{job=\"worker\",cluster_name=~\"$cluster\"}[5m])) or on() vector(0)", "hide": false, "instant": false, "legendFormat": "Worker", "range": true, "refId": "worker" } ], "title": "Request/s - Metadata", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Time series visualization of job execution patterns. Important for understanding cluster background activity levels and identifying periods of high maintenance activity that might impact performance.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 44 }, "id": 69, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum(alluxio_active_job_count{job=\"coordinator\",type=\"running\"})", "instant": false, "legendFormat": "Running Jobs", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum(alluxio_active_job_count{job=\"coordinator\",type=\"waiting\"})", "hide": false, "instant": false, "legendFormat": "Waiting Jobs", "range": true, "refId": "B" } ], "title": "Jobs", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the 90th percentile latency for all request types. This metric is key to understanding the typical user experience and identifying performance degradation.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "ms" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 51 }, "id": 90, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "right", "showLegend": true, "width": 160 }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "histogram_quantile(0.90, sum(rate(alluxio_fuse_call_latency_ms_bucket{job=\"fuse\",cluster_name=~\"$cluster\"}[5m])) by (le, method))", "instant": false, "legendFormat": "{{method}}", "range": true, "refId": "FUSE" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "histogram_quantile(0.90, sum(rate(alluxio_meta_operation_latency_ms_bucket{job=\"worker\",cluster_name=~\"$cluster\"}[5m])) by (le, op))", "hide": false, "instant": false, "legendFormat": "Worker.{{op}}", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "histogram_quantile(0.90, sum(rate(alluxio_s3_api_call_latency_ms_bucket{job=\"worker\",cluster_name=~\"$cluster\"}[5m])) by (le, method))", "hide": false, "instant": false, "legendFormat": "S3.{{method}}", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "histogram_quantile(0.90, sum(rate(alluxio_ufs_latency_ms_bucket{job=\"worker\",cluster_name=~\"$cluster\"}[5m])) by (le, method, ufs_type))", "hide": false, "instant": false, "legendFormat": "UFS.{{method}}-{{ufs_type}}", "range": true, "refId": "C" } ], "title": "Request Latency - P90", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the 99th percentile latency for all request types. This metric is critical for identifying worst-case performance and outliers that may indicate system stress.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "ms" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 51 }, "id": 93, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "right", "showLegend": true, "width": 160 }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "histogram_quantile(0.99, sum(rate(alluxio_fuse_call_latency_ms_bucket{job=\"fuse\",cluster_name=~\"$cluster\"}[5m])) by (le, method))", "instant": false, "legendFormat": "{{method}}", "range": true, "refId": "FUSE" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "histogram_quantile(0.99, sum(rate(alluxio_meta_operation_latency_ms_bucket{job=\"worker\",cluster_name=~\"$cluster\"}[5m])) by (le, op))", "hide": false, "instant": false, "legendFormat": "Worker.{{op}}", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "histogram_quantile(0.99, sum(rate(alluxio_s3_api_call_latency_ms_bucket{job=\"worker\",cluster_name=~\"$cluster\"}[5m])) by (le, method))", "hide": false, "instant": false, "legendFormat": "S3.{{method}}", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "histogram_quantile(0.99, sum(rate(alluxio_ufs_latency_ms_bucket{job=\"worker\",cluster_name=~\"$cluster\"}[5m])) by (le, method, ufs_type))", "hide": false, "instant": false, "legendFormat": "UFS.{{method}}-{{ufs_type}}", "range": true, "refId": "C" } ], "title": "Request Latency - P99", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the estimated network latency for client GetStatus gRPC calls (metadata operations), calculated as the difference between client-observed latency and worker metadata processing latency. This helps identify network-related delays and connectivity issues between clients and the cluster.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "ms" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 58 }, "id": 138, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "right", "showLegend": true, "width": 160 }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "(histogram_quantile(0.90, sum(rate(alluxio_grpc_client_call_latency_ms_bucket{job=\"fuse\",method=\"GetStatus\",instance=~\"$instance\",cluster_name=~\"$cluster\"}[5m])) by (le))\n-\nhistogram_quantile(0.90, sum(rate(alluxio_meta_operation_latency_ms_bucket{job=\"worker\",op=\"getStatus\",cluster_name=~\"$cluster\"}[5m])) by (le))) > 0", "hide": false, "instant": false, "legendFormat": "Latency", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "(histogram_quantile(0.90, sum(rate(alluxio_fuse_call_latency_ms_bucket{job=\"fuse\",method=\"Fuse.Getattr\",cluster_name=~\"$cluster\"}[5m])) by (le))\n-\nhistogram_quantile(0.90, sum(rate(alluxio_meta_operation_latency_ms_bucket{job=\"worker\",op=\"getStatus\",cluster_name=~\"$cluster\"}[5m])) by (le))) > 0", "hide": true, "instant": false, "legendFormat": "Latency (FUSE)", "range": true, "refId": "B" } ], "title": "RPC (Client<->Worker) Latency - P90", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the estimated network latency for client GetStatus gRPC calls (metadata operations), calculated as the difference between client-observed latency and worker metadata processing latency. This helps identify network-related delays and connectivity issues between clients and the cluster.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "ms" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 58 }, "id": 139, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "right", "showLegend": true, "width": 160 }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "(histogram_quantile(0.99, sum(rate(alluxio_grpc_client_call_latency_ms_bucket{job=\"fuse\",method=\"GetStatus\",instance=~\"$instance\",cluster_name=~\"$cluster\"}[5m])) by (le))\n-\nhistogram_quantile(0.99, sum(rate(alluxio_meta_operation_latency_ms_bucket{job=\"worker\",op=\"getStatus\",cluster_name=~\"$cluster\"}[5m])) by (le))) > 0", "hide": false, "instant": false, "legendFormat": "Latency", "range": true, "refId": "A" } ], "title": "RPC (Client<->Worker) Latency - P99", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the success rate for all request types across interfaces. A value below 99% (excluding Not Found errors) typically indicates system issues that require investigation.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "yellow", "value": null }, { "color": "green", "value": 0.9 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 65 }, "id": 107, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "right", "showLegend": true, "width": 160 }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "(sum by (method) (irate(alluxio_fuse_call_latency_ms_count{job=\"fuse\", success=\"true\",cluster_name=~\"$cluster\"}[5m])))\n/\n(sum by (method) (irate(alluxio_fuse_call_latency_ms_count{job=\"fuse\", cluster_name=~\"$cluster\"}[5m])))", "instant": false, "legendFormat": "{{method}}", "range": true, "refId": "FUSE" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "(sum by (method) (irate(alluxio_s3_api_call_latency_ms_count{job=\"worker\", success=\"true\",cluster_name=~\"$cluster\"}[5m]))) \n/\n(sum by (method) (irate(alluxio_s3_api_call_latency_ms_count{job=\"worker\", cluster_name=~\"$cluster\"}[5m])))", "hide": false, "instant": false, "legendFormat": "S3.{{method}}", "range": true, "refId": "S3" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "(sum by (op) (irate(alluxio_meta_operation_total{job=\"worker\",cluster_name=~\"$cluster\"}[5m]) - irate(alluxio_meta_operation_errors_total{job=\"worker\",cluster_name=~\"$cluster\"}[5m])))\n/\n(sum by (op) (irate(alluxio_meta_operation_total{job=\"worker\",cluster_name=~\"$cluster\"}[5m])))", "hide": false, "instant": false, "legendFormat": "Worker.{{op}}", "range": true, "refId": "Worker" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "(sum by (ufs_type) (irate(alluxio_ufs_total{cluster_name=~\"$cluster\"}[5m]) - irate(alluxio_ufs_error_total{cluster_name=~\"$cluster\"}[5m])))\n/\n(sum by (ufs_type) (irate(alluxio_ufs_latency_ms_count{cluster_name=~\"$cluster\"}[5m])))", "hide": false, "instant": false, "legendFormat": "UFS-{{ufs_type}}-{{error_code}}", "range": true, "refId": "UFS" } ], "title": "Request Call - Success Rate", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the failure rate across all request types and interfaces. Spikes in this metric require immediate investigation to identify system or application errors.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 65 }, "id": 88, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "right", "showLegend": true, "width": 160 }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum by (method, state) (irate(alluxio_fuse_result_total{job=\"fuse\",state!=\"SUCCESS\", cluster_name=~\"$cluster\"}[5m]))", "instant": false, "legendFormat": "{{method}}-{{state}}", "range": true, "refId": "FUSE" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum by (method, status) (irate(alluxio_s3_api_call_latency_ms_count{job=\"worker\", success=\"false\",cluster_name=~\"$cluster\"}[5m]))", "hide": false, "instant": false, "legendFormat": "S3.{{method}}-{{status}}", "range": true, "refId": "S3" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum by (op) (irate(alluxio_meta_operation_errors_total{job=\"worker\",cluster_name=~\"$cluster\"}[5m]))", "hide": false, "instant": false, "legendFormat": "Worker.{{op}}", "range": true, "refId": "Worker" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum by (ufs_type,error_code) (irate(alluxio_ufs_error_total{cluster_name=~\"$cluster\"}[5m]))", "hide": false, "instant": false, "legendFormat": "UFS-{{ufs_type}}-{{error_code}}", "range": true, "refId": "UFS" } ], "title": "Request Call - Failures", "type": "timeseries" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 72 }, "id": 21, "panels": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the 90th percentile latency for individual Fuse mount operations. This helps isolate performance issues specific to a particular Fuse client.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "ms" }, "overrides": [] }, "gridPos": { "h": 6, "w": 12, "x": 0, "y": 73 }, "id": 91, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "right", "showLegend": true, "width": 160 }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "histogram_quantile(0.90, rate(alluxio_fuse_call_latency_ms_bucket{job=\"fuse\",instance=~\"$instance\",cluster_name=~\"$cluster\"}[5m]))", "instant": false, "legendFormat": "{{method}}({{instance}})", "range": true, "refId": "FUSE" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "histogram_quantile(\n 0.90,\n sum(\n rate(alluxio_client_netty_read_time_to_receive_first_packet_ms_bucket{instance=~\"$instance\",cluster_name=~\"$cluster\"}[5m])\n ) by (le, instance)\n)", "hide": false, "instant": false, "legendFormat": "__auto", "range": true, "refId": "netty" } ], "title": "Request Latency - P90", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the 99th percentile latency for individual Fuse mount operations. This helps isolate performance issues specific to a particular Fuse client.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "ms" }, "overrides": [] }, "gridPos": { "h": 6, "w": 12, "x": 12, "y": 73 }, "id": 126, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "right", "showLegend": true, "width": 160 }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "histogram_quantile(0.99, rate(alluxio_fuse_call_latency_ms_bucket{job=\"fuse\",instance=~\"$instance\",cluster_name=~\"$cluster\"}[5m]))", "instant": false, "legendFormat": "{{method}}({{instance}})", "range": true, "refId": "FUSE" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "histogram_quantile(\n 0.99,\n sum(\n rate(alluxio_client_netty_read_time_to_receive_first_packet_ms_bucket{instance=~\"$instance\",cluster_name=~\"$cluster\"}[5m])\n ) by (le, instance)\n)", "hide": false, "instant": false, "legendFormat": "__auto", "range": true, "refId": "netty" } ], "title": "Request Latency - P99", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the read throughput for an individual Fuse mount. This is useful for monitoring the read activity of a specific client.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "binBps" }, "overrides": [] }, "gridPos": { "h": 6, "w": 12, "x": 0, "y": 79 }, "id": 37, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "irate(alluxio_data_access_bytes_sum{job=\"fuse\",method=\"read\",instance=~\"$instance\",cluster_name=~\"$cluster\"}[5m])", "instant": false, "legendFormat": "{{instance}}", "range": true, "refId": "FUSE" } ], "title": "Throughput - Read", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the write throughput for an individual Fuse mount. This is useful for monitoring the write activity of a specific client.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "binBps" }, "overrides": [] }, "gridPos": { "h": 6, "w": 12, "x": 12, "y": 79 }, "id": 38, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "irate(alluxio_data_access_bytes_sum{job=\"fuse\",method=\"write\",instance=~\"$instance\",cluster_name=~\"$cluster\"}[5m])", "instant": false, "legendFormat": "{{instance}}", "range": true, "refId": "FUSE" } ], "title": "Throughput - Write", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the read request rate for an individual Fuse mount. This helps understand the read workload from a specific client.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "reqps" }, "overrides": [] }, "gridPos": { "h": 6, "w": 12, "x": 0, "y": 85 }, "id": 39, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "irate(alluxio_data_access_bytes_count{job=\"fuse\",method=\"read\",instance=~\"$instance\",cluster_name=~\"$cluster\"}[5m])", "instant": false, "legendFormat": "{{instance}}", "range": true, "refId": "FUSE" } ], "title": "Request/s - Read", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the write request rate for an individual Fuse mount. This helps understand the write workload from a specific client.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "reqps" }, "overrides": [] }, "gridPos": { "h": 6, "w": 12, "x": 12, "y": 85 }, "id": 40, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "irate(alluxio_data_access_bytes_count{job=\"fuse\",method=\"write\",instance=~\"$instance\",cluster_name=~\"$cluster\"}[5m])", "instant": false, "legendFormat": "{{instance}}", "range": true, "refId": "FUSE" } ], "title": "Request/s - Write", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the failure rate for an individual Fuse mount. This is critical for diagnosing client-specific connectivity or operational issues.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 6, "w": 12, "x": 0, "y": 91 }, "id": 89, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "right", "showLegend": true, "width": 160 }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, "expr": "irate(alluxio_fuse_result_total{job=\"fuse\",state!=\"SUCCESS\", instance=~\"$instance\",cluster_name=~\"$cluster\"}[5m])", "instant": false, "legendFormat": "{{method}}-{{state}}({{instance}})", "range": true, "refId": "A" } ], "title": "FUSE Request Failure", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Read throughput when FUSE operations fall back to underlying filesystem (UFS). High fallback rates indicate cache misses or data not in Alluxio, reducing performance benefits. Should be minimal for optimal performance.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "binBps" }, "overrides": [] }, "gridPos": { "h": 6, "w": 12, "x": 12, "y": 91 }, "id": 25, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": false, "expr": "irate(alluxio_ufs_data_access_bytes_total{job=\"fuse\",instance=~\"$instance\",method=\"read\",cluster_name=~\"$cluster\"}[5m]) or on() vector(0)", "instant": false, "legendFormat": "{{instance}}", "range": true, "refId": "A" } ], "title": "Throughput - UFS Fallback", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the latency distribution of gRPC (metadata ops) and Netty (data ops) client calls. This helps identify RPC communication delays, client-side bottlenecks, and potential network or service responsiveness issues within the workers.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "ms" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 97 }, "id": 115, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "right", "showLegend": true, "width": 160 }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "histogram_quantile(\n 0.99,\n sum(\n rate(alluxio_grpc_client_call_latency_ms_bucket{job=\"fuse\",instance=~\"$instance\",cluster_name=~\"$cluster\"}[5m])\n ) by (le, method, instance)\n)", "instant": false, "legendFormat": "{{method}}({{instance}})", "range": true, "refId": "grpc" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "histogram_quantile(\n 0.99,\n sum(\n rate(alluxio_client_netty_read_time_to_receive_first_packet_ms_bucket{instance=~\"$instance\",cluster_name=~\"$cluster\"}[5m])\n ) by (le, instance)\n)", "hide": false, "instant": false, "legendFormat": "Read-TTFB ({{instance}})", "range": true, "refId": "netty" } ], "title": "Client Request Latency (P99)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the number of concurrent gRPC client calls (metadata ops). This helps identify spikes in metadata request load and potential client-side bottlenecks within the worker.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "reqps" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 97 }, "id": 113, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "right", "showLegend": true, "width": 160 }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "alluxio_grpc_client_concurrency{job=\"fuse\",instance=~\"$instance\",cluster_name=~\"$cluster\"}", "instant": false, "legendFormat": "{{method}}({{instance}})", "range": true, "refId": "FUSE" } ], "title": "Client Request Concurrency", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the error rate of gRPC client calls (metadata operations) and Netty operation calls (data operations). This helps identify failing metadata requests and potential reliability issues in client-worker communication within the cluster.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "yellow", "value": null }, { "color": "green", "value": 0.9 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 104 }, "id": 111, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "right", "showLegend": true, "width": 160 }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum by (instance,method) (irate(alluxio_grpc_client_errors_total{instance=~\"$instance\",cluster_name=~\"$cluster\"}[5m]))\n/\n(sum by (instance,method) (irate(alluxio_grpc_client_errors_total{instance=~\"$instance\",cluster_name=~\"$cluster\"}[5m])) + sum by (instance,method) (irate(alluxio_grpc_client_successes_total{instance=~\"$instance\",cluster_name=~\"$cluster\"}[5m])))", "instant": false, "legendFormat": "{{method}}({{instance}})", "range": true, "refId": "meta" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum by (instance,op) (irate(alluxio_netty_operation_errors_total{instance=~\"$instance\",cluster_name=~\"$cluster\"}[5m]))\n/\n(sum by (instance,op) (irate(alluxio_netty_operations_total{instance=~\"$instance\",cluster_name=~\"$cluster\"}[5m])))", "hide": false, "instant": false, "legendFormat": "{{method}}({{instance}})", "range": true, "refId": "data" } ], "title": "Client Request Failure Rate", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the total number of gRPC client errors (metadata operations) and Netty operation errors (data operations). This helps track failing requests and potential reliability issues in client-worker communication within the cluster.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 104 }, "id": 112, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "right", "showLegend": true, "width": 160 }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "irate(alluxio_grpc_client_errors_total{instance=~\"$instance\",cluster_name=~\"$cluster\"}[5m])", "instant": false, "legendFormat": "{{method}}-{{status_code}}({{instance}})", "range": true, "refId": "a" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "irate(alluxio_netty_operation_errors_total{instance=~\"$instance\",cluster_name=~\"$cluster\"}[5m])", "hide": false, "instant": false, "legendFormat": "{{op}}-{{reason}}({{instance}})", "range": true, "refId": "b" } ], "title": "Client Request Errors", "type": "timeseries" } ], "title": "Fuse", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 73 }, "id": 99, "panels": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the 90th percentile latency for S3 API operations on a per-worker basis. This helps isolate S3 performance issues to specific workers.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "ms" }, "overrides": [] }, "gridPos": { "h": 6, "w": 12, "x": 0, "y": 3 }, "id": 100, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "right", "showLegend": true, "width": 160 }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "histogram_quantile(0.90, rate(alluxio_s3_api_call_latency_ms_bucket{job=\"worker\",instance=~\"$instance\",cluster_name=~\"$cluster\"}[5m]))", "instant": false, "legendFormat": "{{method}}({{instance}})", "range": true, "refId": "FUSE" } ], "title": "Request Latency - P90", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the 99th percentile latency for S3 API operations on a per-worker basis. This helps identify worst-case S3 performance on specific workers.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "ms" }, "overrides": [] }, "gridPos": { "h": 6, "w": 12, "x": 12, "y": 3 }, "id": 101, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "right", "showLegend": true, "width": 160 }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "histogram_quantile(0.99, rate(alluxio_s3_api_call_latency_ms_bucket{job=\"worker\",instance=~\"$instance\",cluster_name=~\"$cluster\"}[5m]))", "instant": false, "legendFormat": "{{method}}({{instance}})", "range": true, "refId": "FUSE" } ], "title": "Request Latency - P99", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the read throughput for S3 API operations on a per-worker basis. This is useful for monitoring S3 read activity on specific workers.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "binBps" }, "overrides": [] }, "gridPos": { "h": 6, "w": 12, "x": 0, "y": 9 }, "id": 102, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "irate(alluxio_s3_api_throughput_bytes_sum{job=\"worker\",method=\"read\",instance=~\"$instance\",cluster_name=~\"$cluster\"}[5m])", "instant": false, "legendFormat": "{{instance}}", "range": true, "refId": "FUSE" } ], "title": "Throughput - Read", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the write throughput for S3 API operations on a per-worker basis. This is useful for monitoring S3 write activity on specific workers.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "binBps" }, "overrides": [] }, "gridPos": { "h": 6, "w": 12, "x": 12, "y": 9 }, "id": 103, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "irate(alluxio_s3_api_throughput_bytes_sum{job=\"worker\",method=\"write\",instance=~\"$instance\",cluster_name=~\"$cluster\"}[5m])", "instant": false, "legendFormat": "{{instance}}", "range": true, "refId": "FUSE" } ], "title": "Throughput - Write", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the read request rate for S3 API operations on a per-worker basis. This helps understand the S3 read workload on specific workers.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "reqps" }, "overrides": [] }, "gridPos": { "h": 6, "w": 12, "x": 0, "y": 15 }, "id": 104, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "irate(alluxio_s3_api_throughput_bytes_count{method=\"read\",job=\"worker\",instance=~\"$instance\",cluster_name=~\"$cluster\"}[5m])", "instant": false, "legendFormat": "{{instance}}", "range": true, "refId": "FUSE" } ], "title": "Request/s - Read", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the write request rate for S3 API operations on a per-worker basis. This helps understand the S3 write workload on specific workers.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "reqps" }, "overrides": [] }, "gridPos": { "h": 6, "w": 12, "x": 12, "y": 15 }, "id": 105, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "irate(alluxio_s3_api_throughput_bytes_count{method=\"write\",job=\"worker\",instance=~\"$instance\",cluster_name=~\"$cluster\"}[5m])", "instant": false, "legendFormat": "{{instance}}", "range": true, "refId": "FUSE" } ], "title": "Request/s - Write", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the failure rate for S3 API operations on a per-worker basis. This is critical for diagnosing S3-specific issues on individual workers.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 6, "w": 12, "x": 0, "y": 21 }, "id": 106, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "right", "showLegend": true, "width": 160 }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "irate(alluxio_s3_api_call_latency_ms_count{job=\"worker\", success=\"false\",instance=~\"$instance\",cluster_name=~\"$cluster\"}[5m])", "instant": false, "legendFormat": "{{method}}-{{status}}({{instance}})", "range": true, "refId": "FUSE" } ], "title": "Request Failure", "type": "timeseries" } ], "title": "S3", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 74 }, "id": 16, "panels": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the amount of cache data storage used by each individual worker. This is useful for identifying uneven storage distribution.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 4 }, "id": 28, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "alluxio_cached_storage_bytes{job=\"worker\",instance=~\"$instance\",cluster_name=~\"$cluster\"}", "instant": false, "legendFormat": "{{instance}}", "range": true, "refId": "A" } ], "title": "Storage (Data) Used", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the amount of cache metadata storage used by each individual worker. This is useful for identifying uneven storage distribution.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 4 }, "id": 134, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "alluxio_metastore_storage_size_bytes{job=\"worker\",instance=~\"$instance\",cluster_name=~\"$cluster\"}", "instant": false, "legendFormat": "{{instance}}-{{dir}}", "range": true, "refId": "A" } ], "title": "Storage (Meta) Used", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the page store device capacity utilization per directory on workers. This helps identify disk space consumption trends and potential storage exhaustion or imbalance within the cluster.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "max": 1, "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 0.8 }, { "color": "red", "value": 0.9 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 4 }, "id": 135, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "1-\n(sum by (instance, dir) (alluxio_page_store_device_available_capacity_bytes{job=\"worker\",instance=~\"$instance\",cluster_name=~\"$cluster\"})\n/\nsum by (instance, dir) (alluxio_page_store_device_total_capacity_bytes{job=\"worker\",instance=~\"$instance\",cluster_name=~\"$cluster\"}))", "hide": false, "instant": false, "legendFormat": "{{instance}}-{{dir}}", "range": true, "refId": "A" } ], "title": "Storage Device Usage", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the number of cached files for each individual worker. This helps in understanding the cache composition on a per-worker basis.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 11 }, "id": 47, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "alluxio_data_cached_files{job=\"worker\",instance=~\"$instance\",cluster_name=~\"$cluster\"}", "hide": false, "instant": false, "legendFormat": "{{instance}}", "range": true, "refId": "A" } ], "title": "Files", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the number of cached pages for each individual worker. This helps in understanding the cache composition on a per-worker basis.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 11 }, "id": 136, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "alluxio_data_cached_pages{job=\"worker\",instance=~\"$instance\",cluster_name=~\"$cluster\"}", "hide": false, "instant": false, "legendFormat": "{{instance}}", "range": true, "refId": "A" } ], "title": "Pages", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the rate of data eviction from the cache for each individual worker. This helps identify which workers are under the most memory pressure.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "binBps" }, "overrides": [] }, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 11 }, "id": 17, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "irate(alluxio_cached_evicted_data_bytes_total{job=\"worker\",instance=~\"$instance\",cluster_name=~\"$cluster\"}[5m])", "instant": false, "legendFormat": "{{instance}}", "range": true, "refId": "A" } ], "title": "Cache Evicted", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the 90th percentile latency for internal metadata operations on a per-worker basis.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "ms" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 18 }, "id": 95, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "right", "showLegend": true, "width": 160 }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "histogram_quantile(0.90, rate(alluxio_meta_operation_latency_ms_bucket{job=\"worker\",instance=~\"$instance\",cluster_name=~\"$cluster\"}[5m]))", "hide": false, "instant": false, "legendFormat": "{{op}}({{instance}})", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "histogram_quantile(\n 0.90,\n sum(\n rate(alluxio_worker_netty_read_time_to_send_first_packet_ms_bucket{job=\"worker\",instance=~\"$instance\",cluster_name=~\"$cluster\"}[5m])\n ) by (le, instance)\n)", "hide": false, "instant": false, "legendFormat": "read-TTFB({{instance}})", "range": true, "refId": "B" } ], "title": "Request Latency - P90", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the 99th percentile latency for internal metadata operations on a per-worker basis.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "ms" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 18 }, "id": 127, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "right", "showLegend": true, "width": 160 }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "histogram_quantile(0.99, rate(alluxio_meta_operation_latency_ms_bucket{job=\"worker\",instance=~\"$instance\",cluster_name=~\"$cluster\"}[5m]))", "hide": false, "instant": false, "legendFormat": "{{op}}({{instance}})", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "histogram_quantile(\n 0.99,\n sum(\n rate(alluxio_worker_netty_read_time_to_send_first_packet_ms_bucket{job=\"worker\",instance=~\"$instance\",cluster_name=~\"$cluster\"}[5m])\n ) by (le, instance)\n)", "hide": false, "instant": false, "legendFormat": "read-TTFB({{instance}})", "range": true, "refId": "B" } ], "title": "Request Latency - P99", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the latency distribution for read operations served by Alluxio workers from the page store or underlying file system (UFS). This helps identify read performance bottlenecks and distinguish slow storage backends or overloaded workers within the cluster.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "ms" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 25 }, "id": 137, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "right", "showLegend": true, "width": 160 }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "histogram_quantile(\n 0.90,\n sum(\n rate(alluxio_worker_netty_read_storage_response_time_ms_bucket{job=\"worker\", cluster_name=~\"$cluster\", instance=~\"$instance\"}[5m])\n ) by (le, instance)\n)", "instant": false, "legendFormat": "Read({{instance}})", "range": true, "refId": "FUSE" } ], "title": "Request (from Page Store or UFS) Latency - P90", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the 90th percentile latency of page store I/O operations (data operations) on workers. This helps identify slow disk I/O performance and potential storage bottlenecks within the cluster.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "µs" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 25 }, "id": 124, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "right", "showLegend": true, "width": 160 }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "histogram_quantile(\n 0.90,\n sum(\n rate(alluxio_page_store_io_latency_microseconds_bucket{job=\"worker\", cluster_name=~\"$cluster\", instance=~\"$instance\"}[5m])\n ) by (le, op, instance)\n)", "hide": false, "instant": false, "legendFormat": "{{op}} ({{instance}})", "range": true, "refId": "A" } ], "title": "PageStore IO Latency - P90", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the read throughput for each individual worker. This is useful for monitoring the read activity and load on each worker.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "binBps" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 32 }, "id": 36, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "irate(alluxio_data_throughput_bytes_total{job=\"worker\",method=\"read\",instance=~\"$instance\",cluster_name=~\"$cluster\"}[5m])", "instant": false, "legendFormat": "{{instance}}", "range": true, "refId": "A" } ], "title": "Read - Throughput", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the write throughput for each individual worker. This is useful for monitoring the write activity and load on each worker.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "binBps" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 32 }, "id": 44, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "irate(alluxio_data_throughput_bytes_total{job=\"worker\",method=\"write\",instance=~\"$instance\",cluster_name=~\"$cluster\"}[5m])", "instant": false, "legendFormat": "{{instance}}", "range": true, "refId": "A" } ], "title": "Write - Throughput", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the read request rate for each individual worker. This helps understand the read workload distribution across workers.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "reqps" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 39 }, "id": 22, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "irate(alluxio_data_access_bytes_count{method=\"read\",job=\"worker\",instance=~\"$instance\",cluster_name=~\"$cluster\"}[5m])", "instant": false, "legendFormat": "{{instance}}", "range": true, "refId": "A" } ], "title": "Read - Request/s", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the write request rate for each individual worker. This helps understand the write workload distribution across workers.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "reqps" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 39 }, "id": 77, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "irate(alluxio_data_access_bytes_count{method=\"write\",job=\"worker\",instance=~\"$instance\",cluster_name=~\"$cluster\"}[5m])", "instant": false, "legendFormat": "{{instance}}", "range": true, "refId": "A" } ], "title": "Write - Request/s", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the active threads and queue length for the read thread pool on each worker. This is useful for diagnosing read performance bottlenecks.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 46 }, "id": 70, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "alluxio_rpc_executor_max_threads{job=\"worker\",executor_name=\"netty-reader\",instance=~\"$instance\",cluster_name=~\"$cluster\"}", "hide": true, "instant": false, "legendFormat": "MaxTheads-{{instance}}", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "alluxio_rpc_executor_active_threads{job=\"worker\",executor_name=\"netty-reader\",instance=~\"$instance\",cluster_name=~\"$cluster\"}", "hide": false, "instant": false, "legendFormat": "ActiveThreads-{{instance}}", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "alluxio_rpc_executor_current_queue_length{job=\"worker\",executor_name=\"netty-reader\",instance=~\"$instance\",cluster_name=~\"$cluster\"}", "hide": false, "instant": false, "legendFormat": "QueueLength-{{instance}}", "range": true, "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "alluxio_rpc_executor_current_threads{job=\"worker\",executor_name=\"netty-reader\",instance=~\"$instance\",cluster_name=~\"$cluster\"}", "hide": true, "instant": false, "legendFormat": "CurrentTheads-{{instance}}", "range": true, "refId": "D" } ], "title": "Read - Threads", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the active threads and queue length for the write thread pool on each worker. This is useful for diagnosing write performance bottlenecks.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 46 }, "id": 78, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "alluxio_rpc_executor_max_threads{job=\"worker\",executor_name=\"netty-writer\",instance=~\"$instance\",cluster_name=~\"$cluster\"}", "hide": true, "instant": false, "legendFormat": "MaxTheads-{{instance}}", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "alluxio_rpc_executor_active_threads{job=\"worker\",executor_name=\"netty-writer\",instance=~\"$instance\",cluster_name=~\"$cluster\"}", "hide": false, "instant": false, "legendFormat": "ActiveThreads-{{instance}}", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "alluxio_rpc_executor_current_queue_length{job=\"worker\",executor_name=\"netty-writer\",instance=~\"$instance\",cluster_name=~\"$cluster\"}", "hide": false, "instant": false, "legendFormat": "QueueLength-{{instance}}", "range": true, "refId": "C" } ], "title": "Write - Threads", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the metadata request rate for each individual worker. This helps identify workers with high metadata load.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "reqps" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 53 }, "id": 26, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "irate(alluxio_meta_operation_total{job=\"worker\",instance=~\"$instance\",cluster_name=~\"$cluster\"}[5m])", "hide": false, "instant": false, "legendFormat": "{{instance}}-{{op}}", "range": true, "refId": "A" } ], "title": "Metadata - Request/s", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the active threads and queue length for the metadata thread pool on each worker. This is useful for diagnosing metadata performance bottlenecks.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 53 }, "id": 79, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "alluxio_rpc_executor_max_threads{job=\"worker\",executor_name=\"grpc-metadata\",instance=~\"$instance\",cluster_name=~\"$cluster\"}", "hide": true, "instant": false, "legendFormat": "MaxTheads-{{instance}}", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "alluxio_rpc_executor_active_threads{job=\"worker\",executor_name=\"grpc-metadata\",instance=~\"$instance\",cluster_name=~\"$cluster\"}", "hide": false, "instant": false, "legendFormat": "ActiveThreads-{{instance}}", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "alluxio_rpc_executor_current_queue_length{job=\"worker\",executor_name=\"grpc-metadata\",instance=~\"$instance\",cluster_name=~\"$cluster\"}", "hide": false, "instant": false, "legendFormat": "QueueLength-{{instance}}", "range": true, "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "alluxio_rpc_executor_current_threads{job=\"worker\",executor_name=\"grpc-metadata\",instance=~\"$instance\",cluster_name=~\"$cluster\"}", "hide": true, "instant": false, "legendFormat": "CurrentTheads-{{instance}}", "range": true, "refId": "D" } ], "title": "Metadata - Threads", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the failure rate for internal metadata operations on a per-worker basis. This helps pinpoint workers that are experiencing internal errors.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 60 }, "id": 92, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "right", "showLegend": true, "width": 160 }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "irate(alluxio_meta_operation_errors_total{job=\"worker\",cluster_name=~\"$cluster\", instance=~\"$instance\"}[5m])", "hide": false, "instant": false, "legendFormat": "{{instance}}-{{op}}", "range": true, "refId": "B" } ], "title": "Request Failure", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the off-heap memory usage for each worker. This is important for monitoring memory resources and preventing out-of-memory errors.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 60 }, "id": 68, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum(alluxio_rocksdb_memory_usage_bytes{job=\"worker\",instance=~\"$instance\",cluster_name=~\"$cluster\"}) by (instance)\n+\nsum(alluxio_netty_direct_memory_usage_bytes{job=\"worker\",instance=~\"$instance\",cluster_name=~\"$cluster\"}) by (instance)\n+\nsum(jvm_memory_used_bytes{job=\"worker\",instance=~\"$instance\",cluster_name=~\"$cluster\",area=\"nonheap\"}) by (instance)\n+\nsum(jvm_buffer_pool_used_bytes{job=\"worker\",instance=~\"$instance\",cluster_name=~\"$cluster\",pool=\"direct\"}) by (instance)\n+\nsum(jvm_buffer_pool_used_bytes{job=\"worker\",instance=~\"$instance\",cluster_name=~\"$cluster\",pool=\"mapped\"}) by (instance)", "hide": false, "instant": false, "legendFormat": "worker-{{instance}}", "range": true, "refId": "A" } ], "title": "Off Heap Memory", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the error rate for PageStore operations on each worker. This can indicate issues with the local cache storage layer.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 67 }, "id": 108, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "right", "showLegend": true, "width": 160 }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "irate(alluxio_page_store_operation_errors_total{job=\"worker\", cluster_name=~\"$cluster\", instance=~\"$instance\"}[5m])", "hide": false, "instant": false, "legendFormat": "{{instance}}-{{op}}-{{cause}}", "range": true, "refId": "B" } ], "title": "PageStore Errors", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the error rate for disk-related PageStore operations on each worker. This can indicate underlying disk hardware or file system issues.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "max": 1, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 0.1 }, { "color": "red", "value": 0.3 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 67 }, "id": 109, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "right", "showLegend": true, "width": 160 }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum by (instance, dir) (irate(alluxio_page_store_dir_operation_errors_total{job=\"worker\", cluster_name=~\"$cluster\", instance=~\"$instance\"}[5m]) or on() vector(0))\n/\nsum by (instance, dir) (irate(alluxio_page_store_dir_operations_total{job=\"worker\", cluster_name=~\"$cluster\", instance=~\"$instance\"}[5m]))", "hide": false, "instant": false, "legendFormat": "{{instance}}-{{dir}}", "range": true, "refId": "B" } ], "title": "PageStore Disk Error Rate", "type": "timeseries" } ], "title": "Worker", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 75 }, "id": 41, "panels": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the 90th percentile latency for UFS operations on a per-worker basis. This helps identify slow UFS interactions from specific workers.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "ms" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 139 }, "id": 97, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "right", "showLegend": true, "width": 160 }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "histogram_quantile(0.90, rate(alluxio_ufs_latency_ms_bucket{instance=~\"$instance\",cluster_name=~\"$cluster\"}[5m]))", "hide": false, "instant": false, "legendFormat": "{{method}} {{ufs_type}} {{instance}}", "range": true, "refId": "C" } ], "title": "Request Latency - P90", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the 99th percentile latency for UFS operations on a per-worker basis. This helps identify worst-case UFS performance from specific workers.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "ms" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 139 }, "id": 98, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "right", "showLegend": true, "width": 160 }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "histogram_quantile(0.99, rate(alluxio_ufs_latency_ms_bucket{instance=~\"$instance\",cluster_name=~\"$cluster\"}[5m]))", "hide": false, "instant": false, "legendFormat": "{{method}} {{ufs_type}} {{instance}}", "range": true, "refId": "C" } ], "title": "Request Latency - P99", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the read throughput from the UFS on a per-worker basis. This is useful for monitoring how much data is being read from the underlying storage.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "binBps" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 146 }, "id": 42, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "irate(alluxio_ufs_data_access_bytes_total{method=\"read\",instance=~\"$instance\",cluster_name=~\"$cluster\"}[5m])", "hide": false, "instant": false, "legendFormat": "{{instance}}", "range": true, "refId": "FUSE fallback" } ], "title": "Throughput - Read", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the write throughput to the UFS on a per-worker basis. This is useful for monitoring how much data is being written to the underlying storage.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "binBps" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 146 }, "id": 43, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "irate(alluxio_ufs_data_access_bytes_total{method=\"write\",instance=~\"$instance\",cluster_name=~\"$cluster\"}[5m])", "hide": false, "instant": false, "legendFormat": "{{instance}}", "range": true, "refId": "FUSE fallback" } ], "title": "Throughput - Write", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the error rate for UFS operations on a per-worker basis. This is critical for diagnosing issues with the underlying storage system.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 153 }, "id": 49, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "irate(alluxio_ufs_error_total{instance=~\"$instance\",cluster_name=~\"$cluster\"}[5m])", "hide": false, "instant": false, "legendFormat": "{{job}}-{{instance}}-{{ufs_type}}-{{error_code}}", "range": true, "refId": "FUSE fallback" } ], "title": "Errors", "type": "timeseries" } ], "title": "UFS", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 76 }, "id": 30, "panels": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Time series visualization of job execution patterns. Important for understanding cluster background activity levels and identifying periods of high maintenance activity that might impact performance.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 }, "id": 130, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "alluxio_active_job_count{job=\"coordinator\",type=\"running\"}", "instant": false, "legendFormat": "{{instance}}-running", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "alluxio_active_job_count{job=\"coordinator\",type=\"waiting\"}", "hide": false, "instant": false, "legendFormat": "{{instance}}-waiting", "range": true, "refId": "B" } ], "title": "Jobs", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Time series visualization of job execution tasks on each worker. Important for understanding worker activity levels and identifying bottlenecks that might impact performance.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 }, "id": 131, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "alluxio_worker_job_task_count{}", "instant": false, "legendFormat": "{{worker}}", "range": true, "refId": "A" } ], "title": "Job Tasks per Worker", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the active threads and queue length for the load job thread pool on each worker. This is useful for diagnosing load job performance bottlenecks.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 14 }, "id": 129, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "alluxio_rpc_executor_max_threads{job=\"worker\",executor_name=\"load-executor\",instance=~\"$instance\",cluster_name=~\"$cluster\"}", "hide": true, "instant": false, "legendFormat": "MaxTheads-{{instance}}", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "alluxio_rpc_executor_active_threads{job=\"worker\",executor_name=\"load-executor\",instance=~\"$instance\",cluster_name=~\"$cluster\"}", "hide": false, "instant": false, "legendFormat": "ActiveThreads-{{instance}}", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "alluxio_rpc_executor_current_queue_length{job=\"worker\",executor_name=\"load-executor\",instance=~\"$instance\",cluster_name=~\"$cluster\"}", "hide": false, "instant": false, "legendFormat": "QueueLength-{{instance}}", "range": true, "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "alluxio_rpc_executor_current_threads{job=\"worker\",executor_name=\"load-executor\",instance=~\"$instance\",cluster_name=~\"$cluster\"}", "hide": true, "instant": false, "legendFormat": "CurrentTheads-{{instance}}", "range": true, "refId": "D" } ], "title": "Job Threads per Worker", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the rate of jobs dispatched per worker.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] } }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 14 }, "id": 32, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum by(worker)(irate(alluxio_distributed_load_job_dispatched_size_total{job=\"coordinator\", instance=~\"$instance\"} [5m]))", "instant": false, "legendFormat": "{{worker}}", "range": true, "refId": "A" } ], "title": "Job Dispatched Per Second", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the load job throughput for each individual worker. This is useful for monitoring the load job activity on each worker.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "binBps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 22 }, "id": 128, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum(irate(alluxio_distributed_load_data_loaded_from_ufs_bytes_total{job=\"worker\",cluster_name=~\"$cluster\"}[5m]))", "instant": false, "legendFormat": "Total", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "irate(alluxio_distributed_load_data_loaded_from_ufs_bytes_total{job=\"worker\",instance=~\"$instance\",cluster_name=~\"$cluster\"}[5m])", "hide": false, "instant": false, "legendFormat": "{{instance}}", "range": true, "refId": "B" } ], "title": "Distributed Load Throughput", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the rate of bytes loaded on workers.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "binBps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 22 }, "id": 33, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum by(instance) (irate(alluxio_distributed_worker_bytes_loaded_bytes_total{job=\"worker\", instance=~\"$instance\"} [5m]))", "instant": false, "legendFormat": "{{instance}}", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum by()(irate(alluxio_distributed_load_job_loaded_bytes_total{job=\"coordinator\", instance=~\"$instance\"} [5m]))", "hide": false, "instant": false, "legendFormat": "Total", "range": true, "refId": "B" } ], "title": "Bytes Loaded on Workers Per Second", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the rate of distributed load operations.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 30 }, "id": 29, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "irate(alluxio_distributed_load_job_scanned_total{job=\"coordinator\", instance=~\"$instance\"}[5m])", "instant": false, "legendFormat": "File & Directory Scanned", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "irate(alluxio_distributed_load_job_processed_total{job=\"coordinator\", instance=~\"$instance\"}[5m])", "hide": false, "instant": false, "legendFormat": "File & Directory Loaded", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "irate(alluxio_distributed_load_job_skipped_total{job=\"coordinator\", instance=~\"$instance\"}[5m])", "hide": false, "instant": false, "legendFormat": "File & Directory Skipped", "range": true, "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum by()(irate(alluxio_distributed_load_job_failure_total{job=\"coordinator\", instance=~\"$instance\"}[5m]))", "hide": false, "instant": false, "legendFormat": "File & Directory Errors", "range": true, "refId": "D" } ], "title": "Distributed Load Operation Counts Per Second", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the failure breakdown by reason and worker.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" } ] } }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 30 }, "id": 31, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum by(reason, worker) (irate(alluxio_distributed_load_job_failure_total{job=\"coordinator\", instance=~\"$instance\"} [5m]))", "instant": false, "legendFormat": "{{reason}}-{{worker}}", "range": true, "refId": "A" } ], "title": "Distributed Load Failure Breakdowns", "type": "timeseries" } ], "title": "Job", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 77 }, "id": 7, "panels": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the total number of worker membership refreshes on each instance. This helps track cluster topology updates and potential instability or frequent reconfiguration of workers within the cluster.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 78 }, "id": 114, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "right", "showLegend": true, "width": 160 }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "irate(alluxio_worker_membership_refresh_count_total{job=~\"$service\",instance=~\"$instance\",cluster_name=~\"$cluster\"}[5m]) > 0", "instant": false, "legendFormat": "{{instance}}", "range": true, "refId": "FUSE" } ], "title": "Active Worker Membership", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the utilization ratio of dynamic resource pools, calculated as current resources over maximum capacity. This helps identify over- or under-utilized pools and potential resource bottlenecks within the cluster.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "max": 1, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 0.8 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 78 }, "id": 118, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "right", "showLegend": true, "width": 160 }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum by (instance, pool_kind, pool_instance) (alluxio_dynamic_resource_pool_current_resources{job=~\"$service\",instance=~\"$instance\",cluster_name=~\"$cluster\"})\n/\nsum by (instance, pool_kind, pool_instance) (alluxio_dynamic_resource_pool_capacity{type=\"max\",job=~\"$service\",instance=~\"$instance\",cluster_name=~\"$cluster\"})", "instant": false, "legendFormat": "{{instance}}-{{pool_instance}}-{{pool_kind}}", "range": true, "refId": "FUSE" } ], "title": "Resource Pool Usage ", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the rate of dynamic resource pool acquisition timeouts. This helps identify contention or delays in resource allocation and potential performance bottlenecks within the cluster.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 85 }, "id": 125, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "right", "showLegend": true, "width": 160 }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "irate(alluxio_dynamic_resource_pool_acquisition_timeouts_total{job=~\"$service\",instance=~\"$instance\",cluster_name=~\"$cluster\"}[5m])", "instant": false, "legendFormat": "{{instance}}-{{pool_kind}}-{{pool_instance}}", "range": true, "refId": "a" } ], "title": "Resource Pool - Acquisition Timeout", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the 99th percentile latency for creating new resources in dynamic resource pools. This helps identify slow resource allocation and potential performance bottlenecks within the cluster.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "ms" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 85 }, "id": 121, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "right", "showLegend": true, "width": 160 }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "histogram_quantile(\n 0.99,\n sum(\n rate(alluxio_dynamic_resource_pool_create_new_resource_latency_ms_bucket{job=~\"$service\",instance=~\"$instance\",cluster_name=~\"$cluster\"}[5m])\n ) by (le, pool_kind, pool_instance, instance)\n)", "instant": false, "legendFormat": "{{instance}}-{{pool_kind}}-{{pool_instance}}", "range": true, "refId": "FUSE" } ], "title": "Resource Pool - Resource Creation Latency (P99)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the CPU time spent by each process.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "µs" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 92 }, "id": 8, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "irate(process_cpu_seconds_total{job=~\"$service\",instance=~\"$instance\",cluster_name=~\"$cluster\"}[5m])", "instant": false, "legendFormat": "{{job}}-{{instance}}", "range": true, "refId": "A" } ], "title": "CPU time spent", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the number of threads for each process.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 92 }, "id": 11, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "jvm_threads_current{job=~\"$service\",instance=~\"$instance\",cluster_name=~\"$cluster\"}", "instant": false, "legendFormat": "{{job}}-{{instance}}", "range": true, "refId": "A" } ], "title": "Threads", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the heap memory usage for each process.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 100 }, "id": 9, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "jvm_memory_used_bytes{area=\"heap\",job=~\"$service\",instance=~\"$instance\",cluster_name=~\"$cluster\"}", "instant": false, "legendFormat": "{{job}}-{{instance}}", "range": true, "refId": "A" } ], "title": "Heap Usage", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the heap memory usage percentage for each process.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "max": 1, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 0.9 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 100 }, "id": 10, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "jvm_memory_used_bytes{area=\"heap\",job=~\"$service\",instance=~\"$instance\",cluster_name=~\"$cluster\"} / jvm_memory_max_bytes{area=\"heap\",job=~\"$service\",instance=~\"$instance\",cluster_name=~\"$cluster\"}", "instant": false, "legendFormat": "{{job}}-{{instance}}", "range": true, "refId": "A" } ], "title": "Heap Usage(%)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the young generation garbage collection time per minute.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 108 }, "id": 12, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "irate(jvm_gc_collection_seconds_sum{gc=\"G1 Young Generation\",job=~\"$service\",instance=~\"$instance\",cluster_name=~\"$cluster\"}[5m]) * 60", "instant": false, "legendFormat": "{{job}}-{{instance}}", "range": true, "refId": "A" } ], "title": "young GC time(per minute)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the young generation garbage collection rate per minute.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 108 }, "id": 13, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "irate(jvm_gc_collection_seconds_count{gc=\"G1 Young Generation\",job=~\"$service\",instance=~\"$instance\",cluster_name=~\"$cluster\"}[5m]) * 60", "instant": false, "legendFormat": "{{job}}-{{instance}}", "range": true, "refId": "A" } ], "title": "young GC rate(per minute)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the old generation garbage collection time per minute.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 116 }, "id": 14, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "irate(jvm_gc_collection_seconds_sum{gc=\"G1 Old Generation\",job=~\"$service\",instance=~\"$instance\",cluster_name=~\"$cluster\"}[5m]) * 60", "instant": false, "legendFormat": "{{job}}-{{instance}}", "range": true, "refId": "A" } ], "title": "old GC time(per minute)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the old generation garbage collection rate per minute.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 116 }, "id": 15, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "irate(jvm_gc_collection_seconds_count{gc=\"G1 Old Generation\",job=~\"$service\",instance=~\"$instance\",cluster_name=~\"$cluster\"}[5m]) * 60", "instant": false, "legendFormat": "{{job}}-{{instance}}", "range": true, "refId": "A" } ], "title": "old GC rate(per minute)", "type": "timeseries" } ], "title": "Process", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 78 }, "id": 53, "panels": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the number of node in ETCD cluster.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "red", "value": null }, { "color": "yellow", "value": 1 }, { "color": "green", "value": 3 } ] } }, "overrides": [] }, "gridPos": { "h": 6, "w": 6, "x": 0, "y": 79 }, "id": 52, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showPercentChange": false, "textMode": "auto", "wideLayout": true }, "pluginVersion": "10.4.5", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum(etcd_server_has_leader{job=\"etcd\"})", "instant": false, "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Up", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the RPC request and failure rates.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "ops" }, "overrides": [] }, "gridPos": { "h": 6, "w": 10, "x": 6, "y": 79 }, "id": 54, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum(rate(grpc_server_started_total{job=\"etcd\",grpc_type=\"unary\"}[5m]))", "instant": false, "legendFormat": "RPC Rate", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum(rate(grpc_server_handled_total{job=\"etcd\",grpc_type=\"unary\",grpc_code!=\"OK\"}[5m]))", "hide": false, "instant": false, "legendFormat": "RPC Failed Rate", "range": true, "refId": "B" } ], "title": "RPC Rate", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the number of active watch and lease streams.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] }, "gridPos": { "h": 6, "w": 8, "x": 16, "y": 79 }, "id": 55, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum(grpc_server_started_total{job=\"etcd\",grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{job=\"etcd\",grpc_service=\"etcdserverpb.Watch\",grpc_type=\"bidi_stream\"})", "instant": false, "legendFormat": "Watch Streams", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum(grpc_server_started_total{job=\"etcd\",grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"}) - sum(grpc_server_handled_total{job=\"etcd\",grpc_service=\"etcdserverpb.Lease\",grpc_type=\"bidi_stream\"})", "hide": false, "instant": false, "legendFormat": "Lease Streams", "range": true, "refId": "B" } ], "title": "Active Streams", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the error rate of Etcd client calls. This helps identify failing Etcd requests and potential reliability or connectivity issues between the cluster and the Etcd service.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "max": 1, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 0.05 }, { "color": "red", "value": 0.2 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 85 }, "id": 120, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true, "width": 160 }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "(sum by (instance, server) (irate(alluxio_etcd_call_errors_total{job=~\"$service\",instance=~\"$instance\",cluster_name=~\"$cluster\"}[5m])) or on() vector(0))\n/\n(sum by (instance, server) (irate(alluxio_etcd_client_calls_total{job=~\"$service\",instance=~\"$instance\",cluster_name=~\"$cluster\"}[5m])))", "instant": false, "legendFormat": "{{instance}}-{{server}}", "range": true, "refId": "rate" } ], "title": "ETCD Client Call Failure Rate", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the 99th percentile latency of Etcd client calls. This helps identify slow Etcd operations and potential performance bottlenecks in Etcd communication within the cluster.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "ms" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 85 }, "id": 116, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true, "width": 160 }, "tooltip": { "hideZeros": false, "mode": "single", "sort": "none" } }, "pluginVersion": "11.4.0-210101", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "histogram_quantile(\n 0.99,\n sum(\n rate(alluxio_etcd_client_call_latency_ms_bucket{job=~\"$service\",instance=~\"$instance\",cluster_name=~\"$cluster\"}[5m])\n ) by (le, instance)\n)", "instant": false, "legendFormat": "{{instance}}", "range": true, "refId": "FUSE" } ], "title": "ETCD Client Call Latency (P99)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the size of the ETCD database.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 6, "w": 8, "x": 0, "y": 92 }, "id": 56, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "etcd_mvcc_db_total_size_in_bytes{job=\"etcd\"}", "instant": false, "legendFormat": "{{instance}}", "range": true, "refId": "A" } ], "title": "DB Size", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the disk sync duration for WAL and backend operations.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] }, "gridPos": { "h": 6, "w": 8, "x": 8, "y": 92 }, "id": 57, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_wal_fsync_duration_seconds_bucket{job=\"etcd\"}[5m])) by (instance, le))", "instant": false, "legendFormat": "{{instance}} WAL fsync", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "histogram_quantile(0.99, sum(rate(etcd_disk_backend_commit_duration_seconds_bucket{job=\"etcd\"}[5m])) by (instance, le))", "hide": false, "instant": false, "legendFormat": "{{instance}} DB fsync", "range": true, "refId": "B" } ], "title": "Disk Sync Duration", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the resident memory usage of the ETCD process.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 6, "w": 8, "x": 16, "y": 92 }, "id": 58, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "process_resident_memory_bytes{job=\"etcd\"}", "instant": false, "legendFormat": "{{instance}} Resident Memory", "range": true, "refId": "A" } ], "title": "Memory", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the client traffic in rates.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 50, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineStyle": { "fill": "solid" }, "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "binBps" }, "overrides": [] }, "gridPos": { "h": 6, "w": 6, "x": 0, "y": 98 }, "id": 59, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "rate(etcd_network_client_grpc_received_bytes_total{job=\"etcd\"}[5m])", "instant": false, "legendFormat": "{{instance}}", "range": true, "refId": "A" } ], "title": "Client Traffic In", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the client traffic out rates.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 50, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "binBps" }, "overrides": [] }, "gridPos": { "h": 6, "w": 6, "x": 6, "y": 98 }, "id": 60, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "rate(etcd_network_client_grpc_sent_bytes_total{job=\"etcd\"}[5m])", "instant": false, "legendFormat": "{{instance}}", "range": true, "refId": "A" } ], "title": "Client Traffic Out", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the peer traffic in rates.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "binBps" }, "overrides": [] }, "gridPos": { "h": 6, "w": 6, "x": 12, "y": 98 }, "id": 61, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum(rate(etcd_network_peer_received_bytes_total{job=\"etcd\"}[5m])) by (instance)", "instant": false, "legendFormat": "{{instance}}", "range": true, "refId": "A" } ], "title": "Peer Traffic In", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the peer traffic out rates.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "binBps" }, "overrides": [] }, "gridPos": { "h": 6, "w": 6, "x": 18, "y": 98 }, "id": 62, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum(rate(etcd_network_peer_sent_bytes_total{job=\"etcd\"}[5m])) by (instance)", "instant": false, "legendFormat": "{{instance}}", "range": true, "refId": "A" } ], "title": "Peer Traffic Out", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the raft proposal metrics including failure rate, pending total, commit rate, and apply rate.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 6, "w": 12, "x": 0, "y": 104 }, "id": 63, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum(rate(etcd_server_proposals_failed_total{job=\"etcd\"}[5m]))", "instant": false, "legendFormat": "Proposal Failure Rate", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum(etcd_server_proposals_pending{job=\"etcd\"})", "hide": false, "instant": false, "legendFormat": "Proposal Pending Total", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum(rate(etcd_server_proposals_committed_total{job=\"etcd\"}[5m]))", "hide": false, "instant": false, "legendFormat": "Proposal Commit Rate", "range": true, "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "sum(rate(etcd_server_proposals_applied_total{job=\"$cluster\"}[5m]))", "hide": false, "instant": false, "legendFormat": "Proposal Apply Rate", "range": true, "refId": "D" } ], "title": "Raft Proposals", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Displays the total number of leader elections per day.\t", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] } }, "overrides": [] }, "gridPos": { "h": 6, "w": 12, "x": 12, "y": 104 }, "id": 64, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "expr": "changes(etcd_server_leader_changes_seen_total{job=\"etcd\"}[1d])", "instant": false, "legendFormat": "{{instance}}", "range": true, "refId": "A" } ], "title": "Total Leader Elections Per Day", "type": "timeseries" } ], "title": "ETCD", "type": "row" } ], "refresh": "", "schemaVersion": 39, "tags": [], "templating": { "list": [ { "allowCustomValue": true, "current": { "selected": false, "text": "Prometheus", "value": "my_unique_uid" }, "hide": 2, "includeAll": false, "multi": false, "name": "DS_PROMETHEUS", "options": [], "query": "prometheus", "refresh": 1, "regex": "", "skipUrlSync": false, "type": "datasource" }, { "allowCustomValue": true, "current": { "selected": false, "text": "All", "value": "$__all" }, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "definition": "label_values(up{job!=\"prometheus\"},job)", "hide": 0, "includeAll": true, "multi": true, "name": "service", "options": [], "query": { "qryType": 1, "query": "label_values(up{job!=\"prometheus\"},job)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 0, "type": "query" }, { "allowCustomValue": true, "current": { "selected": true, "text": [ "All" ], "value": [ "$__all" ] }, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "definition": "label_values(up{job!=\"prometheus\"},instance)", "hide": 0, "includeAll": true, "multi": true, "name": "instance", "options": [], "query": { "qryType": 1, "query": "label_values(up{job!=\"prometheus\"},instance)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 0, "type": "query" }, { "allowCustomValue": true, "current": { "selected": false, "text": "All", "value": "$__all" }, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "definition": "label_values(up{job=\"worker\"},cluster_name)", "hide": 0, "includeAll": true, "multi": true, "name": "cluster", "options": [], "query": { "qryType": 1, "query": "label_values(up{job=\"worker\"},cluster_name)", "refId": "PrometheusVariableQueryEditor-VariableQuery" }, "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 0, "type": "query" } ] }, "time": { "from": "now-3h", "to": "now" }, "timepicker": {}, "timezone": "", "title": "Alluxio", "uid": "e61ebf29-ce0e-4547-95c8-5da322ee3a42", "version": 7, "weekStart": "" }