{ "__inputs": [ { "name": "DS_PROMETHEUS", "label": "Prometheus", "description": "", "type": "datasource", "pluginId": "prometheus", "pluginName": "Prometheus" } ], "__elements": {}, "__requires": [ { "type": "panel", "id": "bargauge", "name": "Bar gauge", "version": "" }, { "type": "panel", "id": "gauge", "name": "Gauge", "version": "" }, { "type": "grafana", "id": "grafana", "name": "Grafana", "version": "9.2.6" }, { "type": "datasource", "id": "prometheus", "name": "Prometheus", "version": "1.0.0" }, { "type": "panel", "id": "stat", "name": "Stat", "version": "" }, { "type": "panel", "id": "state-timeline", "name": "State timeline", "version": "" }, { "type": "panel", "id": "text", "name": "Text", "version": "" }, { "type": "panel", "id": "timeseries", "name": "Time series", "version": "" } ], "annotations": { "list": [ { "builtIn": 1, "datasource": { "type": "grafana", "uid": "-- Grafana --" }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "target": { "limit": 100, "matchAny": false, "tags": [], "type": "dashboard" }, "type": "dashboard" } ] }, "description": "A comprehensive monitoring dashboard based on Mastodon's Prometheus statsd exporter", "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, "id": null, "links": [], "liveNow": true, "panels": [ { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 30, "panels": [], "title": "Mastodon Overview", "type": "row" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "The total amount of HTTP Queries that Mastodon is serving.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 1, "mappings": [], "thresholds": { "mode": "percentage", "steps": [ { "color": "dark-blue", "value": null } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 3, "w": 3, "x": 0, "y": 1 }, "id": 33, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "text": { "valueSize": 42 }, "textMode": "auto" }, "pluginVersion": "9.2.6", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "exemplar": false, "expr": "sort(sum by (mastodon)(rate(mastodon_controller_duration_count[$__rate_interval])))", "instant": false, "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Puma QPS", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "The total amount of SQL Queries that Mastodon is generating, split out in SELECT, DELETE, INSERT and UPDATE.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 1, "mappings": [], "thresholds": { "mode": "percentage", "steps": [ { "color": "dark-blue", "value": null } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 5, "w": 3, "x": 3, "y": 1 }, "id": 25, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "text": { "titleSize": 22, "valueSize": 22 }, "textMode": "auto" }, "pluginVersion": "9.2.6", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum by (operation) (rate(mastodon_db_operation_count{operation=\"select\"}[5m]))", "legendFormat": "__auto", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum by (operation) (rate(mastodon_db_operation_count{operation=\"insert\"}[5m]))", "hide": false, "legendFormat": "__auto", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum by (operation) (rate(mastodon_db_operation_count{operation=\"update\"}[5m]))", "hide": false, "legendFormat": "__auto", "range": true, "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum by (operation) (rate(mastodon_db_operation_count{operation=\"delete\"}[5m]))", "hide": false, "legendFormat": "__auto", "range": true, "refId": "D" } ], "title": "SQL ops/sec", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "The running average latency of SQL Queries that Mastodon is generating, split out in SELECT, DELETE, INSERT and UPDATE.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 1, "mappings": [], "max": 0.25, "thresholds": { "mode": "percentage", "steps": [ { "color": "dark-green", "value": null }, { "color": "#EAB839", "value": 50 }, { "color": "dark-orange", "value": 60 }, { "color": "dark-red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 5, "w": 3, "x": 6, "y": 1 }, "id": 26, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "text": { "titleSize": 22, "valueSize": 22 }, "textMode": "auto" }, "pluginVersion": "9.2.6", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum by (operation) (rate(mastodon_db_operation_sum{operation=\"select\"}[$__rate_interval])) /\nsum by (operation) (rate(mastodon_db_operation_count{operation=\"select\"}[$__rate_interval]))", "legendFormat": "__auto", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum by (operation) (rate(mastodon_db_operation_sum{operation=\"insert\"}[$__rate_interval])) /\nsum by (operation) (rate(mastodon_db_operation_count{operation=\"insert\"}[$__rate_interval]))", "hide": false, "legendFormat": "__auto", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum by (operation) (rate(mastodon_db_operation_sum{operation=\"update\"}[$__rate_interval])) /\nsum by (operation) (rate(mastodon_db_operation_count{operation=\"update\"}[$__rate_interval]))", "hide": false, "legendFormat": "__auto", "range": true, "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum by (operation) (rate(mastodon_db_operation_sum{operation=\"delete\"}[$__rate_interval])) /\nsum by (operation) (rate(mastodon_db_operation_count{operation=\"delete\"}[$__rate_interval]))", "hide": false, "legendFormat": "__auto", "range": true, "refId": "D" } ], "title": "SQL Latency", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "The total amount of Cache Queries that Mastodon is generating.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 1, "mappings": [], "thresholds": { "mode": "percentage", "steps": [ { "color": "dark-blue", "value": null } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 5, "w": 3, "x": 9, "y": 1 }, "id": 27, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "text": { "titleSize": 15, "valueSize": 15 }, "textMode": "auto" }, "pluginVersion": "9.2.6", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "exemplar": false, "expr": "sort(sum by (operation)(rate(mastodon_cache_duration_count[$__rate_interval])))", "instant": false, "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Cache ops/sec", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "The total amount of Cache Queries that Mastodon is generating.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 1, "mappings": [], "max": 0.25, "thresholds": { "mode": "percentage", "steps": [ { "color": "dark-green", "value": null }, { "color": "#EAB839", "value": 50 }, { "color": "dark-orange", "value": 60 }, { "color": "dark-red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 5, "w": 3, "x": 12, "y": 1 }, "id": 28, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "text": { "titleSize": 15, "valueSize": 15 }, "textMode": "auto" }, "pluginVersion": "9.2.6", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "exemplar": false, "expr": "sort(sum by (operation) (rate(mastodon_cache_duration_sum[$__rate_interval])) /\nsum by (operation) (rate(mastodon_cache_duration_count[$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "__auto", "range": true, "refId": "B" } ], "title": "Cache Latency", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "The total amount of worker jobs that Sidekiq is performing.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 1, "mappings": [], "thresholds": { "mode": "percentage", "steps": [ { "color": "dark-blue", "value": null } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 3, "w": 3, "x": 15, "y": 1 }, "id": 37, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "text": { "titleSize": 25, "valueSize": 25 }, "textMode": "auto" }, "pluginVersion": "9.2.6", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "exemplar": false, "expr": "sort(sum by (mastodon)(rate(mastodon_sidekiq_worker_success_total[$__rate_interval])))", "instant": false, "legendFormat": "success", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "exemplar": false, "expr": "sort(sum by (mastodon)(rate(mastodon_sidekiq_worker_failure_total[$__rate_interval])))", "hide": false, "instant": false, "legendFormat": "failure", "range": true, "refId": "B" } ], "title": "Sidekiq Jobs/sec", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "The total amount of worker jobs that Sidekiq is performing.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 1, "mappings": [], "thresholds": { "mode": "percentage", "steps": [ { "color": "dark-blue", "value": null } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 5, "w": 3, "x": 18, "y": 1 }, "id": 55, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "text": { "titleSize": 22, "valueSize": 22 }, "textMode": "auto" }, "pluginVersion": "9.2.6", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "exemplar": false, "expr": "sort(sum by (mastodon)(mastodon_sidekiq_workers_size))", "instant": false, "interval": "", "legendFormat": "workers", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "exemplar": false, "expr": "sort(sum by (mastodon)(mastodon_sidekiq_scheduled_size))", "hide": false, "instant": false, "legendFormat": "scheduled", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "exemplar": false, "expr": "sort(sum by (mastodon)(mastodon_sidekiq_retry_size))", "hide": false, "instant": false, "legendFormat": "retry", "range": true, "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "exemplar": false, "expr": "sort(sum by (mastodon)(mastodon_sidekiq_dead_size))", "hide": false, "instant": false, "legendFormat": "dead", "range": true, "refId": "D" } ], "title": "Sidekiq Queues", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "gridPos": { "h": 5, "w": 3, "x": 21, "y": 1 }, "id": 54, "options": { "code": { "language": "plaintext", "showLineNumbers": false, "showMiniMap": false }, "content": "\n", "mode": "html" }, "pluginVersion": "9.2.6", "transparent": true, "type": "text" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "The running average latency of HTTP Queries that Mastodon is serving.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 1, "mappings": [], "max": 0.25, "thresholds": { "mode": "percentage", "steps": [ { "color": "dark-green", "value": null }, { "color": "#EAB839", "value": 40 }, { "color": "dark-orange", "value": 60 }, { "color": "dark-red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 2, "w": 3, "x": 0, "y": 4 }, "id": 34, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "text": { "valueSize": 35 }, "textMode": "auto" }, "pluginVersion": "9.2.6", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "exemplar": false, "expr": "sum by (mastodon) (rate(mastodon_controller_duration_sum[$__rate_interval])) /\nsum by (mastodon) (rate(mastodon_controller_duration_count[$__rate_interval]))", "instant": false, "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Puma Latency", "type": "stat" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "The running average latency of HTTP Queries that Mastodon is serving.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 1, "mappings": [], "max": 10, "thresholds": { "mode": "percentage", "steps": [ { "color": "dark-green", "value": null }, { "color": "#EAB839", "value": 40 }, { "color": "dark-orange", "value": 60 }, { "color": "dark-red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 2, "w": 3, "x": 15, "y": 4 }, "id": 38, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "text": { "valueSize": 35 }, "textMode": "auto" }, "pluginVersion": "9.2.6", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "exemplar": false, "expr": "sum by (mastodon) (rate(mastodon_sidekiq_worker_processing_time_sum[$__rate_interval])) /\nsum by (mastodon) (rate(mastodon_sidekiq_worker_processing_time_count[$__rate_interval]))", "instant": false, "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Sidekiq Latency", "type": "stat" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 6 }, "id": 36, "panels": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "Rate of queries/sec of API calls served by the Mastodon Webserver", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 7 }, "id": 32, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum by (controller) (rate(mastodon_controller_duration_count{controller=~\"$puma_controller\"}[$__rate_interval]))", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Controller Operations ($puma_controller)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "Rate of response codes/sec of HTTP API calls served by the Mastodon Webserver", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 7 }, "id": 57, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum by (status) (rate(mastodon_controller_status{controller=~\"$puma_controller\"}[$__rate_interval]))", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Controller Return Codes ($puma_controller)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "Current latency of API calls served by the Mastodon Webserver", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 1, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 7 }, "id": 39, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum by (controller) (rate(mastodon_controller_duration_sum{controller=~\"$puma_controller\"}[$__rate_interval])) /\nsum by (controller) (rate(mastodon_controller_duration_count{controller=~\"$puma_controller\"}[$__rate_interval]))", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Controller Latency ($puma_controller)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "Rate of queries/sec of API calls served by the Mastodon Webserver, split out by action as well as controller.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 14 }, "id": 40, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum by (controller,action) (rate(mastodon_controller_duration_count{controller=~\"$puma_controller\",action=~\"$puma_action\"}[$__rate_interval]))", "legendFormat": "{{action}}-{{controller}}", "range": true, "refId": "A" } ], "title": "Action-Controller Operations ($puma_action)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "Rate of response codes/sec of HTTP API calls served by the Mastodon Webserver", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 14 }, "id": 58, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum by (status) (rate(mastodon_controller_status{action=~\"$puma_action\",controller=~\"$puma_controller\"}[$__rate_interval]))", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Action-Controller Return Codes ($puma_action,$puma_controller)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "Current latency of API calls served by the Mastodon Webserver, split out by controller and action", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 1, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 14 }, "id": 41, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum by (controller,action) (rate(mastodon_controller_duration_sum{controller=~\"$puma_controller\",action=~\"$puma_action\"}[$__rate_interval])) /\nsum by (controller,action) (rate(mastodon_controller_duration_count{controller=~\"$puma_controller\",action=~\"$puma_action\"}[$__rate_interval]))", "legendFormat": "{{action}}-{{controller}}", "range": true, "refId": "A" } ], "title": "Action-Controller Latency ($puma_action)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "Rate of queries/sec of API calls served by the Mastodon Webserver, split out by action, format and controller.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 21 }, "id": 42, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum by (controller,action,format) (rate(mastodon_controller_duration_count{controller=~\"$puma_controller\",action=~\"$puma_action\",format=~\"$puma_format\"}[$__rate_interval]))", "legendFormat": "{{action}}-{{format}}-{{controller}}", "range": true, "refId": "A" } ], "title": "Action-Format-Controller Operations ($puma_action,$puma_format)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "Rate of response codes/sec of HTTP API calls served by the Mastodon Webserver", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 21 }, "id": 59, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum by (status) (rate(mastodon_controller_status{action=~\"$puma_action\",format=~\"$puma_format\",controller=~\"$puma_controller\"}[$__rate_interval]))", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Action-Format-Controller Return Codes ($puma_action,$puma_format,$puma_controller)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "Current latency of API calls served by the Mastodon Webserver, split out by controller and action", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 1, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 21 }, "id": 43, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum by (controller,action,format) (rate(mastodon_controller_duration_sum{controller=~\"$puma_controller\",action=~\"$puma_action\",format=~\"$puma_format\"}[$__rate_interval])) /\nsum by (controller,action,format) (rate(mastodon_controller_duration_count{controller=~\"$puma_controller\",action=~\"$puma_action\",format=~\"$puma_format\"}[$__rate_interval]))", "legendFormat": "{{action}}-{{format}}-{{controller}}", "range": true, "refId": "A" } ], "title": "Action-Format-Controller Latency ($puma_action,$puma_format)", "type": "timeseries" } ], "title": "Puma Controller Details ($puma_controller,$puma_action,$puma_format)", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 7 }, "id": 7, "panels": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "Total amount of SQL operations across Mastodon - operations are SELECT, INSERT, DELETE and UPDATE.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 8 }, "id": 4, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum by (operation) (rate(mastodon_db_operation_count{operation=~\"$sql_operation\"}[5m]))", "legendFormat": "{{operation}}", "range": true, "refId": "A" } ], "title": "Database SQL Operations", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "Running latency of SQL operations across Mastodon - operations are SELECT, INSERT, DELETE and UPDATE.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineStyle": { "fill": "solid" }, "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 1, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 8 }, "id": 5, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum by (operation) (rate(mastodon_db_operation_sum{operation=~\"$sql_operation\"}[$__rate_interval])) / \nsum by (operation) (rate(mastodon_db_operation_count{operation=~\"$sql_operation\"}[$__rate_interval]))", "legendFormat": "{{operation}}", "range": true, "refId": "A" } ], "title": "Database SQL Latency", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "The 90th percentile latency over all SELECT, INSERT, UPDATE and DELETE queries as seen by the client.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "max": 1, "thresholds": { "mode": "percentage", "steps": [ { "color": "green" }, { "color": "orange", "value": 60 }, { "color": "red", "value": 70 }, { "color": "dark-red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 4, "w": 8, "x": 16, "y": 8 }, "id": 13, "options": { "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true }, "pluginVersion": "9.2.6", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "avg by (operation) (mastodon_db_operation{quantile=\"0.9\",operation=\"select\"}>0)", "legendFormat": "__auto", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "avg by (operation) (mastodon_db_operation{quantile=\"0.9\",operation=\"insert\"}>0)", "hide": false, "legendFormat": "__auto", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "avg by (operation) (mastodon_db_operation{quantile=\"0.9\",operation=\"update\"}>0)", "hide": false, "legendFormat": "__auto", "range": true, "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "avg by (operation) (mastodon_db_operation{quantile=\"0.9\",operation=\"delete\"}>0)", "hide": false, "legendFormat": "__auto", "range": true, "refId": "D" } ], "title": "90th ptile latency", "type": "gauge" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "The 99th percentile latency over all SELECT, INSERT, UPDATE and DELETE queries as seen by the client.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "max": 2, "thresholds": { "mode": "percentage", "steps": [ { "color": "green" }, { "color": "orange", "value": 60 }, { "color": "red", "value": 70 }, { "color": "dark-red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 4, "w": 8, "x": 16, "y": 12 }, "id": 14, "options": { "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true }, "pluginVersion": "9.2.6", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "avg by (operation) (mastodon_db_operation{quantile=\"0.99\",operation=\"select\"}>0)", "legendFormat": "__auto", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "avg by (operation) (mastodon_db_operation{quantile=\"0.99\",operation=\"insert\"}>0)", "hide": false, "legendFormat": "__auto", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "avg by (operation) (mastodon_db_operation{quantile=\"0.99\",operation=\"update\"}>0)", "hide": false, "legendFormat": "__auto", "range": true, "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "avg by (operation) (mastodon_db_operation{quantile=\"0.99\",operation=\"delete\"}>0)", "hide": false, "legendFormat": "__auto", "range": true, "refId": "D" } ], "title": "99th ptile latency", "type": "gauge" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "Lefthand graph is the number of SQL statements per second, split by table", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 16 }, "id": 10, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "rate(mastodon_db_operation_count{table=~\"$sql_table\",operation=~\"$sql_operation\"}[$__rate_interval])", "legendFormat": "{{table}}.{{operation}}", "range": true, "refId": "A" } ], "title": "SQL Table Operations ($sql_table,$sql_operation)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "Lefthand graph is the latency of database operations specified by table", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 16 }, "id": 11, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "rate(mastodon_db_operation_sum{table=~\"$sql_table\",operation=~\"$sql_operation\"}[$__rate_interval]) / \nrate(mastodon_db_operation_count{table=~\"$sql_table\",operation=~\"$sql_operation\"}[$__rate_interval])", "legendFormat": "{{table}}.{{operation}}", "range": true, "refId": "A" } ], "title": "SQL Table latency ($sql_table,$sql_operation)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "The 90th percentile latency over SELECT, INSERT, UPDATE and DELETE queries to the specified SQL Table as seen by the client.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "max": 1, "thresholds": { "mode": "percentage", "steps": [ { "color": "green" }, { "color": "orange", "value": 60 }, { "color": "red", "value": 70 }, { "color": "dark-red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 4, "w": 8, "x": 16, "y": 16 }, "id": 15, "options": { "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true }, "pluginVersion": "9.2.6", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "avg by (operation) (mastodon_db_operation{quantile=\"0.9\",operation=\"select\",table=~\"$sql_table\"}>0)", "legendFormat": "__auto", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "avg by (operation) (mastodon_db_operation{quantile=\"0.9\",operation=\"insert\",table=~\"$sql_table\"}>0)", "hide": false, "legendFormat": "__auto", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "avg by (operation) (mastodon_db_operation{quantile=\"0.9\",operation=\"update\",table=~\"$sql_table\"}>0)", "hide": false, "legendFormat": "__auto", "range": true, "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "avg by (operation) (mastodon_db_operation{quantile=\"0.9\",operation=\"delete\",table=~\"$sql_table\"}>0)", "hide": false, "legendFormat": "__auto", "range": true, "refId": "D" } ], "title": "90th ptile latency ($sql_table)", "type": "gauge" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "The 99th percentile latency over SELECT, INSERT, UPDATE and DELETE queries to the specified SQL Table as seen by the client.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "max": 2, "thresholds": { "mode": "percentage", "steps": [ { "color": "green" }, { "color": "orange", "value": 60 }, { "color": "red", "value": 70 }, { "color": "dark-red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 4, "w": 8, "x": 16, "y": 20 }, "id": 16, "options": { "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true }, "pluginVersion": "9.2.6", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "avg by (operation) (mastodon_db_operation{quantile=\"0.99\",operation=\"select\",table=~\"$sql_table\"}>0)", "legendFormat": "__auto", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "avg by (operation) (mastodon_db_operation{quantile=\"0.99\",operation=\"insert\",table=~\"$sql_table\"}>0)", "hide": false, "legendFormat": "__auto", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "avg by (operation) (mastodon_db_operation{quantile=\"0.99\",operation=\"update\",table=~\"$sql_table\"}>0)", "hide": false, "legendFormat": "__auto", "range": true, "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "avg by (operation) (mastodon_db_operation{quantile=\"0.99\",operation=\"delete\",table=~\"$sql_table\"}>0)", "hide": false, "legendFormat": "__auto", "range": true, "refId": "D" } ], "title": "99th ptile latency ($sql_table)", "type": "gauge" } ], "title": "SQL Details ($sql_table,$sql_operation)", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 8 }, "id": 18, "panels": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "Cache operations per second from the Mastodon service", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 9 }, "id": 20, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum by (operation)(rate(mastodon_cache_duration_count{operation=~\"$cache_operation\"}[5m]))", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Cache Operations ($cache_operation)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "Cache latency from the Mastodon service", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 1, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green" }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 9 }, "id": 21, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "multi", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum by (operation)(rate(mastodon_cache_duration_sum{operation=~\"$cache_operation\"}[5m])) / \nsum by (operation)(rate(mastodon_cache_duration_count{operation=~\"$cache_operation\"}[5m]))", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Cache Latency ($cache_operation)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "The 90th percentile latency over cache operations that are not hits as seen by the client.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "max": 0.25, "thresholds": { "mode": "percentage", "steps": [ { "color": "green" }, { "color": "orange", "value": 60 }, { "color": "red", "value": 70 }, { "color": "dark-red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 4, "w": 8, "x": 16, "y": 9 }, "id": 22, "options": { "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true }, "pluginVersion": "9.2.6", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "avg by (operation) (mastodon_cache_duration{quantile=\"0.9\",operation=\"read_miss\"}>0)", "hide": false, "legendFormat": "__auto", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "avg by (operation) (mastodon_cache_duration{quantile=\"0.9\",operation=\"increment\"}>0)", "hide": false, "legendFormat": "__auto", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "avg by (operation) (mastodon_cache_duration{quantile=\"0.9\",operation=\"write\"}>0)", "hide": false, "legendFormat": "__auto", "range": true, "refId": "D" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "avg by (operation) (mastodon_cache_duration{quantile=\"0.9\",operation=\"generate\"}>0)", "hide": false, "legendFormat": "__auto", "range": true, "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "avg by (operation) (mastodon_cache_duration{quantile=\"0.9\",operation=\"delete\"}>0)", "hide": false, "legendFormat": "__auto", "range": true, "refId": "E" } ], "title": "90th ptile latency", "type": "gauge" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "The 99th percentile latency over cache operations that are not hits as seen by the client.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "max": 0.8, "thresholds": { "mode": "percentage", "steps": [ { "color": "green" }, { "color": "orange", "value": 60 }, { "color": "red", "value": 70 }, { "color": "dark-red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 4, "w": 8, "x": 16, "y": 13 }, "id": 23, "options": { "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true }, "pluginVersion": "9.2.6", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "avg by (operation) (mastodon_cache_duration{quantile=\"0.99\",operation=\"read_miss\"}>0)", "hide": false, "legendFormat": "__auto", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "avg by (operation) (mastodon_cache_duration{quantile=\"0.99\",operation=\"increment\"}>0)", "hide": false, "legendFormat": "__auto", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "avg by (operation) (mastodon_cache_duration{quantile=\"0.99\",operation=\"write\"}>0)", "hide": false, "legendFormat": "__auto", "range": true, "refId": "D" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "avg by (operation) (mastodon_cache_duration{quantile=\"0.99\",operation=\"generate\"}>0)", "hide": false, "legendFormat": "__auto", "range": true, "refId": "C" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "avg by (operation) (mastodon_cache_duration{quantile=\"0.99\",operation=\"delete\"}>0)", "hide": false, "legendFormat": "__auto", "range": true, "refId": "E" } ], "title": "99th ptile latency", "type": "gauge" } ], "title": "Cache Details ($cache_operation)", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 9 }, "id": 45, "panels": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "Rate of sidekiq worker invocations by worker type, split out by success and failure.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 8, "w": 9, "x": 0, "y": 10 }, "id": 47, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum by (worker)(rate(mastodon_sidekiq_worker_success_total{worker=~\"$sidekiq_worker\"}[$__rate_interval]))", "legendFormat": "{{worker}} Success", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum by (worker)(rate(mastodon_sidekiq_worker_failure_total{worker=~\"$sidekiq_worker\"}[$__rate_interval]))", "hide": false, "legendFormat": "{{worker}} Failure", "range": true, "refId": "B" } ], "title": "Sidekiq Worker ($sidekiq_worker)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "Running latency of sidekiq worker invocations by worker type.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "decimals": 1, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 9, "x": 9, "y": 10 }, "id": 48, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sum by (worker)(rate(mastodon_sidekiq_worker_processing_time_sum{worker=~\"$sidekiq_worker\"}[$__rate_interval]))\n/ sum by (worker)(rate(mastodon_sidekiq_worker_processing_time_count{worker=~\"$sidekiq_worker\"}[$__rate_interval]))\n", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Sidekiq Latency ($sidekiq_worker)", "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "The median, 90th and 99th percentile latency over Sidekiq worker invocations.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "max": 5, "thresholds": { "mode": "percentage", "steps": [ { "color": "green", "value": null }, { "color": "orange", "value": 60 }, { "color": "red", "value": 70 }, { "color": "dark-red", "value": 80 } ] }, "unit": "s" }, "overrides": [ { "matcher": { "id": "byFrameRefID", "options": "B" }, "properties": [ { "id": "max", "value": 30 } ] }, { "matcher": { "id": "byFrameRefID", "options": "C" }, "properties": [ { "id": "max", "value": 90 } ] } ] }, "gridPos": { "h": 4, "w": 6, "x": 18, "y": 10 }, "id": 49, "options": { "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showThresholdLabels": false, "showThresholdMarkers": true }, "pluginVersion": "9.2.6", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "avg by (quantile)(mastodon_sidekiq_worker_processing_time{quantile=\"0.5\",worker=~\"$sidekiq_worker\"}>0)", "hide": false, "legendFormat": "median", "range": true, "refId": "A" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "avg by (quantile)(mastodon_sidekiq_worker_processing_time{quantile=\"0.9\",worker=~\"$sidekiq_worker\"}>0)", "hide": false, "legendFormat": "90th ptile", "range": true, "refId": "B" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "avg by (quantile)(mastodon_sidekiq_worker_processing_time{quantile=\"0.99\",worker=~\"$sidekiq_worker\"}>0)", "hide": false, "interval": "", "legendFormat": "99th ptile", "range": true, "refId": "C" } ], "title": "50,90,99th ptile latency ($sidekiq_worker)", "type": "gauge" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "A total number of worker jobs waiting in the queue, usually due to there not being enough worker threads or sidekiq jobs to handle them in time. This graph here shows the n-count of elements in the queue.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "max": 1000, "thresholds": { "mode": "percentage", "steps": [ { "color": "green", "value": null }, { "color": "dark-yellow", "value": 50 }, { "color": "dark-orange", "value": 60 }, { "color": "dark-red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 4, "w": 6, "x": 18, "y": 14 }, "id": 63, "options": { "displayMode": "lcd", "minVizHeight": 10, "minVizWidth": 0, "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "showUnfilled": true }, "pluginVersion": "9.2.6", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sort(sum by (queue)(mastodon_sidekiq_queue_enqueued))", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Sidekiq Queue Depth", "type": "bargauge" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "A historical graph of the average queue delay for each of the main queues in Sidekiq. If the queues do not have wait-time, they are depicted green. They will go through yellow, orange and become red once the queue is backing up and worker jobs have to wait before being scheduled.", "fieldConfig": { "defaults": { "color": { "mode": "continuous-BlYlRd" }, "custom": { "fillOpacity": 70, "lineWidth": 0, "spanNulls": false }, "mappings": [], "max": 300, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 5, "w": 21, "x": 0, "y": 18 }, "id": 61, "options": { "alignValue": "left", "legend": { "displayMode": "list", "placement": "bottom", "showLegend": true }, "mergeValues": true, "rowHeight": 0.9, "showValue": "never", "tooltip": { "mode": "single", "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "expr": "sort(sum by (queue)(mastodon_sidekiq_queue_latency))", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Sidekiq Queue Delay timeline", "type": "state-timeline" }, { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "description": "The total amount of worker jobs that Sidekiq is performing.", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 1, "mappings": [], "max": 10, "thresholds": { "mode": "percentage", "steps": [ { "color": "dark-green", "value": null }, { "color": "#EAB839", "value": 40 }, { "color": "dark-orange", "value": 60 }, { "color": "dark-red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 5, "w": 3, "x": 21, "y": 18 }, "id": 56, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "text": { "titleSize": 18, "valueSize": 18 }, "textMode": "auto" }, "pluginVersion": "9.2.6", "targets": [ { "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "editorMode": "code", "exemplar": false, "expr": "sort(sum by (queue)(mastodon_sidekiq_queue_latency))", "instant": false, "interval": "", "legendFormat": "__auto", "range": true, "refId": "A" } ], "title": "Sidekiq Queue Delay", "type": "stat" } ], "title": "Sidekiq Details ($sidekiq_worker)", "type": "row" } ], "refresh": "1m", "schemaVersion": 37, "style": "dark", "tags": [], "templating": { "list": [ { "current": {}, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "definition": "mastodon_controller_duration_count", "hide": 0, "includeAll": true, "label": "Puma Controller", "multi": true, "name": "puma_controller", "options": [], "query": { "query": "mastodon_controller_duration_count", "refId": "StandardVariableQuery" }, "refresh": 1, "regex": "/controller=\"([^\"]+)\"/", "skipUrlSync": false, "sort": 1, "type": "query" }, { "current": {}, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "definition": "mastodon_controller_duration_count", "hide": 0, "includeAll": true, "label": "Puma Action", "multi": true, "name": "puma_action", "options": [], "query": { "query": "mastodon_controller_duration_count", "refId": "StandardVariableQuery" }, "refresh": 1, "regex": "/action=\"([^\"]+)\"/", "skipUrlSync": false, "sort": 1, "type": "query" }, { "current": {}, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "definition": "mastodon_controller_duration_count", "hide": 0, "includeAll": true, "label": "Puma Format", "multi": true, "name": "puma_format", "options": [], "query": { "query": "mastodon_controller_duration_count", "refId": "StandardVariableQuery" }, "refresh": 1, "regex": "/format=\"([^\"]+)\"/", "skipUrlSync": false, "sort": 1, "type": "query" }, { "current": {}, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "definition": "mastodon_db_operation", "hide": 0, "includeAll": true, "label": "SQL Table", "multi": true, "name": "sql_table", "options": [], "query": { "query": "mastodon_db_operation", "refId": "StandardVariableQuery" }, "refresh": 1, "regex": "/table=\"([a-z]+).*/", "skipUrlSync": false, "sort": 1, "type": "query" }, { "current": {}, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "definition": "mastodon_db_operation", "hide": 0, "includeAll": true, "label": "SQL Operation", "multi": true, "name": "sql_operation", "options": [], "query": { "query": "mastodon_db_operation", "refId": "StandardVariableQuery" }, "refresh": 1, "regex": "/operation=\"([a-z]+).*/", "skipUrlSync": false, "sort": 1, "type": "query" }, { "current": {}, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "definition": "mastodon_cache_duration_count", "hide": 0, "includeAll": true, "label": "Cache Operation", "multi": true, "name": "cache_operation", "options": [], "query": { "query": "mastodon_cache_duration_count", "refId": "StandardVariableQuery" }, "refresh": 1, "regex": "/operation=\"([a-z_]+).*/", "skipUrlSync": false, "sort": 1, "type": "query" }, { "current": {}, "datasource": { "type": "prometheus", "uid": "PBFA97CFB590B2093" }, "definition": "mastodon_sidekiq_worker_success_total", "hide": 0, "includeAll": true, "label": "Sidekiq Worker", "multi": true, "name": "sidekiq_worker", "options": [], "query": { "query": "mastodon_sidekiq_worker_success_total", "refId": "StandardVariableQuery" }, "refresh": 1, "regex": "/worker=\"([^\\\"]+)/", "skipUrlSync": false, "sort": 1, "type": "query" } ] }, "time": { "from": "now-24h", "to": "now" }, "timepicker": { "nowDelay": "" }, "timezone": "", "title": "Mastodon Stats", "uid": "IXDZT_dVz", "version": 61, "weekStart": "", "gnetId": 17492 }