From be0f0bc52e2b8ded733c446389108d1a5928bd38 Mon Sep 17 00:00:00 2001 From: Joseph Hanson Date: Mon, 21 Oct 2024 11:46:13 -0500 Subject: [PATCH] add grafana --- .../grafana/app/helmrelease.yaml | 332 ++ .../grafana/app/kustomization.yaml | 26 + .../grafana/app/resources/flux-logs.json | 392 ++ .../grafana/app/resources/zfs-details.json | 3680 +++++++++++++++++ kubernetes/apps/observability/grafana/ks.yaml | 20 + .../apps/observability/kustomization.yaml | 1 + 6 files changed, 4451 insertions(+) create mode 100644 kubernetes/apps/observability/grafana/app/helmrelease.yaml create mode 100644 kubernetes/apps/observability/grafana/app/kustomization.yaml create mode 100644 kubernetes/apps/observability/grafana/app/resources/flux-logs.json create mode 100644 kubernetes/apps/observability/grafana/app/resources/zfs-details.json create mode 100644 kubernetes/apps/observability/grafana/ks.yaml diff --git a/kubernetes/apps/observability/grafana/app/helmrelease.yaml b/kubernetes/apps/observability/grafana/app/helmrelease.yaml new file mode 100644 index 00000000..165ab8ee --- /dev/null +++ b/kubernetes/apps/observability/grafana/app/helmrelease.yaml @@ -0,0 +1,332 @@ +--- +# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/helm.toolkit.fluxcd.io/helmrelease_v2beta2.json +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: grafana +spec: + interval: 30m + chartRef: + kind: OCIRepository + name: grafana + namespace: flux-system + driftDetection: + mode: enabled + install: + remediation: + retries: 3 + upgrade: + cleanupOnFail: true + remediation: + retries: 3 + values: + deploymentStrategy: + type: RollingUpdate + testFramework: + enabled: false + serviceMonitor: + enabled: true + metricRelabelings: + - action: labeldrop + regex: (pod) + ingress: + enabled: true + ingressClassName: internal-nginx + hosts: + - &host "grafana.jahanson.tech" + tls: + - hosts: + - *host + topologySpreadConstraints: + - maxSkew: 1 + topologyKey: kubernetes.io/hostname + whenUnsatisfiable: DoNotSchedule + labelSelector: + matchLabels: + app.kubernetes.io/name: grafana + persistence: + enabled: false + plugins: + - grafana-clock-panel + - https://github.com/VictoriaMetrics/victorialogs-datasource/releases/download/v0.4.0/victorialogs-datasource-v0.4.0.zip;victorialogs-datasource + - https://github.com/VictoriaMetrics/victoriametrics-datasource/releases/download/v0.9.0/victoriametrics-datasource-v0.9.0.zip;victoriametrics-datasource + + datasources: + datasources.yaml: + apiVersion: 1 + deleteDatasources: + - { name: Alertmanager, orgId: 1 } + - { name: Loki, orgId: 1 } + - { name: Prometheus, orgId: 1 } + datasources: + - name: Prometheus + type: prometheus + uid: prometheus + access: proxy + url: http://vmsingle-victoria.observability.svc:8429 + isDefault: true + jsonData: + prometheusType: Prometheus + timeInterval: 60s + - name: VictoriaMetrics + type: victoriametrics-datasource + access: proxy + url: http://vmsingle-stack.observability.svc:8429 + - name: VictoriaLogs + type: victorialogs-datasource + access: proxy + url: http://victoria-logs.observability:9428 + - name: Alertmanager + type: alertmanager + uid: alertmanager + access: proxy + url: http://alertmanager.observability.svc.cluster.local:9093 + jsonData: + implementation: prometheus + + dashboardProviders: + dashboardproviders.yaml: + apiVersion: 1 + providers: + - name: default + orgId: 1 + folder: "" + type: file + disableDeletion: false + editable: true + options: + path: /var/lib/grafana/dashboards/default + - name: flux + orgId: 1 + folder: Flux + type: file + disableDeletion: false + editable: true + options: + path: /var/lib/grafana/dashboards/flux + - name: kubernetes + orgId: 1 + folder: Kubernetes + type: file + disableDeletion: false + editable: true + options: + path: /var/lib/grafana/dashboards/kubernetes + - name: nginx + orgId: 1 + folder: Nginx + type: file + disableDeletion: false + editable: true + options: + path: /var/lib/grafana/dashboards/nginx + - name: observability + orgId: 1 + folder: Observability + type: file + disableDeletion: false + editable: true + options: + path: /var/lib/grafana/dashboards/observability + - name: rook-ceph + orgId: 1 + folder: Rook-Ceph + type: file + disableDeletion: false + editable: true + options: + path: /var/lib/grafana/dashboards/rook-ceph + - name: unifi + orgId: 1 + folder: Unifi + type: file + disableDeletion: false + editable: true + options: + path: /var/lib/grafana/dashboards/unifi + dashboards: + default: + cloudflared: + # renovate: depName="Cloudflare Tunnels (cloudflared)" + gnetId: 17457 + revision: 6 + datasource: + - { name: DS_PROMETHEUS, value: Prometheus } + external-dns: + # renovate: depName="External-dns" + gnetId: 15038 + revision: 3 + datasource: Prometheus + minio: + # renovate: depName="MinIO Dashboard" + gnetId: 13502 + revision: 26 + datasource: + - { name: DS_PROMETHEUS, value: Prometheus } + node-exporter-full: + # renovate: depName="Node Exporter Full" + gnetId: 1860 + revision: 37 + datasource: Prometheus + nut-exporter: + # renovate: depName="Prometheus NUT Exporter for DRuggeri" + gnetId: 19308 + revision: 3 + datasource: + - { name: DS_PROMETHEUS, value: Prometheus } + smart: + # renovate: depName="Dashboard for smartctl_exporter" + gnetId: 20204 + revision: 1 + datasource: + - { name: DS_PROMETHEUS, value: Prometheus } + volsync: + # removate: depNmae="VolSync Dashboard" + gnetId: 21356 + revision: 1 + datasource: + - { name: DS_PROMETHEUS, value: Prometheus } + - { name: VAR_REPLICATIONDESTNAME, value: .*-dst } + cert-manager: + url: https://gitlab.com/uneeq-oss/cert-manager-mixin/-/raw/master/dashboards/cert-manager.json?ref_type=heads + datasource: Prometheus + dcgm-exporter: + url: https://raw.githubusercontent.com/NVIDIA/dcgm-exporter/main/grafana/dcgm-exporter-dashboard.json + datasource: Prometheus + external-secrets: + url: https://raw.githubusercontent.com/external-secrets/external-secrets/main/docs/snippets/dashboard.json + datasource: Prometheus + node-feature-discovery: + url: https://raw.githubusercontent.com/kubernetes-sigs/node-feature-discovery/master/examples/grafana-dashboard.json + datasource: Prometheus + flux: + flux-cluster: + url: https://raw.githubusercontent.com/fluxcd/flux2-monitoring-example/main/monitoring/configs/dashboards/cluster.json + datasource: Prometheus + flux-control-plane: + url: https://raw.githubusercontent.com/fluxcd/flux2-monitoring-example/main/monitoring/configs/dashboards/control-plane.json + datasource: Prometheus + kubernetes: + k8s-system-api-server: + url: https://raw.githubusercontent.com/dotdc/grafana-dashboards-kubernetes/master/dashboards/k8s-system-api-server.json + token: '' + k8s-system-coredns: + url: https://raw.githubusercontent.com/dotdc/grafana-dashboards-kubernetes/master/dashboards/k8s-system-coredns.json + token: '' + k8s-views-global: + url: https://raw.githubusercontent.com/dotdc/grafana-dashboards-kubernetes/master/dashboards/k8s-views-global.json + token: '' + k8s-views-namespaces: + url: https://raw.githubusercontent.com/dotdc/grafana-dashboards-kubernetes/master/dashboards/k8s-views-namespaces.json + token: '' + k8s-views-nodes: + url: https://raw.githubusercontent.com/dotdc/grafana-dashboards-kubernetes/master/dashboards/k8s-views-nodes.json + token: '' + k8s-views-pods: + url: https://raw.githubusercontent.com/dotdc/grafana-dashboards-kubernetes/master/dashboards/k8s-views-pods.json + token: '' + nginx: + nginx: + url: https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/grafana/dashboards/nginx.json + datasource: Prometheus + nginx-request-handling-performance: + url: https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/grafana/dashboards/request-handling-performance.json + datasource: Prometheus + observability: + prometheus: + # renovate: depName="Prometheus" + gnetId: 19105 + revision: 5 + datasource: Prometheus + victoria-agent: + # renovate: depName="VictoriaMetrics - vmagent" + gnetId: 12683 + revision: 20 + datasource: Prometheus + victoria-alert: + # renovate: depName="VictoriaMetrics - vmalert" + gnetId: 14950 + revision: 12 + datasource: Prometheus + victoria-operator: + # renovate: depName="VictoriaMetrics - operator" + gnetId: 17869 + revision: 3 + datasource: Prometheus + victoria-single: + # renovate: depName="VictoriaMetrics - single-node" + gnetId: 10229 + revision: 36 + datasource: Prometheus + rook-ceph: + rook-ceph-cluster: + # renovate: depName="Ceph Cluster" + gnetId: 2842 + revision: 18 + datasource: Prometheus + rook-ceph-osd-single: + # renovate: depName="Ceph - OSD (Single)" + gnetId: 5336 + revision: 9 + datasource: Prometheus + rook-ceph-pools: + # renovate: depName="Ceph - Pools" + gnetId: 5342 + revision: 9 + datasource: Prometheus + unifi: + unifi-insights: + # renovate: depName="UniFi-Poller: Client Insights - Prometheus" + gnetId: 11315 + revision: 9 + datasource: Prometheus + unifi-network-sites: + # renovate: depName="UniFi-Poller: Network Sites - Prometheus" + gnetId: 11311 + revision: 5 + datasource: Prometheus + unifi-uap: + # renovate: depName="UniFi-Poller: UAP Insights - Prometheus" + gnetId: 11314 + revision: 10 + datasource: Prometheus + unifi-usw: + # renovate: depName="UniFi-Poller: USW Insights - Prometheus" + gnetId: 11312 + revision: 9 + datasource: Prometheus + + ## Grafana's primary configuration + ## NOTE: values in map will be converted to ini format + ## ref: http://docs.grafana.org/installation/configuration/ + ## + grafana.ini: + analytics: + check_for_updates: false + check_for_plugin_updates: false + reporting_enabled: false + auth: + disable_login_form: true + auth.anonymous: + enabled: true + news: + news_feed_enabled: false + plugins: + allow_loading_unsigned_plugins: "victorialogs-datasource,victoriametrics-datasource" + users: + allow_sign_up: false + auto_assign_org: true + auto_assign_org_role: Admin + + sidecar: + dashboards: + enabled: true + searchNamespace: ALL + folderAnnotation: grafana_folder + provider: + disableDelete: true + foldersFromFilesStructure: true + datasources: + enabled: true + searchNamespace: ALL + enableUniqueFilenames: true diff --git a/kubernetes/apps/observability/grafana/app/kustomization.yaml b/kubernetes/apps/observability/grafana/app/kustomization.yaml new file mode 100644 index 00000000..6e03333e --- /dev/null +++ b/kubernetes/apps/observability/grafana/app/kustomization.yaml @@ -0,0 +1,26 @@ +--- +# yaml-language-server: $schema=https://json.schemastore.org/kustomization +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./helmrelease.yaml +configMapGenerator: + - name: grafana-dashboards + files: + - ./resources/zfs-details.json + options: + annotations: + kustomize.toolkit.fluxcd.io/substitute: disabled + labels: + grafana_dashboard: "1" + - name: flux-dashboards + files: + - ./resources/flux-logs.json + options: + annotations: + kustomize.toolkit.fluxcd.io/substitute: disabled + labels: + grafana_dashboard: "1" + grafana_folder: "Flux" +generatorOptions: + disableNameSuffixHash: true diff --git a/kubernetes/apps/observability/grafana/app/resources/flux-logs.json b/kubernetes/apps/observability/grafana/app/resources/flux-logs.json new file mode 100644 index 00000000..bbcbc91d --- /dev/null +++ b/kubernetes/apps/observability/grafana/app/resources/flux-logs.json @@ -0,0 +1,392 @@ +{ + "__inputs": [ + { + "name": "DS_VICTORIALOGS", + "label": "VictoriaLogs", + "description": "", + "type": "datasource", + "pluginId": "victorialogs-datasource", + "pluginName": "VictoriaLogs" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "11.2.0" + }, + { + "type": "panel", + "id": "logs", + "name": "Logs", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + }, + { + "type": "datasource", + "id": "victorialogs-datasource", + "name": "VictoriaLogs", + "version": "0.4.0" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + }, + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "iconColor": "red", + "name": "flux events", + "target": { + "limit": 100, + "matchAny": false, + "tags": [ + "flux" + ], + "type": "tags" + } + } + ] + }, + "description": "Flux logs collected from Kubernetes, stored in VictoriaLogs", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "victorialogs-datasource", + "uid": "${DS_VICTORIALOGS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "victorialogs-datasource", + "uid": "${DS_VICTORIALOGS}" + }, + "expr": "sum(count_over_time({namespace=~\"$namespace\", stream=~\"$stream\", app =~\"$controller\"} | json | __error__!=\"JSONParserErr\" | level=~\"$level\" |= \"$query\" [$__interval]))", + "instant": false, + "legendFormat": "Log count", + "range": true, + "refId": "A" + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "victorialogs-datasource", + "uid": "${DS_VICTORIALOGS}" + }, + "description": "Logs from services running in Kubernetes", + "gridPos": { + "h": 25, + "w": 24, + "x": 0, + "y": 4 + }, + "id": 2, + "options": { + "dedupStrategy": "numbers", + "enableLogDetails": false, + "prettifyLogMessage": true, + "showCommonLabels": false, + "showLabels": false, + "showTime": false, + "sortOrder": "Descending", + "wrapLogMessage": false + }, + "targets": [ + { + "datasource": { + "type": "victorialogs-datasource", + "uid": "${DS_VICTORIALOGS}" + }, + "expr": "{namespace=~\"$namespace\", stream=~\"$stream\", app =~\"$controller\"} | json | __error__!=\"JSONParserErr\" | level=~\"$level\" |= \"$query\"", + "refId": "A" + } + ], + "type": "logs" + } + ], + "refresh": "10s", + "schemaVersion": 39, + "tags": [ + "flux" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "", + "value": "" + }, + "description": "String to search for", + "hide": 0, + "label": "Search Query", + "name": "query", + "options": [ + { + "selected": true, + "text": "", + "value": "" + } + ], + "query": "", + "skipUrlSync": false, + "type": "textbox" + }, + { + "allValue": "info|error", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "hide": 0, + "includeAll": true, + "multi": false, + "name": "level", + "options": [ + { + "selected": true, + "text": "All", + "value": "$__all" + }, + { + "selected": false, + "text": "info", + "value": "info" + }, + { + "selected": false, + "text": "error", + "value": "error" + } + ], + "query": "info,error", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + }, + { + "allValue": ".+", + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "victorialogs-datasource", + "uid": "${DS_VICTORIALOGS}" + }, + "definition": "label_values(app)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "controller", + "options": [], + "query": "label_values(app)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": ".+", + "current": { + "selected": true, + "text": [ + "flux-system" + ], + "value": [ + "flux-system" + ] + }, + "datasource": { + "type": "victorialogs-datasource", + "uid": "${DS_VICTORIALOGS}" + }, + "definition": "label_values(namespace)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "namespace", + "options": [], + "query": "label_values(namespace)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": ".+", + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "victorialogs-datasource", + "uid": "${DS_VICTORIALOGS}" + }, + "definition": "label_values(stream)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "stream", + "options": [], + "query": "label_values(stream)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "VictoriaLogs", + "value": "VictoriaLogs" + }, + "hide": 0, + "includeAll": false, + "label": "Datasource", + "multi": false, + "name": "DS_VICTORIALOGS", + "options": [], + "query": "victorialogs-datasource", + "queryValue": "VictoriaLogs", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Flux Logs", + "uid": "flux-logs", + "version": 3, + "weekStart": "" +} diff --git a/kubernetes/apps/observability/grafana/app/resources/zfs-details.json b/kubernetes/apps/observability/grafana/app/resources/zfs-details.json new file mode 100644 index 00000000..6f02bce2 --- /dev/null +++ b/kubernetes/apps/observability/grafana/app/resources/zfs-details.json @@ -0,0 +1,3680 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "Prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "11.2.0" + }, + { + "type": "panel", + "id": "heatmap", + "name": "Heatmap", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "ZFS monitoring dashboard for Unraid", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 19376, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 18, + "panels": [], + "title": "General", + "type": "row" + }, + { + "datasource": { + "default": false, + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": false, + "inspect": false + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Used %" + }, + "properties": [ + { + "id": "unit", + "value": "percentunit" + }, + { + "id": "min", + "value": 0 + }, + { + "id": "max", + "value": 1 + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "lcd", + "type": "gauge" + } + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.75 + }, + { + "color": "orange", + "value": 0.85 + }, + { + "color": "red", + "value": 0.9 + } + ] + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Health" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "options": { + "0": { + "color": "green", + "index": 0, + "text": "ONLINE" + }, + "1": { + "color": "yellow", + "index": 1, + "text": "DEGRADED" + }, + "2": { + "color": "red", + "index": 2, + "text": "FAULTED" + }, + "3": { + "color": "#808080", + "index": 3, + "text": "OFFLINE" + }, + "4": { + "color": "#808080", + "index": 4, + "text": "UNAVAIL" + }, + "5": { + "color": "#808080", + "index": 5, + "text": "REMOVED" + }, + "6": { + "color": "#808080", + "index": 6, + "text": "SUSPENDED" + } + }, + "type": "value" + } + ] + }, + { + "id": "custom.cellOptions", + "value": { + "mode": "gradient", + "type": "color-background" + } + }, + { + "id": "custom.align", + "value": "center" + } + ] + } + ] + }, + "gridPos": { + "h": 15, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 95, + "interval": "$interval", + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "frameIndex": 4, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "zfs_dataset_available_bytes", + "format": "table", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "AVAIL", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "zfs_dataset_used_bytes", + "format": "table", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "USED", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "zfs_pool_health", + "format": "table", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "HEALTH", + "useBackend": false + } + ], + "title": "ZFS Datasets", + "transformations": [ + { + "id": "groupBy", + "options": { + "fields": { + "Value #ALLOC": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" + }, + "Value #AVAIL": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" + }, + "Value #B": { + "aggregations": [] + }, + "Value #C": { + "aggregations": [ + "last" + ], + "operation": "aggregate" + }, + "Value #D": { + "aggregations": [ + "last" + ], + "operation": "aggregate" + }, + "Value #E": { + "aggregations": [ + "last" + ], + "operation": "aggregate" + }, + "Value #F": { + "aggregations": [ + "last" + ], + "operation": "aggregate" + }, + "Value #FRAG": { + "aggregations": [ + "last" + ], + "operation": "aggregate" + }, + "Value #FREE": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" + }, + "Value #G": { + "aggregations": [ + "last" + ], + "operation": "aggregate" + }, + "Value #H": { + "aggregations": [ + "last" + ], + "operation": "aggregate" + }, + "Value #HEALTH": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" + }, + "Value #I": { + "aggregations": [ + "last" + ], + "operation": "aggregate" + }, + "Value #J": { + "aggregations": [ + "last" + ], + "operation": "aggregate" + }, + "Value #SIZE": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" + }, + "Value #USED": { + "aggregations": [ + "lastNotNull" + ], + "operation": "aggregate" + }, + "host": { + "aggregations": [] + }, + "name": { + "aggregations": [], + "operation": "groupby" + }, + "path": { + "aggregations": [] + }, + "pool": { + "aggregations": [], + "operation": "groupby" + }, + "state": { + "aggregations": [], + "operation": "groupby" + }, + "vdev": { + "aggregations": [], + "operation": "groupby" + } + } + } + }, + { + "id": "merge", + "options": {} + }, + { + "id": "calculateField", + "options": { + "alias": "Capacity", + "binary": { + "left": "Value #AVAIL (lastNotNull)", + "right": "Value #USED (lastNotNull)" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + }, + { + "id": "calculateField", + "options": { + "alias": "Used %", + "binary": { + "left": "Value #USED (lastNotNull)", + "operator": "/", + "reducer": "sum", + "right": "Capacity" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "host": true, + "path": true, + "pool": true + }, + "includeByName": {}, + "indexByName": { + "Used %": 8, + "Value #C (last)": 6, + "Value #D (last)": 10, + "Value #E (last)": 9, + "Value #F (last)": 7, + "Value #G (last)": 11, + "Value #H (last)": 5, + "Value #I (last)": 12, + "host": 1, + "name": 0, + "path": 4, + "state": 3, + "vdev": 2 + }, + "renameByName": { + "Value #ALLOC (lastNotNull)": "Allocated", + "Value #AVAIL (lastNotNull)": "Available", + "Value #C (last)": "Used", + "Value #D (last)": "CHK Errors", + "Value #E (last)": "Fragmentation", + "Value #F (last)": "Free", + "Value #FRAG (last)": "Fragmentation", + "Value #FREE (lastNotNull)": "Free", + "Value #G (last)": "Read Bytes", + "Value #H (last)": "Size", + "Value #HEALTH (lastNotNull)": "Health", + "Value #I (last)": "Write Bytes", + "Value #SIZE (lastNotNull)": "Size", + "Value #USED (lastNotNull)": "Used", + "host": "", + "name": "Pool", + "pool": "", + "state": "Health", + "vdev": "VDEV" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 31, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 22, + "interval": "1d", + "options": { + "legend": { + "calcs": [ + "diff" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "exemplar": true, + "expr": "delta(zpool_stats_alloc{job='$job', vdev=\"root\"}[1d])", + "interval": "", + "legendFormat": "{{name}}", + "range": true, + "refId": "A" + } + ], + "timeFrom": "30d", + "title": "Pool Growth by Day", + "type": "timeseries" + }, + { + "alert": { + "alertRuleTags": {}, + "conditions": [ + { + "evaluator": { + "params": [ + 2199023255552 + ], + "type": "lt" + }, + "operator": { + "type": "and" + }, + "query": { + "params": [ + "B", + "5m", + "now" + ] + }, + "reducer": { + "params": [], + "type": "avg" + }, + "type": "query" + }, + { + "evaluator": { + "params": [ + 536870912000 + ], + "type": "lt" + }, + "operator": { + "type": "or" + }, + "query": { + "params": [ + "H", + "5m", + "now" + ] + }, + "reducer": { + "params": [], + "type": "avg" + }, + "type": "query" + }, + { + "evaluator": { + "params": [ + 322122547200 + ], + "type": "lt" + }, + "operator": { + "type": "or" + }, + "query": { + "params": [ + "I", + "5m", + "now" + ] + }, + "reducer": { + "params": [], + "type": "avg" + }, + "type": "query" + } + ], + "executionErrorState": "alerting", + "for": "5m", + "frequency": "1m", + "handler": 1, + "message": "ZPool Low on Space ${hddmain - Free} ${ssdnvme- Free} ${ssdsata - Free}", + "name": "Pool Space alert", + "noDataState": "no_data", + "notifications": [] + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": 60000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 251, + "interval": "1m", + "options": { + "legend": { + "calcs": [ + "min", + "lastNotNull", + "diff" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.1.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "exemplar": true, + "expr": "zpool_stats_alloc{job='$job', vdev=\"root\"}", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{name}} - Used", + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "exemplar": true, + "expr": "zpool_stats_size{job='$job', vdev=\"root\"}", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{name}} - Size", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "exemplar": true, + "expr": "zpool_stats_free{job='$job', vdev=\"root\"}", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{name}} - Free", + "refId": "B" + } + ], + "thresholds": [ + { + "colorMode": "critical", + "op": "lt", + "value": 2199023255552, + "visible": true + } + ], + "timeFrom": "24h", + "title": "Pool Space", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 33 + }, + "id": 62, + "panels": [], + "title": "ARC", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 13, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 60000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 42, + "interval": "$interval", + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "node_zfs_arc_data_size{job=\"$job\"}", + "legendFormat": "Data", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "node_zfs_arc_metadata_size{job=\"$job\"}", + "hide": false, + "legendFormat": "Metadata", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "node_zfs_arc_mfu_size{job=\"$job\"}", + "hide": false, + "legendFormat": "Most Frequently Used", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "node_zfs_arc_dnode_size{job=\"$job\"}", + "hide": false, + "legendFormat": "Dnode Size", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "node_zfs_arc_mru_size{job=\"$job\"}", + "hide": false, + "legendFormat": "Most Recently Used", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "node_zfs_arc_size{job=\"$job\"}", + "hide": false, + "legendFormat": "ARC Size", + "range": true, + "refId": "F" + } + ], + "title": "ARC Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 13, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 60000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.+Ratio/" + }, + "properties": [ + { + "id": "unit", + "value": "percentunit" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 41 + }, + "id": 321, + "interval": "$interval", + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "rate(node_zfs_arc_demand_data_hits{job=\"$job\"}[$__rate_interval])", + "legendFormat": "Hits", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "rate(node_zfs_arc_demand_data_misses{job=\"$job\"}[$__rate_interval])", + "hide": false, + "legendFormat": "Misses", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "rate(node_zfs_arc_demand_data_hits{job=\"$job\"}[$__rate_interval])/(rate(node_zfs_arc_demand_data_misses{job=\"$job\"}[$__rate_interval])+rate(node_zfs_arc_demand_data_hits{job=\"$job\"}[$__rate_interval]))", + "hide": false, + "legendFormat": "Hit Ratio", + "range": true, + "refId": "C" + } + ], + "title": "ARC Demand - Data", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 13, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 60000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.+Ratio/" + }, + "properties": [ + { + "id": "unit", + "value": "percentunit" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 48 + }, + "id": 322, + "interval": "$interval", + "options": { + "legend": { + "calcs": [ + "last" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "rate(node_zfs_arc_demand_metadata_hits{job=\"$job\"}[$__rate_interval])", + "legendFormat": "Hits", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "rate(node_zfs_arc_demand_metadata_misses{job=\"$job\"}[$__rate_interval])", + "hide": false, + "legendFormat": "Misses", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "rate(node_zfs_arc_demand_metadata_hits{job=\"$job\"}[$__rate_interval])/(rate(node_zfs_arc_demand_metadata_misses{job=\"$job\"}[$__rate_interval])+rate(node_zfs_arc_demand_metadata_hits{job=\"$job\"}[$__rate_interval]))", + "hide": false, + "legendFormat": "Hit Ratio", + "range": true, + "refId": "C" + } + ], + "title": "ARC Demand - Metadata", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 55 + }, + "id": 40, + "panels": [], + "title": "Scans", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Start" + }, + "properties": [ + { + "id": "unit", + "value": "dateTimeAsLocal" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "End" + }, + "properties": [ + { + "id": "unit", + "value": "dateTimeAsLocal" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Scanned" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Reconstructed" + }, + "properties": [ + { + "id": "unit", + "value": "bytes" + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 56 + }, + "id": 4, + "interval": "$interval", + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "exemplar": false, + "expr": "zpool_scan_stats_end_ts{state=~\"finished|canceled\", job=\"$job\"} * 1000", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "exemplar": false, + "expr": "zpool_scan_stats_start_ts{state=~\"finished|canceled\", job=\"$job\"} * 1000", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "exemplar": false, + "expr": "zpool_scan_stats_examined{state=~\"finished|canceled\", job=\"$job\"}", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "exemplar": false, + "expr": "zpool_scan_stats_errors{state=~\"finished|canceled\", job=\"$job\"}", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "exemplar": false, + "expr": "zpool_scan_stats_processed{state=~\"finished|canceled\", job=\"$job\"}", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "E" + } + ], + "title": "Pools Scans Finished", + "transformations": [ + { + "id": "groupBy", + "options": { + "fields": { + "Value #A": { + "aggregations": [ + "last" + ], + "operation": "aggregate" + }, + "Value #B": { + "aggregations": [ + "last" + ], + "operation": "aggregate" + }, + "Value #C": { + "aggregations": [ + "last" + ], + "operation": "aggregate" + }, + "Value #D": { + "aggregations": [ + "last" + ], + "operation": "aggregate" + }, + "Value #E": { + "aggregations": [ + "last" + ], + "operation": "aggregate" + }, + "function": { + "aggregations": [], + "operation": "groupby" + }, + "host": { + "aggregations": [] + }, + "name": { + "aggregations": [], + "operation": "groupby" + }, + "state": { + "aggregations": [], + "operation": "groupby" + } + } + } + }, + { + "id": "merge", + "options": {} + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": { + "Value #A (last)": 4, + "Value #B (last)": 3, + "Value #C (last)": 5, + "Value #D (last)": 6, + "Value #E (last)": 7, + "function": 1, + "name": 0, + "state": 2 + }, + "renameByName": { + "Value #A (last)": "End", + "Value #B (last)": "Start", + "Value #C (last)": "Scanned", + "Value #D (last)": "Errors", + "Value #E (last)": "Reconstructed", + "function": "Function", + "name": "Pool", + "state": "State" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false, + "minWidth": 50 + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Start" + }, + "properties": [ + { + "id": "unit", + "value": "dateTimeAsLocal" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Rate" + }, + "properties": [ + { + "id": "unit", + "value": "binBps" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "EST" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 62 + }, + "id": 6, + "interval": "$interval", + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "exemplar": false, + "expr": "zpool_scan_stats_start_ts{job=~'$job', state!~\"finished|canceled\"} * 1000", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "exemplar": false, + "expr": "zpool_scan_stats_remaining_t{state!~\"finished|canceled\", job=~\"$job\"}", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "exemplar": false, + "expr": "zpool_scan_stats_errors{job=~'$job', state!~\"finished|canceled\"}", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "exemplar": false, + "expr": "zpool_scan_stats_issued{job=~'$job', state!~\"finished|canceled\"}", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "exemplar": false, + "expr": "zpool_scan_stats_pass_issued{job=~'$job', state!~\"finished|canceled\"}", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "exemplar": false, + "expr": "zpool_scan_stats_processed{job=~'$job',state!~\"finished|canceled\"}", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "exemplar": false, + "expr": "zpool_scan_stats_to_process{job=~'$job',state!~\"finished|canceled\"}", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "G" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "exemplar": false, + "expr": "zpool_scan_stats_rate{job=~'$job',state!~\"finished|canceled\"}", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "H" + } + ], + "title": "Pools Scans In Progress", + "transformations": [ + { + "id": "groupBy", + "options": { + "fields": { + "Value #A": { + "aggregations": [ + "last" + ], + "operation": "aggregate" + }, + "Value #B": { + "aggregations": [ + "last" + ], + "operation": "aggregate" + }, + "Value #C": { + "aggregations": [ + "last" + ], + "operation": "aggregate" + }, + "Value #D": { + "aggregations": [ + "last" + ], + "operation": "aggregate" + }, + "Value #E": { + "aggregations": [ + "last" + ], + "operation": "aggregate" + }, + "Value #F": { + "aggregations": [ + "last" + ], + "operation": "aggregate" + }, + "Value #G": { + "aggregations": [ + "last" + ], + "operation": "aggregate" + }, + "Value #H": { + "aggregations": [ + "last" + ], + "operation": "aggregate" + }, + "Value #I": { + "aggregations": [ + "last" + ], + "operation": "aggregate" + }, + "__name__": { + "aggregations": [ + "last" + ] + }, + "db": { + "aggregations": [] + }, + "function": { + "aggregations": [], + "operation": "groupby" + }, + "name": { + "aggregations": [], + "operation": "groupby" + }, + "state": { + "aggregations": [], + "operation": "groupby" + }, + "zpool_scan_stats_pct_done{function=\"resilver\", job=\"unRaid\", name=\"hddmain\", state=\"scanning\"}": { + "aggregations": [], + "operation": "groupby" + } + } + } + }, + { + "id": "merge", + "options": {} + }, + { + "id": "calculateField", + "options": { + "alias": "Pending", + "binary": { + "left": "Value #B (last)", + "operator": "-", + "reducer": "sum", + "right": "Value #D (last)" + }, + "mode": "binary", + "reduce": { + "include": [ + "Total" + ], + "reducer": "sum" + }, + "replaceFields": false + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Value #G (last)": false + }, + "indexByName": { + "Pending": 9, + "Value #A (last)": 3, + "Value #B (last)": 4, + "Value #C (last)": 5, + "Value #D (last)": 6, + "Value #E (last)": 7, + "Value #F (last)": 8, + "Value #G (last)": 10, + "Value #H (last)": 11, + "function": 1, + "name": 0, + "state": 2 + }, + "renameByName": { + "Value #A (last)": "Start", + "Value #B (last)": "Total", + "Value #C (last)": "Errors", + "Value #D (last)": "Examined", + "Value #E (last)": "Examined Passed", + "Value #F (last)": "Proccessed", + "Value #G (last)": "To Proccess", + "Value #H (last)": "Rate", + "function": "Function", + "name": "Pool", + "state": "State" + } + } + }, + { + "id": "calculateField", + "options": { + "alias": "EST", + "binary": { + "left": "Pending", + "operator": "/", + "reducer": "sum", + "right": "Rate" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + } + ], + "type": "table" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 67 + }, + "id": 26, + "panels": [], + "repeat": "pool", + "repeatDirection": "h", + "title": "Pool Stats - $pool", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 0, + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": 60000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "line" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 68 + }, + "id": 12, + "interval": "$interval", + "options": { + "legend": { + "calcs": [ + "min", + "lastNotNull", + "diff" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.1.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "exemplar": true, + "expr": "zpool_stats_alloc{job='$job', vdev=\"root\", name=\"$pool\"}", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "Used", + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "exemplar": true, + "expr": "zpool_stats_size{job='$job', vdev=\"root\", name=\"$pool\"}", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "Total", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "exemplar": true, + "expr": "zpool_stats_free{job='$job', vdev=\"root\", name=\"$pool\"}", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "Free", + "refId": "B" + } + ], + "timeFrom": "24h", + "title": "Pool Space", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 5, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 120000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "unit", + "value": "iops" + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "C" + }, + "properties": [ + { + "id": "unit", + "value": "iops" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 13, + "x": 0, + "y": 73 + }, + "id": 16, + "interval": "$interval", + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "irate(zpool_stats_read_bytes{job='$job', name=\"$pool\",vdev=\"root\"}[$__rate_interval])", + "legendFormat": "{{name}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "irate(zpool_stats_read_ops{job='$job', name=\"$pool\", vdev=\"root\"}[$__rate_interval])", + "hide": false, + "legendFormat": "{{name}} IOPS", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "irate(zpool_stats_read_errors{job='$job', name=\"$pool\", vdev=\"root\"}[$__rate_interval])", + "hide": false, + "legendFormat": "{{name}} Errors", + "range": true, + "refId": "C" + } + ], + "title": "Pool Reads", + "type": "timeseries" + }, + { + "cards": {}, + "color": { + "cardColor": "#56A64B", + "colorScale": "sqrt", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 11, + "x": 13, + "y": 73 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 399, + "interval": "$interval", + "legend": { + "show": true + }, + "options": { + "calculate": false, + "calculation": {}, + "cellGap": 1, + "cellValues": {}, + "color": { + "exponent": 0.5, + "fill": "#56A64B", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Spectral", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "showValue": "never", + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "decimals": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.2.0", + "reverseYBuckets": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "exemplar": true, + "expr": "sum(increase(zpool_latency_total_read{name=\"$pool\", job=\"$job\"}[$__rate_interval])) by(le)", + "format": "heatmap", + "interval": "", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Total Reads Latency", + "tooltip": { + "show": true, + "showHistogram": true + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "yAxis": { + "decimals": 0, + "format": "s", + "logBase": 1, + "show": true + }, + "yBucketBound": "auto" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 5, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": 120000, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "unit", + "value": "iops" + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "C" + }, + "properties": [ + { + "id": "unit", + "value": "iops" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 13, + "x": 0, + "y": 83 + }, + "id": 323, + "interval": "$interval", + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "irate(zpool_stats_write_bytes{job='$job', name=\"$pool\",vdev=\"root\"}[$__rate_interval])", + "legendFormat": "{{name}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "irate(zpool_stats_write_ops{job='$job', name=\"$pool\", vdev=\"root\"}[$__rate_interval])", + "hide": false, + "legendFormat": "{{name}} IOPS", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "irate(zpool_stats_write_errors{job='$job', name=\"$pool\", vdev=\"root\"}[$__rate_interval])", + "hide": false, + "legendFormat": "{{name}} Errors", + "range": true, + "refId": "C" + } + ], + "title": "Pool Writes", + "type": "timeseries" + }, + { + "cards": {}, + "color": { + "cardColor": "#56A64B", + "colorScale": "sqrt", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 11, + "x": 13, + "y": 83 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 325, + "interval": "$interval", + "legend": { + "show": true + }, + "options": { + "calculate": false, + "calculation": {}, + "cellGap": 1, + "cellValues": {}, + "color": { + "exponent": 0.5, + "fill": "#56A64B", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Spectral", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "showValue": "never", + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "decimals": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.2.0", + "reverseYBuckets": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "exemplar": true, + "expr": "sum by(le) (increase(zpool_latency_total_write{name=\"$pool\", job=\"$job\"}[$__rate_interval]))", + "format": "heatmap", + "interval": "", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Total Writes Latency", + "tooltip": { + "show": true, + "showHistogram": true + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "yAxis": { + "decimals": 0, + "format": "s", + "logBase": 1, + "show": true + }, + "yBucketBound": "auto" + }, + { + "collapsed": false, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 145 + }, + "id": 113, + "panels": [], + "repeat": "pool", + "title": "IO Stats - $pool", + "type": "row" + }, + { + "cards": {}, + "color": { + "cardColor": "#56A64B", + "colorScale": "sqrt", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 146 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 361, + "interval": "$interval", + "legend": { + "show": true + }, + "options": { + "calculate": false, + "calculation": {}, + "cellGap": 1, + "cellValues": {}, + "color": { + "exponent": 0.5, + "fill": "#56A64B", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Spectral", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "showValue": "never", + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "decimals": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.2.0", + "reverseYBuckets": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "exemplar": true, + "expr": "sum (increase(zpool_latency_sync_read{job=\"$job\", name=\"$pool\"}[$__rate_interval])) by(le) ", + "format": "heatmap", + "interval": "", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Sync Reads Latency", + "tooltip": { + "show": true, + "showHistogram": true + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "yAxis": { + "decimals": 0, + "format": "s", + "logBase": 1, + "show": true + }, + "yBucketBound": "auto" + }, + { + "cards": {}, + "color": { + "cardColor": "#56A64B", + "colorScale": "sqrt", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 146 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 362, + "interval": "$interval", + "legend": { + "show": true + }, + "options": { + "calculate": false, + "calculation": {}, + "cellGap": 1, + "cellValues": {}, + "color": { + "exponent": 0.5, + "fill": "#56A64B", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Spectral", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "showValue": "never", + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "decimals": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.2.0", + "reverseYBuckets": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "exemplar": true, + "expr": "sum (increase(zpool_latency_sync_write{job='$job', name=\"$pool\"}[$__rate_interval])) by (le)", + "format": "heatmap", + "interval": "", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Sync Writes Latency", + "tooltip": { + "show": true, + "showHistogram": true + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "yAxis": { + "decimals": 0, + "format": "s", + "logBase": 1, + "show": true + }, + "yBucketBound": "auto" + }, + { + "cards": {}, + "color": { + "cardColor": "#56A64B", + "colorScale": "sqrt", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 156 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 363, + "interval": "$interval", + "legend": { + "show": true + }, + "options": { + "calculate": false, + "calculation": {}, + "cellGap": 1, + "cellValues": {}, + "color": { + "exponent": 0.5, + "fill": "#56A64B", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Spectral", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "showValue": "never", + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "decimals": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.2.0", + "reverseYBuckets": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "exemplar": true, + "expr": "sum(increase(zpool_latency_async_read{job='$job', name=\"$pool\"}[$__rate_interval])) by (le)", + "format": "heatmap", + "interval": "", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Async Reads Latency", + "tooltip": { + "show": true, + "showHistogram": true + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "yAxis": { + "decimals": 0, + "format": "s", + "logBase": 1, + "show": true + }, + "yBucketBound": "auto" + }, + { + "cards": {}, + "color": { + "cardColor": "#56A64B", + "colorScale": "sqrt", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 156 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 364, + "interval": "$interval", + "legend": { + "show": true + }, + "options": { + "calculate": false, + "calculation": {}, + "cellGap": 1, + "cellValues": {}, + "color": { + "exponent": 0.5, + "fill": "#56A64B", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Spectral", + "steps": 128 + }, + "exemplars": { + "color": "#ff00ffb3" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "showValue": "never", + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "decimals": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.2.0", + "reverseYBuckets": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "exemplar": true, + "expr": "sum(increase(zpool_latency_async_write{job='$job',name=\"$pool\"}[$__rate_interval])) by (le)", + "format": "heatmap", + "interval": "", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Async Writes Latency", + "tooltip": { + "show": true, + "showHistogram": true + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "yAxis": { + "decimals": 0, + "format": "s", + "logBase": 1, + "show": true + }, + "yBucketBound": "auto" + }, + { + "cards": {}, + "color": { + "cardColor": "#56A64B", + "colorScale": "sqrt", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 166 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 365, + "interval": "$interval", + "legend": { + "show": true + }, + "options": { + "calculate": false, + "calculation": {}, + "cellGap": 1, + "cellValues": {}, + "color": { + "exponent": 0.5, + "fill": "#56A64B", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Spectral", + "steps": 128 + }, + "exemplars": { + "color": "#ff00ff" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "showValue": "never", + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "decimals": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.2.0", + "reverseYBuckets": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "exemplar": true, + "expr": "sum(increase(zpool_latency_disk_read{job='$job', name=\"$pool\"}[$__rate_interval])) by (le)", + "format": "heatmap", + "interval": "", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Disk Reads Latency", + "tooltip": { + "show": true, + "showHistogram": true + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "yAxis": { + "decimals": 0, + "format": "s", + "logBase": 1, + "show": true + }, + "yBucketBound": "auto" + }, + { + "cards": {}, + "color": { + "cardColor": "#56A64B", + "colorScale": "sqrt", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 166 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 366, + "interval": "$interval", + "legend": { + "show": true + }, + "options": { + "calculate": false, + "calculation": {}, + "cellGap": 1, + "cellValues": {}, + "color": { + "exponent": 0.5, + "fill": "#56A64B", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Spectral", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "showValue": "never", + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "decimals": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.2.0", + "reverseYBuckets": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "exemplar": true, + "expr": "sum(increase(zpool_latency_disk_write{job='$job',name=\"$pool\"}[$__rate_interval])) by (le)", + "format": "heatmap", + "interval": "", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Disk Writes Latency", + "tooltip": { + "show": true, + "showHistogram": true + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "yAxis": { + "decimals": 0, + "format": "s", + "logBase": 1, + "show": true + }, + "yBucketBound": "auto" + }, + { + "cards": {}, + "color": { + "cardColor": "#56A64B", + "colorScale": "sqrt", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 176 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 367, + "interval": "$interval", + "legend": { + "show": true + }, + "options": { + "calculate": false, + "calculation": {}, + "cellGap": 1, + "cellValues": {}, + "color": { + "exponent": 0.5, + "fill": "#56A64B", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Spectral", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "showValue": "never", + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "decimals": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.2.0", + "reverseYBuckets": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "exemplar": true, + "expr": "sum(increase(zpool_latency_scrub{job='$job', name=\"$pool\"}[$__rate_interval])) by (le)", + "format": "heatmap", + "interval": "", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Scrub Latency", + "tooltip": { + "show": true, + "showHistogram": true + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "yAxis": { + "decimals": 0, + "format": "s", + "logBase": 1, + "show": true + }, + "yBucketBound": "auto" + }, + { + "cards": {}, + "color": { + "cardColor": "#56A64B", + "colorScale": "sqrt", + "colorScheme": "interpolateSpectral", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "tsbuckets", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 176 + }, + "heatmap": {}, + "hideZeroBuckets": true, + "highlightCards": true, + "id": 368, + "interval": "$interval", + "legend": { + "show": true + }, + "options": { + "calculate": false, + "calculation": {}, + "cellGap": 1, + "cellValues": {}, + "color": { + "exponent": 0.5, + "fill": "#56A64B", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Spectral", + "steps": 128 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "showValue": "never", + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": true + }, + "yAxis": { + "axisPlacement": "left", + "decimals": 0, + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "11.2.0", + "reverseYBuckets": false, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "exemplar": true, + "expr": "sum(increase(zpool_latency_trim{job='$job', name=\"$pool\"}[$__rate_interval])) by(le)", + "format": "heatmap", + "interval": "", + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Trim Latency", + "tooltip": { + "show": true, + "showHistogram": true + }, + "type": "heatmap", + "xAxis": { + "show": true + }, + "yAxis": { + "decimals": 0, + "format": "s", + "logBase": 1, + "show": true + }, + "yBucketBound": "auto" + } + ], + "refresh": "1m", + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "current": {}, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": "30s", + "value": "30s" + }, + "hide": 2, + "includeAll": false, + "multi": false, + "name": "interval", + "options": [ + { + "selected": true, + "text": "30s", + "value": "30s" + } + ], + "query": "30s", + "skipUrlSync": false, + "type": "custom" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "label_values(zfs_pool_health,pool)", + "hide": 0, + "includeAll": true, + "label": "Pool", + "multi": false, + "name": "pool", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(zfs_pool_health,pool)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(zfs_pool_health,instance)", + "hide": 0, + "includeAll": true, + "label": "Instance", + "multi": false, + "name": "instance", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(zfs_pool_health,instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "ZFS Details", + "uid": "-AweIwvnk", + "version": 5, + "weekStart": "" +} diff --git a/kubernetes/apps/observability/grafana/ks.yaml b/kubernetes/apps/observability/grafana/ks.yaml new file mode 100644 index 00000000..89fbfcf2 --- /dev/null +++ b/kubernetes/apps/observability/grafana/ks.yaml @@ -0,0 +1,20 @@ +--- +# yaml-language-server: $schema=https://ks.hsn.dev/kustomize.toolkit.fluxcd.io/kustomization_v1.json +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: &app grafana + namespace: flux-system +spec: + targetNamespace: observability + commonMetadata: + labels: + app.kubernetes.io/name: *app + path: ./kubernetes/apps/observability/grafana/app + prune: true + sourceRef: + kind: GitRepository + name: theshire + wait: false + interval: 30m + retryInterval: 1m diff --git a/kubernetes/apps/observability/kustomization.yaml b/kubernetes/apps/observability/kustomization.yaml index 963b4e7b..03c54148 100644 --- a/kubernetes/apps/observability/kustomization.yaml +++ b/kubernetes/apps/observability/kustomization.yaml @@ -8,6 +8,7 @@ resources: # Flux-Kustomizations - ./alertmanager/ks.yaml - ./gatus/ks.yaml + - ./grafana/ks.yaml - ./node-exporter/ks.yaml - ./prometheus-operator-crds/ks.yaml - ./unpoller/ks.yaml