From 2b112bdb5042a6cfd4446ee60e972607379a4c09 Mon Sep 17 00:00:00 2001
From: Joseph Hanson <joe@veri.dev>
Date: Fri, 16 Aug 2024 18:55:06 -0500
Subject: [PATCH] adding victoria metrics

---
 .../victoria-metrics/app/helmrelease.yaml     | 201 ++++++++++++++++++
 .../victoria-metrics}/app/kustomization.yaml  |   0
 .../observability/victoria-metrics/ks.yaml    |  26 +++
 .../apps/victoria-metrics/kustomization.yaml  |   9 -
 .../apps/victoria-metrics/namespace.yaml      |   8 -
 .../operator/app/helmrelease.yaml             |  28 ---
 .../operator/cluster/cluster.yaml             |  15 --
 .../operator/cluster/kustomization.yaml       |   6 -
 .../apps/victoria-metrics/operator/ks.yaml    |  52 -----
 .../repositories/helm/victoria-metrics.yaml   |   1 +
 10 files changed, 228 insertions(+), 118 deletions(-)
 create mode 100644 kubernetes/apps/observability/victoria-metrics/app/helmrelease.yaml
 rename kubernetes/apps/{victoria-metrics/operator => observability/victoria-metrics}/app/kustomization.yaml (100%)
 create mode 100644 kubernetes/apps/observability/victoria-metrics/ks.yaml
 delete mode 100644 kubernetes/apps/victoria-metrics/kustomization.yaml
 delete mode 100644 kubernetes/apps/victoria-metrics/namespace.yaml
 delete mode 100644 kubernetes/apps/victoria-metrics/operator/app/helmrelease.yaml
 delete mode 100644 kubernetes/apps/victoria-metrics/operator/cluster/cluster.yaml
 delete mode 100644 kubernetes/apps/victoria-metrics/operator/cluster/kustomization.yaml
 delete mode 100644 kubernetes/apps/victoria-metrics/operator/ks.yaml

diff --git a/kubernetes/apps/observability/victoria-metrics/app/helmrelease.yaml b/kubernetes/apps/observability/victoria-metrics/app/helmrelease.yaml
new file mode 100644
index 00000000..85e3e9bc
--- /dev/null
+++ b/kubernetes/apps/observability/victoria-metrics/app/helmrelease.yaml
@@ -0,0 +1,201 @@
+---
+# yaml-language-server: $schema=https://raw.githubusercontent.com/bjw-s/helm-charts/main/charts/other/app-template/schemas/helmrelease-helm-v2beta2.schema.json
+apiVersion: helm.toolkit.fluxcd.io/v2beta1
+kind: HelmRelease
+metadata:
+  name: victoria-metrics
+spec:
+  interval: 30m
+  chart:
+    spec:
+      chart: victoria-metrics-k8s-stack
+      version: 0.23.2
+      sourceRef:
+        kind: HelmRepository
+        name: victoria-metrics
+  values:
+    fullnameOverride: victoria-metrics
+
+    # VM Operator deployment
+    victoria-metrics-operator:
+      enabled: true
+      operator:
+        # disable_prometheus_converter: false # Ensure we keep enabled the converter to sync prom rules to VM rules
+        enable_converter_ownership: true # Required to allow VM to remove VM rules it imports if a prometheus rule is deleted
+
+    # Single-binary vm cluster
+    vmsingle:
+      enabled: true
+      spec:
+        extraArgs:
+          dedup.minScrapeInterval: 30s
+          maxLabelsPerTimeseries: "90"
+          search.minStalenessInterval: 5m
+          vmalert.proxyURL: http://vmalert-victoria-metrics.observability.svc.cluster.local:8080
+        retentionPeriod: 1y
+        storage:
+          storageClassName: "openebs-zfs"
+          resources:
+            requests:
+              storage: "50Gi"
+          accessModes:
+            - ReadWriteOnce
+      ingress:
+        enabled: true
+        ingressClassName: internal-nginx
+        hosts:
+          - vm.jahanson.tech
+
+    # VM Alerting (however, this just watches & passes alerts to alertmanager)
+    vmalert:
+      enabled: true
+      spec:
+        replicaCount: 1
+        resources:
+          requests:
+            cpu: 50m
+            memory: 128Mi
+          limits:
+            cpu: 150m
+            memory: 256Mi
+        topologySpreadConstraints:
+          - maxSkew: 1
+            topologyKey: kubernetes.io/hostname
+            whenUnsatisfiable: DoNotSchedule
+            labelSelector:
+              matchLabels:
+                app.kubernetes.io/name: vmalert
+        extraArgs:
+          external.url: https://vmalert.jahanson.tech
+        notifiers:
+          - url: http://alertmanager.observability.svc.cluster.local:9093
+      ingress:
+        enabled: true
+        ingressClassName: internal-nginx
+        hosts:
+          - vmalert.jahanson.tech
+
+    # VM Data scraping
+    vmagent:
+      enabled: true
+      spec:
+        replicaCount: 1
+        shardCount: 2
+        scrapeInterval: 30s
+        externalLabels:
+          cluster: main
+        resources:
+          requests:
+            cpu: 50m
+            memory: 256Mi
+          limits:
+            cpu: 400m
+            memory: 512Mi
+        topologySpreadConstraints:
+          - maxSkew: 1
+            topologyKey: kubernetes.io/hostname
+            whenUnsatisfiable: DoNotSchedule
+            labelSelector:
+              matchLabels:
+                app.kubernetes.io/name: vmagent
+        additionalScrapeConfigs:
+          name: vm-additional-scrape-configs
+          key: prometheus-additional.yaml
+      ingress:
+        enabled: true
+        ingressClassName: internal-nginx
+        hosts:
+          - vmagent.jahanson.tech
+
+    # Extra slack templates
+    monzoTemplate:
+      enabled: false
+
+    # Scrape configs
+    kubelet:
+      enabled: true
+      spec:
+        interval: 30s
+        # drop high cardinality label and useless metrics for cadvisor and kubelet
+        metricRelabelConfigs:
+          - action: labeldrop
+            regex: (uid|pod_uid|id)
+          - action: labeldrop
+            regex: (name)
+          - action: drop
+            source_labels: [__name__]
+            regex: (rest_client_request_duration_seconds_bucket|rest_client_request_duration_seconds_sum|rest_client_request_duration_seconds_count)
+          - action: drop
+            source_labels: [__name__]
+            regex: (container_tasks_state|container_memory_failures_total)
+          - action: drop
+            source_labels: [__name__]
+            regex: (container_blkio_device_usage_total)
+          - action: drop
+            source_labels: [__name__]
+            regex: (prober_probe_duration_seconds_bucket)
+        relabelConfigs:
+          - action: labelmap
+            regex: __meta_kubernetes_node_label_(.+)
+          - sourceLabels: [__metrics_path__]
+            targetLabel: metrics_path
+          - targetLabel: "job"
+            replacement: "kubelet"
+    # kubeApiServer:
+    #   enabled: false
+    # kubeControllerManager:
+    #   enabled: false
+    # kubeProxy:
+    #   enabled: false
+    # kubeScheduler:
+    #   enabled: false
+    # kubeEtcd:
+    #   enabled: false
+
+    # Enable deployment of kube-state-metrics
+    # kube-state-metrics:
+    #   enabled: false
+
+    # Enable deployment of prometheus-node-exporter
+    # prometheus-node-exporter:
+    #   enabled: false
+
+    # Enable deployment of grafana
+    # defaultDashboardsEnabled: false
+    # grafana:
+    #   enabled: false
+
+    # Enable deployment of alertmanager
+    # alertmanager:
+    #   enabled: false
+
+    # Prepared sets of default rules
+    # Adjust to what scraping functions you have enabled
+    # i.e. if you dont have kubeapisever setup & enabled, disable
+    #   the kubeApiserver rules below
+    defaultRules:
+      create: true
+      rules:
+        etcd: false
+        general: true
+        k8s: true
+        kubeApiserver: true
+        kubeApiserverAvailability: true
+        kubeApiserverBurnrate: true
+        kubeApiserverHistogram: true
+        kubeApiserverSlos: true
+        kubelet: true
+        kubePrometheusGeneral: true
+        kubePrometheusNodeRecording: true
+        kubernetesApps: true
+        kubernetesResources: true
+        kubernetesStorage: true
+        kubernetesSystem: true
+        kubeScheduler: false
+        kubeStateMetrics: true
+        network: true
+        node: true
+        vmagent: true
+        vmsingle: false
+        vmhealth: true
+        alertmanager: false
diff --git a/kubernetes/apps/victoria-metrics/operator/app/kustomization.yaml b/kubernetes/apps/observability/victoria-metrics/app/kustomization.yaml
similarity index 100%
rename from kubernetes/apps/victoria-metrics/operator/app/kustomization.yaml
rename to kubernetes/apps/observability/victoria-metrics/app/kustomization.yaml
diff --git a/kubernetes/apps/observability/victoria-metrics/ks.yaml b/kubernetes/apps/observability/victoria-metrics/ks.yaml
new file mode 100644
index 00000000..85590ef0
--- /dev/null
+++ b/kubernetes/apps/observability/victoria-metrics/ks.yaml
@@ -0,0 +1,26 @@
+---
+# yaml-language-server: $schema=https://ks.hsn.dev/kustomize.toolkit.fluxcd.io/kustomization_v1.json
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: &app victoria-metrics-stack
+  namespace: flux-system
+spec:
+  targetNamespace: observability
+  commonMetadata:
+    labels:
+      app.kubernetes.io/name: *app
+  dependsOn:
+    - name: external-secrets-stores
+  path: ./kubernetes/apps/observability/victoria-metrics/app
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: homelab
+  wait: false
+  interval: 30m
+  retryInterval: 1m
+  timeout: 5m
+  postBuild:
+    substitute:
+      APP: *app
diff --git a/kubernetes/apps/victoria-metrics/kustomization.yaml b/kubernetes/apps/victoria-metrics/kustomization.yaml
deleted file mode 100644
index ae1229da..00000000
--- a/kubernetes/apps/victoria-metrics/kustomization.yaml
+++ /dev/null
@@ -1,9 +0,0 @@
----
-# yaml-language-server: $schema=https://json.schemastore.org/kustomization
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-resources:
-  # Pre Flux-Kustomizations
-  - ./namespace.yaml
-  # Flux-Kustomizations
-  # - ./vm/ks.yaml
diff --git a/kubernetes/apps/victoria-metrics/namespace.yaml b/kubernetes/apps/victoria-metrics/namespace.yaml
deleted file mode 100644
index 42fe39d2..00000000
--- a/kubernetes/apps/victoria-metrics/namespace.yaml
+++ /dev/null
@@ -1,8 +0,0 @@
----
-apiVersion: v1
-kind: Namespace
-metadata:
-  name: victoria-metrics
-  labels:
-    kustomize.toolkit.fluxcd.io/prune: disabled
-    pgo-enabled-hsn.dev: "true"
diff --git a/kubernetes/apps/victoria-metrics/operator/app/helmrelease.yaml b/kubernetes/apps/victoria-metrics/operator/app/helmrelease.yaml
deleted file mode 100644
index a427d222..00000000
--- a/kubernetes/apps/victoria-metrics/operator/app/helmrelease.yaml
+++ /dev/null
@@ -1,28 +0,0 @@
----
-# yaml-language-server: $schema=https://raw.githubusercontent.com/bjw-s/helm-charts/main/charts/other/app-template/schemas/helmrelease-helm-v2beta2.schema.json
-apiVersion: helm.toolkit.fluxcd.io/v2
-kind: HelmRelease
-metadata:
-  name: victoria-metrics-operator
-  namespace: monitoring
-spec:
-  interval: 30m
-  chart:
-    spec:
-      chart: victoria-metrics-operator
-      version: 0.32.2
-      sourceRef:
-        kind: HelmRepository
-        name: victoria-metrics
-        namespace: flux-system
-  values:
-    fullnameOverride: victoria-metrics
-
-    # VM Operator deployment
-    resources:
-      limits:
-        cpu: 500m
-        memory: 500Mi
-      requests:
-        cpu: 100m
-        memory: 150Mi
diff --git a/kubernetes/apps/victoria-metrics/operator/cluster/cluster.yaml b/kubernetes/apps/victoria-metrics/operator/cluster/cluster.yaml
deleted file mode 100644
index 63ec60c5..00000000
--- a/kubernetes/apps/victoria-metrics/operator/cluster/cluster.yaml
+++ /dev/null
@@ -1,15 +0,0 @@
-# yaml-language-server: $schema=https://ks.hsn.dev/operator.victoriametrics.com/vmsingle_v1beta1.json
-apiVersion: operator.victoriametrics.com/v1beta1
-kind: VMSingle
-metadata:
-  name: vm-single
-  namespace: victoria-metrics
-spec:
-  retentionPeriod: "1y"
-  storage:
-    storageClassName: openebs-zfs
-    accessModes:
-      - ReadWriteOnce
-    resources:
-      requests:
-        storage: 50Gi
diff --git a/kubernetes/apps/victoria-metrics/operator/cluster/kustomization.yaml b/kubernetes/apps/victoria-metrics/operator/cluster/kustomization.yaml
deleted file mode 100644
index a2c786be..00000000
--- a/kubernetes/apps/victoria-metrics/operator/cluster/kustomization.yaml
+++ /dev/null
@@ -1,6 +0,0 @@
----
-# yaml-language-server: $schema=https://json.schemastore.org/kustomization
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-resources:
-  - ./cluster.yaml
diff --git a/kubernetes/apps/victoria-metrics/operator/ks.yaml b/kubernetes/apps/victoria-metrics/operator/ks.yaml
deleted file mode 100644
index 02ca7c9e..00000000
--- a/kubernetes/apps/victoria-metrics/operator/ks.yaml
+++ /dev/null
@@ -1,52 +0,0 @@
----
-# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/kustomize.toolkit.fluxcd.io/kustomization_v1.json
-apiVersion: kustomize.toolkit.fluxcd.io/v1
-kind: Kustomization
-metadata:
-  name: &app victoria-metrics-operator
-  namespace: flux-system
-spec:
-  targetNamespace: victoria-metrics
-  commonMetadata:
-    labels:
-      app.kubernetes.io/name: *app
-  dependsOn:
-    - name: external-secrets-stores
-  path: ./kubernetes/apps/victoria-metrics/operator/app
-  prune: true
-  sourceRef:
-    kind: GitRepository
-    name: homelab
-  wait: false
-  interval: 30m
-  retryInterval: 1m
-  timeout: 5m
-  postBuild:
-    substitute:
-      APP: *app
----
-# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/kustomize.toolkit.fluxcd.io/kustomization_v1.json
-apiVersion: kustomize.toolkit.fluxcd.io/v1
-kind: Kustomization
-metadata:
-  name: &app victoria-metrics-cluster
-  namespace: flux-system
-spec:
-  targetNamespace: victoria-metrics
-  commonMetadata:
-    labels:
-      app.kubernetes.io/name: *app
-  dependsOn:
-    - name: external-secrets-stores
-  path: ./kubernetes/apps/victoria-metrics/operator/cluster
-  prune: true
-  sourceRef:
-    kind: GitRepository
-    name: homelab
-  wait: false
-  interval: 30m
-  retryInterval: 1m
-  timeout: 5m
-  postBuild:
-    substitute:
-      APP: *app
diff --git a/kubernetes/flux/repositories/helm/victoria-metrics.yaml b/kubernetes/flux/repositories/helm/victoria-metrics.yaml
index 89bc7ed9..37ee6b00 100644
--- a/kubernetes/flux/repositories/helm/victoria-metrics.yaml
+++ b/kubernetes/flux/repositories/helm/victoria-metrics.yaml
@@ -4,6 +4,7 @@ apiVersion: source.toolkit.fluxcd.io/v1
 kind: HelmRepository
 metadata:
   name: victoria-metrics
+  namespace: flux-system
 spec:
   interval: 30m
   url: https://victoriametrics.github.io/helm-charts/