adding victoria metrics

This commit is contained in:
Joseph Hanson 2024-08-16 18:55:06 -05:00
parent 889b03771a
commit 2b112bdb50
Signed by: jahanson
SSH key fingerprint: SHA256:vy6dKBECV522aPAwklFM3ReKAVB086rT3oWwiuiFG7o
10 changed files with 228 additions and 118 deletions

View file

@ -0,0 +1,201 @@
---
# yaml-language-server: $schema=https://raw.githubusercontent.com/bjw-s/helm-charts/main/charts/other/app-template/schemas/helmrelease-helm-v2beta2.schema.json
apiVersion: helm.toolkit.fluxcd.io/v2beta1
kind: HelmRelease
metadata:
name: victoria-metrics
spec:
interval: 30m
chart:
spec:
chart: victoria-metrics-k8s-stack
version: 0.23.2
sourceRef:
kind: HelmRepository
name: victoria-metrics
values:
fullnameOverride: victoria-metrics
# VM Operator deployment
victoria-metrics-operator:
enabled: true
operator:
# disable_prometheus_converter: false # Ensure we keep enabled the converter to sync prom rules to VM rules
enable_converter_ownership: true # Required to allow VM to remove VM rules it imports if a prometheus rule is deleted
# Single-binary vm cluster
vmsingle:
enabled: true
spec:
extraArgs:
dedup.minScrapeInterval: 30s
maxLabelsPerTimeseries: "90"
search.minStalenessInterval: 5m
vmalert.proxyURL: http://vmalert-victoria-metrics.observability.svc.cluster.local:8080
retentionPeriod: 1y
storage:
storageClassName: "openebs-zfs"
resources:
requests:
storage: "50Gi"
accessModes:
- ReadWriteOnce
ingress:
enabled: true
ingressClassName: internal-nginx
hosts:
- vm.jahanson.tech
# VM Alerting (however, this just watches & passes alerts to alertmanager)
vmalert:
enabled: true
spec:
replicaCount: 1
resources:
requests:
cpu: 50m
memory: 128Mi
limits:
cpu: 150m
memory: 256Mi
topologySpreadConstraints:
- maxSkew: 1
topologyKey: kubernetes.io/hostname
whenUnsatisfiable: DoNotSchedule
labelSelector:
matchLabels:
app.kubernetes.io/name: vmalert
extraArgs:
external.url: https://vmalert.jahanson.tech
notifiers:
- url: http://alertmanager.observability.svc.cluster.local:9093
ingress:
enabled: true
ingressClassName: internal-nginx
hosts:
- vmalert.jahanson.tech
# VM Data scraping
vmagent:
enabled: true
spec:
replicaCount: 1
shardCount: 2
scrapeInterval: 30s
externalLabels:
cluster: main
resources:
requests:
cpu: 50m
memory: 256Mi
limits:
cpu: 400m
memory: 512Mi
topologySpreadConstraints:
- maxSkew: 1
topologyKey: kubernetes.io/hostname
whenUnsatisfiable: DoNotSchedule
labelSelector:
matchLabels:
app.kubernetes.io/name: vmagent
additionalScrapeConfigs:
name: vm-additional-scrape-configs
key: prometheus-additional.yaml
ingress:
enabled: true
ingressClassName: internal-nginx
hosts:
- vmagent.jahanson.tech
# Extra slack templates
monzoTemplate:
enabled: false
# Scrape configs
kubelet:
enabled: true
spec:
interval: 30s
# drop high cardinality label and useless metrics for cadvisor and kubelet
metricRelabelConfigs:
- action: labeldrop
regex: (uid|pod_uid|id)
- action: labeldrop
regex: (name)
- action: drop
source_labels: [__name__]
regex: (rest_client_request_duration_seconds_bucket|rest_client_request_duration_seconds_sum|rest_client_request_duration_seconds_count)
- action: drop
source_labels: [__name__]
regex: (container_tasks_state|container_memory_failures_total)
- action: drop
source_labels: [__name__]
regex: (container_blkio_device_usage_total)
- action: drop
source_labels: [__name__]
regex: (prober_probe_duration_seconds_bucket)
relabelConfigs:
- action: labelmap
regex: __meta_kubernetes_node_label_(.+)
- sourceLabels: [__metrics_path__]
targetLabel: metrics_path
- targetLabel: "job"
replacement: "kubelet"
# kubeApiServer:
# enabled: false
# kubeControllerManager:
# enabled: false
# kubeProxy:
# enabled: false
# kubeScheduler:
# enabled: false
# kubeEtcd:
# enabled: false
# Enable deployment of kube-state-metrics
# kube-state-metrics:
# enabled: false
# Enable deployment of prometheus-node-exporter
# prometheus-node-exporter:
# enabled: false
# Enable deployment of grafana
# defaultDashboardsEnabled: false
# grafana:
# enabled: false
# Enable deployment of alertmanager
# alertmanager:
# enabled: false
# Prepared sets of default rules
# Adjust to what scraping functions you have enabled
# i.e. if you dont have kubeapisever setup & enabled, disable
# the kubeApiserver rules below
defaultRules:
create: true
rules:
etcd: false
general: true
k8s: true
kubeApiserver: true
kubeApiserverAvailability: true
kubeApiserverBurnrate: true
kubeApiserverHistogram: true
kubeApiserverSlos: true
kubelet: true
kubePrometheusGeneral: true
kubePrometheusNodeRecording: true
kubernetesApps: true
kubernetesResources: true
kubernetesStorage: true
kubernetesSystem: true
kubeScheduler: false
kubeStateMetrics: true
network: true
node: true
vmagent: true
vmsingle: false
vmhealth: true
alertmanager: false

View file

@ -0,0 +1,26 @@
---
# yaml-language-server: $schema=https://ks.hsn.dev/kustomize.toolkit.fluxcd.io/kustomization_v1.json
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: &app victoria-metrics-stack
namespace: flux-system
spec:
targetNamespace: observability
commonMetadata:
labels:
app.kubernetes.io/name: *app
dependsOn:
- name: external-secrets-stores
path: ./kubernetes/apps/observability/victoria-metrics/app
prune: true
sourceRef:
kind: GitRepository
name: homelab
wait: false
interval: 30m
retryInterval: 1m
timeout: 5m
postBuild:
substitute:
APP: *app

View file

@ -1,9 +0,0 @@
---
# yaml-language-server: $schema=https://json.schemastore.org/kustomization
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
# Pre Flux-Kustomizations
- ./namespace.yaml
# Flux-Kustomizations
# - ./vm/ks.yaml

View file

@ -1,8 +0,0 @@
---
apiVersion: v1
kind: Namespace
metadata:
name: victoria-metrics
labels:
kustomize.toolkit.fluxcd.io/prune: disabled
pgo-enabled-hsn.dev: "true"

View file

@ -1,28 +0,0 @@
---
# yaml-language-server: $schema=https://raw.githubusercontent.com/bjw-s/helm-charts/main/charts/other/app-template/schemas/helmrelease-helm-v2beta2.schema.json
apiVersion: helm.toolkit.fluxcd.io/v2
kind: HelmRelease
metadata:
name: victoria-metrics-operator
namespace: monitoring
spec:
interval: 30m
chart:
spec:
chart: victoria-metrics-operator
version: 0.32.2
sourceRef:
kind: HelmRepository
name: victoria-metrics
namespace: flux-system
values:
fullnameOverride: victoria-metrics
# VM Operator deployment
resources:
limits:
cpu: 500m
memory: 500Mi
requests:
cpu: 100m
memory: 150Mi

View file

@ -1,15 +0,0 @@
# yaml-language-server: $schema=https://ks.hsn.dev/operator.victoriametrics.com/vmsingle_v1beta1.json
apiVersion: operator.victoriametrics.com/v1beta1
kind: VMSingle
metadata:
name: vm-single
namespace: victoria-metrics
spec:
retentionPeriod: "1y"
storage:
storageClassName: openebs-zfs
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 50Gi

View file

@ -1,6 +0,0 @@
---
# yaml-language-server: $schema=https://json.schemastore.org/kustomization
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- ./cluster.yaml

View file

@ -1,52 +0,0 @@
---
# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/kustomize.toolkit.fluxcd.io/kustomization_v1.json
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: &app victoria-metrics-operator
namespace: flux-system
spec:
targetNamespace: victoria-metrics
commonMetadata:
labels:
app.kubernetes.io/name: *app
dependsOn:
- name: external-secrets-stores
path: ./kubernetes/apps/victoria-metrics/operator/app
prune: true
sourceRef:
kind: GitRepository
name: homelab
wait: false
interval: 30m
retryInterval: 1m
timeout: 5m
postBuild:
substitute:
APP: *app
---
# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/kustomize.toolkit.fluxcd.io/kustomization_v1.json
apiVersion: kustomize.toolkit.fluxcd.io/v1
kind: Kustomization
metadata:
name: &app victoria-metrics-cluster
namespace: flux-system
spec:
targetNamespace: victoria-metrics
commonMetadata:
labels:
app.kubernetes.io/name: *app
dependsOn:
- name: external-secrets-stores
path: ./kubernetes/apps/victoria-metrics/operator/cluster
prune: true
sourceRef:
kind: GitRepository
name: homelab
wait: false
interval: 30m
retryInterval: 1m
timeout: 5m
postBuild:
substitute:
APP: *app

View file

@ -4,6 +4,7 @@ apiVersion: source.toolkit.fluxcd.io/v1
kind: HelmRepository
metadata:
name: victoria-metrics
namespace: flux-system
spec:
interval: 30m
url: https://victoriametrics.github.io/helm-charts/