From 34e5ef12a47c9d520a5c081e92d32ef339e22955 Mon Sep 17 00:00:00 2001 From: Joseph Hanson Date: Sat, 5 Oct 2024 11:48:28 -0500 Subject: [PATCH] add alert manager --- .../alertmanager/app/externalsecret.yaml | 23 +++++ .../alertmanager/app/helmrelease.yaml | 87 +++++++++++++++++++ .../alertmanager/app/kustomization.yaml | 15 ++++ .../app/resources/alertmanager.yml | 59 +++++++++++++ .../apps/observability/alertmanager/ks.yaml | 23 +++++ .../apps/observability/kustomization.yaml | 1 + 6 files changed, 208 insertions(+) create mode 100644 kubernetes/apps/observability/alertmanager/app/externalsecret.yaml create mode 100644 kubernetes/apps/observability/alertmanager/app/helmrelease.yaml create mode 100644 kubernetes/apps/observability/alertmanager/app/kustomization.yaml create mode 100644 kubernetes/apps/observability/alertmanager/app/resources/alertmanager.yml create mode 100644 kubernetes/apps/observability/alertmanager/ks.yaml diff --git a/kubernetes/apps/observability/alertmanager/app/externalsecret.yaml b/kubernetes/apps/observability/alertmanager/app/externalsecret.yaml new file mode 100644 index 00000000..f74d44d3 --- /dev/null +++ b/kubernetes/apps/observability/alertmanager/app/externalsecret.yaml @@ -0,0 +1,23 @@ +--- +# yaml-language-server: $schema=https://ks.hsn.dev/external-secrets.io/externalsecret_v1beta1.json +apiVersion: external-secrets.io/v1beta1 +kind: ExternalSecret +metadata: + name: alertmanager-secret +spec: + refreshInterval: 5m + secretStoreRef: + kind: ClusterSecretStore + name: onepassword-connect + target: + name: alertmanager-secret + creationPolicy: Owner + data: + - secretKey: pushover_api_token + remoteRef: + key: Pushover + property: alertmanager_token + - secretKey: pushover_api_userkey + remoteRef: + key: Pushover + property: userkey_jahanson diff --git a/kubernetes/apps/observability/alertmanager/app/helmrelease.yaml b/kubernetes/apps/observability/alertmanager/app/helmrelease.yaml new file mode 100644 index 00000000..9671dfb1 --- /dev/null +++ b/kubernetes/apps/observability/alertmanager/app/helmrelease.yaml @@ -0,0 +1,87 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/bjw-s/helm-charts/main/charts/other/app-template/schemas/helmrelease-helm-v2.schema.json +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: alertmanager +spec: + interval: 30m + chart: + spec: + chart: app-template + version: 3.5.0 + interval: 30m + sourceRef: + kind: HelmRepository + name: bjw-s + namespace: flux-system + values: + controllers: + alertmanager: + type: statefulset + annotations: + reloader.stakater.com/auto: "true" + + statefulset: + volumeClaimTemplates: + - name: storage + accessMode: ReadWriteOnce + size: 50Mi + storageClass: ceph-block + globalMounts: + - path: /alertmanager + + containers: + alertmanager: + image: + repository: quay.io/prometheus/alertmanager + tag: v0.27.0 + ports: + - name: http + containerPort: 9093 + probes: + liveness: + enabled: true + readiness: + enabled: true + startup: + enabled: true + spec: + failureThreshold: 30 + periodSeconds: 5 + resources: + requests: + cpu: 11m + memory: 50M + limits: + memory: 99M + + service: + app: + controller: alertmanager + ports: + http: + port: 9093 + + ingress: + app: + className: internal-nginx + hosts: + - host: alertmanager.jahanson.tech + paths: + - path: / + service: + identifier: app + port: http + + persistence: + config: + type: configMap + name: alertmanager-configmap + globalMounts: + - path: /etc/alertmanager + secrets: + type: secret + name: alertmanager-secret + globalMounts: + - path: /etc/secrets diff --git a/kubernetes/apps/observability/alertmanager/app/kustomization.yaml b/kubernetes/apps/observability/alertmanager/app/kustomization.yaml new file mode 100644 index 00000000..11ae520b --- /dev/null +++ b/kubernetes/apps/observability/alertmanager/app/kustomization.yaml @@ -0,0 +1,15 @@ +--- +# yaml-language-server: $schema=https://json.schemastore.org/kustomization +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./externalsecret.yaml + - ./helmrelease.yaml +configMapGenerator: + - name: alertmanager-configmap + files: + - resources/alertmanager.yml +generatorOptions: + disableNameSuffixHash: true + annotations: + kustomize.toolkit.fluxcd.io/substitute: disabled diff --git a/kubernetes/apps/observability/alertmanager/app/resources/alertmanager.yml b/kubernetes/apps/observability/alertmanager/app/resources/alertmanager.yml new file mode 100644 index 00000000..f929a4e3 --- /dev/null +++ b/kubernetes/apps/observability/alertmanager/app/resources/alertmanager.yml @@ -0,0 +1,59 @@ +--- +receivers: + - name: "null" + - name: "pushover" + pushover_configs: + - html: true + token_file: /etc/secrets/pushover_api_token + user_key_file: /etc/secrets/pushover_api_userkey + send_resolved: true + priority: |- + {{ if eq .Status "firing" }}1{{ else }}0{{ end }} + url_title: View in Alert Manager + title: |- + [{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .CommonLabels.alertname }} + message: |- + {{- range .Alerts }} + {{- if ne .Labels.severity "" }} + Severity: {{ .Labels.severity }} + {{- else }} + Severity: N/A + {{- end }} + {{- if ne .Annotations.description "" }} + Description: {{ .Annotations.description }} + {{- else if ne .Annotations.summary "" }} + Summary: {{ .Annotations.summary }} + {{- else if ne .Annotations.message "" }} + Message: {{ .Annotations.message }} + {{- else }} + Description: N/A + {{- end }} + {{- if gt (len .Labels.SortedPairs) 0 }} + Details: + {{- range .Labels.SortedPairs }} + • {{ .Name }}: {{ .Value }} + {{- end }} + {{- end }} + {{- end }} + +route: + group_by: ["alertname", "job"] + group_wait: 30s + group_interval: 5m + repeat_interval: 6h + receiver: "pushover" + routes: + - receiver: "null" + matchers: + - alertname =~ "InfoInhibitor|Watchdog" + - receiver: "pushover" + matchers: + - severity = critical + continue: true + +inhibit_rules: + - source_matchers: + - severity = "critical" + target_matchers: + - severity = "warning" + equal: ["alertname", "namespace"] diff --git a/kubernetes/apps/observability/alertmanager/ks.yaml b/kubernetes/apps/observability/alertmanager/ks.yaml new file mode 100644 index 00000000..1279419f --- /dev/null +++ b/kubernetes/apps/observability/alertmanager/ks.yaml @@ -0,0 +1,23 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/fluxcd-community/flux2-schemas/main/kustomization-kustomize-v1.json +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: &appname alertmanager + namespace: flux-system +spec: + targetNamespace: observability + commonMetadata: + labels: + app.kubernetes.io/name: *appname + interval: 30m + timeout: 5m + path: "./kubernetes/apps/observability/alertmanager/app" + prune: true + sourceRef: + kind: GitRepository + name: theshire + wait: true + dependsOn: + - name: external-secrets-stores + - name: rook-ceph-cluster diff --git a/kubernetes/apps/observability/kustomization.yaml b/kubernetes/apps/observability/kustomization.yaml index 618b7490..ba427dae 100644 --- a/kubernetes/apps/observability/kustomization.yaml +++ b/kubernetes/apps/observability/kustomization.yaml @@ -6,6 +6,7 @@ resources: # Pre Flux-Kustomizations - ./namespace.yaml # Flux-Kustomizations + - ./alertmanager/ks.yaml - ./gatus/ks.yaml - ./prometheus-operator-crds/ks.yaml - ./victoria-metrics/ks.yaml