add alert manager

kubernetes/apps/observability/alertmanager/app/externalsecret.yaml

@@ -0,0 +1,23 @@
+---
+# yaml-language-server: $schema=https://ks.hsn.dev/external-secrets.io/externalsecret_v1beta1.json
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
+metadata:
+  name: alertmanager-secret
+spec:
+  refreshInterval: 5m
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: onepassword-connect
+  target:
+    name: alertmanager-secret
+    creationPolicy: Owner
+  data:
+    - secretKey: pushover_api_token
+      remoteRef:
+        key: Pushover
+        property: alertmanager_token
+    - secretKey: pushover_api_userkey
+      remoteRef:
+        key: Pushover
+        property: userkey_jahanson

kubernetes/apps/observability/alertmanager/app/helmrelease.yaml

@@ -0,0 +1,87 @@
+---
+# yaml-language-server: $schema=https://raw.githubusercontent.com/bjw-s/helm-charts/main/charts/other/app-template/schemas/helmrelease-helm-v2.schema.json
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: alertmanager
+spec:
+  interval: 30m
+  chart:
+    spec:
+      chart: app-template
+      version: 3.5.0
+      interval: 30m
+      sourceRef:
+        kind: HelmRepository
+        name: bjw-s
+        namespace: flux-system
+  values:
+    controllers:
+      alertmanager:
+        type: statefulset
+        annotations:
+          reloader.stakater.com/auto: "true"
+
+        statefulset:
+          volumeClaimTemplates:
+            - name: storage
+              accessMode: ReadWriteOnce
+              size: 50Mi
+              storageClass: ceph-block
+              globalMounts:
+                - path: /alertmanager
+
+        containers:
+          alertmanager:
+            image:
+              repository: quay.io/prometheus/alertmanager
+              tag: v0.27.0
+            ports:
+              - name: http
+                containerPort: 9093
+            probes:
+              liveness:
+                enabled: true
+              readiness:
+                enabled: true
+              startup:
+                enabled: true
+                spec:
+                  failureThreshold: 30
+                  periodSeconds: 5
+            resources:
+              requests:
+                cpu: 11m
+                memory: 50M
+              limits:
+                memory: 99M
+
+    service:
+      app:
+        controller: alertmanager
+        ports:
+          http:
+            port: 9093
+
+    ingress:
+      app:
+        className: internal-nginx
+        hosts:
+          - host: alertmanager.jahanson.tech
+            paths:
+              - path: /
+                service:
+                  identifier: app
+                  port: http
+
+    persistence:
+      config:
+        type: configMap
+        name: alertmanager-configmap
+        globalMounts:
+          - path: /etc/alertmanager
+      secrets:
+        type: secret
+        name: alertmanager-secret
+        globalMounts:
+          - path: /etc/secrets

kubernetes/apps/observability/alertmanager/app/kustomization.yaml

@@ -0,0 +1,15 @@
+---
+# yaml-language-server: $schema=https://json.schemastore.org/kustomization
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ./externalsecret.yaml
+  - ./helmrelease.yaml
+configMapGenerator:
+  - name: alertmanager-configmap
+    files:
+      - resources/alertmanager.yml
+generatorOptions:
+  disableNameSuffixHash: true
+  annotations:
+    kustomize.toolkit.fluxcd.io/substitute: disabled

kubernetes/apps/observability/alertmanager/app/resources/alertmanager.yml

@@ -0,0 +1,59 @@
+---
+receivers:
+  - name: "null"
+  - name: "pushover"
+    pushover_configs:
+      - html: true
+        token_file: /etc/secrets/pushover_api_token
+        user_key_file: /etc/secrets/pushover_api_userkey
+        send_resolved: true
+        priority: |-
+          {{ if eq .Status "firing" }}1{{ else }}0{{ end }}
+        url_title: View in Alert Manager
+        title: |-
+          [{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .CommonLabels.alertname }}
+        message: |-
+          {{- range .Alerts }}
+            {{- if ne .Labels.severity "" }}
+              <b>Severity:</b> <i>{{ .Labels.severity }}</i>
+            {{- else }}
+              <b>Severity:</b> <i>N/A</i>
+            {{- end }}
+            {{- if ne .Annotations.description "" }}
+              <b>Description:</b> <i>{{ .Annotations.description }}</i>
+            {{- else if ne .Annotations.summary "" }}
+              <b>Summary:</b> <i>{{ .Annotations.summary }}</i>
+            {{- else if ne .Annotations.message "" }}
+              <b>Message:</b> <i>{{ .Annotations.message }}</i>
+            {{- else }}
+              <b>Description:</b> <i>N/A</i>
+            {{- end }}
+            {{- if gt (len .Labels.SortedPairs) 0 }}
+              <b>Details:</b>
+              {{- range .Labels.SortedPairs }}
+                • <b>{{ .Name }}:</b> <i>{{ .Value }}</i>
+              {{- end }}
+            {{- end }}
+          {{- end }}
+
+route:
+  group_by: ["alertname", "job"]
+  group_wait: 30s
+  group_interval: 5m
+  repeat_interval: 6h
+  receiver: "pushover"
+  routes:
+    - receiver: "null"
+      matchers:
+        - alertname =~ "InfoInhibitor|Watchdog"
+    - receiver: "pushover"
+      matchers:
+        - severity = critical
+      continue: true
+
+inhibit_rules:
+  - source_matchers:
+      - severity = "critical"
+    target_matchers:
+      - severity = "warning"
+    equal: ["alertname", "namespace"]

kubernetes/apps/observability/alertmanager/ks.yaml

@@ -0,0 +1,23 @@
+---
+# yaml-language-server: $schema=https://raw.githubusercontent.com/fluxcd-community/flux2-schemas/main/kustomization-kustomize-v1.json
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: &appname alertmanager
+  namespace: flux-system
+spec:
+  targetNamespace: observability
+  commonMetadata:
+    labels:
+      app.kubernetes.io/name: *appname
+  interval: 30m
+  timeout: 5m
+  path: "./kubernetes/apps/observability/alertmanager/app"
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: theshire
+  wait: true
+  dependsOn:
+    - name: external-secrets-stores
+    - name: rook-ceph-cluster

kubernetes/apps/observability/kustomization.yaml

@@ -6,6 +6,7 @@ resources:
   # Pre Flux-Kustomizations
   - ./namespace.yaml
   # Flux-Kustomizations
+  - ./alertmanager/ks.yaml
   - ./gatus/ks.yaml
   - ./prometheus-operator-crds/ks.yaml
   - ./victoria-metrics/ks.yaml