add alert manager
This commit is contained in:
parent
b8d4f143b9
commit
34e5ef12a4
SSH key fingerprint:
SHA256:vy6dKBECV522aPAwklFM3ReKAVB086rT3oWwiuiFG7o
6 changed files with 208 additions and 0 deletions
|
|
@ -0,0 +1,23 @@
|
|||
---
|
||||
# yaml-language-server: $schema=https://ks.hsn.dev/external-secrets.io/externalsecret_v1beta1.json
|
||||
apiVersion: external-secrets.io/v1beta1
|
||||
kind: ExternalSecret
|
||||
metadata:
|
||||
name: alertmanager-secret
|
||||
spec:
|
||||
refreshInterval: 5m
|
||||
secretStoreRef:
|
||||
kind: ClusterSecretStore
|
||||
name: onepassword-connect
|
||||
target:
|
||||
name: alertmanager-secret
|
||||
creationPolicy: Owner
|
||||
data:
|
||||
- secretKey: pushover_api_token
|
||||
remoteRef:
|
||||
key: Pushover
|
||||
property: alertmanager_token
|
||||
- secretKey: pushover_api_userkey
|
||||
remoteRef:
|
||||
key: Pushover
|
||||
property: userkey_jahanson
|
||||
|
|
@ -0,0 +1,87 @@
|
|||
---
|
||||
# yaml-language-server: $schema=https://raw.githubusercontent.com/bjw-s/helm-charts/main/charts/other/app-template/schemas/helmrelease-helm-v2.schema.json
|
||||
apiVersion: helm.toolkit.fluxcd.io/v2
|
||||
kind: HelmRelease
|
||||
metadata:
|
||||
name: alertmanager
|
||||
spec:
|
||||
interval: 30m
|
||||
chart:
|
||||
spec:
|
||||
chart: app-template
|
||||
version: 3.5.0
|
||||
interval: 30m
|
||||
sourceRef:
|
||||
kind: HelmRepository
|
||||
name: bjw-s
|
||||
namespace: flux-system
|
||||
values:
|
||||
controllers:
|
||||
alertmanager:
|
||||
type: statefulset
|
||||
annotations:
|
||||
reloader.stakater.com/auto: "true"
|
||||
|
||||
statefulset:
|
||||
volumeClaimTemplates:
|
||||
- name: storage
|
||||
accessMode: ReadWriteOnce
|
||||
size: 50Mi
|
||||
storageClass: ceph-block
|
||||
globalMounts:
|
||||
- path: /alertmanager
|
||||
|
||||
containers:
|
||||
alertmanager:
|
||||
image:
|
||||
repository: quay.io/prometheus/alertmanager
|
||||
tag: v0.27.0
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 9093
|
||||
probes:
|
||||
liveness:
|
||||
enabled: true
|
||||
readiness:
|
||||
enabled: true
|
||||
startup:
|
||||
enabled: true
|
||||
spec:
|
||||
failureThreshold: 30
|
||||
periodSeconds: 5
|
||||
resources:
|
||||
requests:
|
||||
cpu: 11m
|
||||
memory: 50M
|
||||
limits:
|
||||
memory: 99M
|
||||
|
||||
service:
|
||||
app:
|
||||
controller: alertmanager
|
||||
ports:
|
||||
http:
|
||||
port: 9093
|
||||
|
||||
ingress:
|
||||
app:
|
||||
className: internal-nginx
|
||||
hosts:
|
||||
- host: alertmanager.jahanson.tech
|
||||
paths:
|
||||
- path: /
|
||||
service:
|
||||
identifier: app
|
||||
port: http
|
||||
|
||||
persistence:
|
||||
config:
|
||||
type: configMap
|
||||
name: alertmanager-configmap
|
||||
globalMounts:
|
||||
- path: /etc/alertmanager
|
||||
secrets:
|
||||
type: secret
|
||||
name: alertmanager-secret
|
||||
globalMounts:
|
||||
- path: /etc/secrets
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
---
|
||||
# yaml-language-server: $schema=https://json.schemastore.org/kustomization
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- ./externalsecret.yaml
|
||||
- ./helmrelease.yaml
|
||||
configMapGenerator:
|
||||
- name: alertmanager-configmap
|
||||
files:
|
||||
- resources/alertmanager.yml
|
||||
generatorOptions:
|
||||
disableNameSuffixHash: true
|
||||
annotations:
|
||||
kustomize.toolkit.fluxcd.io/substitute: disabled
|
||||
|
|
@ -0,0 +1,59 @@
|
|||
---
|
||||
receivers:
|
||||
- name: "null"
|
||||
- name: "pushover"
|
||||
pushover_configs:
|
||||
- html: true
|
||||
token_file: /etc/secrets/pushover_api_token
|
||||
user_key_file: /etc/secrets/pushover_api_userkey
|
||||
send_resolved: true
|
||||
priority: |-
|
||||
{{ if eq .Status "firing" }}1{{ else }}0{{ end }}
|
||||
url_title: View in Alert Manager
|
||||
title: |-
|
||||
[{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .CommonLabels.alertname }}
|
||||
message: |-
|
||||
{{- range .Alerts }}
|
||||
{{- if ne .Labels.severity "" }}
|
||||
<b>Severity:</b> <i>{{ .Labels.severity }}</i>
|
||||
{{- else }}
|
||||
<b>Severity:</b> <i>N/A</i>
|
||||
{{- end }}
|
||||
{{- if ne .Annotations.description "" }}
|
||||
<b>Description:</b> <i>{{ .Annotations.description }}</i>
|
||||
{{- else if ne .Annotations.summary "" }}
|
||||
<b>Summary:</b> <i>{{ .Annotations.summary }}</i>
|
||||
{{- else if ne .Annotations.message "" }}
|
||||
<b>Message:</b> <i>{{ .Annotations.message }}</i>
|
||||
{{- else }}
|
||||
<b>Description:</b> <i>N/A</i>
|
||||
{{- end }}
|
||||
{{- if gt (len .Labels.SortedPairs) 0 }}
|
||||
<b>Details:</b>
|
||||
{{- range .Labels.SortedPairs }}
|
||||
• <b>{{ .Name }}:</b> <i>{{ .Value }}</i>
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
route:
|
||||
group_by: ["alertname", "job"]
|
||||
group_wait: 30s
|
||||
group_interval: 5m
|
||||
repeat_interval: 6h
|
||||
receiver: "pushover"
|
||||
routes:
|
||||
- receiver: "null"
|
||||
matchers:
|
||||
- alertname =~ "InfoInhibitor|Watchdog"
|
||||
- receiver: "pushover"
|
||||
matchers:
|
||||
- severity = critical
|
||||
continue: true
|
||||
|
||||
inhibit_rules:
|
||||
- source_matchers:
|
||||
- severity = "critical"
|
||||
target_matchers:
|
||||
- severity = "warning"
|
||||
equal: ["alertname", "namespace"]
|
||||
23
kubernetes/apps/observability/alertmanager/ks.yaml
Normal file
23
kubernetes/apps/observability/alertmanager/ks.yaml
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
---
|
||||
# yaml-language-server: $schema=https://raw.githubusercontent.com/fluxcd-community/flux2-schemas/main/kustomization-kustomize-v1.json
|
||||
apiVersion: kustomize.toolkit.fluxcd.io/v1
|
||||
kind: Kustomization
|
||||
metadata:
|
||||
name: &appname alertmanager
|
||||
namespace: flux-system
|
||||
spec:
|
||||
targetNamespace: observability
|
||||
commonMetadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: *appname
|
||||
interval: 30m
|
||||
timeout: 5m
|
||||
path: "./kubernetes/apps/observability/alertmanager/app"
|
||||
prune: true
|
||||
sourceRef:
|
||||
kind: GitRepository
|
||||
name: theshire
|
||||
wait: true
|
||||
dependsOn:
|
||||
- name: external-secrets-stores
|
||||
- name: rook-ceph-cluster
|
||||
|
|
@ -6,6 +6,7 @@ resources:
|
|||
# Pre Flux-Kustomizations
|
||||
- ./namespace.yaml
|
||||
# Flux-Kustomizations
|
||||
- ./alertmanager/ks.yaml
|
||||
- ./gatus/ks.yaml
|
||||
- ./prometheus-operator-crds/ks.yaml
|
||||
- ./victoria-metrics/ks.yaml
|
||||
|
|
|
|||
Loading…
Reference in a new issue