2023-10-03 12:26:06 -05:00
53 changed files with 835 additions and 189 deletions
--- a/.envrc
+++ b/.envrc
@ -1,3 +1,4 @@
 #shellcheck disable=SC2148,SC2155
 export KUBECONFIG="$(expand_path ./kubeconfig)"
 export SOPS_AGE_KEY_FILE="$(expand_path ./age.key)"
+export TALOSCONFIG="$(expand_path ./talos/clusterconfig/talosconfig)"
--- a/.sops.yaml
+++ b/.sops.yaml
@ -9,7 +9,7 @@ creation_rules:
    # Valinor
    age: >-
      age1g786w8t40g9y29l33rfd4jqlwhrgsxsc7ped6uju60k54j0q3enql3kfve
-  - path_regex: (ansible|terraform)/.*\.sops\.ya?ml
+  - path_regex: (ansible|terraform|talos)/.*\.sops\.ya?ml
    # Valinor
    age: >-
      age1g786w8t40g9y29l33rfd4jqlwhrgsxsc7ped6uju60k54j0q3enql3kfve
--- a/.taskfiles/_scripts/wait-for-k8s-job.sh
+++ b/.taskfiles/_scripts/wait-for-k8s-job.sh
@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+JOB_NAME=$1
+NAMESPACE="${2:-default}"
+
+[[ -z "${JOB_NAME}" ]] && echo "Job name not specified" && exit 1
+
+while true; do
+    STATUS="$(kubectl -n "${NAMESPACE}" get pod -l job-name="${JOB_NAME}" -o jsonpath='{.items[*].status.phase}')"
+    if [ "${STATUS}" == "Pending" ]; then
+        break
+    fi
+    sleep 1
+done
--- a/.taskfiles/flux/Taskfile.yaml
+++ b/.taskfiles/flux/Taskfile.yaml
@ -0,0 +1,47 @@
+---
+version: "3"
+
+tasks:
+  gr-sync:
+    desc: Sync all Flux GitRepositories
+    cmds:
+      - |
+        kubectl get gitrepositories --all-namespaces --no-headers | awk '{print $1, $2}' \
+          | xargs -P 4 -L 1 bash -c \
+            'kubectl -n $0 annotate gitrepository/$1 reconcile.fluxcd.io/requestedAt=$(date +%s) --field-manager=flux-client-side-apply --overwrite'
+
+  ks-sync:
+    desc: Sync all Flux Kustomizations
+    cmds:
+      - |
+        kubectl get kustomization --all-namespaces --no-headers | awk '{print $1, $2}' \
+          | xargs -P 4 -L 1 bash -c \
+            'kubectl -n $0 annotate kustomization/$1 reconcile.fluxcd.io/requestedAt="$(date +%s)" --field-manager=flux-client-side-apply --overwrite'
+
+  hr-sync:
+    desc: Sync all Flux HelmReleases
+    cmds:
+      - |
+        kubectl get helmreleases --all-namespaces --no-headers | awk '{print $1, $2}' \
+          | xargs -P 4 -L 1 bash -c \
+            'kubectl -n $0 annotate helmrelease/$1 reconcile.fluxcd.io/requestedAt="$(date +%s)" --overwrite'
+
+  tf-sync:
+    desc: Sync Flux Terraforms
+    cmds:
+      - |
+        kubectl get terraforms --all-namespaces --no-headers | awk '{print $1, $2}' \
+          | xargs -P 4 -L 1 bash -c \
+            'kubectl -n $0 annotate terraform/$1 reconcile.fluxcd.io/requestedAt="$(date +%s)" --overwrite'
+  hr-suspend:
+    desc: Suspend all Flux HelmReleases
+    cmds:
+      - |
+        flux get helmrelease --all-namespaces --no-header | awk '{print $1, $2}' \
+          | xargs -L 1 bash -c 'flux -n $0 suspend helmrelease $1'
+  hr-resume:
+    desc: Resume all Flux HelmReleases
+    cmds:
+      - |
+        flux get helmrelease --all-namespaces --no-header | awk '{print $1, $2}' \
+          | xargs -L 1 bash -c 'flux -n $0 resume helmrelease $1'
--- a/.taskfiles/rook/Taskfile.yaml
+++ b/.taskfiles/rook/Taskfile.yaml
@ -0,0 +1,91 @@
+---
+version: "3"
+
+x-task-vars: &task-vars
+  node: "{{.node}}"
+  ceph_disk: "{{.ceph_disk}}"
+  ts: "{{.ts}}"
+  jobName: "{{.jobName}}"
+
+vars:
+  waitForJobScript: "../_scripts/wait-for-k8s-job.sh"
+  ts: '{{now | date "150405"}}'
+
+tasks:
+  wipe-node-aule:
+    desc: Trigger a wipe of Rook-Ceph data on node "aule"
+    cmds:
+      - task: wipe-disk
+        vars:
+          node: "{{.node}}"
+          ceph_disk: "/dev/disk/by-id/scsi-0HC_Volume_37460833"
+      - task: wipe-data
+        vars:
+          node: "{{.node}}"
+    vars:
+      node: aule
+
+  wipe-node-eonwe:
+    desc: Trigger a wipe of Rook-Ceph data on node "eonwe"
+    cmds:
+      - task: wipe-disk
+        vars:
+          node: "{{.node}}"
+          ceph_disk: "/dev/disk/by-id/scsi-0HC_Volume_37460887"
+      - task: wipe-data
+        vars:
+          node: "{{.node}}"
+    vars:
+      node: eonwe
+
+  wipe-node-arlen:
+    desc: Trigger a wipe of Rook-Ceph data on node "arlen"
+    cmds:
+      - task: wipe-disk
+        vars:
+          node: "{{.node}}"
+          ceph_disk: "/dev/disk/by-id/scsi-0HC_Volume_37460897"
+      - task: wipe-data
+        vars:
+          node: "{{.node}}"
+    vars:
+      node: arlen
+
+  wipe-disk:
+    desc: Wipe all remnants of rook-ceph from a given disk (ex. task rook:wipe-disk node=aule ceph_disk="/dev/nvme0n1")
+    silent: true
+    internal: true
+    cmds:
+      - envsubst < <(cat {{.wipeRookDiskJobTemplate}}) | kubectl apply -f -
+      - bash {{.waitForJobScript}} {{.wipeCephDiskJobName}} default
+      - kubectl -n default wait job/{{.wipeCephDiskJobName}} --for condition=complete --timeout=1m
+      - kubectl -n default logs job/{{.wipeCephDiskJobName}} --container list
+      - kubectl -n default delete job {{.wipeCephDiskJobName}}
+    vars:
+      node: '{{ or .node (fail "`node` is required") }}'
+      ceph_disk: '{{ or .ceph_disk (fail "`ceph_disk` is required") }}'
+      jobName: 'wipe-disk-{{- .node -}}-{{- .ceph_disk | replace "/" "-" -}}-{{- .ts -}}'
+      wipeRookDiskJobTemplate: "WipeDiskJob.tmpl.yaml"
+    env: *task-vars
+    preconditions:
+      - sh: test -f {{.waitForJobScript}}
+      - sh: test -f {{.wipeRookDiskJobTemplate}}
+
+  wipe-data:
+    desc: Wipe all remnants of rook-ceph from a given disk (ex. task rook:wipe-data node=aule)
+    silent: true
+    internal: true
+    cmds:
+      - envsubst < <(cat {{.wipeRookDataJobTemplate}}) | kubectl apply -f -
+      - bash {{.waitForJobScript}} {{.wipeRookDataJobName}} default
+      - kubectl -n default wait job/{{.wipeRookDataJobName}} --for condition=complete --timeout=1m
+      - kubectl -n default logs job/{{.wipeRookDataJobName}} --container list
+      - kubectl -n default delete job {{.wipeRookDataJobName}}
+    vars:
+      node: '{{ or .node (fail "`node` is required") }}'
+      jobName: "wipe-rook-data-{{- .node -}}-{{- .ts -}}"
+      wipeRookDataJobTemplate: "WipeRookDataJob.tmpl.yaml"
+    env: *task-vars
+    preconditions:
+      - sh: test -f {{.waitForJobScript}}
+      - sh: test -f {{.wipeRookDataJobTemplate}}
--- a/.taskfiles/rook/WipeDiskJob.tmpl.yaml
+++ b/.taskfiles/rook/WipeDiskJob.tmpl.yaml
@ -0,0 +1,26 @@
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: "${jobName}"
+  namespace: "default"
+spec:
+  ttlSecondsAfterFinished: 3600
+  template:
+    spec:
+      automountServiceAccountToken: false
+      restartPolicy: Never
+      nodeName: ${node}
+      containers:
+        - name: disk-wipe
+          image: ghcr.io/onedr0p/alpine:3.17.3@sha256:999384960b6114496a5e4036e945141c205d064ce23b87326bd3f8d878c5a9d4
+          securityContext:
+            privileged: true
+          resources: {}
+          command: ["/bin/sh", "-c"]
+          args:
+            - apk add --no-cache sgdisk util-linux parted;
+              sgdisk --zap-all ${ceph_disk};
+              blkdiscard ${ceph_disk};
+              dd if=/dev/zero bs=1M count=10000 oflag=direct of=${ceph_disk};
+              partprobe ${ceph_disk};
--- a/.taskfiles/rook/WipeRookDataJob.tmpl.yaml
+++ b/.taskfiles/rook/WipeRookDataJob.tmpl.yaml
@ -0,0 +1,29 @@
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: "${jobName}"
+  namespace: "default"
+spec:
+  ttlSecondsAfterFinished: 3600
+  template:
+    spec:
+      automountServiceAccountToken: false
+      restartPolicy: Never
+      nodeName: ${node}
+      containers:
+        - name: disk-wipe
+          image: ghcr.io/onedr0p/alpine:3.17.3@sha256:999384960b6114496a5e4036e945141c205d064ce23b87326bd3f8d878c5a9d4
+          securityContext:
+            privileged: true
+          resources: {}
+          command: ["/bin/sh", "-c"]
+          args:
+            - rm -rf /mnt/host_var/lib/rook
+          volumeMounts:
+            - mountPath: /mnt/host_var
+              name: host-var
+      volumes:
+        - name: host-var
+          hostPath:
+            path: /var
--- a/Taskfile.yaml
+++ b/Taskfile.yaml
@ -21,6 +21,12 @@ env:
 includes:
  volsync: .taskfiles/VolSync/Tasks.yaml
  precommit: .taskfiles/PreCommit/Tasks.yaml
+  rook:
+    taskfile: ".taskfiles/rook"
+    dir: .taskfiles/rook
+  flux:
+    dir: .taskfiles/flux
+    taskfile: .taskfiles/flux

 tasks:

--- a/ansible/kubernetes/inventory/group_vars/all/main.yaml
+++ b/ansible/kubernetes/inventory/group_vars/all/main.yaml
@ -1,6 +1,6 @@
 ---
 # renovate: datasource=github-releases depName=k3s-io/k3s
-k3s_release_version: "v1.27.5+k3s1"
+k3s_release_version: "v1.28.2+k3s1"
 k3s_install_hard_links: true
 k3s_become: true
 k3s_etcd_datastore: true
--- a/ansible/kubernetes/inventory/hosts.yaml
+++ b/ansible/kubernetes/inventory/hosts.yaml
@ -6,15 +6,15 @@ kubernetes:
  children:
    master:
      hosts:
-        valinor-1:
+        aule:
          ansible_host: 10.2.0.3
          ceph_drives:
            - /dev/disk/by-id/scsi-0HC_Volume_37231496
-        valinor-2:
+        eonwe:
          ansible_host: 10.2.0.4
          ceph_drives:
            - /dev/disk/by-id/scsi-0HC_Volume_37231521
-        valinor-3:
+        arlen:
          ansible_host: 10.2.0.5
          ceph_drives:
            - /dev/disk/by-id/scsi-0HC_Volume_37231596
--- a/ansible/kubernetes/playbooks/templates/custom-cilium-helmchart.yaml.j2
+++ b/ansible/kubernetes/playbooks/templates/custom-cilium-helmchart.yaml.j2
@ -9,7 +9,7 @@ spec:
  # renovate: datasource=helm
  repo: https://helm.cilium.io/
  chart: cilium
-  version: 1.14.0
+  version: 1.14.2
  targetNamespace: kube-system
  bootstrap: true
  valuesContent: |-
@ -33,7 +33,7 @@ spec:
    ipv4NativeRoutingCIDR: "{{ k3s_server['cluster-cidr'] }}"
    k8sServiceHost: "{{ k3s_registration_address }}"
    k8sServicePort: 6443
-    kubeProxyReplacement: strict
+    kubeProxyReplacement: true
    kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256
    l2announcements:
      enabled: true
--- a/ansible/kubernetes/playbooks/templates/custom-coredns-helmchart.yaml.j2
+++ b/ansible/kubernetes/playbooks/templates/custom-coredns-helmchart.yaml.j2
@ -9,7 +9,7 @@ spec:
  # renovate: datasource=helm
  repo: https://coredns.github.io/helm
  chart: coredns
-  version: 1.24.5
+  version: 1.26.0
  targetNamespace: kube-system
  bootstrap: true
  valuesContent: |-
--- a/kubernetes/apps/cert-manager/cert-manager/app/helmrelease.yaml
+++ b/kubernetes/apps/cert-manager/cert-manager/app/helmrelease.yaml
@ -10,7 +10,7 @@ spec:
  chart:
    spec:
      chart: cert-manager
-      version: v1.12.3
+      version: v1.13.1
      sourceRef:
        kind: HelmRepository
        name: jetstack
--- a/kubernetes/apps/cert-manager/cert-manager/issuers/helmrelease.yaml
+++ b/kubernetes/apps/cert-manager/cert-manager/issuers/helmrelease.yaml
@ -10,7 +10,7 @@ spec:
  chart:
    spec:
      chart: cert-manager-webhook-dnsimple
-      version: 0.0.6
+      version: 0.0.10
      interval: 30m
      sourceRef:
        kind: HelmRepository
@ -33,3 +33,4 @@ spec:
          secretKeyRef:
            name: dnsimple-api-token
            key: letsencrypt-email
+    containerport: 8443
--- a/kubernetes/apps/cert-manager/cert-manager/ks.yaml
+++ b/kubernetes/apps/cert-manager/cert-manager/ks.yaml
@ -27,6 +27,6 @@ spec:
  sourceRef:
    kind: GitRepository
    name: valinor
-  wait: true
+  wait: false
  dependsOn:
    - name: cluster-apps-cert-manager
--- a/kubernetes/apps/kube-system/cilium/app/cilium-l2.yaml
+++ b/kubernetes/apps/kube-system/cilium/app/cilium-l2.yaml
@ -1,20 +0,0 @@
---
-apiVersion: cilium.io/v2alpha1
-kind: CiliumL2AnnouncementPolicy
-metadata:
-  name: policy
-spec:
-  loadBalancerIPs: true
-  interfaces:
-    - ^enp.*
-  nodeSelector:
-    matchLabels:
-      kubernetes.io/os: linux
---
-apiVersion: cilium.io/v2alpha1
-kind: CiliumLoadBalancerIPPool
-metadata:
-  name: pool
-spec:
-  cidrs:
-    - cidr: 10.2.42.0/24
--- a/kubernetes/apps/kube-system/cilium/app/helmrelease.yaml
+++ b/kubernetes/apps/kube-system/cilium/app/helmrelease.yaml
@ -26,89 +26,34 @@ spec:
  uninstall:
    keepHistory: false
  values:
-    autoDirectNodeRoutes: true
-    bpf:
-      masquerade: true
-    bgp:
-      enabled: false
    cluster:
-      name: kubernetes
+      name: valinor
      id: 1
-    containerRuntime:
-      integration: containerd
-      socketPath: /var/run/k3s/containerd/containerd.sock
-    endpointRoutes:
-      enabled: true
-    hubble:
-      enabled: true
-      metrics:
-        enabled:
-          - dns:query
-          - drop
-          - tcp
-          - flow
-          - port-distribution
-          - icmp
-          - http
-        serviceMonitor:
-          enabled: true
-        dashboards:
-          enabled: true
-          annotations:
-            grafana_folder: Cilium
-      relay:
-        enabled: true
-        rollOutPods: true
-        prometheus:
-          serviceMonitor:
-            enabled: true
-      ui:
-        enabled: true
-        rollOutPods: true
-        ingress:
-          enabled: true
-          className: nginx
-          hosts:
-            - &host hubble.valinor.social
-          tls:
-            - hosts:
-                - *host
    ipam:
      mode: kubernetes
-    ipv4NativeRoutingCIDR: 10.32.0.0/16
+    kubeProxyReplacement: true
+    securityContext:
+      capabilities:
+        ciliumAgent:
+          - CHOWN
+          - KILL
+          - NET_ADMIN
+          - NET_RAW
+          - IPC_LOCK
+          - SYS_ADMIN
+          - SYS_RESOURCE
+          - DAC_OVERRIDE
+          - FOWNER
+          - SETGID
+          - SETUID
+        cleanCiliumState:
+          - NET_ADMIN
+          - SYS_ADMIN
+          - SYS_RESOURCE
+    cgroup:
+      autoMount:
+        enabled: false
+      hostRoot: /sys/fs/cgroup
    k8sServiceHost: 10.2.0.6
    k8sServicePort: 6443
-    kubeProxyReplacement: strict
-    kubeProxyReplacementHealthzBindAddr: 0.0.0.0:10256
-    l2announcements:
-      enabled: true
-      leaseDuration: 120s
-      leaseRenewDeadline: 60s
-      leaseRetryPeriod: 1s
-    loadBalancer:
-      algorithm: maglev
-      mode: dsr
-    localRedirectPolicy: true
-    operator:
-      rollOutPods: true
-      prometheus:
-        enabled: true
-        serviceMonitor:
-          enabled: true
-      dashboards:
-        enabled: true
-        annotations:
-          grafana_folder: Cilium
-    prometheus:
-      enabled: true
-      serviceMonitor:
-        enabled: true
-        trustCRDsExist: true
-    dashboards:
-      enabled: true
-      annotations:
-        grafana_folder: Cilium
    rollOutCiliumPods: true
-    securityContext:
-      privileged: true
-    tunnel: disabled
--- a/kubernetes/apps/kube-system/cilium/ks.yaml
+++ b/kubernetes/apps/kube-system/cilium/ks.yaml
@ -1,14 +1,17 @@
 ---
+# yaml-language-server: $schema=https://raw.githubusercontent.com/fluxcd-community/flux2-schemas/main/kustomization-kustomize-v1.json
 apiVersion: kustomize.toolkit.fluxcd.io/v1
 kind: Kustomization
 metadata:
  name: cluster-apps-cilium
  namespace: flux-system
 spec:
-  interval: 10m
+  interval: 30m
+  retryInterval: 1m
+  timeout: 5m
  path: "./kubernetes/apps/kube-system/cilium/app"
  prune: true
  sourceRef:
    kind: GitRepository
    name: valinor
-  wait: true
+  wait: false
--- a/kubernetes/apps/kube-system/hccm/app/externalsecret.yaml
+++ b/kubernetes/apps/kube-system/hccm/app/externalsecret.yaml
@ -0,0 +1,22 @@
+---
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
+metadata:
+  name: hcloud
+  namespace: kube-system
+spec:
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: onepassword-connect
+  target:
+    name: hcloud
+    creationPolicy: Owner
+  data:
+    - secretKey: token
+      remoteRef:
+        key: hetzner
+        property: cloud-api-token
+    - secretKey: network
+      remoteRef:
+        key: hetzner
+        property: cloud-network-name
--- a/kubernetes/apps/kube-system/hccm/app/helmrelease.yaml
+++ b/kubernetes/apps/kube-system/hccm/app/helmrelease.yaml
@ -0,0 +1,24 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/fluxcd-community/flux2-schemas/main/helmrelease-helm-v2beta1.json
+---
+apiVersion: helm.toolkit.fluxcd.io/v2beta1
+kind: HelmRelease
+metadata:
+  name: hccm
+  namespace: kube-system
+spec:
+  interval: 30m
+  chart:
+    spec:
+      chart: hcloud-cloud-controller-manager
+      version: v1.18.0
+      sourceRef:
+        kind: HelmRepository
+        name: hetzner
+        namespace: flux-system
+      interval: 30m
+  values:
+    metrics:
+      enabled: true
+    networking:
+      enabled: true
+      clusterCIDR: 10.244.0.0/16
--- a/kubernetes/apps/kube-system/cilium/app/kustomization.yaml
+++ b/kubernetes/apps/kube-system/cilium/app/kustomization.yaml
@ -1,7 +1,8 @@
+# yaml-language-server: $schema=https://json.schemastore.org/kustomization.json
 ---
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 namespace: kube-system
 resources:
+  - ./externalsecret.yaml
  - ./helmrelease.yaml
-  - ./cilium-l2.yaml
--- a/kubernetes/apps/kube-system/hccm/ks.yaml
+++ b/kubernetes/apps/kube-system/hccm/ks.yaml
@ -0,0 +1,16 @@
+---
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: cluster-apps-hetzner-hccm
+  namespace: flux-system
+  labels:
+    substitution.flux.home.arpa/enabled: "true"
+spec:
+  interval: 10m
+  path: "./kubernetes/apps/kube-system/hccm/app"
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: valinor
+  wait: true
--- a/kubernetes/apps/kube-system/kustomization.yaml
+++ b/kubernetes/apps/kube-system/kustomization.yaml
@ -1,3 +1,4 @@
+# yaml-language-server: $schema=https://json.schemastore.org/kustomization.json
 ---
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
@ -5,5 +6,6 @@ resources:
  # Pre Flux-Kustomizations
  - ./namespace.yaml
  # Flux-Kustomizations
-  - ./metrics-server/ks.yaml
  - ./cilium/ks.yaml
+  - ./metrics-server/ks.yaml
+  - ./hccm/ks.yaml
--- a/kubernetes/apps/kube-system/metrics-server/app/helmrelease.yaml
+++ b/kubernetes/apps/kube-system/metrics-server/app/helmrelease.yaml
@ -18,3 +18,8 @@ spec:
  values:
    metrics:
      enabled: true
+    args:
+      - --kubelet-insecure-tls
+      - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
+      - --kubelet-use-node-status-port
+      - --metric-resolution=15s
--- a/kubernetes/apps/kube-system/metrics-server/app/kustomization.yaml
+++ b/kubernetes/apps/kube-system/metrics-server/app/kustomization.yaml
@ -1,3 +1,4 @@
+# yaml-language-server: $schema=https://json.schemastore.org/kustomization.json
 ---
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
--- a/kubernetes/apps/monitoring/alertmanager/app/helmrelease.yaml
+++ b/kubernetes/apps/monitoring/alertmanager/app/helmrelease.yaml
@ -24,7 +24,7 @@ spec:

    image:
      repository: quay.io/prometheus/alertmanager
-      tag: main@sha256:32e432a57c8ba354f7b42c5a7784033225f86b8b029a29c1f77a8c785ea90ddb
+      tag: main@sha256:7c060ae2a86177fbb4106fddcdd9f2cd494d4415b67ccda71a9fdf11f52e825b

    podAnnotations:
      reloader.stakater.com/auto: "true"
--- a/kubernetes/apps/monitoring/grafana/app/helmrelease.yaml
+++ b/kubernetes/apps/monitoring/grafana/app/helmrelease.yaml
@ -14,7 +14,7 @@ spec:
        kind: HelmRepository
        name: grafana
        namespace: flux-system
-      version: 6.60.0
+      version: 6.60.1
  interval: 30m
  timeout: 20m
  maxHistory: 2
@ -184,7 +184,7 @@ spec:
        root_url: https://grafana.valinor.social

    imageRenderer:
-      enabled: true
+      enabled: false

    ingress:
      enabled: true
--- a/kubernetes/apps/monitoring/kube-prometheus-stack/app/helmrelease.yaml
+++ b/kubernetes/apps/monitoring/kube-prometheus-stack/app/helmrelease.yaml
@ -11,7 +11,7 @@ spec:
  chart:
    spec:
      chart: kube-prometheus-stack
-      version: 51.1.1
+      version: 51.2.0
      sourceRef:
        kind: HelmRepository
        name: prometheus-community
@ -201,20 +201,12 @@ spec:
        enableAdminAPI: true
        walCompression: true
        thanos:
-          image: quay.io/thanos/thanos:v0.32.3
+          image: quay.io/thanos/thanos:v0.32.4
          objectStorageConfig:
-            name: thanos-objstore-secret
+            name: thanos-s3-secret
            key: objstore.yml
        retention: 2d
        retentionSize: 15GB
-        additionalScrapeConfigs:
-          - job_name: statsd-exporter
-            scrape_interval: 1m
-            scrape_timeout: 10s
-            honor_timestamps: true
-            static_configs:
-              - targets:
-                  - statsd-exporter.fediverse.svc.cluster.local:9102 # default zalando postgres cluster
        storageSpec:
          volumeClaimTemplate:
            spec:
@ -222,8 +214,3 @@ spec:
              resources:
                requests:
                  storage: 20Gi
-  valuesFrom:
-    - targetPath: objstoreConfig.config.bucket
-      kind: ConfigMap
-      name: thanos-bucket-v1
-      valuesKey: BUCKET_NAME
--- a/kubernetes/apps/monitoring/node-exporter/app/helmrelease.yaml
+++ b/kubernetes/apps/monitoring/node-exporter/app/helmrelease.yaml
@ -10,7 +10,7 @@ spec:
  chart:
    spec:
      chart: prometheus-node-exporter
-      version: 4.23.1
+      version: 4.23.2
      sourceRef:
        kind: HelmRepository
        name: prometheus-community
--- a/kubernetes/apps/monitoring/thanos/app/externalsecret.yaml
+++ b/kubernetes/apps/monitoring/thanos/app/externalsecret.yaml
@ -0,0 +1,31 @@
+---
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
+metadata:
+  name: thanos
+  namespace: monitoring
+spec:
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: onepassword-connect
+  target:
+    name: thanos-s3-secret
+    creationPolicy: Owner
+    template:
+      engineVersion: v2
+      data:
+        objstore.yml: |-
+          type: s3
+          config:
+            access_key: {{ .minio_thanos_access_key }}
+            bucket: thanos
+            endpoint: {{ .minio_s3_host }}
+            region: us-east-1
+            secret_key: {{ .minio_thanos_secret_key }}
+  dataFrom:
+    - extract:
+        key: minio
+      rewrite:
+        - regexp:
+            source: "(.*)"
+            target: "minio_$1"
--- a/kubernetes/apps/monitoring/thanos/app/helmrelease.yaml
+++ b/kubernetes/apps/monitoring/thanos/app/helmrelease.yaml
@ -11,7 +11,7 @@ spec:
  chart:
    spec:
      chart: thanos
-      version: 12.13.5
+      version: 12.13.6
      sourceRef:
        kind: HelmRepository
        name: bitnami
@ -34,11 +34,8 @@ spec:
    image:
      registry: quay.io
      repository: thanos/thanos
-      tag: v0.32.3
-    objstoreConfig:
-      type: s3
-      config:
-        insecure: true
+      tag: v0.32.4
+    existingObjstoreSecret: thanos-s3-secret
    queryFrontend:
      enabled: true
      replicaCount: 3
@ -71,14 +68,14 @@ spec:
      persistence:
        enabled: true
        storageClass: ceph-block
-        size: 100Gi
+        size: 20Gi
    storegateway:
      enabled: true
      replicaCount: 3
      persistence:
        enabled: true
        storageClass: ceph-block
-        size: 20Gi
+        size: 10Gi
    ruler:
      enabled: true
      replicaCount: 3
@ -99,29 +96,8 @@ spec:
      persistence:
        enabled: true
        storageClass: ceph-block
-        size: 20Gi
+        size: 5Gi
    metrics:
      enabled: true
      serviceMonitor:
        enabled: true
-  valuesFrom:
-    - targetPath: objstoreConfig.config.bucket
-      kind: ConfigMap
-      name: thanos-bucket-v1
-      valuesKey: BUCKET_NAME
-    - targetPath: objstoreConfig.config.endpoint
-      kind: ConfigMap
-      name: thanos-bucket-v1
-      valuesKey: BUCKET_HOST
-    - targetPath: objstoreConfig.config.region
-      kind: ConfigMap
-      name: thanos-bucket-v1
-      valuesKey: BUCKET_REGION
-    - targetPath: objstoreConfig.config.access_key
-      kind: Secret
-      name: thanos-bucket-v1
-      valuesKey: AWS_ACCESS_KEY_ID
-    - targetPath: objstoreConfig.config.secret_key
-      kind: Secret
-      name: thanos-bucket-v1
-      valuesKey: AWS_SECRET_ACCESS_KEY
--- a/kubernetes/apps/monitoring/thanos/app/kustomization.yaml
+++ b/kubernetes/apps/monitoring/thanos/app/kustomization.yaml
@ -4,7 +4,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 namespace: monitoring
 resources:
-  - ./objectbucketclaim.yaml
+  - ./externalsecret.yaml
  - ./helmrelease.yaml
 configMapGenerator:
  - name: thanos-bucket-replicate-dashboard
--- a/kubernetes/apps/monitoring/thanos/app/objectbucketclaim.yaml
+++ b/kubernetes/apps/monitoring/thanos/app/objectbucketclaim.yaml
@ -1,9 +0,0 @@
---
-apiVersion: objectbucket.io/v1alpha1
-kind: ObjectBucketClaim
-metadata:
-  name: thanos-bucket-v1
-  namespace: monitoring
-spec:
-  bucketName: thanos-v1
-  storageClassName: ceph-bucket
--- a/kubernetes/apps/network/ingress-nginx/app/helmrelease.yaml
+++ b/kubernetes/apps/network/ingress-nginx/app/helmrelease.yaml
@ -18,11 +18,11 @@ spec:
    controller:
      replicaCount: 3

-      hostPort:
-        enabled: true
-        ports:
-          http: 81
-          https: 444
+      # hostPort:
+      #   enabled: true
+      #   ports:
+      #     http: 81
+      #     https: 444

      updateStrategy:
        type: Recreate
@ -31,8 +31,12 @@ spec:
        enabled: true
        type: LoadBalancer
        annotations:
-          external-dns.alpha.kubernetes.io/hostname: "ingress.valinor.social"
-          io.cilium/lb-ipam-ips: "10.2.42.1"
+          load-balancer.hetzner.cloud/location: fsn1
+          load-balancer.hetzner.cloud/protocol: tcp
+          load-balancer.hetzner.cloud/name: valinor-nginx
+          load-balancer.hetzner.cloud/use-private-ip: true
+          load-balancer.hetzner.cloud/uses-proxyprotocol: true
+
        externalTrafficPolicy: Local

      publishService:
--- a/kubernetes/apps/rook-ceph/rook-ceph/cluster/helmrelease.yaml
+++ b/kubernetes/apps/rook-ceph/rook-ceph/cluster/helmrelease.yaml
@ -53,15 +53,23 @@ spec:
        config:
          osdsPerDevice: "1"
        nodes:
-          - name: "valinor-1"
+          - name: "aule"
            devices:
-              - name: /dev/disk/by-id/scsi-0HC_Volume_37231496
-          - name: "valinor-2"
+              - name: /dev/disk/by-id/scsi-0HC_Volume_37460833
+          - name: "eonwe"
            devices:
-              - name: /dev/disk/by-id/scsi-0HC_Volume_37231521
-          - name: "valinor-3"
+              - name: /dev/disk/by-id/scsi-0HC_Volume_37460887
+          - name: "arlen"
            devices:
-              - name: /dev/disk/by-id/scsi-0HC_Volume_37231596
+              - name: /dev/disk/by-id/scsi-0HC_Volume_37460897
+      resources:
+        mgr:
+          limits:
+            cpu: "1000m"
+            memory: "1Gi"
+          requests:
+            cpu: "1000m"
+            memory: "1Gi"

    ingress:
      ingressClassName: "nginx"
@ -101,3 +109,68 @@ spec:
            csi.storage.k8s.io/node-stage-secret-name: rook-csi-rbd-node
            csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph
            csi.storage.k8s.io/fstype: ext4
+    cephFileSystems:
+      - name: ceph-filesystem
+        spec:
+          metadataPool:
+            replicated:
+              size: 3
+          dataPools:
+            - failureDomain: host
+              replicated:
+                size: 3
+          metadataServer:
+            activeCount: 1
+            activeStandby: true
+            resources:
+              requests:
+                cpu: "35m"
+                memory: "64M"
+              limits:
+                memory: "144M"
+        storageClass:
+          enabled: true
+          isDefault: false
+          name: ceph-filesystem
+          reclaimPolicy: Delete
+          allowVolumeExpansion: true
+          mountOptions: []
+          parameters:
+            csi.storage.k8s.io/provisioner-secret-name: rook-csi-cephfs-provisioner
+            csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph
+            csi.storage.k8s.io/controller-expand-secret-name: rook-csi-cephfs-provisioner
+            csi.storage.k8s.io/controller-expand-secret-namespace: rook-ceph
+            csi.storage.k8s.io/node-stage-secret-name: rook-csi-cephfs-node
+            csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph
+            csi.storage.k8s.io/fstype: ext4
+    cephObjectStores:
+      - name: ceph-objectstore
+        spec:
+          metadataPool:
+            failureDomain: host
+            replicated:
+              size: 3
+          dataPool:
+            failureDomain: host
+            erasureCoded:
+              dataChunks: 2
+              codingChunks: 1
+          preservePoolsOnDelete: true
+          gateway:
+            port: 80
+            resources:
+              requests:
+                cpu: 100m
+                memory: 128M
+              limits:
+                memory: 2Gi
+            instances: 1
+          healthCheck:
+            bucket:
+              interval: 60s
+        storageClass:
+          enabled: true
+          name: ceph-bucket
+          reclaimPolicy: Delete
+          parameters:
+            region: us-east-1
--- a/kubernetes/apps/security/external-secrets/stores/onepassword/helmrelease.yaml
+++ b/kubernetes/apps/security/external-secrets/stores/onepassword/helmrelease.yaml
@ -23,7 +23,7 @@ spec:

    image:
      repository: docker.io/1password/connect-api
-      tag: 1.7.2@sha256:6aa94cf713f99c0fa58c12ffdd1b160404b4c13a7f501a73a791aa84b608c5a1
+      tag: 1.7.2

    env:
      OP_BUS_PORT: "11220"
--- a/kubernetes/apps/system/kubelet-csr-approver/app/helmrelease.yaml
+++ b/kubernetes/apps/system/kubelet-csr-approver/app/helmrelease.yaml
@ -0,0 +1,21 @@
+---
+# yaml-language-server: $schema=https://raw.githubusercontent.com/fluxcd-community/flux2-schemas/main/helmrelease-helm-v2beta1.json
+apiVersion: helm.toolkit.fluxcd.io/v2beta1
+kind: HelmRelease
+metadata:
+  name: kubelet-csr-approver
+  namespace: system
+spec:
+  interval: 30m
+  chart:
+    spec:
+      chart: kubelet-csr-approver
+      version: 1.0.5
+      sourceRef:
+        kind: HelmRepository
+        name: postfinance
+        namespace: flux-system
+      interval: 30m
+  valuesFrom:
+    - kind: ConfigMap
+      name: kubelet-csr-approver-values
--- a/kubernetes/apps/system/kubelet-csr-approver/app/kustomization.yaml
+++ b/kubernetes/apps/system/kubelet-csr-approver/app/kustomization.yaml
@ -0,0 +1,14 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: system
+resources:
+  - ./helmrelease.yaml
+
+configMapGenerator:
+  - name: kubelet-csr-approver-values
+    files:
+      - values.yaml=./values.yaml
+
+configurations:
+  - kustomizeconfig.yaml
--- a/kubernetes/apps/system/kubelet-csr-approver/app/kustomizeconfig.yaml
+++ b/kubernetes/apps/system/kubelet-csr-approver/app/kustomizeconfig.yaml
@ -0,0 +1,7 @@
+---
+nameReference:
+  - kind: ConfigMap
+    version: v1
+    fieldSpecs:
+      - path: spec/valuesFrom/name
+        kind: HelmRelease
--- a/kubernetes/apps/system/kubelet-csr-approver/app/values.yaml
+++ b/kubernetes/apps/system/kubelet-csr-approver/app/values.yaml
@ -0,0 +1,5 @@
+---
+providerRegex: |
+  ^(eonwe|aule|arlen)$
+
+bypassDnsResolution: true
--- a/kubernetes/apps/system/kubelet-csr-approver/ks.yaml
+++ b/kubernetes/apps/system/kubelet-csr-approver/ks.yaml
@ -0,0 +1,15 @@
+---
+# yaml-language-server: $schema=https://raw.githubusercontent.com/fluxcd-community/flux2-schemas/main/kustomization-kustomize-v1.json
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: cluster-apps-kubelet-csr-approver
+  namespace: flux-system
+spec:
+  interval: 10m
+  path: "./kubernetes/apps/system/kubelet-csr-approver/app"
+  prune: true
+  sourceRef:
+    kind: GitRepository
+    name: valinor
+  wait: true
--- a/kubernetes/apps/system/kustomization.yaml
+++ b/kubernetes/apps/system/kustomization.yaml
@ -6,3 +6,4 @@ resources:
  - ./namespace.yaml
  # Flux-Kustomizations
  - ./reloader/ks.yaml
+  - ./kubelet-csr-approver/ks.yaml
--- a/kubernetes/flux/repositories/helm/hetzner.yaml
+++ b/kubernetes/flux/repositories/helm/hetzner.yaml
@ -0,0 +1,10 @@
+---
+apiVersion: source.toolkit.fluxcd.io/v1beta2
+kind: HelmRepository
+metadata:
+  name: hetzner
+  namespace: flux-system
+spec:
+  interval: 30m
+  url: https://charts.hetzner.cloud
+  timeout: 3m
--- a/kubernetes/flux/repositories/helm/kustomization.yaml
+++ b/kubernetes/flux/repositories/helm/kustomization.yaml
@ -1,3 +1,4 @@
+# yaml-language-server: $schema=https://json.schemastore.org/kustomization.json
 ---
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
@ -12,6 +13,7 @@ resources:
 - external-secrets.yaml
 - fairwinds.yaml
 - grafana.yaml
+ - hetzner.yaml
 - ingress-nginx.yaml
 - jahanson.yaml
 - jetstack.yaml
--- a/requirements.txt
+++ b/requirements.txt
@ -1,5 +1,5 @@
 ansible==8.4.0
-ansible-lint==6.20.0
+ansible-lint==6.20.3
 bcrypt==4.0.1
 jmespath==1.0.1
 netaddr==0.9.0
--- a/talos/clusterconfig/.gitignore
+++ b/talos/clusterconfig/.gitignore
@ -0,0 +1,7 @@
+valinor-aule.hsn.dev.yaml
+valinor-eonwe.hsn.dev.yaml
+valinor-arlen.hsn.dev.yaml
+talosconfig
+valinor-vaire.hsn.dev.yaml
+valinor-nienna.hsn.dev.yaml
+valinor-orome.hsn.dev.yaml
--- a/talos/deploy-integrations.sh
+++ b/talos/deploy-integrations.sh
@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+# shellcheck disable=2312
+pushd integrations >/dev/null 2>&1 || exit 1
+
+#rm -rf cni/charts
+#envsubst < ../../kubernetes/apps/kube-system/cilium/app/values.yaml > cni/values.yaml
+#kustomize build --enable-helm cni | kubectl apply -f -
+#rm cni/values.yaml
+#rm -rf cni/charts
+
+rm -rf kubelet-csr-approver/charts
+envsubst < ../../kubernetes/apps/system/kubelet-csr-approver/app/values.yaml > kubelet-csr-approver/values.yaml
+if ! kubectl get ns system >/dev/null 2>&1; then
+  kubectl create ns system
+fi
+kustomize build --enable-helm kubelet-csr-approver | kubectl apply -f -
+rm kubelet-csr-approver/values.yaml
+rm -rf kubelet-csr-approver/charts
--- a/talos/generate-clusterconfig.sh
+++ b/talos/generate-clusterconfig.sh
@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+
+# Generate a new config using talhelper
+talhelper genconfig --env-file talenv.sops.yaml --secret-file talsecret.sops.yaml --config-file talconfig.yaml
--- a/talos/integrations/cni/kustomization.yaml
+++ b/talos/integrations/cni/kustomization.yaml
@ -0,0 +1,18 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+helmCharts:
+  - name: cilium
+    repo: https://helm.cilium.io/
+    version: 1.14.2
+    releaseName: cilium
+    includeCRDs: true
+    namespace: kube-system
+    valuesFile: values.yaml
+
+commonAnnotations:
+  meta.helm.sh/release-name: cilium
+  meta.helm.sh/release-namespace: kube-system
+commonLabels:
+  app.kubernetes.io/managed-by: Helm
--- a/talos/integrations/kubelet-csr-approver/kustomization.yaml
+++ b/talos/integrations/kubelet-csr-approver/kustomization.yaml
@ -0,0 +1,18 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+helmCharts:
+  - name: kubelet-csr-approver
+    repo: https://postfinance.github.io/kubelet-csr-approver
+    version: 1.0.5
+    releaseName: kubelet-csr-approver
+    includeCRDs: true
+    namespace: system
+    valuesFile: values.yaml
+
+commonAnnotations:
+  meta.helm.sh/release-name: kubelet-csr-approver
+  meta.helm.sh/release-namespace: system
+commonLabels:
+  app.kubernetes.io/managed-by: Helm
--- a/talos/talconfig.yaml
+++ b/talos/talconfig.yaml
@ -0,0 +1,165 @@
+---
+clusterName: ${clusterName}
+
+talosVersion: v1.5.1
+kubernetesVersion: 1.28.1
+endpoint: "https://${clusterName}.hsn.dev:6443"
+
+cniConfig:
+  name: none
+
+additionalApiServerCertSans:
+  - ${clusterEndpointIP}
+
+additionalMachineCertSans:
+  - ${clusterEndpointIP}
+  - ${clusterName}.hsn.dev
+
+nodes:
+  - hostname: aule.hsn.dev
+    disableSearchDomain: true
+    ipAddress: 10.2.0.3
+    controlPlane: true
+    installDiskSelector:
+      busPath: /dev/sda
+    networkInterfaces:
+      - interface: eth0
+        dhcp: true
+      - interface: eth1
+        dhcp: true
+        routes:
+          - network: 10.2.0.0/16
+            gateway: 10.2.1.1 # The route's gateway (if empty, creates link scope route).
+            metric: 2048
+
+  - hostname: eonwe.hsn.dev
+    disableSearchDomain: true
+    ipAddress: 10.2.0.5
+    controlPlane: true
+    installDiskSelector:
+      busPath: /dev/sda
+    networkInterfaces:
+      - interface: eth0
+        dhcp: true
+      - interface: eth1
+        dhcp: true
+        routes:
+          - network: 10.2.0.0/16
+            gateway: 10.2.1.1 # The route's gateway (if empty, creates link scope route).
+            metric: 2048
+
+  - hostname: arlen.hsn.dev
+    disableSearchDomain: true
+    ipAddress: 10.2.0.4
+    controlPlane: true
+    installDiskSelector:
+      busPath: /dev/sda
+    networkInterfaces:
+      - interface: eth0
+        dhcp: true
+      - interface: eth1
+        dhcp: true
+        routes:
+          - network: 10.2.0.0/16
+            gateway: 10.2.1.1 # The route's gateway (if empty, creates link scope route).
+            metric: 2048
+  - hostname: vaire.hsn.dev
+    disableSearchDomain: true
+    ipAddress: 10.2.0.8
+    controlPlane: false
+    installDiskSelector:
+      busPath: /dev/sda
+    networkInterfaces:
+      - interface: eth0
+        dhcp: true
+      - interface: eth1
+        dhcp: true
+        routes:
+          - network: 10.2.0.0/16
+            gateway: 10.2.1.1 # The route's gateway (if empty, creates link scope route).
+            metric: 2048
+  - hostname: nienna.hsn.dev
+    disableSearchDomain: true
+    ipAddress: 10.2.0.9
+    controlPlane: false
+    installDiskSelector:
+      busPath: /dev/sda
+    networkInterfaces:
+      - interface: eth0
+        dhcp: true
+      - interface: eth1
+        dhcp: true
+        routes:
+          - network: 10.2.0.0/16
+            gateway: 10.2.1.1 # The route's gateway (if empty, creates link scope route).
+            metric: 2048
+  - hostname: orome.hsn.dev
+    disableSearchDomain: true
+    ipAddress: 10.2.0.10
+    controlPlane: false
+    installDiskSelector:
+      busPath: /dev/sda
+    networkInterfaces:
+      - interface: eth0
+        dhcp: true
+      - interface: eth1
+        dhcp: true
+        routes:
+          - network: 10.2.0.0/16
+            gateway: 10.2.1.1 # The route's gateway (if empty, creates link scope route).
+            metric: 2048
+
+controlPlane:
+  patches:
+    - |-
+      cluster:
+        allowSchedulingOnMasters: true
+        network:
+          cni:
+            name: none
+        proxy:
+          disabled: true
+        etcd:
+          advertisedSubnets:
+            - 10.2.0.0/24
+
+    - |-
+      - op: remove
+        path: /cluster/apiServer/admissionControl
+
+    - |-
+      machine:
+        features:
+          kubePrism:
+            enabled: true
+            port: 7445
+
+        files:
+          - op: create
+            path: /etc/cri/conf.d/20-customization.part
+            content: |
+              [plugins]
+                [plugins."io.containerd.grpc.v1.cri"]
+                  enable_unprivileged_ports = true
+                  enable_unprivileged_icmp = true
+        kubelet:
+          extraArgs:
+            feature-gates: CronJobTimeZone=true,GracefulNodeShutdown=true,NewVolumeManagerReconstruction=false
+            rotate-server-certificates: "true"
+          extraConfig:
+            maxPods: 150
+          nodeIP:
+            validSubnets:
+                - 10.2.0.0/24
+        network:
+          extraHostEntries:
+            - ip: ${clusterEndpointIP}
+              aliases:
+                - ${clusterName}.hsn.dev
+        sysctls:
+          fs.inotify.max_user_watches: "1048576"
+          fs.inotify.max_user_instances: "8192"
+        time:
+          disabled: false
+          servers:
+            - ntp.hetzner.com
--- a/talos/talenv.sops.yaml
+++ b/talos/talenv.sops.yaml
@ -0,0 +1,22 @@
+clusterName: ENC[AES256_GCM,data:iT5CwpMddw==,iv:st1ajjpRXQiHozpIJqUUwmRe542IiR2aWLEdqkk4W9k=,tag:KOCQ8x28kwNNDUXwOTpulg==,type:str]
+clusterEndpointIP: ENC[AES256_GCM,data:5VXivET/uV4=,iv:SRhLmDfbSlhnb9DsaFXCqiP/Bx4Khi4GdXseyuhuYAw=,tag:BrP3OL/1FwrUyCMWRFB0BQ==,type:str]
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age1g786w8t40g9y29l33rfd4jqlwhrgsxsc7ped6uju60k54j0q3enql3kfve
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBQYkFFdkluSU5heUJLZ1hZ
+            NlVFR0RMdDN5QTU3UjhZQzFGbS83ZXRKOXpRCmJwZTlmQ2drbWp0aFZaZmFad2Nm
+            dkxZV1g0NUozY1laV2N4ellTaEJGVE0KLS0tIEptRWFJZVpYcWR6MGNzeU41Vnpi
+            MTFUZEplYVN5RGhhMGNEcDlGbTVQcjQKktwztZAHGUqoxbGHuAg0dX5Vap+wFVfx
+            ku6Hzg1ZU8Lvd8ODe+4p+RvHSKVll1akgpPVuymCUxl+I6EvH7gEDA==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2023-09-25T17:08:14Z"
+    mac: ENC[AES256_GCM,data:WpwataAKsHlCIH6MN/lBBwBk5sKMCYlIptHXCnoqFCMdzPK8JR86XzeOPpJEN9aXu1wfdve+y1f7r4j7j+8V/eYjKDAYnv1ewsmZm9VfzfIcRAv2BGVANp52OASPCyoTwq9wpv7p/1d+f4C2vCZCarmurroxhGcvb17COFOs1SQ=,iv:2sSA+2NyqaSFA1v/Gp6XyTeaqBt5b5OLALmZ/b2TqJE=,tag:DxbiT2+bwjhOjZ38KQ26vQ==,type:str]
+    pgp: []
+    unencrypted_suffix: _unencrypted
+    version: 3.8.0
--- a/talos/talsecret.sops.yaml
+++ b/talos/talsecret.sops.yaml
@ -0,0 +1,43 @@
+cluster:
+    id: ENC[AES256_GCM,data:K+hrEJHwEkMvD7vP6Dl7g3VZ0LC9Ytxm4us4Dcj7kMz6n5mEUjO7AeK1ZXI=,iv:QwxLybEen4e06QrV2fXq7NZU523tly2QzvEERAO4PDY=,tag:NOrMociqIh+JNzPEpMRthg==,type:str]
+    secret: ENC[AES256_GCM,data:bS0TsqBwbjSZcaMnh/16ZLFmbihpTctaHJQPxfhjmPKL1W0pJ7ivdlk44jc=,iv:qfL7Q26+tNCyTRYxqVGcwNVY+nYrNkylqxv1fDVZIPM=,tag:AszrJ1/igGLHQWVKyGJsAA==,type:str]
+secrets:
+    bootstraptoken: ENC[AES256_GCM,data:P0ZbFPa8yMtDamH307VD5fJnTFgj38A=,iv:5rFtX93mSAhZdRZhV3/ZhUYZvjoEq7aHYbuSxMfsjWo=,tag:xYQakQbO1nVyA6oE0qVfZQ==,type:str]
+    secretboxencryptionsecret: ENC[AES256_GCM,data:HGcJPvrgpWFMTCf1Zo74ek7sZqm8dwa0+EbLcwB3P7/u6mTooOOskONQKA4=,iv:/iOLOaNxdOOv6bwvpJInhfs8sIzaIHsjErIlhTEReds=,tag:u3MdQnaE0+EnFOqSJtSYmw==,type:str]
+trustdinfo:
+    token: ENC[AES256_GCM,data:tfYLX59Xy2Cp9t3GAhqeDus2moEEMns=,iv:xiQOuMWnGnJcr4zTqHOMFGeaVdQNZDg6FTgu9T9NrEU=,tag:A46+Hqq1n9x2WSLQqepIfA==,type:str]
+certs:
+    etcd:
+        crt: ENC[AES256_GCM,data:ZXI0AWHjVzDiq3D+/txZX3hNsi8bNx/wO/JtVUQyXWbn0gPEpwQIQ/Ty+/juYeQoxot7Cq3QHWvmIn5A+EwBcercL0xg6mv4NNLWYpKiEGa37/lC5B1C2R2jG52GARVEbrfrTqTLZgaxJYltHpZE5GYahESAJihmh9F5t5/6pPnGn5yQUlv840ETfVQzn6F46dTp88dveokEZsNN9QGPySdJHJLRNIyVHcM/pJ57bqJdtCQlS/P3vBsamiL44WMYLfRDzsTqd/kJojGRfUbyB65J24uutX8h1W5BcaJgVlfSfFbs+WXPW6RQA+4495f+KPjMWQh3XoIIXcoQt1fsKo6ZQ783Z2H49ORkNArx1Io3GHhxSuzV7kI/losHZGr7Qxz4TcsV1/mJBKsCjRttQ6xI6Mps40j7PG8X1yuX3PFjy8rHrww3QZOpjs1dUin+mJ/YMvEEAIY9UdrqeORNByV4lQinLA5jnKGlxKcpVC2xESE3hULVwp+M5zah1XD0JM3hTzdd2ainn8h+RPF2RMULUopxlFnyPCOfyhk1KaXDGtzwZn3dlF1t86WRqZPsWvxe7k+F1g4pq/KCniaroAGn8ABpXW9xC653NvKPJJHk4vKV+PthalUANOao7WWHFTFSJI0ZpFeIUffGrSCwFeQH6q/vYNku+XNF4ujqDqNY7ZvUvP8k9p33HXs2ElBz+Yx9+a/NUFTW+OmIAd1yntggTVT/UFJ7WmVlNnZ7XchvupQmChef4KCgnukYkh10s5tYFQRvEhn5jUjPechRKRCAQZn40YzpcoAlxQcQbdjb53QUlDGleAb0vMBmZojcUXE2z+6dKejTsrs0pS2LXnp4dk+Lfsi0iS/VmGQhw6t94LrSEaP9tFioHJejkRMrNbiZMmU+I+6SsP1YFJd0RGHJsuRVychiGRAsYxXISO1ihlUEAlrSnV7wENhbtLpA/xw307ZemnIHJhDR1j4I00lx9tqqkTXaIc47OT+qD53KZTcW2mY5Qm1z+GlXVaXIjR1I4Q==,iv:35Ctm7TvpQHdDMqC4hOQZwWC0vZxDWwfI/GyrwCl85E=,tag:LI1CIl0tgUEUUqH8AU6flg==,type:str]
+        key: ENC[AES256_GCM,data:J5ocIosLtz1M2HpCdhzXYbxtOCrSaKbAAoBvzC3jazWY6G2SLP4T/z+pgxicYfvO1liiZg6Ehv+hz5/oDk9E9qNOn2BDR1b3Tn1GuQVvmvW0qPK8EFUht23nAARNTpct6binkq1zt4ei/yuFCWVIkPBnzooNb6jupaRML73JHpV24H7EXO6Qx+5z5SUUycmi8Tjb1oaOCp2sXY5dtzAp14qVlMcKh8SKIkCc2iO5sQvuGKnNZAEketHqatocY/24FMAK6TbRkEqJIPsugX56uc5XlcSqrCu2k4am97sprrSWcwgoRfAFHGG6CejquQXK/1RerwGh5R/RpZXX7I4aHcnN1Yeadr9xZpJlLLS8cj/X+wZd1yYbjac56ZMDRM3ihW/XLMi34K05Rz50WKI8Sw==,iv:dOAgaO+5MmXSsZ/75Gg4TG38c3Lp+bfP/g5z9ycmJrQ=,tag:saCtqLXcmWXoKfohRAH8iQ==,type:str]
+    k8s:
+        crt: ENC[AES256_GCM,data:i1r1OtMITZzP50t/5Fr32Q9P1NDhetE9sYmmH3vHCaiF0rELFqXLkB983KljHIE65jKvljDooOwpRy1fQwf1ssvOgOF+PYbb6LQdLlIpy69eygA9FLJvKbntZAcjuUzl7wAS/+80jq4Ra9+YHu7gzXlKsesBLF7Q9tbi72HAC89jVn8S5k8pMwTftdJlRtdkeqeXDahFLHBfCyyD8RmtNIRDzR8LBpQ80dk61WENY/Dn/WN8t8QQ2F0YUmgTYvwzGotGSA7w2q9RmSXiRspTGXwYF2Pi8VSVSJ1u01kFASjHpq94vN4Mt1B0+IayyM1/F5gnjfWIY2ax+bIF5h9lF7GvCmTdRuN0GuYh04OojnvtIXtr5gfoN7qB2GsWn1unpFud83AMdpcsYTEQJRISwR4+RFPWswwX0TU2uEx1gPwRZDgyb9yOPEPQUwnXDeJoJT6ToM+uku4Kka0fMRdtlm34BLOvDoOPFOCakONi3cjiYxOwOd8o9Wdm1ipfOUiApvbJzRY7KCpaH657rdpM8CKEmYJUH43wC5IqOmoIC7H4DS6uTlnovTt2IGK7Pk7npGJQ1vzs3jz/fCv/pBoIImhzZmwUwVfFE7W2BeZHPy3Ka1WyujBKMqtZve3acIKLwatpXxq/Aivdae4DavlsXMPn0L3KpTB2ySwCZ1dz7olzTAF5RzalttWeYdPxIDm+QYjxOVV/YLlFO1X7OwtWyJzr9gDxChk5cKXDkIO0lgVQ3DmoCpE4T95C7x4CxKuoPP94EWiobVgA2fINmrpuaas3qSAa5YB2oOyG7qQbLvJ+YV48btZzLaIWh2tR1KK0S6fE/ZQWLQapY1cbV7oEGPDxjqDalfvhqixPbbWNQRxblczoigjEoy9437inXJvbCnb92UUTrfDqxHdJ2FekpQ5pAo8Ioq9QCH2cnaiLTcixAsjKKVGUnqYtpO5bCes2aemlPjFurl19jRg4SqOW9xm7MjGzhzjp17b+VgfrgHpYrsBNYdNL/Fc5XVqMfRHwJLyWpMzs952eM2H/mmb0hEgSiqS96eJpr4tK5A==,iv:26IsGB31zf8Ml1rb7rdMHFj+8AjIrCwo/GDtOLYZHVw=,tag:AZQ1+wiT98i1MkLrMzXnKQ==,type:str]
+        key: ENC[AES256_GCM,data:R35atBogRh6eeQrm7xDrKT3/4heuVsY1bNcVqyopx7YRnjML7VU56ngfH5rmdHPum0+eabDb0IfukdPtqMti6psAwwlvQdS/epAYaA+sM7HxXZ5YDuaNT3F8tg+cBcNb7UEiD2BDASWdH/ayjuQ3KkDFmGldIIAWsS9EmVAGHvqSziSDgggVuHwkcUULGjFVtfmhuYlGkKaayOhHJO305iwt5xjkQSULMxay4X9k0dJ5WEf6NJNuT2kB60K9mSVdmqMadm3Q3maU4BODQ3RxcSbjhjfjBtLK9yE9zcV5U94mP8kIDwL4kcjUVc+4qjLGcLoOrvkbvRMZWhPIj1dJp9PSdmzhuH18chT3/DMykA7fcpBM9mK1SRKmhDCVGoyz96q9+jSJzNKtsDcB0eGv5Q==,iv:3Dwawk28ccYFO4+x8P3kF7vVnuksIFiwzsAWDpX4sB4=,tag:Sl5uDFN7V+K8NU3N44o1ew==,type:str]
+    k8saggregator:
+        crt: ENC[AES256_GCM,data:s3hlp1K8dmIuyAFU2FGuovi6SUuH35ooFj/yCFtbllwzVuYT8QFwkdRqK1JF+mdrD9QfEO8ik0r0QrapJR87xvubyFt3pb3FRjCluAz/KuU4xTlCYynQ50yXqAkKWiLh7w5N3yaDVLtav8+aXq9j5asLSZJtUT2v6mjuDoH+mbnUD1Pn5kyNTDDvUIT5HPj94aj4YMR81Al7XFMlM0nhRxhOISBzu5nLQYeUDNrx1MVRIjg5pVTXLByHfFLEyfPwz7wmYv0GQ080NHnq0D4Ws3GAzkqnYxiFG7OrLbsu7GTcye2pQmaLSd8ihq/G5rCktggeTQLPWinR2gTVouqNhQWRjIVcVROtbtALGB8kuJAj3dlA26taPGfa8szYNKCOzgMY8V/RNXsbzjpORG8lLmW+C/046/IX+kQE+5s1FeZ7FUlcZOJXki0nsIQDdwYaOWlj9mkTRYxZYjaWQtcahJBZYXvrkLbVAR/aYWQq7pK+lZJc1aDk26tJhvEFc7vDQAlmhJWKfPPRITM3cBXkhu50FuXRbwyq6HmrW8Ht8ExnYDPWX70993W0mBOo/pRfExApMbQyZuRjUjaoHQ1lqAuW0rsTfXOjE/VKWO0jpJxfWzXWFLLAeVNkbl+skz0iophNhNq0vMrgVWMoR5Zat7N+PBsE6WvGsGcantk9ooYtBxNe1ogSmQqT9rygsNKFGn46NDDaUbn8N4MwLg05dfH8BeokXqax50KEGP6DeH8LyPUXgF7rindJgZ+7atLRkloABdpwkDk85uxvX3F3OS/8GKVsLl2VINpuy50F/MF7MXvCm5EVnw/B9pi1ScX3Iypgu1IG2vwaKbHxdvOtZLGYutvBI8YIgkU1GRZ78SXexUL5turG2dQVV5FBa6G5RYBAgLJRwSRAH/x73RABSLva416llwkbJc98TjhoFmTb2PfuyYMYR/ZYl2XCUpXl,iv:nGJR2zH+9v4aC936Y3yfNkVTCh6F1HZ1mNGWZ1e0Bp0=,tag:DdJP6Ytwa3ab7LXnZFXv5w==,type:str]
+        key: ENC[AES256_GCM,data:zSWKIdW1G5ytG7OeYpy/omYYHMvQM88DTgDjC5tXnwolH6JRHG68dNyJA/+9sOlz6Gzy7X2LLnA/+hCH3/lNfBy+H2q+9u8LsBfYIXzh1LtQC008rdygh0BnFF0wrBP6ge6bLyF61WUIGk3hLjj7bgsXOhxAyEgwN1dLIBsUR6MsRJfIlbXNZ2LCN7TgmU+NbhWUxWU2LnpsCpEWB4EJ75RaTseyDBdwGt9SZuC43GkUFUrwCWQpVusXNAtuAB9VvS9Rimtz4OuhuI30BneMyx7xH1XCzItIm+8Kt6ZPDehVKQqvsyq9wfOhUUf77raxUurmMBWz+qQDohVy3mP/4Z8CmybI1+vEXvedV3ncw9NlUQmqre56MOjPAA47shtqcvdOvgSRTDfdDcWRNdsmIg==,iv:zVrm4Rl4pR+qoE6/oESzT5gvaLH7sZPBKuYkm1pU9KM=,tag:aWq+/8orGZfrmUlH0hIf9w==,type:str]
+    k8sserviceaccount:
+        key: ENC[AES256_GCM,data:9Kxsp7UMZ+WpRiRVtwS/qWt9wuWbazM1fx8uD7JPwUP6Pf9s1/KDtMjmHaR2VvXCPAG9QfxjyGCgltlrm6ytVFpP2i9FEEhkhQjdXY2H4ySG1/TjB3oBLkOnhFFXo/SDtSAbQb6AVgsqMMw9dJykL4p6GyqLoSw4FFATnWRAKGi3bRZ7ZxkhAkFCBr99N4FmSU5THVWmHH6e3Yy3eklfegIArgm2OMBxJCb5s3FfsIRFLd86Gw6IgsgVglCo+lRhajab4nxrhxE4nwffpszfSYq2Ux/ZRujEiLZmd+Qizy4j6Zib7t3NQ2ptmgZJr/cRiWKaniNQrhBLADpEXWSMoakVg1R6hE0Zom7YH2tgof+ghkUp939TpOqUKKG1qGqKenxn+iEgsN0vFDyUL7v4Zg==,iv:a5XN2Unipx5JrfpIYMiRzm5q1Nz2iWBlK4eCaVaBsg4=,tag:VsfZfdqeIPAhFk0clqCh/g==,type:str]
+    os:
+        crt: ENC[AES256_GCM,data:47Y6SogDWQygUOmFEbhmrdR7EPL/PnGH6spsH4IyVGJWP1BVBlHipUl3nxQ4yJwhiNbTRdfwqXGv25mIq4r59evNQWlmLxqe95MGua4326djYrBotuo9zp+m65bfstfdZ2OirHs8gdnlpQgfE+m86/y5Ez4rc3W3/QmDvw2cHU5WmFgHP6gxPy8Uv+RZ/5T3p6e0321mLLw0OVTJd14FED1Aw6gnmQ63E5J2WUDwKKqSALemar+DVBwc6HMoPw06i0oB5Mpt1+kwvPNfa0UvJhaoUXK9KYHIP28+pGBqgxqt6zC40xGOBV8DEpAXKDmEQZPsSaOq4cXsWQ33xO5Oj9qxR4HxZlhfD5bmGRBgxO8RE8LcXWJ4p9ecAgitZ1HTARMurAcqy/eYO1Zn7z8bNxzNUY6FT72CE5a/ujH2xDiiS6mJB0fDzD9f58J/P2s1cLC/1h4IE5WKXmsvy6QAXs3Q1m31D3g0CSTQoo6sNAV3s0kHl0ejDwjutZ5x0MIlemrtcN8WigSxRt7jxDpVxiCvb2KcporyGcCNSFzjpGA3HToThTHQ1IE3iES5CauKUHJlerMJ2tmYgRiAf/pVEaqoovjcRnGY5ZB/QaV8svDdM4u6rg2yEV8yhQ/V80jsDRMJ937CiJJiEJJPXC+mAEQYaPx1/REaGHeLWfyH/UtxRbG/jAZT7mnBW/gQXyZwKsbYhKRi9tEMIq0ur/R3rjw9IMBXoS0o6ZpvD69BW1zAou6Ep9CAhXtVXFqrIgagBKjzYf/H4bTmVZzTIfRhUAcjzWkBj9MzoyWOhP4ogLuAraHoD1t/yui8Y08OPDVCotrWPsHhVg/+exBzs654FqYyfzmpG/xthiQ9YbYIgztjI0d4,iv:7jKIr7PMK/k0uv/B/FAxJoW3mJREeENgTSCs80K1mcs=,tag:+ilIFyLKNfE9I3TardZk3Q==,type:str]
+        key: ENC[AES256_GCM,data:duo8abMvBl20XmCmVGEl3E8/f9/vYt9PxW1E7zGoyRA2JR/0FOmqA7lAGer0lJMvdCJE5pNPo4ltT5Naod259ww5z7vVM/XgGY+zPY2UslrjaFxOXd2HFY2t1VEv1fhr9xFHMQ/8aS07nF9vX5tVEnWI+uqDqbvyVL2ecQSOuR0gsADL4+lpyxDoqgqR2ynDTP5CJ96bfEowH9n6O/UeMNm8KAVd3sCN0K4Y4MvpU1AZN9/s,iv:Pw2hjuTWiDhIMeqpC4D050Ykqpd7FwQxH/jkxAn2wJQ=,tag:J/m8Cvko0V4dCx2Ap9pP8g==,type:str]
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age1g786w8t40g9y29l33rfd4jqlwhrgsxsc7ped6uju60k54j0q3enql3kfve
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAxdk0yeUljL0N1eCtJUlNl
+            RUk0Y1V3M1F2WUhFZE9NRENjNDNQdUNSUXpRCk1KbmtlaEJkZnZHS2FaUDVCMVN6
+            Y3lvSWdpaG9vOVRNdUxjS2dibFNXS00KLS0tIDJ4QTE2VXQ5L2JvTTZ5cFB0blZz
+            d1FDb25DWWVkRmJQdDJXRzlDYjI2b1EK88JtK5D39eJ0vFrHf5ba0dEiNcBIT0w0
+            WGOqOa+LUDhZ10Sa2X/z2IewH1hF+qFceEcXTRBjjmHTTUjn1fdNgQ==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2023-09-25T21:31:24Z"
+    mac: ENC[AES256_GCM,data:XwvzArzbdT+S2txA2Cis3mIpn/ncWEo15yGch57vNDjRlw8ZGLrjneHcbWRThmq84gSbsBh2S2tpiROvT+e+iZ62d1rF2RXusDxY/8a7UXo9ckKY1YVcxQploXmbVadw9FFbaiZkCjGTirrf6SHzPDuN8wAKpfZuVPZG3l4CA8I=,iv:bFED8pWnuLRN4oY1/HvYwFEnAZgrAOp0zETn49XNx1A=,tag:dPUKw38HclNjoLEaPid63g==,type:str]
+    pgp: []
+    unencrypted_suffix: _unencrypted
+    version: 3.8.0