Merge pull request 'Swap storageclasses to openebs zfspv' (#270) from swap-to-zfs into main

Reviewed-on: jahanson/homelab#270

.taskfiles/VolSync/ReplicationDestination.tmpl.yaml

@@ -11,7 +11,7 @@ spec:
     repository: "${rsrc}-restic-secret"
     destinationPVC: "${claim}"
     copyMethod: Direct
-    storageClassName: ceph-block
+    storageClassName: openebs-zfs
     # IMPORTANT NOTE:
     #   Set to the last X number of snapshots to restore from
     previous: ${previous}

kubernetes/apps/database/crunchy-postgres-operator/cluster/postgrescluster.yaml

@@ -41,7 +41,7 @@ spec:
           app.kubernetes.io/name: crunchy-postgres
       replicas: &replica 1
       dataVolumeClaimSpec:
-        storageClassName: openebs-hostpath
+        storageClassName: openebs-zfs
         accessModes:
           - ReadWriteOnce
         resources:

kubernetes/apps/kube-system/alpine-debug.yml

@@ -0,0 +1,20 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: alpine-debug
+  namespace: kube-system
+spec:
+  #  nodeName: nenya
+  containers:
+    - name: alpine-debug
+      image: alpine
+      securityContext:
+        privileged: true
+      command: ["/bin/ash", "-c", "while true; do sleep 10; done"]
+      resources:
+        requests:
+          cpu: 100m
+          memory: 512Mi
+        limits:
+          cpu: 4000m
+          memory: 4000Mi

kubernetes/apps/media/immich/app/postgresCluster.yaml

@@ -44,7 +44,7 @@ spec:
           app.kubernetes.io/name: pgo-${APP}
       replicas: 1
       dataVolumeClaimSpec:
-        storageClassName: openebs-hostpath
+        storageClassName: openebs-zfs
         accessModes:
           - ReadWriteOnce
         resources:

kubernetes/apps/observability/kube-prometheus-stack/app/helmrelease.yaml

@@ -51,7 +51,7 @@ spec:
         storage:
           volumeClaimTemplate:
             spec:
-              storageClassName: openebs-hostpath
+              storageClassName: openebs-zfs
               resources:
                 requests:
                   storage: 1Gi
@@ -149,7 +149,7 @@ spec:
         storageSpec:
           volumeClaimTemplate:
             spec:
-              storageClassName: openebs-hostpath
+              storageClassName: openebs-zfs
               resources:
                 requests:
                   storage: 20Gi

kubernetes/apps/observability/loki/app/helmrelease.yaml

@@ -114,7 +114,7 @@ spec:
       replicas: 1
       persistence:
         size: 20Gi
-        storageClass: openebs-hostpath
+        storageClass: openebs-zfs
     gateway:
       replicas: 1
       image:
@@ -135,7 +135,7 @@ spec:
       replicas: 1
       persistence:
         size: 20Gi
-        storageClass: openebs-hostpath
+        storageClass: openebs-zfs
     sidecar:
       image:
         repository: ghcr.io/kiwigrid/k8s-sidecar

kubernetes/apps/observability/thanos/app/helmrelease.yaml

@@ -72,7 +72,7 @@ spec:
         - --retention.resolution-1h=60d
       persistence: &persistence
         enabled: true
-        storageClass: openebs-hostpath
+        storageClass: openebs-zfs
         size: 10Gi
     query:
       replicas: 1

kubernetes/apps/openebs-system/openebs/app/helmrelease.yaml

@@ -3,16 +3,16 @@
 apiVersion: helm.toolkit.fluxcd.io/v2beta2
 kind: HelmRelease
 metadata:
-  name: openebs
+  name: openebs-zfs
 spec:
   interval: 30m
   chart:
     spec:
-      chart: openebs
+      chart: zfs-localpv
-      version: 3.10.0
+      version: 2.5.1
       sourceRef:
         kind: HelmRepository
-        name: openebs
+        name: openebs-zfs
         namespace: flux-system
   install:
     remediation:
@@ -24,9 +24,5 @@ spec:
   uninstall:
     keepHistory: false
   values:
-    localprovisioner:
+    zfsNode:
-      hostpathClass:
+      encrKeysDir: /var/openebs/keys
-        enabled: true
-        name: openebs-hostpath
-        isDefaultClass: false
-        basePath: /var/openebs/local

kubernetes/apps/openebs-system/openebs/app/kustomization.yaml

@@ -4,3 +4,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
   - ./helmrelease.yaml
+  - ./storageclass.yaml
+  - ./volumesnapshotclass.yaml

kubernetes/apps/openebs-system/openebs/app/storageclass.yaml

@@ -0,0 +1,12 @@
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: openebs-zfs
+provisioner: zfs.csi.openebs.io
+parameters:
+  recordsize: "128k"
+  compression: "off"
+  dedup: "off"
+  fstype: "zfs"
+  poolname: "nahar"

kubernetes/apps/openebs-system/openebs/app/volumesnapshotclass.yaml

@@ -0,0 +1,10 @@
+---
+# yaml-language-server: $schema=https://ks.hsn.dev/snapshot.storage.k8s.io/volumesnapshotclass_v1beta1.json
+kind: VolumeSnapshotClass
+apiVersion: snapshot.storage.k8s.io/v1beta1
+metadata:
+  name: openebs-zfs
+  annotations:
+    snapshot.storage.kubernetes.io/is-default-class: "true"
+driver: zfs.csi.openebs.io
+deletionPolicy: Delete

kubernetes/bootstrap/talos/talconfig.yaml

@@ -3,7 +3,7 @@
 clusterName: homelab
 
 talosVersion: v1.7.1
-kubernetesVersion: 1.28.4
+kubernetesVersion: 1.30.0
 endpoint: "https://10.1.1.57:6443"
 
 additionalApiServerCertSans:
@@ -18,7 +18,7 @@ nodes:
     ipAddress: 10.1.1.61
     controlPlane: true
     installDiskSelector:
-      busPath: /pci0000:20/0000:20:01.2/0000:2d:00.0/nvme/nvme1/nvme1n1
+      busPath: /pci0000:20/0000:20:01.2/0000:2c:00.0/nvme/nvme4/nvme4n1
     networkInterfaces:
       - interface: enp37s0f1
         dhcp: true
@@ -33,9 +33,11 @@ nodes:
       customization:
         systemExtensions:
           officialExtensions:
-            - "siderolabs/amd-ucode"
+            - siderolabs/amd-ucode
-            - "siderolabs/nonfree-kmod-nvidia"
+            - siderolabs/nonfree-kmod-nvidia
-            - "siderolabs/nvidia-container-toolkit"
+            - siderolabs/nvidia-container-toolkit
+            - siderolabs/zfs
+
     patches:
       - |-
         machine:
@@ -134,12 +136,12 @@ controlPlane:
             validSubnets:
                 - 10.1.1.0/24
           extraMounts:
-            - destination: /var/openebs/local
+            - destination: /var/openebs/keys
               options:
                 - bind
                 - rshared
                 - rw
-              source: /var/openebs/local
+              source: /var/openebs/keys
               type: bind
 
     # Custom sysctls

kubernetes/flux/repositories/helm/openebs-zfs.yaml

@@ -0,0 +1,10 @@
+---
+# yaml-language-server: $schema=https://ks.hsn.dev/source.toolkit.fluxcd.io/helmrepository_v1beta2.json
+apiVersion: source.toolkit.fluxcd.io/v1beta2
+kind: HelmRepository
+metadata:
+  name: openebs-zfs
+  namespace: flux-system
+spec:
+  interval: 2h
+  url: https://openebs.github.io/zfs-localpv

kubernetes/templates/postgres-database/postgresCluster.yaml

@@ -36,7 +36,7 @@ spec:
           app.kubernetes.io/name: pgo-${APP}
       replicas: 1
       dataVolumeClaimSpec:
-        storageClassName: openebs-hostpath
+        storageClassName: openebs-zfs
         accessModes:
           - ReadWriteOnce
         resources:

kubernetes/templates/volsync/claim.yaml

@@ -12,4 +12,4 @@ spec:
   resources:
     requests:
       storage: "${VOLSYNC_CAPACITY}"
-  storageClassName: "${VOLSYNC_STORAGECLASS:-ceph-block}"
+  storageClassName: "${VOLSYNC_STORAGECLASS:-openebs-zfs}"

kubernetes/templates/volsync/minio.yaml

@@ -36,11 +36,11 @@ spec:
     copyMethod: "${VOLSYNC_COPYMETHOD:-Snapshot}"
     pruneIntervalDays: 7
     repository: "${APP}-volsync-secret"
-    volumeSnapshotClassName: "${VOLSYNC_SNAPSHOTCLASS:-csi-ceph-blockpool}"
+    volumeSnapshotClassName: "${VOLSYNC_SNAPSHOTCLASS:-openebs-zfs}"
     cacheCapacity: "${VOLSYNC_CACHE_CAPACITY:-4Gi}"
-    cacheStorageClassName: "${VOLSYNC_CACHE_SNAPSHOTCLASS:-openebs-hostpath}"
+    cacheStorageClassName: "${VOLSYNC_CACHE_SNAPSHOTCLASS:-openebs-zfs}"
     cacheAccessModes: ["${VOLSYNC_CACHE_ACCESSMODES:-ReadWriteOnce}"]
-    storageClassName: "${VOLSYNC_STORAGECLASS:-ceph-block}"
+    storageClassName: "${VOLSYNC_STORAGECLASS:-openebs-zfs}"
     accessModes: ["${VOLSYNC_ACCESSMODES:-ReadWriteOnce}"]
     moverSecurityContext:
       runAsUser: ${APP_UID:-568}
@@ -62,11 +62,11 @@ spec:
   restic:
     repository: "${APP}-volsync-secret"
     copyMethod: Snapshot # must be Snapshot
-    volumeSnapshotClassName: "${VOLSYNC_SNAPSHOTCLASS:-csi-ceph-blockpool}"
+    volumeSnapshotClassName: "${VOLSYNC_SNAPSHOTCLASS:-openebs-zfs}"
-    cacheStorageClassName: "${VOLSYNC_CACHE_SNAPSHOTCLASS:-openebs-hostpath}"
+    cacheStorageClassName: "${VOLSYNC_CACHE_SNAPSHOTCLASS:-openebs-zfs}"
     cacheAccessModes: ["${VOLSYNC_CACHE_ACCESSMODES:-ReadWriteOnce}"]
     cacheCapacity: "${VOLSYNC_CACHE_CAPACITY:-4Gi}"
-    storageClassName: "${VOLSYNC_STORAGECLASS:-ceph-block}"
+    storageClassName: "${VOLSYNC_STORAGECLASS:-openebs-zfs}"
     accessModes: ["${VOLSYNC_ACCESSMODES:-ReadWriteOnce}"]
     capacity: "${VOLSYNC_CAPACITY}"
     moverSecurityContext:

kubernetes/templates/volsync/r2.yaml

@@ -36,11 +36,11 @@ spec:
     copyMethod: "${VOLSYNC_COPYMETHOD:-Snapshot}"
     pruneIntervalDays: 7
     repository: "${APP}-volsync-r2-secret"
-    volumeSnapshotClassName: "${VOLSYNC_SNAPSHOTCLASS:-csi-ceph-blockpool}"
+    volumeSnapshotClassName: "${VOLSYNC_SNAPSHOTCLASS:-openebs-zfs}"
     cacheCapacity: "${VOLSYNC_CACHE_CAPACITY:-4Gi}"
-    cacheStorageClassName: "${VOLSYNC_CACHE_SNAPSHOTCLASS:-openebs-hostpath}"
+    cacheStorageClassName: "${VOLSYNC_CACHE_SNAPSHOTCLASS:-openebs-zfs}"
     cacheAccessModes: ["${VOLSYNC_CACHE_ACCESSMODES:-ReadWriteOnce}"]
-    storageClassName: "${VOLSYNC_STORAGECLASS:-ceph-block}"
+    storageClassName: "${VOLSYNC_STORAGECLASS:-openebs-zfs}"
     accessModes: ["${VOLSYNC_ACCESSMODES:-ReadWriteOnce}"]
     moverSecurityContext:
       runAsUser: ${APP_UID:-568}

kubernetes/tools/kbench.yaml

@@ -4,7 +4,7 @@ apiVersion: v1
 metadata:
   name: kbench-pvc
 spec:
-  storageClassName: ceph-block
+  storageClassName: openebs-zfs
   accessModes:
     - ReadWriteOnce
   resources: