From 3c8866251520ff16eb2da63380879f59bfd0661e Mon Sep 17 00:00:00 2001 From: Joseph Hanson Date: Tue, 7 May 2024 19:05:31 -0500 Subject: [PATCH] Swap storageclasses to openebs zfspv --- .../kubernetes}/rook-ceph/kustomization.yaml | 0 .../kubernetes}/rook-ceph/namespace.yaml | 0 .../rook-ceph/rook-ceph/app/helmrelease.yaml | 0 .../rook-ceph/app/kustomization.yaml | 0 ...k-ceph-dashboard-password.secret.sops.yaml | 0 .../rook-ceph/cluster/helmrelease.yaml | 0 .../rook-ceph/cluster/kustomization.yaml | 0 .../kubernetes}/rook-ceph/rook-ceph/ks.yaml | 0 .../VolSync/ReplicationDestination.tmpl.yaml | 2 +- .../cluster/postgrescluster.yaml | 2 +- kubernetes/apps/kube-system/alpine-debug.yml | 20 +++++++++++++++++++ .../media/immich/app/postgresCluster.yaml | 2 +- .../app/helmrelease.yaml | 4 ++-- .../observability/loki/app/helmrelease.yaml | 4 ++-- .../observability/thanos/app/helmrelease.yaml | 2 +- .../openebs/app/helmrelease.yaml | 16 ++++++--------- .../openebs/app/kustomization.yaml | 4 +++- .../openebs/app/storageclass.yaml | 12 +++++++++++ .../openebs/app/volumesnapshotclass.yaml | 10 ++++++++++ kubernetes/bootstrap/talos/talconfig.yaml | 16 ++++++++------- .../flux/repositories/helm/openebs-zfs.yaml | 10 ++++++++++ .../postgres-database/postgresCluster.yaml | 2 +- kubernetes/templates/volsync/claim.yaml | 2 +- kubernetes/templates/volsync/minio.yaml | 12 +++++------ kubernetes/templates/volsync/r2.yaml | 6 +++--- kubernetes/tools/kbench.yaml | 2 +- 26 files changed, 90 insertions(+), 38 deletions(-) rename {kubernetes/apps => .archive/kubernetes}/rook-ceph/kustomization.yaml (100%) rename {kubernetes/apps => .archive/kubernetes}/rook-ceph/namespace.yaml (100%) rename {kubernetes/apps => .archive/kubernetes}/rook-ceph/rook-ceph/app/helmrelease.yaml (100%) rename {kubernetes/apps => .archive/kubernetes}/rook-ceph/rook-ceph/app/kustomization.yaml (100%) rename {kubernetes/apps => .archive/kubernetes}/rook-ceph/rook-ceph/app/rook-ceph-dashboard-password.secret.sops.yaml (100%) rename {kubernetes/apps => .archive/kubernetes}/rook-ceph/rook-ceph/cluster/helmrelease.yaml (100%) rename {kubernetes/apps => .archive/kubernetes}/rook-ceph/rook-ceph/cluster/kustomization.yaml (100%) rename {kubernetes/apps => .archive/kubernetes}/rook-ceph/rook-ceph/ks.yaml (100%) create mode 100644 kubernetes/apps/kube-system/alpine-debug.yml create mode 100644 kubernetes/apps/openebs-system/openebs/app/storageclass.yaml create mode 100644 kubernetes/apps/openebs-system/openebs/app/volumesnapshotclass.yaml create mode 100644 kubernetes/flux/repositories/helm/openebs-zfs.yaml diff --git a/kubernetes/apps/rook-ceph/kustomization.yaml b/.archive/kubernetes/rook-ceph/kustomization.yaml similarity index 100% rename from kubernetes/apps/rook-ceph/kustomization.yaml rename to .archive/kubernetes/rook-ceph/kustomization.yaml diff --git a/kubernetes/apps/rook-ceph/namespace.yaml b/.archive/kubernetes/rook-ceph/namespace.yaml similarity index 100% rename from kubernetes/apps/rook-ceph/namespace.yaml rename to .archive/kubernetes/rook-ceph/namespace.yaml diff --git a/kubernetes/apps/rook-ceph/rook-ceph/app/helmrelease.yaml b/.archive/kubernetes/rook-ceph/rook-ceph/app/helmrelease.yaml similarity index 100% rename from kubernetes/apps/rook-ceph/rook-ceph/app/helmrelease.yaml rename to .archive/kubernetes/rook-ceph/rook-ceph/app/helmrelease.yaml diff --git a/kubernetes/apps/rook-ceph/rook-ceph/app/kustomization.yaml b/.archive/kubernetes/rook-ceph/rook-ceph/app/kustomization.yaml similarity index 100% rename from kubernetes/apps/rook-ceph/rook-ceph/app/kustomization.yaml rename to .archive/kubernetes/rook-ceph/rook-ceph/app/kustomization.yaml diff --git a/kubernetes/apps/rook-ceph/rook-ceph/app/rook-ceph-dashboard-password.secret.sops.yaml b/.archive/kubernetes/rook-ceph/rook-ceph/app/rook-ceph-dashboard-password.secret.sops.yaml similarity index 100% rename from kubernetes/apps/rook-ceph/rook-ceph/app/rook-ceph-dashboard-password.secret.sops.yaml rename to .archive/kubernetes/rook-ceph/rook-ceph/app/rook-ceph-dashboard-password.secret.sops.yaml diff --git a/kubernetes/apps/rook-ceph/rook-ceph/cluster/helmrelease.yaml b/.archive/kubernetes/rook-ceph/rook-ceph/cluster/helmrelease.yaml similarity index 100% rename from kubernetes/apps/rook-ceph/rook-ceph/cluster/helmrelease.yaml rename to .archive/kubernetes/rook-ceph/rook-ceph/cluster/helmrelease.yaml diff --git a/kubernetes/apps/rook-ceph/rook-ceph/cluster/kustomization.yaml b/.archive/kubernetes/rook-ceph/rook-ceph/cluster/kustomization.yaml similarity index 100% rename from kubernetes/apps/rook-ceph/rook-ceph/cluster/kustomization.yaml rename to .archive/kubernetes/rook-ceph/rook-ceph/cluster/kustomization.yaml diff --git a/kubernetes/apps/rook-ceph/rook-ceph/ks.yaml b/.archive/kubernetes/rook-ceph/rook-ceph/ks.yaml similarity index 100% rename from kubernetes/apps/rook-ceph/rook-ceph/ks.yaml rename to .archive/kubernetes/rook-ceph/rook-ceph/ks.yaml diff --git a/.taskfiles/VolSync/ReplicationDestination.tmpl.yaml b/.taskfiles/VolSync/ReplicationDestination.tmpl.yaml index 46be699d..0f7e02c8 100644 --- a/.taskfiles/VolSync/ReplicationDestination.tmpl.yaml +++ b/.taskfiles/VolSync/ReplicationDestination.tmpl.yaml @@ -11,7 +11,7 @@ spec: repository: "${rsrc}-restic-secret" destinationPVC: "${claim}" copyMethod: Direct - storageClassName: ceph-block + storageClassName: openebs-zfs # IMPORTANT NOTE: # Set to the last X number of snapshots to restore from previous: ${previous} diff --git a/kubernetes/apps/database/crunchy-postgres-operator/cluster/postgrescluster.yaml b/kubernetes/apps/database/crunchy-postgres-operator/cluster/postgrescluster.yaml index 188054ac..15b9d76c 100644 --- a/kubernetes/apps/database/crunchy-postgres-operator/cluster/postgrescluster.yaml +++ b/kubernetes/apps/database/crunchy-postgres-operator/cluster/postgrescluster.yaml @@ -41,7 +41,7 @@ spec: app.kubernetes.io/name: crunchy-postgres replicas: &replica 1 dataVolumeClaimSpec: - storageClassName: openebs-hostpath + storageClassName: openebs-zfs accessModes: - ReadWriteOnce resources: diff --git a/kubernetes/apps/kube-system/alpine-debug.yml b/kubernetes/apps/kube-system/alpine-debug.yml new file mode 100644 index 00000000..3b1a44bd --- /dev/null +++ b/kubernetes/apps/kube-system/alpine-debug.yml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Pod +metadata: + name: alpine-debug + namespace: kube-system +spec: + # nodeName: nenya + containers: + - name: alpine-debug + image: alpine + securityContext: + privileged: true + command: ["/bin/ash", "-c", "while true; do sleep 10; done"] + resources: + requests: + cpu: 100m + memory: 512Mi + limits: + cpu: 4000m + memory: 4000Mi diff --git a/kubernetes/apps/media/immich/app/postgresCluster.yaml b/kubernetes/apps/media/immich/app/postgresCluster.yaml index c695d7a0..0cd139c6 100644 --- a/kubernetes/apps/media/immich/app/postgresCluster.yaml +++ b/kubernetes/apps/media/immich/app/postgresCluster.yaml @@ -44,7 +44,7 @@ spec: app.kubernetes.io/name: pgo-${APP} replicas: 1 dataVolumeClaimSpec: - storageClassName: openebs-hostpath + storageClassName: openebs-zfs accessModes: - ReadWriteOnce resources: diff --git a/kubernetes/apps/observability/kube-prometheus-stack/app/helmrelease.yaml b/kubernetes/apps/observability/kube-prometheus-stack/app/helmrelease.yaml index 2dce209e..36d26209 100644 --- a/kubernetes/apps/observability/kube-prometheus-stack/app/helmrelease.yaml +++ b/kubernetes/apps/observability/kube-prometheus-stack/app/helmrelease.yaml @@ -51,7 +51,7 @@ spec: storage: volumeClaimTemplate: spec: - storageClassName: openebs-hostpath + storageClassName: openebs-zfs resources: requests: storage: 1Gi @@ -149,7 +149,7 @@ spec: storageSpec: volumeClaimTemplate: spec: - storageClassName: openebs-hostpath + storageClassName: openebs-zfs resources: requests: storage: 20Gi diff --git a/kubernetes/apps/observability/loki/app/helmrelease.yaml b/kubernetes/apps/observability/loki/app/helmrelease.yaml index 29d379ce..f80ece9c 100644 --- a/kubernetes/apps/observability/loki/app/helmrelease.yaml +++ b/kubernetes/apps/observability/loki/app/helmrelease.yaml @@ -114,7 +114,7 @@ spec: replicas: 1 persistence: size: 20Gi - storageClass: openebs-hostpath + storageClass: openebs-zfs gateway: replicas: 1 image: @@ -135,7 +135,7 @@ spec: replicas: 1 persistence: size: 20Gi - storageClass: openebs-hostpath + storageClass: openebs-zfs sidecar: image: repository: ghcr.io/kiwigrid/k8s-sidecar diff --git a/kubernetes/apps/observability/thanos/app/helmrelease.yaml b/kubernetes/apps/observability/thanos/app/helmrelease.yaml index 70d30a0e..770f1319 100644 --- a/kubernetes/apps/observability/thanos/app/helmrelease.yaml +++ b/kubernetes/apps/observability/thanos/app/helmrelease.yaml @@ -72,7 +72,7 @@ spec: - --retention.resolution-1h=60d persistence: &persistence enabled: true - storageClass: openebs-hostpath + storageClass: openebs-zfs size: 10Gi query: replicas: 1 diff --git a/kubernetes/apps/openebs-system/openebs/app/helmrelease.yaml b/kubernetes/apps/openebs-system/openebs/app/helmrelease.yaml index 5a5e25cd..ea844c7b 100644 --- a/kubernetes/apps/openebs-system/openebs/app/helmrelease.yaml +++ b/kubernetes/apps/openebs-system/openebs/app/helmrelease.yaml @@ -3,16 +3,16 @@ apiVersion: helm.toolkit.fluxcd.io/v2beta2 kind: HelmRelease metadata: - name: openebs + name: openebs-zfs spec: interval: 30m chart: spec: - chart: openebs - version: 3.10.0 + chart: zfs-localpv + version: 2.5.1 sourceRef: kind: HelmRepository - name: openebs + name: openebs-zfs namespace: flux-system install: remediation: @@ -24,9 +24,5 @@ spec: uninstall: keepHistory: false values: - localprovisioner: - hostpathClass: - enabled: true - name: openebs-hostpath - isDefaultClass: false - basePath: /var/openebs/local \ No newline at end of file + zfsNode: + encrKeysDir: /var/openebs/keys diff --git a/kubernetes/apps/openebs-system/openebs/app/kustomization.yaml b/kubernetes/apps/openebs-system/openebs/app/kustomization.yaml index 2d7deaca..0a884217 100644 --- a/kubernetes/apps/openebs-system/openebs/app/kustomization.yaml +++ b/kubernetes/apps/openebs-system/openebs/app/kustomization.yaml @@ -3,4 +3,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - - ./helmrelease.yaml \ No newline at end of file + - ./helmrelease.yaml + - ./storageclass.yaml + - ./volumesnapshotclass.yaml diff --git a/kubernetes/apps/openebs-system/openebs/app/storageclass.yaml b/kubernetes/apps/openebs-system/openebs/app/storageclass.yaml new file mode 100644 index 00000000..bef45453 --- /dev/null +++ b/kubernetes/apps/openebs-system/openebs/app/storageclass.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: openebs-zfs +provisioner: zfs.csi.openebs.io +parameters: + recordsize: "128k" + compression: "off" + dedup: "off" + fstype: "zfs" + poolname: "nahar" diff --git a/kubernetes/apps/openebs-system/openebs/app/volumesnapshotclass.yaml b/kubernetes/apps/openebs-system/openebs/app/volumesnapshotclass.yaml new file mode 100644 index 00000000..4794053e --- /dev/null +++ b/kubernetes/apps/openebs-system/openebs/app/volumesnapshotclass.yaml @@ -0,0 +1,10 @@ +--- +# yaml-language-server: $schema=https://ks.hsn.dev/snapshot.storage.k8s.io/volumesnapshotclass_v1beta1.json +kind: VolumeSnapshotClass +apiVersion: snapshot.storage.k8s.io/v1beta1 +metadata: + name: openebs-zfs + annotations: + snapshot.storage.kubernetes.io/is-default-class: "true" +driver: zfs.csi.openebs.io +deletionPolicy: Delete diff --git a/kubernetes/bootstrap/talos/talconfig.yaml b/kubernetes/bootstrap/talos/talconfig.yaml index 654f3867..48a54bcd 100644 --- a/kubernetes/bootstrap/talos/talconfig.yaml +++ b/kubernetes/bootstrap/talos/talconfig.yaml @@ -3,7 +3,7 @@ clusterName: homelab talosVersion: v1.7.1 -kubernetesVersion: 1.28.4 +kubernetesVersion: 1.30.0 endpoint: "https://10.1.1.57:6443" additionalApiServerCertSans: @@ -18,7 +18,7 @@ nodes: ipAddress: 10.1.1.61 controlPlane: true installDiskSelector: - busPath: /pci0000:20/0000:20:01.2/0000:2d:00.0/nvme/nvme1/nvme1n1 + busPath: /pci0000:20/0000:20:01.2/0000:2c:00.0/nvme/nvme4/nvme4n1 networkInterfaces: - interface: enp37s0f1 dhcp: true @@ -33,9 +33,11 @@ nodes: customization: systemExtensions: officialExtensions: - - "siderolabs/amd-ucode" - - "siderolabs/nonfree-kmod-nvidia" - - "siderolabs/nvidia-container-toolkit" + - siderolabs/amd-ucode + - siderolabs/nonfree-kmod-nvidia + - siderolabs/nvidia-container-toolkit + - siderolabs/zfs + patches: - |- machine: @@ -134,12 +136,12 @@ controlPlane: validSubnets: - 10.1.1.0/24 extraMounts: - - destination: /var/openebs/local + - destination: /var/openebs/keys options: - bind - rshared - rw - source: /var/openebs/local + source: /var/openebs/keys type: bind # Custom sysctls diff --git a/kubernetes/flux/repositories/helm/openebs-zfs.yaml b/kubernetes/flux/repositories/helm/openebs-zfs.yaml new file mode 100644 index 00000000..08657de3 --- /dev/null +++ b/kubernetes/flux/repositories/helm/openebs-zfs.yaml @@ -0,0 +1,10 @@ +--- +# yaml-language-server: $schema=https://ks.hsn.dev/source.toolkit.fluxcd.io/helmrepository_v1beta2.json +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: HelmRepository +metadata: + name: openebs-zfs + namespace: flux-system +spec: + interval: 2h + url: https://openebs.github.io/zfs-localpv diff --git a/kubernetes/templates/postgres-database/postgresCluster.yaml b/kubernetes/templates/postgres-database/postgresCluster.yaml index 2957b83f..dab72c1c 100644 --- a/kubernetes/templates/postgres-database/postgresCluster.yaml +++ b/kubernetes/templates/postgres-database/postgresCluster.yaml @@ -36,7 +36,7 @@ spec: app.kubernetes.io/name: pgo-${APP} replicas: 1 dataVolumeClaimSpec: - storageClassName: openebs-hostpath + storageClassName: openebs-zfs accessModes: - ReadWriteOnce resources: diff --git a/kubernetes/templates/volsync/claim.yaml b/kubernetes/templates/volsync/claim.yaml index 1f2d9e16..34346faf 100644 --- a/kubernetes/templates/volsync/claim.yaml +++ b/kubernetes/templates/volsync/claim.yaml @@ -12,4 +12,4 @@ spec: resources: requests: storage: "${VOLSYNC_CAPACITY}" - storageClassName: "${VOLSYNC_STORAGECLASS:-ceph-block}" + storageClassName: "${VOLSYNC_STORAGECLASS:-openebs-zfs}" diff --git a/kubernetes/templates/volsync/minio.yaml b/kubernetes/templates/volsync/minio.yaml index 5af4c633..c2c96576 100644 --- a/kubernetes/templates/volsync/minio.yaml +++ b/kubernetes/templates/volsync/minio.yaml @@ -36,11 +36,11 @@ spec: copyMethod: "${VOLSYNC_COPYMETHOD:-Snapshot}" pruneIntervalDays: 7 repository: "${APP}-volsync-secret" - volumeSnapshotClassName: "${VOLSYNC_SNAPSHOTCLASS:-csi-ceph-blockpool}" + volumeSnapshotClassName: "${VOLSYNC_SNAPSHOTCLASS:-openebs-zfs}" cacheCapacity: "${VOLSYNC_CACHE_CAPACITY:-4Gi}" - cacheStorageClassName: "${VOLSYNC_CACHE_SNAPSHOTCLASS:-openebs-hostpath}" + cacheStorageClassName: "${VOLSYNC_CACHE_SNAPSHOTCLASS:-openebs-zfs}" cacheAccessModes: ["${VOLSYNC_CACHE_ACCESSMODES:-ReadWriteOnce}"] - storageClassName: "${VOLSYNC_STORAGECLASS:-ceph-block}" + storageClassName: "${VOLSYNC_STORAGECLASS:-openebs-zfs}" accessModes: ["${VOLSYNC_ACCESSMODES:-ReadWriteOnce}"] moverSecurityContext: runAsUser: ${APP_UID:-568} @@ -62,11 +62,11 @@ spec: restic: repository: "${APP}-volsync-secret" copyMethod: Snapshot # must be Snapshot - volumeSnapshotClassName: "${VOLSYNC_SNAPSHOTCLASS:-csi-ceph-blockpool}" - cacheStorageClassName: "${VOLSYNC_CACHE_SNAPSHOTCLASS:-openebs-hostpath}" + volumeSnapshotClassName: "${VOLSYNC_SNAPSHOTCLASS:-openebs-zfs}" + cacheStorageClassName: "${VOLSYNC_CACHE_SNAPSHOTCLASS:-openebs-zfs}" cacheAccessModes: ["${VOLSYNC_CACHE_ACCESSMODES:-ReadWriteOnce}"] cacheCapacity: "${VOLSYNC_CACHE_CAPACITY:-4Gi}" - storageClassName: "${VOLSYNC_STORAGECLASS:-ceph-block}" + storageClassName: "${VOLSYNC_STORAGECLASS:-openebs-zfs}" accessModes: ["${VOLSYNC_ACCESSMODES:-ReadWriteOnce}"] capacity: "${VOLSYNC_CAPACITY}" moverSecurityContext: diff --git a/kubernetes/templates/volsync/r2.yaml b/kubernetes/templates/volsync/r2.yaml index 18c3d6f8..e71e3828 100644 --- a/kubernetes/templates/volsync/r2.yaml +++ b/kubernetes/templates/volsync/r2.yaml @@ -36,11 +36,11 @@ spec: copyMethod: "${VOLSYNC_COPYMETHOD:-Snapshot}" pruneIntervalDays: 7 repository: "${APP}-volsync-r2-secret" - volumeSnapshotClassName: "${VOLSYNC_SNAPSHOTCLASS:-csi-ceph-blockpool}" + volumeSnapshotClassName: "${VOLSYNC_SNAPSHOTCLASS:-openebs-zfs}" cacheCapacity: "${VOLSYNC_CACHE_CAPACITY:-4Gi}" - cacheStorageClassName: "${VOLSYNC_CACHE_SNAPSHOTCLASS:-openebs-hostpath}" + cacheStorageClassName: "${VOLSYNC_CACHE_SNAPSHOTCLASS:-openebs-zfs}" cacheAccessModes: ["${VOLSYNC_CACHE_ACCESSMODES:-ReadWriteOnce}"] - storageClassName: "${VOLSYNC_STORAGECLASS:-ceph-block}" + storageClassName: "${VOLSYNC_STORAGECLASS:-openebs-zfs}" accessModes: ["${VOLSYNC_ACCESSMODES:-ReadWriteOnce}"] moverSecurityContext: runAsUser: ${APP_UID:-568} diff --git a/kubernetes/tools/kbench.yaml b/kubernetes/tools/kbench.yaml index fa201f80..6d48f152 100644 --- a/kubernetes/tools/kbench.yaml +++ b/kubernetes/tools/kbench.yaml @@ -4,7 +4,7 @@ apiVersion: v1 metadata: name: kbench-pvc spec: - storageClassName: ceph-block + storageClassName: openebs-zfs accessModes: - ReadWriteOnce resources: