diff --git a/kubernetes/apps/rook-ceph/kustomization.yaml b/.archive/kubernetes/rook-ceph/kustomization.yaml similarity index 100% rename from kubernetes/apps/rook-ceph/kustomization.yaml rename to .archive/kubernetes/rook-ceph/kustomization.yaml diff --git a/kubernetes/apps/rook-ceph/namespace.yaml b/.archive/kubernetes/rook-ceph/namespace.yaml similarity index 100% rename from kubernetes/apps/rook-ceph/namespace.yaml rename to .archive/kubernetes/rook-ceph/namespace.yaml diff --git a/kubernetes/apps/rook-ceph/rook-ceph/app/helmrelease.yaml b/.archive/kubernetes/rook-ceph/rook-ceph/app/helmrelease.yaml similarity index 100% rename from kubernetes/apps/rook-ceph/rook-ceph/app/helmrelease.yaml rename to .archive/kubernetes/rook-ceph/rook-ceph/app/helmrelease.yaml diff --git a/kubernetes/apps/rook-ceph/rook-ceph/app/kustomization.yaml b/.archive/kubernetes/rook-ceph/rook-ceph/app/kustomization.yaml similarity index 100% rename from kubernetes/apps/rook-ceph/rook-ceph/app/kustomization.yaml rename to .archive/kubernetes/rook-ceph/rook-ceph/app/kustomization.yaml diff --git a/kubernetes/apps/rook-ceph/rook-ceph/app/rook-ceph-dashboard-password.secret.sops.yaml b/.archive/kubernetes/rook-ceph/rook-ceph/app/rook-ceph-dashboard-password.secret.sops.yaml similarity index 100% rename from kubernetes/apps/rook-ceph/rook-ceph/app/rook-ceph-dashboard-password.secret.sops.yaml rename to .archive/kubernetes/rook-ceph/rook-ceph/app/rook-ceph-dashboard-password.secret.sops.yaml diff --git a/kubernetes/apps/rook-ceph/rook-ceph/cluster/helmrelease.yaml b/.archive/kubernetes/rook-ceph/rook-ceph/cluster/helmrelease.yaml similarity index 100% rename from kubernetes/apps/rook-ceph/rook-ceph/cluster/helmrelease.yaml rename to .archive/kubernetes/rook-ceph/rook-ceph/cluster/helmrelease.yaml diff --git a/kubernetes/apps/rook-ceph/rook-ceph/cluster/kustomization.yaml b/.archive/kubernetes/rook-ceph/rook-ceph/cluster/kustomization.yaml similarity index 100% rename from kubernetes/apps/rook-ceph/rook-ceph/cluster/kustomization.yaml rename to .archive/kubernetes/rook-ceph/rook-ceph/cluster/kustomization.yaml diff --git a/kubernetes/apps/rook-ceph/rook-ceph/ks.yaml b/.archive/kubernetes/rook-ceph/rook-ceph/ks.yaml similarity index 100% rename from kubernetes/apps/rook-ceph/rook-ceph/ks.yaml rename to .archive/kubernetes/rook-ceph/rook-ceph/ks.yaml diff --git a/.taskfiles/VolSync/ReplicationDestination.tmpl.yaml b/.taskfiles/VolSync/ReplicationDestination.tmpl.yaml index 46be699..0f7e02c 100644 --- a/.taskfiles/VolSync/ReplicationDestination.tmpl.yaml +++ b/.taskfiles/VolSync/ReplicationDestination.tmpl.yaml @@ -11,7 +11,7 @@ spec: repository: "${rsrc}-restic-secret" destinationPVC: "${claim}" copyMethod: Direct - storageClassName: ceph-block + storageClassName: openebs-zfs # IMPORTANT NOTE: # Set to the last X number of snapshots to restore from previous: ${previous} diff --git a/kubernetes/apps/database/crunchy-postgres-operator/cluster/postgrescluster.yaml b/kubernetes/apps/database/crunchy-postgres-operator/cluster/postgrescluster.yaml index 188054a..15b9d76 100644 --- a/kubernetes/apps/database/crunchy-postgres-operator/cluster/postgrescluster.yaml +++ b/kubernetes/apps/database/crunchy-postgres-operator/cluster/postgrescluster.yaml @@ -41,7 +41,7 @@ spec: app.kubernetes.io/name: crunchy-postgres replicas: &replica 1 dataVolumeClaimSpec: - storageClassName: openebs-hostpath + storageClassName: openebs-zfs accessModes: - ReadWriteOnce resources: diff --git a/kubernetes/apps/kube-system/alpine-debug.yml b/kubernetes/apps/kube-system/alpine-debug.yml new file mode 100644 index 0000000..3b1a44b --- /dev/null +++ b/kubernetes/apps/kube-system/alpine-debug.yml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Pod +metadata: + name: alpine-debug + namespace: kube-system +spec: + # nodeName: nenya + containers: + - name: alpine-debug + image: alpine + securityContext: + privileged: true + command: ["/bin/ash", "-c", "while true; do sleep 10; done"] + resources: + requests: + cpu: 100m + memory: 512Mi + limits: + cpu: 4000m + memory: 4000Mi diff --git a/kubernetes/apps/media/immich/app/postgresCluster.yaml b/kubernetes/apps/media/immich/app/postgresCluster.yaml index c695d7a..0cd139c 100644 --- a/kubernetes/apps/media/immich/app/postgresCluster.yaml +++ b/kubernetes/apps/media/immich/app/postgresCluster.yaml @@ -44,7 +44,7 @@ spec: app.kubernetes.io/name: pgo-${APP} replicas: 1 dataVolumeClaimSpec: - storageClassName: openebs-hostpath + storageClassName: openebs-zfs accessModes: - ReadWriteOnce resources: diff --git a/kubernetes/apps/observability/kube-prometheus-stack/app/helmrelease.yaml b/kubernetes/apps/observability/kube-prometheus-stack/app/helmrelease.yaml index 2dce209..36d2620 100644 --- a/kubernetes/apps/observability/kube-prometheus-stack/app/helmrelease.yaml +++ b/kubernetes/apps/observability/kube-prometheus-stack/app/helmrelease.yaml @@ -51,7 +51,7 @@ spec: storage: volumeClaimTemplate: spec: - storageClassName: openebs-hostpath + storageClassName: openebs-zfs resources: requests: storage: 1Gi @@ -149,7 +149,7 @@ spec: storageSpec: volumeClaimTemplate: spec: - storageClassName: openebs-hostpath + storageClassName: openebs-zfs resources: requests: storage: 20Gi diff --git a/kubernetes/apps/observability/loki/app/helmrelease.yaml b/kubernetes/apps/observability/loki/app/helmrelease.yaml index 29d379c..f80ece9 100644 --- a/kubernetes/apps/observability/loki/app/helmrelease.yaml +++ b/kubernetes/apps/observability/loki/app/helmrelease.yaml @@ -114,7 +114,7 @@ spec: replicas: 1 persistence: size: 20Gi - storageClass: openebs-hostpath + storageClass: openebs-zfs gateway: replicas: 1 image: @@ -135,7 +135,7 @@ spec: replicas: 1 persistence: size: 20Gi - storageClass: openebs-hostpath + storageClass: openebs-zfs sidecar: image: repository: ghcr.io/kiwigrid/k8s-sidecar diff --git a/kubernetes/apps/observability/thanos/app/helmrelease.yaml b/kubernetes/apps/observability/thanos/app/helmrelease.yaml index 70d30a0..770f131 100644 --- a/kubernetes/apps/observability/thanos/app/helmrelease.yaml +++ b/kubernetes/apps/observability/thanos/app/helmrelease.yaml @@ -72,7 +72,7 @@ spec: - --retention.resolution-1h=60d persistence: &persistence enabled: true - storageClass: openebs-hostpath + storageClass: openebs-zfs size: 10Gi query: replicas: 1 diff --git a/kubernetes/apps/openebs-system/openebs/app/helmrelease.yaml b/kubernetes/apps/openebs-system/openebs/app/helmrelease.yaml index 5a5e25c..ea844c7 100644 --- a/kubernetes/apps/openebs-system/openebs/app/helmrelease.yaml +++ b/kubernetes/apps/openebs-system/openebs/app/helmrelease.yaml @@ -3,16 +3,16 @@ apiVersion: helm.toolkit.fluxcd.io/v2beta2 kind: HelmRelease metadata: - name: openebs + name: openebs-zfs spec: interval: 30m chart: spec: - chart: openebs - version: 3.10.0 + chart: zfs-localpv + version: 2.5.1 sourceRef: kind: HelmRepository - name: openebs + name: openebs-zfs namespace: flux-system install: remediation: @@ -24,9 +24,5 @@ spec: uninstall: keepHistory: false values: - localprovisioner: - hostpathClass: - enabled: true - name: openebs-hostpath - isDefaultClass: false - basePath: /var/openebs/local \ No newline at end of file + zfsNode: + encrKeysDir: /var/openebs/keys diff --git a/kubernetes/apps/openebs-system/openebs/app/kustomization.yaml b/kubernetes/apps/openebs-system/openebs/app/kustomization.yaml index 2d7deac..0a88421 100644 --- a/kubernetes/apps/openebs-system/openebs/app/kustomization.yaml +++ b/kubernetes/apps/openebs-system/openebs/app/kustomization.yaml @@ -3,4 +3,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - - ./helmrelease.yaml \ No newline at end of file + - ./helmrelease.yaml + - ./storageclass.yaml + - ./volumesnapshotclass.yaml diff --git a/kubernetes/apps/openebs-system/openebs/app/storageclass.yaml b/kubernetes/apps/openebs-system/openebs/app/storageclass.yaml new file mode 100644 index 0000000..bef4545 --- /dev/null +++ b/kubernetes/apps/openebs-system/openebs/app/storageclass.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: openebs-zfs +provisioner: zfs.csi.openebs.io +parameters: + recordsize: "128k" + compression: "off" + dedup: "off" + fstype: "zfs" + poolname: "nahar" diff --git a/kubernetes/apps/openebs-system/openebs/app/volumesnapshotclass.yaml b/kubernetes/apps/openebs-system/openebs/app/volumesnapshotclass.yaml new file mode 100644 index 0000000..4794053 --- /dev/null +++ b/kubernetes/apps/openebs-system/openebs/app/volumesnapshotclass.yaml @@ -0,0 +1,10 @@ +--- +# yaml-language-server: $schema=https://ks.hsn.dev/snapshot.storage.k8s.io/volumesnapshotclass_v1beta1.json +kind: VolumeSnapshotClass +apiVersion: snapshot.storage.k8s.io/v1beta1 +metadata: + name: openebs-zfs + annotations: + snapshot.storage.kubernetes.io/is-default-class: "true" +driver: zfs.csi.openebs.io +deletionPolicy: Delete diff --git a/kubernetes/bootstrap/talos/talconfig.yaml b/kubernetes/bootstrap/talos/talconfig.yaml index 654f386..48a54bc 100644 --- a/kubernetes/bootstrap/talos/talconfig.yaml +++ b/kubernetes/bootstrap/talos/talconfig.yaml @@ -3,7 +3,7 @@ clusterName: homelab talosVersion: v1.7.1 -kubernetesVersion: 1.28.4 +kubernetesVersion: 1.30.0 endpoint: "https://10.1.1.57:6443" additionalApiServerCertSans: @@ -18,7 +18,7 @@ nodes: ipAddress: 10.1.1.61 controlPlane: true installDiskSelector: - busPath: /pci0000:20/0000:20:01.2/0000:2d:00.0/nvme/nvme1/nvme1n1 + busPath: /pci0000:20/0000:20:01.2/0000:2c:00.0/nvme/nvme4/nvme4n1 networkInterfaces: - interface: enp37s0f1 dhcp: true @@ -33,9 +33,11 @@ nodes: customization: systemExtensions: officialExtensions: - - "siderolabs/amd-ucode" - - "siderolabs/nonfree-kmod-nvidia" - - "siderolabs/nvidia-container-toolkit" + - siderolabs/amd-ucode + - siderolabs/nonfree-kmod-nvidia + - siderolabs/nvidia-container-toolkit + - siderolabs/zfs + patches: - |- machine: @@ -134,12 +136,12 @@ controlPlane: validSubnets: - 10.1.1.0/24 extraMounts: - - destination: /var/openebs/local + - destination: /var/openebs/keys options: - bind - rshared - rw - source: /var/openebs/local + source: /var/openebs/keys type: bind # Custom sysctls diff --git a/kubernetes/flux/repositories/helm/openebs-zfs.yaml b/kubernetes/flux/repositories/helm/openebs-zfs.yaml new file mode 100644 index 0000000..08657de --- /dev/null +++ b/kubernetes/flux/repositories/helm/openebs-zfs.yaml @@ -0,0 +1,10 @@ +--- +# yaml-language-server: $schema=https://ks.hsn.dev/source.toolkit.fluxcd.io/helmrepository_v1beta2.json +apiVersion: source.toolkit.fluxcd.io/v1beta2 +kind: HelmRepository +metadata: + name: openebs-zfs + namespace: flux-system +spec: + interval: 2h + url: https://openebs.github.io/zfs-localpv diff --git a/kubernetes/templates/postgres-database/postgresCluster.yaml b/kubernetes/templates/postgres-database/postgresCluster.yaml index 2957b83..dab72c1 100644 --- a/kubernetes/templates/postgres-database/postgresCluster.yaml +++ b/kubernetes/templates/postgres-database/postgresCluster.yaml @@ -36,7 +36,7 @@ spec: app.kubernetes.io/name: pgo-${APP} replicas: 1 dataVolumeClaimSpec: - storageClassName: openebs-hostpath + storageClassName: openebs-zfs accessModes: - ReadWriteOnce resources: diff --git a/kubernetes/templates/volsync/claim.yaml b/kubernetes/templates/volsync/claim.yaml index 1f2d9e1..34346fa 100644 --- a/kubernetes/templates/volsync/claim.yaml +++ b/kubernetes/templates/volsync/claim.yaml @@ -12,4 +12,4 @@ spec: resources: requests: storage: "${VOLSYNC_CAPACITY}" - storageClassName: "${VOLSYNC_STORAGECLASS:-ceph-block}" + storageClassName: "${VOLSYNC_STORAGECLASS:-openebs-zfs}" diff --git a/kubernetes/templates/volsync/minio.yaml b/kubernetes/templates/volsync/minio.yaml index 5af4c63..c2c9657 100644 --- a/kubernetes/templates/volsync/minio.yaml +++ b/kubernetes/templates/volsync/minio.yaml @@ -36,11 +36,11 @@ spec: copyMethod: "${VOLSYNC_COPYMETHOD:-Snapshot}" pruneIntervalDays: 7 repository: "${APP}-volsync-secret" - volumeSnapshotClassName: "${VOLSYNC_SNAPSHOTCLASS:-csi-ceph-blockpool}" + volumeSnapshotClassName: "${VOLSYNC_SNAPSHOTCLASS:-openebs-zfs}" cacheCapacity: "${VOLSYNC_CACHE_CAPACITY:-4Gi}" - cacheStorageClassName: "${VOLSYNC_CACHE_SNAPSHOTCLASS:-openebs-hostpath}" + cacheStorageClassName: "${VOLSYNC_CACHE_SNAPSHOTCLASS:-openebs-zfs}" cacheAccessModes: ["${VOLSYNC_CACHE_ACCESSMODES:-ReadWriteOnce}"] - storageClassName: "${VOLSYNC_STORAGECLASS:-ceph-block}" + storageClassName: "${VOLSYNC_STORAGECLASS:-openebs-zfs}" accessModes: ["${VOLSYNC_ACCESSMODES:-ReadWriteOnce}"] moverSecurityContext: runAsUser: ${APP_UID:-568} @@ -62,11 +62,11 @@ spec: restic: repository: "${APP}-volsync-secret" copyMethod: Snapshot # must be Snapshot - volumeSnapshotClassName: "${VOLSYNC_SNAPSHOTCLASS:-csi-ceph-blockpool}" - cacheStorageClassName: "${VOLSYNC_CACHE_SNAPSHOTCLASS:-openebs-hostpath}" + volumeSnapshotClassName: "${VOLSYNC_SNAPSHOTCLASS:-openebs-zfs}" + cacheStorageClassName: "${VOLSYNC_CACHE_SNAPSHOTCLASS:-openebs-zfs}" cacheAccessModes: ["${VOLSYNC_CACHE_ACCESSMODES:-ReadWriteOnce}"] cacheCapacity: "${VOLSYNC_CACHE_CAPACITY:-4Gi}" - storageClassName: "${VOLSYNC_STORAGECLASS:-ceph-block}" + storageClassName: "${VOLSYNC_STORAGECLASS:-openebs-zfs}" accessModes: ["${VOLSYNC_ACCESSMODES:-ReadWriteOnce}"] capacity: "${VOLSYNC_CAPACITY}" moverSecurityContext: diff --git a/kubernetes/templates/volsync/r2.yaml b/kubernetes/templates/volsync/r2.yaml index 18c3d6f..e71e382 100644 --- a/kubernetes/templates/volsync/r2.yaml +++ b/kubernetes/templates/volsync/r2.yaml @@ -36,11 +36,11 @@ spec: copyMethod: "${VOLSYNC_COPYMETHOD:-Snapshot}" pruneIntervalDays: 7 repository: "${APP}-volsync-r2-secret" - volumeSnapshotClassName: "${VOLSYNC_SNAPSHOTCLASS:-csi-ceph-blockpool}" + volumeSnapshotClassName: "${VOLSYNC_SNAPSHOTCLASS:-openebs-zfs}" cacheCapacity: "${VOLSYNC_CACHE_CAPACITY:-4Gi}" - cacheStorageClassName: "${VOLSYNC_CACHE_SNAPSHOTCLASS:-openebs-hostpath}" + cacheStorageClassName: "${VOLSYNC_CACHE_SNAPSHOTCLASS:-openebs-zfs}" cacheAccessModes: ["${VOLSYNC_CACHE_ACCESSMODES:-ReadWriteOnce}"] - storageClassName: "${VOLSYNC_STORAGECLASS:-ceph-block}" + storageClassName: "${VOLSYNC_STORAGECLASS:-openebs-zfs}" accessModes: ["${VOLSYNC_ACCESSMODES:-ReadWriteOnce}"] moverSecurityContext: runAsUser: ${APP_UID:-568} diff --git a/kubernetes/tools/kbench.yaml b/kubernetes/tools/kbench.yaml index fa201f8..6d48f15 100644 --- a/kubernetes/tools/kbench.yaml +++ b/kubernetes/tools/kbench.yaml @@ -4,7 +4,7 @@ apiVersion: v1 metadata: name: kbench-pvc spec: - storageClassName: ceph-block + storageClassName: openebs-zfs accessModes: - ReadWriteOnce resources: