Swap storageclasses to openebs zfspv #270

Merged
jahanson merged 1 commit from swap-to-zfs into main 2024-05-07 19:11:28 -05:00
26 changed files with 90 additions and 38 deletions

View file

@ -11,7 +11,7 @@ spec:
repository: "${rsrc}-restic-secret" repository: "${rsrc}-restic-secret"
destinationPVC: "${claim}" destinationPVC: "${claim}"
copyMethod: Direct copyMethod: Direct
storageClassName: ceph-block storageClassName: openebs-zfs
# IMPORTANT NOTE: # IMPORTANT NOTE:
# Set to the last X number of snapshots to restore from # Set to the last X number of snapshots to restore from
previous: ${previous} previous: ${previous}

View file

@ -41,7 +41,7 @@ spec:
app.kubernetes.io/name: crunchy-postgres app.kubernetes.io/name: crunchy-postgres
replicas: &replica 1 replicas: &replica 1
dataVolumeClaimSpec: dataVolumeClaimSpec:
storageClassName: openebs-hostpath storageClassName: openebs-zfs
accessModes: accessModes:
- ReadWriteOnce - ReadWriteOnce
resources: resources:

View file

@ -0,0 +1,20 @@
apiVersion: v1
kind: Pod
metadata:
name: alpine-debug
namespace: kube-system
spec:
# nodeName: nenya
containers:
- name: alpine-debug
image: alpine
securityContext:
privileged: true
command: ["/bin/ash", "-c", "while true; do sleep 10; done"]
resources:
requests:
cpu: 100m
memory: 512Mi
limits:
cpu: 4000m
memory: 4000Mi

View file

@ -44,7 +44,7 @@ spec:
app.kubernetes.io/name: pgo-${APP} app.kubernetes.io/name: pgo-${APP}
replicas: 1 replicas: 1
dataVolumeClaimSpec: dataVolumeClaimSpec:
storageClassName: openebs-hostpath storageClassName: openebs-zfs
accessModes: accessModes:
- ReadWriteOnce - ReadWriteOnce
resources: resources:

View file

@ -51,7 +51,7 @@ spec:
storage: storage:
volumeClaimTemplate: volumeClaimTemplate:
spec: spec:
storageClassName: openebs-hostpath storageClassName: openebs-zfs
resources: resources:
requests: requests:
storage: 1Gi storage: 1Gi
@ -149,7 +149,7 @@ spec:
storageSpec: storageSpec:
volumeClaimTemplate: volumeClaimTemplate:
spec: spec:
storageClassName: openebs-hostpath storageClassName: openebs-zfs
resources: resources:
requests: requests:
storage: 20Gi storage: 20Gi

View file

@ -114,7 +114,7 @@ spec:
replicas: 1 replicas: 1
persistence: persistence:
size: 20Gi size: 20Gi
storageClass: openebs-hostpath storageClass: openebs-zfs
gateway: gateway:
replicas: 1 replicas: 1
image: image:
@ -135,7 +135,7 @@ spec:
replicas: 1 replicas: 1
persistence: persistence:
size: 20Gi size: 20Gi
storageClass: openebs-hostpath storageClass: openebs-zfs
sidecar: sidecar:
image: image:
repository: ghcr.io/kiwigrid/k8s-sidecar repository: ghcr.io/kiwigrid/k8s-sidecar

View file

@ -72,7 +72,7 @@ spec:
- --retention.resolution-1h=60d - --retention.resolution-1h=60d
persistence: &persistence persistence: &persistence
enabled: true enabled: true
storageClass: openebs-hostpath storageClass: openebs-zfs
size: 10Gi size: 10Gi
query: query:
replicas: 1 replicas: 1

View file

@ -3,16 +3,16 @@
apiVersion: helm.toolkit.fluxcd.io/v2beta2 apiVersion: helm.toolkit.fluxcd.io/v2beta2
kind: HelmRelease kind: HelmRelease
metadata: metadata:
name: openebs name: openebs-zfs
spec: spec:
interval: 30m interval: 30m
chart: chart:
spec: spec:
chart: openebs chart: zfs-localpv
version: 3.10.0 version: 2.5.1
sourceRef: sourceRef:
kind: HelmRepository kind: HelmRepository
name: openebs name: openebs-zfs
namespace: flux-system namespace: flux-system
install: install:
remediation: remediation:
@ -24,9 +24,5 @@ spec:
uninstall: uninstall:
keepHistory: false keepHistory: false
values: values:
localprovisioner: zfsNode:
hostpathClass: encrKeysDir: /var/openebs/keys
enabled: true
name: openebs-hostpath
isDefaultClass: false
basePath: /var/openebs/local

View file

@ -3,4 +3,6 @@
apiVersion: kustomize.config.k8s.io/v1beta1 apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization kind: Kustomization
resources: resources:
- ./helmrelease.yaml - ./helmrelease.yaml
- ./storageclass.yaml
- ./volumesnapshotclass.yaml

View file

@ -0,0 +1,12 @@
---
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: openebs-zfs
provisioner: zfs.csi.openebs.io
parameters:
recordsize: "128k"
compression: "off"
dedup: "off"
fstype: "zfs"
poolname: "nahar"

View file

@ -0,0 +1,10 @@
---
# yaml-language-server: $schema=https://ks.hsn.dev/snapshot.storage.k8s.io/volumesnapshotclass_v1beta1.json
kind: VolumeSnapshotClass
apiVersion: snapshot.storage.k8s.io/v1beta1
metadata:
name: openebs-zfs
annotations:
snapshot.storage.kubernetes.io/is-default-class: "true"
driver: zfs.csi.openebs.io
deletionPolicy: Delete

View file

@ -3,7 +3,7 @@
clusterName: homelab clusterName: homelab
talosVersion: v1.7.1 talosVersion: v1.7.1
kubernetesVersion: 1.28.4 kubernetesVersion: 1.30.0
endpoint: "https://10.1.1.57:6443" endpoint: "https://10.1.1.57:6443"
additionalApiServerCertSans: additionalApiServerCertSans:
@ -18,7 +18,7 @@ nodes:
ipAddress: 10.1.1.61 ipAddress: 10.1.1.61
controlPlane: true controlPlane: true
installDiskSelector: installDiskSelector:
busPath: /pci0000:20/0000:20:01.2/0000:2d:00.0/nvme/nvme1/nvme1n1 busPath: /pci0000:20/0000:20:01.2/0000:2c:00.0/nvme/nvme4/nvme4n1
networkInterfaces: networkInterfaces:
- interface: enp37s0f1 - interface: enp37s0f1
dhcp: true dhcp: true
@ -33,9 +33,11 @@ nodes:
customization: customization:
systemExtensions: systemExtensions:
officialExtensions: officialExtensions:
- "siderolabs/amd-ucode" - siderolabs/amd-ucode
- "siderolabs/nonfree-kmod-nvidia" - siderolabs/nonfree-kmod-nvidia
- "siderolabs/nvidia-container-toolkit" - siderolabs/nvidia-container-toolkit
- siderolabs/zfs
patches: patches:
- |- - |-
machine: machine:
@ -134,12 +136,12 @@ controlPlane:
validSubnets: validSubnets:
- 10.1.1.0/24 - 10.1.1.0/24
extraMounts: extraMounts:
- destination: /var/openebs/local - destination: /var/openebs/keys
options: options:
- bind - bind
- rshared - rshared
- rw - rw
source: /var/openebs/local source: /var/openebs/keys
type: bind type: bind
# Custom sysctls # Custom sysctls

View file

@ -0,0 +1,10 @@
---
# yaml-language-server: $schema=https://ks.hsn.dev/source.toolkit.fluxcd.io/helmrepository_v1beta2.json
apiVersion: source.toolkit.fluxcd.io/v1beta2
kind: HelmRepository
metadata:
name: openebs-zfs
namespace: flux-system
spec:
interval: 2h
url: https://openebs.github.io/zfs-localpv

View file

@ -36,7 +36,7 @@ spec:
app.kubernetes.io/name: pgo-${APP} app.kubernetes.io/name: pgo-${APP}
replicas: 1 replicas: 1
dataVolumeClaimSpec: dataVolumeClaimSpec:
storageClassName: openebs-hostpath storageClassName: openebs-zfs
accessModes: accessModes:
- ReadWriteOnce - ReadWriteOnce
resources: resources:

View file

@ -12,4 +12,4 @@ spec:
resources: resources:
requests: requests:
storage: "${VOLSYNC_CAPACITY}" storage: "${VOLSYNC_CAPACITY}"
storageClassName: "${VOLSYNC_STORAGECLASS:-ceph-block}" storageClassName: "${VOLSYNC_STORAGECLASS:-openebs-zfs}"

View file

@ -36,11 +36,11 @@ spec:
copyMethod: "${VOLSYNC_COPYMETHOD:-Snapshot}" copyMethod: "${VOLSYNC_COPYMETHOD:-Snapshot}"
pruneIntervalDays: 7 pruneIntervalDays: 7
repository: "${APP}-volsync-secret" repository: "${APP}-volsync-secret"
volumeSnapshotClassName: "${VOLSYNC_SNAPSHOTCLASS:-csi-ceph-blockpool}" volumeSnapshotClassName: "${VOLSYNC_SNAPSHOTCLASS:-openebs-zfs}"
cacheCapacity: "${VOLSYNC_CACHE_CAPACITY:-4Gi}" cacheCapacity: "${VOLSYNC_CACHE_CAPACITY:-4Gi}"
cacheStorageClassName: "${VOLSYNC_CACHE_SNAPSHOTCLASS:-openebs-hostpath}" cacheStorageClassName: "${VOLSYNC_CACHE_SNAPSHOTCLASS:-openebs-zfs}"
cacheAccessModes: ["${VOLSYNC_CACHE_ACCESSMODES:-ReadWriteOnce}"] cacheAccessModes: ["${VOLSYNC_CACHE_ACCESSMODES:-ReadWriteOnce}"]
storageClassName: "${VOLSYNC_STORAGECLASS:-ceph-block}" storageClassName: "${VOLSYNC_STORAGECLASS:-openebs-zfs}"
accessModes: ["${VOLSYNC_ACCESSMODES:-ReadWriteOnce}"] accessModes: ["${VOLSYNC_ACCESSMODES:-ReadWriteOnce}"]
moverSecurityContext: moverSecurityContext:
runAsUser: ${APP_UID:-568} runAsUser: ${APP_UID:-568}
@ -62,11 +62,11 @@ spec:
restic: restic:
repository: "${APP}-volsync-secret" repository: "${APP}-volsync-secret"
copyMethod: Snapshot # must be Snapshot copyMethod: Snapshot # must be Snapshot
volumeSnapshotClassName: "${VOLSYNC_SNAPSHOTCLASS:-csi-ceph-blockpool}" volumeSnapshotClassName: "${VOLSYNC_SNAPSHOTCLASS:-openebs-zfs}"
cacheStorageClassName: "${VOLSYNC_CACHE_SNAPSHOTCLASS:-openebs-hostpath}" cacheStorageClassName: "${VOLSYNC_CACHE_SNAPSHOTCLASS:-openebs-zfs}"
cacheAccessModes: ["${VOLSYNC_CACHE_ACCESSMODES:-ReadWriteOnce}"] cacheAccessModes: ["${VOLSYNC_CACHE_ACCESSMODES:-ReadWriteOnce}"]
cacheCapacity: "${VOLSYNC_CACHE_CAPACITY:-4Gi}" cacheCapacity: "${VOLSYNC_CACHE_CAPACITY:-4Gi}"
storageClassName: "${VOLSYNC_STORAGECLASS:-ceph-block}" storageClassName: "${VOLSYNC_STORAGECLASS:-openebs-zfs}"
accessModes: ["${VOLSYNC_ACCESSMODES:-ReadWriteOnce}"] accessModes: ["${VOLSYNC_ACCESSMODES:-ReadWriteOnce}"]
capacity: "${VOLSYNC_CAPACITY}" capacity: "${VOLSYNC_CAPACITY}"
moverSecurityContext: moverSecurityContext:

View file

@ -36,11 +36,11 @@ spec:
copyMethod: "${VOLSYNC_COPYMETHOD:-Snapshot}" copyMethod: "${VOLSYNC_COPYMETHOD:-Snapshot}"
pruneIntervalDays: 7 pruneIntervalDays: 7
repository: "${APP}-volsync-r2-secret" repository: "${APP}-volsync-r2-secret"
volumeSnapshotClassName: "${VOLSYNC_SNAPSHOTCLASS:-csi-ceph-blockpool}" volumeSnapshotClassName: "${VOLSYNC_SNAPSHOTCLASS:-openebs-zfs}"
cacheCapacity: "${VOLSYNC_CACHE_CAPACITY:-4Gi}" cacheCapacity: "${VOLSYNC_CACHE_CAPACITY:-4Gi}"
cacheStorageClassName: "${VOLSYNC_CACHE_SNAPSHOTCLASS:-openebs-hostpath}" cacheStorageClassName: "${VOLSYNC_CACHE_SNAPSHOTCLASS:-openebs-zfs}"
cacheAccessModes: ["${VOLSYNC_CACHE_ACCESSMODES:-ReadWriteOnce}"] cacheAccessModes: ["${VOLSYNC_CACHE_ACCESSMODES:-ReadWriteOnce}"]
storageClassName: "${VOLSYNC_STORAGECLASS:-ceph-block}" storageClassName: "${VOLSYNC_STORAGECLASS:-openebs-zfs}"
accessModes: ["${VOLSYNC_ACCESSMODES:-ReadWriteOnce}"] accessModes: ["${VOLSYNC_ACCESSMODES:-ReadWriteOnce}"]
moverSecurityContext: moverSecurityContext:
runAsUser: ${APP_UID:-568} runAsUser: ${APP_UID:-568}

View file

@ -4,7 +4,7 @@ apiVersion: v1
metadata: metadata:
name: kbench-pvc name: kbench-pvc
spec: spec:
storageClassName: ceph-block storageClassName: openebs-zfs
accessModes: accessModes:
- ReadWriteOnce - ReadWriteOnce
resources: resources: