From f73dc34b63e8f23fc43374aa4c5b2fad28d03884 Mon Sep 17 00:00:00 2001 From: Joseph Hanson Date: Tue, 16 Jan 2024 17:46:22 -0600 Subject: [PATCH] Adding rook ceph. --- kubernetes/apps/rook-ceph/kustomization.yaml | 9 + kubernetes/apps/rook-ceph/namespace.yaml | 7 + .../rook-ceph/rook-ceph/app/helmrelease.yaml | 44 ++++ .../rook-ceph/app/kustomization.yaml | 7 + ...k-ceph-dashboard-password.secret.sops.yaml | 26 +++ .../rook-ceph/cluster/helmrelease.yaml | 206 ++++++++++++++++++ .../rook-ceph/cluster/kustomization.yaml | 6 + kubernetes/apps/rook-ceph/rook-ceph/ks.yaml | 42 ++++ 8 files changed, 347 insertions(+) create mode 100644 kubernetes/apps/rook-ceph/kustomization.yaml create mode 100644 kubernetes/apps/rook-ceph/namespace.yaml create mode 100644 kubernetes/apps/rook-ceph/rook-ceph/app/helmrelease.yaml create mode 100644 kubernetes/apps/rook-ceph/rook-ceph/app/kustomization.yaml create mode 100644 kubernetes/apps/rook-ceph/rook-ceph/app/rook-ceph-dashboard-password.secret.sops.yaml create mode 100644 kubernetes/apps/rook-ceph/rook-ceph/cluster/helmrelease.yaml create mode 100644 kubernetes/apps/rook-ceph/rook-ceph/cluster/kustomization.yaml create mode 100644 kubernetes/apps/rook-ceph/rook-ceph/ks.yaml diff --git a/kubernetes/apps/rook-ceph/kustomization.yaml b/kubernetes/apps/rook-ceph/kustomization.yaml new file mode 100644 index 00000000..dcaf9188 --- /dev/null +++ b/kubernetes/apps/rook-ceph/kustomization.yaml @@ -0,0 +1,9 @@ +--- +# yaml-language-server: $schema=https://json.schemastore.org/kustomization +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + # Pre Flux-Kustomizations + - ./namespace.yaml + # Flux-Kustomizations + - ./rook-ceph/ks.yaml diff --git a/kubernetes/apps/rook-ceph/namespace.yaml b/kubernetes/apps/rook-ceph/namespace.yaml new file mode 100644 index 00000000..4f4d74a8 --- /dev/null +++ b/kubernetes/apps/rook-ceph/namespace.yaml @@ -0,0 +1,7 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: rook-ceph + labels: + kustomize.toolkit.fluxcd.io/prune: disabled diff --git a/kubernetes/apps/rook-ceph/rook-ceph/app/helmrelease.yaml b/kubernetes/apps/rook-ceph/rook-ceph/app/helmrelease.yaml new file mode 100644 index 00000000..2f33795d --- /dev/null +++ b/kubernetes/apps/rook-ceph/rook-ceph/app/helmrelease.yaml @@ -0,0 +1,44 @@ +--- +# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/helm.toolkit.fluxcd.io/helmrelease_v2beta2.json +apiVersion: helm.toolkit.fluxcd.io/v2beta2 +kind: HelmRelease +metadata: + name: rook-ceph-operator +spec: + interval: 30m + timeout: 15m + chart: + spec: + chart: rook-ceph + version: v1.13.2 + sourceRef: + kind: HelmRepository + name: rook-ceph + namespace: flux-system + install: + remediation: + retries: 3 + upgrade: + cleanupOnFail: true + remediation: + retries: 3 + uninstall: + keepHistory: false + dependsOn: + - name: snapshot-controller + namespace: storage + values: + csi: + provisioner: + image: registry.k8s.io/sig-storage/csi-provisioner:v3.6.3 + cephFSKernelMountOptions: ms_mode=prefer-crc + enableLiveness: true + serviceMonitor: + enabled: true + monitoring: + enabled: true + resources: + requests: + memory: 128Mi # unchangable + cpu: 100m # unchangable + limits: {} diff --git a/kubernetes/apps/rook-ceph/rook-ceph/app/kustomization.yaml b/kubernetes/apps/rook-ceph/rook-ceph/app/kustomization.yaml new file mode 100644 index 00000000..fb2f8c12 --- /dev/null +++ b/kubernetes/apps/rook-ceph/rook-ceph/app/kustomization.yaml @@ -0,0 +1,7 @@ +--- +# yaml-language-server: $schema=https://json.schemastore.org/kustomization +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./rook-ceph-dashboard-password.secret.sops.yaml + - ./helmrelease.yaml diff --git a/kubernetes/apps/rook-ceph/rook-ceph/app/rook-ceph-dashboard-password.secret.sops.yaml b/kubernetes/apps/rook-ceph/rook-ceph/app/rook-ceph-dashboard-password.secret.sops.yaml new file mode 100644 index 00000000..78c545de --- /dev/null +++ b/kubernetes/apps/rook-ceph/rook-ceph/app/rook-ceph-dashboard-password.secret.sops.yaml @@ -0,0 +1,26 @@ +apiVersion: v1 +kind: Secret +metadata: + name: rook-ceph-dashboard-password +stringData: + password: ENC[AES256_GCM,data:WWTt7SN6ssndLahsOA1gujEeGAM=,iv:YbHGNN+11wA/MLq9vFVM6v4mhPO58JmwXBDj0Qs7+Wk=,tag:5Xn0tqpiIiEt8ZWZHRTM3w==,type:str] +sops: + kms: [] + gcp_kms: [] + azure_kv: [] + hc_vault: [] + age: + - recipient: age1eqlaq205y5jre9hu5hvulywa7w3d4qyxwmafneamxcn7nejesedsf4q9g6 + enc: | + -----BEGIN AGE ENCRYPTED FILE----- + YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAzb2ZpaDd0azNHNTJoUTB6 + VVpKbm94ZEprSHplb2UrQnkzTzdGUEFjcGxBCnhxR1BwNmFIOExtMW5GRkVJWTl5 + blQzSmZ0Tm5CWTk3N25nUUM0dFpKUTQKLS0tIEgwSHNlVXNRdHZvcE10VzExU0hE + L0dGK1lFd0ZSQ0lTcEdMNTBkSDJ6WWsKQuiJmRSLbvmgenlu4F2/CQYCCbZTtS/K + nz7NsY2om+mWMvPSvLAp1pOHDAdFW79ggQAiCyslDi9iOkaD8MOnxQ== + -----END AGE ENCRYPTED FILE----- + lastmodified: "2024-01-16T23:22:39Z" + mac: ENC[AES256_GCM,data:djsWoz/MuUhEKsM03+iaGV/dZUjRAGkiBEz4hROi+rfNWeHLJG2/xXPSKYYgT3h7JOZGh2Gnz7NXiB7TuixlWrAfT2BUBzd+2o9/hzg3xQzLAjApSfZdyap6oafatKxZAR/JHBSw7s0saVNnop9d/DZK4c1Fb1qNKoTrnWqqrF8=,iv:oitjHdZl07CaoBtNtX/sOPLHu7AS/R4YE4TKBJKrUBw=,tag:Br8mBH+mATEwsLzSZmoVYg==,type:str] + pgp: [] + encrypted_regex: ^(data|stringData)$ + version: 3.8.1 diff --git a/kubernetes/apps/rook-ceph/rook-ceph/cluster/helmrelease.yaml b/kubernetes/apps/rook-ceph/rook-ceph/cluster/helmrelease.yaml new file mode 100644 index 00000000..7cc85767 --- /dev/null +++ b/kubernetes/apps/rook-ceph/rook-ceph/cluster/helmrelease.yaml @@ -0,0 +1,206 @@ +--- +# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/helm.toolkit.fluxcd.io/helmrelease_v2beta2.json +apiVersion: helm.toolkit.fluxcd.io/v2beta2 +kind: HelmRelease +metadata: + name: rook-ceph-cluster +spec: + interval: 30m + timeout: 15m + chart: + spec: + chart: rook-ceph-cluster + version: v1.13.2 + sourceRef: + kind: HelmRepository + name: rook-ceph + namespace: flux-system + install: + remediation: + retries: 3 + upgrade: + cleanupOnFail: true + remediation: + retries: 3 + uninstall: + keepHistory: false + dependsOn: + - name: rook-ceph-operator + namespace: rook-ceph + - name: snapshot-controller + namespace: storage + values: + monitoring: + enabled: true + createPrometheusRules: true + ingress: + dashboard: + ingressClassName: internal-nginx + host: + name: &host rook.jahanson.tech + path: / + tls: + - hosts: + - *host + toolbox: + enabled: true + configOverride: | + [global] + bdev_enable_discard = true + bdev_async_discard = true + osd_class_update_on_start = false + cephClusterSpec: + network: + provider: host + connections: + requireMsgr2: true + crashCollector: + disable: false + dashboard: + enabled: true + urlPrefix: / + ssl: false + storage: + useAllNodes: false + useAllDevices: false + config: + osdsPerDevice: "1" + nodes: + - name: narya + devices: + - name: /dev/disk/by-id/nvme-Samsung_SSD_960_EVO_250GB_S3ESNX0K308438J + - name: vilya + devices: + - name: /dev/disk/by-id/nvme-PC300_NVMe_SK_hynix_256GB_EJ75N587410705M4U + - name: gollum + devices: + - name: /dev/disk/by-id/nvme-Samsung_SSD_960_EVO_250GB_S3ESNX0K308457X + placement: + mgr: &placement + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: Exists + mon: *placement + resources: + mgr: + requests: + cpu: 500m + memory: 512Mi + limits: + cpu: 2000m + memory: 2Gi + mon: + requests: + cpu: 500m + memory: 1Gi + limits: + cpu: 4000m + memory: 4Gi + osd: + requests: + cpu: 500m + memory: 4Gi + limits: + cpu: 4000m + memory: 8Gi + cephBlockPools: + - name: ceph-blockpool + spec: + failureDomain: host + replicated: + size: 6 + storageClass: + enabled: true + name: ceph-block + isDefault: true + reclaimPolicy: Delete + allowVolumeExpansion: true + parameters: + imageFormat: "2" + imageFeatures: layering + csi.storage.k8s.io/provisioner-secret-name: rook-csi-rbd-provisioner + csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph + csi.storage.k8s.io/controller-expand-secret-name: rook-csi-rbd-provisioner + csi.storage.k8s.io/controller-expand-secret-namespace: rook-ceph + csi.storage.k8s.io/node-stage-secret-name: rook-csi-rbd-node + csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph + csi.storage.k8s.io/fstype: ext4 + cephBlockPoolsVolumeSnapshotClass: + enabled: true + name: csi-ceph-blockpool + isDefault: false + deletionPolicy: Delete + cephFileSystems: + - name: ceph-filesystem + spec: + metadataPool: + replicated: + size: 3 + dataPools: + - failureDomain: host + replicated: + size: 3 + name: data0 + metadataServer: + activeCount: 1 + activeStandby: true + resources: + requests: + cpu: 1000m + memory: 4Gi + limits: + memory: 4Gi + storageClass: + enabled: true + isDefault: false + name: ceph-filesystem + pool: data0 + reclaimPolicy: Delete + allowVolumeExpansion: true + parameters: + csi.storage.k8s.io/provisioner-secret-name: rook-csi-cephfs-provisioner + csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph + csi.storage.k8s.io/controller-expand-secret-name: rook-csi-cephfs-provisioner + csi.storage.k8s.io/controller-expand-secret-namespace: rook-ceph + csi.storage.k8s.io/node-stage-secret-name: rook-csi-cephfs-node + csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph + csi.storage.k8s.io/fstype: ext4 + cephFileSystemVolumeSnapshotClass: + enabled: true + name: csi-ceph-filesystem + isDefault: false + deletionPolicy: Delete + cephObjectStores: + - name: ceph-objectstore + spec: + metadataPool: + failureDomain: host + replicated: + size: 3 + dataPool: + failureDomain: host + erasureCoded: + dataChunks: 2 + codingChunks: 1 + preservePoolsOnDelete: true + gateway: + port: 80 + resources: + requests: + cpu: 1000m + memory: 1Gi + limits: + memory: 2Gi + instances: 2 + healthCheck: + bucket: + interval: 60s + storageClass: + enabled: true + name: ceph-bucket + reclaimPolicy: Delete + parameters: + region: us-east-1 diff --git a/kubernetes/apps/rook-ceph/rook-ceph/cluster/kustomization.yaml b/kubernetes/apps/rook-ceph/rook-ceph/cluster/kustomization.yaml new file mode 100644 index 00000000..17cbc72b --- /dev/null +++ b/kubernetes/apps/rook-ceph/rook-ceph/cluster/kustomization.yaml @@ -0,0 +1,6 @@ +--- +# yaml-language-server: $schema=https://json.schemastore.org/kustomization +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./helmrelease.yaml diff --git a/kubernetes/apps/rook-ceph/rook-ceph/ks.yaml b/kubernetes/apps/rook-ceph/rook-ceph/ks.yaml new file mode 100644 index 00000000..5c002a5c --- /dev/null +++ b/kubernetes/apps/rook-ceph/rook-ceph/ks.yaml @@ -0,0 +1,42 @@ +--- +# yaml-language-server: $schema=https://ks.hsn.dev/kustomize.toolkit.fluxcd.io/kustomization_v1.json +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: &app rook-ceph + namespace: flux-system +spec: + targetNamespace: rook-ceph + commonMetadata: + labels: + app.kubernetes.io/name: *app + path: ./kubernetes/apps/rook-ceph/rook-ceph/app + prune: false # never should be deleted + sourceRef: + kind: GitRepository + name: home-kubernetes + wait: false + interval: 30m + retryInterval: 1m + timeout: 5m +--- +# yaml-language-server: $schema=https://ks.hsn.dev/kustomize.toolkit.fluxcd.io/kustomization_v1.json +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: &app rook-ceph-cluster + namespace: flux-system +spec: + targetNamespace: rook-ceph + commonMetadata: + labels: + app.kubernetes.io/name: *app + path: ./kubernetes/apps/rook-ceph/rook-ceph/cluster + prune: false # never should be deleted + sourceRef: + kind: GitRepository + name: home-kubernetes + wait: false + interval: 30m + retryInterval: 1m + timeout: 15m