Adding rook ceph.

2024-01-16 17:46:22 -06:00 · 2024-01-16 17:46:22 -06:00 · f73dc34b63
commit f73dc34b63
parent 06c4feb045
8 changed files with 347 additions and 0 deletions
--- a/kubernetes/apps/rook-ceph/kustomization.yaml
+++ b/kubernetes/apps/rook-ceph/kustomization.yaml
@ -0,0 +1,9 @@
+---
+# yaml-language-server: $schema=https://json.schemastore.org/kustomization
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  # Pre Flux-Kustomizations
+  - ./namespace.yaml
+  # Flux-Kustomizations
+  - ./rook-ceph/ks.yaml
--- a/kubernetes/apps/rook-ceph/namespace.yaml
+++ b/kubernetes/apps/rook-ceph/namespace.yaml
@ -0,0 +1,7 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: rook-ceph
+  labels:
+    kustomize.toolkit.fluxcd.io/prune: disabled
--- a/kubernetes/apps/rook-ceph/rook-ceph/app/helmrelease.yaml
+++ b/kubernetes/apps/rook-ceph/rook-ceph/app/helmrelease.yaml
@ -0,0 +1,44 @@
+---
+# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/helm.toolkit.fluxcd.io/helmrelease_v2beta2.json
+apiVersion: helm.toolkit.fluxcd.io/v2beta2
+kind: HelmRelease
+metadata:
+  name: rook-ceph-operator
+spec:
+  interval: 30m
+  timeout: 15m
+  chart:
+    spec:
+      chart: rook-ceph
+      version: v1.13.2
+      sourceRef:
+        kind: HelmRepository
+        name: rook-ceph
+        namespace: flux-system
+  install:
+    remediation:
+      retries: 3
+  upgrade:
+    cleanupOnFail: true
+    remediation:
+      retries: 3
+  uninstall:
+    keepHistory: false
+  dependsOn:
+    - name: snapshot-controller
+      namespace: storage
+  values:
+    csi:
+      provisioner:
+        image: registry.k8s.io/sig-storage/csi-provisioner:v3.6.3
+      cephFSKernelMountOptions: ms_mode=prefer-crc
+      enableLiveness: true
+      serviceMonitor:
+        enabled: true
+    monitoring:
+      enabled: true
+    resources:
+      requests:
+        memory: 128Mi # unchangable
+        cpu: 100m # unchangable
+      limits: {}
--- a/kubernetes/apps/rook-ceph/rook-ceph/app/kustomization.yaml
+++ b/kubernetes/apps/rook-ceph/rook-ceph/app/kustomization.yaml
@ -0,0 +1,7 @@
+---
+# yaml-language-server: $schema=https://json.schemastore.org/kustomization
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ./rook-ceph-dashboard-password.secret.sops.yaml
+  - ./helmrelease.yaml
--- a/kubernetes/apps/rook-ceph/rook-ceph/app/rook-ceph-dashboard-password.secret.sops.yaml
+++ b/kubernetes/apps/rook-ceph/rook-ceph/app/rook-ceph-dashboard-password.secret.sops.yaml
@ -0,0 +1,26 @@
+apiVersion: v1
+kind: Secret
+metadata:
+    name: rook-ceph-dashboard-password
+stringData:
+    password: ENC[AES256_GCM,data:WWTt7SN6ssndLahsOA1gujEeGAM=,iv:YbHGNN+11wA/MLq9vFVM6v4mhPO58JmwXBDj0Qs7+Wk=,tag:5Xn0tqpiIiEt8ZWZHRTM3w==,type:str]
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age1eqlaq205y5jre9hu5hvulywa7w3d4qyxwmafneamxcn7nejesedsf4q9g6
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAzb2ZpaDd0azNHNTJoUTB6
+            VVpKbm94ZEprSHplb2UrQnkzTzdGUEFjcGxBCnhxR1BwNmFIOExtMW5GRkVJWTl5
+            blQzSmZ0Tm5CWTk3N25nUUM0dFpKUTQKLS0tIEgwSHNlVXNRdHZvcE10VzExU0hE
+            L0dGK1lFd0ZSQ0lTcEdMNTBkSDJ6WWsKQuiJmRSLbvmgenlu4F2/CQYCCbZTtS/K
+            nz7NsY2om+mWMvPSvLAp1pOHDAdFW79ggQAiCyslDi9iOkaD8MOnxQ==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2024-01-16T23:22:39Z"
+    mac: ENC[AES256_GCM,data:djsWoz/MuUhEKsM03+iaGV/dZUjRAGkiBEz4hROi+rfNWeHLJG2/xXPSKYYgT3h7JOZGh2Gnz7NXiB7TuixlWrAfT2BUBzd+2o9/hzg3xQzLAjApSfZdyap6oafatKxZAR/JHBSw7s0saVNnop9d/DZK4c1Fb1qNKoTrnWqqrF8=,iv:oitjHdZl07CaoBtNtX/sOPLHu7AS/R4YE4TKBJKrUBw=,tag:Br8mBH+mATEwsLzSZmoVYg==,type:str]
+    pgp: []
+    encrypted_regex: ^(data|stringData)$
+    version: 3.8.1
--- a/kubernetes/apps/rook-ceph/rook-ceph/cluster/helmrelease.yaml
+++ b/kubernetes/apps/rook-ceph/rook-ceph/cluster/helmrelease.yaml
@ -0,0 +1,206 @@
+---
+# yaml-language-server: $schema=https://kubernetes-schemas.pages.dev/helm.toolkit.fluxcd.io/helmrelease_v2beta2.json
+apiVersion: helm.toolkit.fluxcd.io/v2beta2
+kind: HelmRelease
+metadata:
+  name: rook-ceph-cluster
+spec:
+  interval: 30m
+  timeout: 15m
+  chart:
+    spec:
+      chart: rook-ceph-cluster
+      version: v1.13.2
+      sourceRef:
+        kind: HelmRepository
+        name: rook-ceph
+        namespace: flux-system
+  install:
+    remediation:
+      retries: 3
+  upgrade:
+    cleanupOnFail: true
+    remediation:
+      retries: 3
+  uninstall:
+    keepHistory: false
+  dependsOn:
+    - name: rook-ceph-operator
+      namespace: rook-ceph
+    - name: snapshot-controller
+      namespace: storage
+  values:
+    monitoring:
+      enabled: true
+      createPrometheusRules: true
+    ingress:
+      dashboard:
+        ingressClassName: internal-nginx
+        host:
+          name: &host rook.jahanson.tech
+          path: /
+        tls:
+          - hosts:
+              - *host
+    toolbox:
+      enabled: true
+    configOverride: |
+      [global]
+      bdev_enable_discard = true
+      bdev_async_discard = true
+      osd_class_update_on_start = false
+    cephClusterSpec:
+      network:
+        provider: host
+        connections:
+          requireMsgr2: true
+      crashCollector:
+        disable: false
+      dashboard:
+        enabled: true
+        urlPrefix: /
+        ssl: false
+      storage:
+        useAllNodes: false
+        useAllDevices: false
+        config:
+          osdsPerDevice: "1"
+        nodes:
+          - name: narya
+            devices:
+              - name: /dev/disk/by-id/nvme-Samsung_SSD_960_EVO_250GB_S3ESNX0K308438J
+          - name: vilya
+            devices:
+              - name: /dev/disk/by-id/nvme-PC300_NVMe_SK_hynix_256GB_EJ75N587410705M4U
+          - name: gollum
+            devices:
+              - name: /dev/disk/by-id/nvme-Samsung_SSD_960_EVO_250GB_S3ESNX0K308457X
+      placement:
+        mgr: &placement
+          nodeAffinity:
+            requiredDuringSchedulingIgnoredDuringExecution:
+              nodeSelectorTerms:
+                - matchExpressions:
+                    - key: node-role.kubernetes.io/control-plane
+                      operator: Exists
+        mon: *placement
+      resources:
+        mgr:
+          requests:
+            cpu: 500m
+            memory: 512Mi
+          limits:
+            cpu: 2000m
+            memory: 2Gi
+        mon:
+          requests:
+            cpu: 500m
+            memory: 1Gi
+          limits:
+            cpu: 4000m
+            memory: 4Gi
+        osd:
+          requests:
+            cpu: 500m
+            memory: 4Gi
+          limits:
+            cpu: 4000m
+            memory: 8Gi
+    cephBlockPools:
+      - name: ceph-blockpool
+        spec:
+          failureDomain: host
+          replicated:
+            size: 6
+        storageClass:
+          enabled: true
+          name: ceph-block
+          isDefault: true
+          reclaimPolicy: Delete
+          allowVolumeExpansion: true
+          parameters:
+            imageFormat: "2"
+            imageFeatures: layering
+            csi.storage.k8s.io/provisioner-secret-name: rook-csi-rbd-provisioner
+            csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph
+            csi.storage.k8s.io/controller-expand-secret-name: rook-csi-rbd-provisioner
+            csi.storage.k8s.io/controller-expand-secret-namespace: rook-ceph
+            csi.storage.k8s.io/node-stage-secret-name: rook-csi-rbd-node
+            csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph
+            csi.storage.k8s.io/fstype: ext4
+    cephBlockPoolsVolumeSnapshotClass:
+      enabled: true
+      name: csi-ceph-blockpool
+      isDefault: false
+      deletionPolicy: Delete
+    cephFileSystems:
+      - name: ceph-filesystem
+        spec:
+          metadataPool:
+            replicated:
+              size: 3
+          dataPools:
+            - failureDomain: host
+              replicated:
+                size: 3
+              name: data0
+          metadataServer:
+            activeCount: 1
+            activeStandby: true
+            resources:
+              requests:
+                cpu: 1000m
+                memory: 4Gi
+              limits:
+                memory: 4Gi
+        storageClass:
+          enabled: true
+          isDefault: false
+          name: ceph-filesystem
+          pool: data0
+          reclaimPolicy: Delete
+          allowVolumeExpansion: true
+          parameters:
+            csi.storage.k8s.io/provisioner-secret-name: rook-csi-cephfs-provisioner
+            csi.storage.k8s.io/provisioner-secret-namespace: rook-ceph
+            csi.storage.k8s.io/controller-expand-secret-name: rook-csi-cephfs-provisioner
+            csi.storage.k8s.io/controller-expand-secret-namespace: rook-ceph
+            csi.storage.k8s.io/node-stage-secret-name: rook-csi-cephfs-node
+            csi.storage.k8s.io/node-stage-secret-namespace: rook-ceph
+            csi.storage.k8s.io/fstype: ext4
+    cephFileSystemVolumeSnapshotClass:
+      enabled: true
+      name: csi-ceph-filesystem
+      isDefault: false
+      deletionPolicy: Delete
+    cephObjectStores:
+      - name: ceph-objectstore
+        spec:
+          metadataPool:
+            failureDomain: host
+            replicated:
+              size: 3
+          dataPool:
+            failureDomain: host
+            erasureCoded:
+              dataChunks: 2
+              codingChunks: 1
+          preservePoolsOnDelete: true
+          gateway:
+            port: 80
+            resources:
+              requests:
+                cpu: 1000m
+                memory: 1Gi
+              limits:
+                memory: 2Gi
+            instances: 2
+          healthCheck:
+            bucket:
+              interval: 60s
+        storageClass:
+          enabled: true
+          name: ceph-bucket
+          reclaimPolicy: Delete
+          parameters:
+            region: us-east-1
--- a/kubernetes/apps/rook-ceph/rook-ceph/cluster/kustomization.yaml
+++ b/kubernetes/apps/rook-ceph/rook-ceph/cluster/kustomization.yaml
@ -0,0 +1,6 @@
+---
+# yaml-language-server: $schema=https://json.schemastore.org/kustomization
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ./helmrelease.yaml
--- a/kubernetes/apps/rook-ceph/rook-ceph/ks.yaml
+++ b/kubernetes/apps/rook-ceph/rook-ceph/ks.yaml
@ -0,0 +1,42 @@
+---
+# yaml-language-server: $schema=https://ks.hsn.dev/kustomize.toolkit.fluxcd.io/kustomization_v1.json
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: &app rook-ceph
+  namespace: flux-system
+spec:
+  targetNamespace: rook-ceph
+  commonMetadata:
+    labels:
+      app.kubernetes.io/name: *app
+  path: ./kubernetes/apps/rook-ceph/rook-ceph/app
+  prune: false # never should be deleted
+  sourceRef:
+    kind: GitRepository
+    name: home-kubernetes
+  wait: false
+  interval: 30m
+  retryInterval: 1m
+  timeout: 5m
+---
+# yaml-language-server: $schema=https://ks.hsn.dev/kustomize.toolkit.fluxcd.io/kustomization_v1.json
+apiVersion: kustomize.toolkit.fluxcd.io/v1
+kind: Kustomization
+metadata:
+  name: &app rook-ceph-cluster
+  namespace: flux-system
+spec:
+  targetNamespace: rook-ceph
+  commonMetadata:
+    labels:
+      app.kubernetes.io/name: *app
+  path: ./kubernetes/apps/rook-ceph/rook-ceph/cluster
+  prune: false # never should be deleted
+  sourceRef:
+    kind: GitRepository
+    name: home-kubernetes
+  wait: false
+  interval: 30m
+  retryInterval: 1m
+  timeout: 15m