From 3f2c831c660ba76a911091d0f105431af6d01132 Mon Sep 17 00:00:00 2001 From: Joseph Hanson Date: Tue, 22 Oct 2024 14:08:21 -0500 Subject: [PATCH] local volsync backup from s3 --> nfs --- .../kyverno/policies/kustomization.yaml | 1 + .../kyverno/policies/volsync-movers.yaml | 46 ++++++++++++ .../volsync/app/helmrelease.yaml | 47 ++++++------ .../apps/volsync-system/volsync/ks.yaml | 16 ++-- .../flux/repositories/git/kustomization.yaml | 6 ++ kubernetes/flux/repositories/git/volsync.yaml | 17 +++++ .../flux/repositories/kustomization.yaml | 1 + .../templates/volsync/kustomization.yaml | 2 +- kubernetes/templates/volsync/nfs.yaml | 73 +++++++++++++++++++ 9 files changed, 178 insertions(+), 31 deletions(-) create mode 100644 kubernetes/apps/kyverno/kyverno/policies/volsync-movers.yaml create mode 100644 kubernetes/flux/repositories/git/kustomization.yaml create mode 100644 kubernetes/flux/repositories/git/volsync.yaml create mode 100644 kubernetes/templates/volsync/nfs.yaml diff --git a/kubernetes/apps/kyverno/kyverno/policies/kustomization.yaml b/kubernetes/apps/kyverno/kyverno/policies/kustomization.yaml index aa9c709e..c20464ad 100644 --- a/kubernetes/apps/kyverno/kyverno/policies/kustomization.yaml +++ b/kubernetes/apps/kyverno/kyverno/policies/kustomization.yaml @@ -5,3 +5,4 @@ kind: Kustomization resources: - ./remove-cpu-limits.yaml - ./schematic-to-pod.yaml + - ./volsync-movers.yaml diff --git a/kubernetes/apps/kyverno/kyverno/policies/volsync-movers.yaml b/kubernetes/apps/kyverno/kyverno/policies/volsync-movers.yaml new file mode 100644 index 00000000..f238310b --- /dev/null +++ b/kubernetes/apps/kyverno/kyverno/policies/volsync-movers.yaml @@ -0,0 +1,46 @@ +--- +# yaml-language-server: $schema=https://ks.hsn.dev/kyverno.io/clusterpolicy_v1.json +apiVersion: kyverno.io/v1 +kind: ClusterPolicy +metadata: + name: volsync-movers + annotations: + policies.kyverno.io/title: Set custom config on the Volsync mover Jobs + policies.kyverno.io/description: | + This policy sets custom configuration on the Volsync mover Jobs. + policies.kyverno.io/subject: Pod +spec: + generateExistingOnPolicyUpdate: true + rules: + - name: set-volsync-movers-custom-config + match: + any: + - resources: + kinds: ["batch/v1/Job"] + namespaces: ["default"] + selector: + matchLabels: + app.kubernetes.io/created-by: volsync + mutate: + patchStrategicMerge: + spec: + podReplacementPolicy: Failed + podFailurePolicy: + rules: + - action: FailJob + onExitCodes: + containerName: restic + operator: In + values: [11] + template: + spec: + containers: + - name: restic + volumeMounts: + - name: repository + mountPath: /repository + volumes: + - name: repository + nfs: + server: shadowfax.jahanson.tech + path: /nahar/volsync diff --git a/kubernetes/apps/volsync-system/volsync/app/helmrelease.yaml b/kubernetes/apps/volsync-system/volsync/app/helmrelease.yaml index 6265f8b6..055125cc 100644 --- a/kubernetes/apps/volsync-system/volsync/app/helmrelease.yaml +++ b/kubernetes/apps/volsync-system/volsync/app/helmrelease.yaml @@ -1,5 +1,5 @@ --- -# yaml-language-server: $schema=https://raw.githubusercontent.com/fluxcd-community/flux2-schemas/main/helmrelease-helm-v2beta2.json +# yaml-language-server: $schema=https://ks.hsn.dev/helm.toolkit.fluxcd.io/helmrelease_v2.json apiVersion: helm.toolkit.fluxcd.io/v2 kind: HelmRelease metadata: @@ -8,30 +8,33 @@ spec: interval: 30m chart: spec: - chart: volsync - version: 0.10.0 + chart: ./helm/volsync sourceRef: - kind: HelmRepository - name: backube + kind: GitRepository + name: volsync namespace: flux-system - interval: 30m + install: + remediation: + retries: 3 + upgrade: + cleanupOnFail: true + remediation: + strategy: rollback + retries: 3 + dependsOn: + - name: kyverno + namespace: kyverno + - name: snapshot-controller + namespace: volsync-system values: manageCRDs: true metrics: disableAuth: true - - # TODO: Refactor if/when https://github.com/backube/volsync/pull/1054 gets merged - postRenderers: - - kustomize: - patches: - - target: - version: v1 - kind: Deployment - name: volsync - patch: | - - op: add - path: /spec/template/metadata/labels/egress.home.arpa~1apiserver - value: allow - - op: add - path: /spec/template/metadata/labels/egress.home.arpa~1kubedns - value: allow + image: &image + repository: quay.io/backube/volsync + tag: release-0.11 + rclone: *image + restic: *image + rsync: *image + rsync-tls: *image + syncthing: *image diff --git a/kubernetes/apps/volsync-system/volsync/ks.yaml b/kubernetes/apps/volsync-system/volsync/ks.yaml index 7079decb..278aea09 100644 --- a/kubernetes/apps/volsync-system/volsync/ks.yaml +++ b/kubernetes/apps/volsync-system/volsync/ks.yaml @@ -1,22 +1,22 @@ --- -# yaml-language-server: $schema=https://raw.githubusercontent.com/fluxcd-community/flux2-schemas/main/kustomization-kustomize-v1.json +# yaml-language-server: $schema=https://ks.hsn.dev/kustomize.toolkit.fluxcd.io/kustomization_v1.json apiVersion: kustomize.toolkit.fluxcd.io/v1 kind: Kustomization metadata: - name: &appname volsync + name: &app volsync namespace: flux-system spec: targetNamespace: volsync-system commonMetadata: labels: - app.kubernetes.io/name: *appname - interval: 10m - path: "./kubernetes/apps/volsync-system/volsync/app" + app.kubernetes.io/name: *app + dependsOn: + - name: cluster-policies + path: ./kubernetes/apps/volsync-system/volsync/app prune: true sourceRef: kind: GitRepository name: theshire - dependsOn: - - name: snapshot-controller wait: false - timeout: 2m + interval: 30m + timeout: 5m diff --git a/kubernetes/flux/repositories/git/kustomization.yaml b/kubernetes/flux/repositories/git/kustomization.yaml new file mode 100644 index 00000000..9b53b9df --- /dev/null +++ b/kubernetes/flux/repositories/git/kustomization.yaml @@ -0,0 +1,6 @@ +--- +# yaml-language-server: $schema=https://json.schemastore.org/kustomization +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./volsync.yaml diff --git a/kubernetes/flux/repositories/git/volsync.yaml b/kubernetes/flux/repositories/git/volsync.yaml new file mode 100644 index 00000000..661b8ddc --- /dev/null +++ b/kubernetes/flux/repositories/git/volsync.yaml @@ -0,0 +1,17 @@ +--- +# yaml-language-server: $schema=https://ks.hsn.dev/source.toolkit.fluxcd.io/gitrepository_v1.json +apiVersion: source.toolkit.fluxcd.io/v1 +kind: GitRepository +metadata: + name: volsync + namespace: flux-system +spec: + interval: 30m + url: https://github.com/backube/volsync + ref: + branch: release-0.11 + ignore: | + # exclude all + /* + # include kubernetes directory + !/helm/volsync diff --git a/kubernetes/flux/repositories/kustomization.yaml b/kubernetes/flux/repositories/kustomization.yaml index 15d1a6bb..71cbce5e 100644 --- a/kubernetes/flux/repositories/kustomization.yaml +++ b/kubernetes/flux/repositories/kustomization.yaml @@ -4,5 +4,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization namespace: flux-system resources: + - ./git - ./helm - ./oci diff --git a/kubernetes/templates/volsync/kustomization.yaml b/kubernetes/templates/volsync/kustomization.yaml index 12ef510f..768d43bf 100644 --- a/kubernetes/templates/volsync/kustomization.yaml +++ b/kubernetes/templates/volsync/kustomization.yaml @@ -4,5 +4,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization resources: - ./claim.yaml - - ./minio.yaml + - ./nfs.yaml - ./r2.yaml diff --git a/kubernetes/templates/volsync/nfs.yaml b/kubernetes/templates/volsync/nfs.yaml new file mode 100644 index 00000000..3e7a7817 --- /dev/null +++ b/kubernetes/templates/volsync/nfs.yaml @@ -0,0 +1,73 @@ +--- +# yaml-language-server: $schema=https://ks.hsn.dev/external-secrets.io/externalsecret_v1beta1.json +apiVersion: external-secrets.io/v1beta1 +kind: ExternalSecret +metadata: + name: "${APP}-volsync" +spec: + secretStoreRef: + kind: ClusterSecretStore + name: onepassword-connect + target: + name: "${APP}-volsync-secret" + template: + engineVersion: v2 + data: + RESTIC_REPOSITORY: "/repository/${APP}" + RESTIC_PASSWORD: "{{ .RESTIC_PASSWORD }}" + dataFrom: + - extract: + key: volsync-template +--- +# yaml-language-server: $schema=https://ks.hsn.dev/volsync.backube/replicationsource_v1alpha1.json +apiVersion: volsync.backube/v1alpha1 +kind: ReplicationSource +metadata: + name: "${APP}" +spec: + sourcePVC: "${APP}" + trigger: + schedule: "0 * * * *" + restic: + copyMethod: "${VOLSYNC_COPYMETHOD:-Snapshot}" + pruneIntervalDays: 7 + repository: "${APP}-volsync-secret" + volumeSnapshotClassName: "${VOLSYNC_SNAPSHOTCLASS:-csi-ceph-blockpool}" + cacheCapacity: "${VOLSYNC_CACHE_CAPACITY:-4Gi}" + cacheStorageClassName: "${VOLSYNC_CACHE_SNAPSHOTCLASS:-openebs-hostpath}" + cacheAccessModes: ["${VOLSYNC_CACHE_ACCESSMODES:-ReadWriteOnce}"] + storageClassName: "${VOLSYNC_STORAGECLASS:-ceph-block}" + accessModes: ["${VOLSYNC_ACCESSMODES:-ReadWriteOnce}"] + moverSecurityContext: + runAsUser: ${VOLSYNC_PUID:-568} + runAsGroup: ${VOLSYNC_PGID:-568} + fsGroup: ${VOLSYNC_PGID:-568} + retain: + hourly: 24 + daily: 14 +--- +# yaml-language-server: $schema=https://ks.hsn.dev/volsync.backube/replicationdestination_v1alpha1.json +apiVersion: volsync.backube/v1alpha1 +kind: ReplicationDestination +metadata: + name: "${APP}-dst" +spec: + trigger: + manual: restore-once + restic: + repository: "${APP}-volsync-secret" + copyMethod: Snapshot + volumeSnapshotClassName: "${VOLSYNC_SNAPSHOTCLASS:-csi-ceph-blockpool}" + cacheStorageClassName: "${VOLSYNC_CACHE_SNAPSHOTCLASS:-openebs-hostpath}" + cacheAccessModes: ["${VOLSYNC_CACHE_ACCESSMODES:-ReadWriteOnce}"] + cacheCapacity: "${VOLSYNC_CACHE_CAPACITY:-8Gi}" + storageClassName: "${VOLSYNC_STORAGECLASS:-ceph-block}" + accessModes: ["${VOLSYNC_ACCESSMODES:-ReadWriteOnce}"] + capacity: "${VOLSYNC_CAPACITY}" + moverSecurityContext: + runAsUser: ${VOLSYNC_PUID:-568} + runAsGroup: ${VOLSYNC_PGID:-568} + fsGroup: ${VOLSYNC_PGID:-568} + enableFileDeletion: true + cleanupCachePVC: true + cleanupTempPVC: true -- 2.47.0