--- version: "3" x-task-vars: &task-vars rsrc: '{{.rsrc}}' controller: '{{.controller}}' namespace: '{{.namespace}}' claim: '{{.claim}}' ts: '{{.ts}}' kustomization: '{{.kustomization}}' previous: '{{.previous}}' vars: destinationTemplate: "{{.ROOT_DIR}}/.taskfiles/VolSync/ReplicationDestination.tmpl.yaml" wipeJobTemplate: "{{.ROOT_DIR}}/.taskfiles/VolSync/WipeJob.tmpl.yaml" waitForJobScript: "{{.ROOT_DIR}}/.taskfiles/VolSync/wait-for-job.sh" listJobTemplate: "{{.ROOT_DIR}}/.taskfiles/VolSync/ListJob.tmpl.yaml" unlockJobTemplate: "{{.ROOT_DIR}}/.taskfiles/VolSync/UnlockJob.tmpl.yaml" ts: '{{now | date "150405"}}' tasks: list: desc: List all snapshots taken by restic for a given ReplicationSource (ex. task volsync:list rsrc=plex [namespace=default]) silent: true cmds: - envsubst < <(cat {{.listJobTemplate}}) | kubectl apply -f - - bash {{.waitForJobScript}} list-{{.rsrc}}-{{.ts}} {{.namespace}} - kubectl -n {{.namespace}} wait job/list-{{.rsrc}}-{{.ts}} --for condition=complete --timeout=1m - kubectl -n {{.namespace}} logs job/list-{{.rsrc}}-{{.ts}} --container list - kubectl -n {{.namespace}} delete job list-{{.rsrc}}-{{.ts}} vars: rsrc: '{{ or .rsrc (fail "ReplicationSource `rsrc` is required") }}' namespace: '{{.namespace | default "default"}}' env: *task-vars preconditions: - sh: test -f {{.waitForJobScript}} - sh: test -f {{.listJobTemplate}} unlock: desc: Unlocks restic repository for a given ReplicationSource (ex. task volsync:unlock rsrc=plex [namespace=default]) silent: true cmds: - envsubst < <(cat {{.unlockJobTemplate}}) | kubectl apply -f - - bash {{.waitForJobScript}} unlock-{{.rsrc}}-{{.ts}} {{.namespace}} - kubectl -n {{.namespace}} wait job/unlock-{{.rsrc}}-{{.ts}} --for condition=complete --timeout=1m - kubectl -n {{.namespace}} logs job/unlock-{{.rsrc}}-{{.ts}} --container unlock - kubectl -n {{.namespace}} delete job unlock-{{.rsrc}}-{{.ts}} vars: rsrc: '{{ or .rsrc (fail "ReplicationSource `rsrc` is required") }}' namespace: '{{.namespace | default "default"}}' env: *task-vars preconditions: - sh: test -f {{.waitForJobScript}} - sh: test -f {{.unlockJobTemplate}} # To run backup jobs in parallel for all replicationsources: # - kubectl get replicationsources --all-namespaces --no-headers | awk '{print $2, $1}' | xargs --max-procs=4 -l bash -c 'task volsync:snapshot rsrc=$0 namespace=$1' # snapshot: desc: Trigger a Restic ReplicationSource snapshot (ex. task volsync:snapshot rsrc=plex [namespace=default]) cmds: - kubectl -n {{.namespace}} patch replicationsources {{.rsrc}} --type merge -p '{"spec":{"trigger":{"manual":"{{.ts}}"}}}' - bash {{.waitForJobScript}} volsync-src-{{.rsrc}} {{.namespace}} - kubectl -n {{.namespace}} wait job/volsync-src-{{.rsrc}} --for condition=complete --timeout=120m # TODO: Find a way to output logs # Error from server (NotFound): jobs.batch "volsync-src-zzztest" not found # - kubectl -n {{.namespace}} logs job/volsync-src-{{.rsrc}} vars: rsrc: '{{ or .rsrc (fail "ReplicationSource `rsrc` is required") }}' namespace: '{{.namespace | default "default"}}' env: *task-vars preconditions: - sh: test -f {{.waitForJobScript}} - sh: kubectl -n {{.namespace}} get replicationsources {{.rsrc}} msg: "ReplicationSource '{{.rsrc}}' not found in namespace '{{.namespace}}'" # To run restore jobs in parallel for all replicationdestinations: # - kubectl get replicationsources --all-namespaces --no-headers | awk '{print $2, $1}' | xargs --max-procs=2 -l bash -c 'task volsync:restore rsrc=$0 namespace=$1' # restore: desc: Trigger a Restic ReplicationSource restore (ex. task volsync:restore rsrc=plex [namespace=default]) cmds: - task: restore-suspend-app vars: *task-vars - task: restore-wipe-job vars: *task-vars - task: restore-volsync-job vars: *task-vars - task: restore-resume-app vars: *task-vars vars: rsrc: '{{ or .rsrc (fail "Variable `rsrc` is required") }}' namespace: '{{.namespace | default "default"}}' # 1) Query to find the Flux Kustomization associated with the ReplicationSource (rsrc) kustomization: sh: | kubectl -n {{.namespace}} get replicationsource {{.rsrc}} \ -o jsonpath="{.metadata.labels.kustomize\.toolkit\.fluxcd\.io/name}" # 2) Query to find the Claim associated with the ReplicationSource (rsrc) claim: sh: | kubectl -n {{.namespace}} get replicationsource {{.rsrc}} \ -o jsonpath="{.spec.sourcePVC}" # 3) Query to find the controller associated with the PersistentVolumeClaim (claim) controller: sh: | app=$(kubectl -n {{.namespace}} get persistentvolumeclaim {{.claim}} -o jsonpath="{.metadata.labels.app\.kubernetes\.io/name}") if kubectl -n {{ .namespace }} get deployment.apps/$app >/dev/null 2>&1 ; then echo "deployment.apps/$app" else echo "statefulset.apps/$app" fi previous: "{{.previous | default 2}}" env: *task-vars preconditions: - sh: test -f {{.wipeJobTemplate}} - sh: test -f {{.destinationTemplate}} - sh: test -f {{.waitForJobScript}} # Suspend the Flux ks and hr restore-suspend-app: internal: true cmds: - flux -n flux-system suspend kustomization {{.kustomization}} - flux -n {{.namespace}} suspend helmrelease {{.rsrc}} - kubectl -n {{.namespace}} scale {{.controller}} --replicas 0 - kubectl -n {{.namespace}} wait pod --for delete --selector="app.kubernetes.io/name={{.rsrc}}" --timeout=2m env: *task-vars # Wipe the PVC of all data restore-wipe-job: internal: true cmds: - envsubst < <(cat {{.wipeJobTemplate}}) | kubectl apply -f - - bash {{.waitForJobScript}} wipe-{{.rsrc}}-{{.claim}}-{{.ts}} {{.namespace}} - kubectl -n {{.namespace}} wait job/wipe-{{.rsrc}}-{{.claim}}-{{.ts}} --for condition=complete --timeout=120m - kubectl -n {{.namespace}} logs job/wipe-{{.rsrc}}-{{.claim}}-{{.ts}} --container wipe - kubectl -n {{.namespace}} delete job wipe-{{.rsrc}}-{{.claim}}-{{.ts}} env: *task-vars # Create VolSync replicationdestination CR to restore data restore-volsync-job: internal: true cmds: - envsubst < <(cat {{.destinationTemplate}}) | kubectl apply -f - - bash {{.waitForJobScript}} volsync-dst-{{.rsrc}}-{{.claim}}-{{.ts}} {{.namespace}} - kubectl -n {{.namespace}} wait job/volsync-dst-{{.rsrc}}-{{.claim}}-{{.ts}} --for condition=complete --timeout=120m - kubectl -n {{.namespace}} delete replicationdestination {{.rsrc}}-{{.claim}}-{{.ts}} env: *task-vars # Resume Flux ks and hr restore-resume-app: internal: true cmds: - flux -n {{.namespace}} resume helmrelease {{.rsrc}} - flux -n flux-system resume kustomization {{.kustomization}} env: *task-vars