From 391116930bcb0f92b4cf4cd6b2a81be48d29a10b Mon Sep 17 00:00:00 2001
From: Joseph Hanson <joe@veri.dev>
Date: Thu, 8 Aug 2024 07:18:03 -0500
Subject: [PATCH] Converting multus to an app template. Yoink via bjw-s.

---
 .../apps/kube-system/multus/app/crd.yaml      |  45 ++++
 .../kube-system/multus/app/helmrelease.yaml   | 220 ++++++++++++++++--
 .../kube-system/multus/app/kustomization.yaml |   4 +-
 .../apps/kube-system/multus/app/rbac.yaml     |  43 ++++
 4 files changed, 288 insertions(+), 24 deletions(-)
 create mode 100644 kubernetes/apps/kube-system/multus/app/crd.yaml
 create mode 100644 kubernetes/apps/kube-system/multus/app/rbac.yaml

diff --git a/kubernetes/apps/kube-system/multus/app/crd.yaml b/kubernetes/apps/kube-system/multus/app/crd.yaml
new file mode 100644
index 0000000..24b2c58
--- /dev/null
+++ b/kubernetes/apps/kube-system/multus/app/crd.yaml
@@ -0,0 +1,45 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  name: network-attachment-definitions.k8s.cni.cncf.io
+spec:
+  group: k8s.cni.cncf.io
+  scope: Namespaced
+  names:
+    plural: network-attachment-definitions
+    singular: network-attachment-definition
+    kind: NetworkAttachmentDefinition
+    shortNames:
+      - net-attach-def
+  versions:
+    - name: v1
+      served: true
+      storage: true
+      schema:
+        openAPIV3Schema:
+          description: 'NetworkAttachmentDefinition is a CRD schema specified by the Network Plumbing
+            Working Group to express the intent for attaching pods to one or more logical or physical
+            networks. More information available at: https://github.com/k8snetworkplumbingwg/multi-net-spec'
+          type: object
+          properties:
+            apiVersion:
+              description: 'APIVersion defines the versioned schema of this represen
+                tation of an object. Servers should convert recognized schemas to the
+                latest internal value, and may reject unrecognized values. More info:
+                https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              type: string
+            kind:
+              description: 'Kind is a string value representing the REST resource this
+                object represents. Servers may infer this from the endpoint the client
+                submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              type: string
+            metadata:
+              type: object
+            spec:
+              description: 'NetworkAttachmentDefinition spec defines the desired state of a network attachment'
+              type: object
+              properties:
+                config:
+                  description: 'NetworkAttachmentDefinition config is a JSON-formatted CNI configuration'
+                  type: string
diff --git a/kubernetes/apps/kube-system/multus/app/helmrelease.yaml b/kubernetes/apps/kube-system/multus/app/helmrelease.yaml
index 4453cff..79a11c2 100644
--- a/kubernetes/apps/kube-system/multus/app/helmrelease.yaml
+++ b/kubernetes/apps/kube-system/multus/app/helmrelease.yaml
@@ -1,5 +1,5 @@
 ---
-# yaml-language-server: $schema=https://raw.githubusercontent.com/fluxcd-community/flux2-schemas/main/helmrelease-helm-v2beta2.json
+# yaml-language-server: $schema=https://raw.githubusercontent.com/bjw-s/helm-charts/main/charts/other/app-template/schemas/helmrelease-helm-v2.schema.json
 apiVersion: helm.toolkit.fluxcd.io/v2
 kind: HelmRelease
 metadata:
@@ -8,33 +8,207 @@ spec:
   interval: 30m
   chart:
     spec:
-      chart: multus
-      version: 5.0.4
+      chart: app-template
+      version: 3.3.2
+      interval: 30m
       sourceRef:
         kind: HelmRepository
-        name: angelnu
+        name: bjw-s
         namespace: flux-system
-      interval: 30m
+
   values:
-    image:
-      repository: ghcr.io/k8snetworkplumbingwg/multus-cni
-      tag: v4.0.2-thick
+    configMaps:
+      daemon-config:
+        data:
+          daemon-config.json: |
+            {
+              "cniVersion": "0.3.1",
+              "logToStderr": true,
+              "logLevel": "verbose",
+              "binDir": "/opt/cni/bin",
+              "chrootDir": "/hostroot",
+              "cniConfigDir": "/host/etc/cni/net.d",
+              "confDir": "/host/etc/cni/net.d",
+              "multusAutoconfigDir": "/host/etc/cni/net.d",
+              "multusConfigFile": "auto",
+              "socketDir": "/host/run/multus/"
+            }
 
-    cni:
-      image:
-        repository: ghcr.io/angelnu/cni-plugins
-        tag: 1.5.1
+    controllers:
+      uninstall:
+        type: job
 
-      paths:
-        config: /etc/cni/net.d
-        bin: /opt/cni/bin
+        annotations:
+          # This is what defines this resource as a hook. Without this line, the
+          # job is considered part of the release.
+          helm.sh/hook: pre-delete, pre-upgrade
+          helm.sh/hook-weight: "-5"
+          helm.sh/hook-delete-policy: hook-succeeded
 
-    resources:
-      requests:
-        cpu: 5m
-        memory: 200Mi
-      limits:
-        memory: 200Mi
+        pod:
+          hostNetwork: true
 
-    hostPaths:
-      netns: /var/run/netns
+        containers:
+          uninstall:
+            image:
+              repository: alpine
+              tag: 3.20.2
+
+            command:
+              - /bin/sh
+              - -c
+
+            args:
+              - |
+                rm -rf /host/etc/cni/net.d/*multus*
+                rm -rf /host/opt/cni/bin/*multus*
+
+      multus:
+        type: daemonset
+
+        annotations:
+          reloader.stakater.com/auto: "true"
+
+        pod:
+          hostNetwork: true
+          hostPID: true
+
+        containers:
+          multus-daemon:
+            image: &image
+              repository: ghcr.io/k8snetworkplumbingwg/multus-cni
+              tag: v4.1.0-thick
+            env:
+              MULTUS_NODE_NAME:
+                valueFrom:
+                  fieldRef:
+                    fieldPath: spec.nodeName
+            resources:
+              requests:
+                cpu: "5m"
+                memory: "96Mi"
+              limits:
+                memory: "96Mi"
+            securityContext:
+              privileged: true
+
+        initContainers:
+          cni-plugins-installer:
+            image:
+              repository: ghcr.io/angelnu/cni-plugins
+              tag: 1.5.1
+            resources:
+              requests:
+                cpu: "10m"
+                memory: "15Mi"
+            securityContext:
+              capabilities:
+                drop:
+                  - ALL
+            terminationMessagePath: /dev/termination-log
+            terminationMessagePolicy: FallbackToLogsOnError
+
+          multus-shim-installer:
+            image: *image
+            command:
+              - /bin/sh
+              - -c
+            args: |
+              set -x
+              cp /usr/src/multus-cni/bin/multus-shim /host/opt/cni/bin/multus-shim
+            resources:
+              requests:
+                cpu: "10m"
+                memory: "15Mi"
+            securityContext:
+              capabilities:
+                drop:
+                  - ALL
+            terminationMessagePath: /dev/termination-log
+            terminationMessagePolicy: FallbackToLogsOnError
+
+    persistence:
+      cni:
+        type: hostPath
+        hostPath: /etc/cni/net.d
+        advancedMounts:
+          uninstall:
+            uninstall:
+              - path: /host/etc/cni/net.d
+          multus:
+            multus-daemon:
+              - path: /host/etc/cni/net.d
+
+      cnibin:
+        type: hostPath
+        hostPath: /opt/cni/bin
+        advancedMounts:
+          uninstall:
+            uninstall:
+              - path: /host/opt/cni/bin
+          multus:
+            cni-plugins-installer:
+              - path: /host/opt/cni/bin
+            multus-shim-installer:
+              - path: /host/opt/cni/bin
+            multus-daemon:
+              # multus-daemon expects that cnibin path must be identical between pod and container host.
+              # e.g. if the cni bin is in '/opt/cni/bin' on the container host side, then it should be
+              # mount to '/opt/cni/bin' in multus-daemon, not to any other directory, like '/opt/bin' or
+              # '/usr/bin'.
+              - path: /opt/cni/bin
+
+      config:
+        type: configMap
+        name: multus-daemon-config
+        advancedMounts:
+          multus:
+            multus-daemon:
+              - path: /etc/cni/net.d/multus.d
+      hostroot:
+        type: hostPath
+        hostPath: /
+        advancedMounts:
+          multus:
+            multus-daemon:
+              - path: /hostroot
+                mountPropagation: HostToContainer
+      host-run:
+        type: hostPath
+        hostPath: /run
+        advancedMounts:
+          multus:
+            multus-daemon:
+              - path: /host/run
+      host-var-lib-cni-multus:
+        type: hostPath
+        hostPath: /var/lib/cni/multus
+        advancedMounts:
+          multus:
+            multus-daemon:
+              - path: /var/lib/cni/multus
+      host-var-lib-kubelet:
+        type: hostPath
+        hostPath: /var/lib/kubelet
+        advancedMounts:
+          multus:
+            multus-daemon:
+              - path: /var/lib/kubelet
+      host-run-k8s-cni-cncf-io:
+        type: hostPath
+        hostPath: /run/k8s.cni.cncf.io
+        advancedMounts:
+          multus:
+            multus-daemon:
+              - path: /run/k8s.cni.cncf.io
+      host-run-netns:
+        type: hostPath
+        hostPath: /var/run/netns/
+        advancedMounts:
+          multus:
+            multus-daemon:
+              - path: /run/netns/
+                mountPropagation: HostToContainer
+
+    serviceAccount:
+      create: true
diff --git a/kubernetes/apps/kube-system/multus/app/kustomization.yaml b/kubernetes/apps/kube-system/multus/app/kustomization.yaml
index 2d7deac..f6a66c9 100644
--- a/kubernetes/apps/kube-system/multus/app/kustomization.yaml
+++ b/kubernetes/apps/kube-system/multus/app/kustomization.yaml
@@ -3,4 +3,6 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-  - ./helmrelease.yaml
\ No newline at end of file
+  - ./helmrelease.yaml
+  - ./rbac.yaml
+  - ./crd.yaml
diff --git a/kubernetes/apps/kube-system/multus/app/rbac.yaml b/kubernetes/apps/kube-system/multus/app/rbac.yaml
new file mode 100644
index 0000000..6078581
--- /dev/null
+++ b/kubernetes/apps/kube-system/multus/app/rbac.yaml
@@ -0,0 +1,43 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: multus
+rules:
+  - apiGroups: ["k8s.cni.cncf.io"]
+    resources:
+      - '*'
+    verbs:
+      - '*'
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+      - pods/status
+    verbs:
+      - get
+      - list
+      - update
+      - watch
+  - apiGroups:
+      - ""
+      - events.k8s.io
+    resources:
+      - events
+    verbs:
+      - create
+      - patch
+      - update
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: multus
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: multus
+subjects:
+  - kind: ServiceAccount
+    name: multus
+    namespace: network