From 25dbfb4019a262d76c270fa4d9d376c8d728752b Mon Sep 17 00:00:00 2001 From: Joseph Hanson Date: Thu, 18 Apr 2024 14:05:38 -0500 Subject: [PATCH] Manage Cilium via flux again. --- .../cilium/app/bgppeeringpolicy.yaml | 37 +++++++++ .../kube-system/cilium/app/helmrelease.yaml | 75 +++++++++++++++++++ .../kube-system/cilium/app/kustomization.yaml | 13 ++++ .../cilium/app/kustomizeconfig.yaml | 7 ++ .../cilium/app/resources/values.yml | 59 +++++++++++++++ kubernetes/apps/kube-system/cilium/ks.yaml | 19 +++++ .../apps/kube-system/kustomization.yaml | 2 +- 7 files changed, 211 insertions(+), 1 deletion(-) create mode 100644 kubernetes/apps/kube-system/cilium/app/bgppeeringpolicy.yaml create mode 100644 kubernetes/apps/kube-system/cilium/app/helmrelease.yaml create mode 100644 kubernetes/apps/kube-system/cilium/app/kustomization.yaml create mode 100644 kubernetes/apps/kube-system/cilium/app/kustomizeconfig.yaml create mode 100644 kubernetes/apps/kube-system/cilium/app/resources/values.yml create mode 100644 kubernetes/apps/kube-system/cilium/ks.yaml diff --git a/kubernetes/apps/kube-system/cilium/app/bgppeeringpolicy.yaml b/kubernetes/apps/kube-system/cilium/app/bgppeeringpolicy.yaml new file mode 100644 index 00000000..f7c0bc34 --- /dev/null +++ b/kubernetes/apps/kube-system/cilium/app/bgppeeringpolicy.yaml @@ -0,0 +1,37 @@ +--- +# yaml-language-server: $schema=https://ks.hsn.dev/cilium.io%2Fciliumbgppeeringpolicy_v2alpha1.json +apiVersion: cilium.io/v2alpha1 +kind: CiliumBGPPeeringPolicy +# comments courtesy of JJGadgets +# MAKE SURE CRDs ARE INSTALLED IN CLUSTER VIA cilium-config ConfigMap OR Cilium HelmRelease/values.yaml (bgpControlPlane.enabled: true), BEFORE THIS IS APPLIED! +# "CiliumBGPPeeringPolicy" Custom Resource will replace the old MetalLB BGP's "bgp-config" ConfigMap +# "CiliumBGPPeeringPolicy" is used with `bgpControlPlane.enabled: true` which uses GoBGP, NOT the old `bgp.enabled: true` which uses MetalLB +metadata: + name: bgp-loadbalancer-ip-main +spec: + nodeSelector: + matchLabels: + kubernetes.io/os: "linux" # match all Linux nodes, change this to match more granularly if more than 1 PeeringPolicy is to be used throughout cluster + virtualRouters: + - localASN: 64512 + exportPodCIDR: false + serviceSelector: # this replaces address-pools, instead of defining the range of IPs that can be assigned to LoadBalancer services, now services have to match below selectors for their LB IPs to be announced + matchExpressions: + - { + key: thisFakeSelector, + operator: NotIn, + values: ["will-match-and-announce-all-services"], + } + neighbors: + - peerAddress: "10.1.1.1/32" # unlike bgp-config ConfigMap, peerAddress needs to be in CIDR notation + peerASN: 64512 + +--- +# yaml-language-server: $schema=https://ks.hsn.dev/cilium.io/ciliumloadbalancerippool_v2alpha1.json +apiVersion: "cilium.io/v2alpha1" +kind: CiliumLoadBalancerIPPool +metadata: + name: main-pool +spec: + cidrs: + - cidr: 10.45.0.1/24 \ No newline at end of file diff --git a/kubernetes/apps/kube-system/cilium/app/helmrelease.yaml b/kubernetes/apps/kube-system/cilium/app/helmrelease.yaml new file mode 100644 index 00000000..d5024b66 --- /dev/null +++ b/kubernetes/apps/kube-system/cilium/app/helmrelease.yaml @@ -0,0 +1,75 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/fluxcd-community/flux2-schemas/main/helmrelease-helm-v2beta2.json +apiVersion: helm.toolkit.fluxcd.io/v2beta2 +kind: HelmRelease +metadata: + name: cilium + namespace: kube-system +spec: + interval: 30m + chart: + spec: + chart: cilium + version: 1.15.4 + sourceRef: + kind: HelmRepository + name: cilium + namespace: flux-system + install: + remediation: + retries: 3 + upgrade: + cleanupOnFail: true + remediation: + strategy: rollback + retries: 3 + values: + hubble: + serviceMonitor: + enabled: true + relay: + prometheus: + serviceMonitor: + enabled: true + ui: + enabled: true + rollOutPods: true + ingress: + className: "internal-nginx" + hosts: + - &host hubble.jahanson.tech + tls: + - hosts: + - *host + metrics: + enabled: + - dns:query + - drop + - tcp + - flow + - port-distribution + - icmp + - http + serviceMonitor: + enabled: true + dashboards: + enabled: true + annotations: + grafana_folder: Cilium + operator: + prometheus: + enabled: true + serviceMonitor: + enabled: true + dashboards: + enabled: true + annotations: + grafana_folder: Cilium + prometheus: + enabled: true + serviceMonitor: + enabled: true + dashboards: + enabled: true + annotations: + grafana_folder: Cilium diff --git a/kubernetes/apps/kube-system/cilium/app/kustomization.yaml b/kubernetes/apps/kube-system/cilium/app/kustomization.yaml new file mode 100644 index 00000000..52b921ea --- /dev/null +++ b/kubernetes/apps/kube-system/cilium/app/kustomization.yaml @@ -0,0 +1,13 @@ +--- +# yaml-language-server: $schema=https://json.schemastore.org/kustomization +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ./bgppeeringpolicy.yaml + - ./helmrelease.yaml +configMapGenerator: + - name: cilium-helm-values + files: + - values.yaml=./resources/values.yml +configurations: + - kustomizeconfig.yaml \ No newline at end of file diff --git a/kubernetes/apps/kube-system/cilium/app/kustomizeconfig.yaml b/kubernetes/apps/kube-system/cilium/app/kustomizeconfig.yaml new file mode 100644 index 00000000..1fcad09f --- /dev/null +++ b/kubernetes/apps/kube-system/cilium/app/kustomizeconfig.yaml @@ -0,0 +1,7 @@ +--- +nameReference: + - kind: ConfigMap + version: v1 + fieldSpecs: + - path: spec/valuesFrom/name + kind: HelmRelease \ No newline at end of file diff --git a/kubernetes/apps/kube-system/cilium/app/resources/values.yml b/kubernetes/apps/kube-system/cilium/app/resources/values.yml new file mode 100644 index 00000000..0477ead5 --- /dev/null +++ b/kubernetes/apps/kube-system/cilium/app/resources/values.yml @@ -0,0 +1,59 @@ +cluster: + name: homelab + id: 1 +bandwidthManager: + enabled: true + bbr: true +bpf: + masquerade: true + tproxy: true + +autoDirectNodeRoutes: true +ipv4NativeRoutingCIDR: 10.244.0.0/16 +routingMode: native + +loadBalancer: + algorithm: maglev + mode: dsr + +containerRuntime: + integration: containerd + +localRedirectPolicy: true +operator: + rollOutPods: true +ipam: + mode: kubernetes +kubeProxyReplacement: true +k8sServiceHost: 127.0.0.1 +k8sServicePort: 7445 +rollOutCiliumPods: true +cgroup: + automount: + enabled: false + hostRoot: /sys/fs/cgroup +bgp: + enabled: false + announce: + loadbalancerIP: true + podCIDR: false +bgpControlPlane: + enabled: true +securityContext: + capabilities: + ciliumAgent: + - CHOWN + - KILL + - NET_ADMIN + - NET_RAW + - IPC_LOCK + - SYS_ADMIN + - SYS_RESOURCE + - DAC_OVERRIDE + - FOWNER + - SETGID + - SETUID + cleanCiliumState: + - NET_ADMIN + - SYS_ADMIN + - SYS_RESOURCE \ No newline at end of file diff --git a/kubernetes/apps/kube-system/cilium/ks.yaml b/kubernetes/apps/kube-system/cilium/ks.yaml new file mode 100644 index 00000000..9c1ef7f8 --- /dev/null +++ b/kubernetes/apps/kube-system/cilium/ks.yaml @@ -0,0 +1,19 @@ +--- +# yaml-language-server: $schema=https://raw.githubusercontent.com/fluxcd-community/flux2-schemas/main/kustomization-kustomize-v1.json +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: &app cilium + namespace: flux-system +spec: + targetNamespace: kube-system + commonMetadata: + labels: + app.kubernetes.io/name: *app + interval: 10m + path: "./kubernetes/apps/kube-system/cilium/app" + prune: true + sourceRef: + kind: GitRepository + name: homelab + wait: true \ No newline at end of file diff --git a/kubernetes/apps/kube-system/kustomization.yaml b/kubernetes/apps/kube-system/kustomization.yaml index c07d822a..b2b9649f 100644 --- a/kubernetes/apps/kube-system/kustomization.yaml +++ b/kubernetes/apps/kube-system/kustomization.yaml @@ -6,7 +6,7 @@ resources: # Pre Flux-Kustomizations - ./namespace.yaml # Flux-Kustomizations - # - ./cilium/ks.yaml + - ./cilium/ks.yaml - ./descheduler/ks.yaml - ./fstrim/ks.yaml - ./metrics-server/ks.yaml