From 71b4f51a61d7e39140b9feb0322e36e3137cbdb5 Mon Sep 17 00:00:00 2001 From: Joseph Hanson Date: Thu, 8 Aug 2024 10:24:53 -0500 Subject: [PATCH 1/4] bond --- kubernetes/apps/kube-system/cilium/app/resources/values.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/apps/kube-system/cilium/app/resources/values.yml b/kubernetes/apps/kube-system/cilium/app/resources/values.yml index 83ccb6ae..0b59fc20 100644 --- a/kubernetes/apps/kube-system/cilium/app/resources/values.yml +++ b/kubernetes/apps/kube-system/cilium/app/resources/values.yml @@ -17,7 +17,7 @@ cni: exclusive: false containerRuntime: integration: containerd -devices: enp+ +devices: bond+ socketLB: enabled: false # supposed to be default off, but it's enabled anyway, and looks fun lol # TODO: 2024-06-02: temporarily turned off to attempt fixing endpoint creation timeout hostNamespaceOnly: true # KubeVirt compatibility -- 2.47.0 From 14f613025913ce024b7e5c1d4f5ff52d368103f4 Mon Sep 17 00:00:00 2001 From: Joseph Hanson Date: Thu, 8 Aug 2024 10:39:19 -0500 Subject: [PATCH 2/4] Configure bond interface. --- kubernetes/bootstrap/helmfile.yaml | 12 ++++---- kubernetes/bootstrap/readme.md | 2 +- kubernetes/bootstrap/talos/talconfig.yaml | 31 +++++++++++++++------ kubernetes/bootstrap/talos/talenv.sops.yaml | 5 ++-- 4 files changed, 33 insertions(+), 17 deletions(-) diff --git a/kubernetes/bootstrap/helmfile.yaml b/kubernetes/bootstrap/helmfile.yaml index b00901a2..2cf36eed 100644 --- a/kubernetes/bootstrap/helmfile.yaml +++ b/kubernetes/bootstrap/helmfile.yaml @@ -26,9 +26,9 @@ releases: version: 1.2.2 values: ["../apps/kube-system/kubelet-csr-approver/app/helm-values.yaml"] needs: ["cilium"] - - name: spegel - namespace: kube-system - chart: oci://ghcr.io/spegel-org/helm-charts/spegel - version: v0.0.23 - values: ["../apps/kube-system/spegel/app/resources/values.yml"] - wait: true + # - name: spegel + # namespace: kube-system + # chart: oci://ghcr.io/spegel-org/helm-charts/spegel + # version: v0.0.23 + # values: ["../apps/kube-system/spegel/app/resources/values.yml"] + # wait: true diff --git a/kubernetes/bootstrap/readme.md b/kubernetes/bootstrap/readme.md index 3d8b73f0..992f4486 100644 --- a/kubernetes/bootstrap/readme.md +++ b/kubernetes/bootstrap/readme.md @@ -18,7 +18,7 @@ talosctl bootstrap --nodes=10.1.1.61 ### Install Cilium & Spegel ```sh -helmfile apply -f kubernetes/bootstrap/talos/apps/helmfile.yaml +helmfile apply -f kubernetes/bootstrap/helmfile.yaml ``` ## Flux Prep diff --git a/kubernetes/bootstrap/talos/talconfig.yaml b/kubernetes/bootstrap/talos/talconfig.yaml index 8b4fe9b7..fe741010 100644 --- a/kubernetes/bootstrap/talos/talconfig.yaml +++ b/kubernetes/bootstrap/talos/talconfig.yaml @@ -4,13 +4,13 @@ clusterName: homelab talosVersion: v1.8.0-alpha.1 kubernetesVersion: 1.30.2 -endpoint: "https://10.1.1.57:6443" +endpoint: "https://${clusterEndpointIP}:6443" -additionalApiServerCertSans: - - 10.1.1.57 +additionalApiServerCertSans: &san + - ${clusterEndpointIP} + - "127.0.0.1" # KubePrism -additionalMachineCertSans: - - 10.1.1.57 +additionalMachineCertSans: *san nodes: - hostname: shadowfax @@ -24,10 +24,25 @@ nodes: partitions: - mountpoint: /var/mnt/nvme1 networkInterfaces: - - interface: enp37s0f1 + - interface: bond0 dhcp: true - - interface: enp37s0f0 - dhcp: false + bond: + mode: 802.3ad + xmitHashPolicy: layer3+4 + lacpRate: fast + miimon: 100 + deviceSelectors: + - hardwareAddr: 04:42:1a:ef:35:74 + driver: ixgbe + - hardwareAddr: 04:42:1a:ef:35:75 + driver: ixgbe + vlans: + - &vlan-iot + vlanId: 30 + mtu: 1500 + dhcp: true + dhcpOptions: + routeMetric: 4096 kernelModules: - name: nvidia - name: nvidia_uvm diff --git a/kubernetes/bootstrap/talos/talenv.sops.yaml b/kubernetes/bootstrap/talos/talenv.sops.yaml index e75d7498..e531a59c 100644 --- a/kubernetes/bootstrap/talos/talenv.sops.yaml +++ b/kubernetes/bootstrap/talos/talenv.sops.yaml @@ -1,4 +1,5 @@ CLUSTER: ENC[AES256_GCM,data:2U1tPNOF/w==,iv:BE6ZEuh9SJirZ48ICFuf7RqnJhfOOu6PjEXwLDPG6vU=,tag:zk5eyFqcOmui6d70LQ7WtA==,type:str] +clusterEndpointIP: ENC[AES256_GCM,data:1gDw0FqQQZ9/,iv:OQ64In7KPn0nqWran1U2/oEHkHSyQsZNM8/beAN1C1M=,tag:diqiZHPcGZ7DVgZGFKJyJw==,type:str] sops: kms: [] gcp_kms: [] @@ -14,8 +15,8 @@ sops: WUlNeVV1T2YxbjJCRU9ubVJheHNBTnMKzgZCLTz1Qo/91EFcHXxdKGosdRKKN/tB VsfaNH/b5S2N8FN1wQ03Dn2nqwCqwiPAoNo8La/7ZHjzvNiXTCOFmQ== -----END AGE ENCRYPTED FILE----- - lastmodified: "2024-06-18T16:49:44Z" - mac: ENC[AES256_GCM,data:OCyq3X7UvExLRAib1mTY1wPC0nvz4+ZQEdzzQ3NkI47aF/YB01clKZTUu4CfsE6px1V0DlmJCtTLP4HZM+BDIGOCF2SyeyzeKwEgc02avlkdjLgvYjRJF21m8yONipXeiyjh8x8AzE83MSvs9n3zzJkwCHNIwz9zpdEKQWsv2Hk=,iv:uB7Lgrv2ujS893c0KgbQW0jh5YOy0rTV99hvdDzJ1N0=,tag:wZLSEQRtRNuIQ6NPV+SH/g==,type:str] + lastmodified: "2024-08-08T13:48:53Z" + mac: ENC[AES256_GCM,data:SA9jJSCbgApT4qJfFNU/RVHhOX7ZdipQ6OmvBa4YqKEriUPD00ddp0musyQobdM3jrTK6P231FzwxYuAOQ+Y+xgWf+ylLyy/zcsVvdJbIzNPTsKwtC5J5zfhyvQK8fnRNP/3sP16X+jJ41iWF3yrPQ7nG7fGidsUPmpGDnGXKZg=,iv:dpHPBXm0OBeDGxbyMAu0qufoCahJb6u7d5KuHoP2d58=,tag:Mrnb8kGacrRvac5HF/BSvg==,type:str] pgp: [] unencrypted_suffix: _unencrypted version: 3.8.1 -- 2.47.0 From f35bdb4296dfa052fe45efc71a90b9776c07f636 Mon Sep 17 00:00:00 2001 From: Joseph Hanson Date: Thu, 8 Aug 2024 10:44:51 -0500 Subject: [PATCH 3/4] trying a few more settings --- .../kube-system/multus/config/net-attach-iot.yaml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/kubernetes/apps/kube-system/multus/config/net-attach-iot.yaml b/kubernetes/apps/kube-system/multus/config/net-attach-iot.yaml index 92c3a9d4..7a9e2001 100644 --- a/kubernetes/apps/kube-system/multus/config/net-attach-iot.yaml +++ b/kubernetes/apps/kube-system/multus/config/net-attach-iot.yaml @@ -12,14 +12,21 @@ spec: "plugins": [ { "type": "macvlan", - "master": "bond0", + "master": "bond0.30", "mode": "bridge", + "capabilities": { + "ips": true + }, "ipam": { "type": "static", "routes": [ { "dst": "10.1.2.0/24", "gw": "10.1.3.1" } ] } + }, + { + "capabilities": { "mac": true }, + "type": "tuning" } ] - } \ No newline at end of file + } -- 2.47.0 From 7d3aefadf2bc4ca801d1d13ae5c42cfdbb58c685 Mon Sep 17 00:00:00 2001 From: Joseph Hanson Date: Thu, 8 Aug 2024 10:49:22 -0500 Subject: [PATCH 4/4] less noise --- kubernetes/apps/kube-system/multus/app/helmrelease.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/apps/kube-system/multus/app/helmrelease.yaml b/kubernetes/apps/kube-system/multus/app/helmrelease.yaml index 79a11c28..fb00fdc0 100644 --- a/kubernetes/apps/kube-system/multus/app/helmrelease.yaml +++ b/kubernetes/apps/kube-system/multus/app/helmrelease.yaml @@ -24,7 +24,7 @@ spec: { "cniVersion": "0.3.1", "logToStderr": true, - "logLevel": "verbose", + "logLevel": "info", "binDir": "/opt/cni/bin", "chrootDir": "/hostroot", "cniConfigDir": "/host/etc/cni/net.d", -- 2.47.0