From dcb8b63e393b712b52e35d3189c9c0973441e904 Mon Sep 17 00:00:00 2001
From: Joseph Hanson <joe@veri.dev>
Date: Thu, 18 Jan 2024 07:39:04 -0600
Subject: [PATCH] Add nvidia containerd template for nenya.

---
 .../main/playbooks/cluster-installation.yaml  | 12 +++++++
 .../playbooks/files/config.nvidia.toml.tmpl   | 35 +++++++++++++++++++
 2 files changed, 47 insertions(+)
 create mode 100644 ansible/main/playbooks/files/config.nvidia.toml.tmpl

diff --git a/ansible/main/playbooks/cluster-installation.yaml b/ansible/main/playbooks/cluster-installation.yaml
index 88762163..d5b29191 100644
--- a/ansible/main/playbooks/cluster-installation.yaml
+++ b/ansible/main/playbooks/cluster-installation.yaml
@@ -80,6 +80,7 @@
 
     # TODO: Replace this with embedded spegel in the future
     - name: Copy custom containerd configuration
+      when: inventory_hostname != 'nenya'
       notify: Restart Kubernetes
       ansible.builtin.copy:
         src: files/config.toml.tmpl
@@ -88,6 +89,17 @@
         group: root
         mode: "0644"
 
+    - name: Copy custom containerd configuration
+      when: inventory_hostname == 'nenya'
+      notify: Restart Kubernetes
+      ansible.builtin.copy:
+        src: files/config.nvidia.toml.tmpl
+        dest: /var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl
+        owner: root
+        group: root
+        mode: "0644"
+
+
   handlers:
     - name: Restart Kubernetes
       ansible.builtin.systemd:
diff --git a/ansible/main/playbooks/files/config.nvidia.toml.tmpl b/ansible/main/playbooks/files/config.nvidia.toml.tmpl
new file mode 100644
index 00000000..7e2fe10b
--- /dev/null
+++ b/ansible/main/playbooks/files/config.nvidia.toml.tmpl
@@ -0,0 +1,35 @@
+version = 2
+
+[plugins."io.containerd.internal.v1.opt"]
+  path = "/var/lib/rancher/k3s/agent/containerd"
+
+[plugins."io.containerd.grpc.v1.cri"]
+  stream_server_address = "127.0.0.1"
+  stream_server_port = "10010"
+  enable_selinux = false
+  enable_unprivileged_ports = true
+  enable_unprivileged_icmp = true
+  sandbox_image = "registry.k8s.io/pause:3.9"
+
+[plugins."io.containerd.grpc.v1.cri".containerd]
+  snapshotter = "overlayfs"
+  disable_snapshot_annotations = true
+
+
+[plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
+  [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia]
+    privileged_without_host_devices = false
+    runtime_engine = ""
+    runtime_root = ""
+    runtime_type = "io.containerd.runc.v2"
+    [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.nvidia.options]
+      BinaryName = "/usr/bin/nvidia-container-runtime"
+
+[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+  runtime_type = "io.containerd.runc.v2"
+
+[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
+  SystemdCgroup = true
+
+[plugins."io.containerd.grpc.v1.cri".registry]
+  config_path = "/var/lib/rancher/k3s/agent/etc/containerd/certs.d"