feat: motd, systemd notifications, restic backups (#93)

* flesh out impermanence

* glances

* hack

* hacking in plex and tautulli

* hack

* hacking

* feat: motd AND systemd failure notifications

---------

Co-authored-by: Truxnell <9149206+truxnell@users.noreply.github.com>

docs/vm/backups.md

@@ -0,0 +1,52 @@
+Nightly Backups are facilitated by nixos restic module & a helper module ive written.
+
+These run to my NAS 'local' and cloudflare R2 'remote'
+
+They are a systemd timer/service so you can query or trigger a manual run with
+
+```bash
+truxnell@daedalus ~> systemctl status restic-backups-lidarr-local.timer
+● restic-backups-lidarr-local.timer
+     Loaded: loaded (/etc/systemd/system/restic-backups-lidarr-local.timer; enabled; preset: enabled)
+     Active: active (waiting) since Sat 2024-04-13 19:50:23 AEST; 12h ago
+    Trigger: Mon 2024-04-15 03:03:22 AEST; 18h left
+   Triggers: ● restic-backups-lidarr-local.service
+
+truxnell@daedalus ~> systemctl status restic-backups-lidarr-local.service
+○ restic-backups-lidarr-local.service
+     Loaded: loaded (/etc/systemd/system/restic-backups-lidarr-local.service; linked; preset: enabled)
+     Active: inactive (dead) since Sun 2024-04-14 04:20:02 AEST; 4h 14min ago
+TriggeredBy: ● restic-backups-lidarr-local.timer
+    Process: 774197 ExecStartPre=/nix/store/vw03a7pxjj1sf59rk1p65nbv1jjwba1b-unit-script-restic-backups-lidarr-local-pre-start/bin/restic-backups-lidarr-local-pre-start (code=exited, status=0/SUCCESS)
+    Process: 774210 ExecStart=/nix/store/cbg69gn45canlna2fsy7y9g72kv5q9y3-restic-0.16.4/bin/restic backup --exclude-file=/nix/store/bk1cxh78aaxbnh22jcxw18jadhk7j2b7-exclude-patterns --files-from=/run/restic-backups-lidarr-local/includes >
+    Process: 774239 ExecStart=/nix/store/cbg69gn45canlna2fsy7y9g72kv5q9y3-restic-0.16.4/bin/restic forget --prune --keep-daily 7 --keep-weekly 5 --keep-monthly 12 (code=exited, status=0/SUCCESS)
+    Process: 774251 ExecStart=/nix/store/cbg69gn45canlna2fsy7y9g72kv5q9y3-restic-0.16.4/bin/restic check (code=exited, status=0/SUCCESS)
+    Process: 774381 ExecStopPost=/nix/store/nk9a304p38yxfgb6f63s6nq1c4icjplb-unit-script-restic-backups-lidarr-local-post-stop/bin/restic-backups-lidarr-local-post-stop (code=exited, status=0/SUCCESS)
+   Main PID: 774251 (code=exited, status=0/SUCCESS)
+         IP: 0B in, 0B out
+        CPU: 21.961s
+
+```
+
+Checking snapshots
+
+```bash
+truxnell@daedalus ~ [3]> sudo restic-lidarr-local snapshots
+repository a2847581 opened (version 2, compression level auto)
+ID        Time                 Host        Tags        Paths
+----------------------------------------------------------------------------
+aef44e7c  2024-04-13 19:56:14  daedalus                /persist/nixos/lidarr
+b96f4b94  2024-04-14 04:19:41  daedalus                /persist/nixos/lidarr
+----------------------------------------------------------------------------
+```
+
+Testing a restore (would do --target / for a real restore)
+Would just have to pause service, run restore, then re-start service.
+
+```bash
+truxnell@daedalus ~ [1]> sudo restic-lidarr-local restore --target /tmp/lidarr/ latest
+repository a2847581 opened (version 2, compression level auto)
+[0:00] 100.00%  2 / 2 index files loaded
+restoring <Snapshot b96f4b94 of [/persist/nixos/lidarr] at 2024-04-14 04:19:41.533770692 +1000 AEST by root@daedalus> to /tmp/lidarr/
+Summary: Restored 52581 files/dirs (11.025 GiB) in 1:37
+```

docs/vm/features.md

@@ -0,0 +1,8 @@
+stable channel for reliable services, with unstable for desktop apps, containers for 'server' apps
+renovate for automated lockfile and container updates
+strong CI on all PR's to ensure system updates from main branch are reliable
+leans into systemd, meaning everything can be managed, viewed and debugged with a consistent interface (Ive come around to loving systemd...)
+cockpit on all servers for easy viewing of stauts logs, etc
+sops-nix for secrets
+nightly restic backups (diff) to local and cloud, with failure notifications and simple command-line wrapper for restores
+gatus monitoring for apps, dns and servers, dynamicaly built from nix across all enabled nodes

flake.lock

@@ -1,5 +1,46 @@
 {
   "nodes": {
+    "crane": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1699217310,
+        "narHash": "sha256-xpW3VFUG7yE6UE6Wl0dhqencuENSkV7qpnpe9I8VbPw=",
+        "owner": "ipetkov",
+        "repo": "crane",
+        "rev": "d535642bbe6f377077f7c23f0febb78b1463f449",
+        "type": "github"
+      },
+      "original": {
+        "owner": "ipetkov",
+        "ref": "v0.15.0",
+        "repo": "crane",
+        "type": "github"
+      }
+    },
+    "dream2nix": {
+      "inputs": {
+        "nixpkgs": [
+          "nix-inspect",
+          "nci",
+          "nixpkgs"
+        ],
+        "purescript-overlay": "purescript-overlay",
+        "pyproject-nix": "pyproject-nix"
+      },
+      "locked": {
+        "lastModified": 1709959559,
+        "narHash": "sha256-Gb+tUU+clGKVBwiznTQf0emZZ+heALqoVwUgI0O13L8=",
+        "owner": "nix-community",
+        "repo": "dream2nix",
+        "rev": "42838c590971da17a4b6483962707b7fb7b8b9a7",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "dream2nix",
+        "type": "github"
+      }
+    },
     "flake-compat": {
       "flake": false,
       "locked": {
@@ -55,6 +96,49 @@
         "type": "github"
       }
     },
+    "mk-naked-shell": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1681286841,
+        "narHash": "sha256-3XlJrwlR0nBiREnuogoa5i1b4+w/XPe0z8bbrJASw0g=",
+        "owner": "yusdacra",
+        "repo": "mk-naked-shell",
+        "rev": "7612f828dd6f22b7fb332cc69440e839d7ffe6bd",
+        "type": "github"
+      },
+      "original": {
+        "owner": "yusdacra",
+        "repo": "mk-naked-shell",
+        "type": "github"
+      }
+    },
+    "nci": {
+      "inputs": {
+        "crane": "crane",
+        "dream2nix": "dream2nix",
+        "mk-naked-shell": "mk-naked-shell",
+        "nixpkgs": [
+          "nix-inspect",
+          "nixpkgs"
+        ],
+        "parts": "parts",
+        "rust-overlay": "rust-overlay",
+        "treefmt": "treefmt"
+      },
+      "locked": {
+        "lastModified": 1710137478,
+        "narHash": "sha256-+hbUWY1PEItyx3CBOGsHlJEDO2wRY2N1mpBhiLBblck=",
+        "owner": "yusdacra",
+        "repo": "nix-cargo-integration",
+        "rev": "f3cc8751427e16ec48c0467357b3f3979a53ae9c",
+        "type": "github"
+      },
+      "original": {
+        "owner": "yusdacra",
+        "repo": "nix-cargo-integration",
+        "type": "github"
+      }
+    },
     "nix-index-database": {
       "inputs": {
         "nixpkgs": [
@@ -75,6 +159,26 @@
         "type": "github"
       }
     },
+    "nix-inspect": {
+      "inputs": {
+        "nci": "nci",
+        "nixpkgs": "nixpkgs",
+        "parts": "parts_2"
+      },
+      "locked": {
+        "lastModified": 1712895608,
+        "narHash": "sha256-JichXRSfTLfy+7fhbTvA89rQLkqsY2eHgEAeAHWbA9s=",
+        "owner": "bluskript",
+        "repo": "nix-inspect",
+        "rev": "3d0fea2bb246130825548fce331093ee9cc9c20b",
+        "type": "github"
+      },
+      "original": {
+        "owner": "bluskript",
+        "repo": "nix-inspect",
+        "type": "github"
+      }
+    },
     "nix-vscode-extensions": {
       "inputs": {
         "flake-compat": "flake-compat",
@@ -115,16 +219,16 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1712437997,
+        "lastModified": 1709961763,
-        "narHash": "sha256-g0whLLwRvgO2FsyhY8fNk+TWenS3jg5UdlWL4uqgFeo=",
+        "narHash": "sha256-6H95HGJHhEZtyYA3rIQpvamMKAGoa8Yh2rFV29QnuGw=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "e38d7cb66ea4f7a0eb6681920615dfcc30fc2920",
+        "rev": "3030f185ba6a4bf4f18b87f345f104e6a6961f34",
         "type": "github"
       },
       "original": {
         "owner": "nixos",
-        "ref": "nixos-23.11",
+        "ref": "nixos-unstable",
         "repo": "nixpkgs",
         "type": "github"
       }
@@ -161,6 +265,22 @@
         "type": "github"
       }
     },
+    "nixpkgs_2": {
+      "locked": {
+        "lastModified": 1712437997,
+        "narHash": "sha256-g0whLLwRvgO2FsyhY8fNk+TWenS3jg5UdlWL4uqgFeo=",
+        "owner": "nixos",
+        "repo": "nixpkgs",
+        "rev": "e38d7cb66ea4f7a0eb6681920615dfcc30fc2920",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nixos",
+        "ref": "nixos-23.11",
+        "repo": "nixpkgs",
+        "type": "github"
+      }
+    },
     "nur": {
       "locked": {
         "lastModified": 1712573039,
@@ -176,18 +296,143 @@
         "type": "github"
       }
     },
+    "parts": {
+      "inputs": {
+        "nixpkgs-lib": [
+          "nix-inspect",
+          "nci",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1709336216,
+        "narHash": "sha256-Dt/wOWeW6Sqm11Yh+2+t0dfEWxoMxGBvv3JpIocFl9E=",
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "rev": "f7b3c975cf067e56e7cda6cb098ebe3fb4d74ca2",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "type": "github"
+      }
+    },
+    "parts_2": {
+      "inputs": {
+        "nixpkgs-lib": [
+          "nix-inspect",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1709336216,
+        "narHash": "sha256-Dt/wOWeW6Sqm11Yh+2+t0dfEWxoMxGBvv3JpIocFl9E=",
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "rev": "f7b3c975cf067e56e7cda6cb098ebe3fb4d74ca2",
+        "type": "github"
+      },
+      "original": {
+        "owner": "hercules-ci",
+        "repo": "flake-parts",
+        "type": "github"
+      }
+    },
+    "purescript-overlay": {
+      "inputs": {
+        "nixpkgs": [
+          "nix-inspect",
+          "nci",
+          "dream2nix",
+          "nixpkgs"
+        ],
+        "slimlock": "slimlock"
+      },
+      "locked": {
+        "lastModified": 1696022621,
+        "narHash": "sha256-eMjFmsj2G1E0Q5XiibUNgFjTiSz0GxIeSSzzVdoN730=",
+        "owner": "thomashoneyman",
+        "repo": "purescript-overlay",
+        "rev": "047c7933abd6da8aa239904422e22d190ce55ead",
+        "type": "github"
+      },
+      "original": {
+        "owner": "thomashoneyman",
+        "repo": "purescript-overlay",
+        "type": "github"
+      }
+    },
+    "pyproject-nix": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1702448246,
+        "narHash": "sha256-hFg5s/hoJFv7tDpiGvEvXP0UfFvFEDgTdyHIjDVHu1I=",
+        "owner": "davhau",
+        "repo": "pyproject.nix",
+        "rev": "5a06a2697b228c04dd2f35659b4b659ca74f7aeb",
+        "type": "github"
+      },
+      "original": {
+        "owner": "davhau",
+        "ref": "dream2nix",
+        "repo": "pyproject.nix",
+        "type": "github"
+      }
+    },
     "root": {
       "inputs": {
         "home-manager": "home-manager",
         "nix-index-database": "nix-index-database",
+        "nix-inspect": "nix-inspect",
         "nix-vscode-extensions": "nix-vscode-extensions",
         "nixos-hardware": "nixos-hardware",
-        "nixpkgs": "nixpkgs",
+        "nixpkgs": "nixpkgs_2",
         "nixpkgs-unstable": "nixpkgs-unstable",
         "nur": "nur",
         "sops-nix": "sops-nix"
       }
     },
+    "rust-overlay": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1710123130,
+        "narHash": "sha256-EoGL/WSM1M2L099Q91mPKO/FRV2iu2ZLOEp3y5sLfiE=",
+        "owner": "oxalica",
+        "repo": "rust-overlay",
+        "rev": "73aca260afe5d41d3ebce932c8d896399c9d5174",
+        "type": "github"
+      },
+      "original": {
+        "owner": "oxalica",
+        "repo": "rust-overlay",
+        "type": "github"
+      }
+    },
+    "slimlock": {
+      "inputs": {
+        "nixpkgs": [
+          "nix-inspect",
+          "nci",
+          "dream2nix",
+          "purescript-overlay",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1688610262,
+        "narHash": "sha256-Wg0ViDotFWGWqKIQzyYCgayeH8s4U1OZcTiWTQYdAp4=",
+        "owner": "thomashoneyman",
+        "repo": "slimlock",
+        "rev": "b5c6cdcaf636ebbebd0a1f32520929394493f1a6",
+        "type": "github"
+      },
+      "original": {
+        "owner": "thomashoneyman",
+        "repo": "slimlock",
+        "type": "github"
+      }
+    },
     "sops-nix": {
       "inputs": {
         "nixpkgs": [
@@ -223,6 +468,28 @@
         "repo": "default",
         "type": "github"
       }
+    },
+    "treefmt": {
+      "inputs": {
+        "nixpkgs": [
+          "nix-inspect",
+          "nci",
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1710088047,
+        "narHash": "sha256-eSqKs6ZCsX9xJyNYLeMDMrxzIDsYtaWClfZCOp0ok6Y=",
+        "owner": "numtide",
+        "repo": "treefmt-nix",
+        "rev": "720322c5352d7b7bd2cb3601a9176b0e91d1de7d",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "treefmt-nix",
+        "type": "github"
+      }
     }
   },
   "root": "root",

flake.nix

@@ -40,6 +40,9 @@
       url = "github:nix-community/nix-index-database";
       inputs.nixpkgs.follows = "nixpkgs";
     };
+    nix-inspect = {
+      url = "github:bluskript/nix-inspect";
+    };
   };
   outputs =
     { self

nixos/hosts/daedalus/default.nix

@@ -25,6 +25,9 @@
     gatus.enable = true;
     sabnzbd.enable = true;
     qbittorrent.enable = true;
+    prowlarr.enable = true;
+
+    backrest.enable = true;
   };
   mySystem.nasFolder = "/tank";
 
@@ -34,6 +37,9 @@
   };
 
   mySystem.services.nfs.enable = true;
+  mySystem.system.motd.networkInterfaces = [ "eno1" ];
+
+
 
   boot = {
 

nixos/hosts/durandal/default.nix

@@ -15,11 +15,11 @@
     plex.enable = true;
     tautulli.enable = true;
     syncthing.enable = true;
-
   };
 
   mySystem.nfs.nas.enable = true;
   mySystem.persistentFolder = "/persistent/nixos";
+  mySystem.system.motd.networkInterfaces = [ "eno1" ];
 
   boot = {
 

nixos/modules/nixos/containers/arr/lidarr/default.nix

@@ -58,6 +58,7 @@ in
       };
     };
 
+
     mySystem.services.homepage.media-services = mkIf cfg.addToHomepage [
       {
         Lidarr = {
@@ -84,5 +85,11 @@ in
       conditions = [ "[CONNECTED] == true" "[STATUS] == 200" "[RESPONSE_TIME] < 50" ];
     }];
 
+    services.restic.backups."${app}-local" = config.lib.mySystem.mkRestic
+      {
+        inherit app;
+        excludePaths = [ "Backups" ];
+        paths = [ persistentFolder ];
+      };
   };
 }

nixos/modules/nixos/containers/backrest/default.nix

@@ -0,0 +1,75 @@
+{ lib
+, config
+, pkgs
+, ...
+}:
+with lib;
+let
+  app = "backrest";
+  image = "garethgeorge/backrest:v0.17.1@sha256:c2cf1897f5a6972516d7f9ce3adbbbb258fde79c9101d3d01edfaeca0a30ea6c";
+  user = "568"; #string
+  group = "568"; #string
+  port = 9898; #int
+  cfg = config.mySystem.services.${app};
+  persistentFolder = "${config.mySystem.persistentFolder}/${app}";
+in
+{
+  options.mySystem.services.${app} =
+    {
+      enable = mkEnableOption "${app}";
+      addToHomepage = mkEnableOption "Add ${app} to homepage" // { default = true; };
+    };
+
+  config = mkIf cfg.enable {
+    # ensure folder exist and has correct owner/group
+    systemd.tmpfiles.rules = [
+      "d ${persistentFolder}/config 0755 ${user} ${group} -"
+      "d ${persistentFolder}/data 0755 ${user} ${group} -"
+      "d ${persistentFolder}/cache 0755 ${user} ${group} -"
+    ];
+
+    virtualisation.oci-containers.containers.${app} = {
+      image = "${image}";
+      user = "${user}:${group}";
+      environment = {
+        BACKREST_PORT = "9898";
+        BACKREST_DATA = "/data";
+        BACKREST_CONFIG = "/config/config.json";
+        XDG_CACHE_HOME = "/cache";
+      };
+      volumes = [
+        "${persistentFolder}/config:/config:rw"
+        "${persistentFolder}/data:/data:rw"
+        "${persistentFolder}/cache:/cache:rw"
+        "${config.mySystem.nasFolder}/backup/nixos/nixos:/repos:rw"
+        "/etc/localtime:/etc/localtime:ro"
+      ];
+      labels = config.lib.mySystem.mkTraefikLabels {
+        name = app;
+        inherit port;
+      };
+    };
+
+    mySystem.services.homepage.infrastructure-services = mkIf cfg.addToHomepage [
+      {
+        Backrest = {
+          icon = "${app}.png";
+          href = "https://${app}.${config.mySystem.domain}";
+          ping = "https://${app}.${config.mySystem.domain}";
+          description = "Local restic backup browser";
+          container = "${app}";
+        };
+      }
+    ];
+
+    mySystem.services.gatus.monitors = mkIf config.mySystem.services.gatus.enable [{
+
+      name = app;
+      group = "infrastructure";
+      url = "https://${app}.${config.mySystem.domain}";
+      interval = "30s";
+      conditions = [ "[CONNECTED] == true" "[STATUS] == 200" "[RESPONSE_TIME] < 50" ];
+    }];
+
+  };
+}

nixos/modules/nixos/containers/default.nix

@@ -7,5 +7,6 @@
     ./qbittorrent
     ./plex
     ./tautulli
+    ./backrest
   ];
 }

nixos/modules/nixos/default.nix

@@ -1,4 +1,5 @@
 { lib, config, ... }:
+with lib;
 {
   imports = [
     ./system
@@ -11,31 +12,33 @@
     ./lib.nix
   ];
 
-  options.mySystem.persistentFolder = lib.mkOption {
+  options.mySystem.persistentFolder = mkOption {
-    type = lib.types.str;
+    type = types.str;
     description = "persistent folder for nixos mutable files";
     default = "/persist/nixos";
   };
 
-  options.mySystem.nasFolder = lib.mkOption {
+  options.mySystem.nasFolder = mkOption {
-    type = lib.types.str;
+    type = types.str;
     description = "folder where nas mounts reside";
     default = "/mnt/nas";
   };
-  options.mySystem.domain = lib.mkOption {
+  options.mySystem.domain = mkOption {
-    type = lib.types.str;
+    type = types.str;
     description = "domain for hosted services";
     default = "";
   };
-  options.mySystem.internalDomain = lib.mkOption {
+  options.mySystem.internalDomain = mkOption {
-    type = lib.types.str;
+    type = types.str;
     description = "domain for local devices";
     default = "";
   };
 
+
   config = {
     systemd.tmpfiles.rules = [
       "d ${config.mySystem.persistentFolder} 777 - - -" #The - disables automatic cleanup, so the file wont be removed after a period
     ];
+
   };
 }

nixos/modules/nixos/lib.nix

@@ -1,7 +1,7 @@
 { lib, config, ... }:
 {
 
-
+  # build up traefik docker labesl
   lib.mySystem.mkTraefikLabels = options: (
     let
       inherit (options) name;
@@ -27,4 +27,31 @@
     }
   );
 
+  # build a restic restore set
+  lib.mySystem.mkRestic = options: (
+    let
+      excludePath = if builtins.hasAttr "excludePath" options then options.excludePath else [ ];
+
+    in
+    {
+      passwordFile = config.sops.secrets."services/restic/password".path;
+      initialize = true;
+      user = "nah";
+      repository = "/tank/backup/nixos/nixos/${options.app}";
+      exclude = options.excludePaths;
+      inherit (options) paths;
+      timerConfig = {
+        OnCalendar = "01:05";
+        Persistent = true;
+        RandomizedDelaySec = "4h";
+      };
+      pruneOpts = [
+        "--keep-daily 7"
+        "--keep-weekly 5"
+        "--keep-monthly 12"
+      ];
+
+    }
+  );
+
 }

nixos/modules/nixos/services/bind/secrets.sops.yaml

@@ -1,8 +1,8 @@
 system:
     networking:
         bind:
-            trux.dev: ENC[AES256_GCM,data:Qr2DNPQrymbiTaZYPN+aFT+njbzEAO4X1qRiVEo5Lz03GsmQ2ct9bmF5VyNGrjKyBVkjEZVeX5istUYXOf7Opq5NA04N7z8uCZCBU2YuzUEVfkz6PXSU1F3NbdBlika+2iogLpDcJRttg3ciNhSrWDiWSlns/VZBKIxjrRX7yGLvCF71UdgaBdCYiTDQNU2kpHQaYLafftppAtLXzWA8HAp6lxsP8a4RISBvA78o1+fr1WzEffjFd8jbdfr61kWhOIygGhdWbVs0taQ5vLzjd4+eN9YFdjNRiHZdktyrJS1PpteJ8FwAZwulDHdKoMvLhm6PnuSw/iTk/CnEXPiaFOKf8g4NxLOcM6kJUgcfiwqorvY6YYQ1j2DfZ2eyiE1TdC/a27oj4ZqoQSKyn+nuILXRu0ptP7IfNEX1QlAeKl2Gi6qgiZLQ1F/67dzJvLAA0onncMkyGV+cHGSSqmb15aBESJ9T/gcTa8b+JjYj++bSL7b5KedHf5VwmoJ6Tug1qPXxubscGP1mD7i7uxZmgW7UYCStAtx+GyIK7dTlCuAjKoLkGxgiQD1elQjuqwJ0RM2YV9cdWwpeFb78sAOSyUfZKfQZDmnutSWzNInW42yLJoNO8URBWsWvc9lpwDLlAGfPo4BXUfzdG+qX5UCFmfPn6AvcYIcLOUz8vEw2YG3byJ1QUYXRIQ851D+aeBte84P41ZlXwC+ov/A9PXztJlz8phCFJQZpe/6LwUqchtwl6CJAvop6nHcgYnQfIrRJGEKkQDLYWLeDrQtovpDqNgZTNkmk0KXeuPGKjPCrXhvmqs4YYUIyQdvGqqli/OLhs7Zq7GOvSSGNWx/wCrptavgxiG/eGBlc1veDOACPdrV4q8LdTfavCTbzNO8lCq8kkzzL9zN7NJYu2vButkkOsmSQ8y0nubpnsxw26a7okeMx5wh1YTZ9dllKS89zVIuXbwuIxRu5z13RrDoOQWm6UGjgvuoTpIsptPfSSA/EGJvy7AQwsWQsmcgm46MLffoRfhD5p6LgwQVVetGGM4iCOmtUDsnFsBbY7KcyPUAw7KI4NacGlAlsJEkdpAlR8HvL229/J8xuOzDSH45nJ53nwH5Ki5WjurS3ovl+ByMg4v5EX4fEY7bWDsYAYzhhQfZrDWFYvJdNEO7j2Wtl7t8gIPiGPOhOnXExZoL6WAauFKer3N9nOE13xqkTJIL9KAJrf3oE39W4i9fWLa3vMknbyKLUu4ycW9r3ndKPk7KIb3kd2TBNqGXUyc6ZomE369lhGs8oajtv6M24V3d9ggE/eHWjUllq35go4YmJG29i7G3vQR5o9HEFxe+RnCqe7j26/lU48+2xDJZtve4uCvAbkU+awLoKz8/n4PQJ309aPPODOMZgA9HWe+I1V93UHZUZuKc0ySsgykru2+dvN8Z53nRwyZWrn5XxVVi5zf+5y1YK/r9SUq2cklny58jr/r34lACLkanXPR2+RMihumcQz00VrTdpadWAsdSk351O86aY9JxSQDqTgvLRZgz4urUCKo9IxhXs3XigIRQlRwPaxUtZuvuSagTqAFnw6qcU7Js2AL/obQ6R6ciU3W+IGid5DZi02GV4Gwxx0FJdXPmx7jOoGghwrZv00h0GCtT4BCqH7VWdEE9ZPShglOhUaPMcN9N7RdN1+iHq4/4KO0McuoTtVCYhHI4lKkq8NJZBP7y3ft8lcsI+PLKUXN7wdy3lcTCYvS8wOVM7WuBncpqdobD/A1TZZm7czEZqi/y7fDPqsQ1g316XNdVqKHzOdg3ejDFIjQST87ep4JnWwaIEFklassPPCy6BJ7sLLPUux7JuSIgaWyMPAbB7U7SY6N5uZl4A1+f14mMb9qheWBqwDcWQ83R6QMjpoum2es7XxXGB2/9oYopslyM9glWigPMJPLojWZydJpZbIiB7o4yQBp/OFozl+AJLij/IIrbtECug6bap27LjajVKRSjQVwvZeOg+omttYcwR6dlqW/sHMNR5pkDhJ5lXzWeyvO1C1WIg2qFLE2WwrgM0/V9JDZHo8Uk7q3NUmnVOYlUyTyrTR/OKFEas7GYBv+/f9l5h4UqQZWzU7S6cMAFbQXYxcX3YzdDIJ4edmuzDdWCuIAy+dQsu1h7cVCRYO3tNKSDyYJ2XymEtNUu8tewicf2t52AF4UCR/3dRr+rgEFbrTpxn1Sm0O8azMIkBiRZSF/hF8qzWZmCEqvMtxWjnZDSAeHVv58/83ZFRx9M42gLqzlc9WaAvKNtIWgTQDOz7lJ3ymWdHdYaFq8z3aOh2IgOBhwAFee9rP1IXu6ASsl9NjlK9TRy1FrTpMzZKRLtqQgXaQKFs20FeoGQR+as7ZBCfIyANF2a/amjQOlV7nAVMGtpPS91DG7NcejHuVklVwBwZ62FAH8Wlm7VuBSKPCJqlbqO3ixsuHtuhUE1IBJNIR5kQbxtkoDjfaIpWB4m2VBUIxmV6d+Bw7lWII0UZPIjoImZ8QnLe67NotgJfRXx/pFnm4zpcSBGMaHpP5HoXVDWbpTjkTZPP99e9v8uWr1qaJfJEBykwAOCRXGM6qh3rj/uqwwUKC8KH1cqOrG5tKudtdE3sWGpV/7YwJMnHEardDCKuKQtOqtWHerZ8b1euhRqQClOB/HVNerAMO1I6mym31btePgRDfcLGPBjL8hE5wC+iuZlYjl4RUHVvhvCVXUwF9aGEv0u7Ut9WfUj2f4YRinI43KawVeQTIPGUD7k6c9RLwVwz3YujY03ruzgDq1urDtTFI2N2PtFR6iiTzk3R9To6Wodjd8lbk8GWYipg1IKORHaQ3e0wgZPbgBI4tj2xwZN1WHIzvM/MLqu8XFW/fnhV3+TZ8JfjNOXFn29Nn79F8rHuil5DUfBQHJh6z/EWQ9+2Wu/B8OA45JqBO4HpLGKN3JmuCuQeqwJi5Rcd9OBJl2OK0Ccm38uGWyYWGsIIL5SefZaAioMoPZs1T/Z7PejwKDamve7sGTIOXvFMIx2S4glqZ3/da9/GeIqCQ5NRG18slP9jbx9RD2GEytsXDJhWo6bgCfKIhk5j0s3vKnQ1cbMpBbRib+RtPsQGfBDw1IasWg3oGZXPclH/zYd9HZsAZN+qY2ny7VZOCY3wswXvCZjda+pYF9I30LcVIfErM19REQn2L3iUqvPJN/Wj+KfosARZAl984FHlvui7IfJN9EuHEMA7c9s9EuyoNWXi7wOcFrt7CBnURk6nNZwq6RbKtSPVCkrRgY9W2sRPlhOXvRFcVM0vRnXAtkpYX0+sVfMQSrmTeS74ippLLwKItjZMgi/M3ZczQkewdMeFbGa0mUZQbjM/bAYQqQ==,iv:amqQUxzTVYI91hO/xbfuAmclf0KLXMwHa7fStRAA4WU=,tag:rMOqMsMfanq45O4Vm2+SSg==,type:str]
+            trux.dev: ENC[AES256_GCM,data:yItoqCfF/QsOPjCdnQa//+3ULoSFaufvFp1v9i9hKOr2yg+Ep/NzUZ9GzDvpBEqKXwlOPV4TV5YIcHX0RFPioQ8LY4snLRwNSmHs3UjvvIw/jpJpUM8iCHMDzl1xcwZFJ3gnNuS5chRQmO19auMe8YQtYL7f6E28g0uQJE5Pcl2K/flVUut7mKVLJAqXM7VZ8Cua3jgctTnx37zyJsxWLfEK9iAzf4rcZoloGM7eyOW1b2eI0b74FQptxUtsAQSCmjuGVlJBp51RZzvLdxGutDZJXWHe6j6fzNj0lOs0vpMRo7/9exbz2q1yw5xC4XAZQN1vuD61Wx/qGyK8vxYNzrwaYLXBVvE67LivMNzayCXapEr3LducDBdwecwS7SPgao4OGIJMmJERIUUMSxvZIlaY6nQUNBkw9aI8CtrHq1bJ+46juJZtJDfjoKmIVgWpaYcq7k8qKKOHhxqJZ+7755TR7m8vcBe520Mj5QnebX4RwK3bLI9WFRv8vKlXDtcvO+zfutRO2hkc8VSb9tuCw75p8vDPCmD1NdVMgX69RvZMxq9NJgF0aPne+8dnoE0aKCSKbEI/bhy3bE07MJ9CWD9BUBHnt+77CALPF0gWOGkVuE+DWccbaVB0waOeJ/+7bM/9y1/PAIV/avfS2durbMCE9bk+LB7bvNSEzNm7ub8Kb+zQP4w2tgKq0RhopmUoVB8X5NkwwCZ5SPcernIjZcOI1O90avjvIh6H+/jdnEFjNplpxzV+BxZjd36f+qgISmNUm3Fbe/+dVAEQUDVt9hB+llXeou/YUA3Av53ncgImgXt9PaXHUtnpHrZin30eIB181Ysx7JKEp8UA7N72b0dh2/kWG7HJgbBnLNTuQdMU7u9hQMt/pFgH8ezqEO9m8XlDh1A6lw8o5ca9XVcPKD65u4Qfu8hCwH4Luq1QwuNFltc+cZPX68v2+Heq9H9zsnBw+odc4zd+XF0FnTNSdvxi2JAbRZ8tMhltAaxrZbjUiR6HkI1XLGhZaAl+L8t1xlwr9jqNIiM/6FrlGAekZlNgchz7IFO5l1VBHQLF6IAomvQEi3av4ttw0HBi9OqWzaM5IuIo/ZeImjLZpM/HQMn/gKAvvpMIOAK9pEKk5kvU2PCkrXzWStQXUYxGRIp8NRjeDXcs4sfygxtMRKnVDoNRMJzYCg+4bQBplLzLsBe7GTQRsVZJYrdOvf1kWMP4cG7MtddFQIBb2MzG4/z0n/KrDVT5Yxlv6NiTK9NZalONQW7wznSn/siReWIjtgJiHaDTg319rbkcyFvkjujAARnvVqkkIiKi2yxm9psG7JKi7gN6MgUaTUqTEuIvapHg0CzvUqUkhE6V39xLEDC+YwATGKrX+SfQwEUXe8QoCNJpArbIrjhJWDB45FMRawP7U6E3GvM/mwS9GK1ZhVnoxvQZPC6CHYaRTYaEh/L+wMoK91c0fHiS0ATghQYhS5W3oPW3kuZeYStIZBRI7ZxFn5Lv1QhJhgBqZSbSvYRB0tWhQmCx074twY2yANEZH24b/7qPo+plSUuj7r29sQwekh7OULXy9YzaNI6apBPKX3YlorJm1BgZ1k3tTuqwq62spY3niK/vlSIHWpaNKOD4vSar4mwcWnjvLHu3fE7cZnK8HKDKbUdO72O+AH2zeEXWi+njg/548RWKv7+03aSciMU096VCvb4z3OUbp0XRqTQDAyuOI3/1x1f3+AXRdAyD0XZDinnhGeiQ9lAIqfYQJblxE7JxVP4EIm0KM6YyIGySajOf9+Q22pZz3IdsHpoOTMekIK0/Z/DQzgzxvtNnXYhoyOUBCpz5tS2C1FMJ9g9ko71WK2yXApu6Dk3sAPG7zJyoqbK7Y7kPjQGTOLTOdSPK++z42U9QjZDJGAwQ9wgESBJG4k28TPlYthrL84e4kdalo+p056ZzmUmGRGfHdGKt4ufDJtFW/tXpAWfXt72tmxb0mqNgiCxyY5LzRI6iaDmYZkN5L8nL0ZNqJ1MLsqOWLKIP/CiDDtfwZBWCdx++KlEkGxJSG6H/6DqQ3u67DRUCRxv/XGtDWCLqRzhpQsD9ZoFMNesB72TNA+1TR0DoTZnn/TI1BTnK2MsSQHds8KzRXIJhBMlfYTIH7ruJtkUEqpkhWDK9PrGH/vm8dbV72YoGGy32jVy+Kllo3OBuK/xXOcU0AH5dgXdLgoN0qxA3SOYf0AINeAGO4g4zvc8YCTHfgqIJkb/An4HmdZGgGQQSTMsXzXy5EFVmhiTiJHDvNjMklQQKDgJfIbzU+bDtQydDnYEdRxyPWQb4T1SPatEmHlt52uYMyGt/G8fXfkU0lxLn7ZRSuac1Xjax3fKvGHECbWYcE8WLQFwWUFPhThSz+e6vUx7+VrjdX7P/yCF25xPxnDFj+O+enMu2eX0RCuF6AlLkapewdivNlGz/qth9Wa3+sd6Ew3AXfJ3oBwkMFM8JjB8gJPmc7PM+LQKcspcnI6icLdEOIIpOaA2oljra8+xlzOWOSYaQr7hv/BQ5jpSSl+m07NHn8ROrjAololWY6jP6fOcnrWwXRkt83qypKrO8nMf6gWlDybb6QY3hY/rfmm0CGLhywle825DbcDqsXRf8nFWu3vII3kOvcrGtVQ4MaHsWBdywlHMajYUSe40dNx6Z3tefzFruz9LHQ7xD1hJz82M7iZ0QLWP3VODgHzi28/uosOlwKEZ0atn27/ceooGbxGDMZMammpE/ov4jh+i8stFX9rn7Jp/fgJnVwIGABRfOX30VRp71nzcAQbCZiYYeNTtOhsSseFSmOC9t9fnOUH6BmefGtU3thnHfnj31b8UPEditCSAcJfiaggtr4VoCqJNA8pR+gKWz6GEDrou9Ow5F1xV2hBXeHcwEXoDmfKs4ElOqKDXagKuFhVrd9wbChn54J5DlNk0Hicwktvx63bB+5g130pVHuEyuaxSAUyJPnr+0zXaPdmXXcKrtkzC5XgU3o/YkR5BKXhomrYgm91e5yhLUIvgW1SdDWF68P0TRBuUZdH8jgbOJ4CGvHETNIIkiZWw16QGeiieZCQcZlRqKSpHFqcqOAyiHko5vTmBanlcH39tdrqtbf+A2z/N/qUbEPSA1j1pJXBjO8hNni04AM8Rdjh882LF2VBV40EBegLAvdQeNcxhzpD6k+64y/Gg6C/nZZq+tu2F3p2UgKnTyieFJv4YRO3LU+7BinogaFTGQDgGQvAPX0S+Ylo5zlruuRAHsX3nNxPwq2qx31uUIvWPuXY8Vc/Kq8TFQuR2bBL0ouyv/yIj9o8GrFqPw2nbQBzrofUhzVrkiQNYp8S7cwNqNNJyCVRdCHDpsXZsHzQdYMm9CxrK1huGX/IQ+oD9DwqOEZDjkp9bmnToeP4+pwOHivw==,iv:BEhQs3Di4Ot5fUpg8jyoRk8IwUf3cErHt1cL8EBKvwQ=,tag:PFalLUWo5cn/tVXMzdaemg==,type:str]
-            natallan.com: ENC[AES256_GCM,data:LFZgRF1LvGxERKW/oJh//K6KzgN7thRouJB2JDLyn46tR/1QVrhbOomVYynGMJayRC08wMY0nYABrkkVMkzXGWbcR1Qxwi8yWp8eDSGPUqnIvMy5y5N84HC6F6vuTbBkNlxOipAWgTLa4iG5UuIs9Rq4hk2oIDvNM4oeGScaLjbw6oOlMivRCqrzUsrqjnnR2AIkLsbb/NhwlKlfN+r+IzULi8dYtjvz38RJ+pQT7nFD4aq31hCKHEovH/FhP+JOC2Nw95gaa00ORodBqs+tdTgBH/7x6PomjiSfY9V/47XlccgBRR/ax6R8j2Nb0xydxTR2EVt19EUWpAsA18xjkWhefoD5qAPRI+MS2IQGSW8hO0wYVxQ2bCXohw2/AmqACNwar0Otbi7mlSQ+UmsWgh7CaWhWrRR09yljLs5nV/qKZl7v15cuzhyqYRgeQiwYNWYhK8L3HgBOy64sIFp3vcVLzkVnvL95213pg5Z+tXuwiyP3zPwUPHMOvsaajVJZtE4SEe70ivhpwgHJRaQ0KjtupCjZAYOpdOXyuyk3w2DF/kWpBGa04b5cUmIER4uFCbJwxhY5nqtMMdMZ0uzCzHxL0cK4JUWqfgXAJsPkn1qPI0bL1/k77MnSDr5wvaIqH2WtUc3tywBss5EdF7xb51FCRNfnDaGy1NI0465n02HkseyslWvXC9bOqO8VMfUOerxdqDtyPc8EAoJyCaZXHrYxq4DRXFi8pAek9VhBTI7xdWraIFbuWHXUswSeE01zeNVvkFKCP1NIclmNvtngJRwTUESPK27RLJisNM3oMGXOaHvOdz/V1Q==,iv:NoGshUs7G4p9wQ7O+PpxtinwPa0SrdALeK49SkwYLFY=,tag:S4L+h7wkP34DWxB3RpjqyQ==,type:str]
+            natallan.com: ENC[AES256_GCM,data:idIBv+9XJ+nVbTFJ5EkWI4IxEw/Fh4gCXLdg10fft7IM2JLHmBObjwSEqly9CZ9AKAPJCZ/WeXKkG+Ul6wRxk8A1juFdDrOOdCXKJXpyrL7cBgSJWDylvKi7tqs9rV30SgU5hAsKNncPIWVWQApO923GKDzVVRMh8HZKTSy5ZZ8bCNQrlPr8DmkGa5VsUcz//scz1qdxnQeyAMJsrzg5bJPvR+bPjWXb06Q0oW8D4jM/gcm2HBq7POwg6DLEmX86mcYrvVw5LR2PDSXeG+32wugIaNvjHoUM62kB8b/OohTe5yqJcMfiGfEeMPlWAOdOfTZWZjS59BKPU7Ay0KUGKpVNHZv4W1G1Ebn6BCMOz4tNF1fmLdX0IcZSaX/0R58VwezKMBvaBj9t+Xmprx4fhrms4dfGpEzEM6LaeZp+dPH0MzIDPTEbrU28Oi49ppaMCQUG2HxOVoqvUNwAq/cA18WTFcEXTXjVdRo8CGo6eYU5lTK21xS6M/6ciANQtDcAPND6AaH/gJWrOFvHw8a++pLPiLK/7xT4k2TSEYMH1K4EdhVAxAc6sAPSQxvluABEsO3P8JdoLiAHI36lIN6RAkaT+gF7QvJWJtxcNqZbgwV448rcSBkWE82sNDrvcEzUpgrT2r2o+ljAX7NtXTBWps2xEjrg6NfdO3qlnW1TvLNmQTuAoezCUBQvlqNWOoSfM4qi4fqpQsnwFwy1jscYVftmI+W0cmI1dvVDJrudGJf6TtKn8Z6it+dmKCx1rRzUKA75JfC3K66FznsvqlC4TSa02nZxyUz1NxgL+bBCn8Z4PFakskxeFg==,iv:dsbNsqKBpedJuaaKZ9fPukQncCaDda8X1YEvm4ITTsU=,tag:ZRfLQ4yhjuvtiulqW1PCFQ==,type:str]
 sops:
     kms: []
     gcp_kms: []
@@ -12,59 +12,59 @@ sops:
         - recipient: age1lj5vmr02qkudvv2xedfj5tq8x93gllgpr6tzylwdlt7lud4tfv5qfqsd5u
           enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBLUlNkMnpmTjZsRnNBaitv
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBlUStiaEppZFByY1BSTDJM
-            U253MTlac3BueGJ2RG5SdUNkaExOSWYzQWxFCldOemJQUjFJdE04TjVuNGRIV2RN
+            a2tGZFIrWnhIRzJnTmVjeVQzR2NkdUVKczFvCnp4MjN5dlpVcEw0WjhoWTVvRXds
-            NTNocEZtd2tDZmFjVDEwR2liMi92TVEKLS0tIE1SUDd3andzeEFrOGxlZGdSTEIw
+            bkxNM2hpdGlOb3dIbnVsWGplTXNjcVUKLS0tIEdHbUtxL1ZsdEdwaHArcnhrYXkw
-            K1VyWHFSR1cvV25RTXIvR3BrNDZGczgKwaUCUNINj+o7d2DlIcq6V1Ls9ZJqxXQd
+            TkRWdG1YQWlJdjZoM3l3dmlpbjdaVW8Kx7BcZHC7gglnTijk5fhHsk0oMdPIs3Xr
-            L9lSOMTZ7wG2liFnySqCSKSSgQELCzHVRo0njv8LU7JLt2VFAjU2Qw==
+            CPeOTnfAMh5unDqmzIlGi+rS8siDcf4QrkjQWRZK9tJynjzkqv0brw==
             -----END AGE ENCRYPTED FILE-----
         - recipient: age17edew3aahg3t5nte5g0a505sn96vnj8g8gqse8q06ccrrn2n3uysyshu2c
           enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBzOW9wVW9RUzFTdDRZUXJx
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBpaVhkTEJMRVBsYkwyVnp6
-            Uk9Vcjd1UWN2amFOaXM4aHk2dkg5ZXpGNTBjCnA5allSZGZWRVZVNW5GQ09Iekpt
+            VE4xMmFBaWVIbGZHS2RVYjJkSjFrK2M3OVV3CjdaQ21HTUdEbmlralJnTU5hTk9t
-            NDNoTUlZa0VLanBYTjlsREtZeGRmZzAKLS0tIGlNUW9OTHkxVkNrVXVmRkdBUWtz
+            RUNjKzNPRjZTdFA3b1ZObm1mS2hjRjgKLS0tIElyOGNMSHVkNVRIT3d4OURka1BF
-            KytWWnFzSVY3YU40bW90blZuWm1PTFEK4EDj8GtLCWcd5FIEx/fywU+XLVeU0X87
+            NnVNS3EyVkNKd1FKMHBhbzM4V2lnNW8Kz92lN5MJrHkRM48nxfXgkRKX8ARWNDqg
-            UwJfkWaUYbLouqMTSKZZc1wf4KvL1GH1qyEANnjY6EDHf3kBGZMkTw==
+            sNqyXIDX9C+Nq2TqpLYNH7Rw06U35QTHQu7NLd/63/dxJUCcpQIpHg==
             -----END AGE ENCRYPTED FILE-----
         - recipient: age1u4tht685sqg6dkmjyer96r93pl425u6353md6fphpd84jh3jwcusvm7mgk
           enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBUVGxsMC95MW5tZjRzSEQy
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBCanhjZCt1c3Y3aHJaV2lx
-            VmpqT1dSUjdSL25vTmJOVnlTR1lLWUdBdVJrCjk2TVAyWmtHSjVDNEhsVG52RzNu
+            eWNhRE1ja1F2dlJTNG9zbS91Q1FVNWE2L2t3CmFxK0p3S3d1dm5NRGhnMWI5QkND
-            aTNJbjlIbVFucktyd3hVak5weWtORmcKLS0tIFdwSkVGamt6dTNlOGNoVDJ1dy8r
+            VG9jeXNWTXFKKzJIYXhvWkZ2bm8wYmsKLS0tIG1nUzlaVXNLbmNjSVI5dVBDME9D
-            RExTQlByTm5NUFA2aW1tR2lSZ1Q5d1UK8e7BRwBzeOvOUXYFwkgBraP1+vrZ3HvL
+            MTh3bjNvWmFWbGRXSVEyWjlpM215QW8KSen/lWbnH1SbP7qOWARwInwXnI0GUx2m
-            gaMH+5AEH4GiEd34svgjLAtmbSzm2/VNhboxYmAWUk+Ff4jn7+tvmw==
+            ZlWTGZPh5/Q+n6LAC64wRLKAQ+0lw7aE/b0Mf9Ht9XGDg3VizS4Ycw==
             -----END AGE ENCRYPTED FILE-----
         - recipient: age1cp6vegrmqfkuj8nmt2u3z0sur7n0f7e9x9zmdv4zygp8j2pnucpsdkgagc
           enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBSUHBUWnZIcHEyZXVUdGRJ
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA5RjFNVXJnTk9pTjZ4TFZs
-            MC9UT3lEM2hlOXJna1MzRzRvMjVtUThQWjBjCnB6OVVqNDE0dFNMTFBWMGkwSEMx
+            L3Z1THlRZXErQnJ0UjkvblZTR3ZoKzJYNVFnCnJWWDBiSEFzeWdXcVFNb01wTFpG
-            YkRQL1VJYXZjQU00YTJ1eDdPUWREeFEKLS0tIFpYRWtOaTlTWFFOdUd3VXZGNGcr
+            eUVvNGNYVk92MDdMckdKemZjRDdpb2cKLS0tIHdQSEpaRzRsa3JDamE3c1VYKy9D
-            TXZFN1BzMEhkMUlObXdYV3BxbzV5dUEKCVGvWx1ZiU8VEZoFvThef2mfa5QmgYp4
+            MkIyMzNuOEV5TVVSTHB6KzVLS1ZGZncKk6cU+7KIwhVG6pbdifpxu8BSD8vW5WJ3
-            rqgTrivwQv6uwp80i+mGzrVpdqhCYhwYgiQ29M8sGJqJSawbUTZ+5w==
+            WOdwHZdbQ69c8VHeoI9WVVDXD5/ubvU15VNHvDqc0+TgM9epmSxThg==
             -----END AGE ENCRYPTED FILE-----
         - recipient: age1j2r8mypw44uvqhfs53424h6fu2rkr5m7asl7rl3zn3xzva9m3dcqpa97gw
           enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBZMDdGN3pRekxsbmdmOUNL
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBUUEVjRm1udU1QakNsY21x
-            SHZJc0xvV2tvMi83dkdDT1Y0bDVIemt5QTFJCjZKbzAzRElMczBTNmkrU3lnU2lR
+            a2p5dnk3NElVWXNoWmxROEp4S0prN2djY21NCnJySUF1eWFIZENMVVZ6MWVSN3hJ
-            dm5pRDQvOExJTW5NRnZXNXVnSmx6UGcKLS0tIEdlU1hSQW9GaEZFeU1xUUFPcFZI
+            bHZ4eWN6SnVTdUxsdDd3OThOcmtTNm8KLS0tIFU5dkdsQWlKdDZzSFBGa3dZUG9q
-            bFU2bmVvcDNSUUZqRjhXbVQwZDFzWGcKJBad1AlJUOPjvVVqFUuzee14Bkt+Ounu
+            eittWnRlbnhJZ1A5M3o3amY2VFZyMFkKxhqNvCHSVUedEWCeuqIWNLomspQhamzo
-            LhhZ+UviDzZ2El1S4gwBY3Rut3aq/vR7n2EziXjNIz9OJtKW141LOw==
+            0uCqZxCgdkCZjt9aehlI/i+rlHs97+IsZoWILxHMnVN2fGiP1WWhiA==
             -----END AGE ENCRYPTED FILE-----
         - recipient: age1w3jtd4lecn2ng8qxantw33qxl2uasfqfjfpx45u6uweexwtxyq4spwssmh
           enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBGMmVtZUhFUzlpVHNXYnFP
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB6QmFId1pnU1VTZTJhblNm
-            Z3RSa2JYRURQR0ZRZTJjMTg0eVlkNmZGVXpzCnR1TEduZFA2MzhGeStKMHo4SDNW
+            M2Qvbkkyc2wxNFRLVnRoT0UyL0ozeXNuOVV3CmEzUjlFVGh5dEtQV3R5NzgycjVy
-            WFplb2MyQnBjSW1oNlBMQXZWZmZaWTgKLS0tIHA0NVdhbVdncTZZMGU0cjB6cmxr
+            QlQrMjJ1NWFUNTlPUHBGRmZPS1dwU1EKLS0tIGpGTnFKYnd1enV5V1JsQ3dmTU5R
-            Tk9pSnZsNEdmQ1UzSUQyeHEzZkZKMkEK2dAdumuwokijR1Oj6Bt1UXZlk4ZeRWq2
+            MkppYkdxMmQvdVJJaFZ3S3B4ckJDMVEKZQblDxIC5opkR92DupfwI1XdEHlnVsYy
-            beEr19lYPqQVdluR5X/5+JYb+aRQCKN+y3VYDo+7a4bn5967xVilAA==
+            JKxg0pbC/ENrT+uBLjSh9cFyuHMk80V4BQ6xZvzvKX+WLJlEsHrgTA==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2024-04-12T10:42:56Z"
+    lastmodified: "2024-04-13T11:28:15Z"
-    mac: ENC[AES256_GCM,data:QF+mFqddIId16iEkXb6euJW/BUTGkwV+cIuJNWt6/rivHJCGYtd+GX6RgSui/hYKooJP04vFzmcaeilwvbq1/sR+D1exQZZ1p4tG5LFRTrbermPHvcYy7Lel7On1c1OkjPTR8tJ6Vhs1en/FESUeL1JdgAX79IPuxa2WveIPE2s=,iv:pM0d7mPEY2kQ70qE+uiawCTwqFzkY1UhIjrkdKejYOw=,tag:eR5gNrTbZuYz4MHOgk9VJg==,type:str]
+    mac: ENC[AES256_GCM,data:R5uGODnxJC3ihSrzdjzxDHTKC+yXXjAOFbUAEOU67P8eM94RUnr8smP1ZDL2fnjCmzJdTMRDuBpjCtXxUeivNMTg/kK6r56VmQ2i2MDKiX49yPtGYfdUiLPBF/ZG/iwNJZ4m/3GZXAzvW2tYYkVzUU3cvsVdCFuWr1tnbsg9o1Y=,iv:kD0QdKbcr4yt+Ol3EK7O76czbYirgDx3pzPgyNB5GcU=,tag:fJsUOKQm3wUGjtqnO3574Q==,type:str]
     pgp: []
     unencrypted_suffix: _unencrypted
     version: 3.8.1

nixos/modules/nixos/services/default.nix

@@ -13,5 +13,6 @@
     ./bind
     ./glances
     ./syncthing
+    ./restic
   ];
 }

nixos/modules/nixos/services/restic/default.nix

@@ -0,0 +1,38 @@
+{ lib
+, config
+, pkgs
+, ...
+}:
+with lib;
+let
+  cfg = config.mySystem.system.resticBackup;
+in
+{
+  options.mySystem.system.resticBackup.local = {
+    enable = mkEnableOption "Local backups" // { default = true; };
+    location = mkOption
+      {
+        type = types.str;
+        description = "Location for local backups";
+        default = "";
+      };
+  };
+  options.mySystem.resticBackup.remote = {
+    enable = mkEnableOption "remote backups";
+    location = mkOption
+      {
+        type = types.str;
+        description = "Location for remote backups";
+        default = "";
+      };
+  };
+
+  config = mkIf (cfg.local.enable or cfg.remote.enable) {
+    sops.secrets."services/restic/password" = {
+      sopsFile = ./secrets.sops.yaml;
+      owner = "kah";
+      group = "kah";
+    };
+
+  };
+}

nixos/modules/nixos/services/restic/secrets.sops.yaml

@@ -0,0 +1,68 @@
+services:
+    restic:
+        password: ENC[AES256_GCM,data:gq4WW/IwIYQ=,iv:jVVSGQhUhAOOv7tTHOxJgYiw8e9Jfgeg8veeirn4510=,tag:eJPAgiYbTPfW7gnuvCv7JQ==,type:str]
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age1lj5vmr02qkudvv2xedfj5tq8x93gllgpr6tzylwdlt7lud4tfv5qfqsd5u
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBKYnpDMGhaM1FvUjFDazNk
+            QkNCbnhmSFludTFYU2l0WDZaOFhsZ2lCT2kwClNoSDNvaXNydmxubm5ZbUZZNW9W
+            cmJVOWtHdjBvcXBuNTdwSXV6NUo4WDAKLS0tIDZzdk9YTGNyS3gyV21hRXo5WVhW
+            aTRyVmdlYVVGbHJjL1BGdWxqNkxQWHMK29GOjS0tCNOECToZPSUZeyt/cElsynqy
+            Ky1ByYdCkYZ+3IiCFjN2fChA58khWg3mRUuSpYrTZKIcdBFw6oKfhA==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age17edew3aahg3t5nte5g0a505sn96vnj8g8gqse8q06ccrrn2n3uysyshu2c
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBYQXdQcGE3anREVHFuMTh1
+            enI0OHVYRDllYVdQNGllTnAyZ1lOU2RrMnk4CnNQUWt2ZUZsd25YWmdKcC9UNHJu
+            V2FZRURibS8yd0ZQZnFaYUhFWVVUdVEKLS0tIGNyTjVJRWo2ODFZUmhTRzJxdWZ0
+            Y3N6V0ZXRFZpUG5ablhKM29ma2ZOMFEKpCHKEiEx8lGNs9WufBZ1zyajgyBm2hWV
+            DW9Z6FB/Y0pvPLs3tF05qQEQ3LVjcLJ3lJ4fcrbqspNhcfV5vN6sZA==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age1u4tht685sqg6dkmjyer96r93pl425u6353md6fphpd84jh3jwcusvm7mgk
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBhSGloUkI3UnY3S010MlI3
+            UWwyc1I2SmZEdWhWVVlsYysrMDIxdzZaQW5vCjJ1WWlVZ0xRWmMxTGh2S1k1MkVE
+            UTJJWlBvR1R6V1RXcjlSbm4rQmFZcTgKLS0tIDAzRUtNenB1cW10ZFdMY285aVFZ
+            MjVZOWM3SFkvMUtoTEZGZkx4V1ZEcFUK2tDvX173EYvGqLxfsKxrKVv8BDorYJk4
+            etatqb+5KQnEYFgxY3qY4nMdsir74VqdHKkg9rP0/eUbNL0exBTjFA==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age1cp6vegrmqfkuj8nmt2u3z0sur7n0f7e9x9zmdv4zygp8j2pnucpsdkgagc
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBpUzlLNVhaMW94YjI5TU5k
+            S2ZIRVBxRVBUT1Jta3hydndUVzZxbDA4ZldzClUzMUJWQ2JWZk9kQU5LVVRJNWlT
+            U0p5ZnphelExSXU4MHh2d3RxNVo5WmsKLS0tIDNHSC9tM3FaRTYybmJtWUxFYVpD
+            ZU9GTUVpSGVzOWZuZWZTZjR1NWFVY2MKy1od9yzs5BJJF/b5TPsqn5ZGWAVdt6nz
+            lX1owv3vRz9VBjOi9omDKbnSPViOBk8C2+5as52nUdWO/xTsNgO1+A==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age1j2r8mypw44uvqhfs53424h6fu2rkr5m7asl7rl3zn3xzva9m3dcqpa97gw
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBKMVU0UHZMdVJXRW5uMkVz
+            OU9PbzZRRW1aanB3b0xLK1ZsYjZGbElqQm13CjRrbUVxamhvMUdGb1FBU2VyNDl1
+            RmNESmNIYWRaYTBpSnZmQ1dYc1NycHMKLS0tIDlmUFZqKzdETUNrWk1qUkM0NUdo
+            eTBWa1kvUjArK2lEbTJtTm4xQUpGTWcKbpujwUOxwcghfWbP9XWHzfhfGtQhjC63
+            qnZJSKGoFT/DxJiGaF70gQk+Gn+db1MaPKZzQ492lqCSX+T22z7+oQ==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age1w3jtd4lecn2ng8qxantw33qxl2uasfqfjfpx45u6uweexwtxyq4spwssmh
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBHZkJsK1ZXRGo3NVVPaktD
+            M3JHVTByYzlOcGtXTklKSSswNlM2R0E4OFJJCk9ab21TbWJORTJuZHZxcVJrUVZw
+            cDNPb25EQmEzRFRXOE9CUG12UzJQYjAKLS0tIG4rWHY5SjBZNW5qb1kyVGNXN1ls
+            SDNRdTJlL0p5UmREU1ExVm9Nay9laE0KPidvFK33/M1v1/62g3/nO6DdHaM7od3F
+            mXCwxArAEZo738AM88Si9xJAyvXNI2yc+cOJzijtXrUBgvmE8DdoIA==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2024-04-13T09:09:34Z"
+    mac: ENC[AES256_GCM,data:VOB3F3+ssvI+2EucvZ+LX1Hl+702vhB5RVSVeSzQbgmnN+zwuYLksO4rdgOpegPGlENcj5M2CzyRqsiGhyuy9THm/u09Ac2PbPEfWGm72pzuSMPymZQrUJmZDU/Gl0IlIfxQGGOfFdaVnzVl4ynIZuseJDjOZP9ymT8G8/ewSYY=,iv:Czjr4i9JuIO+2Ftl3ENE/XAzsca9rfYCvgy+tggMihY=,tag:4BlY8d29AUh4FluA6eUNeg==,type:str]
+    pgp: []
+    unencrypted_suffix: _unencrypted
+    version: 3.8.1

nixos/modules/nixos/system/default.nix

@@ -7,5 +7,7 @@
     ./nix.nix
     ./zfs.nix
     ./nfs
+    ./motd
+    ./pushover
   ];
 }

nixos/modules/nixos/system/motd/default.nix

@@ -0,0 +1,84 @@
+{ config, lib, pkgs, ... }:
+let
+  motd = pkgs.writeShellScriptBin "motd"
+    ''
+      #! /usr/bin/env bash
+      source /etc/os-release
+      service_status=$(systemctl list-units | grep podman-)
+      RED="\e[31m"
+      GREEN="\e[32m"
+      BOLD="\e[1m"
+      ENDCOLOR="\e[0m"
+      LOAD1=`cat /proc/loadavg | awk {'print $1'}`
+      LOAD5=`cat /proc/loadavg | awk {'print $2'}`
+      LOAD15=`cat /proc/loadavg | awk {'print $3'}`
+
+      MEMORY=`free -m | awk 'NR==2{printf "%s/%sMB (%.2f%%)\n", $3,$2,$3*100 / $2 }'`
+
+      # time of day
+      HOUR=$(date +"%H")
+      if [ $HOUR -lt 12  -a $HOUR -ge 0 ]
+      then    TIME="morning"
+      elif [ $HOUR -lt 17 -a $HOUR -ge 12 ]
+      then    TIME="afternoon"
+      else
+          TIME="evening"
+      fi
+
+
+      uptime=`cat /proc/uptime | cut -f1 -d.`
+      upDays=$((uptime/60/60/24))
+      upHours=$((uptime/60/60%24))
+      upMins=$((uptime/60%60))
+      upSecs=$((uptime%60))
+
+      printf "$BOLD Welcome to $(hostname)!$ENDCOLOR\n"
+      printf "\n"
+      ${lib.strings.concatStrings (lib.lists.forEach cfg.networkInterfaces (x: "printf \"$BOLD  * %-20s$ENDCOLOR %s\\n\" \"IPv4 ${x}\" \"$(ip -4 addr show ${x} | grep -oP '(?<=inet\\s)\\d+(\\.\\d+){3}')\"\n"))}
+      printf "$BOLD  * %-20s$ENDCOLOR %s\n" "Release" "$PRETTY_NAME"
+      printf "$BOLD  * %-20s$ENDCOLOR %s\n" "Kernel" "$(uname -rs)"
+      printf "\n"
+      printf "$BOLD  * %-20s$ENDCOLOR %s\n" "CPU usage" "$LOAD1, $LOAD5, $LOAD15 (1, 5, 15 min)"
+      printf "$BOLD  * %-20s$ENDCOLOR %s\n" "Memory" "$MEMORY"
+      printf "$BOLD  * %-20s$ENDCOLOR %s\n" "System uptime" "$upDays days $upHours hours $upMins minutes $upSecs seconds"
+
+      printf "\n"
+      printf "$BOLDService status$ENDCOLOR\n"
+
+      while IFS= read -r line; do
+        if [[ $line =~ ".scope" ]]; then
+          continue
+        fi
+        if echo "$line" | grep -q 'failed'; then
+          service_name=$(echo $line | awk '{print $2;}' | sed 's/podman-//g')
+          printf "$RED• $ENDCOLOR%-50s $RED[failed]$ENDCOLOR\n" "$service_name"
+        elif echo "$line" | grep -q 'running'; then
+          service_name=$(echo $line | awk '{print $1;}' | sed 's/podman-//g')
+          printf "$GREEN• $ENDCOLOR%-50s $GREEN[active]$ENDCOLOR\n" "$service_name"
+        else
+          echo "service status unknown"
+        fi
+      done <<< "$service_status"
+    '';
+  cfg = config.mySystem.system.motd;
+in
+{
+  options.mySystem.system.motd = {
+    enable = lib.mkEnableOption "MOTD";
+    networkInterfaces = lib.mkOption {
+      description = "Network interfaces to monitor";
+      type = lib.types.listOf lib.types.str;
+      # default = lib.mapAttrsToList (_: val: val.interface)
+      default = [ ];
+    };
+
+  };
+  config = lib.mkIf cfg.enable {
+    environment.systemPackages = [
+      motd
+    ];
+    programs.fish.interactiveShellInit = lib.mkIf config.programs.fish.enable ''
+      motd
+    '';
+  };
+}

nixos/modules/nixos/system/pushover/default.nix

@@ -0,0 +1,53 @@
+{ lib
+, config
+, pkgs
+, ...
+}:
+with lib;
+let
+  cfg = config.mySystem.system.systemd.pushover-alerts;
+in
+{
+  options.mySystem.system.systemd.pushover-alerts.enable = mkEnableOption "Pushover alers for systemd failures" // { default = true; };
+  options.systemd.services = mkOption {
+    type = with types; attrsOf (
+      submodule {
+        config.onFailure = [ "notify-pushover@%n.service" ];
+      }
+    );
+  };
+
+  config = mkIf cfg.enable {
+
+    systemd.services."notify-pushover@" = {
+      enable = true;
+      onFailure = lib.mkForce [ ]; # cant refer to itself on failure
+      description = "Notify on failed unit %i";
+      serviceConfig = {
+        Type = "oneshot";
+        # User = config.users.users.truxnell.name;
+        EnvironmentFile = config.sops.secrets."services/pushover/env".path;
+      };
+
+      # Script calls pushover with some deets.
+      # Here im using the systemd specifier %i passed into the script,
+      # which I can reference with bash $1.
+      script = ''
+        ${pkgs.curl}/bin/curl --fail -s -o /dev/null \
+          --form-string "token=$PUSHOVER_API_KEY" \
+          --form-string "user=$PUSHOVER_USER_KEY" \
+          --form-string "priority=1" \
+          --form-string "html=1" \
+          --form-string "timestamp=$(date +%s)" \
+          --form-string "url=https://$2:9090/system/services#/$1" \
+          --form-string "url_title=View in Cockpit" \
+          --form-string "title=Unit failure: '$1' on $2" \
+          --form-string "message=<b>$1</b> has failed on <b>$2</b><br><u>Journal tail:</u><br><br><i>$(journalctl -u $1 -n 10 -o cat)</i>" \
+          https://api.pushover.net/1/messages.json 2&>1
+
+      '';
+      scriptArgs = "%i %H";
+    };
+
+  };
+}

nixos/profiles/global/secrets.sops.yaml

@@ -1,4 +1,7 @@
-truxnell-password: ENC[AES256_GCM,data:/ZLxonyxqLLRJvVSuPczEkeiOaY/Z/1pmtwnOl8HQAds/hAnTWzVnfaovOP5KbsrS5GohTbHTAL80NOBflB1vZz+pWzhKVBbqQxnmYXGpp3jdO7q6Vo9yKPTnu4ClkFkN2QkX4xmUgSIRQ==,iv:xMhbcgBwqjCeKx0ZfTwORonxaFNZZ9yzBb2F27s0KO0=,tag:legRUJEC2ZXWTHCF0Kb7DA==,type:str]
+services:
+    pushover:
+        env: ENC[AES256_GCM,data:nkiW4SDRCjmKrXTDSOolV1+WJorodjF+2FvBpXRa7PsXMQM+4pgP1Ll4TRZHkrwJ5hpD0X9hnb1wdVUcm/2DU/o4qkFl/ZUGQIiOZRbyirxINeYq7G/0TWJmtx/vw48L,iv:2pqzQDEfXkkA+GCXdk4+2NFOi3OASFqefzVf0YcWkUc=,tag:tNeYcgfsLAaKGVmOsTLPdg==,type:str]
+truxnell-password: ENC[AES256_GCM,data:SQhRB9eQRLbyTF1ebUoGPhWOdfcX3+yMTsIxY+/Tb0dNYAYvFojc+vcULevKS7DteLlRHSOFZS5MaPkgv4+agF8ZCC1Wy6A6KyMd4NGxzt27mE1/tjla2OVIyqoo3ye7hpxLZxW9Feh+Pg==,iv:684OoJRCiLmnfzjijz2CEdFpvlBkGzlTYIpKqbLAgtQ=,tag:5YyHnnX9/i3kp8yZjdP4XQ==,type:str]
 sops:
     kms: []
     gcp_kms: []
@@ -8,59 +11,59 @@ sops:
         - recipient: age1lj5vmr02qkudvv2xedfj5tq8x93gllgpr6tzylwdlt7lud4tfv5qfqsd5u
           enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBkR0xqVzZsZzN4VEFsbXJn
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBKbjFLVktzVVNSeDVuOVpx
-            RW1uNjJqNjVUcjN5c3Z3RkRMbUpERzRidVZBCjdLYis2TzdnMkJGalN5WEJ4NzdC
+            OXVrcU5vK01JbjZvMlh0bi9INFlPU2VTZUFrClRzVGlSSzljcldnVFhZTHhKcWYz
-            eEMvSFhjamllM1VPSGhGWFAwSzltYm8KLS0tIHFLWHlPdkVsYmpWUW5YWmdzUXZQ
+            MHdGdEZBeU1tMDJOU1FVVVlMSFRNUGMKLS0tIGtRdGFaMC83MUc2VDdEOUJKcFYy
-            dCtSdXNWdGdrNGlLcVg3bVBWWEdWeG8KFNFYMPp+uPhlFyDXzps946gowRM+EpnG
+            ZGorL2orMXJ1K3VNek5WaTdkcXpyUDAKUrd8OXnSEvOEHeKY02aMEnQEAK3dHWUg
-            SljpZ2XTMHLZ2ZNHqrkdXaou8H7dDZjafo010I5c+U0/BzYg3GKwlw==
+            /zPECgCQwStiE11erj+mfYhgSeHDx0szQieRj4a+x4KaEItydVOMng==
             -----END AGE ENCRYPTED FILE-----
         - recipient: age17edew3aahg3t5nte5g0a505sn96vnj8g8gqse8q06ccrrn2n3uysyshu2c
           enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB1dWtHWVR4NFNaMS9LM2xr
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBKYmM1cDFHd2JKMlhHU0tu
-            dm9UamZkTmVwOGtaMmdJeU5uZGY5TGJwcnpVCmpyZ201OVpJSmFnT0V3aGluZUxP
+            VSthQnB0Q1VBVjEyRlJzZUtXYW1sSUordml3ClZQQnNQWFFmTnNkeEI2cjJ4OGhW
-            djRTYUJCNis5NHhKbmNtZ3NlQldXdWMKLS0tIEJuSTJPVzhJVW93Qi8zSFFvSzlk
+            VWRqZWU1aFpZc1c4dzdPcWxWek43OEEKLS0tIHNvc0NmalFZWHY1a1I2RkJYR3pR
-            SW9jdk5BaHRhc3puVzE0cEwrMlRMcVEKV5++1oZk48SA2iuxf64NVg1gQo997tM/
+            c1JwckUvQTRxZDlsQThsUHd2VndvQ1kKQAJhEKLV3AcLDhk3BEbjwsLmEC+FFYZt
-            06VqoVuLX1Vqo/InVmWzMJJA6IKSAe1k8eOeoZ7Sbgty7rcd3al1VQ==
+            AZXPbhJVZ2n62yU97IcEZOEs7tcaPFqRQmuEk0caMEj4F3RgF0naOg==
             -----END AGE ENCRYPTED FILE-----
         - recipient: age1u4tht685sqg6dkmjyer96r93pl425u6353md6fphpd84jh3jwcusvm7mgk
           enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBjSVhBNnpKREgrWHFSd0lC
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBJaDdXV2FrMHA0dkxkN2F4
-            bVd4b2ZmblR6Zlp6alEwV01CQm1TWGdFU3o4Ck5PY2R2VU5OSDZxME8zWkkyZ2Yr
+            dCszUG42bkI5bGlzMFJmVy9RV1R1QjZsNjNJCnBBMWVpYWdnN2MzOGZXT2UzRXZw
-            bjJpZmFjUEdHY1hxWUZQSVd1Rlhnb3cKLS0tIEE2SnlkTThtR0pOdmF0SStNOUxi
+            VVlVVmJaUHhQcVNXYVJqdnkzSTJ3TUUKLS0tIFdwUU0wR0c2eXp2NEFxV29DMDIx
-            MnJ2Ri9xb0E5Wis3ejhTVDdtQUVPc28KpDiexpCl7Pocrv1PAHEWVHEFEDUDq4F5
+            K2d1UXZTenZnV0ViR3NOZE5YK1RNRkkK5ForFTQ9G7dvy3gri/nSVkYl4GViM4Ni
-            CsUxpyH4+odoi1Qzj2iDkbuaun5mTER96B/gfXKb8UZqOIygHA/89g==
+            MiTQCriWOb8y0Fbdidc61NHOuGF3Ji3HUE7V065+DpWb43M8Y+w93g==
             -----END AGE ENCRYPTED FILE-----
         - recipient: age1cp6vegrmqfkuj8nmt2u3z0sur7n0f7e9x9zmdv4zygp8j2pnucpsdkgagc
           enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB0bHZmS3FkSnhlNndWOGhU
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBuV0diWWZLOEdVVXhHRzZP
-            eC9oSlEvUEorTHlvKzZ4RjhZNWthUzFTdTFrCktNWUNCYlQvZDY5OC9YRFJ1ei9V
+            TWtmLzQyR1N4ZHRMVVpLVWwrdXlHWjgzVkdJCld6NExvWGVXVzR3cjl2eVJTY01E
-            UEY0MGhDMDVHai9LZzRrYWtqMzNJd0UKLS0tIGtPZHVPOTFXeXJEWTFwVmZoZFdX
+            ZzJjcWJWWklJMUpjUjFJOURLMTBZNG8KLS0tIHpEMzA3enZMNXpmeVNtUjl0Y3lF
-            ckF5M1Z6Z3lRRC9yYVIvaGJsbllSK1EK8JODbe2VZg5ABspZn5eNmvF3pJziVY9X
+            RUR2alVhVC9rbXdYYlB4THRYRTVYSUEK9jiP+9/IMTTEQlWwn+MvL7NgI4Z97YEY
-            B05xe15jisD3k5mXcbolo3wkt78+fBV1M5EYuOYgtwI4bdWp1he+TQ==
+            C+U3mvXVOQ/FA/3hCaD1HALr6WHIV3DGcxacblYT4awAbN4crtfnIA==
             -----END AGE ENCRYPTED FILE-----
         - recipient: age1j2r8mypw44uvqhfs53424h6fu2rkr5m7asl7rl3zn3xzva9m3dcqpa97gw
           enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBDZk8zS3c4alFyL09WSkMr
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB2b01SMXJtTHhib2VrQ0hq
-            aDFhZGhQRVBYUExmV05DbjlJb3B4ZmFmcEcwCkx4aGhRbFNudGFEeHN3Q2xSSkxa
+            YW5JQU85L3dXbDh5b3paZElCOWpwMG9QODBNCkR3UTJLSFZOOEFKUW1jbG5YNmJo
-            SnBxeG40L3YwbkFZL2FLY1hWYTA3N0kKLS0tIEZOM0o4VUFPWm1YUERzTS85Zmsr
+            ZGNaamlsZVJ2K2dxMjRKcFUxY3Y2NncKLS0tIEErUkN1WDgwajd1TjBFOURxQTdK
-            ZHgwR2xPTE5zaGx3dDIxZ3F6allKRjgKgpcA82ZC8WrCF5b9EqkaHvrCQQYEFWXI
+            aW0rSUVwbnJqaHdUTWxYWWRGNm5reDAKlQG5maCq2K8aFJRbuuzv9SyNhrxzjbFQ
-            BxY8+3w3/hqnDiWzlPdwRQGN0J0e2WeIUFzSaQFYpR7kemP3DJ+MtQ==
+            jtO2KoFX2gLFw90YFCsMFbaVO+xTcZQ7FQv0s4ktffudnT5zjuzFAg==
             -----END AGE ENCRYPTED FILE-----
         - recipient: age1w3jtd4lecn2ng8qxantw33qxl2uasfqfjfpx45u6uweexwtxyq4spwssmh
           enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA0YnNXdkZSclc2L2NYMEJs
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB0TzlkSHYwc1F6VWozTUhq
-            MG9LeURhc2VsM1Zxa3dIU0dNSTZpYTJqblUwCjV0QnE4eHljZHU1cGZTTUhLNzkv
+            dVZWVFVPc0F1V1gvV3d0M1NzT3dubWx6OFhRCmlaVHJuRDVRZnUwd0ttWkdYck51
-            WEJyQnFDS1JTRTd4dzJYa21EcW5hekUKLS0tIGMzMC9aVjJkZEZnM1JlTU9uTWdl
+            NUdFOThuSWhLRE1lUW5aOU85TmhXVzgKLS0tIFJXVkc3Q3hmQjNQay9BV3lVZ3Jk
-            OUN5d2lEYnJCdW0vTXJnUWt4d3hCT28KjuBFDRjCyU037UV7s4ZSaMxPhZhUBakG
+            a2NDeTRTWGtxT0wyWEF1djlyQlhQYmcKy/liFdZyxuUp6eI7s+lANV0mcQWHOLFe
-            6IEpCm0U2NYfLAgqDrq9Pn1J9Ut1Q3Uep/UWBfqNET/yARoiXPDTvg==
+            4Cg92W0Xppv+J3W4W+rVwzyFWUrkTnBFfZHnN3nhz18Lm7FR7vs7oQ==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2024-04-12T03:04:28Z"
+    lastmodified: "2024-04-14T00:18:09Z"
-    mac: ENC[AES256_GCM,data:lDxEN6svXfQEHyWRrDcs8XU8Sblxof820ZOcMoVfSMkjgNcx883E6ZKbZlE1lQztlG1RCyvGgpPotjfEN7KgH87IZ3EpUdq6t+4f2ag8T2xnjDNoU1PeiLLTGvd5rt5MeKK3YqhxQ17OKrdvwVDL+wcnZedF9X0vgbpFehBTIhY=,iv:S4cRp1It/BNYknkLk8x75oi615ddXp3FbS7Q5HBtgrg=,tag:9ugtzg2cw5Gc3/KpHbmuFQ==,type:str]
+    mac: ENC[AES256_GCM,data:S4QbCTp+rxSPumolno0FuSNvtvEZpA4E77S2mSliI5y4GJ5n/mx8SY07xbwqMzB3W9EgO0ZT+vvsx4N7jkZPBtr+m12/KwG8NcHZsBdXNi2TRi8CGZlCXFzRNQSjJRiYBMsdKwVCdm6Wxlf/PuCnNj0ShSU0IWaTzlSc0FhSeYM=,iv:Nv+rbtRCXZFAnDi0wzq2/qjdvr7535BkCogBqllmPGQ=,tag:OwTPYR4e8N/qGQvvOjh7SA==,type:str]
     pgp: []
     unencrypted_suffix: _unencrypted
     version: 3.8.1

nixos/profiles/global/sops.nix

@@ -2,5 +2,10 @@
 {
 
   sops.age.sshKeyPaths = [ "${config.mySystem.system.impermanence.sshPath}/ssh_host_ed25519_key" ];
+  # Secret for machine-specific pushover
+  sops.secrets."services/pushover/env" = {
+
+    sopsFile = ./secrets.sops.yaml;
+  };
 
 }

nixos/profiles/role-server.nix

@@ -15,8 +15,10 @@ with lib;
     mySystem.services.rebootRequiredCheck.enable = true;
     mySystem.security.wheelNeedsSudoPassword = false;
     mySystem.services.cockpit.enable = true;
+    mySystem.system.motd.enable = true;
     mySystem.services.gatus.monitors = mkIf config.mySystem.services.gatus.enable [{
 
+
       name = config.networking.hostName;
       group = "servers";
       url = "icmp://${config.networking.hostName}.${config.mySystem.internalDomain}";

nixos/profiles/role-worstation.nix

@@ -1,4 +1,4 @@
-{ config, lib, pkgs, imports, boot, self, ... }:
+{ config, lib, pkgs, imports, boot, self, inputs, ... }:
 # Role for workstations
 # Covers desktops/laptops, expected to have a GUI and do worloads
 # Will have home-manager installs
@@ -79,6 +79,7 @@ with config;
     gh
 
     bind # for dns utils like named-checkconf
+    inputs.nix-inspect.packages.${pkgs.system}.default
   ];
 
   i18n = {