feat: migrating apps, nginx, bye traefik (#130)

* hax

* hax

* shell monitoring

* hax radicale!

* hacking

* haxor

* hax

* hack

* feat: refactor paths etc for impermance

* fix: restic

* hax

* more hax

* feat: migrate z2m

---------

Co-authored-by: Truxnell <9149206+truxnell@users.noreply.github.com>

.github/renovate/autoMerge.json5

@@ -9,11 +9,11 @@
       "schedule": [ "before 4am on Sunday" ],
       "matchUpdateTypes": [ 'minor', 'patch', 'digest'],
       "matchPackageNames": [
-        'ghcr.io/onedr0p/sonarr',
+        // 'ghcr.io/onedr0p/sonarr',
-        'ghcr.io/onedr0p/readarr-nightly',
+        // 'ghcr.io/onedr0p/readarr-nightly',
-        'ghcr.io/onedr0p/radarr',
+        // 'ghcr.io/onedr0p/radarr',
-        'ghcr.io/onedr0p/lidarr',
+        // 'ghcr.io/onedr0p/lidarr',
-        'ghcr.io/onedr0p/prowlarr',
+        // 'ghcr.io/onedr0p/prowlarr',
         'ghcr.io/twin/gatus',
         'vaultwarden/server',
       ],

.vscode/module.code-snippets

@@ -10,13 +10,18 @@
       "with lib;",
       "let",
       "  cfg = config.mySystem.${1}.${2};",
+      "  app = \"${3}\""
+      "  appFolder = \"apps/${app}\";",
+      "  persistentFolder = \"${config.mySystem.persistentFolder}/${appFolder}\";",
+      "  user = app;",
+      "  group = app;",
       "in",
       "{",
-      "  options.mySystem.${1}.${2}.enable = mkEnableOption \"${3}\";",
+      "  options.mySystem.${1}.${2}.enable = mkEnableOption \"${4}\";",
       "",
       "  config = mkIf cfg.enable {",
       "",
-      "  $4{}",
+      "  $5",
       "",
       "  };",
       "}",

docs/installation/install.md

@@ -0,0 +1,182 @@
+## Getting ISO/Image
+
+How to get nix to a system upfront can be done in different ways depending on how much manual work you plan to do for bootstrapping a system. Below is a few ways I've used, but obviously there will be multiple methods and this isn't a comprehensive list.
+
+### Default ISO (2-step)
+
+You can download the **minimal ISO** from [NixOS.org](https://nixos.org/download/) This gets you a basic installer running on tmpfs, which you can then open up for SSH, partition drives, `nixos-install` them, and then reboot into a basic system, ready to then do a second install for whatever config/flake you are doing.
+
+### Custom ISO (1-step)
+
+An alternative is to build a _custom ISO_ or image with a number of extra tools, ssh key pre-installed, etc. If you image a drive with this you get a bootable nixos install immediately, which I find useful for Raspberry Pi's
+
+I have started down this path with the below code:
+
+!!! note
+
+    The below nix for images is Work In Progress!
+
+**[ISO Image (x86_64)](https://github.com/truxnell/nix-config/blob/main/nixos/hosts/images/cd-dvd/default.nix)**
+**[SD Image (aarch64\_)](https://github.com/truxnell/nix-config/blob/main/nixos/hosts/images/sd-image/default.nix)**
+
+From here, you can build them with `nix build`. I have mine in my flake so I could run `nix build .#images.rpi4`. Even better, you can also setup a Github Action to build these images for you and put them in a release for you to download.
+
+### Graphical install
+
+A graphical install is available but this isn't my cup of tea for a system so closely linked to declarative setup from configuration files. There are plenty of guides to guide through a graphical installer.
+
+### Alternate tools (Automated)
+
+You can look at tools like [disko](https://github.com/nix-community/disko) for declarative disk formatting and [nixos-anywhere](https://github.com/nix-community/nixos-anywhere) for installing NixOS, well, anywhere using features like `kexec` to bootstrap nix on any accessiable linux system.
+
+This closes the loop of the manual steps you need to setup a NixOS system - I've chosen to avoid these as I don't want additional complexity in my installs and I'm OK with a few manual steps for the rare occasions I setup new devices.
+
+## Booting into raw system
+
+### Linux x86_64 systems
+
+I'm a fan of [Ventoy.net](https://ventoy.net/en/index.html) for getting ISO's onto systems, as it allows you to load ISO files onto a USB, and from any system you can boot it up into a ISO selector - this way you can have one USB with many ISO's, including rescue-style ISO's ready to go.
+
+### Virtual Machines
+
+I follow the same approach as x86_64 above for testing Nix in VM's.
+
+General settings below for a virtual machine- I stick with larger drives to ensure nix-store & compiling doesn't bite me. 16GB is probably OK too.
+
+**VM Settings:**
+**ISO:** nixos-minimal
+**Hard:** Drive: 32GB
+**RAM:** 2GB
+**EFI:** Enable
+
+!!! warning
+
+    Ensure you have EFI enabled or ensure you align your configuration.nix to your VM's BIOS setup, else you will have issues installing & booting.
+
+For VM's, I then expose port 22 to allow SSH from local machine into vm - mapping host port 3022 to vm guest 22.
+
+### aarch64 (RasPi)
+
+I cant comment on Mac as im not a Apple guy, so this section is for my Raspberry Pi's. I build my custom image and flash it to a sd-card, then boot it - this way I already have ssh open and I can just ssh into it headless. If you use a minimal ISO you will need to have a monitor/keyboard attached, as by default SSH is not enabled in the default ISO until a root password is set.
+
+### Other methods
+
+You could also look at PXE/iPXE/network boot across your entire network so devices with no OS 'drop' into a NixOS live installer.
+
+## Initial install from live image
+
+### Opening SSH
+
+If your live image isn't customised with a root password or ssh key, SSH will be closed until you set one.
+
+```sh
+sudo su
+passwd
+```
+
+`sshd` is now running, so you can now ssh into the vm remotely for the rest of the setup if you prefer
+
+### Partitioning drives
+
+### Standard
+
+Next step is to partition drives. This will vary per host, so `lsblk`, viewing disks in `/dev/`, `/dev/disk/by-id`, `parted` and at times your eyeballs will be useful to identify what drive to partition
+
+Below is a fairly standard setup with a 512MB boot partition, 8GB swap and the rest ext4. If you have the RAM a swap partition is unlikely to be needed.
+
+```sh
+# Partitioning
+parted /dev/sda -- mklabel gpt
+parted /dev/sda -- mkpart root ext4 512MB -8GB
+parted /dev/sda -- mkpart swap linux-swap -8GB 100%
+parted /dev/sda -- mkpart ESP fat32 1MB 512MB
+parted /dev/sda -- set 3 esp on
+```
+
+And then a little light formatting.
+
+```sh
+# Formatting
+mkfs.ext4 -L nixos /dev/sda1
+mkswap -L swap /dev/sda2 # if swap setup
+mkfs.fat -F 32 -n boot /dev/sda3
+```
+
+We will want to mount these ready for the config generation (which will look at the current mount setup and output config for it)
+
+```sh
+# Mounting disks for installation
+mount /dev/disk/by-label/nixos /mnt
+mkdir -p /mnt/boot
+mount /dev/disk/by-label/boot /mnt/boot
+swapon /dev/sda2 # if swap setup
+```
+
+### Impermanence (ZFS)
+
+TBC
+
+### Generating initial nixos config
+
+If this is a fresh machine you've never worked with & **dont have a config ready to push to it**, you'll need to generate a config to get started
+
+The below will generate a config based on the current setup of your machine, and output it to the /mnt folder.
+
+```
+# Generating default configuration
+nixos-generate-config --root /mnt
+```
+
+This will output `configuration.nix` and `hardware-config.nix`.  `configuration.nix` contains a boilerplate with some basics to create a bootable system, mainly importing hardware-config.
+`hardware-config.nix` contains the nix code to setup the kernel on boot with a expected list of modules based on your systems capabilities, as well as the nix to mount the drives as currently setup.
+
+As I gitops my files, I then copy the hardware-config to my machine, and then copy across a bootstrap configuration file with some basics added for install.
+
+```sh
+scp -P 3022 nixos/hosts/bootstrap/configuration.nix root@127.0.0.1:/mnt/etc/nixos/configuration.nix
+scp -P 3022 root@127.0.0.1:/mnt/etc/nixos/hardware-configuration.nix nixos/hosts/nixosvm/hardware-configuration.nix
+```
+
+### Installing nix
+
+```sh
+nixos-install
+reboot
+
+# after machine has rebooted
+nixos-rebuild switch
+```
+
+Set the password for the user that was created.
+Might need to use su?
+
+```sh
+passwd truxnell
+```
+
+Also grab the ssh keys and re-encrypt sops
+
+```sh
+cat /etc/ssh/ssh_host_ed25519_key.pub | ssh-to-age
+```
+
+then run task
+
+Login as user, copy nix git OR for remote machines/servers just `nixos-install --impure --flake github:truxnell/nix-config#<MACHINE_ID>`
+
+```sh
+mkdir .local
+cd .local
+git clone https://github.com/truxnell/nix-config.git
+cd nix-config
+```
+
+Apply config to bootstrapped device
+First time around, MUST APPLY <machinename> with name of host in ./hosts/
+This is because `.. --flake .` looks for a `nixosConfigurations` key with the machines hostname
+The bootstrap machine will be called 'nixos-bootstrap' so the flake by default would resolve `nixosConfigurations.nixos-bootstrap`
+Subsequent rebuilds can be called with the default command as after first build the machines hostname will be changed to the desired machine
+
+```sh
+nixos-rebuild switch --flake .#<machinename>
+```

flake.nix

@@ -225,7 +225,6 @@
             ];
             profileModules = [
               ./nixos/profiles/role-server.nix
-              ./nixos/profiles/impermanence.nix
               { home-manager.users.truxnell = ./nixos/home/truxnell/server.nix; }
             ];
           };
@@ -240,7 +239,6 @@
             ];
             profileModules = [
               ./nixos/profiles/role-server.nix
-              ./nixos/profiles/impermanence.nix
               { home-manager.users.truxnell = ./nixos/home/truxnell/server.nix; }
             ];
           };

images/README.md

@@ -1,31 +0,0 @@
-# ISO Image builds
-
-A minimal NixOS install iso build.
-
-Mainly useful for force-enabling `sshd` with my public key to allow headless deployments.
-
-> https://nixos.wiki/wiki/Creating_a_NixOS_live_CD
-
-## Building
-
-```
-cd iso
-nix-build '<nixpkgs/nixos>' -A config.system.build.isoImage -I nixos-config=iso.nix
-```
-
-# Checking image contents
-
-```
-$ mkdir mnt
-$ sudo mount -o loop result/iso/nixos-*.iso mnt
-$ ls mnt
-boot  EFI  isolinux  nix-store.squashfs  version.txt
-$ umount mnt
-```
-
-# Testing image in QEMU
-
-```
-$ nix-shell -p qemu
-$ qemu-system-x86_64 -enable-kvm -m 256 -cdrom result/iso/nixos-*.iso
-```

images/iso.nix

@@ -1,26 +0,0 @@
-{ config
-, pkgs
-, ...
-}: {
-  imports = [
-    <nixpkgs/nixos/modules/installer/cd-dvd/installation-cd-minimal.nix>
-
-    # Provide an initial copy of the NixOS channel so that the user
-    # doesn't need to run "nix-channel --update" first.
-    # <nixpkgs/nixos/modules/installer/cd-dvd/channel.nix>
-  ];
-
-  environment.systemPackages = [
-    pkgs.jq
-    pkgs.yq
-    pkgs.unixtools.top
-    pkgs.vim
-    pkgs.git
-    pkgs.dnsutils
-  ];
-
-  systemd.services.sshd.wantedBy = pkgs.lib.mkForce [ "multi-user.target" ];
-  users.users.root.openssh.authorizedKeys.keys = [
-    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMZS9J1ydflZ4iJdJgO8+vnN8nNSlEwyn9tbWU9OcysW truxnell@home"
-  ];
-}

nixos/home/truxnell/workstation.nix

@@ -82,15 +82,18 @@ with config;
     # Install these packages for my user
     packages = with pkgs;
       [
+        #apps
         discord
         steam
         spotify
-        brightnessctl
         prusa-slicer
         bitwarden
         yubioath-flutter
         yubikey-manager-qt
+        flameshot
+        vlc
 
+        # cli
         bat
         dbus
         direnv
@@ -99,7 +102,10 @@ with config;
         python3
         fzf
         ripgrep
-        flyctl # fly.io control line
+
+        brightnessctl
+
+
 
       ];
 

nixos/hosts/daedalus/default.nix

@@ -12,12 +12,13 @@
   ];
 
   mySystem.purpose = "Network Attached Storage";
+  mySystem.system.impermanence.enable = true;
   mySystem.services = {
     openssh.enable = true;
 
     #containers
     podman.enable = true;
-    traefik.enable = true;
+    nginx.enable = true;
     sonarr.enable = true;
     radarr.enable = true;
     lidarr.enable = true;
@@ -28,6 +29,8 @@
 
 
   };
+  mySystem.security.acme.enable = true;
+
   mySystem.nasFolder = "/tank";
   mySystem.system.resticBackup.local.location = "/tank/backup/nixos/nixos";
 
@@ -86,7 +89,7 @@
     {
       device = "rpool/safe/persist";
       fsType = "zfs";
-      # neededForBoot = true; # for impermanence
+      neededForBoot = true; # for impermanence
     };
 
   swapDevices =

nixos/hosts/durandal/default.nix

@@ -10,10 +10,9 @@
   mySystem.services = {
     openssh.enable = true;
     podman.enable = true;
-    traefik.enable = true;
+    nginx.enable = true;
-    postgresql.enable = true;
-
 
+    openvscode-server.enable = true;
 
   };
   mySystem.system.systemd.pushover-alerts.enable = false;
@@ -21,6 +20,7 @@
   mySystem.nfs.nas.enable = true;
   mySystem.persistentFolder = "/persistent";
   mySystem.system.motd.networkInterfaces = [ "eno1" ];
+  mySystem.security.acme.enable = true;
 
   # Dev machine
   mySystem.system.resticBackup =

nixos/hosts/shodan/default.nix

@@ -7,10 +7,12 @@
 , ...
 }: {
   mySystem.purpose = "Homelab";
+  mySystem.system.impermanence.enable = true;
   mySystem.services = {
     openssh.enable = true;
     podman.enable = true;
-    traefik.enable = true;
+
+    nginx.enable = true;
 
     gatus.enable = true;
     homepage.enable = true;
@@ -30,9 +32,11 @@
     home-assistant.enable = true;
     openvscode-server.enable = true;
 
-
+    radicale.enable = true;
   };
 
+  mySystem.security.acme.enable = true;
+
   mySystem.nfs.nas.enable = true;
   mySystem.persistentFolder = "/persist";
   mySystem.system.motd.networkInterfaces = [ "enp1s0" ];
@@ -81,7 +85,7 @@
     {
       device = "rpool/safe/persist";
       fsType = "zfs";
-      # neededForBoot = true; # for impermanence
+      neededForBoot = true; # for impermanence
     };
 
   fileSystems."/boot" =

nixos/lib/default.nix

@@ -70,7 +70,7 @@ rec {
         extraOptions = containerExtraOptions;
       };
 
-      systemd.tmpfiles.rules = lib.optionals (lib.attrsets.hasAttrByPath [ "persistence" "folder" ] options) [ "d ${options.persistence.folder} 0755 ${user} ${group} -" ]
+      systemd.tmpfiles.rules = lib.optionals (lib.attrsets.hasAttrByPath [ "persistence" "folder" ] options) [ "d ${options.persistence.folder} 0750 ${user} ${group} -" ]
       ;
 
       # built a entry for homepage

nixos/modules/nixos/containers/arr/lidarr/default.nix

@@ -11,8 +11,9 @@ let
   group = "568"; #string
   port = 8686; #int
   cfg = config.mySystem.services.${app};
-  appFolder = "containers/${app}";
+  appFolder = "/var/lib/${app}";
-  persistentFolder = "${config.mySystem.persistentFolder}/${appFolder}";
+
+  # persistentFolder = "${config.mySystem.persistentFolder}/var/lib/${appFolder}";
 in
 {
   options.mySystem.services.${app} =
@@ -24,7 +25,7 @@ in
   config = mkIf cfg.enable {
     # ensure folder exist and has correct owner/group
     systemd.tmpfiles.rules = [
-      "d ${persistentFolder} 0755 ${user} ${group} -" #The - disables automatic cleanup, so the file wont be removed after a period
+      "d ${appFolder} 0750 ${user} ${group} -" #The - disables automatic cleanup, so the file wont be removed after a period
     ];
 
     sops.secrets."services/${app}/env" = {
@@ -49,18 +50,25 @@ in
       };
       environmentFiles = [ config.sops.secrets."services/${app}/env".path ];
       volumes = [
-        "${persistentFolder}:/config:rw"
+        "${appFolder}:/config:rw"
         "${config.mySystem.nasFolder}/natflix:/media:rw"
         "/etc/localtime:/etc/localtime:ro"
       ];
-      labels = lib.myLib.mkTraefikLabels {
+    };
-        name = app;
+
-        inherit (config.networking) domain;
+    services.nginx.virtualHosts."${app}.${config.networking.domain}" = {
+      useACMEHost = config.networking.domain;
+      forceSSL = true;
+      locations."/" = {
+        proxyPass = "http://${app}:${builtins.toString port}";
+        extraConfig = "resolver 10.88.0.1;";
 
-        inherit port;
       };
     };
 
+     environment.persistence."${config.mySystem.system.impermanence.persistPath}" = lib.mkIf config.mySystem.system.impermanence.enable {
+      directories = [{ directory = appFolder; user = user; group = group; mode = "750"; }];
+    };
 
     mySystem.services.homepage.media = mkIf cfg.addToHomepage [
       {

nixos/modules/nixos/containers/arr/prowlarr/default.nix

@@ -11,8 +11,8 @@ let
   group = "568"; #string
   port = 9696; #int
   cfg = config.mySystem.services.${app};
-  appFolder = "containers/${app}";
+  appFolder = "/var/lib/${app}";
-  persistentFolder = "${config.mySystem.persistentFolder}/${appFolder}";
+  # persistentFolder = "${config.mySystem.persistentFolder}/var/lib/${appFolder}";
 in
 {
   options.mySystem.services.${app} =
@@ -24,7 +24,7 @@ in
   config = mkIf cfg.enable {
     # ensure folder exist and has correct owner/group
     systemd.tmpfiles.rules = [
-      "d ${persistentFolder} 0755 ${user} ${group} -" #The - disables automatic cleanup, so the file wont be removed after a period
+      "d ${appFolder} 0750 ${user} ${group} -" #The - disables automatic cleanup, so the file wont be removed after a period
     ];
 
     sops.secrets."services/${app}/env" = {
@@ -48,15 +48,23 @@ in
       };
       environmentFiles = [ config.sops.secrets."services/${app}/env".path ];
       volumes = [
-        "${persistentFolder}:/config:rw"
+        "${appFolder}:/config:rw"
         "/etc/localtime:/etc/localtime:ro"
       ];
-      labels = lib.myLib.mkTraefikLabels {
-        name = app;
-        inherit (config.networking) domain;
-
-        inherit port;
     };
+
+    services.nginx.virtualHosts."${app}.${config.networking.domain}" = {
+      useACMEHost = config.networking.domain;
+      forceSSL = true;
+      locations."/" = {
+        proxyPass = "http://${app}:${builtins.toString port}";
+        extraConfig = "resolver 10.88.0.1;";
+
+      };
+    };
+
+     environment.persistence."${config.mySystem.system.impermanence.persistPath}" = lib.mkIf config.mySystem.system.impermanence.enable {
+      directories = [{ directory = appFolder; user = user; group = group; mode = "750"; }];
     };
 
     mySystem.services.homepage.media = mkIf cfg.addToHomepage [

nixos/modules/nixos/containers/arr/radarr/default.nix

@@ -11,8 +11,8 @@ let
   group = "568"; #string
   port = 7878; #int
   cfg = config.mySystem.services.${app};
-  appFolder = "containers/${app}";
+  appFolder = "/var/lib/${app}";
-  persistentFolder = "${config.mySystem.persistentFolder}/${appFolder}";
+  # persistentFolder = "${config.mySystem.persistentFolder}/var/lib/${appFolder}";
 in
 {
   options.mySystem.services.${app} =
@@ -24,7 +24,7 @@ in
   config = mkIf cfg.enable {
     # ensure folder exist and has correct owner/group
     systemd.tmpfiles.rules = [
-      "d ${persistentFolder} 0755 ${user} ${group} -" #The - disables automatic cleanup, so the file wont be removed after a period
+      "d ${appFolder} 0750 ${user} ${group} -" #The - disables automatic cleanup, so the file wont be removed after a period
     ];
 
     sops.secrets."services/${app}/env" = {
@@ -49,18 +49,28 @@ in
       };
       environmentFiles = [ config.sops.secrets."services/${app}/env".path ];
       volumes = [
-        "${persistentFolder}:/config:rw"
+        "${appFolder}:/config:rw"
         "${config.mySystem.nasFolder}/natflix:/media:rw"
         "/etc/localtime:/etc/localtime:ro"
       ];
-      labels = lib.myLib.mkTraefikLabels {
-        name = app;
-        inherit (config.networking) domain;
 
-        inherit port;
+    };
+
+    services.nginx.virtualHosts."${app}.${config.networking.domain}" = {
+      useACMEHost = config.networking.domain;
+      forceSSL = true;
+      locations."/" = {
+        proxyPass = "http://${app}:${builtins.toString port}";
+        extraConfig = "resolver 10.88.0.1;";
+
       };
     };
 
+     environment.persistence."${config.mySystem.system.impermanence.persistPath}" = lib.mkIf config.mySystem.system.impermanence.enable {
+      directories = [{ directory = appFolder; user = user; group = group; mode = "750"; }];
+    };
+
+
     mySystem.services.homepage.media = mkIf cfg.addToHomepage [
       {
         Radarr = {

nixos/modules/nixos/containers/arr/readarr/default.nix

@@ -11,8 +11,8 @@ let
   group = "568"; #string
   port = 8787; #int
   cfg = config.mySystem.services.${app};
-  appFolder = "containers/${app}";
+  appFolder = "/var/lib/${app}";
-  persistentFolder = "${config.mySystem.persistentFolder}/${appFolder}";
+  # persistentFolder = "${config.mySystem.persistentFolder}/var/lib/${appFolder}";
 in
 {
   options.mySystem.services.${app} =
@@ -24,7 +24,7 @@ in
   config = mkIf cfg.enable {
     # ensure folder exist and has correct owner/group
     systemd.tmpfiles.rules = [
-      "d ${persistentFolder} 0755 ${user} ${group} -" #The - disables automatic cleanup, so the file wont be removed after a period
+      "d ${appFolder} 0750 ${user} ${group} -" #The - disables automatic cleanup, so the file wont be removed after a period
     ];
 
     sops.secrets."services/${app}/env" = {
@@ -48,16 +48,24 @@ in
       };
       environmentFiles = [ config.sops.secrets."services/${app}/env".path ];
       volumes = [
-        "${persistentFolder}:/config:rw"
+        "${appFolder}:/config:rw"
         "${config.mySystem.nasFolder}/natflix:/media:rw"
         "/etc/localtime:/etc/localtime:ro"
       ];
-      labels = lib.myLib.mkTraefikLabels {
-        name = app;
-        inherit (config.networking) domain;
-
-        inherit port;
     };
+
+    services.nginx.virtualHosts."${app}.${config.networking.domain}" = {
+      useACMEHost = config.networking.domain;
+      forceSSL = true;
+      locations."/" = {
+        proxyPass = "http://${app}:${builtins.toString port}";
+        extraConfig = "resolver 10.88.0.1;";
+
+      };
+    };
+
+     environment.persistence."${config.mySystem.system.impermanence.persistPath}" = lib.mkIf config.mySystem.system.impermanence.enable {
+      directories = [{ directory = appFolder; user = user; group = group; mode = "750"; }];
     };
 
     mySystem.services.homepage.media = mkIf cfg.addToHomepage [

nixos/modules/nixos/containers/arr/sonarr/default.nix

@@ -11,8 +11,8 @@ let
   group = "568"; #string
   port = 8989; #int
   cfg = config.mySystem.services.${app};
-  appFolder = "containers/${app}";
+  appFolder = "/var/lib/${app}";
-  persistentFolder = "${config.mySystem.persistentFolder}/${appFolder}";
+  # persistentFolder = "${config.mySystem.persistentFolder}/var/lib/${appFolder}";
   containerPersistentFolder = "/config";
 in
 {
@@ -25,7 +25,7 @@ in
   config = mkIf cfg.enable {
     # ensure folder exist and has correct owner/group
     systemd.tmpfiles.rules = [
-      "d ${persistentFolder} 0755 ${user} ${group} -" #The - disables automatic cleanup, so the file wont be removed after a period
+      "d ${appFolder} 0750 ${user} ${group} -" #The - disables automatic cleanup, so the file wont be removed after a period
     ];
 
     sops.secrets."services/${app}/env" = {
@@ -51,16 +51,24 @@ in
       };
       environmentFiles = [ config.sops.secrets."services/${app}/env".path ];
       volumes = [
-        "${persistentFolder}:/config:rw"
+        "${appFolder}:/config:rw"
         "${config.mySystem.nasFolder}/natflix:/media:rw"
         "/etc/localtime:/etc/localtime:ro"
       ];
-      labels = lib.myLib.mkTraefikLabels {
-        name = app;
-        inherit (config.networking) domain;
-
-        inherit port;
     };
+
+    services.nginx.virtualHosts."${app}.${config.networking.domain}" = {
+      useACMEHost = config.networking.domain;
+      forceSSL = true;
+      locations."/" = {
+        proxyPass = "http://${app}:${builtins.toString port}";
+        extraConfig = "resolver 10.88.0.1;";
+
+      };
+    };
+
+     environment.persistence."${config.mySystem.system.impermanence.persistPath}" = lib.mkIf config.mySystem.system.impermanence.enable {
+      directories = [{ directory = appFolder; user = user; group = group; mode = "750"; }];
     };
 
     mySystem.services.homepage.media = mkIf cfg.addToHomepage [

nixos/modules/nixos/containers/backrest/default.nix

@@ -11,8 +11,8 @@ let
   group = "568"; #string
   port = 9898; #int
   cfg = config.mySystem.services.${app};
-  appFolder = "containers/${app}";
+  appFolder = "/var/lib/${app}";
-  persistentFolder = "${config.mySystem.persistentFolder}/${appFolder}";
+  # persistentFolder = "${config.mySystem.persistentFolder}/var/lib/${appFolder}";
 in
 {
   options.mySystem.services.${app} =
@@ -24,9 +24,9 @@ in
   config = mkIf cfg.enable {
     # ensure folder exist and has correct owner/group
     systemd.tmpfiles.rules = [
-      "d ${persistentFolder}/config 0755 ${user} ${group} -"
+      "d ${appFolder}/config 0750 ${user} ${group} -"
-      "d ${persistentFolder}/data 0755 ${user} ${group} -"
+      "d ${appFolder}/data 0750 ${user} ${group} -"
-      "d ${persistentFolder}/cache 0755 ${user} ${group} -"
+      "d ${appFolder}/cache 0750 ${user} ${group} -"
     ];
 
     virtualisation.oci-containers.containers.${app} = {
@@ -39,20 +39,24 @@ in
         XDG_CACHE_HOME = "/cache";
       };
       volumes = [
-        "${persistentFolder}/nixos/config:/config:rw"
+        "${appFolder}/nixos/config:/config:rw"
-        "${persistentFolder}/nixos/data:/data:rw"
+        "${appFolder}/nixos/data:/data:rw"
-        "${persistentFolder}/nixos/cache:/cache:rw"
+        "${appFolder}/nixos/cache:/cache:rw"
         "${config.mySystem.nasFolder}/backup/nixos/nixos:/repos:rw"
         "/etc/localtime:/etc/localtime:ro"
       ];
-      labels = lib.myLib.mkTraefikLabels {
+    };
-        name = app;
-        inherit (config.networking) domain;
 
-        inherit port;
+    services.nginx.virtualHosts."${app}.${config.networking.domain}" = {
+      useACMEHost = config.networking.domain;
+      forceSSL = true;
+      locations."/" = {
+        proxyPass = "http://${app}:${builtins.toString port}";
+        extraConfig = "resolver 10.88.0.1;";
       };
     };
 
+
     mySystem.services.homepage.infrastructure = mkIf cfg.addToHomepage [
       {
         Backrest = {

nixos/modules/nixos/containers/cross-seed/default.nix

@@ -11,8 +11,8 @@ let
   group = "568"; #string
   port = 8080; #int
   cfg = config.mySystem.services.${app};
-  appFolder = "containers/${app}";
+  appFolder = "/var/lib/${app}";
-  persistentFolder = "${config.mySystem.persistentFolder}/${appFolder}";
+  # persistentFolder = "${config.mySystem.persistentFolder}/var/lib/${appFolder}";
   configFile = builtins.toFile "config.js" (builtins.toJSON configVar);
 
 in
@@ -25,7 +25,7 @@ in
   config = mkIf cfg.enable {
     # ensure folder exist and has correct owner/group
     systemd.tmpfiles.rules = [
-      "d ${persistentFolder} 0755 ${user} ${group} -" #The - disables automatic cleanup, so the file wont be removed after a period
+      "d ${appFolder} 0750 ${user} ${group} -" #The - disables automatic cleanup, so the file wont be removed after a period
     ];
 
     virtualisation.oci-containers.containers.${app} = {
@@ -33,7 +33,7 @@ in
       user = "${user}:${group}";
       cmd = [ "daemon" ];
       volumes = [
-        "${persistentFolder}:/config:rw"
+        "${appFolder}:/config:rw"
         "${configFile}:/config/config.yaml:ro"
         "/etc/localtime:/etc/localtime:ro"
       ];

nixos/modules/nixos/containers/default.nix

@@ -13,6 +13,6 @@
     ./whoogle
     ./redlib
     ./home-assistant
-    ./node-red
+    # ./calibre
   ];
 }

nixos/modules/nixos/containers/factorio/default.nix

@@ -13,8 +13,8 @@ let
   port = 34203; #int
   port_rcon = 27019; #int
   cfg = config.mySystem.services.${app}.${instance};
-  appFolder = "containers/${app}/${instance}";
+  appFolder = "/var/lib/${app}/${instance}";
-  persistentFolder = "${config.mySystem.persistentFolder}/${appFolder}";
+  # persistentFolder = "${config.mySystem.persistentFolder}/var/lib/${appFolder}";
 in
 {
   options.mySystem.services.${app}.${instance} =
@@ -30,7 +30,7 @@ in
 
     # ensure folder exist and has correct owner/group
     systemd.tmpfiles.rules = [
-      "d ${persistentFolder} 0755 ${user} ${group} -" #The - disables automatic cleanup, so the file wont be removed after a period
+      "d ${appFolder} 0755 ${user} ${group} -" #The - disables automatic cleanup, so the file wont be removed after a period
     ];
     # make user for container
     users = {
@@ -59,7 +59,7 @@ in
       image = "${image}";
       user = "${user}:${group}";
       volumes = [
-        "${persistentFolder}:/factorio:rw"
+        "${appFolder}:/factorio:rw"
         "/etc/localtime:/etc/localtime:ro"
       ];
       environment =
@@ -76,6 +76,9 @@ in
       allowedTCPPorts = [ port ]; # I dont use rcon so not opening that too.
     };
 
+     environment.persistence."${config.mySystem.system.impermanence.persistPath}" = lib.mkIf config.mySystem.system.impermanence.enable {
+      directories = [{ directory = appFolder; user = user; group = group; mode = "750"; }];
+    };
 
 
     mySystem.services.gatus.monitors = mkIf config.mySystem.services.gatus.enable [{

nixos/modules/nixos/containers/gatus/default.nix

@@ -12,8 +12,9 @@ let
   group = "568"; #string
   port = 8080; #int
   cfg = config.mySystem.services.${app};
-  appFolder = "containers/${app}";
+  appFolder = "/var/lib/${app}";
-  persistentFolder = "${config.mySystem.persistentFolder}/${appFolder}";
+
+ # persistentFolder = "${config.mySystem.persistentFolder}/var/lib/${appFolder}";
   containerPersistentFolder = "/config";
   extraEndpoints = [
     # TODO refactor these out into their own file or fake host?
@@ -103,16 +104,19 @@ in
         "${configFile}:/config/config.yaml:ro"
       ];
 
-      labels = lib.myLib.mkTraefikLabels {
-        name = app;
-        inherit (config.networking) domain;
-
-        inherit port;
-      };
-
       extraOptions = [ "--cap-add=NET_RAW" ]; # Required for ping/etc to do monitoring
     };
 
+    services.nginx.virtualHosts."${app}.${config.networking.domain}" = {
+      useACMEHost = config.networking.domain;
+      forceSSL = true;
+      locations."/" = {
+        proxyPass = "http://${app}:${builtins.toString port}";
+        extraConfig = "resolver 10.88.0.1;";
+      };
+    };
+
+
     mySystem.services.homepage.infrastructure = mkIf cfg.addToHomepage [
       {
         "Gatus Internal" = {

nixos/modules/nixos/containers/home-assistant/default-old.nix

@@ -1,67 +0,0 @@
-{ lib
-, config
-, pkgs
-, ...
-}:
-with lib;
-let
-  cfg = config.mySystem.services.home-assistant;
-  app = "Home-assistant";
-  user = "kah";
-  group = "kah";
-  appFolder = "home-assistant";
-  persistentFolder = "${config.mySystem.persistentFolder}/containers/${appFolder}";
-
-in
-{
-  options.mySystem.services.home-assistant.enable = mkEnableOption "home-assistant";
-
-  # running in a container vs nix module mainly
-  # as I know the container is solid. Bit iffy
-  # over the packaging of HA in nix & arguments
-  # from HA dev on nix packaging
-  config = mkIf cfg.enable
-    (lib.recursiveUpdate
-      {
-        sops.secrets."services/${app}/env" = {
-          sopsFile = ./secrets.sops.yaml;
-          owner = user;
-          inherit group;
-          restartUnits = [ "podman-${app}.service" ];
-        };
-      }
-
-      (myLib.mkService
-        {
-          inherit app user group;
-          description = "Home Automation";
-          port = 8123;
-          inherit (config.time) timeZone;
-          # subdomainOverride = "hass";
-          inherit (config.networking) domain;
-          persistence = {
-            folder = persistentFolder;
-            backup = true;
-          };
-          homepage = {
-            icon = "home-assistant.svg";
-            category = "home";
-          };
-          container = {
-            enable = true;
-            image = "ghcr.io/onedr0p/home-assistant:2024.1.5@sha256:64bb3ffa532c3c52563f0e4a4de8d50c889f42a1b0826b35ee1ac728652fb107";
-            env = {
-              HASS_IP = "10.8.20.42";
-            };
-            envFiles = [ config.sops.secrets."services/${app}/env".path ];
-            persistentFolderMount = "/config";
-            addTraefikLabels = true;
-            caps = {
-              # readOnly = true;
-              noNewPrivileges = true;
-              # dropAll = true;
-            };
-          };
-        })
-    );
-}

nixos/modules/nixos/containers/home-assistant/default.nix

@@ -7,12 +7,12 @@ with lib;
 let
   app = "home-assistant";
   image = "ghcr.io/onedr0p/home-assistant:2024.1.5@sha256:64bb3ffa532c3c52563f0e4a4de8d50c889f42a1b0826b35ee1ac728652fb107";
-  user = "568"; #string
+  user = "kah"; #string
-  group = "568"; #string
+  group = "kah"; #string
   port = 8123; #int
   cfg = config.mySystem.services.${app};
-  appFolder = "containers/${app}";
+  appFolder = "/var/lib/${app}";
-  persistentFolder = "${config.mySystem.persistentFolder}/${appFolder}";
+  # persistentFolder = "${config.mySystem.persistentFolder}/var/lib/${appFolder}";
 in
 {
   options.mySystem.services.${app} =
@@ -24,7 +24,7 @@ in
   config = mkIf cfg.enable {
     # ensure folder exist and has correct owner/group
     systemd.tmpfiles.rules = [
-      "d ${persistentFolder} 0755 ${user} ${group} -" #The - disables automatic cleanup, so the file wont be removed after a period
+      "d ${appFolder} 0750 ${user} ${group} -" #The - disables automatic cleanup, so the file wont be removed after a period
     ];
 
     sops.secrets."services/${app}/env" = {
@@ -44,30 +44,38 @@ in
       };
       environmentFiles = [ config.sops.secrets."services/${app}/env".path ];
       volumes = [
-        "${persistentFolder}:/config:rw"
+        "${appFolder}:/config:rw"
         "/etc/localtime:/etc/localtime:ro"
       ];
-      labels = lib.myLib.mkTraefikLabels {
+    };
-        name = app;
-        inherit (config.networking) domain;
 
-        inherit port;
+    services.nginx.virtualHosts."${app}.${config.networking.domain}" = {
+      useACMEHost = config.networking.domain;
+      forceSSL = true;
+      locations."/" = {
+        proxyPass = "http://${app}:${builtins.toString port}";
+        proxyWebsockets = true;
+        extraConfig = "resolver 10.88.0.1;";
       };
     };
 
 
+
     mySystem.services.homepage.media = mkIf cfg.addToHomepage [
       {
-        Lidarr = {
+        ${app} = {
           icon = "${app}.svg";
           href = "https://${app}.${config.mySystem.domain}";
-
           description = "Home automation";
           container = "${app}";
         };
       }
     ];
 
+    environment.persistence."${config.mySystem.system.impermanence.persistPath}" = lib.mkIf config.mySystem.system.impermanence.enable {
+      directories = [{ directory = appFolder; user = "kah"; group = "kah"; mode = "750"; }];
+    };
+
     mySystem.services.gatus.monitors = [{
 
       name = app;

nixos/modules/nixos/containers/home-assistant/secrets.sops.yaml

@@ -1,6 +1,6 @@
 services:
     home-assistant:
-        env: ENC[AES256_GCM,data:SzSN0Gu1RhTL4VDyX5nOPFadNZ4s0grYnQKzZhHb3MwPkYtrinnFKARahZ0dkvnKjOjCxxP6uaYHF/9rBxyRRuxjoOk7Qj4xF9Rr9v5e/YSyJHhIOcX4KTaagLW55t7IK0dmHPX2eimWdUcTh1MKjWKgG6CtRGLYMN4fPTxzTjoLjtGiHD3tLhrDc+vMS4GYDWocBx3FXp8vICbKaLpVsUHT3JUWstQhnadtmCvywgrXid5UoJnWxtpQxR2vf7m7SYjcIRX1iA6/PHAhgZftsMt8Aw4tw8+gR5jJivmkBm3HE2iJwtDJnowvdVeAaLMdpdzDW45a7P++F5Cwr5yQLuKfkPeCkgiD1EzDKjQs4Rnq19+7ZDZT8WCA5wtzwP8+yzjLO0wZfJ4Bda0jd1tR+FeID3NSsU1il8S3bXLegLxmCLnYvzKRdxHrUUFu85xTfe8GYMtllUv8c10zAL9nRmRUPXh7JpXntKEidEy6w0F1iOWsR67yp1qOzlGOu7k0wMAS7QZ5/Op/nBjuugh+mOyABO5CpjnPqIXc19oerkNCHZJOmmHdT7wf3n90yn4j09XyIYDI6e7G+FXa/QwBqO5r0z+ulqHcpyhkklB425Zz5G2QUCBCz3mqJ/RhiWk+g4WwNEEecy31hNIaVl4zwwTlk/iFXdZ938NWp3jZp2zOzhGLlRcFbcuUzvqXqL1tFmIokIE/4AQll1zkmgUFFc/76ppTc4qylNy9z7NxM5/SJoONKW2jx4Tf2sEBIK/LICHgGu2VzBhCSOW+EyWr4vziRT9Kq4TVHLf56k+Sh2SKGFZg4FKkIVl7ZbiQyRp88Jqlna4sZwlA+4bcPMi8BMFj3yREv6t1uSaiUO+OtMpCrREuU9afp4OsQs1QyoED24Zh1TKkpZ7vJ6rYCT/j0EEEsSMEcV/dmIoyawGyM8KB5ZjZIR8LWaLffHARWBcholKANWv9a3kFp0qsqvqVxxNxH3GEqZlFz83igwWOeILeSdZ2BEVKZBP4tmQu8HvUEPHtucRUIbsPSMgRXHJPbmia4w7053OrmATEKNYKVXCspoA0gsPF9AqsH/I67NbpamJ0F1/sc+d0Zw8oJKHexes9WAa3S3bCVwdtPI2cIMNLTA4T7fA5d4yZ6joe+CnHZnprbd7LlQIoK/MEM8bftvGz3Jnk/3fqTOlW8yC4tknsnO44m3LJmQ2rDbWlswO3rQTP/SCpkHcXTgbT5A==,iv:q4C1fJqWWH5RIn42p7VWwonpMaOeahyp2jzb+1pNPr0=,tag:gDytkHSuISDbNqIbfvDnwQ==,type:str]
+        env: ENC[AES256_GCM,data:tQS5aEMYKFjUqf2yD7xzIOEuhVYh9gyMIRNytx13CjCR7s4NheuFroyiDtvRck/lY849JR+m9Tmp35VTeWjUHpzmlxWUDt7oXS6jvTehUTU3dvRjXoQI7Fq2VVvq8V3p3HRhjlTRUS3hHthG5Qv6oRO0C2m0tA6cx1lzk5VPF6yFO+1XWGSsxf453QgpN5jZ5/EIKcEn4qjV6jIaTFY0ZpqnZ9TRjGyPFWDgLItlOt/NUDhRyPx6h1uy7IRCl2WviVU301DS+SNwOtqnqMWHaMNzHtAaofZ/RSmVbLWZOoxFlyM141XmDYaouAroz9aYY3fbGcijnBcI7/8qHsUU23NQWrDodYDo0dmB2HGiob7ajBpxxvyt0V5rKqHZzjeZ32xDu55S3X35uuoMqAwfnxkA39cT8RA4PaKhvQNnxUK2LUhtffcfSLU7i5nNMjqr9I2iDbvdOVtR+SQb3ujJr+ryED0KdYbpz86Ddc3OoZQLNymqTcYypTuCfAjyEuMiGGywVYL+96PukwYF+gifm0VtMjPlhEXsQorHTOehTJty3H2U7BX79Ij4MmxIJCo7V05pF4HpPXWPNkmtsyBB1f6K+eyk93XpCQw4izeqYT10nr6fYOXwDFGOZs8S/vmMGxjVoo3uteHpu925RCptc/y1Nq6D/ly6CZBT0aGeVdxPdnHwiWG52dPBUWemHO5sPqBynU5rFYJBOzADwTbhq+CnmL4EHKLmJ3ldjpemgYXEZ1Cn3HnWFHdCVlkfCLnTC4o3tD3GrO1DTk26PeGpj7cRwMruzGrYpJUauVWfgzxXHFDkAyQnLRiNRCA6ETBNT/NYx1Te4Q4hrRnrBWYI9/eb7OHnaGgty84Vh4AqagInjuK7DXQf6XutEt9x1X2Vms3/qvKnczPKeQhjS1BJ2udCtQ3fZfQJ5a4hW9B9Z5Gl8D4q7BguPzYrpen6mDd1gV71etQphsrVqacM2SPiRRxDQoKP2NgNL5dnlqE8sCaNVeqmacHNPk691U8ELH3tziEqXbH/WWwEOS457grXctCsJs/Rd5R2GypxmeYV3ckAsP9pQxnglH16X+ZFj3qCECcNQw8+1Xp2WiEiIZUDeWC9TKpQhUw2wkjLVyWbRx11Y5u5nt7upieckbj/gPTBc37+YBmhcYF23hWbvd/sfe69iQuDefYQ2arDXoaoDJhtzA==,iv:ZXIxLmtV601lRtcasR3wgrohPsfoK4OH5UZjFgrCjnA=,tag:aOCRQ9Mf4Qx9yxmwvRQCGA==,type:str]
 sops:
     kms: []
     gcp_kms: []
@@ -10,68 +10,68 @@ sops:
         - recipient: age1lj5vmr02qkudvv2xedfj5tq8x93gllgpr6tzylwdlt7lud4tfv5qfqsd5u
           enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBNQzA1VlEzeVVwcDN6ZHpm
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBOMEdLYlN2Q1ZFMW9DcUpD
-            bDhYNlU4VWI5Z2FwUmVNZUJETUhKSkVCcm1vClI0ZmlyekRicUR4Z2lhMVdpUjE2
+            SGlvS3llaW1iM0FDM3paRFFjeXRKMlhRbFVVCnBJMk9URG9FY0Z4bFkrbWQ3UG13
-            VXdrRDhuQWZ1K2Frb1dsWWFvdDBNQWsKLS0tIDgvcTBncUpYcHE4cXJzeUl6K2lY
+            MDFSOExMU0ZuTjNQQUdDT0g4WGdpRnMKLS0tIHFORlV5eVhFKzZyMVlsZW8xUW0v
-            VEJPencraktjeTFMeTJxWUowbUcyeFUKWRIPe0rrbHRpE4d4saqS9xMRzcy2huwg
+            RWhvWXBXak1Rei9vekkwV016OFhhM0kKLWOVpnb+uCMPEkHPrBpbZUUQHLhF3+8+
-            kDXek89pTY+2kYcPiKIQ4FPdOO2abGkRZeNWUZ/Nc/MvUqBhtQoUQA==
+            8v7hA9sc0Kq2zCIDyZtgU537Rq1nWFUvFqBrtjd3LAD/JDNnlFa0qg==
             -----END AGE ENCRYPTED FILE-----
         - recipient: age17edew3aahg3t5nte5g0a505sn96vnj8g8gqse8q06ccrrn2n3uysyshu2c
           enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB3ZnRncGo4VzFieTJIQXFn
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB5eW90MU5rb2R5OWFOWkM5
-            VzFVQVhVSkVyeTFGaEtuYXBnQU5mZDhrS0FFCnZPanhkL2pCTzBEc1IrMGMwTDh6
+            WW1mTUIwVUdNVVlBYk43TXFqVm5XdGM3Vmg4CkJOMllHS3F2NXBSbk9IblorT2Jw
-            eklPS0dOb0dJZEtEMU9YTHNQSURFNmcKLS0tIGRNTUU0MS9HMXdHQ0xXekNwNXMz
+            aEY2a3owMU5wS0J5aDlUUzAyQmNyd3MKLS0tIHBRNS8xVE96S25FekJwOWlVOHdp
-            OUwrYnRXQkEvNXFRTkl2VVFoKy85NVkKvw3PHp1TwvCmXGKCv4mxvJQYhKommZg0
+            R0VMeFYzSFBJeHUxUUo5bVlhQ2RQNTAKTe+h2IJwttc/sdupwJ49dWbLQb3N3hkV
-            e0OAmzasx8ek7HDySvfTuYjQKVSCB46wtojuAMUkPScI/yqJLHJ4rQ==
+            hgoLePD9nGuhTmfPR5KMD0mzjIoche5ZXGuoAIQRDTOzx6yA/wz4xw==
             -----END AGE ENCRYPTED FILE-----
         - recipient: age1u4tht685sqg6dkmjyer96r93pl425u6353md6fphpd84jh3jwcusvm7mgk
           enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSB2N1I4QXczQUZqb1NGdXdk
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSA5RUw2OFhSeDRLWlBLYkxs
-            VWRmTExoeDhXaVJPOWJYMW9IU2NseGxKY0ZzClhMRWJUSldDTzRvQkpBWldXVGVm
+            ZkgyUEQ1cVlJVEtiYTQydEE2Q0JqWEJDcDJVCkpNNDJiSmYrZTdPSnlmYm5YZndP
-            VXBmRFJDMkpaeXNDWE9lQ1Q5OWF6QUkKLS0tIGJ0VUd3OVZJVmRFMS9KdlFhSFdT
+            RysrVnZ1YUw0NTF1SnViM1Q1eFFCN2sKLS0tIENyYnpYV1pKYUwrOUg1MGFPY1R1
-            aU9rQU5mT3NNVjdvbUlaZ1hvbUErenMKyu1/RxivA+uaIjg+NbZUYSvMACyT5Jkq
+            bXNkRFhtQ1lsOEsvNTF3NzZFT3RoZkUKDpTIaasXNJME/IEiJ340JWKmxhqLbrXf
-            Qu6Gq+hzh5++pi+dBn/OHqwn0hCszBYT5MBu3PueZoLAyqLwsuQ9eQ==
+            t+SfYSRShB8hCYqb2RDlp8dvEKK/XtZbOk3MGgrdN3Ldpyc0OahN1A==
             -----END AGE ENCRYPTED FILE-----
         - recipient: age1cp6vegrmqfkuj8nmt2u3z0sur7n0f7e9x9zmdv4zygp8j2pnucpsdkgagc
           enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBXZEp2SFVDcnMvNmxmUERQ
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBoY1J1d3NZb0J2WVQ0NjJC
-            RG9FRnFuWmhCUnE2aktyWEFodExPWGo0OWxFCm5UbU91b1pVbkt3ZnIyNmlvbGxn
+            amUxOUtITmFFcEljOWZGcHM5eHNtSlZZZzFJClFGbVV1S0NabEIzZHBUb1ptNFI3
-            NFI4R2N2a1hJc2lhNEZiRzRCMFpWYUkKLS0tIHJkY1NnNjJwMmxTUDJRK2dIbzcv
+            N2l2YzBhNy9sbDMvQmhQaXVTMnZtODAKLS0tIDFnTWd5eU4wZzQyd1hEU3pQbjl0
-            TWMxL0FndGFqRlVyNFgxUjdIVVVRTUkKqtG3A2QgQtII1F3AhOBVggdtG8V8QezT
+            S2U5T2kwcHpWdlN5Mkt2SCtLbjJrTHMKkghmAdBiWM8NRCtX8KGkgv6qteqmWE0k
-            CTvO9LcCrb4cy2kC/7hC1HoUFVz3CJGlubS0QA9nfB4o3s5qdZ5XvA==
+            o++Vup33Bq+P4d4i8YvSPb0i8e96f1ZrZsxYRBXbsPGFEb8miTfXgQ==
             -----END AGE ENCRYPTED FILE-----
         - recipient: age1ekt5xz7u2xgdzgsrffhd9x22n80cn4thxd8zxjy2ey5vq3ca7gnqz25g5r
           enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBjUGdYb3BFWUJTekdzdHdo
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBTaThPY1huVExNYUxseUFC
-            N0JIblVRSldRTmFPMC9ReHJOVm04ditlSjJVCkpabGRldkVOTTA0RnF1cDZkZ2Ji
+            UlRPUFh6OGRVN0JiNWNkVU9DRFZuQmpVQzJNCkcxYUczOFpmNXM2QS9iL3ZaMjNW
-            UlhuU1NWTFBOSnc2TUI5amNXWUJDNWcKLS0tIG45Q2ZlbzdiZFlvODgrN1JtaTQy
+            RWcrcU80ckk0dnJ3MmtuOUVJSHExNUkKLS0tIEJnenRUUW15ZHRsY1JCZWRuQ1Aw
-            RG00dTI4RVNPL2lqbDVJTzMrODlsS0kKVj9SIj0j+dXB1yDAsshcumTTxx/IaGo9
+            dTVlcU41Y29QTVU2WHdtc2J1UjZOMjQKvHypo0Kf45Uwfvh4mYVQUtOUdHtuIQMV
-            FJ+Lhcb6w8C+e6MFLHFl45ifYcyRV7vqzzdPUzemjyJWm0FqbSeu+Q==
+            fF47SmAPUQ/BQiOkNlSzKD5iqzti/tzeUEIfGfo/ES3olegX7FiCBw==
             -----END AGE ENCRYPTED FILE-----
         - recipient: age1jpeh4s553taxkyxhzlshzqjfrtvmmp5lw0hmpgn3mdnmgzku332qe082dl
           enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBVU3FsMUxGTC9XUTJ4Tys1
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBTTW5JS3hKYkZyQUczMnVa
-            YXgxamo2d3IyTDQyS1orb1ZNbGJaaklGZnlFCmlnUjk1cysySlNHNmpQdERVSTRa
+            YTJ4bS9KbXFFeUJaUythZ3dkTGQ5Wm0wK2dBCnhIQkJaYUJnSTl6WFN5blZqcVBw
-            cFhIV1N1bUs0enJzMVkxYVBnNVdralkKLS0tIEFEak04NENELy9mNk4wVm4wWlVQ
+            ZlFYT3ZWQnIvOXBSQVI3V3pzemw5SmsKLS0tIHVKSVlMTHJaLzN0dGQzV0NRdkM1
-            TmsxQitKMkpNNzlQaTVVWnIzSHE3YUUK6d+F8J//NhP9/R8S9fm8JBI9N76bzD6o
+            MVVaYU9xVWg3ZFNYU1ZWdXE1WTJpOFkK8OisVjQZkEoMRywZPjLA34+U1BuAA14U
-            tLuj1N6MG+R2xONW1C4Eu39K+I5ynNyMPFCo7ACKb1TF91i28V7wtQ==
+            IXNeBI4TRm2Rw0fidloIfLV1mkig9tQ4cC2KO5CyxMXudtH2ijdxbQ==
             -----END AGE ENCRYPTED FILE-----
         - recipient: age1j2r8mypw44uvqhfs53424h6fu2rkr5m7asl7rl3zn3xzva9m3dcqpa97gw
           enc: |
             -----BEGIN AGE ENCRYPTED FILE-----
-            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBUVHFaR3BOaE1LSzFQSnB0
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBEdDZpRFpnbEIvMm5ZM3Ir
-            NnZsMnZud0xzcmV2S3JQUkpkaXlYTFFsa21BCjNYRFhJQ2VITGRRdzYycGFJVDFr
+            UWN6ZjVlZjZrWk44NTdEaUFVODZ4Y3lwWTJrClc5eTdnQ1dkbHBObFpGNStlVWVv
-            TFZ1ZFhjOW5MYThiUlloQW1Wa3lBMFEKLS0tIERrU3R1d2JWM2p6ZktxZ2JoamNq
+            Q0E0c2RmRlBvTmJNV216K000NmVOTWsKLS0tIFhJL3ZSbVVDeVpZcXA0NmhvMkNO
-            WTJJdkNXTy8yYUNQS0RzRHZpQlZjV2sKmLACWwksFbii3K5E9sfTSJq2Z3HMaj/F
+            WUlsU0NQcjE1cnQxbkd4dEtWMWJON2cK1H+ELmL6B2tNmQWRRXrzr40Fjtbxp5wt
-            hsHcpeNPdKtQlWw7ILa6Dl0/0sVAXssRFamb6teEVchipF9KJVATLQ==
+            /gBW58Dyafx4W68i7dPaSsTBzSl0K4VSh2mEC6phj1m2ABbsu/aIAg==
             -----END AGE ENCRYPTED FILE-----
-    lastmodified: "2024-04-29T08:40:40Z"
+    lastmodified: "2024-05-03T04:49:57Z"
-    mac: ENC[AES256_GCM,data:4S2pnHbrARApO/hRWn0ZlgWNNgdzzd2T/fx3LJ8M8n1cE5Us5Tj0yFxJPoiT/1Y+n3aZ/9hKD7AyIXhhrXA9gXk2MLHI1XjYVYkAtlW7264SIt9ZCEuJk4KN7LJnlFvxjF18yjKMBmjl34FCDaD8C+Pv+L3L1wOw/CMHHJ3Ayik=,iv:zT/TS+EDnq9s8mZiGR3rE2EDXwtPV3XUWE3xN2sHPQ4=,tag:Kh+PD7Hg5veeZvTb7Ta8rQ==,type:str]
+    mac: ENC[AES256_GCM,data:01/WpnAa0FvBQNy7PYgOns6W9ykpjQ+k8RcxAEo0tTou20E2veokYOmdGq1VPK1X8rYPbpFevF5rqC1gr4fIDWtzgRGxOf1htiYY0UWOSqyrRqdd/pNtI/FFRR96QSVhKhsKrc7oPJj95Gg5n2SnOfcoQHmHPlW1sIUV0PjedvE=,iv:hc2uuALlqWJftA9RvWHLwKem3nDlqMH7wbf2/TEOA4M=,tag:vVbzp/EUA3PChnj1e86KFg==,type:str]
     pgp: []
     unencrypted_suffix: _unencrypted
     version: 3.8.1

nixos/modules/nixos/containers/homepage/default.nix

@@ -12,8 +12,8 @@ let
   group = "568"; #string
   port = 3000; #int
   cfg = config.mySystem.services.${app};
-  appFolder = "containers/${app}";
+  appFolder = "/var/lib/${app}";
-  persistentFolder = "${config.mySystem.persistentFolder}/${appFolder}";
+ # persistentFolder = "${config.mySystem.persistentFolder}/var/lib/${appFolder}";
 
   # TODO refactor out this sht
   settings =
@@ -276,22 +276,10 @@ in
 
       ];
 
-      # labels = {
-      #   "traefik.enable" = "true";
-      #   "traefik.http.routers.${app}.entrypoints" = "websecure";
-      #   "traefik.http.routers.${app}.middlewares" = "local-ip-only@file";
-      #   "traefik.http.services.${app}.loadbalancer.server.port" = "${toString port}";
-      # };
-      labels = lib.myLib.mkTraefikLabels {
-        name = app;
-        inherit (config.networking) domain;
-
-        inherit port;
-      };
       # not using docker socket for discovery, just
       # building up the apps from a shared key
       # this is a bit more tedious, but more secure
-      # from not exposing docker socet and makes it
+      # from not exposing docker socket and makes it
       # easier to have/move services between hosts
       volumes = [
         "/etc/localtime:/etc/localtime:ro"
@@ -309,6 +297,17 @@ in
       ];
     };
 
+    services.nginx.virtualHosts."${app}.${config.networking.domain}" = {
+      useACMEHost = config.networking.domain;
+      forceSSL = true;
+      locations."/" = {
+        proxyPass = "http://${app}:${builtins.toString port}";
+        extraConfig = "resolver 10.88.0.1;";
+
+      };
+    };
+
+
     mySystem.services.gatus.monitors = [{
       name = app;
       group = "infrastructure";

nixos/modules/nixos/containers/plex/default.nix

@@ -11,8 +11,9 @@ let
   group = "568"; #string
   port = 32400; #int
   cfg = config.mySystem.services.${app};
-  appFolder = "containers/${app}";
+  appFolder = "/var/lib/${app}";
-  persistentFolder = "${config.mySystem.persistentFolder}/${appFolder}";
+
+  ## persistentFolder = "${config.mySystem.persistentFolder}/var/lib/${appFolder}";
 in
 {
   options.mySystem.services.${app} =
@@ -25,16 +26,16 @@ in
     };
 
   config = mkIf cfg.enable {
-    # ensure folder exist and has correct owner/group
+
-    systemd.tmpfiles.rules = [
+     environment.persistence."${config.mySystem.system.impermanence.persistPath}" = lib.mkIf config.mySystem.system.impermanence.enable {
-      "d ${persistentFolder} 0755 ${user} ${group} -" #The - disables automatic cleanup, so the file wont be removed after a period
+      directories = [{ directory = appFolder; user = user; group = group; mode = "750"; }];
-    ];
+    };
 
     virtualisation.oci-containers.containers.${app} = {
       image = "${image}";
       user = "${user}:${group}";
       volumes = [
-        "${persistentFolder}:/config:rw"
+        "${appFolder}:/config:rw"
         "${config.mySystem.nasFolder}/natflix:/data:rw"
         "${config.mySystem.nasFolder}/backup/kubernetes/apps/plex:/config/backup:rw"
         "/dev/dri:/dev/dri" # for hardware transcoding
@@ -44,12 +45,6 @@ in
         PLEX_ADVERTISE_URL = "https://10.8.20.42:32400,https://${app}.${config.mySystem.domain}:443"; # TODO var ip
       };
       ports = [ "${builtins.toString port}:${builtins.toString port}" ]; # expose port
-      labels = lib.myLib.mkTraefikLabels {
-        name = app;
-        inherit (config.networking) domain;
-
-        inherit port;
-      };
     };
     networking.firewall = mkIf cfg.openFirewall {
 
@@ -57,6 +52,17 @@ in
       allowedUDPPorts = [ port ];
     };
 
+    services.nginx.virtualHosts."${app}.${config.networking.domain}" = {
+      useACMEHost = config.networking.domain;
+      forceSSL = true;
+      locations."/" = {
+        proxyPass = "http://${app}:${builtins.toString port}";
+        extraConfig = "resolver 10.88.0.1;";
+
+      };
+    };
+
+
 
     mySystem.services.homepage.media = mkIf cfg.addToHomepage [
       {

nixos/modules/nixos/containers/qbittorrent/default.nix

@@ -12,8 +12,8 @@ let
   port = 8080; #int
   qbit_port = 32189;
   cfg = config.mySystem.services.${app};
-  appFolder = "containers/${app}";
+  appFolder = "/var/lib/${app}";
-  persistentFolder = "${config.mySystem.persistentFolder}/${appFolder}";
+  # persistentFolder = "${config.mySystem.persistentFolder}/var/lib/${appFolder}";
 in
 {
   options.mySystem.services.${app} =
@@ -29,7 +29,7 @@ in
   config = mkIf cfg.enable {
     # ensure folder exist and has correct owner/group
     systemd.tmpfiles.rules = [
-      "d ${persistentFolder} 0755 ${user} ${group} -" #The - disables automatic cleanup, so the file wont be removed after a period
+      "d ${appFolder} 0750 ${user} ${group} -" #The - disables automatic cleanup, so the file wont be removed after a period
     ];
 
     virtualisation.oci-containers.containers.${app} = {
@@ -40,17 +40,27 @@ in
       };
       ports = [ "${builtins.toString qbit_port}:${builtins.toString qbit_port}" ];
       volumes = [
-        "${persistentFolder}:/config:rw"
+        "${appFolder}:/config:rw"
         "${config.mySystem.nasFolder}/natflix:/media:rw"
         "/etc/localtime:/etc/localtime:ro"
       ];
-      labels = lib.myLib.mkTraefikLabels {
+    };
-        name = app;
+
-        inherit (config.networking) domain;
+     environment.persistence."${config.mySystem.system.impermanence.persistPath}" = lib.mkIf config.mySystem.system.impermanence.enable {
+      directories = [{ directory = appFolder; user = user; group = group; mode = "750"; }];
+    };
+
+    services.nginx.virtualHosts."${app}.${config.networking.domain}" = {
+      useACMEHost = config.networking.domain;
+      forceSSL = true;
+      locations."/" = {
+        proxyPass = "http://${app}:${builtins.toString port}";
+        extraConfig = "resolver 10.88.0.1;";
 
-        inherit port;
       };
     };
+
+
     # gotta open up that firewall
     networking.firewall = mkIf cfg.openFirewall {
 

nixos/modules/nixos/containers/redlib/default.nix

@@ -5,65 +5,144 @@
 }:
 with lib;
 let
-  cfg = config.mySystem.services.redlib;
+  cfg = config.mySystem.${category}.${app};
+  app = "redlib";
+  category = "services";
+  description = "reddit alternative frontend";
+  image = "quay.io/redlib/redlib@sha256:7fa92bb9b5a281123ee86a0b77a443939c2ccdabba1c12595dcd671a84cd5a64";
+  user = "nobody"; #string
+  group = "nobody"; #string
+  port = 8080; #int
+  appFolder = "/var/lib/${app}";
+  # persistentFolder = "${config.mySystem.persistentFolder}/var/lib/${appFolder}";
+  host = "${app}" + (if cfg.development then "-dev" else "");
+  url = "${host}.${config.networking.domain}";
 in
 {
-  options.mySystem.services.redlib.enable = mkEnableOption "redlib";
+  options.mySystem.${category}.${app} =
-
-  # fuck /u/spez
-  config =
-    mkIf cfg.enable
-      (myLib.mkService
     {
-          app = "Redlib";
+      enable = mkEnableOption "${app}";
-          description = "Reddit alternate frontend";
+      addToHomepage = mkEnableOption "Add ${app} to homepage" // { default = true; };
-          port = 8080;
+      monitor = mkOption
-          user = "nobody";
+        {
-          group = "nobody";
+          type = lib.types.bool;
-          inherit (config.time) timeZone;
+          description = "Enable gatus monitoring";
-          inherit (config.networking) domain;
+          default = true;
-          homepage = {
-            icon = "libreddit.svg";
-            category = "home";
         };
-          container = {
+      prometheus = mkOption
-            enable = true;
+        {
-            image = "quay.io/redlib/redlib@sha256:7fa92bb9b5a281123ee86a0b77a443939c2ccdabba1c12595dcd671a84cd5a64";
+          type = lib.types.bool;
+          description = "Enable prometheus scraping";
+          default = true;
+        };
+      addToDNS = mkOption
+        {
+          type = lib.types.bool;
+          description = "Add to DNS list";
+          default = true;
+        };
+      development = mkOption
+        {
+          type = lib.types.bool;
+          description = "Development instance";
+          default = false;
+        };
+      backups = mkOption
+        {
+          type = lib.types.bool;
+          description = "Enable local backups";
+          default = true;
+        };
+
+
+    };
+
+  config = mkIf cfg.enable {
+
+    ## Secrets
+    # sops.secrets."${category}/${app}/env" = {
+    #   sopsFile = ./secrets.sops.yaml;
+    #   owner = user;
+    #   group = group;
+    #   restartUnits = [ "${app}.service" ];
+    # };
+
+    users.users.truxnell.extraGroups = [ group ];
+
+
+    # Folder perms
+    # systemd.tmpfiles.rules = [
+    # "d ${appFolder}/ 0750 ${user} ${group} -"
+    # ];
+
+    ## service
+    # services.test= {
+    #   enable = true;
+    # };
+
+    ## container
+    virtualisation.oci-containers.containers = config.lib.mySystem.mkContainer {
+      inherit app image user group;
       env = {
-              REDLIB_DEFAULT_SHOW_NSFW = "on";
+        test = "derp";
-              REDLIB_DEFAULT_USE_HLS = "on";
-              REDLIB_DEFAULT_HIDE_HLS_NOTIFICATION = "on";
       };
-            addTraefikLabels = true;
+      envFiles = [ ];
-            caps = {
+      volumes = [ ];
-              readOnly = true;
+    };
-              noNewPrivileges = true;
+
-              dropAll = true;
+    # homepage integration
+    mySystem.services.homepage.infrastructure = mkIf cfg.addToHomepage [
+      {
+        ${app} = {
+          icon = "${app}.svg";
+          href = "https://${url}";
+          description = description;
+        };
+      }
+    ];
+
+    ### gatus integration
+    mySystem.services.gatus.monitors = mkIf cfg.monitor [
+      {
+        name = app;
+        group = "${category}";
+        url = "https://${url}/settings"; # settings page as pinging the main page is slow/creates requests
+        interval = "1m";
+        conditions = [ "[CONNECTED] == true" "[STATUS] == 200" "[RESPONSE_TIME] < 50" ];
+      }
+    ];
+
+    ### Ingress
+    services.nginx.virtualHosts.${url} = {
+      useACMEHost = config.networking.domain;
+      forceSSL = true;
+      locations."^~ /" = {
+        proxyPass = "http://${app}:${builtins.toString port}";
+        extraConfig = "resolver 10.88.0.1;";
       };
     };
-        });
-  # mkService
-  # app: App Name, string, required
-  # appUrl: App url, string, default "https://APP.DOMAIN"
-  # description: App Description, string, required
-  # image: Container IMage, string, required
-  # port: port, int
-  # timeZone: timezone, required
-  # domain: domain of app, required
-  # addToHomepage: Flag to add to homepage, bool, default false
-  ## HOMEPAGE
-  # homepage.icon: Icon for homepage listing, string, default "app.svg"
 
-  # user: user to run as, string, default 568
+    ### firewall config
-  # group: group to run as, string, default 568
-  # envFiles, files to add as env, list of string, default [ TZ = timeZone ]
 
-  ## CONTAINER
+    # networking.firewall = mkIf cfg.openFirewall {
-  # container.env, env vars for container, attrset, default { }
+    #   allowedTCPPorts = [ port ];
-  # container.addTraefikLabels, flag for adding traefik exposing labels, default true
+    #   allowedUDPPorts = [ port ];
-  # caps.privileged: privileged pod, grant pod high privs, defualt SUPER false.  SUPER DOOPER FALSE
+    # };
-  # caps.readOnly: readonly pod (outside mounted paths etc).  default false
+
-  #
+    ### backups
+    # warnings = [
+    #   (mkIf (!cfg.backups && config.mySystem.purpose != "Development")
+    #     "WARNING: Local backups for ${app} are disabled!")
+    # ];
+
+    # services.restic.backups = config.lib.mySystem.mkRestic
+    #   {
+    #     inherit app user;
+    #     paths = [ appFolder ];
+    #     inherit appFolder;
+
+    #   };
 
 
+  };
 }

nixos/modules/nixos/containers/redlib/default.nix.old

@@ -0,0 +1,68 @@
+{ lib
+, config
+, pkgs
+, ...
+}:
+with lib;
+let
+  cfg = config.mySystem.services.redlib;
+in
+{
+  options.mySystem.services.redlib.enable = mkEnableOption "redlib";
+
+  # fuck /u/spez
+  config =
+    mkIf cfg.enable
+      (myLib.mkService
+        {
+          app = "Redlib";
+          description = "Reddit alternate frontend";
+          port = 8080;
+          user = "nobody";
+          group = "nobody";
+          inherit (config.time) timeZone;
+          inherit (config.networking) domain;
+          homepage = {
+            icon = "libreddit.svg";
+            category = "home";
+          };
+          container = {
+            enable = true;
+            image = "quay.io/redlib/redlib@sha256:7fa92bb9b5a281123ee86a0b77a443939c2ccdabba1c12595dcd671a84cd5a64";
+            env = {
+              REDLIB_DEFAULT_SHOW_NSFW = "on";
+              REDLIB_DEFAULT_USE_HLS = "on";
+              REDLIB_DEFAULT_HIDE_HLS_NOTIFICATION = "on";
+            };
+            caps = {
+              readOnly = true;
+              noNewPrivileges = true;
+              dropAll = true;
+            };
+          };
+        });
+  # mkService
+  # app: App Name, string, required
+  # appUrl: App url, string, default "https://APP.DOMAIN"
+  # description: App Description, string, required
+  # image: Container IMage, string, required
+  # port: port, int
+  # timeZone: timezone, required
+  # domain: domain of app, required
+  # addToHomepage: Flag to add to homepage, bool, default false
+  ## HOMEPAGE
+  # homepage.icon: Icon for homepage listing, string, default "app.svg"
+
+  # user: user to run as, string, default 568
+  # group: group to run as, string, default 568
+  # envFiles, files to add as env, list of string, default [ TZ = timeZone ]
+
+  ## CONTAINER
+  # container.env, env vars for container, attrset, default { }
+  # container.addTraefikLabels, flag for adding traefik exposing labels, default true
+  # caps.privileged: privileged pod, grant pod high privs, defualt SUPER false.  SUPER DOOPER FALSE
+  # caps.readOnly: readonly pod (outside mounted paths etc).  default false
+  #
+
+
+}

nixos/modules/nixos/containers/sabnzbd/default.nix

@@ -11,8 +11,8 @@ let
   group = "568"; #string
   port = 8080; #int
   cfg = config.mySystem.services.${app};
-  appFolder = "containers/${app}";
+  appFolder = "/var/lib/${app}";
-  persistentFolder = "${config.mySystem.persistentFolder}/${appFolder}";
+  # persistentFolder = "${config.mySystem.persistentFolder}/var/lib/${appFolder}";
 in
 {
   options.mySystem.services.${app} =
@@ -24,7 +24,7 @@ in
   config = mkIf cfg.enable {
     # ensure folder exist and has correct owner/group
     systemd.tmpfiles.rules = [
-      "d ${persistentFolder} 0755 ${user} ${group} -" #The - disables automatic cleanup, so the file wont be removed after a period
+      "d ${appFolder} 0750 ${user} ${group} -" #The - disables automatic cleanup, so the file wont be removed after a period
     ];
 
     virtualisation.oci-containers.containers.${app} = {
@@ -34,18 +34,23 @@ in
         SABNZBD__HOST_WHITELIST_ENTRIES = "sabnzbd, sabnzbd.trux.dev";
       };
       volumes = [
-        "${persistentFolder}:/config:rw"
+        "${appFolder}:/config:rw"
         "${config.mySystem.nasFolder}/natflix:/media:rw"
         "/etc/localtime:/etc/localtime:ro"
       ];
-      labels = lib.myLib.mkTraefikLabels {
+    };
-        name = app;
+
-        inherit (config.networking) domain;
+    services.nginx.virtualHosts."${app}.${config.networking.domain}" = {
+      useACMEHost = config.networking.domain;
+      forceSSL = true;
+      locations."/" = {
+        proxyPass = "http://${app}:${builtins.toString port}";
+        extraConfig = "resolver 10.88.0.1;";
 
-        inherit port;
       };
     };
 
+
     mySystem.services.homepage.media = mkIf cfg.addToHomepage [
       {
         Sabnzbd = {
@@ -62,6 +67,10 @@ in
       }
     ];
 
+     environment.persistence."${config.mySystem.system.impermanence.persistPath}" = lib.mkIf config.mySystem.system.impermanence.enable {
+      directories = [{ directory = appFolder; user = user; group = group; mode = "750"; }];
+    };
+
     mySystem.services.gatus.monitors = [{
 
       name = app;

nixos/modules/nixos/containers/searxng/default.nix

@@ -11,8 +11,8 @@ let
   group = "977"; #string
   port = 8080; #int
   cfg = config.mySystem.services.${app};
-  appFolder = "containers/${app}";
+  appFolder = "/var/lib/${app}";
-  persistentFolder = "${config.mySystem.persistentFolder}/${appFolder}";
+  # persistentFolder = "${config.mySystem.persistentFolder}/var/lib/${appFolder}";
   configNix = { use_default_settings = { engines = { keep_only = [ "arch linux wiki" "google" "google images" "google news" "google videos" "google scholar" "google play apps" "duckduckgo" "brave" "startpage" "gitlab" "github" "codeberg" "sourcehut" "bitbucket" "apple app store" "wikipedia" "currency" "docker hub" "ddg definitions" "duckduckgo images" "bandcamp" "deviantart" "tineye" "apple maps" "fdroid" "flickr" "free software directory" "z-library" "lobste.rs" "azlyrics" "openstreetmap" "npm" "pypi" "lib.rs" "nyaa" "reddit" "sepiasearch" "soundcloud" "stackoverflow" "askubuntu" "superuser" "searchcode code" "unsplash" "youtube" "wolframalpha" "mojeek" ]; }; }; engines = [{ name = "brave"; disabled = false; } { name = "startpage"; disabled = false; } { name = "apple app store"; disabled = false; } { name = "ddg definitions"; disabled = false; } { name = "tineye"; disabled = false; } { name = "apple maps"; disabled = false; } { name = "duckduckgo images"; disabled = false; } { name = "fdroid"; disabled = false; } { name = "free software directory"; disabled = false; } { name = "bitbucket"; disabled = false; } { name = "gitlab"; disabled = false; } { name = "codeberg"; disabled = false; } { name = "google play apps"; disabled = false; } { name = "lobste.rs"; disabled = false; } { name = "azlyrics"; disabled = false; } { name = "npm"; disabled = false; } { name = "nyaa"; disabled = false; categories = "videos"; } { name = "searchcode code"; disabled = false; } { name = "mojeek"; disabled = false; } { name = "lib.rs"; disabled = false; } { name = "sourcehut"; disabled = false; }]; general = { instance_name = "NatFlix Search"; enable_metrics = false; }; brand = { new_issue_url = ""; docs_url = ""; public_instances = ""; wiki_url = ""; issue_url = ""; }; search = { safe_search = 0; autocomplete = "duckduckgo"; autocomplete_min = 2; default_lang = "en"; max_page = 0; }; server = { base_url = "https://searxng.\${EXTERNAL_DOMAIN}/"; image_proxy = true; http_protocol_version = "1.1"; method = "GET"; }; ui = { static_use_hash = true; infinite_scroll = true; default_theme = "simple"; theme_args = { simple_style = "dark"; }; }; enabled_plugins = [ "Hash plugin" "Search on category select" "Self Information" "Tracker URL remover" "Open Access DOI rewrite" "Vim-like hotkeys" ]; };
   configFile = builtins.toFile "config.yaml" (builtins.toJSON configNix);
 in
@@ -37,12 +37,6 @@ in
         SEARXNG_BASE_URL = "https://searxng.${config.mySystem.domain}/";
         SEARXNG_URL = "https://searxng.${config.mySystem.domain}";
       };
-      labels = lib.myLib.mkTraefikLabels {
-        name = app;
-        inherit (config.networking) domain;
-
-        inherit port;
-      };
       extraOptions = [
         "--read-only"
         "--tmpfs=/etc/searxng/"
@@ -54,6 +48,16 @@ in
 
     };
 
+    services.nginx.virtualHosts."${app}.${config.networking.domain}" = {
+      useACMEHost = config.networking.domain;
+      forceSSL = true;
+      locations."/" = {
+        proxyPass = "http://${app}:${builtins.toString port}";
+        extraConfig = "resolver 10.88.0.1;";
+
+      };
+    };
+
     mySystem.services.homepage.home = mkIf cfg.addToHomepage [
       {
         Searxng = {

nixos/modules/nixos/containers/tautulli/default.nix

@@ -11,8 +11,8 @@ let
   group = "568"; #string
   port = 8181; #int
   cfg = config.mySystem.services.${app};
-  appFolder = "containers/${app}";
+  appFolder = "/var/lib/${app}";
-  persistentFolder = "${config.mySystem.persistentFolder}/${appFolder}";
+  # persistentFolder = "${config.mySystem.persistentFolder}/var/lib/${appFolder}";
 in
 {
   options.mySystem.services.${app} =
@@ -24,24 +24,32 @@ in
   config = mkIf cfg.enable {
     # ensure folder exist and has correct owner/group
     systemd.tmpfiles.rules = [
-      "d ${persistentFolder} 0755 ${user} ${group} -" #The - disables automatic cleanup, so the file wont be removed after a period
+      "d ${appFolder} 0750 ${user} ${group} -" #The - disables automatic cleanup, so the file wont be removed after a period
     ];
 
     virtualisation.oci-containers.containers.${app} = {
       image = "${image}";
       user = "${user}:${group}";
       volumes = [
-        "${persistentFolder}:/config:rw"
+        "${appFolder}:/config:rw"
         "${config.mySystem.nasFolder}/natflix:/media:rw"
         "${config.mySystem.nasFolder}/backup/kubernetes/apps/tautulli:/config/backup:rw"
         "/etc/localtime:/etc/localtime:ro"
       ];
-      labels = lib.myLib.mkTraefikLabels {
-        name = app;
-        inherit (config.networking) domain;
-
-        inherit port;
     };
+
+    services.nginx.virtualHosts."${app}.${config.networking.domain}" = {
+      useACMEHost = config.networking.domain;
+      forceSSL = true;
+      locations."/" = {
+        proxyPass = "http://${app}:${builtins.toString port}";
+        extraConfig = "resolver 10.88.0.1;";
+
+      };
+    };
+
+     environment.persistence."${config.mySystem.system.impermanence.persistPath}" = lib.mkIf config.mySystem.system.impermanence.enable {
+      directories = [{ directory = appFolder; user = user; group = group; mode = "750"; }];
     };
 
     mySystem.services.homepage.media = mkIf cfg.addToHomepage [

nixos/modules/nixos/containers/whoogle/default.nix

@@ -11,8 +11,8 @@ let
   group = "927"; #string
   port = 5000; #int
   cfg = config.mySystem.services.${app};
-  appFolder = "containers/${app}";
+  appFolder = "/var/lib/${app}";
-  persistentFolder = "${config.mySystem.persistentFolder}/${appFolder}";
+ # persistentFolder = "${config.mySystem.persistentFolder}/var/lib/${appFolder}";
 in
 {
   options.mySystem.services.${app} =
@@ -47,15 +47,19 @@ in
         WHOOGLE_CONFIG_VIEW_IMAGE = "1";
         WHOOGLE_CONFIG_DISABLE = "1";
       };
+    };
 
-      labels = lib.myLib.mkTraefikLabels {
+    services.nginx.virtualHosts."${app}.${config.networking.domain}" = {
-        name = app;
+      useACMEHost = config.networking.domain;
-        inherit (config.networking) domain;
+      forceSSL = true;
+      locations."/" = {
+        proxyPass = "http://${app}:${builtins.toString port}";
+        extraConfig = "resolver 10.88.0.1;";
 
-        inherit port;
       };
     };
 
+
     mySystem.services.homepage.home = mkIf cfg.addToHomepage [
       {
         Whoogle = {

nixos/modules/nixos/default.nix

@@ -10,6 +10,7 @@ with lib;
     ./hardware
     ./containers
     ./lib.nix
+    ./security
   ];
 
   options.mySystem.persistentFolder = mkOption {
@@ -38,6 +39,11 @@ with lib;
     description = "System purpose";
     default = "Production";
   };
+  options.mySystem.monitoring.prometheus.scrapeConfigs = mkOption {
+    type = lib.types.listOf lib.types.attrs;
+    description = "Prometheus scrape targets";
+    default = [ ];
+  };
 
 
   config = {

nixos/modules/nixos/lib.nix

@@ -2,6 +2,36 @@
 with lib;
 {
 
+  # container builder
+  lib.mySystem.mkContainer = options: (
+    let
+      # nix doesnt have an exhausive list of options for oci
+      # so here i try to get a robust list of security options for containers
+      # because everyone needs more tinfoild hat right?  RIGHT?
+
+      containerExtraOptions = lib.optionals (lib.attrsets.attrByPath [ "caps" "privileged" ] false options) [ "--privileged" ]
+        ++ lib.optionals (lib.attrsets.attrByPath [ "caps" "readOnly" ] false options) [ "--read-only" ]
+        ++ lib.optionals (lib.attrsets.attrByPath [ "caps" "tmpfs" ] false options) [ (map (folders: "--tmpfs=${folders}") tmpfsFolders) ]
+        ++ lib.optionals (lib.attrsets.attrByPath [ "caps" "noNewPrivileges" ] false options) [ "--security-opt=no-new-privileges" ]
+        ++ lib.optionals (lib.attrsets.attrByPath [ "caps" "dropAll" ] false options) [ "--cap-drop=ALL" ];
+
+    in
+    {
+      ${options.app} = {
+        image = "${options.image}";
+        user = "${options.user}:${options.group}";
+        environment = {
+          TZ = config.time.timeZone;
+        } // options.env;
+        environmentFiles = lib.attrsets.attrByPath [ "envFiles" ] [ ] options;
+        volumes = [ "/etc/localtime:/etc/localtime:ro" ]
+          ++ lib.attrsets.attrByPath [ "volumes" ] [ ] options;
+
+        extraOptions = containerExtraOptions;
+      };
+    }
+  );
+
 
   # build a restic restore set for both local and remote
   lib.mySystem.mkRestic = options: (
@@ -23,13 +53,14 @@ with lib;
         #
         ${pkgs.restic}/bin/restic unlock --remove-all || true
       '';
+
     in
     {
       # local backup
-      "${options.app}-local" = mkIf config.mySystem.system.resticBackup.local.enable {
+      "${options.app}-local" = {
         inherit pruneOpts timerConfig initialize backupPrepareCommand;
         # Move the path to the zfs snapshot path
-        paths = map (x: "${config.mySystem.persistentFolder}/.zfs/snapshot/restic_nightly_snap/${x}") options.paths;
+        paths = map (x: "${config.mySystem.system.resticBackup.mountPath}/${x}") options.paths;
         passwordFile = config.sops.secrets."services/restic/password".path;
         exclude = excludePath;
         repository = "${config.mySystem.system.resticBackup.local.location}/${options.appFolder}";
@@ -37,10 +68,10 @@ with lib;
       };
 
       # remote backup
-      "${options.app}-remote" = mkIf config.mySystem.system.resticBackup.remote.enable {
+      "${options.app}-remote" = {
         inherit pruneOpts timerConfig initialize backupPrepareCommand;
         # Move the path to the zfs snapshot path
-        paths = map (x: "${config.mySystem.persistentFolder}/.zfs/snapshot/restic_nightly_snap/${x}") options.paths;
+        paths = map (x: "${config.mySystem.system.resticBackup.mountPath}/${x}") options.paths;
         environmentFile = config.sops.secrets."services/restic/env".path;
         passwordFile = config.sops.secrets."services/restic/password".path;
         repository = "${config.mySystem.system.resticBackup.remote.location}/${options.appFolder}";
@@ -51,7 +82,4 @@ with lib;
     }
   );
 
-
-
-
 }

nixos/modules/nixos/security/acme/default.nix

@@ -0,0 +1,47 @@
+{ lib
+, config
+, pkgs
+, ...
+}:
+with lib;
+let
+  cfg = config.mySystem.security.acme;
+  app = "acme";
+  appFolder = "/var/lib/${app}";
+ # persistentFolder = "${config.mySystem.persistentFolder}/var/lib/${appFolder}";
+  user = app;
+  group = app;
+
+in
+{
+  options.mySystem.security.acme.enable = mkEnableOption "acme";
+
+  config = mkIf cfg.enable {
+    sops.secrets = {
+      "security/acme/env".sopsFile = ./secrets.sops.yaml;
+      "security/acme/env".restartUnits = [ "${app}.service" ];
+    };
+
+     environment.persistence."${config.mySystem.system.impermanence.persistPath}" = lib.mkIf config.mySystem.system.impermanence.enable {
+      directories = [ "/var/lib/acme" ];
+    };
+
+
+    security.acme = {
+      acceptTerms = true;
+      defaults.email = "admin@${config.networking.domain}";
+
+      certs.${config.networking.domain} = {
+        extraDomainNames = [
+          "${config.networking.domain}"
+          "*.${config.networking.domain}"
+        ];
+        dnsProvider = "cloudflare";
+        dnsResolver = "1.1.1.1:53";
+        credentialsFile = config.sops.secrets."security/acme/env".path;
+      };
+    };
+
+
+  };
+}

nixos/modules/nixos/security/acme/secrets.sops.yaml

@@ -0,0 +1,77 @@
+security:
+    acme:
+        env: ENC[AES256_GCM,data:cVFlyUdU+4GrDwgEMIctg7LhrwPrF90E5Q4BsidQH+A//5fXiPbKYYnXFzQ1yFqkwOLZr/zvM3Uoqm+kFvRYhNyI4f+GKyW1lGRvFiUYuhNVPJB45A==,iv:Ng3G22Z93FsXgYk7/ufoJGy4Va84LNmhqL10/+hgvPQ=,tag:qGkVIJSbJ9MGXGtUr9KJTw==,type:str]
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age1lj5vmr02qkudvv2xedfj5tq8x93gllgpr6tzylwdlt7lud4tfv5qfqsd5u
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBvYTlvVG5uL3FWUGtZcUhT
+            bW9QdWVqSlV6OVplcThzZlBqbnVsZ3JVL3k4CllqR216ejJCb3JZTmNKK05hbStx
+            QWlUTmdTazV0TGt1RjlaYlB0VnVIdmMKLS0tIEplaTdmTXo1eUZFMzlUYXllcHFw
+            dlU0aU1PZHgwODVSTDNWbW5EWUIyNW8KX1fGSjOehuHw0za6TmO2CC5GSYV015Ld
+            OM4FkUMSRT8Fs4p0aD1Kvf7ZbGHgN3FDelfA4sSfm03zjAl2iE4BHw==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age17edew3aahg3t5nte5g0a505sn96vnj8g8gqse8q06ccrrn2n3uysyshu2c
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBTUDVkbGtVVnBYcnY5bVJ5
+            WjEycE5oYzBRR3NPWGpuWnJER2NvL0lJTUVrCmZhTTliMzJwUmtkcllmay9rbzF6
+            MUg0T0FIUS9nK3lMMERGWGJQR1FwcU0KLS0tIE9vVnBSMnNGVDVtNTJ3WlBZK0Jm
+            WllkTjhvQ1lWenhOeU1GNUc2U2Y2aWMKLhHcgzw41VvOo2rFi8Siv0xX/x1DsKPT
+            fKMUBcZuV94jLTVIecDNftpd0KsN3J1uFFxZD7CUQ9pIGQccf6xYSA==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age1u4tht685sqg6dkmjyer96r93pl425u6353md6fphpd84jh3jwcusvm7mgk
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBFVGdZSitzVDNMa24wT0NF
+            MzhnMTgyZHM2V0VGSHY1VUs3QjlwSlNqV0RNCmVraFpKRmd6bWpCSEZnMll0TnJs
+            Z2ZDdlZSenBsRFpwUlpDK1hOSFoweEEKLS0tIGlsUW82WGR0OHVFOU85U043ODk3
+            MWt3K1Vhb1NpQzhwSVIxL0pITngrbWcKnDR/aq+x3Q/xfomA0czI9+YsWvZaZrIY
+            sQeaXCIPvyHncX9V9/yvKOoyhyN/o2TfMWA3W4swevd9HlNI7a+Ltw==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age1cp6vegrmqfkuj8nmt2u3z0sur7n0f7e9x9zmdv4zygp8j2pnucpsdkgagc
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBGUkVicUx3UkVRV1ZsbXFh
+            b0JtZnpVRTFuclRaZ29vbFRDeGNWbk11eFQwCitaa1luQ1Ryb21WR21kOFhGWVl4
+            b3ZtcGJsSk9IN3RmM0wyY0JXdDJkZU0KLS0tIFVUQXZ0YmlXbHdhK2o2SVZsNGw0
+            N3J3L3ZPUndMWjRKZ1l5Y3F4cCt3UTAK00hUdLNvLyVnPulFc6PmkKJTiwFl0Svg
+            mxqIGwVBKDosXnppVQezamNr3/46V8SglclbhVlYaPvSbVRLFfwmVw==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age1ekt5xz7u2xgdzgsrffhd9x22n80cn4thxd8zxjy2ey5vq3ca7gnqz25g5r
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBpSnJaQVc0SHdKTkdCSUll
+            VDdXQnlXaDBTNkIvZFR6bFJvOW1YOUgwWWkwCmxoWUxUWm00eHF6d3VtNlU5bllm
+            VnF6cHJNK3BIV2orVVl6RDhRcXNZWlEKLS0tIGNWb05qOUN5bU1RYjdJTUVENFNW
+            K002dzZCZVpuWEF1N0k4WUROYzAycGcKDH8IGFufvhaL+WMq+S/tn5ow5T5he+LY
+            VMxXuKkR+cAnCDrumlyaaIX//hjjT9PVS/xCxiYKblnCAHNh8isDUA==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age1jpeh4s553taxkyxhzlshzqjfrtvmmp5lw0hmpgn3mdnmgzku332qe082dl
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBCNHR6amdjMzBqbWVOUm1Z
+            KzhmTDVSbElsTHFtZXNmVTFuSFl0Si9Dd0hJCkp0WmNlUDFEcVh2VERsL0VSUWZs
+            NXZkUkxCa2NvUUJBM0pQS0FJcTA4L2sKLS0tIDhFYm1mWXJCQmU0NWo5a0Y4SW50
+            SFd1Z2VWTG9LK2dyTTB0cnRnOWFKMEkKHD1fi/dm6A9+/KbGcKBeXA4SotaiFYZx
+            1yK6n/YJB/bd77KGV4cIde+4tgZea10Sq5flg24LE91UnaYmEwC5/g==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age1j2r8mypw44uvqhfs53424h6fu2rkr5m7asl7rl3zn3xzva9m3dcqpa97gw
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBCK29MK2FzQ1FiSG4wano3
+            a1lKSUx5RnFvbDN1cEJ0UGl3NTRrRjBrWXlRCmgxWkpYRXlOVTc0YVNYL1hJdGQx
+            VHF6OURlU2tmSGRmaFp1UTRoMVdCeVUKLS0tIENkemhwenBnMTd6cEF3Q3k0OGJU
+            R3hCaS81bWJhVmZ0S1hLLzhDVUVxdXcKhmdOGA8tNlije8D2GfJahLpoSXbVidgA
+            TxkL6bn4ZFtB8Cc3H1jWzmxpFTPeWQGk+MBHB86V+p8pmBur5m1btA==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2024-04-29T12:25:10Z"
+    mac: ENC[AES256_GCM,data:qk0/+Lv14wIGMGx4wmtaEVpNl00htxH5R3D37bgvLI/+X5DCnKay/Wu6GsM2i4Vp2BbnfJHX7dj6+r0VAEc8gRLz0KYpNGVyOZHWzgdLWhDeHNs6+GA7MZTxED4XrTPcNfXwzoK75RNAeSHy0o0nkjl0CYFrF+yvc67rW0VCKMc=,iv:Z2GlCkyKMI4waAMATMiZvdIz08N5OtMIyvCyWwjgRFI=,tag:8VEXwUdOQgcnaQrrODI4TA==,type:str]
+    pgp: []
+    unencrypted_suffix: _unencrypted
+    version: 3.8.1

nixos/modules/nixos/security/default.nix

@@ -0,0 +1,9 @@
+{ lib, config, ... }:
+with lib;
+{
+  imports = [
+    ./acme
+  ];
+
+
+}

nixos/modules/nixos/services/cockpit/default.nix

@@ -30,7 +30,6 @@ in
     systemPackages = with pkgs;
       [
         # (mkIf config.virtualisation.podman.enable nur.repos.procyon.cockpit-podman) # TODO replace only if server runs pods
-
         # nur.repos.dukzcry.cockpit-machines # TODO enable with virtualisation on server
         # nur.repos.dukzcry.libvirt-dbus # TODO enable with virtualisation on server
         # pkgs.virt-manager # TODO enable with virtualisation on server

nixos/modules/nixos/services/default.nix

@@ -20,5 +20,10 @@
     ./postgresql
     ./blocky
     ./openvscode-server
+    ./grafana
+    ./prometheus
+    ./radicale
+    ./node-red
+    ./nginx
   ];
 }

nixos/modules/nixos/services/grafana/default.nix

@@ -0,0 +1,133 @@
+{ lib
+, config
+, pkgs
+, ...
+}:
+with lib;
+let
+  cfg = config.mySystem.services.grafana;
+  app = "grafana";
+  category = "services";
+  description = "Metric visualisation";
+  user = app; #string
+  group = app; #string
+  port = 2342; #int
+  appFolder = "/var/lib/${app}";
+ # persistentFolder = "${config.mySystem.persistentFolder}/var/lib/${appFolder}";
+  host = "${app}" + (if cfg.development then "-dev" else "");
+  url = "${host}.${config.networking.domain}";
+in
+{
+  options.mySystem.${category}.${app} =
+    {
+      enable = mkEnableOption "${app}";
+      addToHomepage = mkEnableOption "Add ${app} to homepage" // { default = true; };
+      monitor = mkOption
+        {
+          type = lib.types.bool;
+          description = "Enable gatus monitoring";
+          default = true;
+        };
+      #   prometheus = mkOption
+      #     {
+      #       type = lib.types.bool;
+      #       description = "Enable prometheus scraping";
+      #       default = true;
+      #     };
+      addToDNS = mkOption
+        {
+          type = lib.types.bool;
+          description = "Add to DNS list";
+          default = true;
+        };
+      development = mkOption
+        {
+          type = lib.types.bool;
+          description = "Development instance";
+          default = false;
+        };
+      backups = mkOption
+        {
+          type = lib.types.bool;
+          description = "Enable local backups";
+          default = true;
+        };
+
+
+    };
+
+  config = mkIf cfg.enable {
+
+    ## Secrets
+    # sops.secrets."${category}/${app}/env" = {
+    #   sopsFile = ./secrets.sops.yaml;
+    #   owner = user;
+    #   group = group;
+    #   restartUnits = [ "${app}.service" ];
+    # };
+
+    users.users.truxnell.extraGroups = [ group ];
+
+    ## service
+    services.grafana = {
+      inherit port;
+      enable = true;
+      domain = host;
+      addr = "127.0.0.1";
+    };
+
+    # homepage integration
+    mySystem.services.homepage.infrastructure = mkIf cfg.addToHomepage [
+      {
+        ${app} = {
+          icon = "${app}.svg";
+          href = "https://${url}";
+          description = description;
+        };
+      }
+    ];
+
+    ### gatus integration
+    mySystem.services.gatus.monitors = mkIf cfg.monitor [
+      {
+        name = app;
+        group = "${category}";
+        url = "https://${url}";
+        interval = "1m";
+        conditions = [ "[CONNECTED] == true" "[STATUS] == 200" "[RESPONSE_TIME] < 50" ];
+      }
+    ];
+
+    ### Ingress
+    services.nginx.virtualHosts.${url} = {
+      useACMEHost = config.networking.domain;
+      forceSSL = true;
+      locations."^~ /" = {
+        proxyPass = "http://127.0.0.1:${builtins.toString port}";
+      };
+    };
+
+    ### firewall config
+
+    # networking.firewall = mkIf cfg.openFirewall {
+    #   allowedTCPPorts = [ port ];
+    #   allowedUDPPorts = [ port ];
+    # };
+
+    ### backups
+    warnings = [
+      (mkIf (!cfg.backups && config.mySystem.purpose != "Development")
+        "WARNING: Local backups for ${app} are disabled!")
+    ];
+
+    services.restic.backups = config.lib.mySystem.mkRestic
+      {
+        inherit app user;
+        paths = [ appFolder ];
+        inherit appFolder;
+
+      };
+
+
+  };
+}

nixos/modules/nixos/services/languagetool/default.nix

@@ -0,0 +1,146 @@
+{ lib
+, config
+, pkgs
+, ...
+}:
+with lib;
+let
+  cfg = config.mySystem.${category}.${app};
+  app = "%{app}";
+  category = "%{cat}";
+  description = "%{description}";
+  image = "%{image}";
+  user = "%{user kah}"; #string
+  group = "%{group kah}"; #string
+  port = %{ port }; #int
+  appFolder = "/var/lib/${app}";
+ # persistentFolder = "${config.mySystem.persistentFolder}/var/lib/${appFolder}";
+  host="${app}" ++ mkIf cfg.development "-dev";
+  url = "${host}.${config.networking.domain}";
+  in
+  {
+  options.mySystem.${category}.${app} =
+  {
+  enable = mkEnableOption "${app}";
+  addToHomepage = mkEnableOption "Add ${app} to homepage" // { default = true; };
+  monitor = mkOption
+  {
+  type = lib.types.bool;
+  description = "Enable gatus monitoring";
+  default = true;
+  };
+  prometheus = mkOption
+  {
+  type = lib.types.bool;
+  description = "Enable prometheus scraping";
+  default = true;
+  };
+  addToDNS = mkOption
+  {
+  type = lib.types.bool;
+  description = "Add to DNS list";
+  default = true;
+  };
+  development = mkOption
+  {
+  type = lib.types.bool;
+  description = "Development instance";
+  default = false;
+  };
+  backupLocal = mkOption
+  {
+  type = lib.types.bool;
+  description = "Enable local backups";
+  default = true;
+  };
+  backupRemote = mkOption
+  {
+  type = lib.types.bool;
+  description = "Enable remote backups";
+  default = true;
+  };
+
+
+  };
+
+  config = mkIf cfg.enable {
+
+  ## Secrets
+  # sops.secrets."${category}/${app}/env" = {
+  #   sopsFile = ./secrets.sops.yaml;
+  #   owner = user;
+  #   group = group;
+  #   restartUnits = [ "${app}.service" ];
+  # };
+
+  users.users.truxnell.extraGroups = [ group ];
+
+
+  # Folder perms - only for containers
+  # systemd.tmpfiles.rules = [
+  # "d ${persistentFolder}/ 0750 ${user} ${group} -"
+  # ];
+
+  ## service
+  # services.test= {
+  #   enable = true;
+  # };
+
+  # homepage integration
+  mySystem.services.homepage.infrastructure = mkIf cfg.addToHomepage [
+  {
+  ${app} = {
+  icon = "${app}.svg";
+  href = "https://${url}";
+  description = description;
+  };
+  }
+  ];
+
+  ### gatus integration
+  mySystem.services.gatus.monitors = mkIf cfg.monitor [
+  {
+  name = app;
+  group = "${category}";
+  url = "https://${url}";
+  interval = "1m";
+  conditions = [ "[CONNECTED] == true" "[STATUS] == 200" "[RESPONSE_TIME] < 50" ];
+  }
+  ];
+
+  ### Ingress
+  services.nginx.virtualHosts.${url} = {
+  useACMEHost = host;
+  forceSSL = true;
+  locations."^~ /" = {
+  proxyPass = "http://127.0.0.1:${builtins.toString port}";
+  };
+  };
+
+  ### firewall config
+
+  # networking.firewall = mkIf cfg.openFirewall {
+  #   allowedTCPPorts = [ port ];
+  #   allowedUDPPorts = [ port ];
+  # };
+
+  ### backups
+  warnings = [
+  (mkIf (!cfg.backupLocal && config.mySystem.purpose != "Development")
+  "WARNING: Local backups for ${app} are disabled!")
+  (mkIf (!cfg.backupRemote && config.mySystem.purpose != "Development")
+  "WARNING: Remote backups for ${app} are disabled!")
+  ];
+
+  services.restic.backups = mkIf cfg.backups config.lib.mySystem.mkRestic
+  {
+  inherit app user;
+  paths = [ appFolder ];
+  inherit appFolder;
+  local=cfg.backupLocal;
+  remote=cfg.backupRemote;
+  };
+
+
+  };
+  }

nixos/modules/nixos/services/monitoring.nix

@@ -40,7 +40,15 @@ in
       config.services.prometheus.exporters.smartctl.port
     ];
 
+    mySystem.monitoring.prometheus.scrapeConfigs = [
+      {
+        job_name = "node-exporter-${config.networking.hostName}";
+        static_configs = [{
+          targets = [ "127.0.0.1:${toString config.services.prometheus.exporters.node.port}" ];
+        }];
+      }
+    ];
+
   };
 
-
 }

nixos/modules/nixos/services/mosquitto/default.nix

@@ -6,10 +6,11 @@
 with lib;
 let
   cfg = config.mySystem.services.mosquitto;
-  persistentFolder = "${config.mySystem.persistentFolder}/nixos/services/mosquitto/";
+  # persistentFolder = "${config.mySystem.persistentFolder}/nixos/services/mosquitto/";
   app = "mosquitto";
   user = app;
   group = app;
+  appFolder = config.services.mosquitto.dataDir;
 in
 {
   options.mySystem.services.mosquitto.enable = mkEnableOption "mosquitto MQTT";
@@ -18,25 +19,19 @@ in
 
     sops.secrets."services/mosquitto/mq/hashedPassword" = {
       sopsFile = ./secrets.sops.yaml;
-      owner = "mosquitto";
+      owner = app;
-      group = "mosquitto";
+      group = app;
       restartUnits = [ "${app}.service" ];
     };
 
-    # ensure folder exist and has correct owner/group
-    systemd.tmpfiles.rules = [
-      "d ${persistentFolder} 0750 ${user} ${group} -" #The - disables automatic cleanup, so the file wont be removed after a period
-    ];
-
 
     services.mosquitto = {
       enable = true;
       # persistance for convienience on restarts
       # but not backed up, there is no data
       # that requires keeping in MQTT
-      dataDir = persistentFolder;
       settings = {
-        persistence_location = "${persistentFolder}";
+        persistence_location = appFolder;
         max_keepalive = 300;
       };
 
@@ -51,6 +46,11 @@ in
         }
       ];
     };
+
+     environment.persistence."${config.mySystem.system.impermanence.persistPath}" = lib.mkIf config.mySystem.system.impermanence.enable {
+      directories = [{ directory = appFolder; user = user; group = group; mode = "750"; }];
+    };
+
     users.users.truxnell.extraGroups = [ "mosquitto" ];
     networking.firewall.allowedTCPPorts = [ 1883 ];
 

nixos/modules/nixos/services/nginx/default.nix

@@ -0,0 +1,72 @@
+{ lib
+, config
+, pkgs
+, ...
+}:
+with lib;
+let
+  cfg = config.mySystem.services.nginx;
+in
+{
+  options.mySystem.services.nginx.enable = mkEnableOption "nginx";
+
+  config = mkIf cfg.enable {
+
+    services.nginx = {
+      enable = true;
+
+      recommendedGzipSettings = true;
+      recommendedOptimisation = true;
+      recommendedProxySettings = true;
+      recommendedTlsSettings = true;
+      recommendedBrotliSettings = true;
+
+      proxyResolveWhileRunning = true; # needed to ensure nginx loads even if it cant resolve vhosts
+
+      statusPage = true;
+      enableReload = true;
+
+      # Only allow PFS-enabled ciphers with AES256
+      sslCiphers = "AES256+EECDH:AES256+EDH:!aNULL";
+
+      appendHttpConfig = ''
+        # Minimize information leaked to other domains
+        add_header 'Referrer-Policy' 'origin-when-cross-origin';
+
+        # Disable embedding as a frame
+        add_header X-Frame-Options DENY;
+
+        # Prevent injection of code in other mime types (XSS Attacks)
+        add_header X-Content-Type-Options nosniff;
+
+
+      '';
+      # TODO add cloudflre IP's when/if I ingest internally.
+      commonHttpConfig = ''
+        add_header X-Clacks-Overhead "GNU Terry Pratchett";
+      '';
+      # provide default host with returning error
+      # else nginx returns the first server
+      # in the config file... >:S
+      virtualHosts = {
+        "_" = {
+          default = true;
+          rejectSSL = true;
+          extraConfig = "return 444;";
+        };
+      };
+
+    };
+
+
+    networking.firewall = {
+
+      allowedTCPPorts = [ 80 443 ];
+      allowedUDPPorts = [ 80 443 ];
+    };
+
+    # required for using acme certs
+    users.users.nginx.extraGroups = [ "acme" ];
+
+  };
+}

nixos/modules/nixos/services/node-red/default.nix

@@ -7,11 +7,11 @@ with lib;
 let
   cfg = config.mySystem.services.node-red;
   app = "node-red";
-  persistentFolder = "${config.mySystem.persistentFolder}/${appFolder}";
+  # persistentFolder = "${config.mySystem.persistentFolder}/var/lib/${appFolder}";
-  appFolder = "apps/${app}";
+  appFolder = config.services.node-red.userDir;
   inherit (config.services.node-red) user;
   inherit (config.services.node-red) group;
-  url = "code-${config.networking.hostName}.${config.networking.domain}";
+  url = "${app}.${config.networking.domain}";
 
 in
 {
@@ -23,46 +23,30 @@ in
 
   config = mkIf cfg.enable {
 
-    # ensure folder exist and has correct owner/group
-    systemd.tmpfiles.rules = [
-      "d ${persistentFolder} 0755 ${user} ${group} -" #The - disables automatic cleanup, so the file wont be removed after a period
-    ];
-
     services.node-red = {
       enable = true;
-      userDir = persistentFolder;
     };
 
-    mySystem.services.traefik.routers = [{
+    services.nginx.virtualHosts."${app}.${config.networking.domain}" = {
-      http.routers.${app} = {
+      useACMEHost = config.networking.domain;
-        rule = "Host(`${app}.${config.mySystem.domain}`)";
+      forceSSL = true;
-        entrypoints = "websecure";
+      locations."/" = {
-        middlewares = "local-ip-only@file";
+        proxyPass = "http://127.0.0.1:${builtins.toString config.services.node-red.port}";
-        service = "${app}";
-      };
-      http.services.${app} = {
-        loadBalancer = {
-          servers = [{
-            url = "http://localhost:1880";
-          }];
       };
     };
 
-    }];
+     environment.persistence."${config.mySystem.system.impermanence.persistPath}" = lib.mkIf config.mySystem.system.impermanence.enable {
+      directories = [{ directory = appFolder; user = user; group = group; mode = "750"; }];
+    };
 
     mySystem.services.homepage.media = mkIf cfg.addToHomepage [
       {
-        code-shodan = {
+        ${app} = {
           icon = "${app}.svg";
           href = "https://${url}";
 
-          description = "Music management";
+          description = "Workflow automation";
           container = "${app}";
-          widget = {
-            type = "${app}";
-            url = "https://${url}";
-            key = "{{HOMEPAGE_VAR_LIDARR__API_KEY}}";
-          };
         };
       }
     ];

nixos/modules/nixos/services/openvscode-server/default.nix

@@ -29,23 +29,15 @@ in
       withoutConnectionToken = true;
     };
 
-    mySystem.services.traefik.routers = [{
+    services.nginx.virtualHosts."code-${config.networking.hostName}.${config.networking.domain}" = {
-      http.routers.${app} = {
+      useACMEHost = config.networking.domain;
-        rule = "Host(`${url}`)";
+      forceSSL = true;
-        entrypoints = "websecure";
+      locations."/" = {
-        middlewares = "local-ip-only@file";
+        proxyPass = "http://127.0.0.1:${builtins.toString config.services.openvscode-server.port}";
-        service = "${app}";
+        proxyWebsockets = true;
-      };
-      http.services.${app} = {
-        loadBalancer = {
-          servers = [{
-            url = "http://localhost:${builtins.toString config.services.openvscode-server.port}";
-          }];
       };
     };
 
-    }];
-
     mySystem.services.homepage.media = mkIf cfg.addToHomepage [
       {
         code-shodan = {

nixos/modules/nixos/services/powerdns/default.nix

@@ -6,7 +6,7 @@
 with lib;
 let
   cfg = config.mySystem.services.powerdns;
-  persistentFolder = "${config.mySystem.persistentFolder}/nixos/pdns";
+  persistentFolder = "${config.mySystem.persistentFolder}/nixos/pdns"; # TODO refactor using bind mounts
   user = "pdns";
   group = "pdns";
   portDns = 5353; # avoiding conflict with adguardhome
@@ -41,7 +41,7 @@ in
 
     # ensure folder exist and has correct owner/group
     systemd.tmpfiles.rules = [
-      "d ${persistentFolder} 0755 ${user} ${group} -" #The - disables automatic cleanup, so the file wont be removed after a period
+      "d ${persistentFolder} 0750 ${user} ${group} -" #The - disables automatic cleanup, so the file wont be removed after a period
     ];
 
     services.powerdns = {

nixos/modules/nixos/services/prometheus/default.nix

@@ -0,0 +1,129 @@
+{ lib
+, config
+, pkgs
+, ...
+}:
+with lib;
+let
+  cfg = config.mySystem.${category}.${app};
+  app = "prometheus";
+  category = "services";
+  description = "Metric ingestion and storage";
+  user = app; #string
+  group = app; #string
+  port = 9001; #int
+  appFolder = "/var/lib/${app}";
+ # persistentFolder = "${config.mySystem.persistentFolder}/var/lib/${appFolder}";
+  host = "${app}" + (if cfg.development then "-dev" else "");
+  url = "${host}.${config.networking.domain}";
+in
+{
+  options.mySystem.${category}.${app} =
+    {
+      enable = mkEnableOption "${app}";
+      addToHomepage = mkEnableOption "Add ${app} to homepage" // { default = true; };
+      monitor = mkOption
+        {
+          type = lib.types.bool;
+          description = "Enable gatus monitoring";
+          default = true;
+        };
+      addToDNS = mkOption
+        {
+          type = lib.types.bool;
+          description = "Add to DNS list";
+          default = true;
+        };
+      development = mkOption
+        {
+          type = lib.types.bool;
+          description = "Development instance";
+          default = false;
+        };
+      backups = mkOption
+        {
+          type = lib.types.bool;
+          description = "Enable local backups";
+          default = true;
+        };
+
+
+    };
+
+  config = mkIf cfg.enable {
+
+    ## Secrets
+    # sops.secrets."${category}/${app}/env" = {
+    #   sopsFile = ./secrets.sops.yaml;
+    #   owner = user;
+    #   group = group;
+    #   restartUnits = [ "${app}.service" ];
+    # };
+
+    users.users.truxnell.extraGroups = [ group ];
+
+    ## service
+    # ref: https://github.com/nmasur/dotfiles/blob/aea33592361215356c0fbe5e9d533906f0a023cc/modules/nixos/services/prometheus.nix#L19
+    # https://github.com/ryan4yin/nix-config/blob/bec52f9d60f493d8bb31f298699dfc99eaf18dcc/hosts/12kingdoms-rakushun/grafana/default.nix#L42
+
+    services.prometheus = {
+      enable = true;
+      port = 9001;
+    };
+
+    # homepage integration
+    mySystem.services.homepage.infrastructure = mkIf cfg.addToHomepage [
+      {
+        ${app} = {
+          icon = "${app}.svg";
+          href = "https://${url}";
+          description = description;
+        };
+      }
+    ];
+
+    ### gatus integration
+    mySystem.services.gatus.monitors = mkIf cfg.monitor [
+      {
+        name = app;
+        group = "${category}";
+        url = "https://${url}";
+        interval = "1m";
+        conditions = [ "[CONNECTED] == true" "[STATUS] == 200" "[RESPONSE_TIME] < 50" ];
+      }
+    ];
+
+    ### Ingress
+    services.nginx.virtualHosts.${url} = {
+      useACMEHost = config.networking.domain;
+      forceSSL = true;
+      locations."^~ /" = {
+        proxyPass = "http://127.0.0.1:${builtins.toString port}";
+      };
+    };
+
+    ### firewall config
+
+    # networking.firewall = mkIf cfg.openFirewall {
+    #   allowedTCPPorts = [ port ];
+    #   allowedUDPPorts = [ port ];
+    # };
+
+    ### backups
+    warnings = [
+      (mkIf (!cfg.backups && config.mySystem.purpose != "Development")
+        "WARNING: Local backups for ${app} are disabled!")
+    ];
+
+    services.restic.backups = config.lib.mySystem.mkRestic
+      {
+        inherit app user;
+        paths = [ appFolder ];
+        inherit appFolder;
+
+      };
+
+
+
+  };
+}

nixos/modules/nixos/services/radicale/default.nix

@@ -0,0 +1,147 @@
+{ lib
+, config
+, pkgs
+, ...
+}:
+with lib;
+let
+  cfg = config.mySystem.${category}.${app};
+  app = "radicale";
+  category = "services";
+  description = "Contact/Calendar managment";
+  #   image = "%{image}";
+  user = app; #string
+  group = app; #string
+  port = 5232; #int
+  appFolder = "/var/lib/${app}";
+ # persistentFolder = "${config.mySystem.persistentFolder}/var/lib/${appFolder}";
+  host = "${app}" + (if cfg.development then "-dev" else "");
+  url = "${host}.${config.networking.domain}";
+in
+{
+  options.mySystem.${category}.${app} =
+    {
+      enable = mkEnableOption "${app}";
+      addToHomepage = mkEnableOption "Add ${app} to homepage" // { default = true; };
+      monitor = mkOption
+        {
+          type = lib.types.bool;
+          description = "Enable gatus monitoring";
+          default = true;
+        };
+      prometheus = mkOption
+        {
+          type = lib.types.bool;
+          description = "Enable prometheus scraping";
+          default = true;
+        };
+      addToDNS = mkOption
+        {
+          type = lib.types.bool;
+          description = "Add to DNS list";
+          default = true;
+        };
+      development = mkOption
+        {
+          type = lib.types.bool;
+          description = "Development instance";
+          default = false;
+        };
+      backups = mkOption
+        {
+          type = lib.types.bool;
+          description = "Enable local backups";
+          default = true;
+        };
+
+
+    };
+
+  config = mkIf cfg.enable {
+
+    ## Secrets
+    sops.secrets."${category}/${app}/htpasswd" = {
+      sopsFile = ./secrets.sops.yaml;
+      owner = user;
+      group = group;
+      restartUnits = [ "${app}.service" ];
+    };
+
+    users.users.truxnell.extraGroups = [ group ];
+
+     environment.persistence."${config.mySystem.system.impermanence.persistPath}" = lib.mkIf config.mySystem.system.impermanence.enable {
+      hideMounts = true;
+      directories = [ "/var/lib/radicale/" ];
+    };
+
+    ## service
+    services.radicale = {
+      enable = true;
+      settings = {
+        server.hosts = [ "0.0.0.0:${builtins.toString port}" ];
+        auth = {
+          type = "htpasswd";
+          htpasswd_filename = config.sops.secrets."${category}/${app}/htpasswd".path;
+          htpasswd_encryption = "plain";
+          realm = "Radicale - Password Required";
+        };
+        storage.filesystem_folder = "/var/lib/radicale/collections"; # TODO impermance/move?
+
+      };
+    };
+
+    # homepage integration
+    mySystem.services.homepage.infrastructure = mkIf cfg.addToHomepage [
+      {
+        ${app} = {
+          icon = "${app}.svg";
+          href = "https://${ url }";
+          description = description;
+        };
+      }
+    ];
+
+    ### gatus integration
+    mySystem.services.gatus.monitors = mkIf cfg.monitor [
+      {
+        name = app;
+        group = "${category}";
+        url = "https://${url}";
+        interval = "1m";
+        conditions = [ "[CONNECTED] == true" "[STATUS] == 200" "[RESPONSE_TIME] < 50" ];
+      }
+    ];
+
+    ### Ingress
+    services.nginx.virtualHosts.${host} = {
+      useACMEHost = config.networking.domain;
+      forceSSL = true;
+      locations."/" = {
+        proxyPass = "http://127.0.0.1:${builtins.toString port}";
+      };
+    };
+
+    ### firewall config
+
+    # networking.firewall = mkIf cfg.openFirewall {
+    #   allowedTCPPorts = [ port ];
+    #   allowedUDPPorts = [ port ];
+    # };
+
+    ### backups
+    warnings = [
+      (mkIf (!cfg.backups && config.mySystem.purpose != "Development")
+        "WARNING: Backups for ${app} are disabled!")
+    ];
+
+    services.restic.backups = mkIf cfg.backups (config.lib.mySystem.mkRestic
+      {
+        inherit app user;
+        paths = [ appFolder ];
+        inherit appFolder;
+
+      });
+
+
+  };
+}

nixos/modules/nixos/services/radicale/secrets.sops.yaml

@@ -0,0 +1,77 @@
+services:
+    radicale:
+        htpasswd: ENC[AES256_GCM,data:1UwfofyQcZicSUVhPumzw5g=,iv:y6Mm588YuxIM5u41hA7gtxAai9+AsvsLIyEBUHx79+8=,tag:jpuuBVlumhRFkQw777W9jA==,type:str]
+sops:
+    kms: []
+    gcp_kms: []
+    azure_kv: []
+    hc_vault: []
+    age:
+        - recipient: age1lj5vmr02qkudvv2xedfj5tq8x93gllgpr6tzylwdlt7lud4tfv5qfqsd5u
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBYc3dCb0t3VkdlVXQvOVVp
+            SFRVMnRuRHdHSGxiOGxTRjJnaHhBWUQ0b2dRCnp6YmNZSTVVdnUvZXpIQ3ZmYjI4
+            cThjRk8zREU4dDhLNjVzeFhLUFVTUkUKLS0tIHJJcHZkSHFKdmVPNkFvZlV5UXpj
+            a0JLb0lWdlJGd3FqRFlXMFNWUk9yQUEKrsZP7IYj5Po07bvj513ZyZwlPnGecYnw
+            dC3LeW6ZhWtFNRl3y4xjZeAE4ghX0TyrGoHEMIjFUszid4sT2iHvnQ==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age17edew3aahg3t5nte5g0a505sn96vnj8g8gqse8q06ccrrn2n3uysyshu2c
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBuRHlaMlU5Y05HUHBxVFc2
+            WG96Rk41VUdYNS9KSmpvK0tiSnk3TUxZMkdNCnExNkVFYVcrOFRsM3g0YTJROFRq
+            VDU2bytVcWlvNUhpNWNCc25ldU5HMEEKLS0tIGswWkZYYnJwZmJXelVQc1R3N1Ju
+            elcvU0ZKMk9ocUlIQk1tVTNPeisxR3MKaeHf2BILzQHFlnFHWUEEILiFR7h/HOoT
+            6Xl018GPmmm+6cjJLNjdvvjcPlxwbakw6fRVdirsP5H3LRdBRDYvMA==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age1u4tht685sqg6dkmjyer96r93pl425u6353md6fphpd84jh3jwcusvm7mgk
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBQckIwK0FBcnhRSXYvTlk2
+            TGZWR29KUmg1ampBWXBRUThVZFIyd2R6bTBJClRJdGVHazlGVDJoSVc3SDlkbVg1
+            U3hHTzFlQWRnekswUmwranUydm90QUkKLS0tIDNNaTdSbGZwOG1hSXlya1FEN2oz
+            Q0xUNDNpUFZHYkhjNncvUjdocmdYVUEKBMLMqLHvoxYciaSQM9cT0WBkIbrXE+x3
+            ZYtDfEQAu9FZvUqVYoGomR/0P1R/NNfhqHb1VAMuLKjB8d3obE0Gvg==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age1cp6vegrmqfkuj8nmt2u3z0sur7n0f7e9x9zmdv4zygp8j2pnucpsdkgagc
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBvMWtIcFJUQTM5UURaNUQx
+            VzJjcDU1QTZZUE5JYTAzOEpPYXhBdkpnVm04ClVHaWRTSTJxUm54NUV6M2o2c3U5
+            OXdmOE9ZT1h6dlN4Z2pVbHFZdENWQ2cKLS0tIHNrZ1hTQUxVSWxWcmYvcnBBbUkx
+            V1pQQUpsNEt0RXBVQUNFY2p5ejhOc1kK5nhUW8lEFa2m38mTGVnQGuRj2DMr+JJ0
+            9oEjLbMkZHfeYKSGDlkOPeG5f7Lp3qCJnJ7FHv6/TlL2tN4yVcUYxA==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age1ekt5xz7u2xgdzgsrffhd9x22n80cn4thxd8zxjy2ey5vq3ca7gnqz25g5r
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBnWXdDUW5RMHhnR3NzamI4
+            ZzhWVHBsWnRPSkJieGpuOEg1VDNzMHpVQXhnCmdGbVJRa3BQU0l0Mmk1aWI3eG0v
+            MlRZdEdkTmxuUnp0QVVSaFdpQVp1Z00KLS0tIHJzZ3BkdEcxUzNDRjZWaUhYOEJs
+            VTF3R2VscUtSSFJEZGN2Qk42TEJKMEEKeohZ/XNFDQSjwmWDSOfUg95S3SXLFqdl
+            hUNGRF32Sno7qE55fTPfK2uffb1Wocyjnt6Otp+Bmu0KUDGeBaBYLA==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age1jpeh4s553taxkyxhzlshzqjfrtvmmp5lw0hmpgn3mdnmgzku332qe082dl
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBpNldwQlNGN25RYW9qanhh
+            bkg3c0VvY0ZFUHBjOE85VHJvODI3Y21GRmhRClFabkEwNWJMMy8rR3lmOUJaTElv
+            a1A4ajJkU3FnTitmZzhUQWNKckE2MUEKLS0tIE5iVUxzM2ZpOGdEc3VJSE54cjF0
+            L1h0TERMSm82N0ZYalB1QmZ0QUMxd2cKminvLq5ok5M44znw1etDknkNho7eohur
+            jgVEWEpn1vL570BVeNwZUcVRW2tUuMGgzznabkWTl19qMaxck//XiA==
+            -----END AGE ENCRYPTED FILE-----
+        - recipient: age1j2r8mypw44uvqhfs53424h6fu2rkr5m7asl7rl3zn3xzva9m3dcqpa97gw
+          enc: |
+            -----BEGIN AGE ENCRYPTED FILE-----
+            YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBraWx2dXpUaEhUS09GWXVx
+            T0daMWlDU2U1NDlCcUVlYkhBUWs2cXBYdHhrCnlTYThNQlJsZGZDRWJ2SUJWRVVW
+            QzBja2I1MWJ2YnN5d0NWMXRSbE43L1EKLS0tIHZETEpOQnZNT1JkYlEzNCtDS1FP
+            UWVaNG0vVXB0YjNXb0ZHeEFaZDVrSmsK3EKc2FhyfB4kG08hfBXRD/pJOyWwmjj8
+            qUns8YxX5KI2dI+P7UNH8uxzpbxhbfKIx1oCGRSsFfLkDfZPtRLKtg==
+            -----END AGE ENCRYPTED FILE-----
+    lastmodified: "2024-04-30T23:25:37Z"
+    mac: ENC[AES256_GCM,data:dadR/CZPmDYJCIFgbWdbu6hjNGrwc1IRk1IucAhVTOkuVQBCMhd4+Qv0CWG5Lc1O4juSblJDQbp9PGO/YCGfJ6jfLplBxlei7LTX45RbaChrPc/bVULSAXDZRjdQKVN/gb2HVgwh2PXXdR0IvYyDao55Repw+Va5qfSSa1FiHfg=,iv:nM1TwKOOB8Qr8f1l/ct32qrAV7dLhclu4SsRk/KcSSM=,tag:6fzeJyS/dPe7Bxm4WYBBwQ==,type:str]
+    pgp: []
+    unencrypted_suffix: _unencrypted
+    version: 3.8.1

nixos/modules/nixos/services/restic/default.nix

@@ -26,8 +26,14 @@ in
           description = "Location for remote backups";
           default = "";
         };
-
     };
+    mountPath = mkOption
+      {
+        type = types.str;
+        description = "Location for  snapshot mount";
+        default = "/mnt/nightly_backup";
+      };
+
   };
 
 
@@ -53,6 +59,14 @@ in
       };
     };
 
+    environment.persistence = mkIf (cfg.local.enable || cfg.remote.enable) {
+      "${config.mySystem.system.impermanence.persistPath}" = {
+        directories = [ "/var/lib/containers" ];
+      };
+    };
+
+
+
     # useful commands:
     # view snapshots - zfs list -t snapshot
 
@@ -76,16 +90,24 @@ in
         timerConfig.Persistent = "true";
       };
 
+      # recreate snapshot and mount, ready for backup
+      # I used mkdir -p over a nix tmpfile, as mkdir -p exits cleanly
+      # if the folder already exists, and tmpfiles complain
+      # if the folder exists and is already mounted.
       services.restic_nightly_snapshot = {
         description = "Nightly ZFS snapshot for Restic";
-        path = with pkgs; [ zfs ];
+        path = with pkgs; [ zfs busybox ];
         serviceConfig.Type = "simple";
         script = ''
+          mkdir -p /mnt/nightly_backup/ && \
+          umount ${cfg.mountPath} || true && \
           zfs destroy rpool/safe/persist@restic_nightly_snap || true && \
-          zfs snapshot rpool/safe/persist@restic_nightly_snap
+          zfs snapshot rpool/safe/persist@restic_nightly_snap && \
+          mount -t zfs rpool/safe/persist@restic_nightly_snap ${cfg.mountPath}
         '';
       };
 
+
     };
   };
 }

nixos/modules/nixos/services/traefik/default.nix

@@ -25,7 +25,7 @@ let
       #   # Forward requests w/ middlewares=authelia@file to authelia.
       #   forwardAuth = {
       #     # address = cfg.autheliaUrl;
-      #     address = "http://localhost:9092/api/verify?rd=https://auth.dhupar.xyz:444/";
+      #     address = "http://127.0.0.1:9092/api/verify?rd=https://auth.dhupar.xyz:444/";
       #     trustForwardHeader = true;
       #     authResponseHeaders = [
       #       "Remote-User"
@@ -38,7 +38,7 @@ let
       # authelia-basic = {
       #   # Forward requests w/ middlewares=authelia-basic@file to authelia.
       #   forwardAuth = {
-      #     address = "http://localhost:9092/api/verify?auth=basic";
+      #     address = "http://127.0.0.1:9092/api/verify?auth=basic";
       #     trustForwardHeader = true;
       #     authResponseHeaders = [
       #       "Remote-User"

nixos/modules/nixos/services/zigbee2mqtt/default.nix

@@ -6,11 +6,12 @@
 with lib;
 let
   cfg = config.mySystem.services.zigbee2mqtt;
-  persistentFolder = "${config.mySystem.persistentFolder}/nixos/services/${app}/";
+  # persistentFolder = "${config.mySystem.persistentFolder}/${appFolder}/";
   app = "zigbee2mqtt";
   user = app;
   group = app;
-
+  appFolder = config.services.zigbee2mqtt.dataDir;
+  port = 8080;
 in
 {
   options.mySystem.services.zigbee2mqtt = {
@@ -21,11 +22,6 @@ in
 
   config = mkIf cfg.enable {
 
-    # ensure folder exist and has correct owner/group
-    systemd.tmpfiles.rules = [
-      "d ${persistentFolder} 0750 ${user} ${group} -" #The - disables automatic cleanup, so the file wont be removed after a period
-    ];
-
     sops.secrets."services/mosquitto/mq/plainPassword.yaml" = {
       sopsFile = ../mosquitto/secrets.sops.yaml;
       owner = config.users.users.zigbee2mqtt.name;
@@ -35,15 +31,13 @@ in
 
     services.zigbee2mqtt = {
       enable = true;
-      dataDir = persistentFolder;
       settings = {
-        advanced.log_level = "debug";
         homeassistant = true;
         permit_join = false;
         include_device_information = true;
         frontend =
           {
-            port = 8080;
+            port = port;
             url = "https://${app}.${config.networking.domain}";
           };
         client_id = "z2m";
@@ -52,32 +46,43 @@ in
         };
         mqtt = {
           server = "mqtt://mqtt.trux.dev:1883";
+          client_id = "z2m";
+          reject_unauthorized = true;
+          keepalive = 60;
+          version = 4;
           user = "mq";
+          base_topic = "zigbee2mqtt";
           password = "!${config.sops.secrets."services/mosquitto/mq/plainPassword.yaml".path} password";
         };
-
+        availability = {
+          active.timeout = 10;
+          passive.timeout = 1500;
         };
+        advanced = {
+          log_level = "debug";
+          network_key = [ 42 88 79 94 97 102 54 190 99 52 160 64 224 107 103 40 ];
+          pan_id = 62782;
+          last_seen = "ISO_8601";
+        };
+        experimental.new_api = true;
+      };
+    };
+
+    environment.persistence."${config.mySystem.system.impermanence.persistPath}" = lib.mkIf config.mySystem.system.impermanence.enable {
+      directories = [{ directory = appFolder; user = user; group = group; mode = "750"; }];
     };
 
     users.users.truxnell.extraGroups = [ app ];
 
-    mySystem.services.traefik.routers = [{
+    services.nginx.virtualHosts."${app}.${config.networking.domain}" = {
-      http.routers.${app} = {
+      useACMEHost = config.networking.domain;
-        rule = "Host(`${app}.${config.mySystem.domain}`)";
+      forceSSL = true;
-        entrypoints = "websecure";
+      locations."/" = {
-        middlewares = "local-ip-only@file";
+        proxyPass = "http://127.0.0.1:${builtins.toString port}";
-        service = "${app}";
+        proxyWebsockets = true;
-      };
-      http.services.${app} = {
-        loadBalancer = {
-          servers = [{
-            url = "http://localhost:8080";
-          }];
       };
     };
 
-    }];
-
 
     mySystem.services.homepage.infrastructure = mkIf cfg.addToHomepage [
       {
@@ -101,11 +106,9 @@ in
 
     services.restic.backups = config.lib.mySystem.mkRestic
       {
-        inherit app;
+        inherit app appFolder;
         user = builtins.toString user;
-        paths = [ persistentFolder ];
+        paths = [ appFolder ];
-        appFolder = app;
-        inherit persistentFolder;
       };
 
 

nixos/modules/nixos/system/default.nix

@@ -6,6 +6,7 @@
     ./systempackages.nix
     ./nix.nix
     ./zfs.nix
+    ./impermanence.nix
     ./nfs
     ./motd
     ./pushover

nixos/modules/nixos/system/impermanence.nix

@@ -0,0 +1,62 @@
+{ lib
+, config
+, ...
+}:
+let
+  cfg = config.mySystem.system.impermanence;
+in
+with lib;
+{
+  options.mySystem.system.impermanence = {
+    enable = mkEnableOption "system impermanence";
+    rootBlankSnapshotName = lib.mkOption {
+      type = lib.types.str;
+      default = "blank";
+    };
+    rootPoolName = lib.mkOption {
+      type = lib.types.str;
+      default = "rpool/local/root";
+    };
+    persistPath = lib.mkOption {
+      type = lib.types.str;
+      default = "/persist";
+    };
+
+  };
+
+
+  config = lib.mkIf cfg.enable {
+    # move ssh keys
+
+    # bind a initrd command to rollback to blank root after boot
+    boot.initrd.postDeviceCommands = lib.mkAfter ''
+      zfs rollback -r ${cfg.rootPoolName}@${cfg.rootBlankSnapshotName}
+    '';
+
+    systemd.tmpfiles.rules = mkIf config.services.openssh.enable [
+      "d /etc/ 0755 root root -" #The - disables automatic cleanup, so the file wont be removed after a period
+      "d /etc/ssh/ 0755 root root -" #The - disables automatic cleanup, so the file wont be removed after a period
+    ];
+
+    environment.persistence."${cfg.persistPath}" = {
+      hideMounts = true;
+      directories =
+        [
+          "/var/log" # persist logs between reboots for debugging
+          "/var/lib/cache" # cache files (restic, nginx, contaienrs)
+          "/var/lib/nixos" # nixos state
+
+        ];
+      files = [
+        "/etc/machine-id"
+        "/etc/adjtime" # hardware clock adjustment
+        # ssh keys
+        "/etc/ssh/ssh_host_ed25519_key"
+        "/etc/ssh/ssh_host_ed25519_key.pub"
+        "/etc/ssh/ssh_host_rsa_key"
+        "/etc/ssh/ssh_host_rsa_key.pub"
+      ];
+    };
+
+  };
+}

nixos/profiles/global.nix

@@ -15,17 +15,6 @@ with lib;
       ./global
     ];
 
-  options.mySystem.system.impermanence = {
-    enable = mkEnableOption "impermanence";
-    # explicitly specify ssh path key
-    # just so I can track where sops-nix needs to find it
-    sshPath = mkOption {
-      type = types.str;
-      default = "/etc/ssh";
-    };
-  };
-
-
   config = {
 
     boot.tmp.cleanOnBoot = true;

nixos/profiles/global/sops.nix

@@ -1,7 +1,7 @@
 { config, ... }:
 {
 
-  sops.age.sshKeyPaths = [ "${config.mySystem.system.impermanence.sshPath}/ssh_host_ed25519_key" ];
+  sops.age.sshKeyPaths = [ "/etc/ssh/ssh_host_ed25519_key" ];
   # Secret for machine-specific pushover
   sops.secrets."services/pushover/env" = {
     sopsFile = ./secrets.sops.yaml;

nixos/profiles/impermanence.nix

@@ -1,74 +0,0 @@
-{ lib
-, config
-, ...
-}:
-let
-  cfg = config.mySystem.system.impermanence;
-in
-with lib;
-{
-  options.mySystem.system.impermanence = {
-    rootBlankSnapshotName = lib.mkOption {
-      type = lib.types.str;
-      default = "blank";
-    };
-    rootPoolName = lib.mkOption {
-      type = lib.types.str;
-      default = "rpool/local/root";
-    };
-    persistPath = lib.mkOption {
-      type = lib.types.str;
-      default = "/persist";
-    };
-
-  };
-
-  config = {
-
-    # move ssh keys
-    mySystem.system.impermanence.sshPath = "${cfg.persistPath}/nixos/etc/ssh";
-    mySystem.system.impermanence.enable = true;
-
-    # bind a initrd command to rollback to blank root after boot
-    boot.initrd.postDeviceCommands = lib.mkAfter ''
-      zfs rollback -r ${cfg.rootPoolName}@${cfg.rootBlankSnapshotName}
-    '';
-
-    # environment.persistence."${cfg.persistPath}/nixos" = {
-    #   hideMounts = true;
-    #   directories =
-    #     [
-    #       "/var/log"
-    #     ];
-    # };
-
-    # move ssh keys to persist folder
-    services.openssh.hostKeys = mkIf config.services.openssh.enable [
-      {
-        path = "${config.mySystem.system.impermanence.sshPath}/ssh_host_ed25519_key";
-        type = "ed25519";
-      }
-      {
-        path = "${config.mySystem.system.impermanence.sshPath}/ssh_host_rsa_key";
-        type = "rsa";
-        bits = 4096;
-      }
-    ];
-
-    # If impermanent, move key location to safe
-    systemd.tmpfiles.rules = mkIf config.services.openssh.enable [
-      "d ${cfg.persistPath}/ 777 root root"
-      "d ${cfg.persistPath}/nixos 777 root root"
-      "d ${cfg.persistPath}/nixos/services 777 root root"
-      "d ${config.mySystem.system.impermanence.sshPath}/ 0755 root root -" #The - disables automatic cleanup, so the file wont be removed after a period
-    ];
-
-    # set machine id for log continuity
-    environment.etc.machine-id.source = "${cfg.persistPath}/nixos/etc/machine-id";
-
-    # keep hardware clock adjustment data
-    environment.etc.adjtime.source = "${cfg.persistPath}/nixos/etc/adjtime";
-
-  };
-
-}

nixos/profiles/role-worstation.nix

@@ -15,7 +15,6 @@ with config;
     # Lets see if fish everywhere is OK on the pi's
     # TODO decide if i drop to bash on pis?
     shell.fish.enable = true;
-    services.cockpit.enable = true;
 
     nfs.nas.enable = true;
     system.resticBackup.local.enable = false;