From fb441942da9b92d16addabc3d20989de161d7c9b Mon Sep 17 00:00:00 2001
From: Callum Wong <mail@callumwong.com>
Date: Sun, 23 Aug 2026 01:31:54 +1000
Subject: [PATCH 01/10] gpu vm

---
 hosts/nixos/liz/_windows-vm.nix |  8 +++-
 hosts/nixos/liz/default.nix     | 20 ++++++++++
 hosts/nixos/vm-gpu/default.nix  | 69 +++++++++++++++++++++++++++++++++
 3 files changed, 96 insertions(+), 1 deletion(-)
 create mode 100644 hosts/nixos/vm-gpu/default.nix

diff --git a/hosts/nixos/liz/_windows-vm.nix b/hosts/nixos/liz/_windows-vm.nix
index 4682b13..75e89c1 100644
--- a/hosts/nixos/liz/_windows-vm.nix
+++ b/hosts/nixos/liz/_windows-vm.nix
@@ -164,7 +164,7 @@ let
     '';
   };
 
-  # QMP powerdown lets Windows unmount NTFS during host shutdown.
+  # Ask Windows to unmount cleanly, then keep ExecStop alive until QEMU exits.
   shutdown = pkgs.writeShellApplication {
     name = "windows-vm-shutdown";
     runtimeInputs = [ pkgs.socat ];
@@ -174,6 +174,12 @@ let
         '{"execute":"qmp_capabilities"}' \
         '{"execute":"system_powerdown"}' \
         | socat - "UNIX-CONNECT:${qmpSocket}" >/dev/null || true
+
+      mainPid="''${MAINPID:-}"
+      [ -n "$mainPid" ] || exit 0
+      while kill -0 "$mainPid" 2>/dev/null; do
+        sleep 1
+      done
     '';
   };
 in
diff --git a/hosts/nixos/liz/default.nix b/hosts/nixos/liz/default.nix
index cf36f90..57e9a0d 100644
--- a/hosts/nixos/liz/default.nix
+++ b/hosts/nixos/liz/default.nix
@@ -29,6 +29,10 @@ in
           n = 2;
           hostname = "vm-coder";
         })
+        (microvmLib.mkHostNetworking {
+          n = 3;
+          hostname = "vm-gpu";
+        })
       ]
       ++ (with nixos; [
         uefi
@@ -95,6 +99,22 @@ in
         restartIfChanged = true;
       };
 
+      microvm.vms.vm-gpu = {
+        flake = inputs.self;
+        autostart = false;
+        restartIfChanged = true;
+      };
+
+      # The ordering dependency makes either guest stop completely before the
+      # other starts when systemd resolves their mutual conflict.
+      systemd.services = {
+        windows-vm = {
+          conflicts = [ "microvm@vm-gpu.service" ];
+          after = [ "microvm@vm-gpu.service" ];
+        };
+        "microvm@vm-gpu".conflicts = [ "windows-vm.service" ];
+      };
+
       _module.args.sshKeys = keys.callum;
 
       system.stateVersion = "25.11";
diff --git a/hosts/nixos/vm-gpu/default.nix b/hosts/nixos/vm-gpu/default.nix
new file mode 100644
index 0000000..2ff529e
--- /dev/null
+++ b/hosts/nixos/vm-gpu/default.nix
@@ -0,0 +1,69 @@
+{ config, microvmLib, ... }:
+let
+  inherit (config.flake.modules) nixos;
+  inherit (config.flake) keys;
+in
+{
+  flake.modules.nixos."hosts/nixos/vm-gpu" =
+    { pkgs, ... }:
+    {
+      imports = [
+        (microvmLib.mkGuestModule {
+          n = 3;
+          hostname = "vm-gpu";
+        })
+      ]
+      ++ (with nixos; [
+        ssh
+      ]);
+
+      system.stateVersion = "25.11";
+
+      microvm = {
+        mem = 4096;
+        vcpu = 4;
+        devices = [
+          {
+            bus = "pci";
+            path = "0000:08:00.0";
+          }
+          {
+            bus = "pci";
+            path = "0000:08:00.1";
+          }
+        ];
+        shares = [
+          {
+            tag = "work";
+            source = "/scratch/gpu";
+            mountPoint = "/work";
+            proto = "virtiofs";
+          }
+        ];
+      };
+
+      services.xserver.videoDrivers = [ "nvidia" ];
+      hardware = {
+        graphics.enable = true;
+        nvidia = {
+          modesetting.enable = false;
+          open = true;
+        };
+      };
+
+      environment = {
+        systemPackages = [
+          (pkgs.llama-cpp.override { cudaSupport = true; })
+          pkgs.ffmpeg-full
+          pkgs.nvtopPackages.nvidia
+        ];
+        variables.LLAMA_CACHE = "/work/llama-cache";
+      };
+
+      systemd.tmpfiles.rules = [
+        "d /work/llama-cache 0755 root root -"
+      ];
+
+      users.users.root.openssh.authorizedKeys.keys = keys.callum;
+    };
+}

From 491c0a03b1c8fa3e1ed8457b6186563cd03bb017 Mon Sep 17 00:00:00 2001
From: Callum Wong <mail@callumwong.com>
Date: Thu, 27 Aug 2026 11:33:00 +1000
Subject: [PATCH 02/10] use fully declarative pattern for vm-gpu

---
 hosts/nixos/liz/default.nix | 59 +++++++++++++++++++++----------------
 1 file changed, 33 insertions(+), 26 deletions(-)

diff --git a/hosts/nixos/liz/default.nix b/hosts/nixos/liz/default.nix
index 57e9a0d..1ea7c1e 100644
--- a/hosts/nixos/liz/default.nix
+++ b/hosts/nixos/liz/default.nix
@@ -75,34 +75,41 @@ in
 
       # https://microvm-nix.github.io/microvm.nix/declarative.html#fully-declarative
 
-      microvm.vms.vm-gallery = {
-        config.imports = [
-          nixos.base
-          nixos.global
-          nixos."hosts/nixos/vm-gallery"
-        ];
-        # nixos.base sets nixpkgs.config, which nixpkgs forbids once pkgs is
-        # instantiated externally; force config-mode's eval-config to build
-        # its own pkgs instead of reusing liz's already-built instance.
-        pkgs = null;
-        restartIfChanged = true;
-      };
+      microvm.vms = {
+        vm-gallery = {
+          config.imports = [
+            nixos.base
+            nixos.global
+            nixos."hosts/nixos/vm-gallery"
+          ];
+          # nixos.base sets nixpkgs.config, which nixpkgs forbids once pkgs is
+          # instantiated externally; force config-mode's eval-config to build
+          # its own pkgs instead of reusing liz's already-built instance.
+          pkgs = null;
+          restartIfChanged = true;
+        };
 
-      microvm.vms.vm-coder = {
-        config.imports = [
-          nixos.base
-          nixos.global
-          nixos."hosts/nixos/vm-coder"
-        ];
-        pkgs = null;
-        nixpkgs = inputs.unstable;
-        restartIfChanged = true;
-      };
+        vm-coder = {
+          config.imports = [
+            nixos.base
+            nixos.global
+            nixos."hosts/nixos/vm-coder"
+          ];
+          pkgs = null;
+          nixpkgs = inputs.unstable;
+          restartIfChanged = true;
+        };
 
-      microvm.vms.vm-gpu = {
-        flake = inputs.self;
-        autostart = false;
-        restartIfChanged = true;
+        vm-gpu = {
+          config.imports = [
+            nixos.base
+            nixos.global
+            nixos."hosts/nixos/vm-gpu"
+          ];
+          pkgs = null;
+          nixpkgs = inputs.unstable;
+          restartIfChanged = true;
+        };
       };
 
       # The ordering dependency makes either guest stop completely before the

From 414f7ee6702244dc5f90bbdd87f48da3d52f17ed Mon Sep 17 00:00:00 2001
From: Callum Wong <mail@callumwong.com>
Date: Thu, 27 Aug 2026 11:33:09 +1000
Subject: [PATCH 03/10] set vm-gpu stateVersion to 26.05

---
 hosts/nixos/vm-gpu/default.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hosts/nixos/vm-gpu/default.nix b/hosts/nixos/vm-gpu/default.nix
index 2ff529e..9e6e98a 100644
--- a/hosts/nixos/vm-gpu/default.nix
+++ b/hosts/nixos/vm-gpu/default.nix
@@ -17,7 +17,7 @@ in
         ssh
       ]);
 
-      system.stateVersion = "25.11";
+      system.stateVersion = "26.05";
 
       microvm = {
         mem = 4096;

From 5c90bf9e4d23c7c5ae3c2efe2487ff5ba9a6e6c1 Mon Sep 17 00:00:00 2001
From: Callum Wong <mail@callumwong.com>
Date: Thu, 27 Aug 2026 11:33:19 +1000
Subject: [PATCH 04/10] add vm-gpu to ciHosts

---
 modules/flake/checks.nix | 1 +
 1 file changed, 1 insertion(+)

diff --git a/modules/flake/checks.nix b/modules/flake/checks.nix
index e33f29e..7fdffcf 100644
--- a/modules/flake/checks.nix
+++ b/modules/flake/checks.nix
@@ -19,6 +19,7 @@ in
       "shama"
       "vm-coder"
       "vm-gallery"
+      "vm-gpu"
     ];
 
     flake.checks.${system} = lib.genAttrs config.flake.ciHosts (

From 2500d883446c6d6991695e792d8977b3f148c699 Mon Sep 17 00:00:00 2001
From: Callum Wong <mail@callumwong.com>
Date: Thu, 27 Aug 2026 12:33:03 +1000
Subject: [PATCH 05/10] pin vm-gpu's standalone check to unstable nixpkgs

liz's declarative microvm guest already builds vm-gpu against
inputs.unstable, but the standalone checks.x86_64-linux.vm-gpu fell
back to the default stable nixpkgs, so CUDA llama-cpp hashed
differently and had to be compiled from source twice per CI run.
---
 hosts/nixos/vm-gpu/default.nix | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/hosts/nixos/vm-gpu/default.nix b/hosts/nixos/vm-gpu/default.nix
index 9e6e98a..813ea80 100644
--- a/hosts/nixos/vm-gpu/default.nix
+++ b/hosts/nixos/vm-gpu/default.nix
@@ -4,6 +4,8 @@ let
   inherit (config.flake) keys;
 in
 {
+  flake.nixpkgs.vm-gpu = "unstable";
+
   flake.modules.nixos."hosts/nixos/vm-gpu" =
     { pkgs, ... }:
     {

From 9a66fb48a66976521fad4ea473433dcf0b88658a Mon Sep 17 00:00:00 2001
From: Callum Wong <mail@callumwong.com>
Date: Thu, 27 Aug 2026 15:03:49 +1000
Subject: [PATCH 06/10] import persistence into vm-gpu

modules/ssh.nix contributes the openssh host keys to the persistence
aspect, not to the ssh aspect, so importing ssh alone left vm-gpu with
no impermanence at all. It regenerated its host keys on every restart,
tripping host key verification for anyone reaching it over ssh.

vm-gallery imports persistence directly and vm-coder picks it up via
the coder aspect; vm-gpu was the only guest missing it.
---
 hosts/nixos/vm-gpu/default.nix | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/hosts/nixos/vm-gpu/default.nix b/hosts/nixos/vm-gpu/default.nix
index 813ea80..602bb3c 100644
--- a/hosts/nixos/vm-gpu/default.nix
+++ b/hosts/nixos/vm-gpu/default.nix
@@ -16,6 +16,8 @@ in
         })
       ]
       ++ (with nixos; [
+        persistence
+
         ssh
       ]);
 

From 13ca6e361ec33b406c33f041a4c05d61dbe6c54c Mon Sep 17 00:00:00 2001
From: Callum Wong <mail@callumwong.com>
Date: Thu, 27 Aug 2026 19:08:18 +1000
Subject: [PATCH 07/10] use llama-cpp service with ling-3.0-tiny

---
 hosts/nixos/vm-gpu/default.nix |   4 +-
 modules/services/llama-cpp.nix | 115 +++++++++++++++++++--------------
 2 files changed, 70 insertions(+), 49 deletions(-)

diff --git a/hosts/nixos/vm-gpu/default.nix b/hosts/nixos/vm-gpu/default.nix
index 602bb3c..f92ac74 100644
--- a/hosts/nixos/vm-gpu/default.nix
+++ b/hosts/nixos/vm-gpu/default.nix
@@ -19,6 +19,8 @@ in
         persistence
 
         ssh
+
+        llama-cpp
       ]);
 
       system.stateVersion = "26.05";
@@ -57,11 +59,9 @@ in
 
       environment = {
         systemPackages = [
-          (pkgs.llama-cpp.override { cudaSupport = true; })
           pkgs.ffmpeg-full
           pkgs.nvtopPackages.nvidia
         ];
-        variables.LLAMA_CACHE = "/work/llama-cache";
       };
 
       systemd.tmpfiles.rules = [
diff --git a/modules/services/llama-cpp.nix b/modules/services/llama-cpp.nix
index 2497285..cc855fe 100644
--- a/modules/services/llama-cpp.nix
+++ b/modules/services/llama-cpp.nix
@@ -1,57 +1,78 @@
 {
-  flake.modules.nixos.llama-cpp =
+  flake.modules.nixos =
+    let
+      port = 9931;
+    in
     {
-      config,
-      lib,
-      pkgs,
-      ...
-    }:
-    {
-      services.llama-cpp = {
-        enable = false;
-        package = pkgs.llama-cpp.override { cudaSupport = true; };
-        settings = {
-          host = "0.0.0.0";
-          port = 8080;
+      llama-cpp =
+        {
+          config,
+          lib,
+          pkgs,
+          ...
+        }:
+        {
+          services.llama-cpp = {
+            enable = true;
+            package = (pkgs.llama-cpp.override { cudaSupport = true; }).overrideAttrs (old: {
+              # drop once nixpkgs bumps past b10472
+              version = "10472";
+              src = pkgs.fetchFromGitHub {
+                owner = "ggml-org";
+                repo = "llama.cpp";
+                tag = "b10472";
+                hash = "sha256-re0WlafJUDZOPNfIq2ECRSctdrDFVc0fXb5iSd7gDR8=";
+                leaveDotGit = true;
+                postFetch = ''
+                  git -C "$out" rev-parse --short HEAD > $out/COMMIT
+                  find "$out" -name .git -print0 | xargs -0 rm -rf
+                '';
+              };
+            });
+            settings = {
+              inherit port;
+              host = "0.0.0.0";
 
-          hf-repo = "unsloth/Qwen3.6-35B-A3B-MTP-GGUF:UD-Q6_K_XL";
-          alias = "qwen3.6-35b-a3b";
+              # https://huggingface.co/bartowski/Ling-3.0-tiny-GGUF
+              hf-repo = "bartowski/Ling-3.0-tiny-GGUF:Q6_K_L";
+              alias = "ling-3.0-tiny";
 
-          n-cpu-moe = 37;
-          n-gpu-layers = 99;
-          kv-offload = true;
-          threads = 12;
-          threads-batch = 12;
-          ctx-size = 262144;
-          flash-attn = "on";
-          cache-type-k = "q8_0";
-          cache-type-v = "q8_0";
-          spec-draft-type-k = "q8_0";
-          spec-draft-type-v = "q8_0";
-          parallel = 1;
+              n-gpu-layers = 999;
+              kv-offload = true;
+              flash-attn = "on";
+              ctx-size = 32768;
+              parallel = 1;
 
-          # https://unsloth.ai/docs/models/qwen3.6#llama.cpp-mtp-guide
-          temp = 0.6;
-          top-p = 0.95;
-          top-k = 20;
-          min-p = 0.00;
-          spec-type = "draft-mtp";
-          spec-draft-n-max = 2;
-        };
-      };
+              temp = 0.6;
+              top-p = 0.95;
+              top-k = 20;
+              min-p = 0.00;
+            };
+          };
 
-      systemd.services.llama-cpp = lib.mkIf config.services.llama-cpp.enable {
-        unitConfig = {
-          Wants = [ "systemd-modules-load.service" ];
-          After = [ "systemd-modules-load.service" ];
-        };
+          systemd.services.llama-cpp = lib.mkIf config.services.llama-cpp.enable {
+            unitConfig = {
+              Wants = [ "systemd-modules-load.service" ];
+              After = [ "systemd-modules-load.service" ];
+            };
 
-        serviceConfig = {
-          ExecStartPre = "${pkgs.bash}/bin/bash -c 'until ${config.hardware.nvidia.package.bin}/bin/nvidia-smi -L >/dev/null 2>&1; do ${pkgs.coreutils}/bin/sleep 1; done'";
-          Environment = lib.mkForce [
-            "LLAMA_CACHE=/var/lib/llama-cpp/cache"
-          ];
-          TimeoutStartSec = "infinity";
+            serviceConfig = {
+              ExecStartPre = "${pkgs.bash}/bin/bash -c 'until ${config.hardware.nvidia.package.bin}/bin/nvidia-smi -L >/dev/null 2>&1; do ${pkgs.coreutils}/bin/sleep 1; done'";
+              TimeoutStartSec = "infinity";
+              DynamicUser = lib.mkForce false;
+              User = "root";
+              Environment = [ "LLAMA_CACHE=/work/llama-cache" ];
+              ReadWritePaths = [ "/work/llama-cache" ];
+            };
+          };
+        };
+      gateway = {
+        modules.gateway.services.llama-cpp = {
+          name = "llama.cpp";
+          domainName = "llama";
+          iconUrl = "https://cdn.jsdelivr.net/gh/selfhst/icons/svg/llama-cpp.svg";
+          addr = "10.0.0.4:${port}";
+          category = "Development";
         };
       };
     };

From a484371a40e33ad477b99aee8934a54408457932 Mon Sep 17 00:00:00 2001
From: Callum Wong <mail@callumwong.com>
Date: Thu, 27 Aug 2026 21:42:48 +1000
Subject: [PATCH 08/10] whoops

---
 modules/services/llama-cpp.nix | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/modules/services/llama-cpp.nix b/modules/services/llama-cpp.nix
index cc855fe..8d9ba3f 100644
--- a/modules/services/llama-cpp.nix
+++ b/modules/services/llama-cpp.nix
@@ -14,7 +14,7 @@
         {
           services.llama-cpp = {
             enable = true;
-            package = (pkgs.llama-cpp.override { cudaSupport = true; }).overrideAttrs (old: {
+            package = (pkgs.llama-cpp.override { cudaSupport = true; }).overrideAttrs {
               # drop once nixpkgs bumps past b10472
               version = "10472";
               src = pkgs.fetchFromGitHub {
@@ -28,7 +28,7 @@
                   find "$out" -name .git -print0 | xargs -0 rm -rf
                 '';
               };
-            });
+            };
             settings = {
               inherit port;
               host = "0.0.0.0";
@@ -71,7 +71,7 @@
           name = "llama.cpp";
           domainName = "llama";
           iconUrl = "https://cdn.jsdelivr.net/gh/selfhst/icons/svg/llama-cpp.svg";
-          addr = "10.0.0.4:${port}";
+          addr = "10.0.0.4:${toString port}";
           category = "Development";
         };
       };

From ee04a4f97d3bf3b764ef0b46709d30870a24b24e Mon Sep 17 00:00:00 2001
From: Callum Wong <mail@callumwong.com>
Date: Thu, 27 Aug 2026 21:43:24 +1000
Subject: [PATCH 09/10] allow llama.cpp port through firewall

---
 modules/services/llama-cpp.nix | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/modules/services/llama-cpp.nix b/modules/services/llama-cpp.nix
index 8d9ba3f..01b6617 100644
--- a/modules/services/llama-cpp.nix
+++ b/modules/services/llama-cpp.nix
@@ -50,6 +50,8 @@
             };
           };
 
+          networking.firewall.allowedTCPPorts = [ port ];
+
           systemd.services.llama-cpp = lib.mkIf config.services.llama-cpp.enable {
             unitConfig = {
               Wants = [ "systemd-modules-load.service" ];

From 17464b5431ff39d5240bc11d47c0b7ad0f09ab87 Mon Sep 17 00:00:00 2001
From: Callum Wong <mail@callumwong.com>
Date: Thu, 27 Aug 2026 23:29:11 +1000
Subject: [PATCH 10/10] llama-cpp: switch to multi-model preset, size contexts
 for 12GB VRAM

---
 modules/services/llama-cpp.nix | 36 ++++++++++++++++++++++++++++++----
 1 file changed, 32 insertions(+), 4 deletions(-)

diff --git a/modules/services/llama-cpp.nix b/modules/services/llama-cpp.nix
index 01b6617..f1daf32 100644
--- a/modules/services/llama-cpp.nix
+++ b/modules/services/llama-cpp.nix
@@ -11,6 +11,36 @@
           pkgs,
           ...
         }:
+        let
+          # https://huggingface.co/blog/ggml-org/model-management-in-llamacpp
+          modelsPreset = pkgs.writeText "llama-cpp-models.ini" ''
+            version = 1
+
+            [ling-3.0-tiny]
+            hf-repo = bartowski/Ling-3.0-tiny-GGUF:Q6_K_L
+            ctx-size = 131072
+            load-on-startup = true
+
+            [hy-mt2-1.8b]
+            hf-repo = tencent/Hy-MT2-1.8B-GGUF:Q8_0
+            ctx-size = 131072
+
+            [lfm2.5-2.6b]
+            hf-repo = LiquidAI/LFM2.5-2.6B-GGUF:F16
+            spec-draft-hf = LiquidAI/LFM2.5-2.6B-DSpark-GGUF:F16
+            spec-type = draft-dspark
+            spec-draft-n-max = 10
+            spec-draft-n-min = 0
+            ctx-size = 131072
+
+            # mmproj automatically downloads
+            [unlimited-ocr]
+            hf-repo = sahilchachra/Unlimited-OCR-GGUF:BF16
+            ctx-size = 8192
+            # deterministic output recommended for OCR
+            temp = 0
+          '';
+        in
         {
           services.llama-cpp = {
             enable = true;
@@ -33,14 +63,12 @@
               inherit port;
               host = "0.0.0.0";
 
-              # https://huggingface.co/bartowski/Ling-3.0-tiny-GGUF
-              hf-repo = "bartowski/Ling-3.0-tiny-GGUF:Q6_K_L";
-              alias = "ling-3.0-tiny";
+              models-preset = modelsPreset;
+              models-max = 1;
 
               n-gpu-layers = 999;
               kv-offload = true;
               flash-attn = "on";
-              ctx-size = 32768;
               parallel = 1;
 
               temp = 0.6;