GPU VM

hosts/nixos/liz/_windows-vm.nix

@@ -164,7 +164,7 @@ let
     '';
   };
 
-  # QMP powerdown lets Windows unmount NTFS during host shutdown.
+  # Ask Windows to unmount cleanly, then keep ExecStop alive until QEMU exits.
   shutdown = pkgs.writeShellApplication {
     name = "windows-vm-shutdown";
     runtimeInputs = [ pkgs.socat ];
@@ -174,6 +174,12 @@ let
         '{"execute":"qmp_capabilities"}' \
         '{"execute":"system_powerdown"}' \
         | socat - "UNIX-CONNECT:${qmpSocket}" >/dev/null || true
+
+      mainPid="''${MAINPID:-}"
+      [ -n "$mainPid" ] || exit 0
+      while kill -0 "$mainPid" 2>/dev/null; do
+        sleep 1
+      done
     '';
   };
 in

hosts/nixos/liz/default.nix

@@ -29,6 +29,10 @@ in
           n = 2;
           hostname = "vm-coder";
         })
+        (microvmLib.mkHostNetworking {
+          n = 3;
+          hostname = "vm-gpu";
+        })
       ]
       ++ (with nixos; [
         uefi
@@ -71,28 +75,51 @@ in
 
       # https://microvm-nix.github.io/microvm.nix/declarative.html#fully-declarative
 
-      microvm.vms.vm-gallery = {
-        config.imports = [
-          nixos.base
-          nixos.global
-          nixos."hosts/nixos/vm-gallery"
-        ];
-        # nixos.base sets nixpkgs.config, which nixpkgs forbids once pkgs is
-        # instantiated externally; force config-mode's eval-config to build
-        # its own pkgs instead of reusing liz's already-built instance.
-        pkgs = null;
-        restartIfChanged = true;
+      microvm.vms = {
+        vm-gallery = {
+          config.imports = [
+            nixos.base
+            nixos.global
+            nixos."hosts/nixos/vm-gallery"
+          ];
+          # nixos.base sets nixpkgs.config, which nixpkgs forbids once pkgs is
+          # instantiated externally; force config-mode's eval-config to build
+          # its own pkgs instead of reusing liz's already-built instance.
+          pkgs = null;
+          restartIfChanged = true;
+        };
+
+        vm-coder = {
+          config.imports = [
+            nixos.base
+            nixos.global
+            nixos."hosts/nixos/vm-coder"
+          ];
+          pkgs = null;
+          nixpkgs = inputs.unstable;
+          restartIfChanged = true;
+        };
+
+        vm-gpu = {
+          config.imports = [
+            nixos.base
+            nixos.global
+            nixos."hosts/nixos/vm-gpu"
+          ];
+          pkgs = null;
+          nixpkgs = inputs.unstable;
+          restartIfChanged = true;
+        };
       };
 
-      microvm.vms.vm-coder = {
-        config.imports = [
-          nixos.base
-          nixos.global
-          nixos."hosts/nixos/vm-coder"
-        ];
-        pkgs = null;
-        nixpkgs = inputs.unstable;
-        restartIfChanged = true;
+      # The ordering dependency makes either guest stop completely before the
+      # other starts when systemd resolves their mutual conflict.
+      systemd.services = {
+        windows-vm = {
+          conflicts = [ "microvm@vm-gpu.service" ];
+          after = [ "microvm@vm-gpu.service" ];
+        };
+        "microvm@vm-gpu".conflicts = [ "windows-vm.service" ];
       };
 
       _module.args.sshKeys = keys.callum;

hosts/nixos/vm-gpu/default.nix

@@ -0,0 +1,73 @@
+{ config, microvmLib, ... }:
+let
+  inherit (config.flake.modules) nixos;
+  inherit (config.flake) keys;
+in
+{
+  flake.nixpkgs.vm-gpu = "unstable";
+
+  flake.modules.nixos."hosts/nixos/vm-gpu" =
+    { pkgs, ... }:
+    {
+      imports = [
+        (microvmLib.mkGuestModule {
+          n = 3;
+          hostname = "vm-gpu";
+        })
+      ]
+      ++ (with nixos; [
+        persistence
+
+        ssh
+
+        llama-cpp
+      ]);
+
+      system.stateVersion = "26.05";
+
+      microvm = {
+        mem = 4096;
+        vcpu = 4;
+        devices = [
+          {
+            bus = "pci";
+            path = "0000:08:00.0";
+          }
+          {
+            bus = "pci";
+            path = "0000:08:00.1";
+          }
+        ];
+        shares = [
+          {
+            tag = "work";
+            source = "/scratch/gpu";
+            mountPoint = "/work";
+            proto = "virtiofs";
+          }
+        ];
+      };
+
+      services.xserver.videoDrivers = [ "nvidia" ];
+      hardware = {
+        graphics.enable = true;
+        nvidia = {
+          modesetting.enable = false;
+          open = true;
+        };
+      };
+
+      environment = {
+        systemPackages = [
+          pkgs.ffmpeg-full
+          pkgs.nvtopPackages.nvidia
+        ];
+      };
+
+      systemd.tmpfiles.rules = [
+        "d /work/llama-cache 0755 root root -"
+      ];
+
+      users.users.root.openssh.authorizedKeys.keys = keys.callum;
+    };
+}

modules/flake/checks.nix

@@ -19,6 +19,7 @@ in
       "shama"
       "vm-coder"
       "vm-gallery"
+      "vm-gpu"
     ];
 
     flake.checks.${system} = lib.genAttrs config.flake.ciHosts (

modules/services/llama-cpp.nix

@@ -1,57 +1,108 @@
 {
-  flake.modules.nixos.llama-cpp =
+  flake.modules.nixos =
+    let
+      port = 9931;
+    in
     {
-      config,
-      lib,
-      pkgs,
-      ...
-    }:
-    {
-      services.llama-cpp = {
-        enable = false;
-        package = pkgs.llama-cpp.override { cudaSupport = true; };
-        settings = {
-          host = "0.0.0.0";
-          port = 8080;
-
-          hf-repo = "unsloth/Qwen3.6-35B-A3B-MTP-GGUF:UD-Q6_K_XL";
-          alias = "qwen3.6-35b-a3b";
-
-          n-cpu-moe = 37;
-          n-gpu-layers = 99;
-          kv-offload = true;
-          threads = 12;
-          threads-batch = 12;
-          ctx-size = 262144;
-          flash-attn = "on";
-          cache-type-k = "q8_0";
-          cache-type-v = "q8_0";
-          spec-draft-type-k = "q8_0";
-          spec-draft-type-v = "q8_0";
-          parallel = 1;
-
-          # https://unsloth.ai/docs/models/qwen3.6#llama.cpp-mtp-guide
-          temp = 0.6;
-          top-p = 0.95;
-          top-k = 20;
-          min-p = 0.00;
-          spec-type = "draft-mtp";
-          spec-draft-n-max = 2;
-        };
-      };
+      llama-cpp =
+        {
+          config,
+          lib,
+          pkgs,
+          ...
+        }:
+        let
+          # https://huggingface.co/blog/ggml-org/model-management-in-llamacpp
+          modelsPreset = pkgs.writeText "llama-cpp-models.ini" ''
+            version = 1
 
-      systemd.services.llama-cpp = lib.mkIf config.services.llama-cpp.enable {
-        unitConfig = {
-          Wants = [ "systemd-modules-load.service" ];
-          After = [ "systemd-modules-load.service" ];
-        };
+            [ling-3.0-tiny]
+            hf-repo = bartowski/Ling-3.0-tiny-GGUF:Q6_K_L
+            ctx-size = 131072
+            load-on-startup = true
+
+            [hy-mt2-1.8b]
+            hf-repo = tencent/Hy-MT2-1.8B-GGUF:Q8_0
+            ctx-size = 131072
+
+            [lfm2.5-2.6b]
+            hf-repo = LiquidAI/LFM2.5-2.6B-GGUF:F16
+            spec-draft-hf = LiquidAI/LFM2.5-2.6B-DSpark-GGUF:F16
+            spec-type = draft-dspark
+            spec-draft-n-max = 10
+            spec-draft-n-min = 0
+            ctx-size = 131072
+
+            # mmproj automatically downloads
+            [unlimited-ocr]
+            hf-repo = sahilchachra/Unlimited-OCR-GGUF:BF16
+            ctx-size = 8192
+            # deterministic output recommended for OCR
+            temp = 0
+          '';
+        in
+        {
+          services.llama-cpp = {
+            enable = true;
+            package = (pkgs.llama-cpp.override { cudaSupport = true; }).overrideAttrs {
+              # drop once nixpkgs bumps past b10472
+              version = "10472";
+              src = pkgs.fetchFromGitHub {
+                owner = "ggml-org";
+                repo = "llama.cpp";
+                tag = "b10472";
+                hash = "sha256-re0WlafJUDZOPNfIq2ECRSctdrDFVc0fXb5iSd7gDR8=";
+                leaveDotGit = true;
+                postFetch = ''
+                  git -C "$out" rev-parse --short HEAD > $out/COMMIT
+                  find "$out" -name .git -print0 | xargs -0 rm -rf
+                '';
+              };
+            };
+            settings = {
+              inherit port;
+              host = "0.0.0.0";
 
-        serviceConfig = {
-          ExecStartPre = "${pkgs.bash}/bin/bash -c 'until ${config.hardware.nvidia.package.bin}/bin/nvidia-smi -L >/dev/null 2>&1; do ${pkgs.coreutils}/bin/sleep 1; done'";
-          Environment = lib.mkForce [
-            "LLAMA_CACHE=/var/lib/llama-cpp/cache"
-          ];
-          TimeoutStartSec = "infinity";
+              models-preset = modelsPreset;
+              models-max = 1;
+
+              n-gpu-layers = 999;
+              kv-offload = true;
+              flash-attn = "on";
+              parallel = 1;
+
+              temp = 0.6;
+              top-p = 0.95;
+              top-k = 20;
+              min-p = 0.00;
+            };
+          };
+
+          networking.firewall.allowedTCPPorts = [ port ];
+
+          systemd.services.llama-cpp = lib.mkIf config.services.llama-cpp.enable {
+            unitConfig = {
+              Wants = [ "systemd-modules-load.service" ];
+              After = [ "systemd-modules-load.service" ];
+            };
+
+            serviceConfig = {
+              ExecStartPre = "${pkgs.bash}/bin/bash -c 'until ${config.hardware.nvidia.package.bin}/bin/nvidia-smi -L >/dev/null 2>&1; do ${pkgs.coreutils}/bin/sleep 1; done'";
+              TimeoutStartSec = "infinity";
+              DynamicUser = lib.mkForce false;
+              User = "root";
+              Environment = [ "LLAMA_CACHE=/work/llama-cache" ];
+              ReadWritePaths = [ "/work/llama-cache" ];
+            };
+          };
+        };
+      gateway = {
+        modules.gateway.services.llama-cpp = {
+          name = "llama.cpp";
+          domainName = "llama";
+          iconUrl = "https://cdn.jsdelivr.net/gh/selfhst/icons/svg/llama-cpp.svg";
+          addr = "10.0.0.4:${toString port}";
+          category = "Development";
         };
       };
     };