Model aware routing

AGENTS.md

@@ -0,0 +1,34 @@
+# Paddler — Stateful Load Balancer for llama.cpp
+
+## Overview
+
+Load balancer + reverse proxy for llama.cpp servers. Distributes requests by available slots, buffers when all busy.
+
+## Key Files
+
+| File | Purpose |
+|------|---------|
+| `src/main.rs` | CLI (balancer / agent subcommands) |
+| `src/balancer/proxy_service.rs` | Proxy logic, model detection (`--check-model`), upstream peer creation |
+| `src/balancer/upstream_peer_pool.rs` | Peer registry, `use_best_peer()`, slot tracking |
+| `src/balancer/upstream_peer.rs` | Peer struct, `Ord` sort (usable → most idle → least processing) |
+| `src/balancer/request_context.rs` | Per-request context, `take_slot()` / `release_slot()` |
+| `src/agent/monitoring_service.rs` | Periodic llama.cpp `/slots` polling |
+| `src/agent/reporting_service.rs` | Agent → balancer WebSocket keepalive |
+| `src/llamacpp/llamacpp_client.rs` | llama.cpp API client |
+
+## Slot Tracking
+
+- Per-request: `take_slot()` / `release_slot()` update counts immediately on request/response
+- Heartbeat: agent polls llama.cpp every N ms (`--monitoring-interval`, default 10000), reconciles drift
+- Peer sort: usable first, then most idle slots DESC, least processing ASC
+
+## Test Infrastructure
+
+- `tests/integration_tests/` — Rust integration tests (cucumber-style)
+- `tests/scripts/test-model-detection.sh` — bash test suite, 19 scenarios
+  - `PADDLER_PROXY_HOST=<host> PADDLER_MODEL=<model> bash tests/scripts/test-model-detection.sh`
+
+## Makefile Targets
+
+`build` · `test` · `integration_tests` · `shell_tests` · `fmt` · `clean`

Cargo.toml

@@ -9,7 +9,7 @@ documentation = "https://github.com/distantmagic/paddler/blob/main/README.md"
 version = "1.2.1"
 
 [workspace]
-members = ["integration_tests"]
+members = ["tests/integration_tests"]
 
 [dependencies]
 actix = "0.13.5"
@@ -32,6 +32,7 @@ serde_json = "1.0.140"
 tokio = { version = "1.45.1", features = ["full"] }
 tokio-stream = { version = "0.1.17", features = ["sync"] }
 url = { version = "2.5.4", features = ["serde"] }
+regex = "1.11.1"
 
 chrono = { version = "0.4.41", optional = true }
 crossterm = { version = "0.28.1", features = ["event-stream"], optional = true }

Makefile

@@ -31,12 +31,16 @@ clean:
 .PHONY: fmt
 fmt: node_modules
 	./jarmuz-fmt.mjs
-	$(MAKE) -C integration_tests fmt
+	$(MAKE) -C tests/integration_tests fmt
 
 .PHONY: integration_tests
 integration_tests:
 	cargo build
-	$(MAKE) -C integration_tests test
+	$(MAKE) -C tests/integration_tests test
+
+.PHONY: shell_tests
+shell_tests:
+	bash tests/scripts/test-model-detection.sh
 
 .PHONY: test
 test: integration_tests

jarmuz/worker-prettier.mjs

@@ -4,7 +4,7 @@ command(`
   npm exec prettier --
     --plugin=prettier-plugin-organize-imports
     --write
-    integration_tests/tests/fixtures
+    tests/integration_tests/tests/fixtures
     jarmuz
     resources
     *.mjs

resources/ts/components/AgentsList.tsx

@@ -1,5 +1,5 @@
 import clsx from "clsx";
-import React, { CSSProperties } from "react";
+import React, { CSSProperties, useState } from "react";
 
 import { type Agent } from "../schemas/Agent";
 
@@ -9,27 +9,145 @@ import {
   agentUsage,
   agentUsage__progress,
   agentsTable,
+  sortIndicator,
+  sortIndicatorAsc,
+  sortIndicatorDesc,
 } from "./Dashboard.module.css";
 
 function formatTimestamp(timestamp: number): string {
   return new Date(timestamp * 1000).toLocaleString();
 }
 
+type SortColumn =
+  | "name"
+  | "model"
+  | "issues"
+  | "llamacppAddr"
+  | "lastUpdate"
+  | "idleSlots"
+  | "processingSlots";
+
+function getSortIndicator(
+  sortConfig: { key: SortColumn; direction: "ascending" | "descending" },
+  currentKey: SortColumn
+): React.ReactNode {
+  if (sortConfig.key !== currentKey) {
+    return null;
+  }
+  const className = clsx(sortIndicator, sortConfig.direction === "ascending" ? sortIndicatorAsc : sortIndicatorDesc);
+  return (
+    <span className={className}>
+      {sortConfig.direction === "ascending" ? "↑" : "↓"}
+    </span>
+  );
+}
+
 export function AgentsList({ agents }: { agents: Array<Agent> }) {
+  const [sortConfig, setSortConfig] = useState<{
+    key: SortColumn;
+    direction: "ascending" | "descending";
+  }>({ key: "name", direction: "ascending" });
+
+  function sortAgents(agents: Array<Agent>): Array<Agent> {
+    const sortableAgents = [...agents];
+    sortableAgents.sort(function (a, b) {
+      const { key, direction } = sortConfig;
+
+      // Helper function to get comparison value based on column type
+      function getValue(agent: Agent, key: SortColumn): string | number {
+        switch (key) {
+          case "name":
+            return agent.status.agent_name || "";
+          case "model":
+            return agent.status.model || "";
+          case "llamacppAddr":
+            return agent.status.external_llamacpp_addr;
+          case "lastUpdate":
+            return agent.last_update.secs_since_epoch;
+          case "idleSlots":
+            return agent.status.slots_idle;
+          case "processingSlots":
+            return agent.status.slots_processing;
+          default:
+            return "";
+        }
+      }
+
+      // Special handling for issues column
+      if (key === "issues") {
+        const hasIssuesA = a.status.error !== null;
+        const hasIssuesB = b.status.error !== null;
+        if (hasIssuesA !== hasIssuesB) {
+          return direction === "ascending" ? (hasIssuesA ? 1 : -1) : (hasIssuesA ? -1 : 1);
+        }
+        const errorA = a.status.error || "";
+        const errorB = b.status.error || "";
+        if (errorA < errorB) return direction === "ascending" ? -1 : 1;
+        if (errorA > errorB) return direction === "ascending" ? 1 : -1;
+        return 0;
+      }
+
+      const valueA = getValue(a, key);
+      const valueB = getValue(b, key);
+
+      // Handle string comparison
+      if (typeof valueA === "string" && typeof valueB === "string") {
+        if (valueA < valueB) return direction === "ascending" ? -1 : 1;
+        if (valueA > valueB) return direction === "ascending" ? 1 : -1;
+        return 0;
+      }
+
+      // Handle numeric comparison
+      if (typeof valueA === "number" && typeof valueB === "number") {
+        if (valueA < valueB) return direction === "ascending" ? -1 : 1;
+        if (valueA > valueB) return direction === "ascending" ? 1 : -1;
+        return 0;
+      }
+
+      return 0;
+    });
+    return sortableAgents;
+  }
+
+  function requestSort(key: SortColumn) {
+    let direction: "ascending" | "descending" = "ascending";
+    if (sortConfig.key === key && sortConfig.direction === "ascending") {
+      direction = "descending";
+    }
+    setSortConfig({ key, direction });
+  }
+
+  const sortedAgents = sortAgents(agents);
+
   return (
     <table className={agentsTable}>
       <thead>
         <tr>
-          <th>Name</th>
-          <th>Issues</th>
-          <th>Llama.cpp address</th>
-          <th>Last update</th>
-          <th>Idle slots</th>
-          <th>Processing slots</th>
+          <th onClick={function () { requestSort("name"); }}>
+            Name{getSortIndicator(sortConfig, "name")}
+          </th>
+          <th onClick={function () { requestSort("model"); }}>
+            Model{getSortIndicator(sortConfig, "model")}
+          </th>
+          <th onClick={function () { requestSort("issues"); }}>
+            Issues{getSortIndicator(sortConfig, "issues")}
+          </th>
+          <th onClick={function () { requestSort("llamacppAddr"); }}>
+            Llama.cpp address{getSortIndicator(sortConfig, "llamacppAddr")}
+          </th>
+          <th onClick={function () { requestSort("lastUpdate"); }}>
+            Last update{getSortIndicator(sortConfig, "lastUpdate")}
+          </th>
+          <th onClick={function () { requestSort("idleSlots"); }}>
+            Idle slots{getSortIndicator(sortConfig, "idleSlots")}
+          </th>
+          <th onClick={function () { requestSort("processingSlots"); }}>
+            Processing slots{getSortIndicator(sortConfig, "processingSlots")}
+          </th>
         </tr>
       </thead>
       <tbody>
-        {agents.map(function ({
+        {sortedAgents.map(function ({
           agent_id,
           last_update,
           quarantined_until,
@@ -47,13 +165,12 @@ export function AgentsList({ agents }: { agents: Array<Agent> }) {
             quarantined_until;
 
           return (
-            <tr
-              className={clsx(agentRow, {
-                [agentRowError]: hasIssues,
-              })}
-              key={agent_id}
-            >
+              <tr
+                className={clsx(agentRow, hasIssues ? agentRowError : undefined)}
+                key={agent_id}
+              >
               <td>{status.agent_name}</td>
+              <td>{status.model}</td>
               <td>
                 {status.error && (
                   <>

resources/ts/components/Dashboard.module.css

@@ -7,11 +7,29 @@
   th {
     border: 1px solid var(--color-border);
     padding: var(--spacing-base);
+    cursor: pointer;
 
     p + p {
       margin-top: var(--spacing-half);
     }
   }
+
+  th:hover {
+    background-color: var(--color-hover);
+  }
+}
+
+.sortIndicator {
+  margin-left: var(--spacing-half);
+  font-size: 0.8em;
+}
+
+.sortIndicatorAsc {
+  color: var(--color-success);
+}
+
+.sortIndicatorDesc {
+  color: var(--color-error);
 }
 
 .agentRow.agentRowError {

resources/ts/schemas/Agent.ts

@@ -5,6 +5,7 @@ import { StatusUpdateSchema } from "./StatusUpdate";
 export const AgentSchema = z
   .object({
     agent_id: z.string(),
+    model: z.string().nullable(),
     last_update: z.object({
       nanos_since_epoch: z.number(),
       secs_since_epoch: z.number(),

resources/ts/schemas/StatusUpdate.ts

@@ -14,6 +14,7 @@ export const StatusUpdateSchema = z
     is_unexpected_response_status: z.boolean().nullable(),
     slots_idle: z.number(),
     slots_processing: z.number(),
+    model: z.string().nullable(),
   })
   .strict();
 

src/agent/monitoring_service.rs

@@ -23,6 +23,7 @@ pub struct MonitoringService {
     monitoring_interval: Duration,
     name: Option<String>,
     status_update_tx: Sender<Bytes>,
+    check_model: bool, // Store the check_model flag
 }
 
 impl MonitoringService {
@@ -32,13 +33,15 @@ impl MonitoringService {
         monitoring_interval: Duration,
         name: Option<String>,
         status_update_tx: Sender<Bytes>,
+        check_model: bool, // Include the check_model flag
     ) -> Result<Self> {
         Ok(MonitoringService {
             external_llamacpp_addr,
             llamacpp_client,
             monitoring_interval,
             name,
             status_update_tx,
+            check_model,
         })
     }
 
@@ -50,6 +53,15 @@ impl MonitoringService {
             .filter(|slot| slot.is_processing)
             .count();
 
+        let model: Option<String> = if self.check_model {
+            match self.llamacpp_client.get_model().await {
+                Ok(model) => model,
+                Err(_) => None,
+            }
+        } else {
+            Some("".to_string())
+        };
+
         StatusUpdate {
             agent_name: self.name.to_owned(),
             error: slots_response.error,
@@ -63,6 +75,7 @@ impl MonitoringService {
             is_unexpected_response_status: slots_response.is_unexpected_response_status,
             slots_idle: slots_response.slots.len() - slots_processing,
             slots_processing,
+            model,
         }
     }
 
@@ -109,4 +122,4 @@ impl Service for MonitoringService {
     fn threads(&self) -> Option<usize> {
         Some(1)
     }
-}
+}