feat(runtime): add scoped workload identities and agent tools

Dockerfile.dev

@@ -12,4 +12,5 @@ ENV PATH=/usr/local/cargo/bin:$PATH
 RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain stable \
     && rustup target add wasm32-unknown-unknown
 COPY --from=builder /out/druid-dev /usr/local/bin/druid-dev
+COPY scripts/build-ui.sh /usr/local/bin/druid-build-ui.sh
 ENTRYPOINT ["druid-dev"]

Makefile

@@ -15,6 +15,8 @@ WSL_HOST_IP ?= $(word 1,$(shell hostname -I 2>/dev/null))
 DRUID_WORKER_CALLBACK_URL_WINDOWS ?= http://$(WSL_HOST_IP):8083
 DRUID_WATCH_ARGS ?= daemon --runtime kubernetes --listen 127.0.0.1:8081 --public-listen 127.0.0.1:8082 --worker-callback-listen $(DRUID_WORKER_CALLBACK_LISTEN) --worker-callback-url $(DRUID_WORKER_CALLBACK_URL) --unsafe-allow-unauthenticated-management --unsafe-allow-unauthenticated-public --k8s-namespace $(DRUID_K8S_NAMESPACE) --k8s-pull-image $(DRUID_K8S_PULL_IMAGE)
 export DRUID_K8S_UI_S3_BUCKET ?= druid-ui
+export DRUID_WORKLOAD_API_URL ?= http://host.docker.internal:8082
+export DRUID_AUTH_JWKS_URL ?= http://127.0.0.1:3010/v2/jwks
 export DRUID_K8S_UI_S3_PUBLIC_BASE_URL ?= http://127.0.0.1:9000/druid-ui
 export DRUID_K8S_UI_S3_REGION ?= us-east-1
 export DRUID_K8S_UI_S3_ENDPOINT ?= http://host.docker.internal:9000

README.md

@@ -67,6 +67,55 @@ There is also websocket support for stdout. TTY is also supported.
 
 Runtime selection is daemon-only: start the daemon with `druid daemon --runtime docker`, then use `druid` to create, run, and inspect scrolls without passing a runtime. Docker runtime state stays in SQLite under the runtime state directory. Scroll specs and runtime data live together in one runtime root.
 
+Agent tools are exposed through `/:id/mcp` and normal REST resource routes.
+Existing queue, command-run/removal and UI-publication routes are registered once
+and accept either their existing owner authentication or scoped workload credentials.
+Other owner REST endpoints are unchanged.
+
+REST paths below are relative to `/:id/api/v1`. Tool arguments are JSON bodies;
+GET limits use `tail_lines` or `timeout_seconds` query parameters. Workload
+responses use the same result objects as MCP. Existing owner handlers preserve
+their previous response shapes and status codes.
+
+| MCP tool | REST operation |
+| --- | --- |
+| `list_commands` | `GET /commands` |
+| `create_command` | `POST /commands` |
+| `remove_command` | `DELETE /scroll/commands/:command` |
+| `run_command` | `POST /command` |
+| `wait_for_command` | `GET /commands/:command/wait` |
+| `get_queue` | `GET /queue` |
+| `read_logs` | `GET /consoles/:console/logs` |
+| `search_logs` | `POST /consoles/:console/logs/search` |
+| `send_stdin` | `POST /consoles/:console/stdin` |
+| `get_deployment_status` | `GET /status` |
+| `publish_ui` | `POST /ui/packages/:scope/publish` |
+
+`remove_command` requires `commands:remove`, stops the command and removes its
+definition, queue/status and runtime containers. Its console/log access is also
+removed; deployment data and other commands are retained. New OpenHands commands
+request this permission; existing Kubernetes agents require explicit recreation
+to gain it.
+
+Kubernetes authorizes each tool request when it is accepted. An accepted command
+wait may finish after its caller's credential is revoked (at most 30 seconds);
+subsequent requests must pass authentication again.
+
+Commands have no execution IDs. `run_command` acknowledges the command name and
+`wait_for_command` observes its current status. Restarted/replaced runs are not
+distinguished, and delayed status events from older containers may update the
+current command status.
+
+**Docker agent-tool caveat:** HTTP/MCP tool endpoints on the Docker runtime are
+unauthenticated and ignore command permissions. Anyone who can reach them can
+invoke every tool for any deployment, including command creation/execution,
+stdin, log access and UI publication. Docker creates no workload credentials or
+identity registry. Use it only for trusted, single-user local development; keep
+the daemon proxy bound to loopback and do not expose it to untrusted networks or
+multi-tenant workloads. Container workloads may also reach these endpoints.
+Existing authentication on unrelated dashboard, management and worker routes is
+unchanged. Kubernetes retains Pod-bound authentication and per-tool permissions.
+
 Kubernetes runtime support is available with `druid daemon --runtime kubernetes` for in-cluster daemons or out-of-cluster daemons using kubeconfig. It stores daemon scroll state in ConfigMaps, materializes OCI artifacts through `druid worker pull` Jobs, and uses kubelet pod stats for procedure-level traffic checks. See `docs/kubernetes_runtime.md` for kubeconfig, RBAC, and PVC setup.
 
 ## Documentation

api/openapi.yaml

@@ -242,21 +242,28 @@ components:
     PublishUIPackageRequest:
       type: object
       properties:
-        path:
-          type: string
-          description: Optional scroll-root-relative .wasm path. Defaults to private/dist/app.wasm or public/dist/app.wasm.
+        with_build:
+          type: boolean
+          default: false
+          description: Run the scope's declared buildcommand as a dependency before uploading its declared WASM/OpenUI pair.
 
     RuntimeUIPackage:
       type: object
       required:
         - url
         - path
         - updated_at
+        - openui_url
+        - openui_path
       properties:
         url:
           type: string
         path:
           type: string
+        openui_url:
+          type: string
+        openui_path:
+          type: string
         updated_at:
           type: string
           format: date-time

apps/druid-dev/adapters/cli/build_launcher_test.go

@@ -0,0 +1,105 @@
+package cli
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/highcard-dev/daemon/internal/devapi"
+)
+
+func TestBuildLauncherWaitsAndRereadsRotatedCredential(t *testing.T) {
+	if _, err := exec.LookPath("python3"); err != nil {
+		t.Skip("python3 is required by the dev image")
+	}
+	for _, fail := range []bool{false, true} {
+		t.Run(map[bool]string{false: "success", true: "failure"}[fail], func(t *testing.T) {
+			tokenFile := filepath.Join(t.TempDir(), "token")
+			if err := os.WriteFile(tokenFile, []byte("before"), 0600); err != nil {
+				t.Fatal(err)
+			}
+			var waits atomic.Int32
+			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				if r.URL.Path == "/deployment/api/v1/command" {
+					if r.Header.Get("Authorization") != "Bearer before" {
+						t.Error("incorrect admission credential")
+					}
+					var input map[string]string
+					if err := json.NewDecoder(r.Body).Decode(&input); err != nil || input["command"] != "build_ui" {
+						t.Error("wrong command payload")
+					}
+					if err := os.WriteFile(tokenFile, []byte("after"), 0600); err != nil {
+						t.Error(err)
+					}
+					w.Write([]byte(`{"accepted":true}`))
+					return
+				}
+				if r.URL.Path != "/deployment/api/v1/commands/build_ui/wait" || r.Header.Get("Authorization") != "Bearer after" {
+					t.Error("wait did not use rotated credential or configured endpoint")
+				}
+				state := "done"
+				if waits.Add(1) == 1 {
+					state = "running"
+				} else if fail {
+					state = "error"
+				}
+				json.NewEncoder(w).Encode(map[string]any{"status": state})
+			}))
+			defer server.Close()
+			command := exec.Command("sh", "../../../../scripts/build-ui.sh", "build_ui")
+			command.Env = append(os.Environ(), "DRUID_API_URL="+server.URL+"/deployment", "DRUID_TOKEN_FILE="+tokenFile)
+			output, err := command.CombinedOutput()
+			if (err != nil) != fail || waits.Load() != 2 {
+				t.Fatalf("err=%v waits=%d output=%s", err, waits.Load(), output)
+			}
+			if strings.Contains(string(output), "before") || strings.Contains(string(output), "after") {
+				t.Fatal("credential leaked")
+			}
+		})
+	}
+}
+
+func TestWatcherRerunsFiniteBuildWithoutWatchingItsOwnArtifacts(t *testing.T) {
+	root := t.TempDir()
+	if err := os.Mkdir(filepath.Join(root, "private"), 0755); err != nil {
+		t.Fatal(err)
+	}
+	server := newDevServer(root, devAuth{}, "private/custom.wasm", "private/layout.openui")
+	defer server.stopWatch()
+	err := server.startWatch(devapi.WatchModeRequest{WatchPaths: []string{"private"}, WorkingDirectory: ".", Command: []string{"sh", "-ec", "printf x >> count; printf wasm > private/custom.wasm; printf ui > private/layout.openui"}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	waitCount := func(want int) {
+		t.Helper()
+		deadline := time.Now().Add(3 * time.Second)
+		for time.Now().Before(deadline) {
+			data, _ := os.ReadFile(filepath.Join(root, "count"))
+			if len(data) == want {
+				return
+			}
+			time.Sleep(10 * time.Millisecond)
+		}
+		t.Fatalf("did not reach %d builds", want)
+	}
+	waitCount(1)
+	time.Sleep(700 * time.Millisecond)
+	data, _ := os.ReadFile(filepath.Join(root, "count"))
+	if len(data) != 1 {
+		t.Fatal("artifacts triggered another build")
+	}
+	if err := os.WriteFile(filepath.Join(root, "private", "source.rs"), []byte("source"), 0644); err != nil {
+		t.Fatal(err)
+	}
+	waitCount(2)
+	if string(server.watchResponse().Status) != statusReady {
+		t.Fatal("finite build disabled watcher")
+	}
+}

apps/druid-dev/adapters/cli/root.go

@@ -43,6 +43,7 @@ type options struct {
 	watchPaths     []string
 	watchWorkdir   string
 	watchCommand   []string
+	watchOutputs   []string
 }
 
 func NewRootCommand() *cobra.Command {
@@ -63,6 +64,7 @@ func NewRootCommand() *cobra.Command {
 	cmd.Flags().StringArrayVar(&opt.watchPaths, "watch-path", nil, "Runtime-relative path to watch (repeatable)")
 	cmd.Flags().StringVar(&opt.watchWorkdir, "watch-working-directory", ".", "Runtime-relative working directory for the watcher command")
 	cmd.Flags().StringArrayVar(&opt.watchCommand, "watch-command", nil, "Watcher command argument (repeatable, preserves argument boundaries)")
+	cmd.Flags().StringArrayVar(&opt.watchOutputs, "watch-output", nil, "Built artifact path to notify without triggering another build")
 	_ = cmd.MarkFlagRequired("root")
 	return cmd
 }
@@ -85,7 +87,7 @@ func run(ctx context.Context, opt options) error {
 			return err
 		}
 	}
-	server := newDevServer(root, auth)
+	server := newDevServer(root, auth, opt.watchOutputs...)
 	watchRequest, err := startupWatchRequest(opt)
 	if err != nil {
 		return err
@@ -145,16 +147,16 @@ type devAuth struct {
 	ownerID   string
 }
 
-type noopQueue struct{}
+type buildQueue struct{ run func() error }
 
-func (noopQueue) AddTempItem(string) error                     { return nil }
-func (noopQueue) AddTempItemWithWait(string) error             { return nil }
-func (noopQueue) GetQueue() map[string]domain.ScrollLockStatus { return nil }
+func (q buildQueue) AddTempItem(string) error                     { return nil }
+func (q buildQueue) AddTempItemWithWait(string) error             { return q.run() }
+func (q buildQueue) GetQueue() map[string]domain.ScrollLockStatus { return nil }
 
 type noopScroll struct{}
 
 func (noopScroll) GetCommand(string) (*domain.CommandInstructionSet, error) {
-	return nil, fmt.Errorf("watcher does not run daemon commands")
+	return &domain.CommandInstructionSet{}, nil
 }
 func (noopScroll) GetCurrent() *domain.Scroll { return nil }
 func (noopScroll) GetFile() *domain.File      { return &domain.File{} }
@@ -178,10 +180,18 @@ type devServer struct {
 	config  watchConfig
 	cancel  context.CancelFunc
 	watchID uint64
+	build   func() error
 }
 
-func newDevServer(root string, auth devAuth) *devServer {
-	return &devServer{root: root, auth: auth, watch: coreservices.NewDevService(noopQueue{}, noopScroll{}), status: statusDisabled}
+func newDevServer(root string, auth devAuth, outputs ...string) *devServer {
+	server := &devServer{root: root, auth: auth, status: statusDisabled}
+	server.watch = coreservices.NewDevService(buildQueue{run: func() error {
+		server.mu.RLock()
+		build := server.build
+		server.mu.RUnlock()
+		return build()
+	}}, noopScroll{}, outputs...)
+	return server
 }
 
 func newApp(server *devServer) *fiber.App {
@@ -317,54 +327,36 @@ func (s *devServer) startWatch(request devapi.WatchModeRequest) error {
 	s.watchID++
 	watchID := s.watchID
 	s.mu.Unlock()
-	if err := s.watch.StartWatching(s.root, paths...); err != nil {
-		s.setWatchError(err)
-		logger.Log().Error("Failed to start Druid development watcher", zap.Error(err))
+	ctx, cancel := context.WithCancel(context.Background())
+	s.mu.Lock()
+	s.build = func() error {
+		child := exec.CommandContext(ctx, request.Command[0], request.Command[1:]...)
+		child.Dir = workdir
+		child.Stdout, child.Stderr = os.Stdout, os.Stderr
+		err := child.Run()
+		s.mu.Lock()
+		if s.watchID == watchID && ctx.Err() == nil {
+			s.status, s.lastErr = statusReady, ""
+			if err != nil {
+				s.status, s.lastErr = statusError, err.Error()
+			}
+		}
+		s.mu.Unlock()
 		return err
 	}
-	ctx, cancel := context.WithCancel(context.Background())
-	child := exec.CommandContext(ctx, request.Command[0], request.Command[1:]...)
-	child.Dir = workdir
-	child.Stdout, child.Stderr = os.Stdout, os.Stderr
-	if err := child.Start(); err != nil {
+	s.mu.Unlock()
+	if err := s.watch.SetHotReloadCommands([]string{"build"}); err != nil {
 		cancel()
-		_ = s.watch.StopWatching()
-		s.setWatchError(err)
-		logger.Log().Error("Failed to start Druid development build process", zap.Error(err))
 		return err
 	}
-	logger.Log().Info("Druid development build process started",
-		zap.Int("pid", child.Process.Pid),
-		zap.String("working_directory", workdir),
-		zap.Strings("command", request.Command),
-	)
 	s.mu.Lock()
-	s.cancel = cancel
-	s.status = statusReady
+	s.cancel, s.status = cancel, statusReady
 	s.mu.Unlock()
-	go func() {
-		err := child.Wait()
-		s.mu.Lock()
-		if s.watchID == watchID {
-			s.cancel = nil
-			if err != nil && ctx.Err() == nil {
-				s.status = statusError
-				s.lastErr = err.Error()
-			} else {
-				s.status = statusDisabled
-				s.lastErr = ""
-			}
-		}
-		s.mu.Unlock()
-		if err != nil && ctx.Err() == nil {
-			logger.Log().Error("Druid development build process exited with an error", zap.Error(err))
-		} else {
-			logger.Log().Info("Druid development build process stopped")
-		}
-		if ctx.Err() == nil {
-			_ = s.watch.StopWatching()
-		}
-	}()
+	if err := s.watch.StartWatching(s.root, paths...); err != nil {
+		cancel()
+		s.setWatchError(err)
+		return err
+	}
 	return nil
 }
 

apps/druid/adapters/cli/client/procedure_test.go

@@ -156,6 +156,6 @@ func (f *fakeProcedureDaemon) GetScrollUIPackages(ctx context.Context, id string
 	return map[string]api.RuntimeUIPackage{}, nil
 }
 
-func (f *fakeProcedureDaemon) PublishScrollUIPackage(ctx context.Context, id string, scope string, path string) (*api.RuntimeScroll, error) {
+func (f *fakeProcedureDaemon) PublishScrollUIPackage(ctx context.Context, id string, scope string, withBuild bool) (*api.RuntimeScroll, error) {
 	return &api.RuntimeScroll{Id: id}, nil
 }

apps/druid/adapters/cli/client/register.go

@@ -24,7 +24,7 @@ type RuntimeDaemon interface {
 	GetScrollRoutingTargets(ctx context.Context, id string) ([]api.RuntimeRoutingTarget, error)
 	ApplyScrollRouting(ctx context.Context, id string, assignments []api.RuntimeRouteAssignment) (*api.RuntimeScroll, error)
 	GetScrollUIPackages(ctx context.Context, id string) (map[string]api.RuntimeUIPackage, error)
-	PublishScrollUIPackage(ctx context.Context, id string, scope string, path string) (*api.RuntimeScroll, error)
+	PublishScrollUIPackage(ctx context.Context, id string, scope string, withBuild bool) (*api.RuntimeScroll, error)
 }
 
 type Config struct {

apps/druid/adapters/cli/client/routing_publish_test.go

@@ -201,6 +201,6 @@ func (f *fakeRoutingDaemon) GetScrollUIPackages(ctx context.Context, id string)
 	return map[string]api.RuntimeUIPackage{}, nil
 }
 
-func (f *fakeRoutingDaemon) PublishScrollUIPackage(ctx context.Context, id string, scope string, path string) (*api.RuntimeScroll, error) {
+func (f *fakeRoutingDaemon) PublishScrollUIPackage(ctx context.Context, id string, scope string, withBuild bool) (*api.RuntimeScroll, error) {
 	return &api.RuntimeScroll{Id: id, Status: api.RuntimeScrollStatusCreated}, nil
 }