Go back to top-level labels to properly support Swarm worker nodes

README.md

@@ -94,10 +94,13 @@ services:
       - /dev/nvidia0:/dev/nvidia0
       - /dev/nvidiactl:/dev/nvidiactl
       - /dev/nvidia-uvm:/dev/nvidia-uvm
+    # Use top-level labels: so the daemon can read them on both manager and
+    # worker nodes. deploy.labels: are service-level metadata only visible to
+    # the Docker API on manager nodes.
+    labels:
+      swarm-device-access.enable: "true"
+      swarm-device-access.device-allow: "/dev/nvidia*"
     deploy:
-      labels:
-        swarm-device-access.enable: "true"
-        swarm-device-access.device-allow: "/dev/nvidia*"
       mode: replicated
       replicas: 1
 ```
@@ -150,11 +153,26 @@ Consumer services opt in and narrow their allowed device set with labels:
 | `swarm-device-access.device-allow` | Comma-separated globs | Allow only matching `/dev/...` paths. Empty means inherit.     |
 | `swarm-device-access.device-deny`  | Comma-separated globs | Deny matching `/dev/...` paths. Deny overrides allow.          |
 
-Declare these labels under `deploy.labels:` (the Swarm service spec — the
-natural home for service-level metadata, alongside Traefik / Homepage / other
-label-driven tooling). The daemon also reads top-level `labels:` if you need to
-override a service-wide value on a single task; per-container values win on
-conflict.
+Declare these labels under top-level `labels:` in your service definition.
+Docker copies top-level labels into each task container, so the daemon can read
+them on every node — including worker nodes.
+
+> **Note:** Do not use `deploy.labels:` for these labels. `deploy.labels:` are
+> Swarm service metadata that only the Docker Swarm API (manager nodes) can
+> read. The daemon cannot see them on worker nodes and will warn you if it
+> detects them on a manager.
+
+### Worker and Manager Nodes
+
+The daemon auto-detects its role at startup by querying the local Docker API.
+On **worker nodes** it skips the Swarm service inspect entirely (no spurious
+warnings). On **manager nodes** it additionally reads service-level metadata
+for backward compatibility — but emits a warning if it finds
+`swarm-device-access.*` labels under `deploy.labels:`, advising you to move
+them to top-level `labels:`.
+
+If the node role changes (promote/demote), restart the daemon so it re-detects
+its role.
 
 If you bind-mount a directory (for example `source: /dev/dri`), the
 `device-allow` glob is evaluated **per child node** inside that directory —

cmd/swarm-device-access/main.go

@@ -111,6 +111,23 @@ func run() int {
 
 	recorder := observability.NewRecorder()
 
+	isSwarmManager := false
+
+	nodeInfo, infoErr := cli.Info(rootCtx)
+	if infoErr != nil {
+		log.Warn(
+			"could not query docker info; assuming worker node (service-label inspection disabled)",
+			"err",
+			infoErr,
+		)
+	} else {
+		isSwarmManager = nodeInfo.Swarm.ControlAvailable
+		log.Info("swarm role detected",
+			"manager", isSwarmManager,
+			"local_node_state", nodeInfo.Swarm.LocalNodeState,
+		)
+	}
+
 	// Start optional observability servers before the main loop so they are
 	// reachable during startup enumeration.
 	if *metricsAddr != "" {
@@ -122,11 +139,12 @@ func run() int {
 	}
 
 	proc := &processor.Processor{
-		Inspector: cli,
-		Cfg:       store,
-		Metrics:   recorder,
-		HostRoot:  hostRootPath,
-		ProcRoot:  "/",
+		Inspector:      cli,
+		Cfg:            store,
+		Metrics:        recorder,
+		HostRoot:       hostRootPath,
+		ProcRoot:       "/",
+		IsSwarmManager: isSwarmManager,
 	}
 
 	runErr := daemon.Run(rootCtx, daemon.Options{

docs/architecture.md

@@ -117,10 +117,11 @@ The daemon uses a two-level policy:
 
 **Per-container policy** (Docker labels):
 
-Declare these labels under `deploy.labels:` in your Swarm stack file — that is
-the service spec, the natural home alongside Traefik / Homepage / other
-label-driven tooling. The daemon also reads top-level `labels:` when present;
-per-container (task-level) values win on conflict.
+Declare these labels under top-level `labels:` in your Swarm stack file.
+Docker copies top-level labels into each task container so the daemon can read
+them on both manager and worker nodes. Do **not** use `deploy.labels:` — those
+are Swarm service metadata only accessible via the manager API; the daemon
+cannot see them on worker nodes and will warn if it finds them on a manager.
 
 | Label                              | Description                                                    |
 |------------------------------------|----------------------------------------------------------------|

examples/audio-passthrough/docker-compose.yaml

@@ -15,10 +15,11 @@ services:
       # /dev/snd is a directory; the daemon walks it and applies a rule per device.
       - /dev/snd:/dev/snd
     command: [ "aplay", "-l" ]
+    # Use top-level labels: — deploy.labels: are only visible on manager nodes.
+    labels:
+      swarm-device-access.enable: "true"
+      swarm-device-access.device-allow: "/dev/snd/*"
     deploy:
-      labels:
-        swarm-device-access.enable: "true"
-        swarm-device-access.device-allow: "/dev/snd/*"
       mode: replicated
       replicas: 1
   swarm-device-access:

examples/gpu-passthrough/docker-compose.yaml

@@ -19,11 +19,12 @@ services:
       - /dev/nvidiactl:/dev/nvidiactl
       - /dev/nvidia-uvm:/dev/nvidia-uvm
     command: [ "nvidia-smi" ]
+    # Use top-level labels: — deploy.labels: are only visible on manager nodes.
+    # Required when daemon runs in opt-in mode (the default).
+    labels:
+      swarm-device-access.enable: "true"
+      swarm-device-access.device-allow: "/dev/nvidia*"
     deploy:
-      labels:
-        # Required when daemon runs in opt-in mode (the default).
-        swarm-device-access.enable: "true"
-        swarm-device-access.device-allow: "/dev/nvidia*"
       mode: replicated
       replicas: 1
   swarm-device-access:

examples/smartctl-exporter/docker-compose.yaml

@@ -0,0 +1,70 @@
+# smartctl-exporter passthrough example for Docker Swarm
+#
+# Runs smartctl-exporter as a global service so it can read disk health
+# metrics from /dev/sg* and /dev/nvme* on every storage node.
+# The swarm-device-access daemon injects cgroup BPF device-allow rules at
+# container start, replacing the privileged: true that Compose would use.
+#
+# Prerequisites on each node:
+#   - /dev/sd*, /dev/nvme*, or /dev/sg* devices present
+#   - /run/udev populated by the host udevd
+#   - swarm-device-access daemon running (global service below)
+#
+# Deploy:
+#   docker stack deploy -c docker-compose.yaml smartctl-example
+
+services:
+  smartctl-exporter:
+    image: prometheuscommunity/smartctl-exporter:v0.14.0
+    volumes:
+      # Bind-mount the full /dev tree; daemon will allow each disk device.
+      - /dev:/dev:ro
+      - /run/udev:/run/udev:ro
+    command:
+      - --smartctl.interval=5m
+      - --smartctl.rescan=30m
+      - --web.listen-address=:9633
+      - --smartctl.device-exclude=^/dev/(loop|md|zram)
+    user: "0:0"
+    cap_add:
+      - SYS_RAWIO
+      - SYS_ADMIN
+    security_opt:
+      - no-new-privileges=true
+    # Use top-level labels: — deploy.labels: are only visible on manager nodes.
+    labels:
+      swarm-device-access.enable: "true"
+      swarm-device-access.device-allow: "/dev/sd*,/dev/nvme*,/dev/sg*,/dev/disk/*"
+    deploy:
+      mode: global
+
+  swarm-device-access:
+    image: docker:29
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+    command:
+      - run
+      - -i
+      - --rm
+      - --name=swarm-device-access
+      - --privileged
+      - --cgroupns=host
+      - --pid=host
+      - --userns=host
+      - -v
+      - /sys:/host/sys
+      - -v
+      - /var/run/docker.sock:/var/run/docker.sock
+      - -v
+      - /dev:/dev
+      # Enable reload-after-daemon-reload (optional):
+      # - -v
+      # - /run/dbus/system_bus_socket:/run/dbus/system_bus_socket
+      # Restrict to disk devices only (optional):
+      # - -device-allow=/dev/sd*,/dev/nvme*,/dev/sg*,/dev/disk/*
+      - ghcr.io/leinardi/swarm-device-access:latest
+    entrypoint: docker
+    deploy:
+      mode: global
+      restart_policy:
+        condition: any

examples/usb-passthrough/docker-compose.yaml

@@ -44,9 +44,10 @@ services:
       # Bind-mount the entire USB bus; the daemon will apply rules to each device.
       - /dev/bus/usb:/dev/bus/usb
     command: [ "lsusb" ]
+    # Use top-level labels: — deploy.labels: are only visible on manager nodes.
+    labels:
+      swarm-device-access.enable: "true"
+      swarm-device-access.device-allow: "/dev/bus/usb/*"
     deploy:
-      labels:
-        swarm-device-access.enable: "true"
-        swarm-device-access.device-allow: "/dev/bus/usb/*"
       mode: replicated
       replicas: 1

examples/v4l2-passthrough/docker-compose.yaml

@@ -52,9 +52,10 @@ services:
       - -t
       - "5"
       - /output/capture.mp4
+    # Use top-level labels: — deploy.labels: are only visible on manager nodes.
+    labels:
+      swarm-device-access.enable: "true"
+      swarm-device-access.device-allow: "/dev/video*"
     deploy:
-      labels:
-        swarm-device-access.enable: "true"
-        swarm-device-access.device-allow: "/dev/video*"
       mode: replicated
       replicas: 1

internal/policy/policy.go

@@ -273,3 +273,23 @@ func UnknownLabels(labels map[string]string) []string {
 
 	return unknown
 }
+
+// KnownLabels returns a sorted list of swarm-device-access.* keys from labels
+// that are recognized by this package (i.e. the inverse of UnknownLabels).
+func KnownLabels(labels map[string]string) []string {
+	var known []string
+
+	for k := range labels {
+		if _, ok := knownLabels[k]; ok {
+			known = append(known, k)
+		}
+	}
+
+	if len(known) == 0 {
+		return nil
+	}
+
+	sort.Strings(known)
+
+	return known
+}

internal/processor/processor.go

@@ -62,11 +62,12 @@ type deviceRuleKey struct {
 // host root (typically "/host"). ProcRoot is used for /proc lookups ("/" in
 // production, temp dir in tests).
 type Processor struct {
-	Inspector DockerInspector
-	Cfg       *config.Store
-	Metrics   *observability.Recorder
-	HostRoot  string
-	ProcRoot  string
+	Inspector      DockerInspector
+	Cfg            *config.Store
+	Metrics        *observability.Recorder
+	HostRoot       string
+	ProcRoot       string
+	IsSwarmManager bool
 }
 
 // ProcessContainer inspects a container and applies cgroup BPF device-allow
@@ -93,37 +94,7 @@ func (p *Processor) ProcessContainer(ctx context.Context, containerID string) er
 		containerLabels = info.Config.Labels
 	}
 
-	var (
-		serviceLabels map[string]string
-		svc           swarm.Service
-	)
-
-	if serviceID := containerLabels[swarmServiceIDLabel]; serviceID != "" {
-		var svcErr error
-
-		svc, _, svcErr = p.Inspector.ServiceInspectWithRaw(
-			ctx,
-			serviceID,
-			swarm.ServiceInspectOptions{},
-		)
-		if svcErr != nil {
-			log.Warn("could not inspect parent service; using container labels only",
-				"id", containerID,
-				"service_id", serviceID,
-				"err", svcErr,
-			)
-		} else {
-			serviceLabels = svc.Spec.Labels
-
-			for _, unknownKey := range policy.UnknownLabels(serviceLabels) {
-				log.Warn("unrecognized swarm-device-access label on parent service",
-					"id", containerID,
-					"service_id", serviceID,
-					"label", unknownKey,
-				)
-			}
-		}
-	}
+	svc, serviceLabels := p.resolveServiceLabels(ctx, containerID, containerLabels)
 
 	effectiveLabels := policy.MergeLabels(serviceLabels, containerLabels)
 
@@ -236,6 +207,69 @@ func (p *Processor) ProcessContainer(ctx context.Context, containerID string) er
 	return errors.Join(allErrs...)
 }
 
+// resolveServiceLabels fetches parent service labels on manager nodes. On
+// worker nodes (IsSwarmManager=false) it returns immediately with zero values
+// so ProcessContainer can continue using container-level labels only.
+func (p *Processor) resolveServiceLabels(
+	ctx context.Context,
+	containerID string,
+	containerLabels map[string]string,
+) (svc swarm.Service, serviceLabels map[string]string) {
+	log := logger.L()
+
+	if !p.IsSwarmManager {
+		return svc, nil
+	}
+
+	serviceID := containerLabels[swarmServiceIDLabel]
+	if serviceID == "" {
+		return svc, nil
+	}
+
+	var svcErr error
+
+	svc, _, svcErr = p.Inspector.ServiceInspectWithRaw(
+		ctx,
+		serviceID,
+		swarm.ServiceInspectOptions{},
+	)
+	if svcErr != nil {
+		log.Warn("could not inspect parent service; using container labels only",
+			"id", containerID,
+			"service_id", serviceID,
+			"err", svcErr,
+		)
+
+		return swarm.Service{}, nil
+	}
+
+	serviceLabels = svc.Spec.Labels
+
+	for _, unknownKey := range policy.UnknownLabels(serviceLabels) {
+		log.Warn("unrecognized swarm-device-access label on parent service",
+			"id", containerID,
+			"service_id", serviceID,
+			"label", unknownKey,
+		)
+	}
+
+	for _, knownKey := range policy.KnownLabels(serviceLabels) {
+		log.Warn(
+			"swarm-device-access label set via deploy.labels on parent service; move to top-level labels: so worker nodes can read it",
+			"id",
+			containerID,
+			"service_id",
+			serviceID,
+			"service",
+			svc.Spec.Name,
+			"label",
+			knownKey,
+		)
+	}
+
+	return svc, serviceLabels
+}
+
 // applyRulesToCgroup logs and (unless dryRun) attaches the collected device
 // rules to the cgroup at cgroupPath via a single AddDeviceRules call.
 func (p *Processor) applyRulesToCgroup(