Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 9 additions & 3 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -90,14 +90,14 @@ services:
cuda-worker:
image: nvidia/cuda:12.4.0-base-ubuntu22.04
command: ["nvidia-smi"]
labels:
swarm-device-access.enable: "true"
swarm-device-access.device-allow: "/dev/nvidia*"
volumes:
- /dev/nvidia0:/dev/nvidia0
- /dev/nvidiactl:/dev/nvidiactl
- /dev/nvidia-uvm:/dev/nvidia-uvm
deploy:
labels:
swarm-device-access.enable: "true"
swarm-device-access.device-allow: "/dev/nvidia*"
mode: replicated
replicas: 1
```
Expand Down Expand Up @@ -150,6 +150,12 @@ Consumer services opt in and narrow their allowed device set with labels:
| `swarm-device-access.device-allow` | Comma-separated globs | Allow only matching `/dev/...` paths. Empty means inherit. |
| `swarm-device-access.device-deny` | Comma-separated globs | Deny matching `/dev/...` paths. Deny overrides allow. |

Declare these labels under `deploy.labels:` (the Swarm service spec — the
natural home for service-level metadata, alongside Traefik / Homepage / other
label-driven tooling). The daemon also reads top-level `labels:` if you need to
override a service-wide value on a single task; per-container values win on
conflict.

Global `-device-allow` and `-device-deny` define the broadest access the daemon
may grant. Per-container labels can only narrow that access. Deny rules always
win.
Expand Down
5 changes: 5 additions & 0 deletions docs/architecture.md
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,11 @@ The daemon uses a two-level policy:

**Per-container policy** (Docker labels):

Declare these labels under `deploy.labels:` in your Swarm stack file — that is
the service spec, the natural home alongside Traefik / Homepage / other
label-driven tooling. The daemon also reads top-level `labels:` when present;
per-container (task-level) values win on conflict.

| Label | Description |
|------------------------------------|----------------------------------------------------------------|
| `swarm-device-access.enable` | `true` to opt in, `false` to explicitly opt out. |
Expand Down
4 changes: 3 additions & 1 deletion docs/testing.md
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,9 @@ or deployment behavior:

3. Start a consumer container with `--label swarm-device-access.enable=true` and
a real `/dev/...` bind mount and confirm the daemon logs `device mount detected`
and `adding device rule`.
and `adding device rule`. (For Swarm stacks, the equivalent placement is
`deploy.labels:` in the service spec — `docker service create --label` writes
to the same location.)

4. If the host uses cgroup v2, confirm a `BPF_CGROUP_DEVICE` program is attached
to the consumer cgroup with `bpftool`.
Expand Down
6 changes: 3 additions & 3 deletions examples/audio-passthrough/docker-compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -11,14 +11,14 @@
services:
audio-worker:
image: ubuntu:24.04
labels:
swarm-device-access.enable: "true"
swarm-device-access.device-allow: "/dev/snd/*"
volumes:
# /dev/snd is a directory; the daemon walks it and applies a rule per device.
- /dev/snd:/dev/snd
command: [ "aplay", "-l" ]
deploy:
labels:
swarm-device-access.enable: "true"
swarm-device-access.device-allow: "/dev/snd/*"
mode: replicated
replicas: 1
swarm-device-access:
Expand Down
8 changes: 4 additions & 4 deletions examples/gpu-passthrough/docker-compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -14,16 +14,16 @@
services:
cuda-worker:
image: nvidia/cuda:12.4.0-base-ubuntu22.04
# Required when daemon runs in opt-in mode (the default).
labels:
swarm-device-access.enable: "true"
swarm-device-access.device-allow: "/dev/nvidia*"
volumes:
- /dev/nvidia0:/dev/nvidia0
- /dev/nvidiactl:/dev/nvidiactl
- /dev/nvidia-uvm:/dev/nvidia-uvm
command: [ "nvidia-smi" ]
deploy:
labels:
# Required when daemon runs in opt-in mode (the default).
swarm-device-access.enable: "true"
swarm-device-access.device-allow: "/dev/nvidia*"
mode: replicated
replicas: 1
swarm-device-access:
Expand Down
6 changes: 3 additions & 3 deletions examples/usb-passthrough/docker-compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -40,13 +40,13 @@ services:

usb-worker:
image: ubuntu:24.04
labels:
swarm-device-access.enable: "true"
swarm-device-access.device-allow: "/dev/bus/usb/*"
volumes:
# Bind-mount the entire USB bus; the daemon will apply rules to each device.
- /dev/bus/usb:/dev/bus/usb
command: [ "lsusb" ]
deploy:
labels:
swarm-device-access.enable: "true"
swarm-device-access.device-allow: "/dev/bus/usb/*"
mode: replicated
replicas: 1
6 changes: 3 additions & 3 deletions examples/v4l2-passthrough/docker-compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -40,9 +40,6 @@ services:

v4l2-worker:
image: linuxserver/ffmpeg:latest # dclint disable-line service-image-require-explicit-tag
labels:
swarm-device-access.enable: "true"
swarm-device-access.device-allow: "/dev/video*"
volumes:
- /dev/video0:/dev/video0
- /tmp/output:/output
Expand All @@ -56,5 +53,8 @@ services:
- "5"
- /output/capture.mp4
deploy:
labels:
swarm-device-access.enable: "true"
swarm-device-access.device-allow: "/dev/video*"
mode: replicated
replicas: 1
42 changes: 42 additions & 0 deletions internal/policy/policy.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,9 @@ package policy

import (
"fmt"
"maps"
"path/filepath"
"sort"
"strconv"
"strings"
)
Expand Down Expand Up @@ -231,3 +233,43 @@ func matchAny(patterns []string, path string) bool {

return false
}

// knownLabels is the set of recognized swarm-device-access.* label keys.
var knownLabels = map[string]struct{}{
LabelEnable: {},
LabelDeviceAllow: {},
LabelDeviceDeny: {},
}

// MergeLabels returns a merged label map: service labels as base, container
// labels win on conflict. Nil inputs are treated as empty maps.
func MergeLabels(service, container map[string]string) map[string]string {
merged := make(map[string]string, len(service)+len(container))

maps.Copy(merged, service)
maps.Copy(merged, container)

return merged
}

// UnknownLabels returns a sorted slice of keys in labels that start with
// LabelPrefix but are not in the known label set. Returns nil when none.
func UnknownLabels(labels map[string]string) []string {
var unknown []string

for k := range labels {
if strings.HasPrefix(k, LabelPrefix) {
if _, ok := knownLabels[k]; !ok {
unknown = append(unknown, k)
}
}
}

if len(unknown) == 0 {
return nil
}

sort.Strings(unknown)

return unknown
}
128 changes: 128 additions & 0 deletions internal/policy/policy_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -328,3 +328,131 @@ func sliceEqual(a, b []string) bool {

return true
}

func TestMergeLabels(t *testing.T) {
t.Parallel()

cases := []struct {
name string
service map[string]string
container map[string]string
want map[string]string
}{
{
name: "both nil",
want: map[string]string{},
},
{
name: "nil service",
container: map[string]string{"a": "1"},
want: map[string]string{"a": "1"},
},
{
name: "nil container",
service: map[string]string{"a": "1"},
want: map[string]string{"a": "1"},
},
{
name: "disjoint union",
service: map[string]string{"a": "1"},
container: map[string]string{"b": "2"},
want: map[string]string{"a": "1", "b": "2"},
},
{
name: "container wins on conflict",
service: map[string]string{"a": "service"},
container: map[string]string{"a": "container"},
want: map[string]string{"a": "container"},
},
}

for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()

got := policy.MergeLabels(tc.service, tc.container)

if len(got) != len(tc.want) {
t.Fatalf("MergeLabels() len=%d, want %d; got=%v", len(got), len(tc.want), got)
}

for k, wantV := range tc.want {
if got[k] != wantV {
t.Errorf("key %q: got %q, want %q", k, got[k], wantV)
}
}
})
}
}

func TestUnknownLabels(t *testing.T) {
t.Parallel()

cases := []struct {
name string
labels map[string]string
want []string
}{
{
name: "nil map",
labels: nil,
want: nil,
},
{
name: "empty map",
labels: map[string]string{},
want: nil,
},
{
name: "no swarm-device-access keys",
labels: map[string]string{"traefik.enable": "true"},
want: nil,
},
{
name: "only known keys",
labels: map[string]string{
policy.LabelEnable: "true",
policy.LabelDeviceAllow: "/dev/snd/*",
policy.LabelDeviceDeny: "/dev/sda",
},
want: nil,
},
{
name: "mixed known and unknown",
labels: map[string]string{
policy.LabelEnable: "true",
policy.LabelPrefix + "enabled": "true",
policy.LabelPrefix + "zzz": "x",
},
want: []string{
policy.LabelPrefix + "enabled",
policy.LabelPrefix + "zzz",
},
},
{
name: "unknown sorted deterministically",
labels: map[string]string{
policy.LabelPrefix + "zzz": "1",
policy.LabelPrefix + "aaa": "2",
policy.LabelPrefix + "mmm": "3",
},
want: []string{
policy.LabelPrefix + "aaa",
policy.LabelPrefix + "mmm",
policy.LabelPrefix + "zzz",
},
},
}

for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()

got := policy.UnknownLabels(tc.labels)

if !sliceEqual(got, tc.want) {
t.Errorf("UnknownLabels() = %v, want %v", got, tc.want)
}
})
}
}
Loading
Loading