From f308eb08f619755cc651de5c8b1e6966ad45a9cc Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 15 Jul 2026 22:59:04 +0800 Subject: [PATCH 1/4] chore(deps): bump cocoon v0.4.6 -> v0.5.1 (0f6c21b) Picks up shared-package fixes cocoon-macos gets for free: flock poll 100ms->2ms (#114), store Update fsyncs off the flock hold (#126), /proc-scan tolerance for vanishing pids (#129), per-NIC CNI teardown (#111). Adapts the two ReflinkCopy call sites to the new SyncMode arg (utils.Sync = preserve the prior fsync). --- cmd/vm/datadisk.go | 2 +- cmd/vm/utils.go | 2 +- go.mod | 2 +- go.sum | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/vm/datadisk.go b/cmd/vm/datadisk.go index e5fba45..da75cbe 100644 --- a/cmd/vm/datadisk.go +++ b/cmd/vm/datadisk.go @@ -133,7 +133,7 @@ func copyDataDisks(dir string, src []string) ([]string, error) { paths := make([]string, 0, len(src)) for _, srcPath := range src { dst := filepath.Join(dir, filepath.Base(srcPath)) - if err := utils.ReflinkCopy(dst, srcPath); err != nil { + if err := utils.ReflinkCopy(dst, srcPath, utils.Sync); err != nil { return nil, fmt.Errorf("copy data disk %s: %w", filepath.Base(srcPath), err) } paths = append(paths, dst) diff --git a/cmd/vm/utils.go b/cmd/vm/utils.go index 5cdee8e..cba8de9 100644 --- a/cmd/vm/utils.go +++ b/cmd/vm/utils.go @@ -78,7 +78,7 @@ func scaffoldVM(cmd *cobra.Command, name, image, varsSrc, varsName string) (dir, return "", "", "", "", err } ovmfVars = filepath.Join(dir, varsName) - if err = utils.ReflinkCopy(ovmfVars, varsSrc); err != nil { + if err = utils.ReflinkCopy(ovmfVars, varsSrc, utils.Sync); err != nil { return "", "", "", "", fmt.Errorf("copy OVMF_VARS: %w", err) } return dir, overlay, ovmfVars, digest, nil diff --git a/go.mod b/go.mod index 87601fe..885423a 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/cocoonstack/cocoon-macos go 1.26.4 require ( - github.com/cocoonstack/cocoon v0.4.6-0.20260704025747-d9fa4dfb5105 + github.com/cocoonstack/cocoon v0.5.2-0.20260713182614-0f6c21b5b6f6 github.com/docker/go-units v0.5.0 github.com/opencontainers/image-spec v1.1.1 github.com/projecteru2/core v0.0.0-20241016125006-ff909eefe04c diff --git a/go.sum b/go.sum index 9557d2d..d6cfda4 100644 --- a/go.sum +++ b/go.sum @@ -8,8 +8,8 @@ github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZe github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30= github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= -github.com/cocoonstack/cocoon v0.4.6-0.20260704025747-d9fa4dfb5105 h1:kY2QEEK+RiJflgTGRfvRZJvN4X87d36dkrneyOZiAMI= -github.com/cocoonstack/cocoon v0.4.6-0.20260704025747-d9fa4dfb5105/go.mod h1:Rl/SAzj1RbyL8XJaIyWiHdT96yK2D1e0xCr8flVqIcA= +github.com/cocoonstack/cocoon v0.5.2-0.20260713182614-0f6c21b5b6f6 h1:OKpbRm7j/WoON1iB26jiIshYwzLibu+5f/p49Um0rzc= +github.com/cocoonstack/cocoon v0.5.2-0.20260713182614-0f6c21b5b6f6/go.mod h1:Rl/SAzj1RbyL8XJaIyWiHdT96yK2D1e0xCr8flVqIcA= github.com/containernetworking/cni v1.3.0 h1:v6EpN8RznAZj9765HhXQrtXgX+ECGebEYEmnuFjskwo= github.com/containernetworking/cni v1.3.0/go.mod h1:Bs8glZjjFfGPHMw6hQu82RUgEPNGEaBb9KS5KtNMnJ4= github.com/containernetworking/plugins v1.9.0 h1:Mg3SXBdRGkdXyFC4lcwr6u2ZB2SDeL6LC3U+QrEANuQ= From 00e2c3e183acd37600956f33e8a85cc43e263c65 Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 15 Jul 2026 23:07:18 +0800 Subject: [PATCH 2/4] fix(net): quiesce host NICs on stop/rm to end the idle-TAP softirq storm Port of cocoon #130. A stopped CNI VM keeps its netns/TAP for a fast restart, but the dead VMM leaves the TAP carrier-less while its veth stays up on the bridge; tc mirred redirect then fires against the down device per LAN broadcast, storming softirqs until the host soft-locks. Quiesce on stop and on rm (before Delete), Unquiesce on start, via the network provider. cocoon's bridge backend no-ops Quiesce, so --net tap|bridge additionally toggles the host TAP link directly. --- cmd/vm/lifecycle.go | 2 ++ cmd/vm/net_linux.go | 57 +++++++++++++++++++++++++++++++++++++++++++-- cmd/vm/net_other.go | 4 ++++ 3 files changed, 61 insertions(+), 2 deletions(-) diff --git a/cmd/vm/lifecycle.go b/cmd/vm/lifecycle.go index ed3ee8d..21d9d15 100644 --- a/cmd/vm/lifecycle.go +++ b/cmd/vm/lifecycle.go @@ -64,6 +64,7 @@ func (h *Handler) Start(cmd *cobra.Command, args []string) error { if err := h.launch(cmd, dir, r); err != nil { return err } + unquiesceNet(cmd, r) fmt.Printf("%s (pid %d)\n", n, r.PID) return nil }); err != nil { @@ -85,6 +86,7 @@ func (h *Handler) Stop(cmd *cobra.Command, args []string) error { return err } terminate(ctx, r, grace) + quiesceNet(cmd, r) stopVNCProxy(ctx, dir) r.PID, r.VNCDisp, r.VNCPass = 0, -1, "" // VNC is launch-scoped: gone with the qemu it belonged to return saveRec(dir, r) diff --git a/cmd/vm/net_linux.go b/cmd/vm/net_linux.go index cf01e3e..e93cbdd 100644 --- a/cmd/vm/net_linux.go +++ b/cmd/vm/net_linux.go @@ -83,8 +83,16 @@ func teardownNet(cmd *cobra.Command, r *record) { // warn instead of failing: rm must proceed, but a leaked TAP/netns should leave a trail if provider, err := newProvider(cmd, r); err != nil { logger.Warnf(ctx, "teardown network for %s: %v", r.VMID, err) - } else if _, err := provider.Delete(ctx, []string{r.VMID}); err != nil { - logger.Warnf(ctx, "teardown network for %s: %v", r.VMID, err) + } else { + // quiesce before Delete: a killed VMM leaves the TAP carrier-less while its veth stays up on + // the bridge, so tc mirred redirect storms host softirqs against the down device in the window + // before Delete removes the redirect (long enough to soft-lock a host on a busy bridge). + if err := provider.Quiesce(ctx, r.VMID); err != nil { + logger.Warnf(ctx, "quiesce network for %s: %v", r.VMID, err) + } + if _, err := provider.Delete(ctx, []string{r.VMID}); err != nil { + logger.Warnf(ctx, "teardown network for %s: %v", r.VMID, err) + } } // CleanupTAPs runs unconditionally: it removes bt-* by name and must not be gated on // newProvider succeeding (rm has no --bridge flag), or an auto-created TAP would leak. @@ -93,6 +101,51 @@ func teardownNet(cmd *cobra.Command, r *record) { } } +// quiesceNet brings a stopped VM's auto-created host NICs down so the carrier-less TAP a dead VMM +// leaves behind can't storm host softirqs (CNI: tc mirred redirect firing against the down device +// per broadcast packet). unquiesceNet reverses it on restart. Best-effort, gated on ownership like +// teardownNet — never touches a user-supplied --tap. +func quiesceNet(cmd *cobra.Command, r *record) { + if !r.TapOwned { + return + } + ctx := cliutil.CommandContext(cmd) + logger := log.WithFunc("cmd.vm.quiesceNet") + if provider, err := newProvider(cmd, r); err != nil { + logger.Warnf(ctx, "quiesce network for %s: %v", r.VMID, err) + } else if err := provider.Quiesce(ctx, r.VMID); err != nil { + logger.Warnf(ctx, "quiesce network for %s: %v", r.VMID, err) + } + setTapLink(ctx, r, "down") +} + +func unquiesceNet(cmd *cobra.Command, r *record) { + if !r.TapOwned { + return + } + ctx := cliutil.CommandContext(cmd) + logger := log.WithFunc("cmd.vm.unquiesceNet") + if provider, err := newProvider(cmd, r); err != nil { + logger.Warnf(ctx, "unquiesce network for %s: %v", r.VMID, err) + } else if err := provider.Unquiesce(ctx, r.VMID); err != nil { + logger.Warnf(ctx, "unquiesce network for %s: %v", r.VMID, err) + } + setTapLink(ctx, r, "up") +} + +// setTapLink toggles the host TAP's admin state for --net tap|bridge, where cocoon's bridge backend +// no-ops Quiesce and QEMU opens the TAP with script=no (never touching its link): on stop the +// readerless TAP would otherwise sit up on the bridge, and on restart stay down until raised. CNI's +// TAP lives in a netns and is handled by the provider, so this is scoped to the host-side modes. +func setTapLink(ctx context.Context, r *record, state string) { + if r.Tap == "" || (r.NetMode != netTAP && r.NetMode != netBridge) { + return + } + if err := exec.CommandContext(ctx, "ip", "link", "set", r.Tap, state).Run(); err != nil { + log.WithFunc("cmd.vm.setTapLink").Warnf(ctx, "set tap %s %s: %v", r.Tap, state, err) + } +} + // ensureNetnsLoopback brings up lo inside the CNI netns. A freshly-created netns has its loopback // DOWN, so qemu's -vnc 127.0.0.1:N (and any other loopback bind) fails with EADDRNOTAVAIL until lo // is up. No-op outside CNI (no netns). Shells out to `ip` because the qemu launch already runs via diff --git a/cmd/vm/net_other.go b/cmd/vm/net_other.go index 8e053d5..64838ba 100644 --- a/cmd/vm/net_other.go +++ b/cmd/vm/net_other.go @@ -19,6 +19,10 @@ func provisionNet(_ *cobra.Command, _ *record) (tap, netns, mac string, err erro func teardownNet(_ *cobra.Command, _ *record) {} +func quiesceNet(_ *cobra.Command, _ *record) {} + +func unquiesceNet(_ *cobra.Command, _ *record) {} + // launchCmd builds the qemu exec; qemu-system-x86_64 is the authoritative VMM with no Go-native // equivalent (and off Linux there is no netns to enter). func launchCmd(_ *record, args []string) *exec.Cmd { From e5bd110adf0a5d724fdb21122949878131716b55 Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 15 Jul 2026 23:16:51 +0800 Subject: [PATCH 3/4] test(vm): pin the shared stop/rm force->grace mapping --force maps to grace 0 (immediate SIGKILL), default to the ACPI grace window; mirrors cocoon f6ebebc which pinned the same mapping with a test. --- cmd/vm/utils_test.go | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 cmd/vm/utils_test.go diff --git a/cmd/vm/utils_test.go b/cmd/vm/utils_test.go new file mode 100644 index 0000000..1e3e6e4 --- /dev/null +++ b/cmd/vm/utils_test.go @@ -0,0 +1,35 @@ +package vm + +import ( + "testing" + "time" + + "github.com/spf13/cobra" +) + +// TestGraceFromFlags pins the shared stop/rm force mapping: --force is an immediate SIGKILL +// (grace 0), everything else waits the ACPI grace window. +func TestGraceFromFlags(t *testing.T) { + tests := []struct { + name string + force bool + want time.Duration + }{ + {name: "force is immediate", force: true, want: 0}, + {name: "default waits the grace window", force: false, want: stopGracePeriod}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cmd := &cobra.Command{} + cmd.Flags().Bool("force", false, "") + if tt.force { + if err := cmd.Flags().Set("force", "true"); err != nil { + t.Fatalf("set force: %v", err) + } + } + if got := graceFromFlags(cmd); got != tt.want { + t.Errorf("graceFromFlags(force=%v): got %v, want %v", tt.force, got, tt.want) + } + }) + } +} From 4a2f1130e1ecd48f2e19c0d850a587be66bedd5b Mon Sep 17 00:00:00 2001 From: CMGS Date: Wed, 15 Jul 2026 23:34:31 +0800 Subject: [PATCH 4/4] refactor(net): toggle the host TAP via netlink instead of shelling out to ip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit /simplify follow-up on the quiesce port: setTapLink flips the link with netlink.LinkSetUp/Down (already a dependency, promoted to direct) rather than exec-ing `ip link set` — it never enters a netns, so the fork-safety rationale that keeps launchCmd/ensureNetnsLoopback on `ip netns exec` does not apply, and it matches how cocoon toggles the same link. Gate on r.Netns (host vs netns TAP) instead of re-listing bridge-backed modes, and de-duplicate the idle-TAP softirq-storm rationale across teardownNet/quiesceNet/setTapLink. --- cmd/vm/net_linux.go | 42 +++++++++++++++++++++++++----------------- go.mod | 2 +- 2 files changed, 26 insertions(+), 18 deletions(-) diff --git a/cmd/vm/net_linux.go b/cmd/vm/net_linux.go index e93cbdd..298e799 100644 --- a/cmd/vm/net_linux.go +++ b/cmd/vm/net_linux.go @@ -12,6 +12,7 @@ import ( "github.com/projecteru2/core/log" "github.com/spf13/cobra" + "github.com/vishvananda/netlink" "github.com/cocoonstack/cocoon/cmd/cliutil" "github.com/cocoonstack/cocoon/config" @@ -84,9 +85,8 @@ func teardownNet(cmd *cobra.Command, r *record) { if provider, err := newProvider(cmd, r); err != nil { logger.Warnf(ctx, "teardown network for %s: %v", r.VMID, err) } else { - // quiesce before Delete: a killed VMM leaves the TAP carrier-less while its veth stays up on - // the bridge, so tc mirred redirect storms host softirqs against the down device in the window - // before Delete removes the redirect (long enough to soft-lock a host on a busy bridge). + // quiesce before Delete closes the same idle-TAP softirq window quiesceNet guards, for the gap + // between the VMM dying and Delete dropping the redirect. if err := provider.Quiesce(ctx, r.VMID); err != nil { logger.Warnf(ctx, "quiesce network for %s: %v", r.VMID, err) } @@ -101,10 +101,9 @@ func teardownNet(cmd *cobra.Command, r *record) { } } -// quiesceNet brings a stopped VM's auto-created host NICs down so the carrier-less TAP a dead VMM -// leaves behind can't storm host softirqs (CNI: tc mirred redirect firing against the down device -// per broadcast packet). unquiesceNet reverses it on restart. Best-effort, gated on ownership like -// teardownNet — never touches a user-supplied --tap. +// quiesceNet brings a stopped VM's owned host NICs down so a dead VMM's carrier-less TAP can't storm +// host softirqs (tc mirred redirect firing against the down device per broadcast packet) — CNI via +// the provider's veths, tap/bridge via setTapLink. unquiesceNet reverses it on start. func quiesceNet(cmd *cobra.Command, r *record) { if !r.TapOwned { return @@ -116,7 +115,7 @@ func quiesceNet(cmd *cobra.Command, r *record) { } else if err := provider.Quiesce(ctx, r.VMID); err != nil { logger.Warnf(ctx, "quiesce network for %s: %v", r.VMID, err) } - setTapLink(ctx, r, "down") + setTapLink(ctx, r, false) } func unquiesceNet(cmd *cobra.Command, r *record) { @@ -130,19 +129,28 @@ func unquiesceNet(cmd *cobra.Command, r *record) { } else if err := provider.Unquiesce(ctx, r.VMID); err != nil { logger.Warnf(ctx, "unquiesce network for %s: %v", r.VMID, err) } - setTapLink(ctx, r, "up") + setTapLink(ctx, r, true) } -// setTapLink toggles the host TAP's admin state for --net tap|bridge, where cocoon's bridge backend -// no-ops Quiesce and QEMU opens the TAP with script=no (never touching its link): on stop the -// readerless TAP would otherwise sit up on the bridge, and on restart stay down until raised. CNI's -// TAP lives in a netns and is handled by the provider, so this is scoped to the host-side modes. -func setTapLink(ctx context.Context, r *record, state string) { - if r.Tap == "" || (r.NetMode != netTAP && r.NetMode != netBridge) { +// setTapLink flips a host TAP's admin state (down on stop, up on start). QEMU opens it with script=no +// and cocoon's bridge backend no-ops Quiesce, so cocoon-macos owns the toggle; a CNI TAP lives in a +// netns (r.Netns != "") and is the provider's job, so this only reaches host-netns TAPs. +func setTapLink(ctx context.Context, r *record, up bool) { + if r.Tap == "" || r.Netns != "" { return } - if err := exec.CommandContext(ctx, "ip", "link", "set", r.Tap, state).Run(); err != nil { - log.WithFunc("cmd.vm.setTapLink").Warnf(ctx, "set tap %s %s: %v", r.Tap, state, err) + logger := log.WithFunc("cmd.vm.setTapLink") + link, err := netlink.LinkByName(r.Tap) + if err != nil { + logger.Warnf(ctx, "find tap %s: %v", r.Tap, err) + return + } + set := netlink.LinkSetUp + if !up { + set = netlink.LinkSetDown + } + if err := set(link); err != nil { + logger.Warnf(ctx, "set tap %s up=%v: %v", r.Tap, up, err) } } diff --git a/go.mod b/go.mod index 885423a..cefbd12 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/opencontainers/image-spec v1.1.1 github.com/projecteru2/core v0.0.0-20241016125006-ff909eefe04c github.com/spf13/cobra v1.10.2 + github.com/vishvananda/netlink v1.3.1 golang.org/x/sync v0.21.0 howett.net/plist v1.0.1 oras.land/oras-go/v2 v2.6.1 @@ -36,7 +37,6 @@ require ( github.com/rogpeppe/go-internal v1.13.1 // indirect github.com/rs/zerolog v1.34.0 // indirect github.com/spf13/pflag v1.0.10 // indirect - github.com/vishvananda/netlink v1.3.1 // indirect github.com/vishvananda/netns v0.0.5 // indirect golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect golang.org/x/net v0.50.0 // indirect