From d90565cc2e23e37c96ab5169f430a7db2154e1d7 Mon Sep 17 00:00:00 2001 From: Alexander Cueva Date: Mon, 31 Aug 2026 17:35:09 -0700 Subject: [PATCH] Add block, goroutine, mutex, and trace subcommands to runsc profile Implement the remaining runtime profiling subcommands for `runsc profile`: - `goroutine`: Dumps the current stack traces of all running goroutines in Sentry. - `block`: Collects a goroutine blocking contention profile. - `mutex`: Collects a mutex contention profile. - `trace`: Collects an execution trace. PiperOrigin-RevId: 974170204 --- runsc/boot/controller.go | 11 ++-- runsc/cmd/profile/BUILD | 4 ++ runsc/cmd/profile/block.go | 99 +++++++++++++++++++++++++++++++ runsc/cmd/profile/goroutine.go | 96 ++++++++++++++++++++++++++++++ runsc/cmd/profile/mutex.go | 99 +++++++++++++++++++++++++++++++ runsc/cmd/profile/profile.go | 4 ++ runsc/cmd/profile/trace.go | 99 +++++++++++++++++++++++++++++++ runsc/container/container_test.go | 76 +++++++++++++++++++++++- runsc/sandbox/sandbox.go | 9 +++ 9 files changed, 490 insertions(+), 7 deletions(-) create mode 100644 runsc/cmd/profile/block.go create mode 100644 runsc/cmd/profile/goroutine.go create mode 100644 runsc/cmd/profile/mutex.go create mode 100644 runsc/cmd/profile/trace.go diff --git a/runsc/boot/controller.go b/runsc/boot/controller.go index ae644c4f5d9..e42447bef6a 100644 --- a/runsc/boot/controller.go +++ b/runsc/boot/controller.go @@ -162,11 +162,12 @@ const ( // Profiling related commands (see pprof.go for more details). const ( - ProfileCPU = "Profile.CPU" - ProfileHeap = "Profile.Heap" - ProfileBlock = "Profile.Block" - ProfileMutex = "Profile.Mutex" - ProfileTrace = "Profile.Trace" + ProfileCPU = "Profile.CPU" + ProfileHeap = "Profile.Heap" + ProfileGoroutine = "Profile.Goroutine" + ProfileBlock = "Profile.Block" + ProfileMutex = "Profile.Mutex" + ProfileTrace = "Profile.Trace" ) // Logging related commands (see logging.go for more details). diff --git a/runsc/cmd/profile/BUILD b/runsc/cmd/profile/BUILD index 4313a5e0caa..dd9b1f60726 100644 --- a/runsc/cmd/profile/BUILD +++ b/runsc/cmd/profile/BUILD @@ -8,9 +8,13 @@ package( go_library( name = "profile", srcs = [ + "block.go", "cpu.go", + "goroutine.go", "heap.go", + "mutex.go", "profile.go", + "trace.go", ], visibility = [ "//runsc:__subpackages__", diff --git a/runsc/cmd/profile/block.go b/runsc/cmd/profile/block.go new file mode 100644 index 00000000000..75d4e7b1114 --- /dev/null +++ b/runsc/cmd/profile/block.go @@ -0,0 +1,99 @@ +// Copyright 2026 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package profile + +import ( + "context" + "os" + "time" + + "github.com/google/subcommands" + "gvisor.dev/gvisor/runsc/cmd/util" + "gvisor.dev/gvisor/runsc/config" + "gvisor.dev/gvisor/runsc/container" + "gvisor.dev/gvisor/runsc/flag" +) + +// block implements subcommands.Command for the "block" command. +type block struct { + duration time.Duration + output string +} + +// Name implements subcommands.Command. +func (*block) Name() string { + return "block" +} + +// Synopsis implements subcommands.Command. +func (*block) Synopsis() string { + return "collects a goroutine blocking profile of a running sandbox" +} + +// Usage implements subcommands.Command. +func (*block) Usage() string { + return "block [flags] - collects a block profile of the sandbox\n" +} + +// SetFlags implements subcommands.Command. +func (b *block) SetFlags(f *flag.FlagSet) { + f.DurationVar(&b.duration, "duration", 30*time.Second, "amount of time to collect the block profile") + f.StringVar(&b.output, "output", "", "path to write block profile output (default: stdout)") +} + +// Execute implements subcommands.Command. +func (b *block) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcommands.ExitStatus { + if f.NArg() != 1 { + if f.Usage != nil { + f.Usage() + } + return subcommands.ExitUsageError + } + + id := f.Arg(0) + conf := args[0].(*config.Config) + + opts := container.LoadOpts{ + SkipCheck: true, + RootContainer: true, + } + cont, err := container.Load(conf.RootDir, container.FullID{ContainerID: id}, opts) + if err != nil { + util.Fatalf("loading sandbox: %v", err) + } + + if !cont.IsSandboxRunning() { + util.Fatalf("container sandbox is not running") + } + + out := os.Stdout + if b.output != "" { + f, err := os.OpenFile(b.output, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) + if err != nil { + util.Fatalf("failed to open output file: %v", err) + } + defer f.Close() + out = f + } + + util.Infof("Collecting block profile for sandbox %q (%v)...", cont.Sandbox.ID, b.duration) + if err := cont.Sandbox.BlockProfile(out, b.duration); err != nil { + util.Fatalf("block profiling failed: %v", err) + } + if b.output != "" { + util.Infof("Block profile written to %s", b.output) + } + return subcommands.ExitSuccess +} diff --git a/runsc/cmd/profile/goroutine.go b/runsc/cmd/profile/goroutine.go new file mode 100644 index 00000000000..b88d864c2f4 --- /dev/null +++ b/runsc/cmd/profile/goroutine.go @@ -0,0 +1,96 @@ +// Copyright 2026 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package profile + +import ( + "context" + "os" + + "github.com/google/subcommands" + "gvisor.dev/gvisor/runsc/cmd/util" + "gvisor.dev/gvisor/runsc/config" + "gvisor.dev/gvisor/runsc/container" + "gvisor.dev/gvisor/runsc/flag" +) + +// goroutine implements subcommands.Command for the "goroutine" command. +type goroutine struct { + output string +} + +// Name implements subcommands.Command. +func (*goroutine) Name() string { + return "goroutine" +} + +// Synopsis implements subcommands.Command. +func (*goroutine) Synopsis() string { + return "dumps the current stack traces of all running goroutines in a sandbox" +} + +// Usage implements subcommands.Command. +func (*goroutine) Usage() string { + return "goroutine [flags] - collects a goroutine profile of the sandbox\n" +} + +// SetFlags implements subcommands.Command. +func (g *goroutine) SetFlags(f *flag.FlagSet) { + f.StringVar(&g.output, "output", "", "path to write goroutine profile output (default: stdout)") +} + +// Execute implements subcommands.Command. +func (g *goroutine) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcommands.ExitStatus { + if f.NArg() != 1 { + if f.Usage != nil { + f.Usage() + } + return subcommands.ExitUsageError + } + + id := f.Arg(0) + conf := args[0].(*config.Config) + + opts := container.LoadOpts{ + SkipCheck: true, + RootContainer: true, + } + cont, err := container.Load(conf.RootDir, container.FullID{ContainerID: id}, opts) + if err != nil { + util.Fatalf("loading sandbox: %v", err) + } + + if !cont.IsSandboxRunning() { + util.Fatalf("container sandbox is not running") + } + + out := os.Stdout + if g.output != "" { + f, err := os.OpenFile(g.output, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) + if err != nil { + util.Fatalf("failed to open output file: %v", err) + } + defer f.Close() + out = f + } + + util.Infof("Collecting goroutine profile for sandbox %q...", cont.Sandbox.ID) + if err := cont.Sandbox.GoroutineProfile(out); err != nil { + util.Fatalf("goroutine profiling failed: %v", err) + } + if g.output != "" { + util.Infof("Goroutine profile written to %s", g.output) + } + return subcommands.ExitSuccess +} diff --git a/runsc/cmd/profile/mutex.go b/runsc/cmd/profile/mutex.go new file mode 100644 index 00000000000..6188e44faac --- /dev/null +++ b/runsc/cmd/profile/mutex.go @@ -0,0 +1,99 @@ +// Copyright 2026 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package profile + +import ( + "context" + "os" + "time" + + "github.com/google/subcommands" + "gvisor.dev/gvisor/runsc/cmd/util" + "gvisor.dev/gvisor/runsc/config" + "gvisor.dev/gvisor/runsc/container" + "gvisor.dev/gvisor/runsc/flag" +) + +// mutex implements subcommands.Command for the "mutex" command. +type mutex struct { + duration time.Duration + output string +} + +// Name implements subcommands.Command. +func (*mutex) Name() string { + return "mutex" +} + +// Synopsis implements subcommands.Command. +func (*mutex) Synopsis() string { + return "collects a mutex contention profile of a running sandbox" +} + +// Usage implements subcommands.Command. +func (*mutex) Usage() string { + return "mutex [flags] - collects a mutex profile of the sandbox\n" +} + +// SetFlags implements subcommands.Command. +func (m *mutex) SetFlags(f *flag.FlagSet) { + f.DurationVar(&m.duration, "duration", 30*time.Second, "amount of time to collect the mutex profile") + f.StringVar(&m.output, "output", "", "path to write mutex profile output (default: stdout)") +} + +// Execute implements subcommands.Command. +func (m *mutex) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcommands.ExitStatus { + if f.NArg() != 1 { + if f.Usage != nil { + f.Usage() + } + return subcommands.ExitUsageError + } + + id := f.Arg(0) + conf := args[0].(*config.Config) + + opts := container.LoadOpts{ + SkipCheck: true, + RootContainer: true, + } + cont, err := container.Load(conf.RootDir, container.FullID{ContainerID: id}, opts) + if err != nil { + util.Fatalf("loading sandbox: %v", err) + } + + if !cont.IsSandboxRunning() { + util.Fatalf("container sandbox is not running") + } + + out := os.Stdout + if m.output != "" { + f, err := os.OpenFile(m.output, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) + if err != nil { + util.Fatalf("failed to open output file: %v", err) + } + defer f.Close() + out = f + } + + util.Infof("Collecting mutex profile for sandbox %q (%v)...", cont.Sandbox.ID, m.duration) + if err := cont.Sandbox.MutexProfile(out, m.duration); err != nil { + util.Fatalf("mutex profiling failed: %v", err) + } + if m.output != "" { + util.Infof("Mutex profile written to %s", m.output) + } + return subcommands.ExitSuccess +} diff --git a/runsc/cmd/profile/profile.go b/runsc/cmd/profile/profile.go index 8b8fc3f0b7d..339344c1bc0 100644 --- a/runsc/cmd/profile/profile.go +++ b/runsc/cmd/profile/profile.go @@ -68,7 +68,11 @@ func createCommander(f *flag.FlagSet) *subcommands.Commander { cdr := subcommands.NewCommander(f, "profile") cdr.Register(cdr.HelpCommand(), "") cdr.Register(cdr.FlagsCommand(), "") + cdr.Register(new(block), "") cdr.Register(new(cpu), "") + cdr.Register(new(goroutine), "") cdr.Register(new(heap), "") + cdr.Register(new(mutex), "") + cdr.Register(new(trace), "") return cdr } diff --git a/runsc/cmd/profile/trace.go b/runsc/cmd/profile/trace.go new file mode 100644 index 00000000000..5c3442cebef --- /dev/null +++ b/runsc/cmd/profile/trace.go @@ -0,0 +1,99 @@ +// Copyright 2026 The gVisor Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package profile + +import ( + "context" + "os" + "time" + + "github.com/google/subcommands" + "gvisor.dev/gvisor/runsc/cmd/util" + "gvisor.dev/gvisor/runsc/config" + "gvisor.dev/gvisor/runsc/container" + "gvisor.dev/gvisor/runsc/flag" +) + +// trace implements subcommands.Command for the "trace" command. +type trace struct { + duration time.Duration + output string +} + +// Name implements subcommands.Command. +func (*trace) Name() string { + return "trace" +} + +// Synopsis implements subcommands.Command. +func (*trace) Synopsis() string { + return "collects an execution trace of a running sandbox" +} + +// Usage implements subcommands.Command. +func (*trace) Usage() string { + return "trace [flags] - collects an execution trace of the sandbox\n" +} + +// SetFlags implements subcommands.Command. +func (t *trace) SetFlags(f *flag.FlagSet) { + f.DurationVar(&t.duration, "duration", 30*time.Second, "amount of time to collect the execution trace") + f.StringVar(&t.output, "output", "", "path to write trace profile output (default: stdout)") +} + +// Execute implements subcommands.Command. +func (t *trace) Execute(_ context.Context, f *flag.FlagSet, args ...any) subcommands.ExitStatus { + if f.NArg() != 1 { + if f.Usage != nil { + f.Usage() + } + return subcommands.ExitUsageError + } + + id := f.Arg(0) + conf := args[0].(*config.Config) + + opts := container.LoadOpts{ + SkipCheck: true, + RootContainer: true, + } + cont, err := container.Load(conf.RootDir, container.FullID{ContainerID: id}, opts) + if err != nil { + util.Fatalf("loading sandbox: %v", err) + } + + if !cont.IsSandboxRunning() { + util.Fatalf("container sandbox is not running") + } + + out := os.Stdout + if t.output != "" { + f, err := os.OpenFile(t.output, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) + if err != nil { + util.Fatalf("failed to open output file: %v", err) + } + defer f.Close() + out = f + } + + util.Infof("Collecting execution trace for sandbox %q (%v)...", cont.Sandbox.ID, t.duration) + if err := cont.Sandbox.Trace(out, t.duration); err != nil { + util.Fatalf("execution trace profiling failed: %v", err) + } + if t.output != "" { + util.Infof("Execution trace written to %s", t.output) + } + return subcommands.ExitSuccess +} diff --git a/runsc/container/container_test.go b/runsc/container/container_test.go index 809e945f23d..97359168fe0 100644 --- a/runsc/container/container_test.go +++ b/runsc/container/container_test.go @@ -4428,7 +4428,7 @@ func TestProfileLive(t *testing.T) { t.Fatalf("error starting container: %v", err) } - // Test live CPU profiling. + // Test live CPU profiling (runsc profile cpu). cpuFile, err := os.Create(filepath.Join(t.TempDir(), "live_cpu.pprof")) if err != nil { t.Fatalf("creating cpu file: %v", err) @@ -4446,7 +4446,7 @@ func TestProfileLive(t *testing.T) { t.Errorf("live CPU profile file is empty") } - // Test live Heap profiling. + // Test live Heap profiling (runsc profile heap). heapFile, err := os.Create(filepath.Join(t.TempDir(), "live_heap.pprof")) if err != nil { t.Fatalf("creating heap file: %v", err) @@ -4463,6 +4463,78 @@ func TestProfileLive(t *testing.T) { if fi.Size() == 0 { t.Errorf("live Heap profile file is empty") } + + // Test live Goroutine profiling (runsc profile goroutine). + goroutineFile, err := os.Create(filepath.Join(t.TempDir(), "live_goroutine.txt")) + if err != nil { + t.Fatalf("creating goroutine file: %v", err) + } + defer goroutineFile.Close() + + if err := cont.Sandbox.GoroutineProfile(goroutineFile); err != nil { + t.Fatalf("GoroutineProfile: %v", err) + } + fi, err = goroutineFile.Stat() + if err != nil { + t.Fatalf("stat goroutine file: %v", err) + } + if fi.Size() == 0 { + t.Errorf("live Goroutine profile file is empty") + } + + // Test live Block profiling (runsc profile block). + blockFile, err := os.Create(filepath.Join(t.TempDir(), "live_block.pprof")) + if err != nil { + t.Fatalf("creating block file: %v", err) + } + defer blockFile.Close() + + if err := cont.Sandbox.BlockProfile(blockFile, 500*time.Millisecond); err != nil { + t.Fatalf("BlockProfile: %v", err) + } + fi, err = blockFile.Stat() + if err != nil { + t.Fatalf("stat block file: %v", err) + } + if fi.Size() == 0 { + t.Errorf("live Block profile file is empty") + } + + // Test live Mutex profiling (runsc profile mutex). + mutexFile, err := os.Create(filepath.Join(t.TempDir(), "live_mutex.pprof")) + if err != nil { + t.Fatalf("creating mutex file: %v", err) + } + defer mutexFile.Close() + + if err := cont.Sandbox.MutexProfile(mutexFile, 500*time.Millisecond); err != nil { + t.Fatalf("MutexProfile: %v", err) + } + fi, err = mutexFile.Stat() + if err != nil { + t.Fatalf("stat mutex file: %v", err) + } + if fi.Size() == 0 { + t.Errorf("live Mutex profile file is empty") + } + + // Test live Trace profiling (runsc profile trace). + traceFile, err := os.Create(filepath.Join(t.TempDir(), "live_trace.out")) + if err != nil { + t.Fatalf("creating trace file: %v", err) + } + defer traceFile.Close() + + if err := cont.Sandbox.Trace(traceFile, 500*time.Millisecond); err != nil { + t.Fatalf("Trace: %v", err) + } + fi, err = traceFile.Stat() + if err != nil { + t.Fatalf("stat trace file: %v", err) + } + if fi.Size() == 0 { + t.Errorf("live Trace profile file is empty") + } } // TestSaveSystemdCgroup emulates a sandbox saving while configured with the diff --git a/runsc/sandbox/sandbox.go b/runsc/sandbox/sandbox.go index bcc7287f62a..ce6cf92ecfd 100644 --- a/runsc/sandbox/sandbox.go +++ b/runsc/sandbox/sandbox.go @@ -2179,6 +2179,15 @@ func (s *Sandbox) HeapProfile(f *os.File, delay time.Duration) error { return s.call(boot.ProfileHeap, &opts, nil) } +// GoroutineProfile writes a goroutine stack dump to the given file. +func (s *Sandbox) GoroutineProfile(f *os.File) error { + log.Debugf("Goroutine profile %q", s.ID) + opts := control.GoroutineProfileOpts{ + FilePayload: urpc.FilePayload{Files: []*os.File{f}}, + } + return s.call(boot.ProfileGoroutine, &opts, nil) +} + // CPUProfile collects a CPU profile. func (s *Sandbox) CPUProfile(f *os.File, duration time.Duration) error { log.Debugf("CPU profile %q", s.ID)