From 2fcd262c4036ffa529fcfbfd158e2ed44f2c4bce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B5=B5=E7=BF=94=E5=AE=87?= Date: Sat, 18 Jul 2026 19:44:02 +0800 Subject: [PATCH] Add eBPF CPU profiling path from stack sampling to Pyroscope export. Wire probe CGO and a minimal BCC symbolizer, and extend cgoreceiver to periodically encode pprof and push process_cpu profiles when ebpf_profiling.endpoint.url is set. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: 赵翔宇 --- .../docker/kindling-collector-config.yml | 16 + collector/go.mod | 104 +++--- collector/go.sum | 134 +++++--- .../pyroscopeexporter/simple_sender.go | 145 ++++++++ .../pyroscopeexporter/simple_sender_test.go | 30 ++ .../component/receiver/cgoreceiver/cgo_func.h | 163 +++++---- .../receiver/cgoreceiver/cgoreceiver.go | 39 ++- .../component/receiver/cgoreceiver/config.go | 38 ++- .../receiver/cgoreceiver/profiling.go | 322 ++++++++++++++++++ collector/pkg/ebpf/pprof/pprof.go | 129 +++++++ collector/pkg/ebpf/pprof/pprof_test.go | 50 +++ deploy/README.md | 14 +- deploy/agent/kindling-collector-config.yml | 18 +- docs/ebpf_cpu_profiling.md | 94 +++++ probe/CMakeLists.txt | 3 +- probe/cmake/modules/bcc_minimal.cmake | 59 ++++ probe/src/CMakeLists.txt | 14 +- probe/src/cgo/cgo_func.cpp | 49 +++ probe/src/cgo/cgo_func.h | 31 +- probe/src/cgo/kindling.cpp | 46 +++ probe/src/cgo/kindling.h | 10 + .../symbolizer/bcc_per_pid_symbolizer.cpp | 304 +++++++++++++++++ .../core/symbolizer/bcc_per_pid_symbolizer.h | 81 +++++ .../core/symbolizer/kindling_symbolizer.cpp | 95 ++++++ .../src/core/symbolizer/kindling_symbolizer.h | 33 ++ 25 files changed, 1849 insertions(+), 172 deletions(-) create mode 100644 collector/pkg/component/consumer/exporter/pyroscopeexporter/simple_sender.go create mode 100644 collector/pkg/component/consumer/exporter/pyroscopeexporter/simple_sender_test.go create mode 100644 collector/pkg/component/receiver/cgoreceiver/profiling.go create mode 100644 collector/pkg/ebpf/pprof/pprof.go create mode 100644 collector/pkg/ebpf/pprof/pprof_test.go create mode 100644 docs/ebpf_cpu_profiling.md create mode 100644 probe/cmake/modules/bcc_minimal.cmake create mode 100644 probe/src/core/symbolizer/bcc_per_pid_symbolizer.cpp create mode 100644 probe/src/core/symbolizer/bcc_per_pid_symbolizer.h create mode 100644 probe/src/core/symbolizer/kindling_symbolizer.cpp create mode 100644 probe/src/core/symbolizer/kindling_symbolizer.h diff --git a/collector/docker/kindling-collector-config.yml b/collector/docker/kindling-collector-config.yml index a9e4c4a28..a2af247fb 100644 --- a/collector/docker/kindling-collector-config.yml +++ b/collector/docker/kindling-collector-config.yml @@ -40,6 +40,22 @@ receivers: - "containerd" - "dockerd" - "containerd-shim" + # eBPF CPU stack sampling → Pyroscope. Empty url disables the feature. + # See docs/ebpf_cpu_profiling.md + ebpf_profiling: + endpoint: + # Example: "http://pyroscope:4040/" + url: "" + remote_timeout: 10s + min_backoff_period: 500ms + max_backoff_period: 30s + max_backoff_retries: 10 + sender_worker_count: 2 + sender_queue_capacity: 200 + # Empty / omit = sample all processes (CAPTURE_ALL). + # Non-empty = only these host PIDs (PROFILING_ENABLED whitelist). + # Example: pids: [1234, 5678] + pids: [] analyzers: cpuanalyzer: diff --git a/collector/go.mod b/collector/go.mod index ed260efd9..65d03e12f 100644 --- a/collector/go.mod +++ b/collector/go.mod @@ -14,67 +14,94 @@ require ( github.com/olivere/elastic/v6 v6.2.1 github.com/orcaman/concurrent-map v0.0.0-20210501183033-44dafcb38ecc github.com/pkg/errors v0.9.1 - github.com/prometheus/client_golang v1.11.1 + github.com/prometheus/client_golang v1.17.0 github.com/shirou/gopsutil v3.21.11+incompatible github.com/spf13/viper v1.10.1 - github.com/stretchr/testify v1.7.1 + github.com/stretchr/testify v1.8.1 github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74 - go.opentelemetry.io/otel v1.2.0 + go.opentelemetry.io/otel v1.7.0 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v0.25.0 - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.2.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.7.0 go.opentelemetry.io/otel/exporters/prometheus v0.25.0 go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v0.25.0 go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.2.0 - go.opentelemetry.io/otel/metric v0.25.0 - go.opentelemetry.io/otel/sdk v1.2.0 + go.opentelemetry.io/otel/metric v0.30.0 + go.opentelemetry.io/otel/sdk v1.7.0 go.opentelemetry.io/otel/sdk/export/metric v0.25.0 go.opentelemetry.io/otel/sdk/metric v0.25.0 - go.opentelemetry.io/otel/trace v1.2.0 - go.uber.org/atomic v1.9.0 + go.opentelemetry.io/otel/trace v1.7.0 + go.uber.org/atomic v1.10.0 go.uber.org/multierr v1.7.0 go.uber.org/zap v1.19.1 golang.org/x/net v0.23.0 golang.org/x/sys v0.18.0 gopkg.in/natefinch/lumberjack.v2 v2.0.0 - k8s.io/api v0.21.5 - k8s.io/apimachinery v0.21.5 - k8s.io/client-go v0.21.5 + k8s.io/api v0.24.2 + k8s.io/apimachinery v0.24.2 + k8s.io/client-go v0.24.2 ) require ( - github.com/mitchellh/mapstructure v1.4.3 - golang.org/x/sync v0.1.0 + github.com/mitchellh/mapstructure v1.5.0 + golang.org/x/sync v0.3.0 +) + +require ( + connectrpc.com/connect v1.14.0 + github.com/google/pprof v0.0.0-20220608213341-c488b8fa1db3 + github.com/grafana/dskit v0.0.0-20240104111617-ea101a3b86eb + github.com/grafana/pyroscope/api v0.4.0 + github.com/klauspost/compress v1.17.3 + github.com/prometheus/common v0.44.0 + github.com/prometheus/prometheus v0.37.9 + google.golang.org/protobuf v1.32.0 +) + +// Pin versions that prometheus/pyroscope would otherwise upgrade past the collector baseline. +replace ( + go.opentelemetry.io/otel => go.opentelemetry.io/otel v1.2.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace => go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.2.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc => go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.2.0 + go.opentelemetry.io/otel/metric => go.opentelemetry.io/otel/metric v0.25.0 + go.opentelemetry.io/otel/sdk => go.opentelemetry.io/otel/sdk v1.2.0 + go.opentelemetry.io/otel/trace => go.opentelemetry.io/otel/trace v1.2.0 + go.opentelemetry.io/proto/otlp => go.opentelemetry.io/proto/otlp v0.10.0 + k8s.io/api => k8s.io/api v0.21.5 + k8s.io/apimachinery => k8s.io/apimachinery v0.21.5 + k8s.io/client-go => k8s.io/client-go v0.21.5 ) require ( github.com/beorn7/perks v1.0.1 // indirect - github.com/cenkalti/backoff/v4 v4.1.1 // indirect + github.com/cenkalti/backoff/v4 v4.1.3 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/fsnotify/fsnotify v1.5.1 // indirect - github.com/go-logr/logr v0.4.0 // indirect + github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/go-logr/logr v1.2.3 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/golang/protobuf v1.5.3 // indirect - github.com/google/go-cmp v0.5.9 // indirect - github.com/google/gofuzz v1.1.0 // indirect + github.com/google/go-cmp v0.6.0 // indirect + github.com/google/gofuzz v1.2.0 // indirect github.com/googleapis/gnostic v0.4.1 // indirect + github.com/gorilla/mux v1.8.0 // indirect + github.com/grafana/regexp v0.0.0-20221122212121-6b5c0a4cb7fd // indirect github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect github.com/hashicorp/errwrap v1.0.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect - github.com/imdario/mergo v0.3.5 // indirect + github.com/imdario/mergo v0.3.12 // indirect + github.com/josharian/intern v1.0.0 // indirect github.com/josharian/native v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/magiconair/properties v1.8.5 // indirect - github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/mdlayher/socket v0.4.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pelletier/go-toml v1.9.4 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_model v0.2.0 // indirect - github.com/prometheus/common v0.26.0 // indirect - github.com/prometheus/procfs v0.6.0 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/prometheus/client_model v0.5.0 // indirect + github.com/prometheus/procfs v0.11.1 // indirect github.com/spf13/afero v1.6.0 // indirect github.com/spf13/cast v1.4.1 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect @@ -84,23 +111,24 @@ require ( github.com/tklauser/numcpus v0.4.0 // indirect github.com/yusufpapurcu/wmi v1.2.2 // indirect go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.25.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.2.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.7.0 // indirect go.opentelemetry.io/otel/internal/metric v0.25.0 // indirect - go.opentelemetry.io/proto/otlp v0.10.0 // indirect - golang.org/x/oauth2 v0.7.0 // indirect + go.opentelemetry.io/proto/otlp v0.16.0 // indirect + golang.org/x/oauth2 v0.11.0 // indirect golang.org/x/term v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect - golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba // indirect + golang.org/x/time v0.1.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect - google.golang.org/grpc v1.56.3 // indirect - google.golang.org/protobuf v1.30.0 // indirect + google.golang.org/genproto v0.0.0-20231106174013-bbf56f31fb17 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20231120223509-83a465c0220f // indirect + google.golang.org/grpc v1.59.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/ini.v1 v1.66.2 // indirect + gopkg.in/ini.v1 v1.66.4 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect - gopkg.in/yaml.v3 v3.0.0 // indirect - k8s.io/klog/v2 v2.8.0 // indirect - k8s.io/utils v0.0.0-20201110183641-67b214c5f920 // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.1.2 // indirect - sigs.k8s.io/yaml v1.2.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/klog/v2 v2.70.0 // indirect + k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect + sigs.k8s.io/yaml v1.3.0 // indirect ) diff --git a/collector/go.sum b/collector/go.sum index 6797327a8..2b1c97b4f 100644 --- a/collector/go.sum +++ b/collector/go.sum @@ -20,6 +20,8 @@ cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIA cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +connectrpc.com/connect v1.14.0 h1:PDS+J7uoz5Oui2VEOMcfz6Qft7opQM9hPiKvtGC01pA= +connectrpc.com/connect v1.14.0/go.mod h1:uoAq5bmhhn43TwhaKdGKN/bZcGtzPW1v+ngDTn5u+8s= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest/autorest v0.11.12/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw= @@ -53,8 +55,9 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/cenkalti/backoff/v4 v4.1.1 h1:G2HAfAmvm/GcKan2oOQpBXOd2tT2G57ZnZGWa1PxPBQ= github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= +github.com/cenkalti/backoff/v4 v4.1.3 h1:cFAlzYUlVYDysBEH2T5hyJZMh3+5+WCBvSnK6Q8UtC4= +github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= @@ -93,8 +96,8 @@ github.com/fortytw2/leaktest v1.2.0 h1:cj6GCiwJDH7l3tMHLjZDo0QqPtrXJiWSI9JgpeQKw github.com/fortytw2/leaktest v1.2.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= -github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -106,8 +109,10 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= -github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc= github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= +github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= @@ -162,26 +167,37 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20220608213341-c488b8fa1db3 h1:mpL/HvfIgIejhVwAfxBQkwEjlhP5o0O9RAeTAjpwzxc= +github.com/google/pprof v0.0.0-20220608213341-c488b8fa1db3/go.mod h1:gSuNB+gJaOiQKLEZ+q+PK9Mq3SOzhRcw2GsGS/FhYDk= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gnostic v0.4.1 h1:DLJCy1n/vrD4HPjOvYcT8aYQXpPIzoRZONaYwyycI+I= github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= +github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grafana/dskit v0.0.0-20240104111617-ea101a3b86eb h1:AWE6+kvtE18HP+lRWNUCyvymyrFSXs6TcS2vXIXGIuw= +github.com/grafana/dskit v0.0.0-20240104111617-ea101a3b86eb/go.mod h1:kkWM4WUV230bNG3urVRWPBnSJHs64y/0RmWjftnnn0c= +github.com/grafana/pyroscope/api v0.4.0 h1:J86DxoNeLOvtJhB1Cn65JMZkXe682D+RqeoIUiYc/eo= +github.com/grafana/pyroscope/api v0.4.0/go.mod h1:MFnZNeUM4RDsDOnbgKW3GWoLSBpLzMMT9nkvhHHo81o= +github.com/grafana/regexp v0.0.0-20221122212121-6b5c0a4cb7fd h1:PpuIBO5P3e9hpqBD0O/HjhShYuM6XE0i/lbE6J94kww= +github.com/grafana/regexp v0.0.0-20221122212121-6b5c0a4cb7fd/go.mod h1:M5qHK+eWfAv8VR/265dIuEpL3fNfeC21tXXp9itM24A= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= @@ -198,8 +214,11 @@ github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/imdario/mergo v0.3.5 h1:JboBksRwiiAJWvIYJVo46AfV+IAIKZpfrSzVKj42R4Q= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= +github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/josharian/native v0.0.0-20200817173448-b6b71def0850/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= github.com/josharian/native v1.0.0 h1:Ts/E8zCSEsG17dUqv7joXJFybuMLjQfWE04tsBODTxk= github.com/josharian/native v1.0.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= @@ -224,6 +243,8 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.17.3 h1:qkRjuerhUU1EmXLYGkSH6EZL+vPSxIrYjLNAK4slzwA= +github.com/klauspost/compress v1.17.3/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= @@ -231,6 +252,7 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -240,10 +262,12 @@ github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaW github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e h1:hB2xlXdHp/pmPZq0y3QnmWAArdw9PqbmotexnWx/FU8= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/mdlayher/ethtool v0.0.0-20210210192532-2b88debcdd43/go.mod h1:+t7E0lkKfbBsebllff1xdTmyJt8lH37niI6kwFk9OTo= github.com/mdlayher/genetlink v1.0.0/go.mod h1:0rJ0h4itni50A86M2kHcgS85ttZazNt7a8H2a2cw0Gc= github.com/mdlayher/netlink v0.0.0-20190409211403-11939a169225/go.mod h1:eQB3mZE4aiYnlUsyGGCOpPETfdQq4Jhsgf1fk3cwQaA= @@ -262,8 +286,8 @@ github.com/mdlayher/socket v0.0.0-20210307095302-262dc9984e00/go.mod h1:GAFlyu4/ github.com/mdlayher/socket v0.4.0 h1:280wsy40IC9M9q1uPGcLBwXpcTQDtoGwVt+BNoITxIw= github.com/mdlayher/socket v0.4.0/go.mod h1:xxFqz5GRCUN3UEOm9CZqEJsAbe1C8OwSK46NlmWuVoc= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs= -github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= @@ -276,7 +300,6 @@ github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8m github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/olivere/elastic v6.2.37+incompatible h1:UfSGJem5czY+x/LqxgeCBgjDn6St+z8OnsCuxwD3L0U= github.com/olivere/elastic v6.2.37+incompatible/go.mod h1:J+q1zQJTgAz9woqsbVRqGeB5G1iqDKVBWLNSYW8yfJ8= @@ -298,30 +321,37 @@ github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_golang v1.11.1 h1:+4eQaD7vAZ6DsfsxB15hbE0odUjGI5ARs9yskGu1v4s= -github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= +github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= +github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= -github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= +github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= +github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI= +github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= +github.com/prometheus/prometheus v0.37.9 h1:xcf1PKMUEg+OmiqQV5XkWyoLXYPV0R8YpC9ywrTRTp8= +github.com/prometheus/prometheus v0.37.9/go.mod h1:qrrW4duOJdlqRG9XZdd2lkPe9iDnFrWoSHv13SArq60= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= @@ -344,14 +374,18 @@ github.com/spf13/viper v1.10.1/go.mod h1:IGlFPqhNAPKRxohIzWpI5QEy4kuI7tcl5WvR+8q github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tklauser/go-sysconf v0.3.10 h1:IJ1AZGZRWbY8T5Vfk04D9WOA5WSejdflXxP03OUqALw= @@ -399,14 +433,13 @@ go.opentelemetry.io/otel/sdk/metric v0.25.0 h1:J+Ta+4IAA5W9AdWhGQLfciEpavBqqSkBz go.opentelemetry.io/otel/sdk/metric v0.25.0/go.mod h1:G4xzj4LvC6xDDSsVXpvRVclQCbofGGg4ZU2VKKtDRfg= go.opentelemetry.io/otel/trace v1.2.0 h1:Ys3iqbqZhcf28hHzrm5WAquMkDHNZTUkw7KHbuNjej0= go.opentelemetry.io/otel/trace v1.2.0/go.mod h1:N5FLswTubnxKxOJHM7XZC074qpeEdLy3CgAVsdMucK0= -go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.opentelemetry.io/proto/otlp v0.10.0 h1:n7brgtEbDvXEgGyKKo8SobKT1e9FewlDtXzkVP5djoE= go.opentelemetry.io/proto/otlp v0.10.0/go.mod h1:zG20xCK0szZ1xdokeSOwEcmlXu+x9kkdRe6N1DhKcfU= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= -go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/goleak v1.1.11-0.20210813005559-691160354723 h1:sHOAIxRGBp443oHZIPB+HsUGaksVCXVQENPxwTfQdH4= +go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= +go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.7.0 h1:zaiO/rmgFjbmCXdSYJWQcdvOCsthmdaHfr3Gm2Kx4Ec= go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= @@ -496,8 +529,8 @@ golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.7.0 h1:qe6s0zUXlPX80/dITx3440hWZ7GwMwgDDyrSGTPJG/g= -golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= +golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU= +golang.org/x/oauth2 v0.11.0/go.mod h1:LdF7O/8bLR/qWK9DrpXmbHLTouvRHK0SgJl0GmDBchk= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -507,8 +540,8 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -561,9 +594,9 @@ golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210525143221-35b2ab0089ea/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= @@ -583,8 +616,9 @@ golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.1.0 h1:xYY+Bajn2a7VBmTM5GikTmnK8ZuX8YgnQCqZpbBNtmA= +golang.org/x/time v0.1.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -661,8 +695,12 @@ google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= -google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= +google.golang.org/genproto v0.0.0-20231106174013-bbf56f31fb17 h1:wpZ8pe2x1Q3f2KyT5f8oP/fa9rHAKgFPr/HZdNuS+PQ= +google.golang.org/genproto v0.0.0-20231106174013-bbf56f31fb17/go.mod h1:J7XzRzVy1+IPwWHZUzoD0IccYZIrXILAQpc+Qy9CMhY= +google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17 h1:JpwMPBpFN3uKhdaekDpiNlImDdkUAyiJ6ez/uxGaUSo= +google.golang.org/genproto/googleapis/api v0.0.0-20231106174013-bbf56f31fb17/go.mod h1:0xJLfVdJqpAPl8tDg1ujOCGzx6LFLttXT5NhllGOXY4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231120223509-83a465c0220f h1:ultW7fxlIvee4HYrtnaRPon9HpEgFk5zYpmfMgtKB5I= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231120223509-83a465c0220f/go.mod h1:L9KNLi232K1/xB6f7AlSX692koaRnKaWSR0stBki0Yc= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -675,8 +713,8 @@ google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTp google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= -google.golang.org/grpc v1.56.3 h1:8I4C0Yq1EjstUzUJzpcRVbuYA2mODtEmpWiQoN/b2nc= -google.golang.org/grpc v1.56.3/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= +google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= +google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -689,20 +727,20 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= +google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/ini.v1 v1.66.2 h1:XfR1dOYubytKy4Shzc2LHrrGhU0lDCfDGG1yLPmpgsI= -gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.66.4 h1:SsAcf+mM7mRZo2nJNGt8mZCjG8ZRaNGMURJw7BsIST4= +gopkg.in/ini.v1 v1.66.4/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= @@ -717,8 +755,8 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0 h1:hjy8E9ON/egN1tAYqKb61G10WtihqetD4sz2H+8nIeA= -gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -733,16 +771,20 @@ k8s.io/client-go v0.21.5 h1:zkVidiWVgciPKYqWpMFMjCUF+4rRXcfkKoyQS1Ue21k= k8s.io/client-go v0.21.5/go.mod h1:EUornVlr3rBrPKXUoMPNggJdEQmvFNMpYO3Kb6432kw= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= -k8s.io/klog/v2 v2.8.0 h1:Q3gmuM9hKEjefWFFYF0Mat+YyFJvsUyYuwyNNJ5C9Ts= k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= +k8s.io/klog/v2 v2.70.0 h1:GMmmjoFOrNepPN0ZeGCzvD2Gh5IKRwdFx8W5PBxVTQU= +k8s.io/klog/v2 v2.70.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7/go.mod h1:wXW5VT87nVfh/iLV8FpR2uDvrFyomxbtb1KivDbvPTE= -k8s.io/utils v0.0.0-20201110183641-67b214c5f920 h1:CbnUZsM497iRC5QMVkHwyl8s2tB3g7yaSHkYPkpgelw= k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 h1:HNSDgDCrr/6Ly3WEGKZftiE7IY19Vz2GdbOCyI4qqhc= +k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/structured-merge-diff/v4 v4.1.2 h1:Hr/htKFmJEbtMgS/UD0N+gtgctAqz81t3nu+sPzynno= sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= -sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= +sigs.k8s.io/structured-merge-diff/v4 v4.2.1 h1:bKCqE9GvQ5tiVHn5rfn1r+yao3aLQEaLzkkmAkf+A6Y= +sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= +sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/collector/pkg/component/consumer/exporter/pyroscopeexporter/simple_sender.go b/collector/pkg/component/consumer/exporter/pyroscopeexporter/simple_sender.go new file mode 100644 index 000000000..e2663a2e3 --- /dev/null +++ b/collector/pkg/component/consumer/exporter/pyroscopeexporter/simple_sender.go @@ -0,0 +1,145 @@ +package pyroscopeexporter + +import ( + "context" + "fmt" + "net/http" + "strings" + "time" + + "connectrpc.com/connect" + "github.com/grafana/dskit/backoff" + pushv1 "github.com/grafana/pyroscope/api/gen/proto/go/push/v1" + "github.com/grafana/pyroscope/api/gen/proto/go/push/v1/pushv1connect" + typesv1 "github.com/grafana/pyroscope/api/gen/proto/go/types/v1" + "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/model/labels" +) + +// RawSample holds a gzip-compressed pprof payload. +type RawSample struct { + RawProfile []byte `json:"raw_profile"` + ID string `json:"id"` +} + +func BulkSend(series []*pushv1.RawProfileSeries, sender *AsyncSender) error { + req := connect.NewRequest(&pushv1.PushRequest{Series: series}) + return sender.Enqueue(req) +} + +func PrepareDataForTransmission(lbs labels.Labels, samples []*RawSample) ([]*typesv1.LabelPair, []*pushv1.RawSample, error) { + b := labels.NewBuilder(nil) + for _, l := range lbs { + if strings.HasPrefix(l.Name, model.ReservedLabelPrefix) && l.Name != labels.MetricName && l.Name != "__delta__" { + continue + } + b.Set(l.Name, l.Value) + } + var protoLabels []*typesv1.LabelPair + for _, l := range b.Labels() { + protoLabels = append(protoLabels, &typesv1.LabelPair{Name: l.Name, Value: l.Value}) + } + var protoSamples []*pushv1.RawSample + for _, s := range samples { + protoSamples = append(protoSamples, &pushv1.RawSample{RawProfile: s.RawProfile}) + } + return protoLabels, protoSamples, nil +} + +type AsyncSender struct { + q chan *connect.Request[pushv1.PushRequest] + client pushv1connect.PusherServiceClient + timeout time.Duration + minBackoff time.Duration + maxBackoff time.Duration + maxRetries int + workers int + ctx context.Context + cancel context.CancelFunc +} + +func NewAsyncSender(endpointURL string, timeout time.Duration, workers, queueCap int, + minB, maxB time.Duration, maxRetries int, +) *AsyncSender { + tr := &http.Transport{ + MaxIdleConns: 128, + MaxIdleConnsPerHost: workers * 4, + MaxConnsPerHost: workers * 4, + IdleConnTimeout: 90 * time.Second, + } + httpClient := &http.Client{Timeout: timeout, Transport: tr} + c := pushv1connect.NewPusherServiceClient( + httpClient, + endpointURL, + connect.WithSendCompression("gzip"), + connect.WithCompressMinBytes(1024), + ) + ctx, cancel := context.WithCancel(context.Background()) + s := &AsyncSender{ + q: make(chan *connect.Request[pushv1.PushRequest], queueCap), + client: c, + timeout: timeout, + minBackoff: minB, + maxBackoff: maxB, + maxRetries: maxRetries, + workers: workers, + ctx: ctx, + cancel: cancel, + } + for i := 0; i < workers; i++ { + go s.worker() + } + return s +} + +func (s *AsyncSender) Stop() { s.cancel(); close(s.q) } + +func (s *AsyncSender) Enqueue(req *connect.Request[pushv1.PushRequest]) error { + select { + case s.q <- req: + return nil + default: + return fmt.Errorf("flamegraph send queue full") + } +} + +func (s *AsyncSender) worker() { + for { + select { + case <-s.ctx.Done(): + return + case req, ok := <-s.q: + if !ok { + return + } + bo := backoff.New(s.ctx, backoff.Config{ + MinBackoff: s.minBackoff, MaxBackoff: s.maxBackoff, MaxRetries: s.maxRetries, + }) + for bo.Ongoing() { + ctx, cancel := context.WithTimeout(s.ctx, s.timeout) + _, err := s.client.Push(ctx, req) + cancel() + if err == nil || !shouldRetry(err) { + break + } + bo.Wait() + } + } + } +} + +func shouldRetry(err error) bool { + code := connect.CodeOf(err) + switch code { + case connect.CodeDeadlineExceeded, + connect.CodeUnknown, + connect.CodeResourceExhausted, + connect.CodeInternal, + connect.CodeUnavailable, + connect.CodeDataLoss, + connect.CodeAborted: + return true + default: + return false + } +} diff --git a/collector/pkg/component/consumer/exporter/pyroscopeexporter/simple_sender_test.go b/collector/pkg/component/consumer/exporter/pyroscopeexporter/simple_sender_test.go new file mode 100644 index 000000000..6e9f788ca --- /dev/null +++ b/collector/pkg/component/consumer/exporter/pyroscopeexporter/simple_sender_test.go @@ -0,0 +1,30 @@ +package pyroscopeexporter + +import ( + "testing" + + "github.com/prometheus/prometheus/model/labels" + "github.com/stretchr/testify/require" +) + +func TestPrepareDataForTransmission(t *testing.T) { + lbs := labels.Labels{ + {Name: "__name__", Value: "process_cpu"}, + {Name: "__internal__", Value: "drop-me"}, + {Name: "pid", Value: "123"}, + } + samples := []*RawSample{{RawProfile: []byte("gzip-pprof")}} + protoLabels, protoSamples, err := PrepareDataForTransmission(lbs, samples) + require.NoError(t, err) + require.Len(t, protoSamples, 1) + require.Equal(t, []byte("gzip-pprof"), protoSamples[0].RawProfile) + + got := map[string]string{} + for _, l := range protoLabels { + got[l.Name] = l.Value + } + require.Equal(t, "process_cpu", got["__name__"]) + require.Equal(t, "123", got["pid"]) + _, hasInternal := got["__internal__"] + require.False(t, hasInternal) +} diff --git a/collector/pkg/component/receiver/cgoreceiver/cgo_func.h b/collector/pkg/component/receiver/cgoreceiver/cgo_func.h index b32bcdaed..cfb9b5750 100644 --- a/collector/pkg/component/receiver/cgoreceiver/cgo_func.h +++ b/collector/pkg/component/receiver/cgoreceiver/cgo_func.h @@ -5,77 +5,120 @@ #ifndef SYSDIG_CGO_FUNC_H #define SYSDIG_CGO_FUNC_H +#include + +/* Must match agent-libs driver/bpf/types.h layout. */ +struct bpf_profile_data { + uint32_t pid; + uint64_t user_stack[127]; + uint64_t kernel_stack[127]; + uint64_t count; +} __attribute__((aligned(8))); +struct sample_key { + uint32_t pid; + int32_t kernel_stack_id; + int32_t user_stack_id; + char comm[16]; +} __attribute__((aligned(8))); +struct sample_key_set { + uint32_t nr_keys; + struct sample_key *keys; +} __attribute__((aligned(8))); + +struct capture_statistics_for_go { + int evts; + int drops; + int drops_buffer; + int drops_pf; + int drops_bug; + int preemptions; + int suppressed; + int tids_suppressed; +}; + +struct event_params_for_subscribe { + char *name; + char *value; +}; + +struct kindling_event_t_for_go { + uint64_t timestamp; + char *name; + uint32_t category; + uint16_t paramsNumber; + uint64_t latency; + struct KeyValue { + char *key; + char *value; + uint32_t len; + uint32_t valueType; + } userAttributes[16]; + struct event_context { + struct thread_info { + uint32_t pid; + uint32_t tid; + uint32_t uid; + uint32_t gid; + char *comm; + char *containerId; + } tinfo; + struct fd_info { + int32_t num; + uint32_t fdType; + char *filename; + char *directory; + uint32_t protocol; + uint8_t role; + uint32_t sip[4]; + uint32_t dip[4]; + uint32_t sport; + uint32_t dport; + uint64_t source; + uint64_t destination; + } fdInfo; + } context; +}; + #ifdef __cplusplus extern "C" { #endif int runForGo(); -int getEventsByInterval(int interval, void** kindlingEvent, void* count); +int getEventsByInterval(int interval, void **kindlingEvent, void *count); void suppressEventsCommForGo(char *comm); -void subEventForGo(char* eventName, char* category, void *params); +void subEventForGo(char *eventName, char *category, void *params); int startProfile(); int stopProfile(); -char* startAttachAgent(int pid); -char* stopAttachAgent(int pid); +char *startAttachAgent(int pid); +char *stopAttachAgent(int pid); void startProfileDebug(int pid, int tid); void stopProfileDebug(); -void getCaptureStatistics(struct capture_statistics_for_go* stats); +void getCaptureStatistics(struct capture_statistics_for_go *stats); void catchSignalUp(); + +/* CPU continuous profiling */ +int getProfileData(struct sample_key key, struct bpf_profile_data *sample); +int getProfileKeys(struct sample_key_set *set); +void freeProfileKeys(struct sample_key_set *set); +void clearProfileMap(); +void clearStacksMap(); +void startCpuSampeling(); +void startCpuSampelingAll(); +void stopCpuSampeling(); +int setProfilePids(const uint32_t *pids, uint32_t n); +int symbolizerInit(void); +void symbolizerShutdown(void); +int symbolizeProcessNamesWithFallback( + uint32_t pid, + const uint64_t *addrs, uint32_t n, + int debug_syms, int perf_map, int map_files, + char ***out_names, uint32_t *out_n); +int symbolizeKernelNames( + const uint64_t *addrs, uint32_t n, + char ***out_names, uint32_t *out_n); +void freeNames(char **names, uint32_t n); +void maintainBccCaches(int64_t seconds); #ifdef __cplusplus } - #endif -#endif //SYSDIG_CGO_FUNC_H -struct capture_statistics_for_go{ - int evts; - int drops; - int drops_buffer; - int drops_pf; - int drops_bug; - int preemptions; - int suppressed; - int tids_suppressed; -}; -struct event_params_for_subscribe { - char *name; - char *value; -}; - -struct kindling_event_t_for_go{ - uint64_t timestamp; - char *name; - uint32_t category; - uint16_t paramsNumber; - uint64_t latency; - struct KeyValue { - char *key; - char* value; - uint32_t len; - uint32_t valueType; - }userAttributes[16]; - struct event_context { - struct thread_info { - uint32_t pid; - uint32_t tid; - uint32_t uid; - uint32_t gid; - char *comm; - char *containerId; - }tinfo; - struct fd_info { - int32_t num; - uint32_t fdType; - char *filename; - char *directory; - uint32_t protocol; - uint8_t role; - uint32_t sip[4]; - uint32_t dip[4]; - uint32_t sport; - uint32_t dport; - - uint64_t source; - uint64_t destination; - }fdInfo; - }context; -}; +#endif // SYSDIG_CGO_FUNC_H diff --git a/collector/pkg/component/receiver/cgoreceiver/cgoreceiver.go b/collector/pkg/component/receiver/cgoreceiver/cgoreceiver.go index 7f84e4325..a623e3418 100644 --- a/collector/pkg/component/receiver/cgoreceiver/cgoreceiver.go +++ b/collector/pkg/component/receiver/cgoreceiver/cgoreceiver.go @@ -42,6 +42,11 @@ type CgoReceiver struct { stats eventCounter probeCounter *probeCounter probeCounterMutex sync.RWMutex + + // eBPF CPU profiling / Pyroscope + symOnce sync.Once + symInited bool + symOpts SymOptions } func NewCgoReceiver(config interface{}, telemetry *component.TelemetryTools, analyzerManager *analyzerpackage.Manager) receiver.Receiver { @@ -77,32 +82,32 @@ func (r *CgoReceiver) Start() error { go r.consumeEvents() go r.startGetEvents() go r.getCaptureStatisticsByInterval(15 * time.Second) + r.EnablePeriodicFlamegraphSampling(15 * time.Second) return nil } func (r *CgoReceiver) startGetEvents() { var count = 0 - var pKindlingEvents [1000]unsafe.Pointer - r.shutdownWG.Add(1) + var pKindlingEvents [1000]unsafe.Pointer + r.shutdownWG.Add(1) - for { - select { - case <-r.stopCh: - r.shutdownWG.Done() - return - default: - evt_count := int(C.getEventsByInterval(C.int(100000000), &pKindlingEvents[0], (unsafe.Pointer)(&count))) - for i := 0; i < evt_count; i++ { - event := convertEvent((*CKindlingEventForGo)(pKindlingEvents[i])) - r.eventChannel <- event - r.stats.add(event.Name, 1) - } - } - } + for { + select { + case <-r.stopCh: + r.shutdownWG.Done() + return + default: + evt_count := int(C.getEventsByInterval(C.int(100000000), &pKindlingEvents[0], (unsafe.Pointer)(&count))) + for i := 0; i < evt_count; i++ { + event := convertEvent((*CKindlingEventForGo)(pKindlingEvents[i])) + r.eventChannel <- event + r.stats.add(event.Name, 1) + } + } + } } - func (r *CgoReceiver) consumeEvents() { r.shutdownWG.Add(1) for { diff --git a/collector/pkg/component/receiver/cgoreceiver/config.go b/collector/pkg/component/receiver/cgoreceiver/config.go index 4dbfcf4f8..cbba88535 100644 --- a/collector/pkg/component/receiver/cgoreceiver/config.go +++ b/collector/pkg/component/receiver/cgoreceiver/config.go @@ -1,8 +1,11 @@ package cgoreceiver +import "time" + type Config struct { - SubscribeInfo []SubEvent `mapstructure:"subscribe"` - ProcessFilterInfo ProcessFilter `mapstructure:"process_filter"` + SubscribeInfo []SubEvent `mapstructure:"subscribe"` + ProcessFilterInfo ProcessFilter `mapstructure:"process_filter"` + EbpfProfiling EbpfProfilingConfig `mapstructure:"ebpf_profiling"` } type SubEvent struct { @@ -15,6 +18,25 @@ type ProcessFilter struct { Comms []string `mapstructure:"comms"` } +// EbpfEndpointOptions is the Pyroscope push endpoint configuration. +type EbpfEndpointOptions struct { + URL string `mapstructure:"url"` + RemoteTimeout time.Duration `mapstructure:"remote_timeout"` + MinBackoff time.Duration `mapstructure:"min_backoff_period"` + MaxBackoff time.Duration `mapstructure:"max_backoff_period"` + MaxBackoffRetries int `mapstructure:"max_backoff_retries"` +} + +// EbpfProfilingConfig configures eBPF CPU stack sampling → Pyroscope export. +// Empty endpoint.url disables the feature. +// Empty Pids means CAPTURE_ALL; non-empty enables whitelist sampling only. +type EbpfProfilingConfig struct { + Endpoint EbpfEndpointOptions `mapstructure:"endpoint"` + SenderWorkerCount int `mapstructure:"sender_worker_count"` + SenderQueueCapacity int `mapstructure:"sender_queue_capacity"` + Pids []uint32 `mapstructure:"pids"` +} + func NewDefaultConfig() *Config { return &Config{ SubscribeInfo: []SubEvent{ @@ -82,5 +104,17 @@ func NewDefaultConfig() *Config { ProcessFilterInfo: ProcessFilter{ Comms: []string{"kindling-collec", "containerd", "dockerd", "containerd-shim"}, }, + EbpfProfiling: EbpfProfilingConfig{ + Endpoint: EbpfEndpointOptions{ + URL: "", + RemoteTimeout: 10 * time.Second, + MinBackoff: 500 * time.Millisecond, + MaxBackoff: 30 * time.Second, + MaxBackoffRetries: 10, + }, + SenderWorkerCount: 2, + SenderQueueCapacity: 200, + Pids: nil, + }, } } diff --git a/collector/pkg/component/receiver/cgoreceiver/profiling.go b/collector/pkg/component/receiver/cgoreceiver/profiling.go new file mode 100644 index 000000000..9a63db2dc --- /dev/null +++ b/collector/pkg/component/receiver/cgoreceiver/profiling.go @@ -0,0 +1,322 @@ +package cgoreceiver + +/* +#cgo LDFLAGS: -L ./ -lkindling -lstdc++ -ldl +#cgo CFLAGS: -I . +#include +#include +#include +#include "cgo_func.h" +*/ +import "C" +import ( + "bytes" + "fmt" + "os" + "strconv" + "time" + "unsafe" + + exporter "github.com/Kindling-project/kindling/collector/pkg/component/consumer/exporter/pyroscopeexporter" + "github.com/Kindling-project/kindling/collector/pkg/ebpf/pprof" + pushv1 "github.com/grafana/pyroscope/api/gen/proto/go/push/v1" + "github.com/prometheus/prometheus/model/labels" + "go.uber.org/zap" + "google.golang.org/protobuf/proto" +) + +type SymOptions struct { + DebugSyms bool + PerfMap bool + MapFiles bool +} + +func (r *CgoReceiver) makeProfilingSender() (*exporter.AsyncSender, error) { + cfgEndpoint := r.cfg.EbpfProfiling.Endpoint + if cfgEndpoint.URL == "" { + r.telemetry.Logger.Error("eBPF profiling endpoint URL is empty; skip periodic sampling") + return nil, fmt.Errorf("profiling endpoint url is empty") + } + if cfgEndpoint.RemoteTimeout == 0 { + cfgEndpoint.RemoteTimeout = 10 * time.Second + } + minBackoff := cfgEndpoint.MinBackoff + if minBackoff == 0 { + minBackoff = 500 * time.Millisecond + } + maxBackoff := cfgEndpoint.MaxBackoff + if maxBackoff == 0 { + maxBackoff = 30 * time.Second + } + maxRetries := cfgEndpoint.MaxBackoffRetries + if maxRetries == 0 { + maxRetries = 10 + } + workers := r.cfg.EbpfProfiling.SenderWorkerCount + if workers <= 0 { + workers = 2 + } + queueCap := r.cfg.EbpfProfiling.SenderQueueCapacity + if queueCap <= 0 { + queueCap = 200 + } + return exporter.NewAsyncSender(cfgEndpoint.URL, cfgEndpoint.RemoteTimeout, workers, queueCap, minBackoff, maxBackoff, maxRetries), nil +} + +// EnablePeriodicFlamegraphSampling runs CPU stack sampling and pushes to Pyroscope. +// Empty ebpf_profiling.pids uses CAPTURE_ALL; non-empty uses PROFILING_ENABLED whitelist. +func (r *CgoReceiver) EnablePeriodicFlamegraphSampling(interval time.Duration) { + r.initSymbolizer() + sender, err := r.makeProfilingSender() + if err != nil { + r.telemetry.Logger.Error("Failed to init profiling sender", zap.Error(err)) + r.closeSymbolizer() + return + } + + go func() { + defer func() { + sender.Stop() + r.closeSymbolizer() + }() + r.startCpuSampling() + roundNumber := 0 + for { + timer := time.NewTimer(interval) + select { + case <-r.stopCh: + timer.Stop() + r.stopCpuSampling() + r.telemetry.Logger.Info("Periodic CPU sampling stopped") + return + case <-timer.C: + } + currentTime := time.Now() + if err := r.collectCpuProfiles(sender); err != nil { + r.telemetry.Logger.Error("Failed to collect periodic CPU profile", zap.Error(err)) + } + roundNumber++ + if roundNumber%10 == 0 { + C.clearStacksMap() + C.maintainBccCaches(C.int64_t(60)) + } + r.telemetry.Logger.Info("CPU FlamegraphSampling once cost", zap.Duration("cost", time.Since(currentTime))) + } + }() +} + +func (r *CgoReceiver) collectCpuProfiles(sender *exporter.AsyncSender) error { + var keySet C.struct_sample_key_set + defer C.freeProfileKeys(&keySet) + if ret := C.getProfileKeys(&keySet); ret != 0 { + return fmt.Errorf("getProfileKeys failed with ret: %d", ret) + } + if keySet.nr_keys == 0 { + r.telemetry.Logger.Info("no profile data available") + return nil + } + + builders := pprof.NewProfileBuilders(97) + keysSlice := (*[1 << 30]C.struct_sample_key)(unsafe.Pointer(keySet.keys))[:keySet.nr_keys:keySet.nr_keys] + pidSet := make(map[int64]struct{}, keySet.nr_keys) + for i := uint32(0); i < uint32(keySet.nr_keys); i++ { + sample := C.struct_bpf_profile_data{} + if ret := C.getProfileData(keysSlice[i], &sample); ret != 0 { + continue + } + stack, err := r.MergeStack(&sample, C.GoString(&keysSlice[i].comm[0])) + if err != nil || stack == nil { + if err != nil { + r.telemetry.Logger.Error("MergeStack err: ", zap.Error(err)) + } + continue + } + pidSet[int64(sample.pid)] = struct{}{} + builder := builders.BuilderForTarget(uint64(sample.pid), nil) + builder.AddSample(stack, uint64(sample.count)) + } + + labeler := NewLabeler(r.telemetry.GetZapLogger(), "process_cpu") + perPidLabels := labeler.BuildLabelsForPIDs(pidSet) + r.telemetry.Logger.Info("Cpu profiling", zap.Int("builders.Builders", len(builders.Builders))) + return r.sendPprofSeries(sender, builders, perPidLabels) +} + +func (r *CgoReceiver) startCpuSampling() { + pids := r.cfg.EbpfProfiling.Pids + if len(pids) == 0 { + r.telemetry.Logger.Info("eBPF CPU profiling: CAPTURE_ALL (no pids configured)") + C.startCpuSampelingAll() + return + } + r.telemetry.Logger.Info("eBPF CPU profiling: whitelist mode", zap.Int("pid_count", len(pids))) + if ret := C.setProfilePids((*C.uint32_t)(unsafe.Pointer(&pids[0])), C.uint32_t(len(pids))); ret != 0 { + r.telemetry.Logger.Error("setProfilePids failed", zap.Int("ret", int(ret))) + return + } + C.startCpuSampeling() +} + +func (r *CgoReceiver) stopCpuSampling() { + C.stopCpuSampeling() +} + +func (r *CgoReceiver) initSymbolizer() { + r.symOnce.Do(func() { + if C.symbolizerInit() != 0 { + r.telemetry.Logger.Panic("InitSymbolizer failed", zap.Error(fmt.Errorf("kindling_symbolizer_init failed"))) + } + r.symInited = true + }) +} + +func (r *CgoReceiver) closeSymbolizer() { + if r.symInited { + C.symbolizerShutdown() + r.symInited = false + } +} + +func (r *CgoReceiver) SymbolizeProcessNames(pid uint32, addrs []uint64) ([]string, error) { + if len(addrs) == 0 { + return nil, fmt.Errorf("kindling_symbolize_process_names addrs == 0") + } + var cNames **C.char + var outN C.uint32_t + rc := C.symbolizeProcessNamesWithFallback( + C.uint32_t(pid), + (*C.uint64_t)(unsafe.Pointer(&addrs[0])), C.uint32_t(len(addrs)), + boolToCInt(r.symOpts.DebugSyms), boolToCInt(r.symOpts.PerfMap), boolToCInt(r.symOpts.MapFiles), + &cNames, &outN, + ) + if rc != 0 { + return nil, fmt.Errorf("kindling_symbolize_process_names failed") + } + defer C.freeNames(cNames, outN) + return cStringsToSlice(cNames, outN), nil +} + +func (r *CgoReceiver) SymbolizeKernelNames(addrs []uint64) ([]string, error) { + if len(addrs) == 0 { + return nil, nil + } + var cNames **C.char + var outN C.uint32_t + rc := C.symbolizeKernelNames( + (*C.uint64_t)(unsafe.Pointer(&addrs[0])), C.uint32_t(len(addrs)), + &cNames, &outN, + ) + if rc != 0 { + return nil, fmt.Errorf("kindling_symbolize_kernel_names failed") + } + defer C.freeNames(cNames, outN) + return cStringsToSlice(cNames, outN), nil +} + +func (r *CgoReceiver) MergeStack(sample *C.struct_bpf_profile_data, comm string) ([]string, error) { + userNames, err := r.SymbolizeProcessNames(uint32(sample.pid), (*[127]uint64)(unsafe.Pointer(&sample.user_stack))[:]) + if err != nil { + return nil, err + } + kernelNames, err := r.SymbolizeKernelNames((*[127]uint64)(unsafe.Pointer(&sample.kernel_stack))[:]) + if err != nil { + return nil, err + } + stack := make([]string, 0, len(userNames)+len(kernelNames)+1) + stack = append(stack, kernelNames...) + stack = append(stack, userNames...) + stack = append(stack, comm) + if len(stack) == 1 { + return nil, nil + } + return stack, nil +} + +type Labeler struct { + Base labels.Labels + logger *zap.Logger +} + +func NewLabeler(logger *zap.Logger, profileName string) *Labeler { + hostIp := os.Getenv("MY_NODE_IP") + return &Labeler{ + Base: labels.Labels{ + {Name: "__name__", Value: profileName}, + {Name: "__delta__", Value: "false"}, + {Name: "service_name", Value: "kindling"}, + {Name: "host_ip", Value: hostIp}, + }, + logger: logger, + } +} + +// BuildLabelsForPIDs builds minimal per-pid labels (no PID control-plane container lookup). +func (l *Labeler) BuildLabelsForPIDs(pidSet map[int64]struct{}) map[int64]labels.Labels { + res := make(map[int64]labels.Labels, len(pidSet)) + for pid := range pidSet { + lbs := append(append(labels.Labels(nil), l.Base...), labels.Label{ + Name: "pid", Value: strconv.FormatInt(pid, 10), + }) + res[pid] = lbs + } + return res +} + +func (r *CgoReceiver) sendPprofSeries(sender *exporter.AsyncSender, builders *pprof.ProfileBuilders, perPidLabels map[int64]labels.Labels) error { + const ( + maxSeriesPerReq = 80 + maxBytesPerReq = 1 << 17 // 128 kB + ) + curBytes := 0 + protoSeries := make([]*pushv1.RawProfileSeries, 0, maxSeriesPerReq) + flush := func() { + if len(protoSeries) == 0 { + return + } + _copy := make([]*pushv1.RawProfileSeries, len(protoSeries)) + copy(_copy, protoSeries) + _ = exporter.BulkSend(_copy, sender) + protoSeries = protoSeries[:0] + curBytes = 0 + } + + for pid, builder := range builders.Builders { + buf := bytes.NewBuffer(nil) + _, err := builder.Write(buf) + if err != nil { + return fmt.Errorf("pprof encode %w", err) + } + rawProfile := buf.Bytes() + realSamples := []*exporter.RawSample{{RawProfile: rawProfile}} + lbs := append(labels.Labels(nil), perPidLabels[int64(pid)]...) + protoLabels, rawSample, _ := exporter.PrepareDataForTransmission(lbs, realSamples) + if len(protoSeries) >= maxSeriesPerReq || curBytes >= maxBytesPerReq { + flush() + } + series := &pushv1.RawProfileSeries{Labels: protoLabels, Samples: rawSample} + protoSeries = append(protoSeries, series) + curBytes += proto.Size(series) + } + flush() + return nil +} + +func cStringsToSlice(cNames **C.char, n C.uint32_t) []string { + count := int(n) + if count == 0 || cNames == nil { + return nil + } + arr := (*[1 << 30]*C.char)(unsafe.Pointer(cNames))[:count:count] + out := make([]string, count) + for i := 0; i < count; i++ { + out[i] = C.GoString(arr[i]) + } + return out +} + +func boolToCInt(b bool) C.int { + if b { + return 1 + } + return 0 +} diff --git a/collector/pkg/ebpf/pprof/pprof.go b/collector/pkg/ebpf/pprof/pprof.go new file mode 100644 index 000000000..32859fef1 --- /dev/null +++ b/collector/pkg/ebpf/pprof/pprof.go @@ -0,0 +1,129 @@ +package pprof + +import ( + "fmt" + "io" + "sync" + "time" + + "github.com/google/pprof/profile" + "github.com/klauspost/compress/gzip" + "github.com/prometheus/prometheus/model/labels" +) + +var ( + gzipWriterPool = sync.Pool{ + New: func() any { + res, err := gzip.NewWriterLevel(io.Discard, gzip.BestSpeed) + if err != nil { + panic(err) + } + return res + }, + } +) + +type ProfileBuilders struct { + Builders map[uint64]*ProfileBuilder + SampleRate int +} + +func NewProfileBuilders(sampleRate int) *ProfileBuilders { + return &ProfileBuilders{Builders: make(map[uint64]*ProfileBuilder), SampleRate: sampleRate} +} + +func (b ProfileBuilders) BuilderForTarget(hash uint64, labels labels.Labels) *ProfileBuilder { + res := b.Builders[hash] + if res != nil { + return res + } + + builder := &ProfileBuilder{ + locations: make(map[string]*profile.Location), + functions: make(map[string]*profile.Function), + Labels: labels, + Profile: &profile.Profile{ + Mapping: []*profile.Mapping{{ID: 1}}, + SampleType: []*profile.ValueType{ + {Type: "cpu", Unit: "nanoseconds"}, + }, + Period: time.Second.Nanoseconds() / int64(b.SampleRate), + PeriodType: &profile.ValueType{Type: "cpu", Unit: "nanoseconds"}, + TimeNanos: time.Now().UnixNano(), + }, + } + + b.Builders[hash] = builder + return builder +} + +type ProfileBuilder struct { + locations map[string]*profile.Location + functions map[string]*profile.Function + Profile *profile.Profile + Labels labels.Labels +} + +func (p *ProfileBuilder) AddSample(stacktrace []string, value uint64) { + sample := &profile.Sample{ + Value: []int64{int64(value) * p.Profile.Period}, + Location: make([]*profile.Location, len(stacktrace)), + } + for i, s := range stacktrace { + sample.Location[i] = p.addLocation(s) + } + p.Profile.Sample = append(p.Profile.Sample, sample) +} + +func (p *ProfileBuilder) addLocation(function string) *profile.Location { + loc, ok := p.locations[function] + if ok { + return loc + } + + id := uint64(len(p.Profile.Location) + 1) + loc = &profile.Location{ + ID: id, + Mapping: p.Profile.Mapping[0], + Line: []profile.Line{ + {Function: p.addFunction(function)}, + }, + } + p.Profile.Location = append(p.Profile.Location, loc) + p.locations[function] = loc + return loc +} + +func (p *ProfileBuilder) addFunction(function string) *profile.Function { + f, ok := p.functions[function] + if ok { + return f + } + + id := uint64(len(p.Profile.Function) + 1) + f = &profile.Function{ + ID: id, + Name: function, + } + p.Profile.Function = append(p.Profile.Function, f) + p.functions[function] = f + return f +} + +func (p *ProfileBuilder) Write(dst io.Writer) (int64, error) { + gzipWriter := gzipWriterPool.Get().(*gzip.Writer) + gzipWriter.Reset(dst) + defer func() { + gzipWriter.Reset(io.Discard) + gzipWriterPool.Put(gzipWriter) + }() + err := p.Profile.WriteUncompressed(gzipWriter) + if err != nil { + return 0, fmt.Errorf("ebpf profile encode %w", err) + } + err = gzipWriter.Close() + if err != nil { + return 0, fmt.Errorf("ebpf profile encode %w", err) + } + return 0, nil +} diff --git a/collector/pkg/ebpf/pprof/pprof_test.go b/collector/pkg/ebpf/pprof/pprof_test.go new file mode 100644 index 000000000..c62b3c1e2 --- /dev/null +++ b/collector/pkg/ebpf/pprof/pprof_test.go @@ -0,0 +1,50 @@ +package pprof + +import ( + "bytes" + "testing" + "time" + + "github.com/google/pprof/profile" + "github.com/prometheus/prometheus/model/labels" + "github.com/stretchr/testify/require" +) + +func TestBackAndForth(t *testing.T) { + const sampleRate = 97 + period := time.Second.Nanoseconds() / int64(sampleRate) + + builders := NewProfileBuilders(97) + + builder := builders.BuilderForTarget(1, labels.Labels{}) + builder.AddSample([]string{"a", "b", "c"}, 239) + builder.AddSample([]string{"a", "b", "d"}, 4242) + + buf := bytes.NewBuffer(nil) + _, err := builder.Write(buf) + require.NoError(t, err) + + rawProfile := buf.Bytes() + + parsed, err := profile.Parse(bytes.NewBuffer(rawProfile)) + require.NoError(t, err) + require.NotNil(t, parsed) + require.Equal(t, 2, len(parsed.Sample)) + require.Equal(t, 4, len(parsed.Function)) + require.Equal(t, 4, len(parsed.Location)) + + stacks := map[string]int64{} + for _, sample := range parsed.Sample { + stack := "" + for i, location := range sample.Location { + if i != 0 { + stack += ";" + } + stack += location.Line[0].Function.Name + } + stacks[stack] = sample.Value[0] + } + + require.Equal(t, 239*period, stacks["a;b;c"]) + require.Equal(t, 4242*period, stacks["a;b;d"]) +} diff --git a/deploy/README.md b/deploy/README.md index 13a166108..4b883b2d0 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -2,7 +2,19 @@ The files under this directory are needed when releasing the Kindling. ## agent -This directory contains files used for deploying `kindling-agent` in Kubernetes. +This directory contains files used for deploying `kindling-agent` in Kubernetes. + +### eBPF CPU profiling (Pyroscope) + +Optional continuous CPU stack sampling is configured under +`receivers.cgoreceiver.ebpf_profiling` in `agent/kindling-collector-config.yml`. + +- Leave `endpoint.url` empty to keep the feature **off** (default). +- Set it to a Pyroscope base URL (e.g. `http://pyroscope:4040/`) to enable push of `process_cpu` profiles. + +See [docs/ebpf_cpu_profiling.md](../docs/ebpf_cpu_profiling.md) for privileges, +BCC build deps, and label details. Stock `kindling-deploy.yml` (`privileged`, +`hostPID`, host `/sys`) is required for this path — same as existing eBPF tracing. ## grafana-with-plugins This directory contains files used for deploying `Grafana` in Kubernetes. diff --git a/deploy/agent/kindling-collector-config.yml b/deploy/agent/kindling-collector-config.yml index a5cb47a04..e55635716 100644 --- a/deploy/agent/kindling-collector-config.yml +++ b/deploy/agent/kindling-collector-config.yml @@ -40,6 +40,22 @@ receivers: - "containerd" - "dockerd" - "containerd-shim" + # eBPF CPU stack sampling → Pyroscope. Empty url disables the feature. + # See docs/ebpf_cpu_profiling.md + ebpf_profiling: + endpoint: + # Example: "http://pyroscope:4040/" + url: "" + remote_timeout: 10s + min_backoff_period: 500ms + max_backoff_period: 30s + max_backoff_retries: 10 + sender_worker_count: 2 + sender_queue_capacity: 200 + # Empty / omit = sample all processes (CAPTURE_ALL). + # Non-empty = only these host PIDs (PROFILING_ENABLED whitelist). + # Example: pids: [1234, 5678] + pids: [] analyzers: cpuanalyzer: @@ -132,7 +148,7 @@ processors: k8smetadataprocessor: # Set "enable" false if you want to run the agent in the non-Kubernetes environment. # Otherwise, the agent will panic if it can't connect to the API-server. - enable: true + enable: false kube_auth_type: serviceAccount kube_config_dir: /root/.kube/config # GraceDeletePeriod controls the delay interval after receiving delete event. diff --git a/docs/ebpf_cpu_profiling.md b/docs/ebpf_cpu_profiling.md new file mode 100644 index 000000000..2e7c84c00 --- /dev/null +++ b/docs/ebpf_cpu_profiling.md @@ -0,0 +1,94 @@ + # eBPF CPU Profiling → Pyroscope + +Kindling can sample CPU call stacks with eBPF (~97Hz), symbolize them with a +minimal BCC symbolizer baked into `libkindling`, and push gzip pprof profiles to +[Grafana Pyroscope](https://grafana.com/oss/pyroscope/) every **15 seconds**. + +This path is **independent** of the existing Camera On/OffCPU flame graph +(`cpuanalyzer` → `cameraexporter` → `camera-front`). + +## Enable / disable + +Edit the collector ConfigMap / config file: + +- `deploy/agent/kindling-collector-config.yml` +- `collector/docker/kindling-collector-config.yml` + +```yaml +receivers: + cgoreceiver: + ebpf_profiling: + endpoint: + # Empty = disabled (default). Non-empty = start sampling + push. + url: "http://pyroscope:4040/" + # Empty / omit = sample all non-kernel processes (CAPTURE_ALL). + # Non-empty = only these host PIDs (whitelist + PROFILING_ENABLED). + pids: [] + # Example: pids: [1234, 5678] +``` + +Apply via the usual agent ConfigMap (`kindlingcfg`) and restart the agent pod. + +**Sampling scope** (static, read at startup only): + +| `pids` | Behavior | +|--------|----------| +| omitted / `[]` | `CAPTURE_ALL` — all non-kernel threads | +| `[pid, ...]` | whitelist — only those host PIDs | + +There is no runtime hot-reload of the PID list; change YAML and restart the agent. + +## What gets pushed + +| Item | Value | +|------|--------| +| Profile name | `process_cpu` (`__name__`) | +| Labels | `service_name=kindling`, `host_ip` (`MY_NODE_IP`), `pid` | +| Interval | 15s (hardcoded in `cgoreceiver.Start`) | +| Sample rate | 97Hz | + +## Runtime requirements + +The stock `deploy/agent/kindling-deploy.yml` already satisfies these: + +- `privileged: true` (eBPF + `perf_event_open`) +- `hostPID: true` / host `/proc` mounts (user-space symbolization) +- host `/sys` mount; `kindling-probe-loader` mounts debugfs under `/sys/kernel/debug` if missing + +Without those, CPU stack sampling will not work even if `url` is set. + +## Probe build notes (BCC symbolizer) + +CPU symbolization uses a **vendored BCC v0.30 subset** (`bcc_syms_minimal`), not +a system `libbcc` package. + +| Piece | Path | +|-------|------| +| Fetch/build | `probe/cmake/modules/bcc_minimal.cmake` | +| Sources | `probe/src/core/symbolizer/` | +| Link into | `libkindling.so` (`probe/src/CMakeLists.txt`) | + +Build-host packages (in addition to existing probe deps such as Qt5 / cmake): + +- `elfutils-devel` / `libelf-dev` +- `zlib-devel` / `zlib1g-dev` +- network access at **cmake configure** time (FetchContent downloads BCC v0.30.0) + +The agent runtime image only needs the prebuilt `libkindling.so`; BCC is linked in statically. + +## Acceptance checklist + +| Check | How | +|-------|-----| +| Default off | Leave `endpoint.url` empty; agent logs skip / does not start CPU sampling | +| Push works | Set `url` to Pyroscope base (e.g. `http://:4040/`); after ~15s UI shows `process_cpu` | +| Symbols | Flame graph has function names or at least `module+offset` frames | +| Stop cleanly | Stop agent; sampling stops (no stuck perf attachments / growing maps) | + +Full eBPF→Pyroscope verification needs a privileged agent (see Runtime requirements) plus a reachable Pyroscope. + +## Related code + +- Collector loop: `collector/pkg/component/receiver/cgoreceiver/profiling.go` +- Sender: `collector/pkg/component/consumer/exporter/pyroscopeexporter/simple_sender.go` +- pprof builder: `collector/pkg/ebpf/pprof/` diff --git a/probe/CMakeLists.txt b/probe/CMakeLists.txt index 257f71614..01821cf39 100644 --- a/probe/CMakeLists.txt +++ b/probe/CMakeLists.txt @@ -16,7 +16,7 @@ # limitations under the License. # -cmake_minimum_required(VERSION 2.8.1) +cmake_minimum_required(VERSION 3.16) add_compile_options(-fPIC) add_compile_options(-pie) @@ -73,5 +73,6 @@ find_package(Qt5 COMPONENTS REQUIRED) include(agent-libs) +include(bcc_minimal) add_subdirectory(src) diff --git a/probe/cmake/modules/bcc_minimal.cmake b/probe/cmake/modules/bcc_minimal.cmake new file mode 100644 index 000000000..8b29ec8bc --- /dev/null +++ b/probe/cmake/modules/bcc_minimal.cmake @@ -0,0 +1,59 @@ +cmake_minimum_required(VERSION 3.16) + +option(BCC_FETCH "Fetch bcc sources at configure time" ON) + +include(ExternalProject) + +## zlib: static -fPIC +#set(_ZLIB_BUILD ${CMAKE_BINARY_DIR}/ext/zlib/build) +#ExternalProject_Add(ext_zlib +# URL https://zlib.net/zlib-1.3.1.tar.gz +# SOURCE_DIR ${CMAKE_BINARY_DIR}/ext/zlib/src +# BINARY_DIR ${_ZLIB_BUILD} +# CMAKE_ARGS -DBUILD_SHARED_LIBS=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_BUILD_TYPE=Release +# BUILD_COMMAND ${CMAKE_COMMAND} --build . -- -j +# INSTALL_COMMAND "") +#add_library(ZLIB::z STATIC IMPORTED) +#set_target_properties(ZLIB::z PROPERTIES IMPORTED_LOCATION ${_ZLIB_BUILD}/libz.a) +#add_dependencies(ZLIB::z ext_zlib) + +## elfutils libelf: static -fPIC，仅构建 libelf +#set(_ELF_SRC ${CMAKE_BINARY_DIR}/ext/elfutils/src) +#ExternalProject_Add(ext_elfutils +# URL https://sourceware.org/elfutils/ftp/0.190/elfutils-0.190.tar.bz2 +# SOURCE_DIR ${_ELF_SRC} +# CONFIGURE_COMMAND ./configure --enable-static --disable-shared +# --disable-debuginfod --disable-libdebuginfod +# --without-lzma --without-bzlib --without-zstd +# CFLAGS=-fPIC +# BUILD_COMMAND make -j +# INSTALL_COMMAND "" +# BUILD_IN_SOURCE 1) +#add_library(LIBELF::elf STATIC IMPORTED) +#set_target_properties(LIBELF::elf PROPERTIES IMPORTED_LOCATION ${_ELF_SRC}/libelf/.libs/libelf.a) +#add_dependencies(LIBELF::elf ext_elfutils) + +# 获取 bcc 源码（配置期可见），只编最小子集 +include(FetchContent) +FetchContent_Declare(bcc_src + URL https://github.com/iovisor/bcc/archive/refs/tags/v0.30.0.tar.gz) +FetchContent_GetProperties(bcc_src) +if(NOT bcc_src_POPULATED) + FetchContent_Populate(bcc_src) # 仅下载/解压，不 add_subdirectory +endif() +set(BCC_SRC_DIR ${bcc_src_SOURCE_DIR}) +set(BCC_INCLUDE_DIR "${BCC_SRC_DIR}/src/cc") + + +add_library(bcc_syms_minimal STATIC + ${BCC_SRC_DIR}/src/cc/bcc_syms.cc + ${BCC_SRC_DIR}/src/cc/bcc_elf.c + ${BCC_SRC_DIR}/src/cc/bcc_perf_map.c + ${BCC_SRC_DIR}/src/cc/bcc_proc.c + ${BCC_SRC_DIR}/src/cc/bcc_zip.c) +set_target_properties(bcc_syms_minimal PROPERTIES POSITION_INDEPENDENT_CODE ON) +target_include_directories(bcc_syms_minimal PUBLIC "${BCC_INCLUDE_DIR}") +#target_link_libraries(bcc_syms_minimal PUBLIC LIBELF::elf ZLIB::z) +find_library(LIBELF_LIB elf REQUIRED) +find_library(ZLIB_LIB z REQUIRED) +target_link_libraries(bcc_syms_minimal PUBLIC ${LIBELF_LIB} ${ZLIB_LIB}) diff --git a/probe/src/CMakeLists.txt b/probe/src/CMakeLists.txt index 31b2c5cca..51b8c8a63 100644 --- a/probe/src/CMakeLists.txt +++ b/probe/src/CMakeLists.txt @@ -1,4 +1,3 @@ - include(libscap) include(libsinsp) @@ -6,18 +5,23 @@ include_directories(.) include_directories("${LIBSINSP_INCLUDE_DIRS}") include_directories("../cgo") include_directories("../converter") +include_directories("./core") + set(SOURCE_FILES cgo/cgo_func.cpp cgo/kindling.cpp converter/cpu_converter.cpp - cgo/catch_sig.cpp cgo/catch_sig.h - cgo/utils.cpp cgo/utils.h) + cgo/catch_sig.cpp cgo/catch_sig.h + cgo/utils.cpp cgo/utils.h + core/symbolizer/bcc_per_pid_symbolizer.cpp + core/symbolizer/kindling_symbolizer.cpp) add_library(kindling SHARED ${SOURCE_FILES}) set(CMAKE_POSITION_INDEPENDENT_CODE ON) target_link_libraries(kindling sinsp - Qt5::Core) + Qt5::Core + bcc_syms_minimal) install(TARGETS kindling - DESTINATION bin) \ No newline at end of file + DESTINATION bin) diff --git a/probe/src/cgo/cgo_func.cpp b/probe/src/cgo/cgo_func.cpp index de9e8ed45..e5902a168 100644 --- a/probe/src/cgo/cgo_func.cpp +++ b/probe/src/cgo/cgo_func.cpp @@ -5,6 +5,7 @@ #include "cgo_func.h" #include "kindling.h" #include "catch_sig.h" +#include int runForGo() { return init_probe(); } @@ -27,3 +28,51 @@ void stopProfileDebug() { stop_profile_debug(); } void getCaptureStatistics(struct capture_statistics_for_go* stats) { get_capture_statistics(stats); } void catchSignalUp() { sig_set_up(); } +int getProfileData(struct sample_key key, struct bpf_profile_data* sample) { + return get_profile_data(key, sample); +} + +int getProfileKeys(struct sample_key_set *set) { + return get_profile_keys(set); +} + +void freeProfileKeys(struct sample_key_set *set) { + if (set && set->keys) { + free(set->keys); + set->keys = NULL; + } + if (set) set->nr_keys = 0; +} + +void clearProfileMap() { clear_profile_map(); } +void clearStacksMap() { clear_stacks_map(); } + +/* Keep ENABLED for API parity; open-source default path uses All. */ +void startCpuSampeling() { start_cpu_sampeling(); } +void startCpuSampelingAll() { start_cpu_sampeling_all(); } +void stopCpuSampeling() { stop_cpu_sampeling(); } + +int setProfilePids(const uint32_t *pids, uint32_t n) { + return set_profile_pids(pids, n); +} + +int symbolizerInit(void) { return symbolizer_init(); } +void symbolizerShutdown(void) { symbolizer_shutdown(); } + +int symbolizeProcessNamesWithFallback( + uint32_t pid, + const uint64_t *addrs, uint32_t n, + int debug_syms, int perf_map, int map_files, + char ***out_names, uint32_t *out_n) { + return symbolize_process_names_with_fallback( + pid, addrs, n, debug_syms, perf_map, map_files, out_names, out_n); +} + +int symbolizeKernelNames( + const uint64_t *addrs, uint32_t n, + char ***out_names, uint32_t *out_n) { + return symbolize_kernel_names(addrs, n, out_names, out_n); +} + +void freeNames(char **names, uint32_t n) { free_names(names, n); } +void maintainBccCaches(int64_t seconds) { maintain_caches(seconds); } diff --git a/probe/src/cgo/cgo_func.h b/probe/src/cgo/cgo_func.h index d9bee3d91..8f0415ca6 100644 --- a/probe/src/cgo/cgo_func.h +++ b/probe/src/cgo/cgo_func.h @@ -5,6 +5,10 @@ #ifndef SYSDIG_CGO_FUNC_H #define SYSDIG_CGO_FUNC_H +#include +#include "kindling.h" +#include "core/symbolizer/kindling_symbolizer.h" + #ifdef __cplusplus extern "C" { #endif @@ -20,8 +24,33 @@ void startProfileDebug(int pid, int tid); void stopProfileDebug(); void getCaptureStatistics(struct capture_statistics_for_go* stats); void catchSignalUp(); + +/* CPU continuous profiling CGO surface (default sampling: CAPTURE_ALL). */ +int getProfileData(struct sample_key key, struct bpf_profile_data* sample); +int getProfileKeys(struct sample_key_set *set); +void freeProfileKeys(struct sample_key_set *set); +void clearProfileMap(); +void clearStacksMap(); +void startCpuSampeling(); +void startCpuSampelingAll(); +void stopCpuSampeling(); +int setProfilePids(const uint32_t *pids, uint32_t n); + +int symbolizerInit(void); +void symbolizerShutdown(void); +int symbolizeProcessNamesWithFallback( + uint32_t pid, + const uint64_t *addrs, uint32_t n, + int debug_syms, int perf_map, int map_files, + char ***out_names, uint32_t *out_n); +int symbolizeKernelNames( + const uint64_t *addrs, uint32_t n, + char ***out_names, uint32_t *out_n); +void freeNames(char **names, uint32_t n); +void maintainBccCaches(int64_t seconds); + #ifdef __cplusplus } #endif -#endif // SYSDIG_CGO_FUNC_H \ No newline at end of file +#endif // SYSDIG_CGO_FUNC_H diff --git a/probe/src/cgo/kindling.cpp b/probe/src/cgo/kindling.cpp index 0dc361eb0..b6ed9e346 100644 --- a/probe/src/cgo/kindling.cpp +++ b/probe/src/cgo/kindling.cpp @@ -1167,3 +1167,49 @@ void get_capture_statistics(struct capture_statistics_for_go* stats) { printf("Number of threads currently being suppressed: %d \n", stats->tids_suppressed); fflush(stdout); } + +void clear_profile_map() { + if (inspector) inspector->clear_profile_map(); +} + +void clear_stacks_map() { + if (inspector) inspector->clear_stacks_map(); +} + +int get_profile_data(struct sample_key key, struct bpf_profile_data* sample) { + if (!inspector) return -1; + return inspector->get_profile_data(key, sample); +} + +int get_profile_keys(struct sample_key_set *set) { + if (!inspector) return -1; + return inspector->get_profile_keys(set); +} + +void start_cpu_sampeling() { + if (inspector) inspector->set_cpu_sampling(PROFILING_ENABLED); +} + +void start_cpu_sampeling_all() { + if (inspector) inspector->set_cpu_sampling(PROFILING_CAPTURE_ALL); +} + +void stop_cpu_sampeling() { + if (inspector) inspector->set_cpu_sampling(PROFILING_DISABLED); +} + +int set_profile_pids(const uint32_t *pids, uint32_t n) { + if (!inspector) + return -1; + inspector->clear_profile_pid_config(); + if (!pids || n == 0) + return 0; + struct pid_config cfg = {}; + cfg.type = PROFILING_TYPE_FRAMEPOINTERS; + cfg.collect = 1; + for (uint32_t i = 0; i < n; i++) { + if (inspector->set_profile_pid_config(pids[i], &cfg) != 0) + return -1; + } + return 0; +} diff --git a/probe/src/cgo/kindling.h b/probe/src/cgo/kindling.h index 97ddf5993..ddace5765 100644 --- a/probe/src/cgo/kindling.h +++ b/probe/src/cgo/kindling.h @@ -52,6 +52,16 @@ void attach_agent(int64_t pid, char* error_message, bool is_attach); void get_capture_statistics(struct capture_statistics_for_go* stats); +/* CPU continuous profiling (eBPF stack sampling → Pyroscope). */ +void clear_profile_map(); +void clear_stacks_map(); +int get_profile_data(struct sample_key key, struct bpf_profile_data* sample); +int get_profile_keys(struct sample_key_set *set); +void start_cpu_sampeling(); +void start_cpu_sampeling_all(); +void stop_cpu_sampeling(); +int set_profile_pids(const uint32_t *pids, uint32_t n); + uint16_t get_protocol(scap_l4_proto proto); uint16_t get_type(ppm_param_type type); uint16_t get_kindling_source(uint16_t etype); diff --git a/probe/src/core/symbolizer/bcc_per_pid_symbolizer.cpp b/probe/src/core/symbolizer/bcc_per_pid_symbolizer.cpp new file mode 100644 index 000000000..17d6434a3 --- /dev/null +++ b/probe/src/core/symbolizer/bcc_per_pid_symbolizer.cpp @@ -0,0 +1,304 @@ +#include "bcc_per_pid_symbolizer.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +namespace { + // 直接从 [p, q) 解析十六进制 + static inline bool parseHexU64Span(const char* p, const char* q, uint64_t& out) { + if (!p || !q || p >= q) return false; + uint64_t v = 0; + while (p < q) { + unsigned char c = static_cast(*p++); + uint64_t d; + if (c >= '0' && c <= '9') d = c - '0'; + else if (c >= 'a' && c <= 'f') d = 10 + (c - 'a'); + else if (c >= 'A' && c <= 'F') d = 10 + (c - 'A'); + else return false; + v = (v << 4) | d; + } + out = v; + return true; + } + + // 以小写十六进制将 value 追加到 dst 末尾（不带 0x 前缀） + static inline void appendHexLower(std::string& dst, uint64_t value) { + char buf[16]; + int idx = 16; + if (value == 0) { dst.push_back('0'); return; } + while (value != 0 && idx > 0) { + unsigned int digit = static_cast(value & 0xF); + value >>= 4; + buf[--idx] = static_cast(digit < 10 ? ('0' + digit) : ('a' + (digit - 10))); + } + dst.append(buf + idx, 16 - idx); + } +} + +BccPerPidSymbolizer::BccPerPidSymbolizer() { + kcache_ = bcc_symcache_new(-1, nullptr); +} +BccPerPidSymbolizer::~BccPerPidSymbolizer() { + { + for (auto& item : pid2cache_) if (item.second.cache) bcc_free_symcache(item.second.cache, item.first); + pid2cache_.clear(); + lru_.clear(); + } + if (kcache_) { bcc_free_symcache(kcache_, -1); kcache_ = nullptr; } +} + +void BccPerPidSymbolizer::lruTouch(uint32_t pid, CacheEntry& entry) { + if (entry.inLru) { + lru_.splice(lru_.end(), lru_, entry.lruIt); + return; + } + lru_.push_back(pid); + entry.lruIt = --lru_.end(); + entry.inLru = true; +} + +void BccPerPidSymbolizer::set_max_pid_caches(size_t n) { + max_pid_caches_ = n == 0 ? 1 : n; + while (pid2cache_.size() > max_pid_caches_) { + uint32_t evictPid = lru_.front(); + lru_.pop_front(); + auto it2 = pid2cache_.find(evictPid); + bcc_free_symcache(it2->second.cache, (int)evictPid); + pid2cache_.erase(it2); + } +} + +void* BccPerPidSymbolizer::getOrCreateProcCache(uint32_t pid, bool debugSyms) { + auto now = std::chrono::steady_clock::now(); + auto it = pid2cache_.find(pid); + if (it != pid2cache_.end() && it->second.cache) { + it->second.lastUsed = now; + lruTouch(pid, it->second); + return it->second.cache; + } + bcc_symbol_option opt{}; + opt.use_debug_file = debugSyms ? 1 : 0; + opt.check_debug_file_crc = 1; + opt.lazy_symbolize = 1; + opt.use_symbol_type = 0xFFFFFFFF; + void* c = bcc_symcache_new((int)pid, &opt); + if(!c){ + return nullptr; + } + + pid2cache_[pid] = CacheEntry(c, now); + lruTouch(pid, pid2cache_[pid]); + refreshMaps(pid); + + if (pid2cache_.size() > max_pid_caches_) { + uint32_t evictPid = lru_.front(); + lru_.pop_front(); + auto it2 = pid2cache_.find(evictPid); + if (it2 != pid2cache_.end()) { + if (it2->second.cache) bcc_free_symcache(it2->second.cache, (int)evictPid); + pid2cache_.erase(it2); + } + } + return c; +} + +int BccPerPidSymbolizer::resolveProcess(uint32_t pid, const uint64_t* addrs, size_t n, + std::vector& out, const ResolveOptions& opt) { + std::unique_lock lock(mu_); + if (!addrs || n == 0) { out.clear(); return 0; } + void* cache = getOrCreateProcCache(pid, opt.debugSyms); + if (!cache) return -1; + + out.resize(n); + for (size_t i = 0; i < n; i++) { + bcc_symbol sym{}; + int rc = bcc_symcache_resolve(cache, addrs[i], &sym); + if (rc == 0) { + const char* nm = sym.demangle_name ? sym.demangle_name : sym.name; + if (nm) out[i].assign(nm); else out[i].clear(); + bcc_symbol_free_demangle_name(&sym); + } else { + if (opt.formatUnresolvedAsModulePlusOffset) { + uint64_t fb; + if (fallbackByMaps(pid, addrs[i], fb)) { + out[i].clear(); + if (sym.module) { + const char* mod = sym.module; + size_t ml = std::strlen(mod); + out[i].reserve(ml + 3 + 16); + out[i].append(mod, ml); + } else { + out[i].reserve(6 + 3 + 16); + out[i].append("", 6); + } + out[i].append("+0x", 3); + appendHexLower(out[i], fb); + } else { + out[i].clear(); + out.resize(i); + break; + } + } else { + if(sym.module){ + out[i].assign(sym.module); + }else{ + out[i].clear(); + out.resize(i); + break; + } + } + } + } + return 0; +} + +int BccPerPidSymbolizer::resolveKernel(const uint64_t* addrs, size_t n, + std::vector& out) { + if (!addrs || n == 0) { out.clear(); return 0; } + if (!kcache_) return -1; + out.resize(n); + for (size_t i = 0; i < n; i++) { + bcc_symbol sym{}; + int rc = bcc_symcache_resolve(kcache_, addrs[i], &sym); + if (rc == 0) { + const char* nm = sym.demangle_name ? sym.demangle_name : sym.name; + out[i] = nm ? nm : ""; + bcc_symbol_free_demangle_name(&sym); + } else { + out[i].clear(); + out.resize(i); + break; + } + } + return 0; +} + +bool BccPerPidSymbolizer::refreshPid(uint32_t pid) { + auto it = pid2cache_.find(pid); + if (it != pid2cache_.end() && it->second.cache) bcc_symcache_refresh(it->second.cache); + else return false; + // 同步刷新 maps + it->second.lastUsed = std::chrono::steady_clock::now(); + lruTouch(pid, it->second); + refreshMaps(pid); + return true; +} + +void BccPerPidSymbolizer::releasePid(uint32_t pid) { + auto it = pid2cache_.find(pid); + if (it != pid2cache_.end()) { + if (it->second.cache) { + bcc_free_symcache(it->second.cache, (int)pid); + lru_.erase(it->second.lruIt); + } + pid2cache_.erase(it); + } +} + +void BccPerPidSymbolizer::maintainCaches(std::chrono::seconds ttl) { + std::unique_lock lock(mu_); + auto now = std::chrono::steady_clock::now(); + // 因为 LRU 顺序与 lastUsed 一致（每次 touch 都挪到尾），可以从头部按需清理 + auto it = lru_.begin(); + while (it != lru_.end()) { + uint32_t pid = *it; + auto mit = pid2cache_.find(pid); + if (mit == pid2cache_.end()) { it = lru_.erase(it); continue; } + if ((now - mit->second.lastUsed) > ttl) { + if (mit->second.cache) bcc_free_symcache(mit->second.cache, (int)pid); + it = lru_.erase(it); + pid2cache_.erase(mit); + } else { + break; + } + } + + // 将剩余（活跃）pid 收集后再统一刷新，避免遍历中移动 LRU 迭代器, 刷新失败会删除。 + std::vector toRefresh; + toRefresh.reserve(lru_.size()); + for (auto it2 = it; it2 != lru_.end(); ++it2) toRefresh.push_back(*it2); + + for (uint32_t rpid : toRefresh) { + if(!refreshPid(rpid))releasePid(rpid); + } + + if (kcache_)bcc_symcache_refresh(kcache_); + else kcache_ = bcc_symcache_new(-1, nullptr);//todo: 报错原因未分类，如果因为权限原因 应该不再重新创建。 +} + +void BccPerPidSymbolizer::refreshMaps(uint32_t pid) { + std::ostringstream oss; + oss << "/proc/" << pid << "/maps"; + std::ifstream ifs(oss.str()); + if (!ifs) return; + + auto it = pid2cache_.find(pid); + if (it == pid2cache_.end()) { + fprintf(stderr, "pid = %u, bcc cache is NULL!\n", pid); + return; + } + + auto& cache = it->second; + std::vector regions; + std::string line; + while (std::getline(ifs, line)) { + const char* s = line.c_str(); + const char* e = s + line.size(); + // start-end + const char* dash = (const char*)::memchr(s, '-', (size_t)(e - s)); + if (!dash) continue; + const char* p = dash + 1; + while (p < e && *p == ' ') ++p; // should not be spaces, safeguard + const char* sp1 = p; while (p < e && *p != ' ') ++p; const char* sp1_end = p; // end hex + while (p < e && *p == ' ') ++p; + const char* perms_beg = p; while (p < e && *p != ' ') ++p; const char* perms_end = p; + while (p < e && *p == ' ') ++p; + const char* off_beg = p; while (p < e && *p != ' ') ++p; const char* off_end = p; + + uint64_t start = 0, endv = 0, pgoff = 0; + if (!parseHexU64Span(s, dash, start)) continue; + if (!parseHexU64Span(sp1, sp1_end, endv)) continue; + if (!parseHexU64Span(off_beg, off_end, pgoff)) continue; + + bool x = false; + for (const char* t = perms_beg; t < perms_end; ++t) { if (*t == 'x') { x = true; break; } } + + MapRegion r; + r.start = start; + r.end = endv; + r.pgoff = pgoff; + r.isExecutable = x; + regions.push_back(std::move(r)); + } + cache.maps = std::move(regions); +} + +bool BccPerPidSymbolizer::fallbackByMaps(uint32_t pid, uint64_t addr, uint64_t& off) { + auto it = pid2cache_.find(pid); + if (it == pid2cache_.end() || it->second.maps.empty()) { + return false; + } + + const auto& regions = it->second.maps; + // 二分查找：找到最后一个 start <= addr 的区间，并检查 addr < end + size_t lo = 0, hi = regions.size(); + while (lo < hi) { + size_t mid = lo + (hi - lo) / 2; + if (regions[mid].start <= addr) { + lo = mid + 1; + } else { + hi = mid; + } + } + if (lo == 0) return false; + const auto& r = regions[lo - 1]; + if (!(addr < r.end)) return false; + + off = (addr - r.start) + r.pgoff; + return true; +} diff --git a/probe/src/core/symbolizer/bcc_per_pid_symbolizer.h b/probe/src/core/symbolizer/bcc_per_pid_symbolizer.h new file mode 100644 index 000000000..0b29af532 --- /dev/null +++ b/probe/src/core/symbolizer/bcc_per_pid_symbolizer.h @@ -0,0 +1,81 @@ +#pragma once +#include +#include +#include +#include +#include +extern "C" { +#include "bcc_syms.h" +} +#define MAX_PID_CACHES_DEFAULT 2048 + +struct ResolveOptions { + bool debugSyms = true; // bcc_symbol_option.use_debug_file + bool formatUnresolvedAsModulePlusOffset = true; // 轻回退：若有 module 则拼 "module+0xoff" +}; + +class BccPerPidSymbolizer { +public: + BccPerPidSymbolizer(); + ~BccPerPidSymbolizer(); + + // 进程符号解析 + int resolveProcess(uint32_t pid, const uint64_t* addrs, size_t n, + std::vector& out, const ResolveOptions& opt); + + // 内核符号解析 + int resolveKernel(const uint64_t* addrs, size_t n, std::vector& out); + + // 显式刷新某 pid（dlopen/perf-map 变化时调用） + bool refreshPid(uint32_t pid); + + // 释放某 pid 的缓存（进程退出/长时间不用） + void releasePid(uint32_t pid); + + // 定期清理：超过 ttl 未使用的缓存回收 + void maintainCaches(std::chrono::seconds ttl); + + // 设置/获取 LRU 容量（单位：pid 数） + void set_max_pid_caches(size_t n); + +private: + void* getOrCreateProcCache(uint32_t pid, bool debugSyms); + static std::string formatModulePlusOffset(const bcc_symbol& sym); + + // -------------------- /proc//maps 回退缓存 -------------------- + struct MapRegion { + uint64_t start = 0; + uint64_t end = 0; + uint64_t pgoff = 0; // 文件内偏移（十六进制列） + //std::string path; // 用bcc module代替, 映射文件路径（可能为空/匿名） + bool isExecutable = false; // perms 是否包含 'x' + }; + + // 解析并刷新指定 pid 的 maps 缓存 + void refreshMaps(uint32_t pid); + + // 使用 maps 回退：将 addr 映射为 "basename+0xoffset"，成功返回 true + bool fallbackByMaps(uint32_t pid, uint64_t addr, uint64_t& out); + + struct CacheEntry { + void* cache = nullptr; + std::chrono::steady_clock::time_point lastUsed; + std::vector maps; + std::list::iterator lruIt; // 指向 lru_ 中该 pid 的位置 + bool inLru = false; + CacheEntry() : cache(nullptr), lastUsed(), maps() {} + CacheEntry(void* c, std::chrono::steady_clock::time_point t) + : cache(c), lastUsed(t), maps(), inLru(false) {} + }; + + // LRU 触达：将 pid 移动到末尾，必要时插入 + void lruTouch(uint32_t pid, CacheEntry& entry); + + // 当前容量（可运行时修改） + size_t max_pid_caches_ = MAX_PID_CACHES_DEFAULT; + + std::mutex mu_; + std::unordered_map pid2cache_; + std::list lru_; // LRU 次序：front 最久未用，back 最近使用 + void* kcache_ = nullptr; // kernel cache (pid = -1) +}; diff --git a/probe/src/core/symbolizer/kindling_symbolizer.cpp b/probe/src/core/symbolizer/kindling_symbolizer.cpp new file mode 100644 index 000000000..9a1148878 --- /dev/null +++ b/probe/src/core/symbolizer/kindling_symbolizer.cpp @@ -0,0 +1,95 @@ +#include "kindling_symbolizer.h" +#include +#include +#include +#include +#include + +static std::unique_ptr g_sym; + +static size_t trim_trailing_zeros(const uint64_t* addrs, size_t n) { + while (n > 0 && addrs[n - 1] == 0) n--; + return n; +} + +static int copy_out(const std::vector& names, char*** out_names, uint32_t* out_n) { + const size_t cnt = names.size(); + char** arr = cnt ? (char**)std::calloc(cnt , sizeof(char*)) : nullptr; + if (!arr && cnt) return -1; + for (size_t i = 0; i < cnt; i++) { + const std::string& s = names[i]; + arr[i] = (char*)std::malloc(s.size() + 1); + if (!arr[i]) { for (size_t j = 0; j < i; j++) std::free(arr[j]); std::free(arr); return -1; } + std::memcpy(arr[i], s.c_str(), s.size()); + arr[i][s.size()] = '\0'; + } + *out_names = arr; + *out_n = (uint32_t)cnt; + return 0; +} + +int symbolizer_init() { + if (!g_sym) g_sym.reset(new BccPerPidSymbolizer()); + return g_sym ? 0 : -1; +} + +void symbolizer_shutdown() { + g_sym.reset(); +} + +int symbolize_process_names( + uint32_t pid, + const uint64_t* addrs, size_t n, + int debug_syms, int /*perf_map*/, int /*map_files*/, + char*** out_names, uint32_t* out_n, ResolveOptions* opt) { + + if (!addrs || n == 0) { *out_names = nullptr; *out_n = 0; return 0; } + if (!g_sym && symbolizer_init() != 0) return -1; + + n = trim_trailing_zeros(addrs, n); + if (n == 0) { *out_names = nullptr; *out_n = 0; return 0; } + + std::vector names; + int rc = g_sym->resolveProcess(pid, addrs, n, names, *opt); + if (rc != 0) return rc; + return copy_out(names, out_names, out_n); +} + +int symbolize_process_names_with_fallback( + uint32_t pid, + const uint64_t* addrs, size_t n, + int debug_syms, int /*perf_map*/, int /*map_files*/, + char*** out_names, uint32_t* out_n) { + ResolveOptions opt; + opt.debugSyms = (debug_syms != 0); + opt.formatUnresolvedAsModulePlusOffset = true; + return symbolize_process_names(pid, addrs, n, debug_syms, 0, 0, out_names, out_n, &opt); +} + +int symbolize_kernel_names( + const uint64_t* addrs, size_t n, + char*** out_names, uint32_t* out_n) { + + if (!addrs || n == 0) { *out_names = nullptr; *out_n = 0; return 0; } + if (!g_sym && symbolizer_init() != 0) return -1; + + n = trim_trailing_zeros(addrs, n); + if (n == 0) { *out_names = nullptr; *out_n = 0; return 0; } + + std::vector names; + int rc = g_sym->resolveKernel(addrs, n, names); + if (rc != 0) return rc; + return copy_out(names, out_names, out_n); +} + +void free_names(char** names, uint32_t n) { + if (!names) return; + for (uint32_t i = 0; i < n; i++) std::free(names[i]); + std::free(names); +} + +void maintain_caches(int64_t seconds){ + if (!g_sym) return; + std::chrono::seconds ttl{seconds}; + g_sym->maintainCaches(ttl); +} \ No newline at end of file diff --git a/probe/src/core/symbolizer/kindling_symbolizer.h b/probe/src/core/symbolizer/kindling_symbolizer.h new file mode 100644 index 000000000..b04d8e7ce --- /dev/null +++ b/probe/src/core/symbolizer/kindling_symbolizer.h @@ -0,0 +1,33 @@ +// +// Created by zhaoxiangyu on 25-8-18. +// + +#ifndef KINDLING_PROBE_KINDLING_SYMBOLIZER_H +#define KINDLING_PROBE_KINDLING_SYMBOLIZER_H + +#include +#include "symbolizer/bcc_per_pid_symbolizer.h" + +int symbolizer_init(); +void symbolizer_shutdown(); + +int symbolize_process_names( + uint32_t pid, + const uint64_t* addrs, size_t n, + int debug_syms, int perf_map, int map_files, + char*** out_names, uint32_t* out_n, ResolveOptions* opt); + +int symbolize_process_names_with_fallback( + uint32_t pid, + const uint64_t *addrs, size_t n, + int debug_syms, int perf_map, int map_files, + char ***out_names, uint32_t *out_n); + +int symbolize_kernel_names( + const uint64_t *addrs, size_t n, + char ***out_names, uint32_t *out_n); + +void free_names(char **names, uint32_t n); + +void maintain_caches(int64_t seconds); +#endif //KINDLING_PROBE_KINDLING_SYMBOLIZER_H