Add ditributed tracing to walshadow

Author: harshil-goel

Cargo.lock

@@ -62,6 +62,10 @@ toml = { version = "1", default-features = false, features = ["parse", "serde"]
 tracing = "0.1"
 tracing-subscriber = { version = "0.3", default-features = false, features = ["env-filter", "fmt", "ansi"] }
 zstd = "0.13"
+opentelemetry = "0.32"
+opentelemetry_sdk = { version = "0.32", features = ["rt-tokio"] }
+opentelemetry-otlp = { version = "0.32", default-features = false, features = ["grpc-tonic", "trace"] }
+tracing-opentelemetry = { version = "0.33", default-features = false }
 
 [dev-dependencies]
 tempfile = "3"

Cargo.toml

@@ -21,7 +21,7 @@ source ./state.env   # PUBLIC_IP, KEY_NAME, ...
 source ../lib.sh
 
 IMAGE="${IMAGE:-walshadow:local}"
-FLUSH_TIMEOUT_MS="${FLUSH_TIMEOUT_MS:-50}"
+FLUSH_TIMEOUT_MS="${FLUSH_TIMEOUT_MS:-1000}"
 node_ssh_setup
 
 SRC_PRIV="${SOURCE_PRIVATE_IP:-$(read_state_var ../ec2-source-pg/state.env SOURCE_PRIVATE_IP)}"

bench/ec2/ec2-walshadow/deploy.sh

@@ -1,6 +1,7 @@
-# Config for the walshadow daemon host. SSH + Prometheus metrics (:9484) from
-# the operator only; it emits into ClickHouse OUTBOUND over the VPC.
+# Config for the walshadow daemon host. SSH + walshadow /metrics (:9484) +
+# the co-located observability UIs — Grafana (:3000), Prometheus (:9090) —
+# all from the operator only; it emits into ClickHouse OUTBOUND over the VPC.
 NODE_NAME=walshadow-daemon
-SG_DESC="walshadow daemon: SSH + Prometheus metrics from my IP"
-INGRESS="22:my 9484:my"
+SG_DESC="walshadow daemon: SSH + metrics + grafana/prometheus from my IP"
+INGRESS="22:my 9484:my 3000:my 9090:my"
 NODE_NEXT="next:      ./deploy.sh ships the image + config and starts the daemon"

bench/ec2/ec2-walshadow/node.env

@@ -33,14 +33,33 @@ sudo sysctl -w kernel.perf_event_paranoid=-1 kernel.kptr_restrict=0 >/dev/null 2
 PIDS=$(sudo docker top walshadow -eo pid --no-headers 2>/dev/null | awk '{print $1}' | paste -sd,)
 [ -n "$PIDS" ] || PIDS=$(sudo pgrep -f 'walshadow-stream|postgres' 2>/dev/null | paste -sd,)
 [ -n "$PIDS" ] || { echo "no walshadow processes found — is the daemon up?" >&2; exit 1; }
-# The walshadow-stream daemon (decode/insert engine) for the single-PID eBPF profiler.
+# The walshadow-stream daemon (decode/insert engine) for the single-PID eBPF
+# profiler. Required and must be live — profiling with no walshadow-stream
+# process is meaningless, so fail loudly here rather than producing empty output.
 WS=$(sudo pgrep -f 'walshadow-stream' 2>/dev/null | head -1 || true)
 [ -n "$WS" ] || WS=$(sudo docker inspect -f '{{.State.Pid}}' walshadow 2>/dev/null || true)
-echo "walshadow pids (perf): $PIDS    walshadow-stream pid (eBPF): ${WS:-none}"
+{ [ -n "$WS" ] && sudo test -d "/proc/$WS"; } \
+  || { echo "walshadow-stream daemon not running — nothing to profile (is the walshadow container up?)" >&2; exit 1; }
+echo "walshadow pids (perf): $PIDS    walshadow-stream pid (eBPF): $WS"
 
 sudo nohup bash -c "
-  { [ -n \"$WS\" ] && profile-bpfcc -F 99 -f -p $WS $DUR > $OUT/oncpu-walshadow-$TS.folded 2>$OUT/oncpu-$TS.log ; } &
-  perf record -F 99 -g -p $PIDS -o $OUT/perf-$TS.data -- sleep $DUR 2>>$OUT/perf-$TS.log \
+  profile-bpfcc -F 99 -f -p $WS $DUR > $OUT/oncpu-walshadow-$TS.folded 2>$OUT/oncpu-$TS.log &
+  # Re-filter the PID list to processes still alive RIGHT NOW: perf -p aborts the
+  # whole record if any one PID has exited (transient PG backends churn between
+  # the snapshot above and this point) → 'Couldn't create thread/CPU maps'. The
+  # walshadow-stream PID is asserted live above, so the list is never empty.
+  LIVE=\"\"; for x in \$(echo '$PIDS' | tr ',' ' '); do [ -d /proc/\$x ] && LIVE=\"\$LIVE,\$x\"; done; LIVE=\${LIVE#,}
+  # DWARF call-graph: walshadow links musl and has no frame pointers, so -g/fp
+  # unwinding dead-ends at the leaf (memcpy/[unknown]). DWARF unwinds via the
+  # binary's .eh_frame (present — default panic=unwind). Tuning that matters:
+  #   * dwarf,65528 — large per-sample stack dump. Rust async frames are big;
+  #     a small dump (8192) truncates the unwind so callers above memcpy are lost.
+  #   * -m 64M — ring buffer big enough for those dumps (avoids a wakeup storm).
+  #   * --no-buildid* — skip perf's end-of-record build-id/addr2line finalization,
+  #     which fails on this host and left the file unfinalized ('data size 0').
+  #     Safe: we run `perf script` on this same host, resolving symbols live.
+  perf record -F 99 --call-graph dwarf,65528 -m 64M --no-buildid --no-buildid-cache \
+    -p \"\$LIVE\" -o $OUT/perf-$TS.data -- sleep $DUR 2>>$OUT/perf-$TS.log \
     || echo 'perf record failed (see log)' >>$OUT/perf-$TS.log
   wait
   chown -R ubuntu $OUT