Add ditributed tracing to walshadow

Author: harshil-goel

Cargo.lock

@@ -61,6 +61,10 @@ toml = { version = "1", default-features = false, features = ["parse", "serde"]
 tracing = "0.1"
 tracing-subscriber = { version = "0.3", default-features = false, features = ["env-filter", "fmt", "ansi"] }
 zstd = "0.13"
+opentelemetry = "0.32"
+opentelemetry_sdk = { version = "0.32", features = ["rt-tokio"] }
+opentelemetry-otlp = { version = "0.32", default-features = false, features = ["grpc-tonic", "trace"] }
+tracing-opentelemetry = { version = "0.33", default-features = false }
 
 [dev-dependencies]
 tempfile = "3"

Cargo.toml

@@ -14,10 +14,10 @@ write_files:
           email       String,
           _lsn        UInt64,
           _xid        UInt32,
-          _op         Enum8('insert' = 1, 'update' = 2, 'delete' = 3),
-          _commit_ts  DateTime64(6, 'UTC')
+          _commit_ts  DateTime64(6, 'UTC'),
+          _is_deleted Bool
       )
-      ENGINE = ReplacingMergeTree(_lsn)
+      ENGINE = ReplacingMergeTree(_lsn, _is_deleted)
       ORDER BY id;
 runcmd:
   - export DEBIAN_FRONTEND=noninteractive

bench/ec2/ec2-clickhouse/cloud-init.yaml

@@ -21,7 +21,11 @@ source ./state.env   # PUBLIC_IP, KEY_NAME, ...
 source ../lib.sh
 
 IMAGE="${IMAGE:-walshadow:local}"
-FLUSH_TIMEOUT_MS="${FLUSH_TIMEOUT_MS:-50}"
+FLUSH_TIMEOUT_MS="${FLUSH_TIMEOUT_MS:-1000}"
+JAEGER_IMAGE="${JAEGER_IMAGE:-jaegertracing/all-in-one:1.57}"
+JAEGER_MAX_TRACES="${JAEGER_MAX_TRACES:-50000}"
+JAEGER_MEMORY="${JAEGER_MEMORY:-1g}"
+TRACE_SAMPLE_RATIO="${TRACE_SAMPLE_RATIO:-0.01}"
 node_ssh_setup
 
 SRC_PRIV="${SOURCE_PRIVATE_IP:-$(read_state_var ../ec2-source-pg/state.env SOURCE_PRIVATE_IP)}"
@@ -65,16 +69,61 @@ columns = [
 ]
 EOF
 
+# Memory-bound Jaeger (in-memory storage capped at JAEGER_MAX_TRACES, RAM at
+# JAEGER_MEMORY) on walshadow-net so the daemon ships spans to it by name.
+"${SSH[@]}" "sudo docker network inspect walshadow-net >/dev/null 2>&1 || sudo docker network create walshadow-net"
+"${SSH[@]}" "sudo docker rm -f jaeger >/dev/null 2>&1 || true; sudo docker run -d --name jaeger --restart unless-stopped \
+  --network walshadow-net \
+  --memory '$JAEGER_MEMORY' \
+  -e COLLECTOR_OTLP_ENABLED=true \
+  -e SPAN_STORAGE_TYPE=memory \
+  -e MEMORY_MAX_TRACES='$JAEGER_MAX_TRACES' \
+  -p 16686:16686 -p 4317:4317 \
+  $JAEGER_IMAGE >/dev/null && echo 'jaeger started'"
+
 "${SSH[@]}" "sudo docker rm -f walshadow >/dev/null 2>&1 || true; sudo docker run -d --name walshadow --restart unless-stopped \
+  --network walshadow-net \
   -e RUST_LOG='warn,walshadow=info' \
   -e WALSHADOW_SOURCE_HOST='$SRC_PRIV' \
   -e WALSHADOW_SOURCE_PORT=5432 \
+  -e OTEL_EXPORTER_OTLP_ENDPOINT='http://jaeger:4317' \
   -v /opt/walshadow/ch-config.toml:/etc/walshadow/ch-config.toml:ro \
   -v walshadow-data:/var/lib/walshadow \
   -p 9484:9484 \
-  $IMAGE >/dev/null && echo started"
+  $IMAGE --trace-sample-ratio '$TRACE_SAMPLE_RATIO' >/dev/null && echo started"
+
+# Grafana + Prometheus: only uploaded/recreated when FORCE_METRICS=1.
+if [ "${FORCE_METRICS:-0}" = "1" ]; then
+  GRAFANA_IMAGE="${GRAFANA_IMAGE:-grafana/grafana:13.0.2}"
+  PROM_IMAGE="${PROM_IMAGE:-prom/prometheus:v3.12.0}"
+  REPO_ROOT="$(cd ../../.. && pwd)"
+
+  "${SSH[@]}" "sudo docker network inspect walshadow-net >/dev/null 2>&1 || sudo docker network create walshadow-net"
+  "${SSH[@]}" "sudo docker network connect walshadow-net walshadow 2>/dev/null || true"
+
+  tar -C "$REPO_ROOT/docker" -czf - grafana prometheus \
+    | "${SSH[@]}" "sudo install -d /opt/walshadow/obs && sudo tar -C /opt/walshadow/obs -xzf -"
+  # compose hostname 'clickhouse' -> private IP
+  "${SSH[@]}" "sudo grep -rl clickhouse /opt/walshadow/obs 2>/dev/null | sudo xargs -r sed -i 's/clickhouse:/$CH_PRIV:/g; s#//clickhouse#//$CH_PRIV#g'"
+
+  "${SSH[@]}" "sudo docker rm -f prometheus >/dev/null 2>&1 || true; sudo docker run -d --name prometheus --restart unless-stopped \
+    --network walshadow-net \
+    -v /opt/walshadow/obs/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro \
+    -p 9090:9090 \
+    $PROM_IMAGE >/dev/null && echo 'prometheus started'"
+
+  "${SSH[@]}" "sudo docker rm -f grafana >/dev/null 2>&1 || true; sudo docker run -d --name grafana --restart unless-stopped \
+    --network walshadow-net \
+    -e GF_AUTH_ANONYMOUS_ENABLED=true -e GF_AUTH_ANONYMOUS_ORG_ROLE=Admin \
+    -e GF_INSTALL_PLUGINS=grafana-clickhouse-datasource \
+    -v /opt/walshadow/obs/grafana/provisioning:/etc/grafana/provisioning:ro \
+    -v /opt/walshadow/obs/grafana/dashboards:/var/lib/grafana/dashboards:ro \
+    -p 3000:3000 \
+    $GRAFANA_IMAGE >/dev/null && echo 'grafana started'"
+fi
 
 echo
 echo "=== deployed ==="
 echo "metrics:  curl http://$PUBLIC_IP:9484/metrics"
+echo "traces:   http://$PUBLIC_IP:16686  (jaeger)"
 echo "logs:     ssh -i $PEM ubuntu@$PUBLIC_IP 'sudo docker logs -f walshadow'"