chore(ci): add telemetry-enabled perf task with local HTTPS server MONGOSH-3454

.evergreen.yml

@@ -14,17 +14,17 @@ post:
       shell: bash
       script: |
         mkdir -p npm-logs
-        tar cvzf npm-logs.tgz npm-logs
+        tar czf npm-logs.tgz npm-logs
 
         if [ -d "src/tmp/mongodb-runner/logs" ]; then
-          tar cvzf mongodb-runner-logs.tgz -C src/tmp/mongodb-runner logs
+          tar czf mongodb-runner-logs.tgz -C src/tmp/mongodb-runner logs
         else
           echo "Directory src/tmp/logs does not exist. Skipping."
         fi
 
         if [ -d "$HOME/.mongodb/mongosh" ]; then
           TARGET="$PWD/mongosh-logs.tgz"
-          (cd $HOME/.mongodb && tar cvzf "$TARGET" mongosh/*_log)
+          (cd $HOME/.mongodb && tar czf "$TARGET" mongosh/*_log)
         else
           echo "Directory ~/.mongodb/mongosh does not exist. Skipping."
         fi
@@ -4902,49 +4902,20 @@ functions:
           .evergreen/run-evergreen-release.sh publish
 
   run_perf_tests:
-    - command: shell.exec
+    - command: subprocess.exec
       params:
         working_dir: src
-        shell: bash
-        script: |
-          set -e
-          set -x
-          npx -y mongodb-runner exec -t standalone --version=7.0.x-enterprise -- \
-            sh -c 'MONGOSH_SMOKE_TEST_SERVER="$MONGODB_URI" ./dist/mongosh --perfTests > perf_results.json'
+        binary: bash
+        args: [".evergreen/perf-test.sh"]
         env:
           DISTRO_ID: ${distro_id}
-    - command: shell.exec
+          MONGOSH_PERF_TELEMETRY: ${mongosh_perf_telemetry|}
+    - command: subprocess.exec
       params:
-        script: |
-          # We use the requester expansion to determine whether the data is from a mainline evergreen run or not
-          if [ "${requester}" == "commit" ]; then
-            is_mainline=true
-          else
-            is_mainline=false
-          fi
-
-          # Parse the username out of the order_id. Patches append the username. The Signal Processing Service (SPS) endpoint does not need the other information.
-          parsed_order_id=$(echo "${revision_order_id}" | awk -F'_' '{print $NF}')
-
-          # Submit the performance data to the SPS endpoint
-          response=$(curl -s -w "\nHTTP_STATUS:%{http_code}" -X 'POST' \
-            "https://performance-monitoring-api.corp.mongodb.com/raw_perf_results/cedar_report?project=${project_id}&version=${version_id}&variant=${build_variant}&order=$parsed_order_id&task_name=${task_name}&task_id=${task_id}&execution=${execution}&mainline=$is_mainline" \
-            -H 'accept: application/json' \
-            -H 'Content-Type: application/json' \
-            -d @src/perf_results.json)
-
-          http_status=$(echo "$response" | grep "HTTP_STATUS" | awk -F':' '{print $2}')
-          response_body=$(echo "$response" | sed '/HTTP_STATUS/d')
-
-          # We want to throw an error if the data was not successfully submitted
-          if [ "$http_status" -ne 200 ]; then
-            echo "Error: Received HTTP status $http_status"
-            echo "Response Body: $response_body"
-            exit 1
-          fi
+        add_expansions_to_env: true
+        binary: bash
+        args: ["src/.evergreen/perf-send.sh"]
 
-          echo "Response Body: $response_body"
-          echo "HTTP Status: $http_status"
 
 # Tasks will show up as the individual blocks in the Evergreen UI that can
 # pass or fail.
@@ -6349,6 +6320,24 @@ tasks:
         vars:
           executable_os_id: "linux-x64"
       - func: run_perf_tests
+  # Same benchmarks with telemetry enabled against a local HTTPS sink, so
+  # mainline produces with/without-telemetry perf series (MONGOSH-3454).
+  - name: perf_tests_linux_x64_telemetry
+    tags: ["perf-test"]
+    depends_on:
+      - name: compile_artifact
+        variant: build_linux_x64
+    commands:
+      - func: checkout
+      - func: install
+        vars:
+          node_js_version: "24.18.0"
+      - func: download_compiled_artifact
+        vars:
+          executable_os_id: "linux-x64"
+      - func: run_perf_tests
+        vars:
+          mongosh_perf_telemetry: "1"
 
   ###
   # EXECUTABLE CONNECTIVITY TESTS
@@ -12991,3 +12980,4 @@ buildvariants:
     run_on: rhel90-dbx-perf-large
     tasks:
       - name: perf_tests_linux_x64
+      - name: perf_tests_linux_x64_telemetry

.evergreen/evergreen.yml.in

@@ -24,17 +24,17 @@ post:
       shell: bash
       script: |
         mkdir -p npm-logs
-        tar cvzf npm-logs.tgz npm-logs
+        tar czf npm-logs.tgz npm-logs
 
         if [ -d "src/tmp/mongodb-runner/logs" ]; then
-          tar cvzf mongodb-runner-logs.tgz -C src/tmp/mongodb-runner logs
+          tar czf mongodb-runner-logs.tgz -C src/tmp/mongodb-runner logs
         else
           echo "Directory src/tmp/logs does not exist. Skipping."
         fi
 
         if [ -d "$HOME/.mongodb/mongosh" ]; then
           TARGET="$PWD/mongosh-logs.tgz"
-          (cd $HOME/.mongodb && tar cvzf "$TARGET" mongosh/*_log)
+          (cd $HOME/.mongodb && tar czf "$TARGET" mongosh/*_log)
         else
           echo "Directory ~/.mongodb/mongosh does not exist. Skipping."
         fi
@@ -1058,49 +1058,20 @@ functions:
           .evergreen/run-evergreen-release.sh publish
 
   run_perf_tests:
-    - command: shell.exec
+    - command: subprocess.exec
       params:
         working_dir: src
-        shell: bash
-        script: |
-          set -e
-          set -x
-          npx -y mongodb-runner exec -t standalone --version=7.0.x-enterprise -- \
-            sh -c 'MONGOSH_SMOKE_TEST_SERVER="$MONGODB_URI" ./dist/mongosh --perfTests > perf_results.json'
+        binary: bash
+        args: [".evergreen/perf-test.sh"]
         env:
           DISTRO_ID: ${distro_id}
-    - command: shell.exec
+          MONGOSH_PERF_TELEMETRY: ${mongosh_perf_telemetry|}
+    - command: subprocess.exec
       params:
-        script: |
-          # We use the requester expansion to determine whether the data is from a mainline evergreen run or not
-          if [ "${requester}" == "commit" ]; then
-            is_mainline=true
-          else
-            is_mainline=false
-          fi
-
-          # Parse the username out of the order_id. Patches append the username. The Signal Processing Service (SPS) endpoint does not need the other information.
-          parsed_order_id=$(echo "${revision_order_id}" | awk -F'_' '{print $NF}')
-
-          # Submit the performance data to the SPS endpoint
-          response=$(curl -s -w "\nHTTP_STATUS:%{http_code}" -X 'POST' \
-            "https://performance-monitoring-api.corp.mongodb.com/raw_perf_results/cedar_report?project=${project_id}&version=${version_id}&variant=${build_variant}&order=$parsed_order_id&task_name=${task_name}&task_id=${task_id}&execution=${execution}&mainline=$is_mainline" \
-            -H 'accept: application/json' \
-            -H 'Content-Type: application/json' \
-            -d @src/perf_results.json)
-
-          http_status=$(echo "$response" | grep "HTTP_STATUS" | awk -F':' '{print $2}')
-          response_body=$(echo "$response" | sed '/HTTP_STATUS/d')
-
-          # We want to throw an error if the data was not successfully submitted
-          if [ "$http_status" -ne 200 ]; then
-            echo "Error: Received HTTP status $http_status"
-            echo "Response Body: $response_body"
-            exit 1
-          fi
+        add_expansions_to_env: true
+        binary: bash
+        args: ["src/.evergreen/perf-send.sh"]
 
-          echo "Response Body: $response_body"
-          echo "HTTP Status: $http_status"
 
 # Tasks will show up as the individual blocks in the Evergreen UI that can
 # pass or fail.
@@ -1346,6 +1317,24 @@ tasks:
         vars:
           executable_os_id: "linux-x64"
       - func: run_perf_tests
+  # Same benchmarks with telemetry enabled against a local HTTPS sink, so
+  # mainline produces with/without-telemetry perf series (MONGOSH-3454).
+  - name: perf_tests_linux_x64_telemetry
+    tags: ["perf-test"]
+    depends_on:
+      - name: compile_artifact
+        variant: build_linux_x64
+    commands:
+      - func: checkout
+      - func: install
+        vars:
+          node_js_version: "<% out(NODE_JS_VERSION_TARGET) %>"
+      - func: download_compiled_artifact
+        vars:
+          executable_os_id: "linux-x64"
+      - func: run_perf_tests
+        vars:
+          mongosh_perf_telemetry: "1"
 
   ###
   # EXECUTABLE CONNECTIVITY TESTS
@@ -1848,3 +1837,4 @@ buildvariants:
     run_on: rhel90-dbx-perf-large
     tasks:
       - name: perf_tests_linux_x64
+      - name: perf_tests_linux_x64_telemetry

.evergreen/perf-get.sh

@@ -0,0 +1,36 @@
+#! /usr/bin/env bash
+
+# Fetch raw performance results for a given Evergreen version/patch from the
+# Signal Processing Service (SPS). This is the same data submitted by
+# perf-send.sh; SPS exposes it read-only, keyed by version id, at:
+#
+#   GET ${PERF_API}/raw_perf_results/versions/<version-id>
+#
+# Reachability is network-gated (corp network / VPN), not token-authed.
+#
+# Usage:
+#   .evergreen/perf-get.sh <version-id>
+#
+#   <version-id>  Evergreen version (or patch) id. This is the 'version' field
+#                 embedded in the perf artifact filenames and the 'version_id'
+#                 expansion of the run.
+#
+# Environment:
+#   PERF_API  Override the base URL (default: the corp SPS endpoint).
+#
+# The raw JSON is written to stdout; filter it with jq as needed, e.g.:
+#   .evergreen/perf-get.sh 6a617fb73e1d2a0007d3c6ca \
+#     | jq '[.[] | select(.info.task_name | test("telemetry"))]'
+
+set -euo pipefail
+
+PERF_API="${PERF_API:-https://performance-monitoring-api.corp.mongodb.com}"
+
+version_id="${1:-}"
+
+if [ -z "$version_id" ]; then
+  echo "usage: perf-get.sh <version-id>" >&2
+  exit 2
+fi
+
+curl -sS -f -H 'accept: application/json' "${PERF_API}/raw_perf_results/versions/${version_id}"

.evergreen/perf-send.sh

@@ -0,0 +1,33 @@
+#! /usr/bin/env bash
+
+# shellcheck disable=SC2154 # requester, revision_order_id, project_id, version_id, build_variant, task_name, task_id and execution are Evergreen expansions provided via add_expansions_to_env in evergreen.yml.in
+
+# We use the requester expansion to determine whether the data is from a mainline evergreen run or not
+if [ "${requester}" == "commit" ]; then
+  is_mainline=true
+else
+  is_mainline=false
+fi
+
+# Parse the username out of the order_id. Patches append the username. The Signal Processing Service (SPS) endpoint does not need the other information.
+parsed_order_id=$(echo "${revision_order_id}" | awk -F'_' '{print $NF}')
+
+# Submit the performance data to the SPS endpoint
+response=$(curl -s -w "\nHTTP_STATUS:%{http_code}" -X 'POST' \
+"https://performance-monitoring-api.corp.mongodb.com/raw_perf_results/cedar_report?project=${project_id}&version=${version_id}&variant=${build_variant}&order=$parsed_order_id&task_name=${task_name}&task_id=${task_id}&execution=${execution}&mainline=$is_mainline" \
+  -H 'accept: application/json' \
+  -H 'Content-Type: application/json' \
+  -d @src/perf_results.json)
+
+http_status=$(echo "$response" | grep "HTTP_STATUS" | awk -F':' '{print $2}')
+response_body=$(echo "$response" | sed '/HTTP_STATUS/d')
+
+# We want to throw an error if the data was not successfully submitted
+if [ "$http_status" -ne 200 ]; then
+  echo "Error: Received HTTP status $http_status"
+  echo "Response Body: $response_body"
+  exit 1
+fi
+
+echo "Response Body: $response_body"
+echo "HTTP Status: $http_status"

.evergreen/perf-test.sh

@@ -0,0 +1,35 @@
+#! /usr/bin/env bash
+
+set -euxo pipefail
+
+if [ "${MONGOSH_PERF_TELEMETRY:-}" = "1" ]; then
+  rm -f telemetry-sink-endpoint.txt telemetry-sink-events.ldjson
+  node .evergreen/telemetry-sink.mts telemetry-sink-endpoint.txt telemetry-sink-events.ldjson &
+  echo $! > telemetry-sink.pid
+  trap 'kill "$(cat telemetry-sink.pid)" 2>/dev/null || true' EXIT
+  for _ in $(seq 1 50); do
+    [ -s telemetry-sink-endpoint.txt ] && break
+    sleep 0.2
+  done
+  [ -s telemetry-sink-endpoint.txt ] # fail if the sink never came up
+  MONGOSH_TELEMETRY_ENDPOINT="$(cat telemetry-sink-endpoint.txt)"
+  export MONGOSH_TELEMETRY_ENDPOINT
+  export SSL_CERT_FILE="$PWD/packages/testing/certificates/ca.crt"
+fi
+
+# shellcheck disable=SC2016 # $MONGODB_URI is deliberately expanded by the inner shell, which mongodb-runner invokes with the URI in its environment
+npx -y mongodb-runner exec -t standalone --version=7.0.x-enterprise -- \
+  sh -c 'MONGOSH_SMOKE_TEST_SERVER="$MONGODB_URI" ./dist/mongosh --perfTests > perf_results.json'
+
+if [ "${MONGOSH_PERF_TELEMETRY:-}" = "1" ]; then
+  kill "$(cat telemetry-sink.pid)" || true
+  wait "$(cat telemetry-sink.pid)" 2>/dev/null || true
+
+  echo "=== telemetry sink events (LDJSON) ==="
+  cat telemetry-sink-events.ldjson
+
+  echo "=== telemetry sink event summary ==="
+  event_count=$(grep -c '"event":' telemetry-sink-events.ldjson || true)
+  echo "total=$event_count"
+  [ "$event_count" -gt 10 ]
+fi