From a1dc054c1aec02afce7c69213d0aa4d2f0ae1f4f Mon Sep 17 00:00:00 2001 From: Sonu Kapoor Date: Wed, 29 Apr 2026 22:08:11 -0400 Subject: [PATCH] test: add native production smoke checks --- .github/workflows/test-and-build.yml | 8 + Makefile | 8 +- docs/production-readiness.md | 101 ++++++++++++ scripts/run-production-readiness-smoke.sh | 144 ++++++++++++++++++ .../fixtures/http-loopback.js | 37 +++++ .../fixtures/module-load.js | 25 +++ 6 files changed, 322 insertions(+), 1 deletion(-) create mode 100644 docs/production-readiness.md create mode 100755 scripts/run-production-readiness-smoke.sh create mode 100644 tests/production_readiness/fixtures/http-loopback.js create mode 100644 tests/production_readiness/fixtures/module-load.js diff --git a/.github/workflows/test-and-build.yml b/.github/workflows/test-and-build.yml index 62aa6a699..d28455ea1 100644 --- a/.github/workflows/test-and-build.yml +++ b/.github/workflows/test-and-build.yml @@ -115,6 +115,10 @@ jobs: path: test-edge.log if-no-files-found: warn + - name: Production readiness smoke + run: | + make test-production-readiness + - name: sccache stats if: always() run: sccache --show-stats || true @@ -350,6 +354,10 @@ jobs: run: | make test-only TEST_JOBS=4 + - name: Production readiness smoke + run: | + make test-production-readiness + - name: sccache stats if: always() run: sccache --show-stats || true diff --git a/Makefile b/Makefile index 0ac53a209..99d26796b 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: build build-wasix build-napi-wasmer-cli test-wasix-napi-cli test-wasix-safe-mode test test-only check-portability clean-dist dist dist-only framework-test framework-test-reset +.PHONY: build build-wasix build-napi-wasmer-cli test-wasix-napi-cli test-wasix-safe-mode test-production-readiness test test-only check-portability clean-dist dist dist-only framework-test framework-test-reset UNAME_S := $(shell uname -s) BUILD_NAPI_DIR ?= build-v8-napi @@ -67,6 +67,12 @@ test-wasix-napi-cli: test-wasix-safe-mode: python3 ./scripts/test-wasix-safe-mode.py --wasmer-bin "$(WASMER_BIN)" --package-dir "$(WASIX_PACKAGE_DIR)" +test-production-readiness: $(EDGE_BINARY) + EDGE_BINARY="$(EDGE_BINARY)" \ + WASMER_BIN="$(WASMER_BIN)" \ + EDGE_WASMER_PACKAGE="$(EDGE_WASMER_PACKAGE)" \ + ./scripts/run-production-readiness-smoke.sh + $(EDGE_BINARY): $(MAKE) build diff --git a/docs/production-readiness.md b/docs/production-readiness.md new file mode 100644 index 000000000..058d8a575 --- /dev/null +++ b/docs/production-readiness.md @@ -0,0 +1,101 @@ +# Production Readiness Smoke Checks + +This document turns part of the 0.x production-readiness roadmap into concrete +smoke checks that can be run repeatedly before releases and after runtime +changes. + +The goal is not to declare the runtime production-ready. These checks cover a +small native baseline for CLI startup, eval, built-in module loading, HTTP +server lifecycle, and repeated cold starts. Stress testing, low-resource +behavior, platform packaging, provider deployment, and broader WASIX validation +remain separate work. + +## 0.x Readiness Matrix + +| Area | Current validation | Status | Next evidence needed | +|---|---|---|---| +| Fast cold starts | Startup benchmarks and startup trace investigation | Partial | Release-gated cold-start thresholds per platform | +| Native runtime smoke | `make test-production-readiness` native smoke checks | Covered first pass | Keep the smoke path green on Linux and macOS | +| WASIX / safe mode smoke | Optional safe-mode smoke in `make test-production-readiness` | Partial | Required CI safe-mode pass for packaged WASIX artifacts | +| HTTPS / TLS in safe mode | Existing `make test-wasix-safe-mode` network/TLS smoke | Partial | Keep HTTPS smoke attached to WASIX package validation | +| Module loading | Smoke fixture validates common built-ins | Covered first pass | Add dependency-heavy package case | +| HTTP server lifecycle | Smoke fixture starts a loopback server and exits | Covered first pass | Add request-rate stress coverage | +| Stress testing | Not covered by this PR | Not covered | Many requests/second and long-running process checks | +| Low-resource behavior | Not covered by this PR | Not covered | Memory/CPU-constrained smoke runner | +| Stable unofficial N-API APIs | Existing N-API and internal binding tests | Partial | Public compatibility surface checklist | +| Platform binaries | CI produces Linux, macOS ARM, and WASIX artifacts | Partial | Windows, Android, iOS example app coverage | +| Production provider use | Roadmap target only | Not covered | Provider deployment report with blocker list | + +## Smoke Harness + +Run the native production-readiness smoke suite: + +```bash +make test-production-readiness +``` + +By default this uses `./build-edge/edge`. Override the binary with: + +```bash +EDGE_BINARY=/path/to/edge make test-production-readiness +``` + +The harness writes a Markdown summary and per-case logs to: + +```text +artifacts/production-readiness// +``` + +These generated artifacts are intentionally ignored by git. Copy relevant +summaries into issue or PR comments when they add context beyond the CI log. + +Each case has a timeout so a stuck server lifecycle or runtime invocation fails +the smoke run instead of hanging indefinitely. Override it with: + +```bash +EDGE_PRODUCTION_READINESS_CASE_TIMEOUT=60 make test-production-readiness +``` + +## Optional Safe-Mode Coverage + +Safe mode requires Wasmer and a WASIX package. Enable the small no-network +safe-mode smoke set with: + +```bash +EDGE_PRODUCTION_READINESS_SAFE=1 make test-production-readiness +``` + +Use a specific Wasmer binary or package with: + +```bash +EDGE_PRODUCTION_READINESS_SAFE=1 \ +WASMER_BIN=/path/to/wasmer \ +EDGE_WASMER_PACKAGE=wasmer/edgejs@= \ +make test-production-readiness +``` + +For network/TLS safe-mode coverage, keep using: + +```bash +make test-wasix-safe-mode +``` + +That path already covers HTTP, HTTPS, TLS, and related WASIX behavior through +the packaged runtime. + +## Release Evidence Expectations + +Before a 0.x production-readiness release, the project should be able to point +to public evidence for: + +- native smoke results on supported desktop/server platforms +- WASIX safe-mode smoke results for the published package +- HTTPS/TLS safe-mode results +- startup benchmark trend +- at least one stress-test result +- low-resource behavior notes +- release artifact availability by platform +- an explicit unresolved-blocker list + +This smoke path is only a starting point for the native checks in that broader +release-readiness picture. diff --git a/scripts/run-production-readiness-smoke.sh b/scripts/run-production-readiness-smoke.sh new file mode 100755 index 000000000..6084046f3 --- /dev/null +++ b/scripts/run-production-readiness-smoke.sh @@ -0,0 +1,144 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +EDGE_BINARY="${EDGE_BINARY:-${ROOT_DIR}/build-edge/edge}" +ARTIFACT_ROOT="${EDGE_PRODUCTION_READINESS_ARTIFACT_DIR:-${ROOT_DIR}/artifacts/production-readiness}" +RUN_ID="${EDGE_PRODUCTION_READINESS_RUN_ID:-$(date -u +%Y%m%dT%H%M%SZ)}" +OUT_DIR="${ARTIFACT_ROOT}/${RUN_ID}" +SUMMARY="${OUT_DIR}/summary.md" +COLD_START_RUNS="${EDGE_PRODUCTION_READINESS_COLD_START_RUNS:-5}" +CASE_TIMEOUT="${EDGE_PRODUCTION_READINESS_CASE_TIMEOUT:-30}" +SAFE_MODE="${EDGE_PRODUCTION_READINESS_SAFE:-0}" +WASMER_BIN="${WASMER_BIN:-wasmer}" +EDGE_WASMER_PACKAGE="${EDGE_WASMER_PACKAGE:-}" + +mkdir -p "${OUT_DIR}" + +case_index=0 +passed=0 +failed=0 + +write_header() { + { + printf '# Production Readiness Smoke Report\n\n' + printf '| Field | Value |\n' + printf '|---|---|\n' + printf '| Run ID | `%s` |\n' "${RUN_ID}" + printf '| Edge binary | `%s` |\n' "${EDGE_BINARY}" + printf '| Safe mode requested | `%s` |\n' "${SAFE_MODE}" + printf '| Cold-start runs | `%s` |\n' "${COLD_START_RUNS}" + printf '| Case timeout | `%ss` |\n\n' "${CASE_TIMEOUT}" + printf '## Results\n\n' + printf '| Case | Status | Log |\n' + printf '|---|---|---|\n' + } > "${SUMMARY}" +} + +append_result() { + local name="$1" + local status="$2" + local log_name="$3" + printf '| `%s` | %s | `%s` |\n' "${name}" "${status}" "${log_name}" >> "${SUMMARY}" +} + +run_case() { + local name="$1" + shift + case_index=$((case_index + 1)) + local safe_name + safe_name="$(printf '%02d-%s.log' "${case_index}" "${name}" | tr ' /:' '---')" + local log_path="${OUT_DIR}/${safe_name}" + local pid + local elapsed=0 + local status=0 + + printf '[run] %s\n' "${name}" + "$@" >"${log_path}" 2>&1 & + pid=$! + + while kill -0 "${pid}" 2>/dev/null; do + if [[ "${elapsed}" -ge "${CASE_TIMEOUT}" ]]; then + printf 'case timed out after %ss\n' "${CASE_TIMEOUT}" >> "${log_path}" + kill "${pid}" 2>/dev/null || true + wait "${pid}" 2>/dev/null || true + status=124 + break + fi + sleep 1 + elapsed=$((elapsed + 1)) + done + + if [[ "${status}" -eq 0 ]]; then + wait "${pid}" || status=$? + fi + + if [[ "${status}" -eq 0 ]]; then + append_result "${name}" "PASS" "${safe_name}" + passed=$((passed + 1)) + printf '[ok] %s\n' "${name}" + else + append_result "${name}" "FAIL" "${safe_name}" + failed=$((failed + 1)) + printf '[fail] %s\n' "${name}" >&2 + sed -n '1,120p' "${log_path}" >&2 + fi +} + +run_edge() { + "${EDGE_BINARY}" "$@" +} + +run_safe_edge() { + local args=("${EDGE_BINARY}" "--safe" "--wasmer-bin" "${WASMER_BIN}") + if [[ -n "${EDGE_WASMER_PACKAGE}" ]]; then + args+=("--wasmer-package" "${EDGE_WASMER_PACKAGE}") + fi + args+=("$@") + "${args[@]}" +} + +run_repeated_cold_start() { + local i + for ((i = 1; i <= COLD_START_RUNS; i++)); do + "${EDGE_BINARY}" -e "" + done + printf 'production-readiness:cold-start %s\n' "${COLD_START_RUNS}" +} + +write_footer() { + { + printf '\n## Summary\n\n' + printf '%s\n' "- Passed: ${passed}" + printf '%s\n' "- Failed: ${failed}" + printf '%s\n' "- Artifact directory: \`${OUT_DIR}\`" + } >> "${SUMMARY}" +} + +if [[ ! -x "${EDGE_BINARY}" ]]; then + printf 'error: Edge binary is missing or not executable: %s\n' "${EDGE_BINARY}" >&2 + exit 1 +fi + +write_header + +run_case "native-version" run_edge --version +run_case "native-eval" run_edge -e "console.log('production-readiness:eval')" +run_case "native-module-load" run_edge "${ROOT_DIR}/tests/production_readiness/fixtures/module-load.js" +run_case "native-http-loopback" run_edge "${ROOT_DIR}/tests/production_readiness/fixtures/http-loopback.js" +run_case "native-repeated-cold-start" run_repeated_cold_start + +if [[ "${SAFE_MODE}" == "1" || "${SAFE_MODE}" == "true" ]]; then + run_case "safe-eval" run_safe_edge -e "console.log('production-readiness:safe-eval')" + run_case "safe-microtask" run_safe_edge -e "console.log('A'); queueMicrotask(() => console.log('B')); console.log('C');" +else + printf '[skip] safe-mode smoke; set EDGE_PRODUCTION_READINESS_SAFE=1 to enable\n' +fi + +write_footer + +printf 'Production readiness smoke report: %s\n' "${SUMMARY}" + +if [[ "${failed}" -ne 0 ]]; then + exit 1 +fi diff --git a/tests/production_readiness/fixtures/http-loopback.js b/tests/production_readiness/fixtures/http-loopback.js new file mode 100644 index 000000000..fef518715 --- /dev/null +++ b/tests/production_readiness/fixtures/http-loopback.js @@ -0,0 +1,37 @@ +'use strict'; + +const http = require('node:http'); + +const server = http.createServer((req, res) => { + if (req.url !== '/ready') { + res.statusCode = 404; + res.end('not-found'); + return; + } + res.end('ok'); +}); + +server.listen(0, '127.0.0.1', () => { + const { port } = server.address(); + http.get({ hostname: '127.0.0.1', port, path: '/ready' }, (res) => { + let body = ''; + res.setEncoding('utf8'); + res.on('data', (chunk) => { + body += chunk; + }); + res.on('end', () => { + server.close(() => { + if (res.statusCode !== 200 || body !== 'ok') { + console.error(`unexpected http response: ${res.statusCode} ${body}`); + process.exit(1); + } + console.log('production-readiness:http-loopback'); + }); + }); + }).on('error', (err) => { + server.close(() => { + console.error(err && (err.stack || err.message || err)); + process.exit(1); + }); + }); +}); diff --git a/tests/production_readiness/fixtures/module-load.js b/tests/production_readiness/fixtures/module-load.js new file mode 100644 index 000000000..0f5182f93 --- /dev/null +++ b/tests/production_readiness/fixtures/module-load.js @@ -0,0 +1,25 @@ +'use strict'; + +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const zlib = require('node:zlib'); + +const payload = Buffer.from('production-readiness'); +const compressed = zlib.gzipSync(payload); +const roundtrip = zlib.gunzipSync(compressed).toString('utf8'); + +if (roundtrip !== 'production-readiness') { + throw new Error(`zlib roundtrip mismatch: ${roundtrip}`); +} + +const root = path.parse(process.cwd()).root; +if (!root || typeof os.platform() !== 'string') { + throw new Error('basic platform/module checks failed'); +} + +if (typeof fs.existsSync !== 'function') { + throw new Error('node:fs did not load correctly'); +} + +console.log('production-readiness:module-load');