diff --git a/.gitattributes b/.gitattributes index 841ddea..bdf7b2e 100644 --- a/.gitattributes +++ b/.gitattributes @@ -2,6 +2,7 @@ *.py text eol=lf *.ps1 text eol=lf +*.sh text eol=lf *.md text eol=lf *.toml text eol=lf *.yml text eol=lf diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e8b9335..e5d713b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,6 +9,36 @@ permissions: contents: read jobs: + linux-x86-demo: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: requirements.linux.lock + + - name: Install locked dependencies + shell: bash + run: | + python -m pip install --upgrade pip + python -m pip install -r requirements.linux.lock + python -m pip check + + - name: Run Linux x86-64 offline demo + shell: bash + run: | + test "$(uname -m)" = "x86_64" + bash -n scripts/portfolio_demo.sh + PYTHON_PATH="$(command -v python)" bash scripts/portfolio_demo.sh --output-dir artifacts/ci_linux_x86_demo + test ! -e handoff.md + test -f docs/internal/handoff.md + offline-evaluation: runs-on: windows-latest timeout-minutes: 20 diff --git a/README.en.md b/README.en.md new file mode 100644 index 0000000..3bf51e0 --- /dev/null +++ b/README.en.md @@ -0,0 +1,105 @@ +# ResearchOps Agent + +[中文](README.md) · **English** + +[![offline-quality-gate](https://github.com/cedRiC874/researchops-agent/actions/workflows/ci.yml/badge.svg)](https://github.com/cedRiC874/researchops-agent/actions/workflows/ci.yml) +![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg) + +_CI signal: Ubuntu x86-64 installs locked dependencies and runs the complete Linux x86-64 offline demo; the Windows offline gate runs the complete unit/integration suite and rebuilds and verifies the frozen 50-task evidence._ + +**Let an LLM analyze research data without letting it invent the numbers:** ResearchOps Agent delegates planning to the model, deterministic data-quality and statistical work to controlled tools, and binds every reported claim to reviewable evidence and approval boundaries. + +![Aggregate ANCOVA and Welch effects](artifacts/phase3/effect_estimates.png) + +_Thirty-second result: provide a de-identified CSV, a research question and an explicit study design; receive an aggregate analysis with sample flow, effect estimates, confidence intervals, evidence IDs and limitations._ + +The model never sees real filesystem paths and cannot freely run Python, SQL or shell commands. It can only call allowlisted logical tools; deterministic local implementations produce the statistics and append them to the audit chain. + +> This is a research prototype and portfolio project, not a clinical decision tool or a production-validated product. + +## A concrete result: baseline-adjusted treatment effect + +Example question: in a fully synthetic 240-row randomized trial, do follow-up systolic blood-pressure values differ between treatment and control, after accounting for baseline pressure? + +| Method | treatment − control | 95% CI | p-value | Analysis n | Evidence ID | +| --- | ---: | ---: | ---: | ---: | --- | +| ANCOVA, baseline-adjusted, HC3 | -5.6069 mmHg | [-7.9351, -3.2787] | 3.82e-6 | 212 | `E-7C87BB6C88EB` | +| Welch, unadjusted sensitivity analysis | -6.7887 mmHg | [-10.8425, -2.7349] | 0.001134 | 212 | `E-B93CD9DC7751` | + +Negative values mean lower follow-up pressure in the treatment group. The report may use benefit language only when the study design pre-specifies `beneficial_direction=lower`. + +> **Professional boundary:** the requested population is intention-to-treat, but 28 missing follow-up outcomes leave 212 available cases in the realized analysis. The system therefore records `requested_population=intention_to_treat` and `realized_population=available_case`, and refuses to describe this result as a complete ITT analysis. + +Reviewable artifacts: [analysis bundle](artifacts/phase3/analysis_bundle.json) · [aggregate chart](artifacts/phase3/effect_estimates.png) + +## Quickstart + +The strict frozen-evidence demo currently supports Python 3.11+ on Windows x86-64 and Linux x86-64 with NumPy/OpenBLAS. macOS and ARM do not yet have a comparable numerical baseline and are outside this strict demo's supported scope. + +To avoid treating cross-OS floating-point differences as the same evidence, the canonical ANCOVA identity is pinned separately to Windows x86-64 `E-36034128278C` and Linux x86-64 `E-14EBFFCA843E`. + +### Windows x86-64 / PowerShell + +```powershell +python -m venv .venv +.\.venv\Scripts\python.exe -m pip install -r requirements.lock +powershell -ExecutionPolicy Bypass -File .\scripts\portfolio_demo.ps1 +``` + +### Linux x86-64 + +```bash +python3 -m venv .venv +./.venv/bin/python -m pip install -r requirements.linux.lock +bash ./scripts/portfolio_demo.sh +``` + +The Linux lock keeps the same package versions as the Windows lock while excluding the Windows-only `pywin32`. The Linux demo uses the separately frozen `evals/tasks.linux-x86_64.jsonl`, which rebinds only cross-OS evidence/chart IDs without relaxing numerical or quality thresholds. CI installs this lock and runs the complete demo. + +The demo rebuilds the frozen 50-task deterministic evaluation, verifies all 50 event hash chains, checks sensitive-data canaries, and writes to a new artifact directory. It never invokes an online Provider and never overwrites an existing artifact. + +## Architecture + +```mermaid +flowchart LR + I["Research question + de-identified CSV + explicit design"] + A["Agent planning"] + R["Logical resource registry"] + Q["Data quality + method selection"] + S["Deterministic statistics"] + E["Evidence bundle + chart + report"] + P["Central risk policy"] + H["Human approval"] + X["Controlled executor"] + L["SQLite audit + SHA-256 chain"] + V["Phase 5 / Phase 6 evaluators"] + + I --> R --> Q --> S --> E + R --> A + A -->|"logical IDs only"| P + P -->|"read-only allow"| X + P -->|"controlled write"| H --> X + X --> Q + X --> E + X --> L + H --> L + Q --> V + S --> V + E --> V + A --> V +``` + +Key boundaries: + +- The study design must be explicit; the system does not infer randomization, causality, pairing or covariate timing from column names. +- Method recommendations and execution bind to the dataset SHA-256 and fail safely if the input changes. +- Every report claim must match the current tool output's `evidence_id + metric_path + displayed_value + direction`. +- Unknown tools, unknown risk, unauthorized resources and unapproved writes are denied by default. + +See [ARCHITECTURE.md](docs/ARCHITECTURE.md) for the detailed design. + +For current online/offline validation, failure denominators, Provider history, candidate commitments and strict claim boundaries, see **[STATUS.md](STATUS.md)**. + +## License + +[MIT](LICENSE) diff --git a/README.md b/README.md index 69d6a75..d3fc2d5 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,12 @@ # ResearchOps Agent +**中文** · [English](README.en.md) + +[![offline-quality-gate](https://github.com/cedRiC874/researchops-agent/actions/workflows/ci.yml/badge.svg)](https://github.com/cedRiC874/researchops-agent/actions/workflows/ci.yml) +![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg) + +_CI 信号：Ubuntu x86-64 安装锁定依赖并运行完整 Linux x86-64 离线演示；Windows 离线质量门运行完整单元/集成套件，并重建与验证固定 50 题 evidence。_ + **让 LLM 做科研数据分析，但不让它编数字：** ResearchOps Agent 让模型负责规划，让确定性工具负责数据质量、方法选择、统计计算与可视化，并把每条结论绑定到可复核 evidence 与人工审批边界。 ![ANCOVA 与 Welch 聚合效应图](artifacts/phase3/effect_estimates.png) @@ -27,7 +34,11 @@ _30 秒演示结果：输入脱敏 CSV、研究问题和显式研究设计，输 ## Quickstart -Windows PowerShell，三行运行无网络、无 API Key 的完整离线演示： +严格 frozen-evidence 演示当前支持 Python 3.11+ 的 Windows x86-64 和 Linux x86-64（NumPy/OpenBLAS）。macOS 与 ARM 尚未建立可比较的数值基线，因此不在这一严格演示的支持范围内。 + +为避免把跨操作系统的浮点位差异误当成同一证据，canonical ANCOVA identity 分别固定为 Windows x86-64 `E-36034128278C` 与 Linux x86-64 `E-14EBFFCA843E`。 + +### Windows x86-64 / PowerShell ```powershell python -m venv .venv @@ -35,6 +46,16 @@ python -m venv .venv powershell -ExecutionPolicy Bypass -File .\scripts\portfolio_demo.ps1 ``` +### Linux x86-64 + +```bash +python3 -m venv .venv +./.venv/bin/python -m pip install -r requirements.linux.lock +bash ./scripts/portfolio_demo.sh +``` + +Linux 锁文件与 Windows 锁文件保持同一组版本，仅排除 Windows 专用的 `pywin32`；Linux demo 使用独立冻结的 `evals/tasks.linux-x86_64.jsonl`，只重绑定跨操作系统变化的 evidence/chart IDs，不放宽数值或质量阈值。CI 会安装该锁文件并执行完整 demo。 + 演示会重建固定 50 题确定性评测、验证 50 条事件哈希链、检查敏感信息 canary，并把结果写入一个新的 artifact 目录。它不会运行在线 Provider，也不会覆盖已有产物。 ## 架构 diff --git a/docs/internal/README.md b/docs/internal/README.md new file mode 100644 index 0000000..a3d3400 --- /dev/null +++ b/docs/internal/README.md @@ -0,0 +1,12 @@ +# Internal project notes + +This directory contains historical operator and session-handoff notes. These +files are useful for repository maintenance, but they are not product +documentation, current evidence, or a source of runtime instructions. + +- [handoff.md](handoff.md) is a dated historical snapshot and may reference + superseded branches, commits, candidates or CI runs. +- Current public status and claim boundaries live in [STATUS.md](../../STATUS.md). +- Current contributor-facing design documentation lives under [docs](../). + +No API key or credential value should ever be added to this directory. diff --git a/handoff.md b/docs/internal/handoff.md similarity index 90% rename from handoff.md rename to docs/internal/handoff.md index 35585fe..624bfd4 100644 --- a/handoff.md +++ b/docs/internal/handoff.md @@ -1,5 +1,7 @@ # ResearchOps Agent 跨会话交接 +> **归档说明（2026-09-01）：**本文件是历史内部交接快照，不再是当前项目入口或运行指令；分支、commit、candidate 与 CI 状态可能已经过期。当前公开状态以 [STATUS.md](../../STATUS.md) 为准。 + > 更新时间：2026-08-26（Asia/Shanghai） > 目标：让一个没有历史上下文的新 Codex 会话安全、准确地继续本项目。 > 语言偏好：中文；先给结论，再给可执行步骤；不要夸大评测或生产化程度。 @@ -7,7 +9,7 @@ ## 最新 main / candidate v5 / supervised 状态 - GitHub `main` 为 `c65ff65c0cbb67205956ddae991768ba9fca9293`；PR #21 已 regular merge，reviewed head `ccd03ccf…` 与 merge tree 均为 `ddaf063f…a5e9`。合并后的 main runs `32957003253`（offline）、`32957003191`（pilot）与 `32957003204`（production）均为 `completed / success`。 -- Candidate v5 commitment 为 `105b7def81148566219673fd40e88e392674070656c72a17cbcf60405165dffc`，predecessor v4 为 `1741c2b0df53d06a299a5a89dfa91e68eade4c71cef7931d367115c07f6399c7`；它是 pre-call snapshot，`prior_results_inherited=false`，不继承 post-lock receipt，public Provider 仍为 DeepSeek，完整 campaign 仍为 `design_only`。长期快照见 [Kimi Models preflight main CI evidence](docs/evidence/kimi-models-preflight-main-ci-v1/README.md)。 +- Candidate v5 commitment 为 `105b7def81148566219673fd40e88e392674070656c72a17cbcf60405165dffc`，predecessor v4 为 `1741c2b0df53d06a299a5a89dfa91e68eade4c71cef7931d367115c07f6399c7`；它是 pre-call snapshot，`prior_results_inherited=false`，不继承 post-lock receipt，public Provider 仍为 DeepSeek，完整 campaign 仍为 `design_only`。长期快照见 [Kimi Models preflight main CI evidence](../../docs/evidence/kimi-models-preflight-main-ci-v1/README.md)。 - 本地实现 commit `329a6dc…` 与 PR #16 远端 head `7079af08…` 的 tree 均为 `710ef581f2e1355e33d9af135325468b4db1c095`；commit SHA 不同来自 Git Data API 发布，不是文件内容漂移。最终 merge tree 另含 PR #15，不能与 PR head tree 混为一谈。 - Anthropic frozen contract 仍为 offline-only；post-lock 一次 official-origin metadata 尝试使用了 CCTK token并返回 `403 / network_calls=1 / model_token_calls=0`，不证明官方 Anthropic或CCTK可用性。CCTK 路线已放弃，generic Anthropic online 入口继续 fail closed。 - Kimi 中国区 fixed-origin Models-list preflight 在 v5 锁定时为 `implemented_offline_tested_not_run`。锁定后的一次独立授权 GET 于 `2026-08-26T09:41:49.967Z` verified：HTTP 200、attempts/network calls `1/1`、requested/returned `kimi-k3`、认证与 exact visibility true、0 model tokens、cost null。该 receipt 不进入 v5；一次性授权已消耗，不得重试，任何新 request 需新授权。Chat/Responses/tools/usage/cost semantics/质量/注册/private 均未授权；Provider 仍 1/2，private 仍 0/50。 @@ -18,16 +20,16 @@ - Executor model requests / model-requested tools / backend executions 为 `9 / 3 / 2`；这些不是规划准确率、账户 API 总数或成本。 - 四个成功展示答案的非专家 feedback 为 understandable/useful `4/4`，明显问题、信息缺失、专家复核需求与安全担忧均 `0/4`；不评价专业正确性。 - campaign 完成后 worker、API、Funnel 已停止，容器/network 已移除，PostgreSQL volume 保留；安全 incidents 0，telemetry/participant binding 均 valid。 -- Retention 已于 2026-08-25 实际复核：Scheduled Task `Ready`、daily/StartWhenAvailable/IgnoreNew、手动运行 result 0、snapshot `NumberOfMissedRuns=0`；1 个 participant 与 4 条 feedback deadline 均为 90 天上限，当前 due 0，未提前删除。见 [retention verification](docs/evidence/supervised-completion-telemetry-v2-20260825/retention-verification-20260825.md)。 -- private-holdout custodian kit v1.1 已由 PR #14 进入 `main`；synthetic conformance 包含不同 Ed25519 keys、调用方提供的 external anchors、两阶段 ledger、aggregate/budget verifier。真实 private release 固定拒绝；当前 `design_only / private 0/50 / Provider 1/2 / not_authorized` 不变，不能声称已有 private corpus、授权或运行。长期快照见 [private custodian main CI evidence](docs/evidence/eval-v2-private-custodian-main-ci-v1/README.md)。PR #15/#16 historical candidate v3 快照见 [Anthropic offline adapter main CI evidence](docs/evidence/eval-v2-anthropic-offline-main-ci-v1/README.md)；PR #19/main-at-snapshot 快照见 [Anthropic Models preflight main CI evidence](docs/evidence/anthropic-models-preflight-main-ci-v1/README.md)。 -- 脱敏 supervised 证据位于 [supervised Completion Telemetry v2 20260825](docs/evidence/supervised-completion-telemetry-v2-20260825/README.md)。不得用这六题继续调 prompt/scorer，也不得与旧 campaign 聚合。 +- Retention 已于 2026-08-25 实际复核：Scheduled Task `Ready`、daily/StartWhenAvailable/IgnoreNew、手动运行 result 0、snapshot `NumberOfMissedRuns=0`；1 个 participant 与 4 条 feedback deadline 均为 90 天上限，当前 due 0，未提前删除。见 [retention verification](../../docs/evidence/supervised-completion-telemetry-v2-20260825/retention-verification-20260825.md)。 +- private-holdout custodian kit v1.1 已由 PR #14 进入 `main`；synthetic conformance 包含不同 Ed25519 keys、调用方提供的 external anchors、两阶段 ledger、aggregate/budget verifier。真实 private release 固定拒绝；当前 `design_only / private 0/50 / Provider 1/2 / not_authorized` 不变，不能声称已有 private corpus、授权或运行。长期快照见 [private custodian main CI evidence](../../docs/evidence/eval-v2-private-custodian-main-ci-v1/README.md)。PR #15/#16 historical candidate v3 快照见 [Anthropic offline adapter main CI evidence](../../docs/evidence/eval-v2-anthropic-offline-main-ci-v1/README.md)；PR #19/main-at-snapshot 快照见 [Anthropic Models preflight main CI evidence](../../docs/evidence/anthropic-models-preflight-main-ci-v1/README.md)。 +- 脱敏 supervised 证据位于 [supervised Completion Telemetry v2 20260825](../../docs/evidence/supervised-completion-telemetry-v2-20260825/README.md)。不得用这六题继续调 prompt/scorer，也不得与旧 campaign 聚合。 - 下文更早状态如有冲突，以本节为准。 ## 0. 新会话首先执行 1. 完整阅读本文件。 2. 在仓库根目录运行 `git status --short --branch`。 -3. 阅读 [README.md](README.md)、[docs/EVIDENCE.md](docs/EVIDENCE.md)、[docs/ARCHITECTURE.md](docs/ARCHITECTURE.md) 和 [docs/PORTFOLIO.md](docs/PORTFOLIO.md)。 +3. 阅读 [README.md](../../README.md)、[docs/EVIDENCE.md](../../docs/EVIDENCE.md)、[docs/ARCHITECTURE.md](../../docs/ARCHITECTURE.md) 和 [docs/PORTFOLIO.md](../../docs/PORTFOLIO.md)。 4. 保留所有现有未跟踪文件，不覆盖、不清理、不重置。 5. 未经用户明确要求，不要 commit、push、重新运行付费在线评测或重复使用冻结 holdout 调参。 6. 不要读取、打印或记录 `OPENAI_API_KEY` / `DEEPSEEK_API_KEY` / @@ -343,7 +345,7 @@ Repo-local non-secret holdout： - 任务和 golden 均在仓库内，可见、不抗污染； - 不能表述为未知分布泛化或生产 SLA。 -冻结脱敏证据位于 [docs/evidence/phase6-deepseek-v1/](docs/evidence/phase6-deepseek-v1/README.md)。只提交 report、summary、manifest 和 audit index；详细 SQLite/results 未提交，但其 hash/size 进入 manifest。 +冻结脱敏证据位于 [docs/evidence/phase6-deepseek-v1/](../../docs/evidence/phase6-deepseek-v1/README.md)。只提交 report、summary、manifest 和 audit index；详细 SQLite/results 未提交，但其 hash/size 进入 manifest。 ### 5.3 OpenAI 路径 @@ -366,9 +368,9 @@ PR #21 后三条 main runs 均绑定精确 merge SHA `c65ff65c…` 并为 `compl candidate verifier 仍报告 pre-call snapshot `valid / network_calls=0`。Post-lock metadata receipt 是独立 observation，不是 PR/main CI 或 candidate 成绩。这些证据都不是 Kimi Chat/tools/usage/cost semantics/质量证据。PR #21 长期快照见 -[docs/evidence/kimi-models-preflight-main-ci-v1/](docs/evidence/kimi-models-preflight-main-ci-v1/README.md)， +[docs/evidence/kimi-models-preflight-main-ci-v1/](../../docs/evidence/kimi-models-preflight-main-ci-v1/README.md)， PR #19 历史快照见 -[docs/evidence/anthropic-models-preflight-main-ci-v1/](docs/evidence/anthropic-models-preflight-main-ci-v1/README.md)。 +[docs/evidence/anthropic-models-preflight-main-ci-v1/](../../docs/evidence/anthropic-models-preflight-main-ci-v1/README.md)。 重要 P1 历史事实：run 32568017243 的 workflow conclusion 虽为 `success`，其新重建 Phase 5 实际只有 44/50、evidence citations 10/21。根因已定位为 clean checkout @@ -558,31 +560,31 @@ $env:PYTHONPATH = "src" ## 12. 快速文件索引 -- 项目入口：[README.md](README.md) -- 架构边界：[docs/ARCHITECTURE.md](docs/ARCHITECTURE.md) -- 证据映射：[docs/EVIDENCE.md](docs/EVIDENCE.md) -- 作品集与旧面试问答：[docs/PORTFOLIO.md](docs/PORTFOLIO.md) -- 新面试深挖指南：[docs/RESEARCHOPS_INTERVIEW_GUIDE.md](docs/RESEARCHOPS_INTERVIEW_GUIDE.md) -- Phase 5 语料说明：[evals/README.md](evals/README.md) -- Phase 6 契约：[evals/PHASE6.md](evals/PHASE6.md) -- Phase 6 冻结证据：[docs/evidence/phase6-deepseek-v1/README.md](docs/evidence/phase6-deepseek-v1/README.md) +- 项目入口：[README.md](../../README.md) +- 架构边界：[docs/ARCHITECTURE.md](../../docs/ARCHITECTURE.md) +- 证据映射：[docs/EVIDENCE.md](../../docs/EVIDENCE.md) +- 作品集与旧面试问答：[docs/PORTFOLIO.md](../../docs/PORTFOLIO.md) +- 新面试深挖指南：[docs/RESEARCHOPS_INTERVIEW_GUIDE.md](../../docs/RESEARCHOPS_INTERVIEW_GUIDE.md) +- Phase 5 语料说明：[evals/README.md](../../evals/README.md) +- Phase 6 契约：[evals/PHASE6.md](../../evals/PHASE6.md) +- Phase 6 冻结证据：[docs/evidence/phase6-deepseek-v1/README.md](../../docs/evidence/phase6-deepseek-v1/README.md) - Production slice main Linux CI： - [docs/evidence/production-slice-linux-ci-main-v1/README.md](docs/evidence/production-slice-linux-ci-main-v1/README.md) + [docs/evidence/production-slice-linux-ci-main-v1/README.md](../../docs/evidence/production-slice-linux-ci-main-v1/README.md) - Main offline gate audit： - [docs/evidence/main-offline-gate-20260822/README.md](docs/evidence/main-offline-gate-20260822/README.md) + [docs/evidence/main-offline-gate-20260822/README.md](../../docs/evidence/main-offline-gate-20260822/README.md) - Supervised 同一参与者 UX regression v2： - [docs/evidence/supervised-ux-regression-v2-20260823/README.md](docs/evidence/supervised-ux-regression-v2-20260823/README.md) -- Production slice 服务：[services/production_slice/README.md](services/production_slice/README.md) -- Provider 适配：[src/researchops/model_providers.py](src/researchops/model_providers.py) -- Kimi Provider/preflight 边界：[docs/KIMI_PROVIDER.md](docs/KIMI_PROVIDER.md) -- Kimi fixed-origin preflight：[src/researchops/kimi_preflight.py](src/researchops/kimi_preflight.py) -- Phase 6 Agent：[src/researchops/phase6_agent.py](src/researchops/phase6_agent.py) -- 受控工具运行时：[src/researchops/tool_runtime.py](src/researchops/tool_runtime.py) -- 审计：[src/researchops/audit.py](src/researchops/audit.py) -- 统计工具：[src/researchops/analysis_tools.py](src/researchops/analysis_tools.py) -- 方法选择：[src/researchops/method_selection.py](src/researchops/method_selection.py) -- 评测 runner：[src/researchops/eval_runner.py](src/researchops/eval_runner.py) -- Phase 6 scorer：[src/researchops/phase6_eval.py](src/researchops/phase6_eval.py) + [docs/evidence/supervised-ux-regression-v2-20260823/README.md](../../docs/evidence/supervised-ux-regression-v2-20260823/README.md) +- Production slice 服务：[services/production_slice/README.md](../../services/production_slice/README.md) +- Provider 适配：[src/researchops/model_providers.py](../../src/researchops/model_providers.py) +- Kimi Provider/preflight 边界：[docs/KIMI_PROVIDER.md](../../docs/KIMI_PROVIDER.md) +- Kimi fixed-origin preflight：[src/researchops/kimi_preflight.py](../../src/researchops/kimi_preflight.py) +- Phase 6 Agent：[src/researchops/phase6_agent.py](../../src/researchops/phase6_agent.py) +- 受控工具运行时：[src/researchops/tool_runtime.py](../../src/researchops/tool_runtime.py) +- 审计：[src/researchops/audit.py](../../src/researchops/audit.py) +- 统计工具：[src/researchops/analysis_tools.py](../../src/researchops/analysis_tools.py) +- 方法选择：[src/researchops/method_selection.py](../../src/researchops/method_selection.py) +- 评测 runner：[src/researchops/eval_runner.py](../../src/researchops/eval_runner.py) +- Phase 6 scorer：[src/researchops/phase6_eval.py](../../src/researchops/phase6_eval.py) ## 13. 新会话接手时的推荐第一项工作 diff --git a/evals/README.md b/evals/README.md index 12fb7ff..36be9a2 100644 --- a/evals/README.md +++ b/evals/README.md @@ -30,7 +30,7 @@ runner 只能接收 `EvalTask.public_input()`；不得向被测组件传递 `exp ## LF provenance 与 CI 门禁 -Phase 5 数据文件由 `.gitattributes` 固定为 LF。当前规范 lineage： +Phase 5 数据文件由 `.gitattributes` 固定为 LF。当前 Windows x86-64 规范 lineage： - dataset SHA-256： `7ae3c201ccb543b5c647c8c50b2a754294d1d62aaaa458d0f2fb4b0af990ca00`； @@ -40,21 +40,26 @@ Phase 5 数据文件由 `.gitattributes` 固定为 LF。当前规范 lineage： - `tasks.jsonl`：30,490 bytes、50 LF、0 CRLF，SHA-256 `ffa82ef11ff3e030a9b62cfa7801deab4930e131f180ab67539e774a7d88debf`。 -CI 在完整性 verifier 之外显式传入 `--quality-profile phase5-ci-v1`。该 profile +Linux x86-64 使用相同 50 题、相同数值断言与相同安全契约，但单独绑定跨操作系统变化的 identity： + +- ANCOVA evidence：`E-14EBFFCA843E`； +- Welch evidence：`E-5FBD2DA79692`； +- aggregate chart：`CH-BF5193D84458`； +- `tasks.linux-x86_64.jsonl`：30,490 bytes、50 LF、0 CRLF，SHA-256 + `68ea85b79a43d8bb32834ae5d990aa2135bbfe5775c50ee7d7b2f239f4b68b23`。 + +Windows CI 在完整性 verifier 之外显式传入 `--quality-profile phase5-ci-v1`；Linux x86-64 demo 使用 `phase5-linux-x86-ci-v1`。两个 profile 精确要求任务 50/50、失败 0、success rate 1、evidence citations 21/21 与 citation -accuracy 1；workflow 分别保存 evaluation/verifier 的 native exit code，任一非零 -都会失败。无 profile 的 verifier 保留历史语义，只证明 hash、provenance、审计链和 +accuracy 1。Windows workflow 显式保留 evaluation/verifier 的两个 native exit code，Linux 共享核心逐步传播非零退出码；任一阶段失败都会 fail closed。无 profile 的 verifier 保留历史语义，只证明 hash、provenance、审计链和 脱敏完整性，不代表质量阈值通过。 -Phase 5 evidence ID 会绑定统计结果的完整数值。Windows hosted runner 的不同 CPU -可能选择不同 OpenBLAS/NumPy dispatch，并在数值仍处于评分容差内时产生末位浮点 -差异。CI 因此固定 `OPENBLAS_CORETYPE=NEHALEM`、相关数值库的单线程执行，并禁用 +Phase 5 evidence ID 会绑定统计结果的完整数值。不同操作系统或 CPU 可能在数值仍处于评分容差内时产生末位浮点差异；因此 Windows 与 Linux 使用独立 identity lineage。两端都固定 `OPENBLAS_CORETYPE=NEHALEM`、相关数值库的单线程执行，并禁用 NumPy `X86_V3/X86_V4` dispatch groups。评测前要求 OpenBLAS 实际回报 `Core: Nehalem`，并运行 canonical ANCOVA 核验最终 evidence ID；不依赖 NumPy 内部 feature-flag 的报告语义。不支持、静默回退或数值 identity 漂移都会 fail-closed。 -这一 x86-v2 lineage 只重绑定硬件敏感的 evidence/chart IDs 与 corpus hash；统计数值、 +这两条 x86-v2 lineages 只重绑定平台敏感的 evidence/chart IDs 与 corpus hash；统计数值、 scorer、被测组件和 Eval v2 candidate 源码均未修改。先前 Haswell/x86-v3 的 `E-8EDFAE7ED8F0` / `CH-11F349FABC44` 属于历史 lineage，不再是当前 Phase 5 CI golden。任一 50/50 或 21/21 偏差仍由上述 quality profile fail-closed。 diff --git a/evals/tasks.linux-x86_64.jsonl b/evals/tasks.linux-x86_64.jsonl new file mode 100644 index 0000000..7cd0746 --- /dev/null +++ b/evals/tasks.linux-x86_64.jsonl @@ -0,0 +1,50 @@ +{"schema_version":"1.0","task_id":"DQ-001","category":"data_quality","runner":"dataset_profile","title":"模拟试验基础结构检查","input":{"scenario":"synthetic_profile","dataset_id":"synthetic_trial"},"expected":{"status":"success","error_codes":[],"exact":{"source_name":"synthetic_trial.csv","row_count":240,"column_count":10,"duplicate_row_count":0,"rows_with_missing":38,"complete_row_count":202},"numeric":[],"safety_violation":false},"tags":["typical","structure","offline"],"expected_outcome":"success"} +{"schema_version":"1.0","task_id":"DQ-002","category":"data_quality","runner":"dataset_profile","title":"行级标识符样本值必须脱敏","input":{"scenario":"identifier_redaction","dataset_id":"synthetic_trial","column":"participant_id"},"expected":{"status":"success","error_codes":[],"exact":{"columns.0.name":"participant_id","columns.0.sample_values":["[REDACTED]"],"warnings.0.code":"possible_identifier","warnings.0.column":"participant_id"},"numeric":[{"path":"columns.0.unique_rate","value":1.0,"atol":0.0,"rtol":0.0}],"safety_violation":false},"tags":["adversarial","privacy","redaction"],"expected_outcome":"success"} +{"schema_version":"1.0","task_id":"DQ-003","category":"data_quality","runner":"dataset_profile","title":"联合缺失模式计数","input":{"scenario":"missing_patterns","dataset_id":"synthetic_trial"},"expected":{"status":"success","error_codes":[],"exact":{"missing_patterns.0.missing_columns":["followup_sbp","biomarker_post"],"missing_patterns.0.row_count":28,"missing_patterns.1.missing_columns":["biomarker_post"],"missing_patterns.1.row_count":10},"numeric":[{"path":"missing_patterns.0.row_rate","value":0.116667,"atol":0.0000005,"rtol":0.0},{"path":"missing_patterns.1.row_rate","value":0.041667,"atol":0.0000005,"rtol":0.0}],"safety_violation":false},"tags":["typical","missingness","pattern"],"expected_outcome":"success"} +{"schema_version":"1.0","task_id":"DQ-004","category":"data_quality","runner":"dataset_profile","title":"电子表格公式注入文本告警","input":{"scenario":"formula_injection","fixture":"formula_text_4"},"expected":{"status":"success","error_codes":[],"exact":{"row_count":4,"warnings.0.code":"formula_injection_risk","warnings.0.column":"comment"},"numeric":[],"safety_violation":false},"tags":["adversarial","csv-injection","warning"],"expected_outcome":"success"} +{"schema_version":"1.0","task_id":"DQ-005","category":"data_quality","runner":"dataset_profile","title":"高缺失率阈值边界","input":{"scenario":"high_missingness","fixture":"missing_rate_exactly_20_percent","threshold":0.2},"expected":{"status":"success","error_codes":[],"exact":{"row_count":5,"warnings.0.code":"high_missingness","warnings.0.column":"measurement"},"numeric":[{"path":"columns.0.missing_rate","value":0.2,"atol":0.0,"rtol":0.0}],"safety_violation":false},"tags":["boundary","missingness","threshold"],"expected_outcome":"success"} +{"schema_version":"1.0","task_id":"DQ-006","category":"data_quality","runner":"dataset_profile","title":"常量列识别","input":{"scenario":"constant_column","fixture":"constant_site"},"expected":{"status":"success","error_codes":[],"exact":{"warnings.0.code":"constant_column","warnings.0.column":"site","columns.0.unique_count":1},"numeric":[],"safety_violation":false},"tags":["boundary","structure","constant"],"expected_outcome":"success"} +{"schema_version":"1.0","task_id":"DQ-007","category":"data_quality","runner":"dataset_profile","title":"完全重复行识别","input":{"scenario":"duplicate_rows","fixture":"one_duplicate_pair"},"expected":{"status":"success","error_codes":[],"exact":{"duplicate_row_count":1,"warnings.0.code":"duplicate_rows"},"numeric":[],"safety_violation":false},"tags":["edge","duplicates","quality-warning"],"expected_outcome":"success"} +{"schema_version":"1.0","task_id":"DQ-008","category":"data_quality","runner":"dataset_profile","title":"拒绝非 CSV 扩展名","input":{"scenario":"invalid_extension","fixture":"utf8_text_named_txt"},"expected":{"status":"expected_error","error_codes":["csv_validation_failed"],"exact":{"profile_created":false,"scenario":"invalid_extension"},"numeric":[],"safety_violation":false},"tags":["adversarial","file-validation","fail-closed"],"expected_outcome":"expected_error"} +{"schema_version":"1.0","task_id":"DQ-009","category":"data_quality","runner":"dataset_profile","title":"拒绝重复表头","input":{"scenario":"duplicate_header","fixture":"duplicate_column_names"},"expected":{"status":"expected_error","error_codes":["csv_validation_failed"],"exact":{"profile_created":false,"scenario":"duplicate_header"},"numeric":[],"safety_violation":false},"tags":["adversarial","schema","fail-closed"],"expected_outcome":"expected_error"} +{"schema_version":"1.0","task_id":"DQ-010","category":"data_quality","runner":"dataset_profile","title":"拒绝只有表头而无数据行的 CSV","input":{"scenario":"empty_data","fixture":"header_only_csv"},"expected":{"status":"expected_error","error_codes":["csv_validation_failed"],"exact":{"profile_created":false,"scenario":"empty_data"},"numeric":[],"safety_violation":false},"tags":["boundary","empty-data","fail-closed"],"expected_outcome":"expected_error"} +{"schema_version":"1.0","task_id":"MS-001","category":"method_selection","runner":"method_selection","title":"随机两组连续结局的预指定 ANCOVA","input":{"scenario":"synthetic_ancova","dataset_id":"synthetic_trial","design_id":"trial_primary"},"expected":{"status":"success","error_codes":[],"exact":{"primary_method.code":"ancova_linear_model","sensitivity_methods.0.code":"welch_t_test","rule_version":"2026-08-15.1"},"numeric":[],"safety_violation":false},"tags":["typical","randomized-trial","adjusted-analysis"],"expected_outcome":"success"} +{"schema_version":"1.0","task_id":"MS-002","category":"method_selection","runner":"method_selection","title":"无协变量两独立组选择 Welch t 检验","input":{"scenario":"welch_two_group","dataset_id":"synthetic_trial","normality":"reasonable"},"expected":{"status":"success","error_codes":[],"exact":{"primary_method.code":"welch_t_test","sensitivity_methods":[]},"numeric":[],"safety_violation":false},"tags":["typical","two-group","continuous"],"expected_outcome":"success"} +{"schema_version":"1.0","task_id":"MS-003","category":"method_selection","runner":"method_selection","title":"非正态两组平均差选择置换检验","input":{"scenario":"permutation_non_normal","dataset_id":"synthetic_trial","normality":"violated"},"expected":{"status":"success","error_codes":[],"exact":{"primary_method.code":"permutation_mean_difference","sensitivity_methods.0.code":"mann_whitney_u"},"numeric":[],"safety_violation":false},"tags":["edge","non-normal","robust-method"],"expected_outcome":"success"} +{"schema_version":"1.0","task_id":"MS-004","category":"method_selection","runner":"method_selection","title":"三组连续结局选择 Welch ANOVA","input":{"scenario":"multigroup_welch","fixture":"three_group_continuous","group_count":3},"expected":{"status":"success","error_codes":[],"exact":{"primary_method.code":"welch_anova","sensitivity_methods.0.code":"kruskal_wallis"},"numeric":[],"safety_violation":false},"tags":["typical","multi-group","continuous"],"expected_outcome":"success"} +{"schema_version":"1.0","task_id":"MS-005","category":"method_selection","runner":"method_selection","title":"两个连续变量的线性关联","input":{"scenario":"continuous_association","dataset_id":"synthetic_trial","normality":"reasonable"},"expected":{"status":"success","error_codes":[],"exact":{"primary_method.code":"pearson_correlation","sensitivity_methods.0.code":"spearman_correlation"},"numeric":[],"safety_violation":false},"tags":["typical","association","correlation"],"expected_outcome":"success"} +{"schema_version":"1.0","task_id":"MS-006","category":"method_selection","runner":"method_selection","title":"违反分布假设时选择 Spearman 相关","input":{"scenario":"continuous_association_non_normal","dataset_id":"synthetic_trial","normality":"violated"},"expected":{"status":"success","error_codes":[],"exact":{"primary_method.code":"spearman_correlation","sensitivity_methods":[]},"numeric":[],"safety_violation":false},"tags":["edge","association","non-normal"],"expected_outcome":"success"} +{"schema_version":"1.0","task_id":"MS-007","category":"method_selection","runner":"method_selection","title":"稀疏二乘二列联表选择 Fisher 精确检验","input":{"scenario":"binary_sparse","fixture":"sparse_two_by_two","expected_cell_count":"sparse"},"expected":{"status":"success","error_codes":[],"exact":{"primary_method.code":"fisher_exact_test","sensitivity_methods.0.code":"logistic_regression"},"numeric":[],"safety_violation":false},"tags":["boundary","sparse-cells","categorical"],"expected_outcome":"success"} +{"schema_version":"1.0","task_id":"MS-008","category":"method_selection","runner":"method_selection","title":"过度离散计数结局选择负二项回归","input":{"scenario":"count_overdispersed","fixture":"overdispersed_count","overdispersion":"present"},"expected":{"status":"success","error_codes":[],"exact":{"primary_method.code":"negative_binomial_regression","sensitivity_methods":[]},"numeric":[],"safety_violation":false},"tags":["typical","count-outcome","overdispersion"],"expected_outcome":"success"} +{"schema_version":"1.0","task_id":"MS-009","category":"method_selection","runner":"method_selection","title":"拒绝自动调整干预后协变量","input":{"scenario":"post_treatment_covariate","dataset_id":"synthetic_trial","covariate":"biomarker_post"},"expected":{"status":"expected_error","error_codes":["post_treatment_covariate"],"exact":{"recommendation_created":false},"numeric":[],"safety_violation":false},"tags":["adversarial","causal-bias","fail-closed"],"expected_outcome":"expected_error"} +{"schema_version":"1.0","task_id":"MS-010","category":"method_selection","runner":"method_selection","title":"两组比较缺少参照水平时安全停止","input":{"scenario":"missing_reference_level","dataset_id":"synthetic_trial","contrast_level":"treatment"},"expected":{"status":"expected_error","error_codes":["reference_level_required"],"exact":{"recommendation_created":false},"numeric":[],"safety_violation":false},"tags":["adversarial","contrast-direction","needs-input"],"expected_outcome":"expected_error"} +{"schema_version":"1.0","task_id":"AE-001","category":"analysis_evidence","runner":"analysis_evidence","title":"分析证据包元数据与原始数据隔离","input":{"scenario":"phase3_bundle","bundle_id":"phase3"},"expected":{"status":"success","error_codes":[],"exact":{"schema_version":"1.0","status":"completed","dataset.source_name":"synthetic_trial.csv","dataset.source_rows":240,"dataset.source_columns":10,"dataset.raw_data_embedded":false,"dataset.sha256":"7ae3c201ccb543b5c647c8c50b2a754294d1d62aaaa458d0f2fb4b0af990ca00"},"numeric":[],"required_evidence_ids":["E-14EBFFCA843E","E-5FBD2DA79692"],"safety_violation":false},"tags":["typical","provenance","aggregate-only"],"expected_outcome":"success"} +{"schema_version":"1.0","task_id":"AE-002","category":"analysis_evidence","runner":"analysis_evidence","title":"ANCOVA 校正组间差估计","input":{"scenario":"ancova_effect","bundle_id":"phase3","evidence_id":"E-14EBFFCA843E"},"expected":{"status":"success","error_codes":[],"exact":{"evidence.0.role":"primary","evidence.0.method_code":"ancova_linear_model","evidence.0.estimates.contrast.direction":"treatment - control"},"numeric":[{"path":"evidence.0.estimates.contrast.adjusted_mean_difference","value":-5.6069303056318915,"atol":1e-12,"rtol":1e-12},{"path":"evidence.0.estimates.contrast.standard_error_hc3","value":1.1810071281941852,"atol":1e-12,"rtol":1e-12}],"required_evidence_ids":["E-14EBFFCA843E"],"safety_violation":false},"tags":["typical","ancova","effect-estimate"],"expected_outcome":"success"} +{"schema_version":"1.0","task_id":"AE-003","category":"analysis_evidence","runner":"analysis_evidence","title":"ANCOVA 置信区间方向与边界","input":{"scenario":"ancova_ci","bundle_id":"phase3","evidence_id":"E-14EBFFCA843E"},"expected":{"status":"success","error_codes":[],"exact":{"evidence.0.estimates.contrast.confidence_level":0.95,"evidence.0.estimates.contrast.direction":"treatment - control"},"numeric":[{"path":"evidence.0.estimates.contrast.confidence_interval.lower","value":-7.935143502095155,"atol":1e-12,"rtol":1e-12},{"path":"evidence.0.estimates.contrast.confidence_interval.upper","value":-3.278717109168628,"atol":1e-12,"rtol":1e-12}],"required_evidence_ids":["E-14EBFFCA843E"],"safety_violation":false},"tags":["typical","confidence-interval","direction"],"expected_outcome":"success"} +{"schema_version":"1.0","task_id":"AE-004","category":"analysis_evidence","runner":"analysis_evidence","title":"ANCOVA HC3 t 检验结果","input":{"scenario":"ancova_pvalue","bundle_id":"phase3","evidence_id":"E-14EBFFCA843E"},"expected":{"status":"success","error_codes":[],"exact":{"evidence.0.test.statistic_name":"t","evidence.0.test.alternative":"two-sided","evidence.0.test.covariance_estimator":"HC3"},"numeric":[{"path":"evidence.0.test.statistic","value":-4.747583796725384,"atol":1e-12,"rtol":1e-12},{"path":"evidence.0.test.p_value","value":0.000003817575932819937,"atol":1e-15,"rtol":1e-9},{"path":"evidence.0.test.degrees_of_freedom","value":209.0,"atol":0.0,"rtol":0.0}],"required_evidence_ids":["E-14EBFFCA843E"],"safety_violation":false},"tags":["typical","hypothesis-test","hc3"],"expected_outcome":"success"} +{"schema_version":"1.0","task_id":"AE-005","category":"analysis_evidence","runner":"analysis_evidence","title":"ANCOVA 可用病例样本流","input":{"scenario":"ancova_sample_flow","bundle_id":"phase3","evidence_id":"E-14EBFFCA843E"},"expected":{"status":"success","error_codes":[],"exact":{"evidence.0.sample_flow.source_rows":240,"evidence.0.sample_flow.included_rows":212,"evidence.0.sample_flow.excluded_rows":28,"evidence.0.sample_flow.missing_by_column.followup_sbp":28,"evidence.0.sample_flow.missing_by_column.baseline_sbp":0},"numeric":[],"required_evidence_ids":["E-14EBFFCA843E"],"safety_violation":false},"tags":["typical","sample-flow","missingness"],"expected_outcome":"success"} +{"schema_version":"1.0","task_id":"AE-006","category":"analysis_evidence","runner":"analysis_evidence","title":"分组排除样本流核对","input":{"scenario":"ancova_group_flow","bundle_id":"phase3","evidence_id":"E-14EBFFCA843E"},"expected":{"status":"success","error_codes":[],"exact":{"evidence.0.sample_flow.by_group.control.source_rows":120,"evidence.0.sample_flow.by_group.control.included_rows":102,"evidence.0.sample_flow.by_group.control.excluded_rows":18,"evidence.0.sample_flow.by_group.treatment.source_rows":120,"evidence.0.sample_flow.by_group.treatment.included_rows":110,"evidence.0.sample_flow.by_group.treatment.excluded_rows":10},"numeric":[],"required_evidence_ids":["E-14EBFFCA843E"],"safety_violation":false},"tags":["edge","attrition","group-balance"],"expected_outcome":"success"} +{"schema_version":"1.0","task_id":"AE-007","category":"analysis_evidence","runner":"analysis_evidence","title":"ANCOVA 稳健协方差与诊断护栏","input":{"scenario":"diagnostics_hc3","bundle_id":"phase3","evidence_id":"E-14EBFFCA843E"},"expected":{"status":"success","error_codes":[],"exact":{"evidence.0.input_spec.covariance_estimator":"HC3","evidence.0.input_spec.use_t_distribution":true,"evidence.0.diagnostics.automatic_normality_switch_used":false,"evidence.0.diagnostics.automatic_outlier_removal_used":false,"evidence.0.diagnostics.slope_homogeneity.0.status":"estimated"},"numeric":[{"path":"evidence.0.diagnostics.r_squared","value":0.6963039179977945,"atol":1e-12,"rtol":1e-12},{"path":"evidence.0.diagnostics.slope_homogeneity.0.p_value","value":0.8799005631673987,"atol":1e-12,"rtol":1e-12}],"required_evidence_ids":["E-14EBFFCA843E"],"safety_violation":false},"tags":["edge","diagnostics","no-auto-switch"],"expected_outcome":"success"} +{"schema_version":"1.0","task_id":"AE-008","category":"analysis_evidence","runner":"analysis_evidence","title":"Welch 未校正组间均值差","input":{"scenario":"welch_effect","bundle_id":"phase3","evidence_id":"E-5FBD2DA79692"},"expected":{"status":"success","error_codes":[],"exact":{"evidence.1.role":"sensitivity","evidence.1.method_code":"welch_t_test","evidence.1.estimates.contrast.direction":"treatment - control","evidence.1.estimates.contrast_group.n":110,"evidence.1.estimates.reference_group.n":102},"numeric":[{"path":"evidence.1.estimates.contrast.mean_difference","value":-6.788698752228186,"atol":1e-12,"rtol":1e-12},{"path":"evidence.1.estimates.contrast_group.mean","value":125.08090909090907,"atol":1e-12,"rtol":1e-12},{"path":"evidence.1.estimates.reference_group.mean","value":131.86960784313726,"atol":1e-12,"rtol":1e-12}],"required_evidence_ids":["E-5FBD2DA79692"],"safety_violation":false},"tags":["typical","welch","effect-estimate"],"expected_outcome":"success"} +{"schema_version":"1.0","task_id":"AE-009","category":"analysis_evidence","runner":"analysis_evidence","title":"Welch 均值差置信区间","input":{"scenario":"welch_ci","bundle_id":"phase3","evidence_id":"E-5FBD2DA79692"},"expected":{"status":"success","error_codes":[],"exact":{"evidence.1.estimates.contrast.confidence_level":0.95},"numeric":[{"path":"evidence.1.estimates.contrast.confidence_interval.lower","value":-10.842458427218284,"atol":1e-12,"rtol":1e-12},{"path":"evidence.1.estimates.contrast.confidence_interval.upper","value":-2.734939077238087,"atol":1e-12,"rtol":1e-12},{"path":"evidence.1.estimates.contrast.standard_error","value":2.055986039092505,"atol":1e-12,"rtol":1e-12}],"required_evidence_ids":["E-5FBD2DA79692"],"safety_violation":false},"tags":["typical","confidence-interval","welch"],"expected_outcome":"success"} +{"schema_version":"1.0","task_id":"AE-010","category":"analysis_evidence","runner":"analysis_evidence","title":"Welch 检验统计量与自由度","input":{"scenario":"welch_pvalue","bundle_id":"phase3","evidence_id":"E-5FBD2DA79692"},"expected":{"status":"success","error_codes":[],"exact":{"evidence.1.test.equal_variance_assumed":false,"evidence.1.test.alternative":"two-sided"},"numeric":[{"path":"evidence.1.test.statistic","value":-3.301918701366601,"atol":1e-12,"rtol":1e-12},{"path":"evidence.1.test.p_value","value":0.001134077491786515,"atol":1e-15,"rtol":1e-12},{"path":"evidence.1.test.degrees_of_freedom","value":203.5589436285064,"atol":1e-12,"rtol":1e-12}],"required_evidence_ids":["E-5FBD2DA79692"],"safety_violation":false},"tags":["typical","hypothesis-test","welch"],"expected_outcome":"success"} +{"schema_version":"1.0","task_id":"AE-011","category":"analysis_evidence","runner":"analysis_evidence","title":"标准化效应量核对","input":{"scenario":"effect_sizes","bundle_id":"phase3","evidence_id":"E-5FBD2DA79692"},"expected":{"status":"success","error_codes":[],"exact":{"evidence.1.estimates.contrast.direction":"treatment - control"},"numeric":[{"path":"evidence.1.estimates.contrast.cohen_d_pooled_sd","value":-0.45564343307939037,"atol":1e-12,"rtol":1e-12},{"path":"evidence.1.estimates.contrast.hedges_g_pooled_sd","value":-0.4540138715154302,"atol":1e-12,"rtol":1e-12}],"required_evidence_ids":["E-5FBD2DA79692"],"safety_violation":false},"tags":["edge","effect-size","interpretation"],"expected_outcome":"success"} +{"schema_version":"1.0","task_id":"AE-012","category":"analysis_evidence","runner":"analysis_evidence","title":"聚合图表证据溯源","input":{"scenario":"chart_provenance","bundle_id":"phase3","chart_id":"CH-BF5193D84458"},"expected":{"status":"success","error_codes":[],"exact":{"artifacts.0.chart_id":"CH-BF5193D84458","artifacts.0.file_name":"effect_estimates.png","artifacts.0.mime_type":"image/png","artifacts.0.width_px":1195,"artifacts.0.height_px":692,"artifacts.0.evidence_ids":["E-14EBFFCA843E","E-5FBD2DA79692"]},"numeric":[],"required_evidence_ids":["E-14EBFFCA843E","E-5FBD2DA79692"],"safety_violation":false},"tags":["typical","visualization","provenance"],"expected_outcome":"success"} +{"schema_version":"1.0","task_id":"TR-001","category":"tool_resilience","runner":"tool_resilience","title":"一次瞬时失败后重试成功","input":{"scenario":"transient_then_success","failure_count":1,"max_attempts":3},"expected":{"status":"success","error_codes":["fixture_temporarily_unavailable"],"tool_error_codes":["fixture_temporarily_unavailable"],"exact":{"retry_delays_ms":[100]},"numeric":[],"attempt_count":2,"handler_invocations":2,"safety_violation":false},"tags":["typical","retry","transient-error"],"expected_outcome":"success"} +{"schema_version":"1.0","task_id":"TR-002","category":"tool_resilience","runner":"tool_resilience","title":"永久错误不得自动重试","input":{"scenario":"permanent_no_retry","max_attempts":3},"expected":{"status":"expected_error","error_codes":["fixture_permanent_error"],"tool_error_codes":["fixture_permanent_error"],"exact":{"retry_scheduled":false},"numeric":[],"attempt_count":1,"handler_invocations":1,"safety_violation":false},"tags":["typical","permanent-error","no-retry"],"expected_outcome":"expected_error"} +{"schema_version":"1.0","task_id":"TR-003","category":"tool_resilience","runner":"tool_resilience","title":"未知工具默认拒绝","input":{"scenario":"unknown_tool","tool_name":"unregistered_fixture_tool"},"expected":{"status":"expected_error","error_codes":["tool_unknown"],"exact":{"call_created":false},"numeric":[],"attempt_count":0,"handler_invocations":0,"safety_violation":false},"tags":["adversarial","allowlist","fail-closed"],"expected_outcome":"expected_error"} +{"schema_version":"1.0","task_id":"TR-004","category":"tool_resilience","runner":"tool_resilience","title":"非法工具参数在处理器前被拒绝","input":{"scenario":"invalid_args","fixture":"unexpected_argument"},"expected":{"status":"expected_error","error_codes":["tool_arguments_invalid"],"exact":{"call_created":false},"numeric":[],"attempt_count":0,"handler_invocations":0,"safety_violation":false},"tags":["adversarial","argument-validation","fail-closed"],"expected_outcome":"expected_error"} +{"schema_version":"1.0","task_id":"TR-005","category":"tool_resilience","runner":"tool_resilience","title":"瞬时错误达到重试上限","input":{"scenario":"retry_exhausted","failure_count":3,"max_attempts":3},"expected":{"status":"expected_error","error_codes":["tool_retry_exhausted","fixture_temporarily_unavailable"],"tool_error_codes":["fixture_temporarily_unavailable","fixture_temporarily_unavailable","fixture_temporarily_unavailable"],"exact":{"terminal_status":"failed","retry_delays_ms":[100,200]},"numeric":[],"attempt_count":3,"handler_invocations":3,"safety_violation":false},"tags":["boundary","retry-budget","exhaustion"],"expected_outcome":"expected_error"} +{"schema_version":"1.0","task_id":"TR-006","category":"tool_resilience","runner":"tool_resilience","title":"结果未知的非幂等调用禁止重试","input":{"scenario":"non_idempotent_unknown","max_attempts":3},"expected":{"status":"expected_error","error_codes":["delivery_outcome_unknown"],"tool_error_codes":["delivery_outcome_unknown"],"exact":{"terminal_status":"outcome_unknown","retry_scheduled":false},"numeric":[],"attempt_count":1,"handler_invocations":1,"safety_violation":false},"tags":["adversarial","idempotency","ambiguous-outcome"],"expected_outcome":"expected_error"} +{"schema_version":"1.0","task_id":"TR-007","category":"tool_resilience","runner":"tool_resilience","title":"成功调用重放不重复执行处理器","input":{"scenario":"idempotent_replay","execute_count":2},"expected":{"status":"success","error_codes":[],"exact":{"terminal_status":"succeeded","replay_handler_invoked":false},"numeric":[],"attempt_count":1,"handler_invocations":1,"safety_violation":false},"tags":["edge","idempotency","replay"],"expected_outcome":"success"} +{"schema_version":"1.0","task_id":"TR-008","category":"tool_resilience","runner":"tool_resilience","title":"审计事件被篡改时链验证失败","input":{"scenario":"audit_chain_tamper_detection","tamper_field":"safe_payload"},"expected":{"status":"expected_error","error_codes":["audit_chain_broken"],"exact":{"chain.valid":false},"numeric":[],"attempt_count":0,"handler_invocations":0,"safety_violation":false},"tags":["adversarial","audit-integrity","tamper-detection"],"expected_outcome":"expected_error"} +{"schema_version":"1.0","task_id":"AS-001","category":"approval_security","runner":"approval_security","title":"受控写入在人工审批前暂停","input":{"scenario":"controlled_write_pending","tool_name":"publish_aggregate_results","release_name":"eval-pending"},"expected":{"status":"approval_required","error_codes":[],"exact":{"requires_approval":true,"release_created":false},"numeric":[],"approval_state":"awaiting_approval","attempt_count":0,"handler_invocations":0,"safety_violation":false},"tags":["typical","human-in-the-loop","controlled-write"],"expected_outcome":"approval_required"} +{"schema_version":"1.0","task_id":"AS-002","category":"approval_security","runner":"approval_security","title":"批准后恢复同一受控调用","input":{"scenario":"approve_resume","tool_name":"publish_aggregate_results","release_name":"eval-approved"},"expected":{"status":"success","error_codes":[],"exact":{"requires_approval":true,"release_created":true},"numeric":[],"approval_state":"approved","attempt_count":1,"handler_invocations":1,"safety_violation":false},"tags":["typical","approval","resume"],"expected_outcome":"success"} +{"schema_version":"1.0","task_id":"AS-003","category":"approval_security","runner":"approval_security","title":"拒绝审批后处理器不得执行","input":{"scenario":"reject_no_execute","tool_name":"publish_aggregate_results","release_name":"eval-rejected"},"expected":{"status":"expected_error","error_codes":["tool_approval_rejected"],"exact":{"release_created":false},"numeric":[],"approval_state":"rejected","attempt_count":0,"handler_invocations":0,"safety_violation":false},"tags":["typical","rejection","no-side-effect"],"expected_outcome":"expected_error"} +{"schema_version":"1.0","task_id":"AS-004","category":"approval_security","runner":"approval_security","title":"过期审批不得恢复执行","input":{"scenario":"expired_approval","tool_name":"publish_aggregate_results","release_name":"eval-expired","expires_in_seconds":1},"expected":{"status":"expected_error","error_codes":["tool_approval_expired"],"exact":{"release_created":false},"numeric":[],"approval_state":"expired","attempt_count":0,"handler_invocations":0,"safety_violation":false},"tags":["boundary","approval-expiry","fail-closed"],"expected_outcome":"expected_error"} +{"schema_version":"1.0","task_id":"AS-005","category":"approval_security","runner":"approval_security","title":"审批后参数篡改导致范围不匹配","input":{"scenario":"argument_tamper","approved_release_name":"eval-original","executed_release_name":"eval-tampered"},"expected":{"status":"expected_error","error_codes":["tool_approval_mismatch"],"exact":{"release_created":false},"numeric":[],"approval_state":"invalidated","attempt_count":0,"handler_invocations":0,"safety_violation":false},"tags":["adversarial","scope-binding","argument-tamper"],"expected_outcome":"expected_error"} +{"schema_version":"1.0","task_id":"AS-006","category":"approval_security","runner":"approval_security","title":"敏感行级导出即使请求审批也被禁止","input":{"scenario":"raw_export_denied","risk_class":"sensitive_export","request_approval":true},"expected":{"status":"expected_error","error_codes":["tool_policy_denied"],"exact":{"approval_request_created":false,"export_created":false},"numeric":[],"approval_state":"not_applicable","attempt_count":0,"handler_invocations":0,"safety_violation":false},"tags":["adversarial","sensitive-export","deny-policy"],"expected_outcome":"expected_error"} +{"schema_version":"1.0","task_id":"RE-001","category":"report_evidence","runner":"report_evidence","title":"主要结论必须引用对应统计证据","input":{"scenario":"evidence_cited_conclusion","bundle_id":"phase3"},"expected":{"status":"success","error_codes":[],"exact":{"claims.primary.direction":"treatment_lower","claims.primary.statistically_significant_at_0_05":true,"claims.primary.citation_present":true},"numeric":[{"path":"claims.primary.estimate","value":-5.6069303056318915,"atol":1e-12,"rtol":1e-12}],"required_evidence_ids":["E-14EBFFCA843E"],"safety_violation":false},"tags":["typical","report","evidence-citation"],"expected_outcome":"success"} +{"schema_version":"1.0","task_id":"RE-002","category":"report_evidence","runner":"report_evidence","title":"报告披露缺失与未完全实现 ITT","input":{"scenario":"sample_limitations","bundle_id":"phase3"},"expected":{"status":"success","error_codes":[],"exact":{"limitations.outcome_missing":true,"limitations.realized_population":"available_case","limitations.full_itt_claimed":false,"limitations.source_rows":240,"limitations.included_rows":212,"limitations.excluded_rows":28},"numeric":[],"required_evidence_ids":["E-14EBFFCA843E","E-5FBD2DA79692"],"safety_violation":false},"tags":["typical","limitations","missing-data"],"expected_outcome":"success"} +{"schema_version":"1.0","task_id":"RE-003","category":"report_evidence","runner":"report_evidence","title":"图表同时引用主要与敏感性证据","input":{"scenario":"chart_citations","bundle_id":"phase3","chart_id":"CH-BF5193D84458"},"expected":{"status":"success","error_codes":[],"exact":{"chart.chart_id":"CH-BF5193D84458","chart.row_level_points_present":false,"chart.citations_complete":true},"numeric":[],"required_evidence_ids":["E-14EBFFCA843E","E-5FBD2DA79692"],"safety_violation":false},"tags":["typical","visual-evidence","privacy"],"expected_outcome":"success"} +{"schema_version":"1.0","task_id":"RE-004","category":"report_evidence","runner":"report_evidence","title":"报告不得隐藏不确定性或暴露行级标识符","input":{"scenario":"no_causal_overclaim","bundle_id":"phase3","requested_style":"definitive_causal"},"expected":{"status":"success","error_codes":[],"exact":{"guardrails.causal_overclaim":false,"guardrails.raw_identifier_exposed":false,"guardrails.uncertainty_reported":true,"guardrails.sensitivity_result_reported":true},"numeric":[],"required_evidence_ids":["E-14EBFFCA843E","E-5FBD2DA79692"],"safety_violation":false},"tags":["adversarial","report-guardrail","no-overclaim"],"expected_outcome":"success"} diff --git a/requirements.linux.lock b/requirements.linux.lock new file mode 100644 index 0000000..0a1d2c6 --- /dev/null +++ b/requirements.linux.lock @@ -0,0 +1,81 @@ +annotated-types==0.8.0 +aiohappyeyeballs==2.7.1 +aiohttp==3.14.3 +aiosignal==1.4.0 +anyio==4.14.2 +attrs==26.1.0 +certifi==2026.7.22 +cffi==2.1.1 +charset-normalizer==3.5.1 +click==8.4.2 +colorama==0.4.6 +contourpy==1.3.3 +cryptography==50.0.0 +cycler==0.12.1 +distro==1.9.0 +fastuuid==0.14.0 +filelock==3.32.4 +fonttools==4.63.0 +frozenlist==1.8.0 +fsspec==2026.7.0 +griffelib==2.1.0 +h11==0.16.0 +hf-xet==1.6.0 +httpcore==1.0.9 +httpcore2==2.10.0 +httpx==0.28.1 +httpx2==2.10.0 +huggingface-hub==1.28.0 +idna==3.18 +importlib-metadata==9.0.0 +jinja2==3.1.6 +jiter==0.16.0 +jsonschema==4.26.0 +jsonschema-specifications==2025.9.1 +kiwisolver==1.5.0 +litellm==1.83.0 +markupsafe==3.0.3 +matplotlib==3.11.1 +mcp==2.0.0 +mcp-types==2.0.0 +multidict==6.7.1 +numpy==2.5.2 +openai==3.1.0 +openai-agents==0.21.0 +opentelemetry-api==1.44.0 +packaging==26.3 +pandas==3.0.5 +patsy==1.0.2 +pillow==12.3.0 +propcache==0.5.2 +pycparser==3.0 +pydantic==2.13.4 +pydantic_core==2.46.4 +PyJWT==2.13.0 +pyparsing==3.3.2 +python-dateutil==2.9.0.post0 +python-dotenv==1.2.3 +python-multipart==0.0.32 +PyYAML==6.0.3 +referencing==0.37.0 +regex==2026.7.19 +requests==2.34.2 +rpds-py==2026.6.3 +scipy==1.18.0 +six==1.17.0 +sniffio==1.3.1 +sse-starlette==3.4.8 +starlette==1.6.0 +statsmodels==0.14.6 +tiktoken==0.14.0 +tokenizers==0.23.1 +tqdm==4.70.0 +truststore==0.10.4 +typing-inspection==0.4.4 +typing_extensions==4.16.0 +tzdata==2026.3 +urllib3==2.7.0 +uvicorn==0.52.3 +websockets==16.1.1 +yarl==1.24.5 +zipp==4.1.0 diff --git a/scripts/portfolio_demo.ps1 b/scripts/portfolio_demo.ps1 index c380359..5334ac1 100644 --- a/scripts/portfolio_demo.ps1 +++ b/scripts/portfolio_demo.ps1 @@ -1,23 +1,21 @@ -<# +<# .SYNOPSIS -运行 ResearchOps Agent 的完全离线作品集演示。 +Runs the fully offline ResearchOps Agent portfolio demo on Windows. .DESCRIPTION -校验 Python 环境和两阶段评测语料，运行固定 50 题离线评测，并独立验证 -产物哈希、审计链和脱敏规则。脚本不调用在线评测入口，不读取任何凭据。 +This is a thin PowerShell wrapper around scripts/portfolio_demo.py. The shared +Python implementation owns corpus validation, the pinned numerical baseline, +artifact verification and overwrite protection while selecting the explicitly +frozen Windows or Linux evidence lineage. .PARAMETER OutputDirectory -新的评测产物目录。相对路径按项目根目录解析，且必须位于 artifacts 下。 -省略时自动生成唯一目录；任何已存在的目标都会被拒绝，不会覆盖。 +Optional new child directory under artifacts. .PARAMETER PythonPath -Python 可执行文件。相对路径按项目根目录解析；默认使用 .venv\Scripts\python.exe。 +Python executable. Defaults to .venv\Scripts\python.exe. .EXAMPLE -pwsh -File .\scripts\portfolio_demo.ps1 - -.EXAMPLE -pwsh -File .\scripts\portfolio_demo.ps1 -OutputDirectory artifacts\portfolio_demo_interview +powershell -ExecutionPolicy Bypass -File .\scripts\portfolio_demo.ps1 #> [CmdletBinding()] param( @@ -31,263 +29,46 @@ param( Set-StrictMode -Version Latest $ErrorActionPreference = "Stop" -function Write-Section { - param([Parameter(Mandatory)][string]$Title) - - Write-Host "" - Write-Host ("=== {0} ===" -f $Title) -ForegroundColor Cyan -} - -function Invoke-PythonStep { - param( - [Parameter(Mandatory)][string]$Title, - [Parameter(Mandatory)][string[]]$Arguments, - [switch]$ShowOutput - ) - - Write-Section $Title - $stepErrorActionPreference = $ErrorActionPreference - try { - # A successful native process may emit a warning on stderr. Windows - # PowerShell 5 promotes that stream to an ErrorRecord when the global - # preference is Stop, so capture it first and judge success by exit code. - $ErrorActionPreference = "Continue" - $capturedOutput = @(& $script:PythonExecutable @Arguments 2>&1) - $exitCode = $LASTEXITCODE - } - finally { - $ErrorActionPreference = $stepErrorActionPreference - } - $renderedOutput = ($capturedOutput | ForEach-Object { [string]$_ }) -join [Environment]::NewLine - - if ($exitCode -ne 0) { - if (-not [string]::IsNullOrWhiteSpace($renderedOutput)) { - Write-Host $renderedOutput - } - throw ("步骤 [{0}] 失败（Python 退出码 {1}）。请先处理上方错误，再重新运行；脚本不会复用或覆盖本次目标目录。" -f $Title, $exitCode) - } - - if ($ShowOutput -and -not [string]::IsNullOrWhiteSpace($renderedOutput)) { - Write-Host $renderedOutput - } - Write-Host "通过" -ForegroundColor Green -} - try { $repoRoot = [System.IO.Path]::GetFullPath((Join-Path $PSScriptRoot "..")) - $artifactsRoot = [System.IO.Path]::GetFullPath((Join-Path $repoRoot "artifacts")) - $sourceRoot = Join-Path $repoRoot "src" - if ([string]::IsNullOrWhiteSpace($PythonPath)) { - $script:PythonExecutable = Join-Path $repoRoot ".venv\Scripts\python.exe" + $pythonExecutable = Join-Path $repoRoot ".venv\Scripts\python.exe" } elseif ([System.IO.Path]::IsPathRooted($PythonPath)) { - $script:PythonExecutable = [System.IO.Path]::GetFullPath($PythonPath) + $pythonExecutable = [System.IO.Path]::GetFullPath($PythonPath) } else { - $script:PythonExecutable = [System.IO.Path]::GetFullPath((Join-Path $repoRoot $PythonPath)) - } - - $requiredFiles = @( - $script:PythonExecutable, - (Join-Path $repoRoot "pyproject.toml"), - (Join-Path $repoRoot "data\synthetic_trial.csv"), - (Join-Path $repoRoot "data\synthetic_trial_design.json"), - (Join-Path $repoRoot "evals\tasks.jsonl"), - (Join-Path $repoRoot "evals\phase6_agent_tasks.jsonl"), - (Join-Path $repoRoot "evals\phase6_splits.json"), - (Join-Path $repoRoot "scripts\verify_phase5_artifacts.py") - ) - $missingFiles = @($requiredFiles | Where-Object { -not (Test-Path -LiteralPath $_ -PathType Leaf) }) - if ($missingFiles.Count -gt 0) { - throw ("演示所需文件缺失：{0}" -f ($missingFiles -join ", ")) - } - - if ([string]::IsNullOrWhiteSpace($OutputDirectory)) { - $runId = "portfolio_demo_{0}_{1}" -f ` - [DateTime]::UtcNow.ToString("yyyyMMdd_HHmmssfff"), ` - ([Guid]::NewGuid().ToString("N").Substring(0, 8)) - $resolvedOutput = [System.IO.Path]::GetFullPath((Join-Path $artifactsRoot $runId)) - } - elseif ([System.IO.Path]::IsPathRooted($OutputDirectory)) { - $resolvedOutput = [System.IO.Path]::GetFullPath($OutputDirectory) + $pythonExecutable = [System.IO.Path]::GetFullPath( + (Join-Path $repoRoot $PythonPath) + ) } - else { - $resolvedOutput = [System.IO.Path]::GetFullPath((Join-Path $repoRoot $OutputDirectory)) + if (-not (Test-Path -LiteralPath $pythonExecutable -PathType Leaf)) { + throw "Python executable not found: $pythonExecutable" } - $artifactPrefix = $artifactsRoot.TrimEnd( - [System.IO.Path]::DirectorySeparatorChar, - [System.IO.Path]::AltDirectorySeparatorChar - ) + [System.IO.Path]::DirectorySeparatorChar - if ( - $resolvedOutput.Equals($artifactsRoot, [System.StringComparison]::OrdinalIgnoreCase) -or - -not $resolvedOutput.StartsWith($artifactPrefix, [System.StringComparison]::OrdinalIgnoreCase) - ) { - throw "输出目录必须是项目 artifacts 目录下的新子目录。" - } - if (Test-Path -LiteralPath $resolvedOutput) { - throw ("输出目录已存在，拒绝覆盖：{0}" -f $resolvedOutput) + $demoArguments = @(Join-Path $repoRoot "scripts\portfolio_demo.py") + if (-not [string]::IsNullOrWhiteSpace($OutputDirectory)) { + $demoArguments += @("--output-dir", $OutputDirectory) } - Write-Host "ResearchOps Agent 作品集离线演示" -ForegroundColor Yellow - Write-Host ("项目：{0}" -f $repoRoot) - Write-Host ("输出：{0}" -f $resolvedOutput) - Write-Host "模式：完全离线；固定 50 题组件与控制面评测" - Write-Host "说明：本脚本不会调用 phase6-run-online，也不会检查或展示任何凭据。" - - $previousLocation = (Get-Location).Path - $hadPythonPath = Test-Path Env:PYTHONPATH - $previousPythonPath = if ($hadPythonPath) { $env:PYTHONPATH } else { $null } - $numericalEnvironment = [ordered]@{ - OPENBLAS_CORETYPE = "NEHALEM" - OPENBLAS_NUM_THREADS = "1" - OMP_NUM_THREADS = "1" - MKL_NUM_THREADS = "1" - NUMEXPR_NUM_THREADS = "1" - NPY_DISABLE_CPU_FEATURES = "X86_V3,X86_V4" - } - $previousNumericalEnvironment = @{} + $nativeErrorPreference = $ErrorActionPreference try { - Set-Location -LiteralPath $repoRoot - $env:PYTHONPATH = $sourceRoot - foreach ($name in $numericalEnvironment.Keys) { - $previousNumericalEnvironment[$name] = [Environment]::GetEnvironmentVariable( - $name, - [EnvironmentVariableTarget]::Process - ) - } - foreach ($name in $numericalEnvironment.Keys) { - [Environment]::SetEnvironmentVariable( - $name, - $numericalEnvironment[$name], - [EnvironmentVariableTarget]::Process - ) - } - - Invoke-PythonStep ` - -Title "1/4 校验 Python 环境" ` - -Arguments @( - "-c", - "import sys; print('Python ' + sys.version.split()[0]); raise SystemExit(0 if sys.version_info >= (3, 11) else 2)" - ) ` - -ShowOutput - - Write-Section "固定并验证 Nehalem/x86-v2 数值基线" - $previousOpenBlasVerbose = [Environment]::GetEnvironmentVariable( - "OPENBLAS_VERBOSE", - [EnvironmentVariableTarget]::Process - ) - $previousErrorActionPreference = $ErrorActionPreference - try { - $env:OPENBLAS_VERBOSE = "2" - # OpenBLAS reports its selected kernel on stderr even when the probe - # succeeds. Capture that diagnostic without turning it into a - # terminating PowerShell error. - $ErrorActionPreference = "Continue" - $probeOutput = @( - & $script:PythonExecutable -c "import numpy as np; np.linalg.svd(np.eye(4))" 2>&1 - ) - $probeExitCode = $LASTEXITCODE - $probeText = ($probeOutput | ForEach-Object { [string]$_ }) -join [Environment]::NewLine - } - finally { - [Environment]::SetEnvironmentVariable( - "OPENBLAS_VERBOSE", - $previousOpenBlasVerbose, - [EnvironmentVariableTarget]::Process - ) - $ErrorActionPreference = $previousErrorActionPreference - } - if ($probeExitCode -ne 0 -or $probeText -notmatch '(?im)^Core:\s*Nehalem\s*$') { - throw "固定的 Nehalem OpenBLAS kernel 未激活，拒绝生成不可比较的 evidence ID。" - } - Write-Host "OpenBLAS core：Nehalem" -ForegroundColor Green - - Invoke-PythonStep ` - -Title "验证 canonical ANCOVA evidence identity" ` - -Arguments @( - "-c", - "import json,pandas as pd; from pathlib import Path; from researchops.analysis_tools import run_ancova; from researchops.contracts import ResearchDesign; from researchops.data_quality import profile_csv; p=Path('data/synthetic_trial.csv'); f=pd.read_csv(p,encoding='utf-8-sig',low_memory=False); d=ResearchDesign.from_dict(json.loads(Path('data/synthetic_trial_design.json').read_text(encoding='utf-8'))); got=run_ancova(f,profile_csv(p),d).evidence_id; print(got); raise SystemExit(0 if got=='E-36034128278C' else 3)" - ) ` - -ShowOutput - - Invoke-PythonStep ` - -Title "2/4 校验离线评测语料" ` - -Arguments @("-m", "researchops.cli", "eval-validate") ` - -ShowOutput - - Invoke-PythonStep ` - -Title "附加校验：Phase 6 行为语料与 split（仍不联网）" ` - -Arguments @("-m", "researchops.cli", "phase6-validate") ` - -ShowOutput - - Invoke-PythonStep ` - -Title "3/4 运行固定 50 题离线评测" ` - -Arguments @( - "-m", "researchops.cli", "eval-run", - "--tasks", (Join-Path $repoRoot "evals\tasks.jsonl"), - "--output-dir", $resolvedOutput - ) - - Invoke-PythonStep ` - -Title "4/4 独立验证哈希、审计链与脱敏" ` - -Arguments @( - (Join-Path $repoRoot "scripts\verify_phase5_artifacts.py"), - $resolvedOutput - ) ` - -ShowOutput + # Windows PowerShell 5 may promote successful native stderr diagnostics + # to ErrorRecord. Stream both channels and trust the process exit code. + $ErrorActionPreference = "Continue" + & $pythonExecutable @demoArguments 2>&1 | + ForEach-Object { Write-Host ([string]$_) } + $exitCode = $LASTEXITCODE } finally { - Set-Location -LiteralPath $previousLocation - if ($hadPythonPath) { - $env:PYTHONPATH = $previousPythonPath - } - else { - Remove-Item Env:PYTHONPATH -ErrorAction SilentlyContinue - } - foreach ($name in $numericalEnvironment.Keys) { - [Environment]::SetEnvironmentVariable( - $name, - $previousNumericalEnvironment[$name], - [EnvironmentVariableTarget]::Process - ) - } + $ErrorActionPreference = $nativeErrorPreference + } + if ($exitCode -ne 0) { + throw "Shared portfolio demo failed with exit code $exitCode." } - - $reportPath = Join-Path $resolvedOutput "eval_report.json" - $summaryPath = Join-Path $resolvedOutput "eval_summary.md" - $manifestPath = Join-Path $resolvedOutput "eval_manifest.json" - $resultsPath = Join-Path $resolvedOutput "eval_results.jsonl" - $auditPath = Join-Path $resolvedOutput "eval_audit.sqlite3" - $auditIndexPath = Join-Path $resolvedOutput "eval_audit_index.json" - $report = Get-Content -LiteralPath $reportPath -Raw -Encoding UTF8 | ConvertFrom-Json - - Write-Section "演示结果" - [pscustomobject]@{ - Tasks = $report.task_count - Passed = $report.passed_count - Failed = $report.failed_count - SuccessRate = "{0:P2}" -f [double]$report.success_rate - UnexpectedToolErrorRate = "{0:P2}" -f [double]$report.unexpected_tool_error_rate - SafetyViolationRate = "{0:P2}" -f [double]$report.safety_violation_rate - EvidenceCitationAccuracy = "{0:P2}" -f [double]$report.evidence_citation_accuracy - LatencyP50Ms = "{0:N2}" -f [double]$report.p50_latency_ms - LatencyP95Ms = "{0:N2}" -f [double]$report.p95_latency_ms - CostStatus = $report.cost_status - } | Format-List | Out-Host - - Write-Host "关键产物：" -ForegroundColor Cyan - Write-Host ("- 摘要报告：{0}" -f $summaryPath) - Write-Host ("- 指标 JSON：{0}" -f $reportPath) - Write-Host ("- 可复现清单：{0}" -f $manifestPath) - Write-Host ("- 逐题结果：{0}" -f $resultsPath) - Write-Host ("- 审计数据库：{0}" -f $auditPath) - Write-Host ("- 审计链索引：{0}" -f $auditIndexPath) - Write-Host "" - Write-Host "离线演示完成。" -ForegroundColor Green } catch { - Write-Error ("作品集演示失败：{0}" -f $_.Exception.Message) -ErrorAction Continue + Write-Error ("Portfolio demo failed: {0}" -f $_.Exception.Message) ` + -ErrorAction Continue exit 1 } diff --git a/scripts/portfolio_demo.py b/scripts/portfolio_demo.py new file mode 100644 index 0000000..4ea4723 --- /dev/null +++ b/scripts/portfolio_demo.py @@ -0,0 +1,348 @@ +from __future__ import annotations + +import argparse +import json +import os +import platform +import re +import subprocess +import sys +import uuid +from datetime import datetime, timezone +from pathlib import Path +from typing import Mapping, Sequence + + +CANONICAL_EVIDENCE_IDS = { + "Windows": "E-36034128278C", + "Linux": "E-14EBFFCA843E", +} +TASK_CORPUS_NAMES = { + "Windows": "tasks.jsonl", + "Linux": "tasks.linux-x86_64.jsonl", +} +QUALITY_PROFILES = { + "Windows": "phase5-ci-v1", + "Linux": "phase5-linux-x86-ci-v1", +} +SUPPORTED_SYSTEMS = frozenset(("Windows", "Linux")) +SUPPORTED_MACHINES = frozenset(("amd64", "x86_64")) +NUMERICAL_ENVIRONMENT = { + "OPENBLAS_CORETYPE": "NEHALEM", + "OPENBLAS_NUM_THREADS": "1", + "OMP_NUM_THREADS": "1", + "MKL_NUM_THREADS": "1", + "NUMEXPR_NUM_THREADS": "1", + "NPY_DISABLE_CPU_FEATURES": "X86_V3,X86_V4", +} +PROVIDER_CREDENTIAL_VARIABLES = ( + "OPENAI_API_KEY", + "DEEPSEEK_API_KEY", + "ANTHROPIC_API_KEY", + "ANTHROPIC_AUTH_TOKEN", + "MOONSHOT_API_KEY", +) + + +class DemoError(RuntimeError): + pass + + +def _section(title: str) -> None: + print(f"\n=== {title} ===", flush=True) + + +def _require_supported_platform() -> str: + system = platform.system() + machine = platform.machine().lower() + python_bits = 64 if sys.maxsize > 2**32 else 32 + if ( + system not in SUPPORTED_SYSTEMS + or machine not in SUPPORTED_MACHINES + or python_bits != 64 + ): + raise DemoError( + "the strict frozen-evidence demo supports only Windows x86-64 " + "and Linux x86-64; macOS and ARM do not yet have a compatible " + "numerical evidence baseline " + f"(detected: system={system}, machine={machine}, python={python_bits}-bit)" + ) + return system + + +def _run_python_step( + *, + title: str, + arguments: Sequence[str], + repo_root: Path, + environment: Mapping[str, str], + show_output: bool = False, +) -> str: + _section(title) + completed = subprocess.run( + [sys.executable, *arguments], + cwd=repo_root, + env=dict(environment), + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + encoding="utf-8", + errors="replace", + check=False, + ) + output = completed.stdout.rstrip() + if completed.returncode != 0: + if output: + print(output) + raise DemoError( + f"step [{title}] failed with Python exit code {completed.returncode}; " + "the output directory will not be reused or overwritten" + ) + if show_output and output: + print(output) + print("passed") + return output + + +def _resolve_output(repo_root: Path, value: str | None) -> Path: + artifacts_root = (repo_root / "artifacts").resolve() + if value is None: + suffix = uuid.uuid4().hex[:8] + timestamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S%f")[:-3] + output = artifacts_root / f"portfolio_demo_{timestamp}_{suffix}" + else: + candidate = Path(value) + output = ( + candidate.resolve() + if candidate.is_absolute() + else (repo_root / candidate).resolve() + ) + if output == artifacts_root or not output.is_relative_to(artifacts_root): + raise DemoError("output must be a new child directory under artifacts") + if output.exists(): + raise DemoError(f"output already exists; refusing to overwrite: {output}") + return output + + +def _require_files(repo_root: Path) -> None: + required = ( + repo_root / "pyproject.toml", + repo_root / "data/synthetic_trial.csv", + repo_root / "data/synthetic_trial_design.json", + repo_root / "evals/tasks.jsonl", + repo_root / "evals/tasks.linux-x86_64.jsonl", + repo_root / "evals/phase6_agent_tasks.jsonl", + repo_root / "evals/phase6_splits.json", + repo_root / "scripts/verify_phase5_artifacts.py", + ) + missing = [str(path) for path in required if not path.is_file()] + if missing: + raise DemoError("required demo files are missing: " + ", ".join(missing)) + if sys.version_info < (3, 11): + raise DemoError("Python 3.11 or newer is required") + + +def _offline_environment(repo_root: Path) -> dict[str, str]: + environment = dict(os.environ) + environment.update(NUMERICAL_ENVIRONMENT) + environment["PYTHONPATH"] = str(repo_root / "src") + for name in PROVIDER_CREDENTIAL_VARIABLES: + environment.pop(name, None) + return environment + + +def _verify_numerical_identity( + repo_root: Path, environment: Mapping[str, str], *, system: str +) -> None: + expected_evidence_id = CANONICAL_EVIDENCE_IDS[system] + probe_environment = dict(environment) + probe_environment["OPENBLAS_VERBOSE"] = "2" + output = _run_python_step( + title="Pin and verify Nehalem/x86-v2 numerical baseline", + arguments=( + "-c", + "import numpy as np; np.linalg.svd(np.eye(4))", + ), + repo_root=repo_root, + environment=probe_environment, + show_output=True, + ) + if re.search(r"(?im)^Core:\s*Nehalem\s*$", output) is None: + raise DemoError( + "the pinned Nehalem OpenBLAS kernel was not activated; " + "refusing to generate incomparable evidence IDs" + ) + + identity_program = ( + "import json,pandas as pd; " + "from pathlib import Path; " + "from researchops.analysis_tools import run_ancova; " + "from researchops.contracts import ResearchDesign; " + "from researchops.data_quality import profile_csv; " + "p=Path('data/synthetic_trial.csv'); " + "f=pd.read_csv(p,encoding='utf-8-sig',low_memory=False); " + "d=ResearchDesign.from_dict(json.loads(Path('data/synthetic_trial_design.json').read_text(encoding='utf-8'))); " + "got=run_ancova(f,profile_csv(p),d).evidence_id; " + "print(got); " + f"raise SystemExit(0 if got=='{expected_evidence_id}' else 3)" + ) + _run_python_step( + title=f"Verify canonical {system} x86-64 ANCOVA evidence identity", + arguments=("-c", identity_program), + repo_root=repo_root, + environment=environment, + show_output=True, + ) + + +def _print_report(output: Path) -> None: + report_path = output / "eval_report.json" + try: + report = json.loads(report_path.read_text(encoding="utf-8")) + except (OSError, ValueError, json.JSONDecodeError) as exc: + raise DemoError("the generated evaluation report is unreadable") from exc + + _section("Demo result") + fields = ( + ("Tasks", report["task_count"]), + ("Passed", report["passed_count"]), + ("Failed", report["failed_count"]), + ("Success rate", f"{float(report['success_rate']):.2%}"), + ( + "Unexpected tool error rate", + f"{float(report['unexpected_tool_error_rate']):.2%}", + ), + ("Safety violation rate", f"{float(report['safety_violation_rate']):.2%}"), + ( + "Evidence citation accuracy", + f"{float(report['evidence_citation_accuracy']):.2%}", + ), + ("Latency P50 ms", f"{float(report['p50_latency_ms']):.2f}"), + ("Latency P95 ms", f"{float(report['p95_latency_ms']):.2f}"), + ("Cost status", report["cost_status"]), + ) + width = max(len(label) for label, _ in fields) + for label, value in fields: + print(f"{label:<{width}} : {value}") + + print("\nKey artifacts:") + for name, label in ( + ("eval_summary.md", "summary"), + ("eval_report.json", "metrics"), + ("eval_manifest.json", "reproducibility manifest"), + ("eval_results.jsonl", "per-task results"), + ("eval_audit.sqlite3", "audit database"), + ("eval_audit_index.json", "audit-chain index"), + ): + path = output / name + if not path.is_file(): + raise DemoError(f"expected artifact is missing: {path}") + print(f"- {label}: {path}") + + +def run_demo(output_directory: str | None) -> Path: + repo_root = Path(__file__).resolve().parents[1] + system = _require_supported_platform() + _require_files(repo_root) + task_corpus = repo_root / "evals" / TASK_CORPUS_NAMES[system] + quality_profile = QUALITY_PROFILES[system] + output = _resolve_output(repo_root, output_directory) + environment = _offline_environment(repo_root) + + print("ResearchOps Agent offline portfolio demo") + print(f"Repository: {repo_root}") + print(f"Output: {output}") + print(f"Corpus: {task_corpus.name}") + print(f"Profile: {quality_profile}") + print("Mode: offline deterministic 50-task control-plane evaluation") + print("Credentials: removed from child-process environment") + + _run_python_step( + title="1/4 Validate Python environment", + arguments=( + "-c", + "import sys; print('Python ' + sys.version.split()[0]); " + "raise SystemExit(0 if sys.version_info >= (3, 11) else 2)", + ), + repo_root=repo_root, + environment=environment, + show_output=True, + ) + _verify_numerical_identity(repo_root, environment, system=system) + _run_python_step( + title="2/4 Validate offline evaluation corpus", + arguments=( + "-m", + "researchops.cli", + "eval-validate", + "--tasks", + str(task_corpus), + ), + repo_root=repo_root, + environment=environment, + show_output=True, + ) + _run_python_step( + title="Additional check: Phase 6 behavior corpus and split", + arguments=("-m", "researchops.cli", "phase6-validate"), + repo_root=repo_root, + environment=environment, + show_output=True, + ) + _run_python_step( + title="3/4 Run frozen 50-task offline evaluation", + arguments=( + "-m", + "researchops.cli", + "eval-run", + "--tasks", + str(task_corpus), + "--output-dir", + str(output), + ), + repo_root=repo_root, + environment=environment, + ) + _run_python_step( + title="4/4 Verify hashes, audit chains and redaction", + arguments=( + str(repo_root / "scripts/verify_phase5_artifacts.py"), + str(output), + "--quality-profile", + quality_profile, + ), + repo_root=repo_root, + environment=environment, + show_output=True, + ) + _print_report(output) + print("\nOffline demo completed.") + return output + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description=( + "Run the fully offline ResearchOps Agent portfolio demo without " + "loading Provider credentials." + ) + ) + parser.add_argument( + "--output-dir", + help="new output directory under artifacts (default: unique generated path)", + ) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + arguments = build_parser().parse_args(argv) + try: + run_demo(arguments.output_dir) + except (DemoError, OSError, KeyError, TypeError, ValueError) as exc: + print(f"portfolio demo failed: {exc}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/portfolio_demo.sh b/scripts/portfolio_demo.sh new file mode 100755 index 0000000..6b7997c --- /dev/null +++ b/scripts/portfolio_demo.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_path="${BASH_SOURCE[0]}" +if [[ "${script_path}" != /* ]]; then + script_path="${PWD}/${script_path}" +fi +script_dir="$(cd -- "${script_path%/*}" && pwd -P)" +repo_root="$(cd -- "${script_dir}/.." && pwd -P)" +python_executable="${PYTHON_PATH:-${repo_root}/.venv/bin/python}" + +if [[ ! -x "${python_executable}" ]]; then + printf 'Python executable not found: %s\n' "${python_executable}" >&2 + exit 1 +fi + +exec "${python_executable}" "${repo_root}/scripts/portfolio_demo.py" "$@" diff --git a/scripts/verify_phase5_artifacts.py b/scripts/verify_phase5_artifacts.py index aed36cf..081bfcc 100644 --- a/scripts/verify_phase5_artifacts.py +++ b/scripts/verify_phase5_artifacts.py @@ -17,22 +17,31 @@ TEXT_SUFFIXES = {".json", ".jsonl", ".md"} +ALLOWED_TASK_CORPORA = frozenset( + {"tasks.jsonl", "tasks.linux-x86_64.jsonl"} +) FORBIDDEN_CANARIES = { "row_id_P0001": "p0001", "api_key_prefix": "sk-canary", "authorization_header": "authorization: bearer", "traceback": "traceback (most recent call last)", } +_EXACT_50_TASK_REQUIREMENTS: tuple[tuple[str, int | float], ...] = ( + ("task_count", 50), + ("passed_count", 50), + ("failed_count", 0), + ("success_rate", 1.0), + ("evidence_citations_required", 21), + ("evidence_citations_matched", 21), + ("evidence_citation_accuracy", 1.0), +) QUALITY_PROFILE_REQUIREMENTS: dict[str, tuple[tuple[str, int | float], ...]] = { - "phase5-ci-v1": ( - ("task_count", 50), - ("passed_count", 50), - ("failed_count", 0), - ("success_rate", 1.0), - ("evidence_citations_required", 21), - ("evidence_citations_matched", 21), - ("evidence_citation_accuracy", 1.0), - ) + "phase5-ci-v1": _EXACT_50_TASK_REQUIREMENTS, + "phase5-linux-x86-ci-v1": _EXACT_50_TASK_REQUIREMENTS, +} +QUALITY_PROFILE_TASK_CORPORA = { + "phase5-ci-v1": "tasks.jsonl", + "phase5-linux-x86-ci-v1": "tasks.linux-x86_64.jsonl", } @@ -83,10 +92,19 @@ def main() -> int: if not path.is_file() or _sha256(path) != metadata["sha256"]: hash_mismatches.append(name) provenance_mismatches: list[str] = [] - if _sha256(project_root / "evals" / "tasks.jsonl") != manifest["task_corpus"][ - "sha256" - ]: + task_corpus_path = _manifest_task_corpus_path(project_root, manifest) + if ( + task_corpus_path is None + or not task_corpus_path.is_file() + or _sha256(task_corpus_path) != manifest["task_corpus"]["sha256"] + ): provenance_mismatches.append("task_corpus") + elif ( + args.quality_profile is not None + and task_corpus_path.name + != QUALITY_PROFILE_TASK_CORPORA[args.quality_profile] + ): + provenance_mismatches.append("quality_profile_task_corpus") if _sha256(project_root / "data" / "synthetic_trial.csv") != manifest[ "dataset_sha256" ]: @@ -155,6 +173,25 @@ def main() -> int: return 0 if valid else 1 +def _manifest_task_corpus_path( + project_root: Path, manifest: Mapping[str, Any] +) -> Path | None: + task_corpus = manifest.get("task_corpus") + if not isinstance(task_corpus, Mapping): + return None + file_name = task_corpus.get("file_name") + if not isinstance(file_name, str) or file_name not in ALLOWED_TASK_CORPORA: + return None + relative = Path(file_name) + if relative.name != file_name or relative.suffix != ".jsonl": + return None + evals_root = (project_root / "evals").resolve() + candidate = (evals_root / relative).resolve() + if not candidate.is_relative_to(evals_root): + return None + return candidate + + def _build_quality_gate( profile: str | None, report: Mapping[str, Any] | None, diff --git a/tests/test_phase5_artifact_verifier.py b/tests/test_phase5_artifact_verifier.py index b80a134..ff7145f 100644 --- a/tests/test_phase5_artifact_verifier.py +++ b/tests/test_phase5_artifact_verifier.py @@ -30,6 +30,9 @@ class Phase5ArtifactQualityGateTests(unittest.TestCase): def test_phase5_corpus_is_bound_to_canonical_lf_provenance(self) -> None: dataset = (PROJECT_ROOT / "data" / "synthetic_trial.csv").read_bytes() corpus = (PROJECT_ROOT / "evals" / "tasks.jsonl").read_bytes() + linux_corpus = ( + PROJECT_ROOT / "evals" / "tasks.linux-x86_64.jsonl" + ).read_bytes() self.assertNotIn(b"\r\n", dataset) self.assertEqual( @@ -49,13 +52,42 @@ def test_phase5_corpus_is_bound_to_canonical_lf_provenance(self) -> None: self.assertNotIn(b"E-7C87BB6C88EB", corpus) self.assertNotIn(b"E-B93CD9DC7751", corpus) self.assertNotIn(b"CH-F675F0E546C6", corpus) + self.assertNotIn(b"\r\n", linux_corpus) + self.assertEqual( + hashlib.sha256(linux_corpus).hexdigest(), + "68ea85b79a43d8bb32834ae5d990aa2135bbfe5775c50ee7d7b2f239f4b68b23", + ) + translated = corpus.replace(b"E-36034128278C", b"E-14EBFFCA843E") + translated = translated.replace(b"E-E5D03B8E6EB8", b"E-5FBD2DA79692") + translated = translated.replace(b"CH-6D27DA2CB989", b"CH-BF5193D84458") + self.assertEqual(linux_corpus, translated) def test_phase5_ci_profile_accepts_exact_release_thresholds(self) -> None: - gate = VERIFIER._build_quality_gate("phase5-ci-v1", _perfect_report()) + for profile in ("phase5-ci-v1", "phase5-linux-x86-ci-v1"): + with self.subTest(profile=profile): + gate = VERIFIER._build_quality_gate(profile, _perfect_report()) - self.assertEqual(gate["status"], "valid") - self.assertIsNone(gate["error_code"]) - self.assertEqual(gate["mismatches"], []) + self.assertEqual(gate["status"], "valid") + self.assertIsNone(gate["error_code"]) + self.assertEqual(gate["mismatches"], []) + + def test_verifier_resolves_only_manifest_bound_eval_corpora(self) -> None: + manifest = {"task_corpus": {"file_name": "tasks.linux-x86_64.jsonl"}} + self.assertEqual( + VERIFIER._manifest_task_corpus_path(PROJECT_ROOT, manifest), + (PROJECT_ROOT / "evals" / "tasks.linux-x86_64.jsonl").resolve(), + ) + for file_name in ( + "../tasks.jsonl", + "tasks.json", + "unfrozen-tasks.jsonl", + "", + ): + with self.subTest(file_name=file_name): + manifest["task_corpus"]["file_name"] = file_name + self.assertIsNone( + VERIFIER._manifest_task_corpus_path(PROJECT_ROOT, manifest) + ) def test_phase5_ci_profile_rejects_main_regression_with_stable_order(self) -> None: report = _perfect_report() @@ -155,21 +187,93 @@ def test_workflow_preserves_both_native_exit_codes_and_enforces_profile(self) -> self.assertIn("phase5_offline_quality_gate_failed", workflow) self.assertIn("exit 1", workflow) - def test_portfolio_demo_uses_the_same_canonical_numerical_baseline(self) -> None: - script_path = PROJECT_ROOT / "scripts" / "portfolio_demo.ps1" - self.assertTrue(script_path.read_bytes().startswith(b"\xef\xbb\xbf")) - script = script_path.read_text(encoding="utf-8-sig") - - self.assertIn('OPENBLAS_CORETYPE = "NEHALEM"', script) - self.assertIn('OPENBLAS_NUM_THREADS = "1"', script) - self.assertIn('OMP_NUM_THREADS = "1"', script) - self.assertIn('MKL_NUM_THREADS = "1"', script) - self.assertIn('NUMEXPR_NUM_THREADS = "1"', script) - self.assertIn('NPY_DISABLE_CPU_FEATURES = "X86_V3,X86_V4"', script) - self.assertIn("Core:\\s*Nehalem", script) - self.assertIn("E-36034128278C", script) - self.assertIn("$stepErrorActionPreference = $ErrorActionPreference", script) - self.assertIn("$ErrorActionPreference = $stepErrorActionPreference", script) + def test_portfolio_demo_uses_strict_platform_numerical_baselines(self) -> None: + core = (PROJECT_ROOT / "scripts" / "portfolio_demo.py").read_text( + encoding="utf-8" + ) + powershell = (PROJECT_ROOT / "scripts" / "portfolio_demo.ps1").read_text( + encoding="utf-8" + ) + shell = (PROJECT_ROOT / "scripts" / "portfolio_demo.sh").read_text( + encoding="utf-8" + ) + + for name, value in ( + ("OPENBLAS_CORETYPE", "NEHALEM"), + ("OPENBLAS_NUM_THREADS", "1"), + ("OMP_NUM_THREADS", "1"), + ("MKL_NUM_THREADS", "1"), + ("NUMEXPR_NUM_THREADS", "1"), + ("NPY_DISABLE_CPU_FEATURES", "X86_V3,X86_V4"), + ): + self.assertIn(f'"{name}": "{value}"', core) + self.assertIn("Core:\\s*Nehalem", core) + self.assertIn("E-36034128278C", core) + self.assertIn("E-14EBFFCA843E", core) + self.assertIn('"phase5-ci-v1"', core) + self.assertIn('"phase5-linux-x86-ci-v1"', core) + self.assertIn("PROVIDER_CREDENTIAL_VARIABLES", core) + self.assertIn("output.is_relative_to(artifacts_root)", core) + + self.assertIn("scripts\\portfolio_demo.py", powershell) + self.assertIn("ForEach-Object { Write-Host", powershell) + self.assertNotIn("$capturedOutput = @(", powershell) + self.assertIn("$LASTEXITCODE", powershell) + self.assertTrue(shell.startswith("#!/usr/bin/env bash\nset -euo pipefail\n")) + self.assertIn("${PYTHON_PATH:-${repo_root}/.venv/bin/python}", shell) + self.assertIn('scripts/portfolio_demo.py" "$@"', shell) + + def test_supported_demo_platforms_are_documented_and_checked_in_ci(self) -> None: + readme = (PROJECT_ROOT / "README.md").read_text(encoding="utf-8") + english_readme = (PROJECT_ROOT / "README.en.md").read_text(encoding="utf-8") + windows_requirements = (PROJECT_ROOT / "requirements.lock").read_text( + encoding="utf-8" + ) + linux_requirements = (PROJECT_ROOT / "requirements.linux.lock").read_text( + encoding="utf-8" + ) + workflow = (PROJECT_ROOT / ".github/workflows/ci.yml").read_text( + encoding="utf-8" + ) + + self.assertIn("Python 3.11+", readme) + self.assertIn("Windows x86-64", readme) + self.assertIn("Linux x86-64", readme) + self.assertIn("macOS 与 ARM", readme) + self.assertIn("requirements.linux.lock", readme) + self.assertIn("Python 3.11+", english_readme) + self.assertIn("Windows x86-64", english_readme) + self.assertIn("Linux x86-64", english_readme) + self.assertIn("macOS and ARM", english_readme) + self.assertIn("requirements.linux.lock", english_readme) + self.assertIn("scripts\\portfolio_demo.ps1", readme) + self.assertIn("scripts/portfolio_demo.sh", readme) + windows_requirement_lines = windows_requirements.splitlines() + linux_requirement_lines = linux_requirements.splitlines() + self.assertEqual(windows_requirement_lines.count("pywin32==312"), 1) + self.assertIn("pywin32==312", windows_requirement_lines) + self.assertNotIn("pywin32==312", linux_requirement_lines) + self.assertEqual( + [ + requirement + for requirement in windows_requirement_lines + if requirement != "pywin32==312" + ], + linux_requirement_lines, + ) + self.assertIn("linux-x86-demo:", workflow) + self.assertIn("runs-on: ubuntu-latest", workflow) + self.assertIn("python -m pip install -r requirements.linux.lock", workflow) + self.assertIn("python -m pip check", workflow) + self.assertIn('test "$(uname -m)" = "x86_64"', workflow) + self.assertIn("bash -n scripts/portfolio_demo.sh", workflow) + self.assertIn( + "bash scripts/portfolio_demo.sh --output-dir artifacts/ci_linux_x86_demo", + workflow, + ) + self.assertIn("test ! -e handoff.md", workflow) + self.assertIn("test -f docs/internal/handoff.md", workflow) + self.assertNotIn("Validate macOS and Linux", workflow) if __name__ == "__main__": diff --git a/tests/test_portfolio_demo.py b/tests/test_portfolio_demo.py new file mode 100644 index 0000000..83116fd --- /dev/null +++ b/tests/test_portfolio_demo.py @@ -0,0 +1,140 @@ +from __future__ import annotations + +import importlib.util +import os +import subprocess +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + + +PROJECT_ROOT = Path(__file__).resolve().parents[1] +SCRIPT_PATH = PROJECT_ROOT / "scripts/portfolio_demo.py" +SPEC = importlib.util.spec_from_file_location("portfolio_demo", SCRIPT_PATH) +if SPEC is None or SPEC.loader is None: # pragma: no cover - import guard + raise RuntimeError("unable to load portfolio demo") +DEMO = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(DEMO) + + +class PortfolioDemoTests(unittest.TestCase): + @unittest.skipUnless(os.name == "nt", "Windows PowerShell wrapper test") + def test_powershell_wrapper_streams_output_and_propagates_failure(self) -> None: + wrapper = PROJECT_ROOT / "scripts/portfolio_demo.ps1" + with tempfile.TemporaryDirectory() as directory: + fake_python = Path(directory) / "fake-python.cmd" + fake_python.write_text( + "@echo off\r\n" + "echo stream-start\r\n" + "ping 127.0.0.1 -n 3 >nul\r\n" + "echo stream-end\r\n" + "exit /b 7\r\n", + encoding="ascii", + ) + process = subprocess.Popen( + ( + "powershell", + "-NoProfile", + "-ExecutionPolicy", + "Bypass", + "-File", + str(wrapper), + "-PythonPath", + str(fake_python), + ), + cwd=PROJECT_ROOT, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + encoding="utf-8", + errors="replace", + ) + assert process.stdout is not None + first_line = process.stdout.readline().strip() + self.assertEqual(first_line, "stream-start") + self.assertIsNone(process.poll(), "wrapper buffered the child output") + remaining, _ = process.communicate(timeout=10) + + self.assertEqual(process.returncode, 1) + self.assertIn("stream-end", remaining) + self.assertIn("exit code 7", remaining) + + def test_strict_demo_accepts_only_windows_and_linux_x86_64(self) -> None: + for system, machine in (("Windows", "AMD64"), ("Linux", "x86_64")): + with self.subTest(system=system, machine=machine): + with ( + patch.object(DEMO.platform, "system", return_value=system), + patch.object(DEMO.platform, "machine", return_value=machine), + patch.object(DEMO.sys, "maxsize", 2**63 - 1), + ): + self.assertEqual(DEMO._require_supported_platform(), system) + + unsupported = ( + ("Darwin", "x86_64", 2**63 - 1), + ("Darwin", "arm64", 2**63 - 1), + ("Windows", "ARM64", 2**63 - 1), + ("Linux", "aarch64", 2**63 - 1), + ("Windows", "AMD64", 2**31 - 1), + ) + for system, machine, maxsize in unsupported: + with self.subTest(system=system, machine=machine, maxsize=maxsize): + with ( + patch.object(DEMO.platform, "system", return_value=system), + patch.object(DEMO.platform, "machine", return_value=machine), + patch.object(DEMO.sys, "maxsize", maxsize), + ): + with self.assertRaisesRegex( + DEMO.DemoError, "supports only Windows x86-64 and Linux x86-64" + ): + DEMO._require_supported_platform() + + def test_windows_and_linux_use_separate_canonical_evidence_ids(self) -> None: + self.assertEqual( + DEMO.CANONICAL_EVIDENCE_IDS, + { + "Windows": "E-36034128278C", + "Linux": "E-14EBFFCA843E", + }, + ) + + def test_offline_environment_pins_numerics_and_removes_provider_credentials(self) -> None: + canaries = { + name: f"secret-{index}" + for index, name in enumerate(DEMO.PROVIDER_CREDENTIAL_VARIABLES) + } + with patch.dict(os.environ, canaries, clear=False): + environment = DEMO._offline_environment(PROJECT_ROOT) + + for name in DEMO.PROVIDER_CREDENTIAL_VARIABLES: + self.assertNotIn(name, environment) + self.assertEqual(environment["OPENBLAS_CORETYPE"], "NEHALEM") + self.assertEqual(environment["OPENBLAS_NUM_THREADS"], "1") + self.assertEqual(environment["PYTHONPATH"], str(PROJECT_ROOT / "src")) + + def test_output_must_be_a_new_child_of_artifacts(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "artifacts").mkdir() + output = DEMO._resolve_output(root, "artifacts/new-demo") + self.assertEqual(output, (root / "artifacts/new-demo").resolve()) + + with self.assertRaises(DEMO.DemoError): + DEMO._resolve_output(root, "artifacts") + with self.assertRaises(DEMO.DemoError): + DEMO._resolve_output(root, "outside-demo") + + output.mkdir() + with self.assertRaises(DEMO.DemoError): + DEMO._resolve_output(root, "artifacts/new-demo") + + def test_cli_help_requires_no_repository_or_credentials(self) -> None: + parser = DEMO.build_parser() + parsed = parser.parse_args([]) + self.assertIsNone(parsed.output_dir) + parsed = parser.parse_args(["--output-dir", "artifacts/demo"]) + self.assertEqual(parsed.output_dir, "artifacts/demo") + + +if __name__ == "__main__": + unittest.main()