From a4632274dd93d1c9755f42766be53f6dc8f362e0 Mon Sep 17 00:00:00 2001 From: gronbow <59046437+gronbow@users.noreply.github.com> Date: Sun, 2 Aug 2026 22:45:49 +0800 Subject: [PATCH] Add GeoSkills v0.4 workflow core --- .github/workflows/tests.yml | 5 + .gitignore | 1 + AGENTS.md | 5 + CHANGELOG.md | 17 + README.md | 78 +- requirements-dev.txt | 1 - skills/geoskills/SKILL.md | 188 +- skills/geoskills/agents/openai.yaml | 4 +- .../classification/tas-lemaitre-2002.json | 2 +- .../examples/geoskills_major_workflow.yaml | 78 + .../examples/geoskills_ree_workflow.yaml | 74 + .../examples/geoskills_spider_workflow.yaml | 102 ++ .../examples/synthetic_major_element_data.csv | 20 +- .../references/workflow-and-recipe.md | 264 +++ skills/geoskills/scripts/check_environment.py | 72 +- skills/geoskills/scripts/geoskills.py | 289 ++++ .../scripts/geoskills_core/__init__.py | 125 ++ .../scripts/geoskills_core/adapters.py | 666 ++++++++ .../scripts/geoskills_core/analytes.py | 327 ++++ .../scripts/geoskills_core/errors.py | 79 + .../scripts/geoskills_core/export.py | 437 +++++ skills/geoskills/scripts/geoskills_core/io.py | 402 +++++ .../scripts/geoskills_core/plotting.py | 192 +++ .../scripts/geoskills_core/recipe.py | 1267 ++++++++++++++ .../scripts/geoskills_core/registry.py | 323 ++++ .../scripts/geoskills_core/reports.py | 406 +++++ .../scripts/geoskills_core/validation.py | 299 ++++ .../scripts/geoskills_core/version.py | 6 + .../scripts/geoskills_core/workflow.py | 1513 +++++++++++++++++ .../geoskills/scripts/plot_geochem_common.py | 31 +- skills/geoskills/scripts/plot_harker.py | 74 +- skills/geoskills/scripts/plot_ree.py | 145 +- skills/geoskills/scripts/plot_spider.py | 8 + skills/geoskills/scripts/plot_tas.py | 31 +- skills/geoskills/scripts/requirements.txt | 2 + tests/test_core_io.py | 229 +++ tests/test_core_output.py | 388 +++++ tests/test_environment_check.py | 44 +- tests/test_plot_core_integration.py | 48 + tests/test_plot_harker.py | 30 + tests/test_plot_ree.py | 104 +- tests/test_recipe_registry.py | 555 ++++++ tests/test_v03_cli_contract.py | 53 + tests/test_workflow_cli.py | 514 ++++++ tests/test_workflow_science.py | 611 +++++++ tests/test_workflow_security.py | 517 ++++++ 46 files changed, 10398 insertions(+), 228 deletions(-) create mode 100644 skills/geoskills/examples/geoskills_major_workflow.yaml create mode 100644 skills/geoskills/examples/geoskills_ree_workflow.yaml create mode 100644 skills/geoskills/examples/geoskills_spider_workflow.yaml create mode 100644 skills/geoskills/references/workflow-and-recipe.md create mode 100644 skills/geoskills/scripts/geoskills.py create mode 100644 skills/geoskills/scripts/geoskills_core/__init__.py create mode 100644 skills/geoskills/scripts/geoskills_core/adapters.py create mode 100644 skills/geoskills/scripts/geoskills_core/analytes.py create mode 100644 skills/geoskills/scripts/geoskills_core/errors.py create mode 100644 skills/geoskills/scripts/geoskills_core/export.py create mode 100644 skills/geoskills/scripts/geoskills_core/io.py create mode 100644 skills/geoskills/scripts/geoskills_core/plotting.py create mode 100644 skills/geoskills/scripts/geoskills_core/recipe.py create mode 100644 skills/geoskills/scripts/geoskills_core/registry.py create mode 100644 skills/geoskills/scripts/geoskills_core/reports.py create mode 100644 skills/geoskills/scripts/geoskills_core/validation.py create mode 100644 skills/geoskills/scripts/geoskills_core/version.py create mode 100644 skills/geoskills/scripts/geoskills_core/workflow.py create mode 100644 tests/test_core_io.py create mode 100644 tests/test_core_output.py create mode 100644 tests/test_plot_core_integration.py create mode 100644 tests/test_recipe_registry.py create mode 100644 tests/test_v03_cli_contract.py create mode 100644 tests/test_workflow_cli.py create mode 100644 tests/test_workflow_science.py create mode 100644 tests/test_workflow_security.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 960e727..b2e8e04 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -32,5 +32,10 @@ jobs: - name: Compile scripts run: python -m compileall -q skills/geoskills/scripts tests + - name: Check unified local workflow + run: | + python skills/geoskills/scripts/geoskills.py version + python skills/geoskills/scripts/geoskills.py self-check --dev + - name: Run tests run: python -m pytest -q diff --git a/.gitignore b/.gitignore index a096fb9..31153bf 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ venv/ # Local/private data and generated output local_data/ outputs/ +skills/geoskills/examples/generated/ tmp/ # Editor and operating-system files diff --git a/AGENTS.md b/AGENTS.md index e7096e4..0beebff 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -10,5 +10,10 @@ - Add tests for numerical rules, invalid input, missing values, non-positive values, and exported files. - Do not claim the skill is research-ready until scientific fixtures and end-to-end tests pass. - Preserve the published v0.3.0 REE, spider, Harker, and volcanic TAS workflows. +- Keep the v0.4 unified entry point, recipe, plan, report, and fixed diagram registry backward compatible within their versioned schemas. +- Never set a recipe confirmation to `true` without an explicit human confirmation. +- Keep default plans and reports free of absolute paths, sample identifiers, and source values. +- Invalidate saved plans whenever the recipe, input content, tool/interface version, selected task, style, or scientific asset changes. +- Commit multi-task outputs only as a complete directory; a failed task must not leave a partial replacement. - Add post-v0.3.0 diagram families only through separate scientific review and validation; defer isotope and tectonic-discrimination diagrams until those workflows are explicitly reviewed. - Require explicit volcanic applicability and composition-basis declarations before TAS plotting; never silently classify intrusive or otherwise out-of-scope samples. diff --git a/CHANGELOG.md b/CHANGELOG.md index ebf999e..019bc9c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,23 @@ This file records the user-visible changes in each public GeoSkills release. +## v0.4.0 — Unreleased + +- Added one provider-neutral `geoskills.py` entry point with version, environment self-check, review-only plan, and guarded run commands. +- Added strict `geoskills.recipe/v1` YAML recipes for explicit local input, worksheet, layout, canonical column mapping, units, output profile, style preset, confirmations, and up to 32 tasks. +- Added a fixed registry for the four already reviewed diagram families; this release does not add a new scientific diagram. +- Added content-addressed plans covering the recipe, input file, selected tasks, styles, tool/interface versions, and versioned scientific assets. Changed inputs or references invalidate old plans. +- Added shareable JSON reports and Chinese QA summaries without absolute paths or source values, plus an opt-in local-reproducible profile for sensitive plotted-data CSV files. +- Added whole-directory atomic multi-task output: every task succeeds before the new result replaces an existing complete bundle. +- Added shared table I/O, analyte mapping, validation, plotting style, export, error, and report modules while retaining the v0.3 command-line contracts. +- Preserved the reviewed K2O→K, P2O5→P, and TiO2→Ti spider conversions without allowing other implicit conversions. +- Added a beginner-oriented Harker + TAS multi-task recipe and workflow documentation. +- Added strict unit/header identity checks, group-subset validation, finite X–Y pair handling, and protected overwrite/recovery rules. +- Added adaptive in-frame REE/spider legends and verified SVG/PDF plus 600 dpi PNG/TIFF publication artifacts. +- Added Chinese missing-input guidance and refined TAS field-label placement for clearer beginner and visual review. +- Removed exact source-data extrema from shareable reports and added report-level safeguards against future extrema leakage. +- Validated the local candidate with 196 automated tests plus synthetic and published-data regression runs; GitHub publication remains gated on user review. + ## v0.3.0 — 2026-07-30 The reviewed v0.2.0 spider-diagram milestone was not tagged separately; it is first published as part of this complete v0.3.0 release. diff --git a/README.md b/README.md index 81c3758..c311d6f 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ GeoSkills 是一个面向地质学与地球化学研究的本地 Agent Skill。 -当前正式版本为 [v0.3.0](https://github.com/gronbow/GeoSkills/releases/tag/v0.3.0),包含球粒陨石标准化 REE 配分图、微量元素蛛网图、Harker 变化图,以及带适用性保护的火山岩 TAS 分类图。 +当前公开正式版本为 [v0.3.0](https://github.com/gronbow/GeoSkills/releases/tag/v0.3.0)。本分支正在审核 v0.4.0 开发候选版:它不增加新图解,而是为已有的 REE、蛛网图、Harker 和 TAS 增加统一配方、运行前计划、隐私报告和多任务原子输出。 ## 当前可以做什么 @@ -16,13 +16,17 @@ GeoSkills 是一个面向地质学与地球化学研究的本地 Agent Skill。 - 以 SiO2 或用户指定变量为横轴,一次生成一幅或多幅 Harker 变化图; - 使用 SiO2 与 Na2O + K2O 绘制火山岩 TAS 图,并输出逐样品分类与边界复核状态; - 生成投稿尺寸的 REE、蛛网图、Harker 或 TAS 图件,并输出 SVG、PDF、600 dpi TIFF、600 dpi PNG; -- 同步输出实际绘图数据 CSV 和机器可读的 JSON 运行报告; -- 可选择完整四边框,以及经过碰撞检查的图内图例; +- 用一个 YAML 配方明确记录数据文件、列映射、单位、科学参数和人工确认项; +- 在出图前生成不含源数据值的计划,数据或参考文件变化后旧计划自动失效; +- 用一个配方从同一数据表运行一个或多个图件任务,所有任务成功后才提交完整输出; +- 输出机器可读的 JSON 报告和中文 QA 摘要; +- 可在 `local-reproducible` 模式保留实际绘图数据 CSV,或在 `shareable` 模式省略该敏感文件; +- 统一工作流默认使用完整四边框,并自动安排图例以尽量避免遮挡数据或超出图幅; - 全程在本地处理数据,绘图脚本不请求网络服务。 ## 安装到 Codex -公开仓库的默认分支 `main` 包含经过审核和自动测试的正式 v0.3.0: +公开仓库的默认分支 `main` 当前包含经过审核和自动测试的正式 v0.3.0: ```text https://github.com/gronbow/GeoSkills @@ -36,9 +40,15 @@ https://github.com/gronbow/GeoSkills 如果设置了 `CODEX_HOME`,则复制到 `%CODEX_HOME%\skills\geoskills`。复制完成后,重新开启一个 Codex 任务即可使用。 -## Windows 快速开始 +### 普通用户:直接在 Codex 中使用 -需要 Python 3.11 或 3.12。先运行 `python --version` 确认版本,然后在 PowerShell 中进入项目目录并安装依赖: +如果只想用自己的数据出图,不需要手动输入后文的开发命令。重新开启一个 Codex 任务,上传或指定本地数据文件,然后说明希望使用 GeoSkills 绘制哪类图。GeoSkills 会先检查本地环境和数据,再起草配方与计划供你审核;它不会替你把尚未核对的科学确认项设为 `true`。 + +如果环境检查提示缺少依赖,可以让 Codex 解释缺少什么,并在你同意后完成本地安装。 + +## 开发者:Windows 本地命令测试 + +以下命令用于开发、审核或手动复现工作流,均应在完整的 GeoSkills 仓库根目录运行。需要 Python 3.11 或 3.12。先运行 `python --version` 确认版本,然后在 PowerShell 中创建独立环境并安装依赖: ```powershell python -m venv .venv @@ -47,6 +57,52 @@ python -m venv .venv 如果系统只提供 Python Launcher,可以把第一条命令改为 `py -3.12 -m venv .venv`。 +## v0.4 统一工作流 + +可以把“配方”理解为一张实验记录表,把“计划”理解为正式运行前的核对清单: + +```text +数据 + 配方 → plan(只检查,不出图)→ 人工审核 → run(一次性生成完整结果) +``` + +若要用命令行复现流程,先检查环境: + +```powershell +.\.venv\Scripts\python.exe skills\geoskills\scripts\geoskills.py self-check +``` + +仓库提供三个可以直接配合合成数据运行的示例配方: + +| 配方 | 任务 | +|---|---| +| `geoskills_ree_workflow.yaml` | REE 配分图 | +| `geoskills_spider_workflow.yaml` | 微量元素蛛网图 | +| `geoskills_major_workflow.yaml` | 同时生成 Harker 和 TAS | + +例如,先为主量元素双任务配方生成计划: + +```powershell +.\.venv\Scripts\python.exe skills\geoskills\scripts\geoskills.py plan skills\geoskills\examples\geoskills_major_workflow.yaml --output outputs\major-plan.json +``` + +此命令不会生成图件。终端只显示状态、计划编号、计划文件名和问题摘要;完整的任务、列映射、单位、参考文件、图件尺寸和输出模式保存在 `outputs\major-plan.json`。请让 Codex 打开并概括该文件,或用文本编辑器查看。状态为 `ready` 且内容经人工核对后,再运行: + +```powershell +.\.venv\Scripts\python.exe skills\geoskills\scripts\geoskills.py run skills\geoskills\examples\geoskills_major_workflow.yaml --plan outputs\major-plan.json +``` + +`--plan outputs\major-plan.json` 只指定计划文件的位置。图件位置由配方中的 `output.directory` 决定,并相对于配方文件所在目录解析。因此,本示例的图件位于 `skills\geoskills\examples\generated\geoskills-major-workflow\`,而不在 `outputs\`。 + +如果 `run` 返回 `review`,图件已经生成,但存在边界样品等必须人工复核的科学状态。`blocked` 或 `needs_confirmation` 则表示安全检查尚未通过,不会提交新的最终图件。 + +默认 `shareable` 模式不保留逐样品绘图数据 CSV,便于分享图件与报告;需要完全本地复现时,可在配方中改为 `local-reproducible`。该模式会保留敏感 CSV,不应直接上传公开仓库。将 `plotted_data_export_reviewed` 设为 `true`,只表示已经核对这种导出后果,不表示允许把数据上传到模型或第三方服务。 + +示例配方中的确认项只适用于仓库内已审核的合成数据。把配方复制给自己的数据后,应先把确认项改为 `false`,实际核对完成后再逐项确认。 + +配方字段、四类任务示例、返回状态和常见错误见 [v0.4 配方与安全运行流程](skills/geoskills/references/workflow-and-recipe.md)。对于旧项目或高级排错,下列 v0.3 单脚本命令仍保持兼容。 + +## v0.3 单脚本兼容命令 + 先检查示例数据: ```powershell @@ -122,6 +178,8 @@ GeoSkills/ │ ├── examples/ │ ├── references/ │ └── scripts/ +│ ├── geoskills.py +│ └── geoskills_core/ └── tests/ ``` @@ -133,13 +191,17 @@ GeoSkills/ - 原始地幔表保留文献 Table 1 的 Cs、Pb 数值,并明确提示原文脚注中的 modified 版本;程序不会静默混用。 - Harker 图只展示变量间的协变关系;默认不添加拟合线,也不从相关性单独推断岩浆过程。 - TAS 边界使用带文献与版本信息的本地资产。恰好位于边界上的样品标为 `review_required`,不会静默选择一侧。 -- TAS 仅用于明确确认的火山岩;`as-reported` 结果标为初步分类,不能替代无水归一化后的专业判断。 +- TAS 仅用于明确确认的火山岩;`as-reported` 结果标为初步分类,不能替代无水归一化后的专业判断,并且必须由用户明确设置 `provisional_classification_accepted: true`。 - 图形可以展示富集程度、斜率和平行性,但不能单独证明岩浆源区、部分熔融、分离结晶或构造环境。 +- 统一计划和可分享报告默认不含绝对路径、样品编号或源数据值。 +- 配方、数据、工具版本或内置参考文件发生变化时,必须重新生成计划。 +- 多任务结果以完整目录为单位提交;中途失败不会留下半套新输出,也不会破坏原有完整目录。 +- 覆盖运行只替换带有效 GeoSkills 运行报告的旧结果目录;普通同名文件夹会被保护。 - `local_data/` 和 `outputs/` 已排除在 Git 之外;不要提交私人或未发表数据。 ## 当前状态 -GeoSkills v0.3.0 是当前正式版本。它整合了 v0.1.0 的 REE 工作流、经过审核的微量元素蛛网图,以及新增的 Harker 和火山岩 TAS 工作流。完整版本通过了 69 项自动测试、已发表数据的 Harker 验证、合成火山岩 TAS 验证、导出审计、人工图面复核,以及 Ubuntu/Windows 上 Python 3.11/3.12 的 GitHub Actions 检查。版本变化见 [CHANGELOG.md](CHANGELOG.md)。 +GeoSkills v0.3.0 仍是当前公开正式版本。v0.4.0 开发候选版已在本地完成统一入口、严格配方、固定图解注册表、计划指纹、隐私报告和多任务原子输出,并已通过 196 项自动测试、合成数据回归和已发表数据回归。该候选版尚未上传 GitHub,需经用户最终审核批准。版本变化见 [CHANGELOG.md](CHANGELOG.md)。 ## 许可 diff --git a/requirements-dev.txt b/requirements-dev.txt index e31277a..9219e2f 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,3 +1,2 @@ -r skills/geoskills/scripts/requirements.txt pytest>=8,<10 -PyYAML>=6,<7 diff --git a/skills/geoskills/SKILL.md b/skills/geoskills/SKILL.md index 136a9be..7954420 100644 --- a/skills/geoskills/SKILL.md +++ b/skills/geoskills/SKILL.md @@ -1,102 +1,94 @@ --- name: geoskills -description: "Inspect whole-rock geochemical tables and create validated, submission-oriented REE patterns, trace-element spider diagrams, Harker variation diagrams, or volcanic TAS classification plots from CSV, TXT, or Excel data. Use GeoSkills to identify sample, group, element, and oxide columns; verify ppm or wt% units and invalid values; apply versioned Sun and McDonough (1989) references; require explicit TAS applicability declarations; export editable SVG/PDF plus 600 dpi TIFF/PNG and exact plotted data; or explain why a table cannot yet be plotted safely. Do not use the current version for isotope, tectonic-discrimination, or other diagram families." +description: "Create validated, submission-oriented geochemical figures from local CSV, TXT, or Excel tables. Use GeoSkills for Sun and McDonough (1989) chondrite-normalized REE patterns, primitive-mantle or N-MORB-normalized trace-element spider diagrams, customizable Harker diagrams, and guarded volcanic TAS classification; for explicit column/unit mapping; for a reviewable multi-task plotting recipe; or for editable SVG/PDF plus high-resolution TIFF/PNG and privacy-safe QA reports. Do not use it for isotope or tectonic-discrimination diagrams." --- -# GeoSkills v0.3.0 - -Create reproducible REE patterns, trace-element spider diagrams, Harker variation diagrams, and volcanic TAS plots through deterministic local Python scripts. Use the language model to select and explain the workflow, never to calculate normalized ratios or classification fields manually. - -## Release status - -Treat v0.3.0 as the current released workflow for REE patterns, trace-element spider diagrams, Harker variation diagrams, and guarded volcanic TAS classification. These workflows have passed scientific fixtures, real-data or synthetic-data validation as appropriate, export QA, Skill validation, cross-platform automated tests, and user review. Do not imply support for isotope or tectonic-discrimination diagrams. - -## Route the request - -- For chondrite-normalized La–Lu patterns, use the REE workflow. -- For multi-element primitive-mantle or N-MORB-normalized patterns, use the spider workflow. -- For one X variable against one or more oxides/elements, use the Harker workflow. -- For volcanic-rock classification by SiO2 and total alkalis, use the TAS workflow. -- Stop and clarify when the requested diagram, unit, reference composition, sample column, or group column is ambiguous. - -## REE workflow - -1. Run `scripts/check_environment.py`. -2. Run `scripts/inspect_data.py INPUT`. -3. Confirm the worksheet, sample identifier, optional group, ppm units, and REE mapping. -4. Run `scripts/normalize_ree.py INPUT --output OUTPUT.csv`. -5. Run `scripts/plot_ree.py INPUT --output-dir OUTPUT_DIR`. -6. Inspect the final-size PNG/TIFF and return the complete figure bundle. - -Use `--axes-frame full --legend-layout inside-auto` when a four-sided frame and collision-checked in-axes legend are requested. The plotter safely falls back to an outside-right legend. - -## Spider workflow - -1. Run `scripts/check_environment.py`. -2. Run `scripts/inspect_spider_data.py INPUT`. -3. Confirm the worksheet, sample identifier, optional group, direct elemental ppm columns, and any supported oxide columns. -4. Choose `pm-sm89`, `pm-sm89-modified`, or `nmorb-sm89`; do not select a reference from the apparent shape of the data. -5. Run `scripts/normalize_spider.py INPUT --reference pm-sm89-modified --output OUTPUT.csv`. -6. Run `scripts/plot_spider.py INPUT --reference pm-sm89-modified --output-dir OUTPUT_DIR`. -7. Inspect the final-size PNG/TIFF, the normalized source-data CSV, conversion records, warnings, and JSON report. - -The spider plot defaults to a full frame, no background grid, and `inside-auto` legend placement. It exports editable SVG/PDF, 600 dpi LZW TIFF, 600 dpi PNG, exact normalized source data, and a machine-readable report from the same figure object. - -## Harker workflow - -1. Run `scripts/check_environment.py`. -2. Run `scripts/inspect_major_data.py INPUT`. -3. Confirm the worksheet, sample identifier, optional group, the X variable, requested Y variables, and each analyte's unit. -4. Run `scripts/plot_harker.py INPUT --x SiO2 --y TiO2,Al2O3,Fe2O3T,MgO,CaO,Na2O,K2O,P2O5 --output-dir OUTPUT_DIR`. -5. Inspect every panel at final size and return the SVG/PDF/TIFF/PNG, plotted-data CSV, and JSON report. - -`SiO2` is the conventional default X variable. The user may select another validated analyte explicitly. Do not add regression lines or assign a petrogenetic process from correlation alone. - -## TAS workflow - -1. Run `scripts/check_environment.py`. -2. Run `scripts/inspect_major_data.py INPUT`. -3. Confirm that the samples are volcanic and ask whether values are already anhydrous-normalized or should be plotted as reported. -4. Run `scripts/plot_tas.py INPUT --confirm-volcanic --composition-basis anhydrous-normalized --output-dir OUTPUT_DIR`. -5. Inspect the final-size figure, classification CSV, boundary/outside statuses, warnings, and JSON report. - -TAS uses `SiO2` versus `Na2O + K2O` in wt%. Never infer volcanic applicability from sample names. Do not use the volcanic fields for plutonic rocks, carbonatites, kimberlites, lamproites, or strongly altered compositions. Treat `as-reported` classifications as provisional and points on a boundary as `review_required`. - -## Scientific guardrails - -- Require explicit wt% units for major oxides and ppm units for direct elemental concentrations. -- Preserve blanks and below-detection-limit states as missing observations. Never replace them with zero or an invented detection limit. -- Reject negative concentrations and duplicate analyte mappings. Permit true zero only on linear Harker/TAS axes. -- Convert only explicit `K2O`, `P2O5`, and `TiO2` wt% columns to K, P, and Ti ppm. Record the CIAAW/IUPAC atomic weights, formula, and factor used. -- Reject finite zero and negative values on logarithmic axes. -- Preserve the cited Sun and McDonough (1989) incompatibility order even when the user selects a subset. -- Use only the versioned local assets `PrimitiveMantle_SM89`, `PrimitiveMantleModified_SM89`, and `NMORB_SM89` for the spider workflow. -- Use the source footnote's modified primitive mantle as the spider default. Keep the printed and modified variants separate; do not silently replace Cs or Pb, and warn when an affected element is plotted with the printed variant. -- Set log limits from finite positive ratios, add a declared margin, round to clean decimal bounds without clipping, and show unity only when it lies inside the range. -- Use the versioned `TAS_LeMaitre2002_Volcanic_CombinedT` asset; keep `Trachyte/Trachydacite` and `Tephrite/Basanite` unresolved when normative information is absent. -- Round Harker limits outward to clean values without clipping data. Keep TAS at its declared fixed model limits so the field geometry is not distorted. -- Use colour plus line style for groups and sample symbols so colour is not the sole identifier. -- Keep user data local; plotting scripts must not make network requests. -- Describe enrichment, depletion, slopes, covariation, and visible anomalies conservatively. Do not assign a unique source, melting process, mineral control, alteration history, or tectonic setting from one plot. - -## Related resources - -| Resource | Use it when | -|---|---| -| [scripts/inspect_data.py](scripts/inspect_data.py) | Inspect REE input | -| [scripts/normalize_ree.py](scripts/normalize_ree.py) | Normalize REE to C1 chondrite | -| [scripts/plot_ree.py](scripts/plot_ree.py) | Create the REE figure bundle | -| [scripts/inspect_spider_data.py](scripts/inspect_spider_data.py) | Inspect trace elements, units, BDL states, and supported oxides | -| [scripts/normalize_spider.py](scripts/normalize_spider.py) | Normalize trace elements to primitive mantle or N-MORB | -| [scripts/plot_spider.py](scripts/plot_spider.py) | Create the spider-diagram figure bundle | -| [assets/normalization/primitive-mantle-sm89.json](assets/normalization/primitive-mantle-sm89.json) | Audit primitive-mantle values and the Cs/Pb footnote | -| [assets/normalization/primitive-mantle-modified-sm89.json](assets/normalization/primitive-mantle-modified-sm89.json) | Audit the explicit footnote-modified Cs/Pb variant | -| [assets/normalization/nmorb-sm89.json](assets/normalization/nmorb-sm89.json) | Audit N-MORB values | -| [scripts/inspect_major_data.py](scripts/inspect_major_data.py) | Inspect major/trace analytes, units, missing states, and Harker/TAS readiness | -| [scripts/plot_harker.py](scripts/plot_harker.py) | Create customizable Harker grids | -| [scripts/plot_tas.py](scripts/plot_tas.py) | Create guarded volcanic TAS figures and classifications | -| [assets/classification/tas-lemaitre-2002.json](assets/classification/tas-lemaitre-2002.json) | Audit TAS fields, labels, applicability, and reference provenance | -| [references/data-contract.md](references/data-contract.md) | Audit the REE input contract | -| [references/scientific-method.md](references/scientific-method.md) | Audit the REE method | -| [references/spider-method.md](references/spider-method.md) | Audit spider input, normalization, conversion, plotting, and interpretation rules | -| [references/major-elements-and-tas.md](references/major-elements-and-tas.md) | Audit Harker/TAS input, plotting, classification, and interpretation rules | +# GeoSkills v0.4 workflow + +Use deterministic local Python for all table reading, normalization, classification, plotting, and export. The language model may guide choices and explain results, but must not calculate normalized ratios, convert oxides, or classify TAS fields manually. + +## Default workflow + +Prefer the unified `scripts/geoskills.py` workflow: + +1. Run `self-check`. +2. Inspect the user's table locally. +3. Draft a versioned YAML recipe with explicit input layout, column mappings, units, output profile, tasks, and confirmations. +4. Keep every unverified confirmation as `false`. Never mark a scientific or data confirmation `true` merely to make the workflow continue. +5. Run `plan`. This checks the recipe, input, selected analytes, scientific assets, and expected outputs without creating figures. +6. Explain any `blocked` or `needs_confirmation` issue in plain language. Revise only after the user supplies the missing information. +7. Show the ready plan's diagram types, reference choices, dimensions, output profile, and plan ID. Obtain the user's approval before `run`. +8. Run the approved, unchanged plan. +9. Inspect final-size PNG/TIFF output, JSON reports, and Chinese QA summaries before returning the bundle. + +Read [references/workflow-and-recipe.md](references/workflow-and-recipe.md) when creating or explaining a recipe. + +## Route each task + +- `ree`: chondrite-normalized La–Lu patterns. +- `spider`: multi-element patterns normalized to `pm-sm89`, `pm-sm89-modified`, or `nmorb-sm89`. +- `harker`: one validated X analyte against one or more validated Y analytes. +- `tas`: volcanic-rock classification using SiO2 and Na2O + K2O. + +Stop if the requested diagram is outside this fixed registry. Do not imply support for isotope, tectonic-discrimination, or other diagram families. + +## Scientific confirmation gates + +Require the user or a qualified reviewer to confirm: + +- input structure and worksheet; +- sample and optional group columns; +- canonical-to-source column mappings; +- wt% for major oxides and ppm for direct elemental concentrations; +- whether plotted source data may be retained; +- for TAS, that the samples are volcanic and whether the basis is `anhydrous-normalized` or `as-reported`. + +Never infer TAS applicability from sample names or values. Do not apply volcanic TAS fields to plutonic rocks, carbonatites, kimberlites, lamproites, or strongly altered compositions. Treat `as-reported` TAS results as provisional and boundary cases as requiring review. + +## Data and value safeguards + +- Preserve blanks and below-detection-limit states as missing. Never replace them with zero or an invented detection limit. +- Reject negative concentrations and duplicate analyte mappings. +- Permit true zero only on linear Harker or TAS axes. +- Reject finite zero and negative values on logarithmic REE or spider axes. +- Convert only explicit `K2O`, `P2O5`, and `TiO2` wt% columns to K, P, and Ti ppm, using the reviewed deterministic conversion and recording its provenance. +- Preserve the Sun and McDonough (1989) element order even for a subset. +- Do not select a normalization reference from the apparent curve shape. +- Keep the printed and footnote-modified primitive-mantle variants separate. +- Round Harker and logarithmic limits outward without clipping data. Keep TAS at the model's fixed limits. + +## Figure and interpretation safeguards + +- Use colour plus marker or line style so colour is not the only identifier. +- Keep editable text in SVG/PDF and export TIFF/PNG from the same figure. +- Use full borders and collision-checked in-axes legends in the unified workflow. +- Describe only visible enrichment, depletion, slope, anomaly, clustering, scatter, and covariation. +- Do not assign a unique source, melting process, mineral control, alteration history, fractional-crystallization path, or tectonic setting from one diagram. +- Treat Harker correlation as covariation, not proof of a petrogenetic process. + +Read the relevant method file before explaining a scientific result: + +- REE: [references/scientific-method.md](references/scientific-method.md) +- Spider: [references/spider-method.md](references/spider-method.md) +- Harker/TAS: [references/major-elements-and-tas.md](references/major-elements-and-tas.md) + +## Output and privacy + +Use `shareable` unless the user explicitly needs the exact plotted-data CSV: + +- `shareable` returns SVG, PDF, TIFF, PNG, JSON, and QA Markdown without the plotted-data CSV. +- `local-reproducible` also retains the plotted-data CSV and marks it sensitive. + +Reports must not expose absolute paths, sample identifiers, or source values. Keep all processing local; do not send user tables to a model, analytics service, or third-party server. Clearly disclose any future remote processing before it occurs. + +The workflow commits a multi-task output directory only after every task succeeds. Do not bypass stale-plan checks or overwrite an existing bundle unless the user explicitly approves replacement. + +## Legacy compatibility + +The reviewed v0.3 scripts remain available for regression checks and advanced one-off debugging: + +- `inspect_data.py`, `normalize_ree.py`, `plot_ree.py` +- `inspect_spider_data.py`, `normalize_spider.py`, `plot_spider.py` +- `inspect_major_data.py`, `plot_harker.py`, `plot_tas.py` + +Prefer the unified recipe workflow for ordinary use because it adds explicit mapping, plan review, privacy profiles, and all-or-nothing multi-task output. Do not silently mix unified and legacy outputs in one result bundle. diff --git a/skills/geoskills/agents/openai.yaml b/skills/geoskills/agents/openai.yaml index 1ad54af..a41931f 100644 --- a/skills/geoskills/agents/openai.yaml +++ b/skills/geoskills/agents/openai.yaml @@ -1,4 +1,4 @@ interface: display_name: "GeoSkills" - short_description: "Create validated REE, spider, Harker, and TAS plots" - default_prompt: "Use $geoskills to inspect my geochemical table and create a validated REE, trace-element spider, Harker, or volcanic TAS figure bundle." + short_description: "Plan and create validated local geochemistry plots" + default_prompt: "Use $geoskills to inspect my local geochemical table, draft a reviewable plotting recipe, and create an approved REE, spider, Harker, or volcanic TAS figure bundle without uploading my data." diff --git a/skills/geoskills/assets/classification/tas-lemaitre-2002.json b/skills/geoskills/assets/classification/tas-lemaitre-2002.json index 3060d5d..444212a 100644 --- a/skills/geoskills/assets/classification/tas-lemaitre-2002.json +++ b/skills/geoskills/assets/classification/tas-lemaitre-2002.json @@ -107,7 +107,7 @@ "id": "S2", "name": "Basaltic trachyandesite", "plot_label": "Basaltic\ntrachyandesite", - "label_position": [53.5, 7.2], + "label_position": [53.4, 8.15], "polygon": [[49.4, 7.3], [53.0, 9.3], [57.0, 5.9], [52.0, 5.0]] }, { diff --git a/skills/geoskills/examples/geoskills_major_workflow.yaml b/skills/geoskills/examples/geoskills_major_workflow.yaml new file mode 100644 index 0000000..1b28d73 --- /dev/null +++ b/skills/geoskills/examples/geoskills_major_workflow.yaml @@ -0,0 +1,78 @@ +# 本示例中的 true 只适用于同目录的合成数据。 +# 替换为自己的数据后,请先将所有 confirmations 改为 false, +# 完成人工核对后再逐项设为 true;这不表示允许上传数据。 +schema_version: geoskills.recipe/v1 + +input: + file: synthetic_major_element_data.csv + sheet: null + layout: row-per-sample + +columns: + sample_id: Sample + group: Group + mapping: + SiO2: SiO2_wt% + TiO2: TiO2_wt% + Al2O3: Al2O3_wt% + Fe2O3T: Fe2O3T_wt% + MnO: MnO_wt% + MgO: MgO_wt% + CaO: CaO_wt% + Na2O: Na2O_wt% + K2O: K2O_wt% + P2O5: P2O5_wt% + Rb: Rb_ppm + Sr: Sr_ppm + Zr: Zr_ppm + units: + major_oxides: wt% + trace_elements: ppm + +output: + directory: generated/geoskills-major-workflow + report_profile: shareable + +presets: + journal-main: + extends: publication-double-column + style: + width_mm: 183 + height_mm: 120 + dpi: 600 + +confirmations: + input_structure_reviewed: true + column_mapping_reviewed: true + units_reviewed: true + plotted_data_export_reviewed: true + +tasks: + - id: harker-main + diagram: harker + stem: figure-harker + preset: journal-main + parameters: + x: SiO2 + y: + - TiO2 + - Al2O3 + - Fe2O3T + - MgO + - CaO + - Na2O + - K2O + - P2O5 + groups: all + confirmations: {} + + - id: tas-main + diagram: tas + stem: figure-tas + preset: journal-main + parameters: + composition_basis: anhydrous-normalized + groups: all + confirmations: + volcanic_samples: true + composition_basis_reviewed: true diff --git a/skills/geoskills/examples/geoskills_ree_workflow.yaml b/skills/geoskills/examples/geoskills_ree_workflow.yaml new file mode 100644 index 0000000..e739fcb --- /dev/null +++ b/skills/geoskills/examples/geoskills_ree_workflow.yaml @@ -0,0 +1,74 @@ +# 本示例中的 true 只适用于同目录的合成数据。 +# 替换为自己的数据后,请先将所有 confirmations 改为 false, +# 完成人工核对后再逐项设为 true;这不表示允许上传数据。 +schema_version: geoskills.recipe/v1 + +input: + file: synthetic_ree_data.csv + sheet: null + layout: row-per-sample + +columns: + sample_id: Sample + group: Group + mapping: + La: La_ppm + Ce: Ce_ppm + Pr: Pr_ppm + Nd: Nd_ppm + Sm: Sm_ppm + Eu: Eu_ppm + Gd: Gd_ppm + Tb: Tb_ppm + Dy: Dy_ppm + Ho: Ho_ppm + Er: Er_ppm + Tm: Tm_ppm + Yb: Yb_ppm + Lu: Lu_ppm + units: + major_oxides: wt% + trace_elements: ppm + +output: + directory: generated/geoskills-ree-workflow + report_profile: shareable + +presets: + journal-main: + extends: publication-double-column + style: + width_mm: 183 + height_mm: 120 + dpi: 600 + +confirmations: + input_structure_reviewed: true + column_mapping_reviewed: true + units_reviewed: true + plotted_data_export_reviewed: true + +tasks: + - id: ree-main + diagram: ree + stem: figure-ree + preset: journal-main + parameters: + reference: chondrite-sm89 + elements: + - La + - Ce + - Pr + - Nd + - Sm + - Eu + - Gd + - Tb + - Dy + - Ho + - Er + - Tm + - Yb + - Lu + groups: all + confirmations: {} diff --git a/skills/geoskills/examples/geoskills_spider_workflow.yaml b/skills/geoskills/examples/geoskills_spider_workflow.yaml new file mode 100644 index 0000000..7727739 --- /dev/null +++ b/skills/geoskills/examples/geoskills_spider_workflow.yaml @@ -0,0 +1,102 @@ +# 本示例中的 true 只适用于同目录的合成数据。 +# 替换为自己的数据后,请先将所有 confirmations 改为 false, +# 完成人工核对后再逐项设为 true;这不表示允许上传数据。 +schema_version: geoskills.recipe/v1 + +input: + file: synthetic_spider_data.csv + sheet: null + layout: row-per-sample + +columns: + sample_id: Sample + group: Group + mapping: + Rb: Rb_ppm + Ba: Ba_ppm + Th: Th_ppm + U: U_ppm + Nb: Nb_ppm + Ta: Ta_ppm + K2O: K2O_wt% + La: La_ppm + Ce: Ce_ppm + Pb: Pb_ppm + Pr: Pr_ppm + Sr: Sr_ppm + P2O5: P2O5_wt% + Nd: Nd_ppm + Sm: Sm_ppm + Zr: Zr_ppm + Hf: Hf_ppm + Eu: Eu_ppm + TiO2: TiO2_wt% + Gd: Gd_ppm + Tb: Tb_ppm + Dy: Dy_ppm + Y: Y_ppm + Ho: Ho_ppm + Er: Er_ppm + Tm: Tm_ppm + Yb: Yb_ppm + Lu: Lu_ppm + units: + major_oxides: wt% + trace_elements: ppm + +output: + directory: generated/geoskills-spider-workflow + report_profile: shareable + +presets: + journal-main: + extends: publication-double-column + style: + width_mm: 183 + height_mm: 120 + dpi: 600 + +confirmations: + input_structure_reviewed: true + column_mapping_reviewed: true + units_reviewed: true + plotted_data_export_reviewed: true + +tasks: + - id: spider-main + diagram: spider + stem: figure-spider + preset: journal-main + parameters: + reference: pm-sm89-modified + elements: + - Rb + - Ba + - Th + - U + - Nb + - Ta + - K + - La + - Ce + - Pb + - Pr + - Sr + - P + - Nd + - Sm + - Zr + - Hf + - Eu + - Ti + - Gd + - Tb + - Dy + - Y + - Ho + - Er + - Tm + - Yb + - Lu + groups: all + confirmations: {} diff --git a/skills/geoskills/examples/synthetic_major_element_data.csv b/skills/geoskills/examples/synthetic_major_element_data.csv index 24c8a1c..3c6c2a2 100644 --- a/skills/geoskills/examples/synthetic_major_element_data.csv +++ b/skills/geoskills/examples/synthetic_major_element_data.csv @@ -1,11 +1,11 @@ Sample,Group,SiO2_wt%,TiO2_wt%,Al2O3_wt%,Fe2O3T_wt%,MnO_wt%,MgO_wt%,CaO_wt%,Na2O_wt%,K2O_wt%,P2O5_wt%,Rb_ppm,Sr_ppm,Zr_ppm -SYN-01,Series A,47.0,1.45,16.2,10.8,0.18,7.8,10.2,2.5,0.5,0.28,22,430,105 -SYN-02,Series A,50.0,1.20,16.8,9.4,0.16,5.9,8.6,3.5,2.5,0.24,48,390,145 -SYN-03,Series A,54.0,0.98,17.1,8.1,0.14,4.1,7.1,3.0,1.0,0.20,62,350,170 -SYN-04,Series B,54.0,0.92,17.5,7.9,0.13,3.7,6.8,4.2,3.0,0.18,85,320,205 -SYN-05,Series B,59.5,0.70,16.9,6.4,0.11,2.4,5.1,3.5,1.3,0.14,105,270,240 -SYN-06,Series B,59.0,0.68,17.0,6.2,0.10,2.2,4.8,5.0,4.0,0.13,132,245,275 -SYN-07,Series C,66.0,0.42,15.8,4.6,0.08,1.2,3.2,4.0,1.5,0.09,155,190,315 -SYN-08,Series C,74.0,0.20,13.9,2.5,0.05,0.4,1.4,4.0,2.2,0.04,190,115,365 -SYN-09,Series C,49.0,1.10,17.8,8.8,0.15,4.8,7.4,6.0,4.3,0.22,118,360,230 -SYN-10,Series C,58.0,0.55,18.1,5.5,0.09,1.8,3.8,7.0,7.0,0.11,175,205,340 +SYN-01,Series A,48.4986,1.4962,16.7165,11.1444,0.1857,8.0487,10.5252,2.5797,0.5159,0.2891,22,430,105 +SYN-02,Series A,50.8647,1.2208,17.0905,9.5626,0.1628,6.0020,8.7487,3.5605,2.5432,0.2442,48,390,145 +SYN-03,Series A,56.4145,1.0238,17.8646,8.4622,0.1463,4.2833,7.4175,3.1341,1.0447,0.2090,62,350,170 +SYN-04,Series B,54.9171,0.9356,17.7972,8.0342,0.1322,3.7628,6.9155,4.2713,3.0510,0.1831,85,320,205 +SYN-05,Series B,61.9469,0.7288,17.5950,6.6632,0.1145,2.4987,5.3097,3.6439,1.3535,0.1458,105,270,240 +SYN-06,Series B,59.5298,0.6861,17.1527,6.2557,0.1009,2.2198,4.8431,5.0449,4.0359,0.1311,132,245,275 +SYN-07,Series C,68.1185,0.4335,16.3072,4.7477,0.0826,1.2385,3.3027,4.1284,1.5481,0.0928,155,190,315 +SYN-08,Series C,74.9823,0.2027,14.0845,2.5332,0.0507,0.4053,1.4186,4.0531,2.2292,0.0404,190,115,365 +SYN-09,Series C,49.2116,1.1048,17.8769,8.8380,0.1506,4.8207,7.4320,6.0259,4.3186,0.2209,118,360,230 +SYN-10,Series C,56.8906,0.5395,17.7538,5.3948,0.0883,1.7656,3.7273,6.8661,6.8661,0.1079,175,205,340 diff --git a/skills/geoskills/references/workflow-and-recipe.md b/skills/geoskills/references/workflow-and-recipe.md new file mode 100644 index 0000000..735f6b3 --- /dev/null +++ b/skills/geoskills/references/workflow-and-recipe.md @@ -0,0 +1,264 @@ +# GeoSkills v0.4 配方与安全运行流程 + +本页解释统一工作流。它不改变 REE、蛛网图、Harker 或 TAS 的科学计算,只把已经审核过的绘图器放进同一套可检查、可重复的流程中。 + +## 先理解三个词 + +- **配方(recipe)**:类似实验记录表。它写明数据文件、列名、单位、图解参数、输出位置和人工确认项。 +- **计划(plan)**:GeoSkills 检查配方和数据后生成的“运行前清单”。此阶段不出图。 +- **运行(run)**:只有计划状态为 `ready`,且配方、数据和内置参考都没有改变时,才生成图件。 + +计划含有输入文件和科学参考文件的 SHA-256 校验值。任何内容变化都会产生新的计划编号,因此旧计划不能静默用于新数据。 + +## 推荐流程 + +### 普通 Codex 用户 + +上传或指定本地数据文件后,直接要求 Codex 使用 GeoSkills 检查数据并起草配方。GeoSkills 应先运行 `self-check`,再生成计划供你审核;普通用户不需要手动输入下面的开发命令。任何科学或数据确认项都必须由人实际核对,不能由模型为了继续运行而代填为 `true`。 + +### 开发者或手动复现 + +以下命令都从完整的 GeoSkills 仓库根目录运行。Windows PowerShell 使用仓库内的独立 Python 环境: + +```powershell +.\.venv\Scripts\python.exe skills\geoskills\scripts\geoskills.py self-check +.\.venv\Scripts\python.exe skills\geoskills\scripts\geoskills.py plan my-work\recipe.yaml --output my-work\plan.json +.\.venv\Scripts\python.exe skills\geoskills\scripts\geoskills.py run my-work\recipe.yaml --plan my-work\plan.json +``` + +macOS 或 Linux 也从仓库根目录运行: + +```bash +python3 skills/geoskills/scripts/geoskills.py self-check +python3 skills/geoskills/scripts/geoskills.py plan my-work/recipe.yaml --output my-work/plan.json +python3 skills/geoskills/scripts/geoskills.py run my-work/recipe.yaml --plan my-work/plan.json +``` + +终端中的 `plan` 结果只提供状态、计划编号、计划文件名和问题摘要。任务、列映射、单位、科学参考、样式和预期输出保存在 `--output` 指定的计划 JSON 中;请让 Codex 打开并概括该文件,或用文本编辑器查看。不要手工修改计划 JSON,也不要为了让程序继续运行而直接把确认项改成 `true`;只有在人确实核对过对应内容后才能修改配方、重新生成计划。 + +`examples/` 中的确认项只针对随仓库发布的合成数据。复制示例给自己的数据时,应先把确认项重置为 `false`。 + +## 命令与返回状态 + +统一命令只在标准输出写一条 JSON,方便 Codex、DeepSeek 或其他本地 Agent 稳定读取: + +- `version`:显示工具版本和图解接口版本; +- `self-check`:检查运行依赖;默认不显示本地可执行文件路径; +- `self-check --dev`:额外检查测试依赖; +- `self-check --include-paths`:仅在明确需要排错时显示本地 Python 路径; +- `plan RECIPE --output PLAN`:检查并保存计划,不生成图件; +- `plan ... --task TASK_ID`:只规划指定任务,可重复提供; +- `run RECIPE --plan PLAN`:复核计划后运行; +- `--overwrite-plan` 或 `--overwrite`:明确允许替换已有计划或完整输出目录。 + +退出码: + +- `0`:`ready`; +- `1`:配置、环境或执行错误; +- `2`:`blocked`、`needs_confirmation` 或 `review`。 + +`review` 与阻断不同:如果 `run` 返回 `review`,完整图件已经生成并提交到输出目录,但存在边界样品等必须人工复核的科学状态。`blocked` 或 `needs_confirmation` 表示尚未通过安全门,不会提交新的最终图件。退出码 `2` 用于防止自动流程把这些状态误认为投稿就绪。 + +## 最小配方结构 + +```yaml +schema_version: geoskills.recipe/v1 + +input: + file: data.csv + sheet: null + layout: row-per-sample + +columns: + sample_id: Sample + group: Group + mapping: + La: La_ppm + Ce: Ce_ppm + Pr: Pr_ppm + Nd: Nd_ppm + Sm: Sm_ppm + units: + major_oxides: wt% + trace_elements: ppm + +output: + directory: geoskills-output + report_profile: shareable + +presets: {} + +confirmations: + input_structure_reviewed: true + column_mapping_reviewed: true + units_reviewed: true + plotted_data_export_reviewed: true + +tasks: + - id: ree-main + diagram: ree + stem: figure-ree + preset: publication-double-column + parameters: + reference: chondrite-sm89 + elements: [La, Ce, Pr, Nd, Sm] + groups: all + confirmations: {} +``` + +`columns.mapping` 的方向固定为“标准分析物名称 → 原始表列名”。GeoSkills 不会用近似拼写猜测映射。 + +`plotted_data_export_reviewed: true` 只表示已经核对 `report_profile` 的导出后果:`shareable` 不保留逐样品 CSV,`local-reproducible` 会保留敏感 CSV。它不表示允许把数据上传到模型、分析服务或第三方服务器。 + +## 输入 + +- `input.file` 必须是配方目录内的 `.csv`、`.txt` 或 `.xlsx`; +- `input.sheet` 在多工作表 Excel 中必须明确填写; +- `input.layout` 可为 `row-per-sample`、`analyte-per-row` 或 `auto`; +- 路径必须是安全相对路径,不能使用网址、绝对路径、环境变量、通配符或 `..`; +- `output.directory` 必须是专用子目录,不能写成 `.`,也不能包含原始输入文件; +- 计划 JSON 不能覆盖配方或原始输入,也不能放在最终输出目录内; +- 一个配方最多 32 个任务,配方文件最大 256 KiB。 + +`input.file` 和 `output.directory` 都相对于配方文件所在目录解析;命令行中的 `--output` 和 `--plan` 则相对于当前命令目录解析。例如,配方位于 `my-work/recipe.yaml` 且写有 `output.directory: results` 时,图件会进入 `my-work/results/`。 + +论文补充材料常见的“分析物在行、样品在列”表可使用 `analyte-per-row`。只有结构唯一且可确认时才会自动转置。 + +## 单位与固定换算 + +- 主量氧化物必须明确为 `wt%`; +- 直接元素浓度必须明确为 `ppm`; +- K、P、Ti 若没有直接 ppm 列,可分别映射 `K2O`、`P2O5`、`TiO2` 的 wt% 列,蛛网图绘图器再使用已审核的固定化学计量换算; +- 不支持其他隐式单位换算。 + +如果原始列名已经明确写出分析物或单位,例如 `La_ppb` 或 `Ce_ppm`,但配方把它声明为另一单位或另一分析物,计划会直接阻止运行。GeoSkills 不会把这种冲突当作“用户自定义映射”而静默改名;需要先回到原始数据和方法说明核对。 + +## 任务 + +每个任务需要唯一的 `id` 和 `stem`。 + +REE: + +```yaml +diagram: ree +parameters: + reference: chondrite-sm89 + elements: [La, Ce, Pr, Nd, Sm, Eu, Gd, Tb, Dy, Ho, Er, Tm, Yb, Lu] + groups: all +``` + +蛛网图: + +```yaml +diagram: spider +parameters: + reference: pm-sm89-modified + elements: [Rb, Ba, Th, U, Nb, Ta, K, La, Ce, Pb, Pr, Sr, P, Nd, Sm, Zr, Hf, Eu, Ti] + groups: all +``` + +Harker: + +```yaml +diagram: harker +parameters: + x: SiO2 + y: [TiO2, Al2O3, Fe2O3T, MgO, CaO, Na2O, K2O, P2O5] + groups: all +``` + +TAS: + +```yaml +diagram: tas +parameters: + composition_basis: anhydrous-normalized + groups: all +confirmations: + volcanic_samples: true + composition_basis_reviewed: true +``` + +TAS 的两个专属确认项不能由模型根据文件名或数值自行推断。使用 `as-reported` 时还必须由用户明确接受其初步分类性质,并提供第三个确认项: + +```yaml +diagram: tas +parameters: + composition_basis: as-reported + groups: all +confirmations: + volcanic_samples: true + composition_basis_reviewed: true + provisional_classification_accepted: true +``` + +只有在用户理解并接受“该分类只是初步结果”后,才能将 `provisional_classification_accepted` 设为 `true`。 + +## 图件预设 + +内置预设: + +- `publication-double-column`:183 × 120 mm,600 dpi; +- `review-preview`:150 × 100 mm,300 dpi,适合快速审核。 + +可以建立自定义预设,但只能继承上述预设并覆盖 `width_mm`、`height_mm` 和 `dpi`: + +```yaml +presets: + journal-main: + extends: publication-double-column + style: + width_mm: 183 + height_mm: 120 + dpi: 600 +``` + +统一工作流默认使用完整四边框,并自动安排图例以尽量避免遮挡数据或超出图幅。v0.4 配方中的自定义预设只负责图幅宽度、高度和 dpi。 + +## 输出与隐私 + +`shareable`: + +- 输出 SVG、PDF、TIFF、PNG、JSON 报告和中文 QA 摘要; +- 不保留逐样品绘图数据 CSV; +- 报告不含绝对路径或源数据值。 + +`local-reproducible`: + +- 额外保留 `*.source_data.csv`; +- CSV 被标记为敏感的绘图源数据,不应公开分享; +- JSON 和 QA 报告本身仍保持不含绝对路径和源数据值。 + +`output.directory` 是相对于配方目录的安全路径,不是相对于计划 JSON 的位置。计划文件和图件目录可以位于不同位置。 + +多任务运行先写入同一文件系统上的临时目录。只有全部任务成功,整个输出目录才一次性替换到最终位置。任一任务失败时,新结果全部取消,已有完整输出不会被半成品覆盖。 + +运行时会先在私有临时目录建立输入快照,再用同一快照完成校验和绘图。`--overwrite` 只允许替换带有有效 `run.report.json` 的 GeoSkills 完整输出;普通文件夹即使同名也会被保护并拒绝覆盖。 + +典型目录: + +```text +geoskills-output/ +├── run.report.json +├── run.qa.md +├── ree-main/ +│ ├── figure-ree.svg +│ ├── figure-ree.pdf +│ ├── figure-ree.tiff +│ ├── figure-ree.png +│ ├── figure-ree.report.json +│ └── figure-ree.qa.md +└── harker-main/ + └── ... +``` + +## 常见非 `ready` 状态与停止原因 + +- `needs_confirmation`:配方中仍有 `false` 的人工确认项; +- `blocked`:输入结构、列映射、单位、所需分析物或科学适用性未通过; +- `review`:图件已经生成,但边界样品或其他科学状态必须人工复核; +- 旧计划失效:配方、输入文件、任务参数、内置参考或工具版本发生变化; +- 输出已存在:需要先审核旧结果,再明确使用 `--overwrite`; +- 多工作表未选择:在 `input.sheet` 填写准确工作表名称。 + +错误不是“程序不听话”,而是可复现研究中的安全门。应修正原因并重新生成计划,不应绕过检查。 diff --git a/skills/geoskills/scripts/check_environment.py b/skills/geoskills/scripts/check_environment.py index d79e416..8816ece 100644 --- a/skills/geoskills/scripts/check_environment.py +++ b/skills/geoskills/scripts/check_environment.py @@ -1,43 +1,101 @@ #!/usr/bin/env python3 -"""Report whether the local Python environment can support GeoSkills development.""" +"""Report whether the local Python environment can run or develop GeoSkills.""" from __future__ import annotations +import argparse import importlib.util +from importlib import metadata import json import platform import sys -REQUIRED_MODULES = { +RUNTIME_MODULES = { "numpy": "numerical calculations", "pandas": "CSV, TXT, and table handling", "matplotlib": "figure generation and export", + "PIL": "raster figure validation", "openpyxl": "Excel .xlsx input", + "yaml": "versioned YAML plotting recipes", +} +DEVELOPMENT_MODULES = { "pytest": "automated tests", } -def main() -> int: +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description=( + "Check the local GeoSkills runtime. Use --dev only when developing " + "or running the automated test suite." + ) + ) + parser.add_argument( + "--dev", + action="store_true", + help="also require developer-only test dependencies", + ) + parser.add_argument( + "--include-paths", + action="store_true", + help="include the local Python executable path in the diagnostic report", + ) + return parser.parse_args() + + +def module_version(name: str) -> str | None: + package_name = { + "yaml": "PyYAML", + "PIL": "Pillow", + }.get(name, name) + try: + return metadata.version(package_name) + except metadata.PackageNotFoundError: + return None + + +def build_report( + *, + dev: bool = False, + include_paths: bool = False, +) -> dict[str, object]: + """Build a JSON-safe environment report for the CLI and unified workflow.""" + required_modules = dict(RUNTIME_MODULES) + if dev: + required_modules.update(DEVELOPMENT_MODULES) modules = { name: { "available": importlib.util.find_spec(name) is not None, "purpose": purpose, + "version": module_version(name), } - for name, purpose in REQUIRED_MODULES.items() + for name, purpose in required_modules.items() } missing = [name for name, details in modules.items() if not details["available"]] - report = { + report: dict[str, object] = { + "check_mode": "development" if dev else "runtime", "python_version": platform.python_version(), - "python_executable": sys.executable, + "python_implementation": platform.python_implementation(), + "platform": platform.system(), "ready": not missing, "missing_modules": missing, "modules": modules, } + if include_paths: + report["python_executable"] = sys.executable + return report + + +def main() -> int: + args = parse_args() + report = build_report(dev=args.dev, include_paths=args.include_paths) print(json.dumps(report, ensure_ascii=False, indent=2)) + missing = report["missing_modules"] if missing: print( - "GeoSkills environment is not ready; missing: " + ", ".join(missing), + "GeoSkills environment is not ready; missing: " + + ", ".join(str(item) for item in missing), file=sys.stderr, ) return 1 diff --git a/skills/geoskills/scripts/geoskills.py b/skills/geoskills/scripts/geoskills.py new file mode 100644 index 0000000..4b9d8e5 --- /dev/null +++ b/skills/geoskills/scripts/geoskills.py @@ -0,0 +1,289 @@ +#!/usr/bin/env python3 +"""Unified, machine-readable GeoSkills v0.4 command line.""" + +from __future__ import annotations + +import argparse +import json +import runpy +import sys +from pathlib import Path +from typing import Any, NoReturn + + +SCRIPTS_DIR = Path(__file__).resolve().parent +TOOL_VERSION = str( + runpy.run_path( + str(SCRIPTS_DIR / "geoskills_core" / "version.py") + )["VERSION"] +) +DIAGRAM_API_VERSION = "geoskills.diagram/v1" +CLI_REPORT_SCHEMA_VERSION = "geoskills.cli-report/v1" + + +def _issue(code: str, severity: str, message: str) -> dict[str, str]: + return {"code": code, "severity": severity, "message": message} + + +def envelope( + command: str, + status: str, + *, + result: dict[str, Any] | None = None, + issues: list[dict[str, Any]] | None = None, +) -> dict[str, Any]: + return { + "schema_version": CLI_REPORT_SCHEMA_VERSION, + "command": command, + "status": status, + "tool": { + "name": "GeoSkills", + "version": TOOL_VERSION, + }, + "result": result or {}, + "issues": issues or [], + } + + +def emit(document: dict[str, Any]) -> None: + print( + json.dumps( + document, + ensure_ascii=False, + sort_keys=True, + allow_nan=False, + ) + ) + + +def exit_code(status: str) -> int: + if status == "ready": + return 0 + if status in {"blocked", "needs_confirmation", "review"}: + return 2 + return 1 + + +class JsonArgumentParser(argparse.ArgumentParser): + """Return malformed command lines as the same JSON contract.""" + + def error(self, message: str) -> NoReturn: + command = ( + sys.argv[1] + if len(sys.argv) > 1 and not sys.argv[1].startswith("-") + else "unknown" + ) + emit( + envelope( + command, + "error", + issues=[_issue("E900", "error", message)], + ) + ) + raise SystemExit(1) + + +def build_parser() -> argparse.ArgumentParser: + parser = JsonArgumentParser( + description="GeoSkills 本地地球化学绘图工作流。", + ) + commands = parser.add_subparsers(dest="command", required=True) + commands.add_parser("version", help="显示工具与接口版本") + + self_check = commands.add_parser( + "self-check", + help="检查本地运行环境", + ) + self_check.add_argument( + "--dev", + action="store_true", + help="同时检查开发测试依赖", + ) + self_check.add_argument( + "--include-paths", + action="store_true", + help="明确允许诊断结果包含本地 Python 路径", + ) + + plan = commands.add_parser( + "plan", + help="检查配方和输入并保存可审核计划,不出图", + ) + plan.add_argument("recipe", type=Path, help="YAML 配方文件") + plan.add_argument( + "--output", + type=Path, + required=True, + help="计划 JSON 文件", + ) + plan.add_argument( + "--task", + action="append", + dest="tasks", + help="只选择该任务 ID;可重复使用", + ) + plan.add_argument( + "--overwrite-plan", + action="store_true", + help="明确替换已有计划文件", + ) + + run = commands.add_parser( + "run", + help="复核计划未变化后,一次性生成完整输出", + ) + run.add_argument("recipe", type=Path, help="YAML 配方文件") + run.add_argument( + "--plan", + type=Path, + required=True, + help="先前生成并审核过的计划 JSON", + ) + run.add_argument( + "--overwrite", + action="store_true", + help="明确原子替换已有完整输出目录", + ) + return parser + + +def command_version() -> tuple[dict[str, Any], int]: + document = envelope( + "version", + "ready", + result={ + "tool_version": TOOL_VERSION, + "diagram_api_version": DIAGRAM_API_VERSION, + }, + ) + return document, 0 + + +def command_self_check( + *, + dev: bool, + include_paths: bool, +) -> tuple[dict[str, Any], int]: + from check_environment import build_report + + report = build_report(dev=dev, include_paths=include_paths) + status = "ready" if bool(report["ready"]) else "error" + if include_paths: + report["privacy_warning"] = ( + "该诊断由用户明确要求,包含本地 Python 可执行文件路径;" + "分享前请先检查。" + ) + issues = ( + [] + if status == "ready" + else [ + _issue( + "E901", + "error", + "本地环境缺少运行所需模块。", + ) + ] + ) + return ( + envelope( + "self-check", + status, + result=report, + issues=issues, + ), + exit_code(status), + ) + + +def command_plan(args: argparse.Namespace) -> tuple[dict[str, Any], int]: + from geoskills_core.workflow import create_plan + + report = create_plan( + args.recipe, + args.output, + selected_task_ids=args.tasks, + overwrite=args.overwrite_plan, + ) + status = str(report["status"]) + result = { + key: value + for key, value in report.items() + if key not in {"status", "issues"} + } + return ( + envelope( + "plan", + status, + result=result, + issues=report["issues"], + ), + exit_code(status), + ) + + +def command_run(args: argparse.Namespace) -> tuple[dict[str, Any], int]: + from geoskills_core.workflow import execute_plan + + report = execute_plan( + args.recipe, + args.plan, + overwrite=args.overwrite, + ) + status = str(report["status"]) + result = { + key: value + for key, value in report.items() + if key not in {"status", "issues"} + } + return ( + envelope( + "run", + status, + result=result, + issues=report["issues"], + ), + exit_code(status), + ) + + +def main() -> int: + if hasattr(sys.stdout, "reconfigure"): + sys.stdout.reconfigure(encoding="utf-8") + if hasattr(sys.stderr, "reconfigure"): + sys.stderr.reconfigure(encoding="utf-8") + args = build_parser().parse_args() + try: + if args.command == "version": + document, code = command_version() + elif args.command == "self-check": + document, code = command_self_check( + dev=args.dev, + include_paths=args.include_paths, + ) + elif args.command == "plan": + document, code = command_plan(args) + else: + document, code = command_run(args) + except Exception: + document = envelope( + str(args.command), + "error", + issues=[ + _issue( + "E999", + "error", + "GeoSkills 遇到未预期错误;请运行 self-check 并保存命令输出。", + ) + ], + ) + code = 1 + emit(document) + print( + f"GeoSkills {args.command}: {document['status']}.", + file=sys.stderr, + ) + return code + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/geoskills/scripts/geoskills_core/__init__.py b/skills/geoskills/scripts/geoskills_core/__init__.py new file mode 100644 index 0000000..4753650 --- /dev/null +++ b/skills/geoskills/scripts/geoskills_core/__init__.py @@ -0,0 +1,125 @@ +"""Public, provider-neutral core helpers for GeoSkills.""" + +from .analytes import ( + DEFAULT_ANALYTE_REGISTRY, + GROUP_NAME_ALIASES, + MAJOR_OXIDE_ORDER, + REE_ORDER, + SAMPLE_NAME_ALIASES, + SPIDER_ELEMENT_ORDER, + TRACE_ELEMENT_ORDER, + AnalyteDefinition, + AnalyteMatch, + AnalyteRegistry, + clean_name, + infer_unit, + match_analyte, + normalize_unit, + strip_unit_suffix, +) +from .errors import ( + FileSizeError, + GeoSkillsError, + InputValidationError, + PlottingError, + TableReadError, + TextEncodingError, + UnsupportedFormatError, + WorksheetError, +) +from .plotting import ( + FORMATS, + GROUP_COLORS, + LINE_STYLES, + MARKERS, + PUBLICATION_DOUBLE_COLUMN, + REVIEW_PREVIEW, + STYLE_PRESETS, + PlotStyleError, + configure_boxed_legend, + group_style_map, + mm_to_inches, + publication_style, + publication_styled, + style_for_index, +) +from .io import ( + MAX_FILE_SIZE_BYTES, + SUPPORTED_SUFFIXES, + adapt_transposed_table, + build_source_metadata, + read_table, + resolve_sheet, + sniff_text_format, + validate_input_file, +) +from .validation import ( + ColumnMapping, + Issue, + Severity, + automatic_column_mappings, + inspect_column_mappings, + issue, + problem_examples, + status_from_issues, + validate_column_mappings, + validate_table_structure, +) + +__all__ = [ + "AnalyteDefinition", + "AnalyteMatch", + "AnalyteRegistry", + "ColumnMapping", + "DEFAULT_ANALYTE_REGISTRY", + "FileSizeError", + "FORMATS", + "GROUP_NAME_ALIASES", + "GROUP_COLORS", + "GeoSkillsError", + "InputValidationError", + "Issue", + "LINE_STYLES", + "MAJOR_OXIDE_ORDER", + "MAX_FILE_SIZE_BYTES", + "MARKERS", + "PUBLICATION_DOUBLE_COLUMN", + "PlotStyleError", + "PlottingError", + "REE_ORDER", + "SAMPLE_NAME_ALIASES", + "SPIDER_ELEMENT_ORDER", + "SUPPORTED_SUFFIXES", + "STYLE_PRESETS", + "Severity", + "TRACE_ELEMENT_ORDER", + "TableReadError", + "TextEncodingError", + "UnsupportedFormatError", + "WorksheetError", + "adapt_transposed_table", + "automatic_column_mappings", + "build_source_metadata", + "clean_name", + "configure_boxed_legend", + "group_style_map", + "infer_unit", + "inspect_column_mappings", + "issue", + "match_analyte", + "normalize_unit", + "problem_examples", + "publication_style", + "publication_styled", + "read_table", + "resolve_sheet", + "sniff_text_format", + "status_from_issues", + "strip_unit_suffix", + "style_for_index", + "validate_column_mappings", + "validate_input_file", + "validate_table_structure", + "mm_to_inches", + "REVIEW_PREVIEW", +] diff --git a/skills/geoskills/scripts/geoskills_core/adapters.py b/skills/geoskills/scripts/geoskills_core/adapters.py new file mode 100644 index 0000000..922f6e1 --- /dev/null +++ b/skills/geoskills/scripts/geoskills_core/adapters.py @@ -0,0 +1,666 @@ +"""Thin, fixed adapters between v0.4 recipes and the reviewed v0.3 plotters.""" + +from __future__ import annotations + +import shutil +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Mapping + +import numpy as np +import pandas as pd + +from .analytes import ( + DEFAULT_ANALYTE_REGISTRY, + infer_unit, + match_analyte, + normalize_unit, +) +from .errors import GeoSkillsError +from .io import adapt_transposed_table, read_table, validate_input_file +from .validation import ColumnMapping, validate_column_mappings + + +class AdapterError(GeoSkillsError): + """A recipe cannot be translated safely to a reviewed plotter.""" + + default_code = "E400" + + +@dataclass(frozen=True) +class PreparedInput: + """One temporary canonical table and its share-safe provenance.""" + + path: Path + source: dict[str, Any] + sample_column: str + group_column: str | None + mappings: tuple[ColumnMapping, ...] + + +def _unit_for( + canonical: str, + source_spec: object, + units: Mapping[str, object], +) -> str: + if isinstance(source_spec, Mapping) and source_spec.get("unit") is not None: + return normalize_unit(source_spec["unit"]) + if canonical in units: + return normalize_unit(units[canonical]) + definition = DEFAULT_ANALYTE_REGISTRY.get(canonical) + if definition is None: + return "unknown" + family_key = ( + "major_oxides" + if definition.kind == "major_oxide" + else "trace_elements" + ) + return normalize_unit(units.get(family_key, "unknown")) + + +def _source_column(source_spec: object) -> str: + if isinstance(source_spec, Mapping): + value = source_spec.get("source") + else: + value = source_spec + if value is None or not str(value).strip(): + raise AdapterError( + "列映射必须提供非空原始列名。", + code="E401", + ) + return str(value) + + +def prepare_mapped_input( + recipe: Mapping[str, Any], + *, + recipe_path: Path, + work_dir: Path, +) -> PreparedInput: + """Create a temporary canonical CSV without changing the user's source.""" + + recipe_directory = recipe_path.resolve().parent + input_value = str(recipe["input"]["file"]) + input_path = (recipe_directory / input_value).resolve() + try: + input_path.relative_to(recipe_directory) + except ValueError as exc: + raise AdapterError( + "输入文件必须位于配方目录或其子目录中。", + code="E402", + ) from exc + + layout = recipe["input"].get("layout", "auto") + transposer = None if layout == "row-per-sample" else adapt_transposed_table + validated_input = validate_input_file(input_path) + work_dir.mkdir(parents=True, exist_ok=True) + snapshot_path = work_dir / f"source_snapshot{validated_input.suffix.lower()}" + shutil.copyfile(validated_input, snapshot_path) + frame, source = read_table( + snapshot_path, + recipe["input"].get("sheet"), + transposer=transposer, + ) + source["filename"] = validated_input.name + if frame is None: + raise AdapterError( + "Excel 文件包含多个工作表;必须在配方中明确 input.sheet。", + code="E403", + details={"sheet_names": source.get("sheet_names", [])}, + ) + if ( + layout == "analyte-per-row" + and source.get("layout") != "column_per_sample_transposed" + ): + raise AdapterError( + "配方声明 analyte-per-row,但未识别出唯一可转置结构。", + code="E404", + ) + + columns = recipe["columns"] + sample_source = str(columns["sample_id"]) + group_value = columns.get("group") + group_source = None if group_value is None else str(group_value) + available = {str(column) for column in frame.columns} + if sample_source not in available: + raise AdapterError( + "配方指定的样品编号列不存在。", + code="E405", + details={"column": sample_source}, + ) + if group_source is not None and group_source not in available: + raise AdapterError( + "配方指定的分组列不存在。", + code="E406", + details={"column": group_source}, + ) + if group_source == sample_source: + raise AdapterError( + "样品编号列和分组列不能是同一列。", + code="E407", + ) + + mapping_config = columns.get("mapping", {}) + units = columns.get("units", {}) + mappings = tuple( + ColumnMapping( + source_column=_source_column(source_spec), + canonical_analyte=str(canonical), + unit=_unit_for(str(canonical), source_spec, units), + origin="recipe", + ) + for canonical, source_spec in mapping_config.items() + ) + mapping_issues = validate_column_mappings( + mappings, + available_columns=frame.columns, + ) + if mapping_issues: + raise AdapterError( + "列映射未通过安全检查。", + code="E408", + details={"issues": [item.to_dict() for item in mapping_issues]}, + ) + header_conflicts: list[dict[str, str]] = [] + for mapping in mappings: + header_unit = infer_unit(mapping.source_column) + declared_unit = normalize_unit(mapping.unit) + matched = match_analyte(mapping.source_column) + if header_unit != "unknown" and header_unit != declared_unit: + header_conflicts.append( + { + "source_column": mapping.source_column, + "canonical_analyte": mapping.canonical_analyte, + "header_unit": header_unit, + "declared_unit": declared_unit, + "conflict": "unit", + } + ) + if ( + matched is not None + and matched.canonical != mapping.canonical_analyte + ): + header_conflicts.append( + { + "source_column": mapping.source_column, + "canonical_analyte": mapping.canonical_analyte, + "header_analyte": matched.canonical, + "conflict": "analyte", + } + ) + if header_conflicts: + raise AdapterError( + "原始列名中的明确分析物或单位与配方声明不一致;GeoSkills 不会猜测、交换或自动换算。", + code="E415", + details={"conflicts": header_conflicts}, + ) + + used_sources = [sample_source] + if group_source is not None: + used_sources.append(group_source) + used_sources.extend(item.source_column for item in mappings) + duplicates = sorted( + { + value + for value in used_sources + if used_sources.count(value) > 1 + } + ) + if duplicates: + raise AdapterError( + "同一原始列不能同时承担多个角色。", + code="E409", + details={"columns": duplicates}, + ) + + canonical = pd.DataFrame({"Sample": frame[sample_source]}) + if group_source is not None: + canonical["Group"] = frame[group_source] + for mapping in mappings: + canonical[f"{mapping.canonical_analyte}_{mapping.unit}"] = frame[ + mapping.source_column + ] + + mapped_path = work_dir / "mapped_input.csv" + canonical.to_csv(mapped_path, index=False) + safe_source = { + key: value + for key, value in source.items() + if key + in { + "filename", + "file_sha256", + "size_bytes", + "format", + "sheet", + "layout", + } + } + safe_source["row_count"] = int(canonical.shape[0]) + safe_source["column_count"] = int(canonical.shape[1]) + return PreparedInput( + path=mapped_path, + source=safe_source, + sample_column="Sample", + group_column="Group" if group_source is not None else None, + mappings=mappings, + ) + + +def _safe_issues(report: Mapping[str, Any]) -> list[dict[str, str]]: + results: list[dict[str, str]] = [] + for item in report.get("issues", []): + severity = str(item.get("severity", "review")) + if severity not in {"info", "warning", "review", "error"}: + severity = "review" + results.append( + { + "code": str(item.get("code", "E499")), + "severity": severity, + "message": str(item.get("message", "需要人工复核。")), + } + ) + return results + + +def inspect_task( + task: Mapping[str, Any], + prepared: PreparedInput, +) -> dict[str, Any]: + """Run the appropriate reviewed inspector without importing plotters.""" + + diagram = str(task["diagram"]) + parameters = task.get("parameters", {}) + selected_groups = parameters.get("groups", "all") + missing_groups: list[str] = [] + selected_frame: pd.DataFrame | None = None + inspection_path = prepared.path + if selected_groups != "all": + if prepared.group_column is None: + missing_groups = [str(item) for item in selected_groups] + else: + canonical_frame = pd.read_csv(prepared.path) + group_text = ( + canonical_frame[prepared.group_column] + .astype("string") + .str.strip() + ) + available_groups = { + str(value) + for value in group_text.dropna() + if str(value).strip() + } + missing_groups = sorted( + str(item) + for item in selected_groups + if str(item) not in available_groups + ) + if not missing_groups: + requested_groups = {str(item) for item in selected_groups} + selected_frame = canonical_frame.loc[ + group_text.isin(requested_groups) + ].copy() + if diagram in {"ree", "spider"}: + inspection_path = _task_input_path(task, prepared) + elif diagram in {"ree", "spider", "harker", "tas"}: + selected_frame = pd.read_csv(prepared.path) + + if diagram == "ree": + from inspect_data import inspect_path + + report = inspect_path( + inspection_path, + requested_sample_column=prepared.sample_column, + requested_group_column=prepared.group_column, + ) + recognized = [ + item["element"] + for item in report.get("ree", {}).get("recognized", []) + ] + elif diagram == "spider": + from inspect_spider_data import inspect_spider_path + + report = inspect_spider_path( + inspection_path, + requested_sample_column=prepared.sample_column, + requested_group_column=prepared.group_column, + ) + recognized = [ + item["element"] + for item in report.get("trace_elements", {}).get("recognized", []) + ] + elif diagram in {"harker", "tas"}: + from inspect_major_data import inspect_major_path + + report = inspect_major_path( + prepared.path, + requested_sample_column=prepared.sample_column, + requested_group_column=prepared.group_column, + ) + recognized = [ + item["analyte"] + for item in report.get("analytes", {}).get("recognized", []) + ] + else: + raise AdapterError( + "图件类型不在固定注册表中。", + code="E410", + details={"diagram": diagram}, + ) + + issues = _safe_issues(report) + requested: list[str] = [] + if diagram in {"ree", "spider"}: + requested = [str(value) for value in parameters.get("elements", [])] + elif diagram == "harker": + requested = [ + str(parameters.get("x", "")), + *[str(value) for value in parameters.get("y", [])], + ] + elif diagram == "tas": + requested = ["SiO2", "Na2O", "K2O"] + missing = sorted( + {value for value in requested if value and value not in set(recognized)} + ) + if missing: + issues.append( + { + "code": "E411", + "severity": "review", + "message": "任务所需分析项目未全部映射到输入表。", + "field": "columns.mapping", + "suggested_action": "补充列映射并重新生成计划。", + } + ) + if missing_groups: + issues.append( + { + "code": "E412", + "severity": "review", + "message": "任务选择的分组未在输入表中找到。", + "field": "tasks.parameters.groups", + "suggested_action": "核对分组名称和 columns.group 后重新生成计划。", + } + ) + pattern_blocked = False + if diagram in {"ree", "spider"} and not missing and not missing_groups: + inspection_frame = pd.read_csv(inspection_path) + canonical_columns = { + mapping.canonical_analyte: ( + f"{mapping.canonical_analyte}_{mapping.unit}" + ) + for mapping in prepared.mappings + } + spider_sources = {"K": "K2O", "P": "P2O5", "Ti": "TiO2"} + positive_found = False + for analyte in (str(value) for value in parameters.get("elements", [])): + source_analyte = analyte + if ( + diagram == "spider" + and source_analyte not in canonical_columns + ): + source_analyte = spider_sources.get(analyte, analyte) + column = canonical_columns.get(source_analyte) + if column is None: + continue + values = pd.to_numeric(inspection_frame[column], errors="coerce") + if bool((values.notna() & np.isfinite(values) & (values > 0)).any()): + positive_found = True + break + if not positive_found: + pattern_blocked = True + issues.append( + { + "code": "E419", + "severity": "review", + "message": "所选样品和元素没有任何可绘制的正有限值。", + "field": "tasks.parameters.groups", + "suggested_action": "检查分组、所选元素、缺失值和非正值。", + } + ) + harker_blocked = False + if diagram == "harker": + requested_y = [str(value) for value in parameters.get("y", [])] + if len(requested_y) > 9: + harker_blocked = True + issues.append( + { + "code": "E416", + "severity": "review", + "message": "Harker 图一次最多绘制 9 个 Y 变量。", + "field": "tasks.parameters.y", + "suggested_action": "拆分为多个 Harker 任务后重新生成计划。", + } + ) + if not missing and not missing_groups and selected_frame is not None: + canonical_columns = { + mapping.canonical_analyte: ( + f"{mapping.canonical_analyte}_{mapping.unit}" + ) + for mapping in prepared.mappings + } + x_name = str(parameters.get("x", "")) + x_column = canonical_columns.get(x_name) + insufficient: list[str] = [] + if x_column is not None: + x_values = pd.to_numeric( + selected_frame[x_column], errors="coerce" + ) + for y_name in requested_y: + y_column = canonical_columns.get(y_name) + if y_column is None: + continue + y_values = pd.to_numeric( + selected_frame[y_column], errors="coerce" + ) + paired = ( + x_values.notna() + & y_values.notna() + & np.isfinite(x_values) + & np.isfinite(y_values) + ) + if int(paired.sum()) < 2: + insufficient.append(y_name) + if insufficient: + harker_blocked = True + issues.append( + { + "code": "E417", + "severity": "review", + "message": "Harker 图的部分 X–Y 组合少于 2 对完整有限值。", + "field": "tasks.parameters.y", + "suggested_action": "检查缺失值、分组筛选或拆分任务后重新生成计划。", + } + ) + tas_blocked = False + if ( + diagram == "tas" + and not missing + and not missing_groups + and selected_frame is not None + ): + canonical_columns = { + mapping.canonical_analyte: ( + f"{mapping.canonical_analyte}_{mapping.unit}" + ) + for mapping in prepared.mappings + } + silica = pd.to_numeric( + selected_frame[canonical_columns["SiO2"]], errors="coerce" + ) + sodium = pd.to_numeric( + selected_frame[canonical_columns["Na2O"]], errors="coerce" + ) + potassium = pd.to_numeric( + selected_frame[canonical_columns["K2O"]], errors="coerce" + ) + finite = ( + silica.notna() + & sodium.notna() + & potassium.notna() + & np.isfinite(silica) + & np.isfinite(sodium) + & np.isfinite(potassium) + ) + if int(finite.sum()) == 0: + tas_blocked = True + issues.append( + { + "code": "E418", + "severity": "review", + "message": "所选样品没有可用于 TAS 分类的完整有限坐标。", + "field": "tasks.parameters.groups", + "suggested_action": "检查分组筛选以及 SiO2、Na2O、K2O 数据。", + } + ) + status = str(report.get("status", "blocked")) + if ( + missing + or missing_groups + or pattern_blocked + or harker_blocked + or tas_blocked + or status != "ready" + ): + status = "blocked" + return { + "task_id": str(task["id"]), + "diagram": diagram, + "status": status, + "recognized_analytes": recognized, + "issues": issues, + } + + +def _comma_list(value: object) -> str | None: + if value is None or value == "all": + return None + if isinstance(value, str): + return value + return ",".join(str(item) for item in value) + + +def _task_input_path( + task: Mapping[str, Any], + prepared: PreparedInput, +) -> Path: + """Create a private per-task subset for REE/spider group selection.""" + + if str(task["diagram"]) not in {"ree", "spider"}: + return prepared.path + selected = task.get("parameters", {}).get("groups", "all") + if selected == "all": + return prepared.path + if prepared.group_column is None: + raise AdapterError( + "按组筛选前必须明确分组列。", + code="E413", + ) + frame = pd.read_csv(prepared.path) + group_text = frame[prepared.group_column].astype("string").str.strip() + requested = {str(item) for item in selected} + subset = frame.loc[group_text.isin(requested)].copy() + if subset.empty: + raise AdapterError( + "所选分组没有可绘制样品。", + code="E414", + ) + task_id = str(task["id"]) + subset_path = prepared.path.parent / f"mapped_input.{task_id}.csv" + subset.to_csv(subset_path, index=False) + return subset_path + + +def run_task( + task: Mapping[str, Any], + prepared: PreparedInput, + *, + output_dir: Path, +) -> dict[str, Any]: + """Call one fixed v0.3 plotter with fully expanded recipe parameters.""" + + diagram = str(task["diagram"]) + parameters = dict(task.get("parameters", {})) + style = dict(task.get("style", {})) + task_input_path = _task_input_path(task, prepared) + output_dir.mkdir(parents=True, exist_ok=False) + common = { + "input_path": task_input_path, + "output_dir": output_dir, + "stem": str(task["stem"]), + "requested_sheet": None, + "requested_sample_column": prepared.sample_column, + "requested_group_column": prepared.group_column, + "title": parameters.get("title"), + "dpi": int(style["dpi"]), + "overwrite": False, + "style_preset": str(style["base_preset"]), + } + + if diagram == "ree": + from plot_ree import plot_path + + return plot_path( + **common, + requested_elements=_comma_list(parameters["elements"]), + width_mm=float(style["width_mm"]), + height_mm=float(style["height_mm"]), + y_margin=float(style.get("y_margin", 0.08)), + axes_frame=str(style.get("axes_frame", "full")), + legend_layout=str(style.get("legend_layout", "inside-auto")), + grid_style=str(style.get("grid_style", "none")), + ) + if diagram == "spider": + from plot_spider import plot_spider_path + + return plot_spider_path( + **common, + reference_key=str(parameters["reference"]), + requested_elements=_comma_list(parameters["elements"]), + width_mm=float(style["width_mm"]), + height_mm=float(style["height_mm"]), + y_margin=float(style.get("y_margin", 0.08)), + axes_frame=str(style.get("axes_frame", "full")), + legend_layout=str(style.get("legend_layout", "inside-auto")), + grid_style=str(style.get("grid_style", "none")), + ) + if diagram == "harker": + from plot_harker import plot_harker_path + + return plot_harker_path( + **common, + requested_x=str(parameters["x"]), + requested_y=_comma_list(parameters["y"]), + requested_groups=_comma_list(parameters.get("groups", "all")), + width_mm=float(style["width_mm"]), + height_mm=( + None + if style.get("height_mm") is None + else float(style["height_mm"]) + ), + columns=( + None + if parameters.get("columns") is None + else int(parameters["columns"]) + ), + axes_frame=str(style.get("axes_frame", "full")), + margin_fraction=float(style.get("margin_fraction", 0.06)), + ) + if diagram == "tas": + from plot_tas import plot_tas_path + + confirmations = task.get("confirmations", {}) + return plot_tas_path( + **common, + requested_groups=_comma_list(parameters.get("groups", "all")), + confirm_volcanic=bool(confirmations.get("volcanic_samples")), + composition_basis=str(parameters["composition_basis"]), + width_mm=float(style["width_mm"]), + height_mm=float(style["height_mm"]), + legend_layout=str(style.get("legend_layout", "inside-auto")), + ) + raise AdapterError( + "图件类型不在固定注册表中。", + code="E410", + details={"diagram": diagram}, + ) diff --git a/skills/geoskills/scripts/geoskills_core/analytes.py b/skills/geoskills/scripts/geoskills_core/analytes.py new file mode 100644 index 0000000..d50f3cb --- /dev/null +++ b/skills/geoskills/scripts/geoskills_core/analytes.py @@ -0,0 +1,327 @@ +"""Canonical analyte and unit names shared by GeoSkills workflows.""" + +from __future__ import annotations + +import re +import unicodedata +from dataclasses import dataclass +from typing import Iterable + + +REE_ORDER = ( + "La", + "Ce", + "Pr", + "Nd", + "Sm", + "Eu", + "Gd", + "Tb", + "Dy", + "Ho", + "Er", + "Tm", + "Yb", + "Lu", +) + +SPIDER_ELEMENT_ORDER = ( + "Cs", + "Tl", + "Rb", + "Ba", + "W", + "Th", + "U", + "Nb", + "Ta", + "K", + "La", + "Ce", + "Pb", + "Pr", + "Mo", + "Sr", + "P", + "Nd", + "F", + "Sm", + "Zr", + "Hf", + "Eu", + "Sn", + "Sb", + "Ti", + "Gd", + "Tb", + "Dy", + "Li", + "Y", + "Ho", + "Er", + "Tm", + "Yb", + "Lu", +) + +MAJOR_OXIDE_ORDER = ( + "SiO2", + "TiO2", + "Al2O3", + "Fe2O3T", + "FeOT", + "Fe2O3", + "FeO", + "MnO", + "MgO", + "CaO", + "Na2O", + "K2O", + "P2O5", + "Cr2O3", + "NiO", + "H2O", + "H2O+", + "H2O-", + "CO2", + "LOI", + "Total", +) + +EXTRA_TRACE_ELEMENTS = ( + "Sc", + "V", + "Cr", + "Co", + "Ni", + "Cu", + "Zn", + "Ga", + "Ge", + "As", + "Se", + "Br", + "Ag", + "Cd", + "In", + "Te", +) + +TRACE_ELEMENT_ORDER = tuple( + dict.fromkeys((*EXTRA_TRACE_ELEMENTS, *SPIDER_ELEMENT_ORDER)) +) + +SAMPLE_NAME_ALIASES = frozenset( + { + "id", + "sample", + "samplecode", + "sampleid", + "samplename", + "sampleno", + "samplenumber", + "specimen", + "specimenid", + } +) + +GROUP_NAME_ALIASES = frozenset( + { + "area", + "group", + "lithology", + "lithologicalgroup", + "locality", + "location", + "region", + "rocktype", + "samplegroup", + "suite", + } +) + + +def clean_name(value: object) -> str: + """Convert a label to a Unicode-normalized comparison key.""" + normalized = unicodedata.normalize("NFKC", str(value)).lower() + return re.sub(r"[^a-z0-9]+", "", normalized) + + +def _analyte_key(value: object) -> str: + """Normalize analytes while preserving the H2O +/- distinction.""" + normalized = unicodedata.normalize("NFKC", str(value)) + normalized = re.sub(r"h2o\s*\+", "h2oplus", normalized, flags=re.I) + normalized = re.sub(r"h2o\s*[-−]\s*$", "h2ominus", normalized, flags=re.I) + return clean_name(normalized) + + +_UNIT_SUFFIX_PATTERN = re.compile( + r"[\s_\-\(\[]*(?:" + r"ppm|mg\s*/\s*kg|" + r"ppb|(?:u|µ|μ)g\s*/\s*kg|" + r"wt\s*\.?\s*(?:%|pct|percent)|weight\s*percent" + r")[\s\)\]]*\s*$", + re.IGNORECASE, +) + +_UNIT_PATTERNS = ( + ( + "ppm", + re.compile(r"(?:^|[^a-z])(?:ppm|mg\s*/\s*kg)(?:$|[^a-z])", re.I), + ), + ( + "ppb", + re.compile( + r"(?:^|[^a-z])(?:ppb|(?:u|µ|μ)g\s*/\s*kg)(?:$|[^a-z])", + re.I, + ), + ), + ( + "wt%", + re.compile( + r"(?:^|[^a-z])(?:wt\s*\.?\s*(?:%|pct|percent)|" + r"weight\s*percent)(?:$|[^a-z])", + re.I, + ), + ), +) + + +def infer_unit(value: object) -> str: + """Infer a unit only when the label contains an explicit unit marker.""" + normalized = unicodedata.normalize("NFKC", str(value)) + for unit, pattern in _UNIT_PATTERNS: + if pattern.search(normalized): + return unit + return "unknown" + + +def normalize_unit(value: object) -> str: + """Normalize one explicit unit token to ppm, ppb, wt%, or unknown.""" + normalized = unicodedata.normalize("NFKC", str(value)).strip().lower() + if normalized in {"unknown", ""}: + return "unknown" + return infer_unit(f"({normalized})") + + +def strip_unit_suffix(value: object) -> str: + """Remove one supported unit suffix while keeping the analyte label.""" + normalized = unicodedata.normalize("NFKC", str(value)).strip() + return _UNIT_SUFFIX_PATTERN.sub("", normalized).strip() + + +@dataclass(frozen=True) +class AnalyteDefinition: + """One canonical analyte and its accepted exact aliases.""" + + canonical: str + kind: str + required_unit: str + aliases: tuple[str, ...] = () + + +@dataclass(frozen=True) +class AnalyteMatch: + """The deterministic result of matching one table label.""" + + canonical: str + kind: str + required_unit: str + source_label: str + explicit_unit: str + + def to_dict(self) -> dict[str, str]: + return { + "analyte": self.canonical, + "kind": self.kind, + "required_unit": self.required_unit, + "source_label": self.source_label, + "explicit_unit": self.explicit_unit, + } + + +class AnalyteRegistry: + """Exact-alias registry; it never guesses from partial label matches.""" + + def __init__(self, definitions: Iterable[AnalyteDefinition]) -> None: + self._definitions: dict[str, AnalyteDefinition] = {} + self._aliases: dict[str, str] = {} + for definition in definitions: + if definition.canonical in self._definitions: + raise ValueError( + f"Duplicate analyte definition: {definition.canonical}" + ) + self._definitions[definition.canonical] = definition + for alias in (definition.canonical, *definition.aliases): + key = _analyte_key(alias) + previous = self._aliases.get(key) + if previous is not None and previous != definition.canonical: + raise ValueError( + f"Alias {alias!r} maps to both {previous} and " + f"{definition.canonical}" + ) + self._aliases[key] = definition.canonical + + @property + def canonical_names(self) -> tuple[str, ...]: + return tuple(self._definitions) + + def get(self, canonical: str) -> AnalyteDefinition | None: + """Return a definition by its exact canonical name.""" + return self._definitions.get(canonical) + + def match(self, value: object) -> AnalyteMatch | None: + source_label = unicodedata.normalize("NFKC", str(value)).strip() + canonical = self._aliases.get( + _analyte_key(strip_unit_suffix(source_label)) + ) + if canonical is None: + return None + definition = self._definitions[canonical] + return AnalyteMatch( + canonical=definition.canonical, + kind=definition.kind, + required_unit=definition.required_unit, + source_label=source_label, + explicit_unit=infer_unit(source_label), + ) + + +_MAJOR_ALIASES = { + "SiO2": ("silica",), + "Fe2O3T": ("TFe2O3", "Fe2O3 total", "Fe2O3 tot"), + "FeOT": ("TFeO", "FeO total", "FeO tot"), + "LOI": ("loss on ignition",), +} + + +def _default_definitions() -> tuple[AnalyteDefinition, ...]: + definitions: list[AnalyteDefinition] = [] + for analyte in MAJOR_OXIDE_ORDER: + definitions.append( + AnalyteDefinition( + canonical=analyte, + kind="major_oxide", + required_unit="wt%", + aliases=_MAJOR_ALIASES.get(analyte, ()), + ) + ) + major_names = set(MAJOR_OXIDE_ORDER) + for analyte in TRACE_ELEMENT_ORDER: + if analyte not in major_names: + definitions.append( + AnalyteDefinition( + canonical=analyte, + kind="trace_element", + required_unit="ppm", + ) + ) + return tuple(definitions) + + +DEFAULT_ANALYTE_REGISTRY = AnalyteRegistry(_default_definitions()) + + +def match_analyte(value: object) -> AnalyteMatch | None: + """Match a column/row label with the default GeoSkills registry.""" + return DEFAULT_ANALYTE_REGISTRY.match(value) diff --git a/skills/geoskills/scripts/geoskills_core/errors.py b/skills/geoskills/scripts/geoskills_core/errors.py new file mode 100644 index 0000000..f944cb7 --- /dev/null +++ b/skills/geoskills/scripts/geoskills_core/errors.py @@ -0,0 +1,79 @@ +"""Expected, machine-readable errors used by the GeoSkills core.""" + +from __future__ import annotations + +from typing import Any, Mapping + + +class GeoSkillsError(Exception): + """Base class for an expected error that can be shown to a user.""" + + default_code = "E000" + + def __init__( + self, + message: str, + *, + code: str | None = None, + details: Mapping[str, Any] | None = None, + suggested_action: str | None = None, + ) -> None: + super().__init__(message) + self.code = code or self.default_code + self.details = dict(details or {}) + self.suggested_action = suggested_action + + def to_issue(self) -> dict[str, Any]: + """Return a JSON-serializable error issue.""" + result: dict[str, Any] = { + "code": self.code, + "severity": "error", + "message": str(self), + } + if self.details: + result["details"] = self.details + if self.suggested_action: + result["suggested_action"] = self.suggested_action + return result + + +class InputValidationError(GeoSkillsError): + """The input path or table does not meet the public data contract.""" + + default_code = "E100" + + +class UnsupportedFormatError(InputValidationError): + """The input file extension is not supported.""" + + default_code = "E101" + + +class FileSizeError(InputValidationError): + """The input file exceeds the configured safety limit.""" + + default_code = "E102" + + +class TextEncodingError(InputValidationError): + """The text file is neither valid UTF-8 nor valid GB18030.""" + + default_code = "E103" + + +class WorksheetError(InputValidationError): + """A requested Excel worksheet does not exist.""" + + default_code = "E104" + + +class TableReadError(InputValidationError): + """A supported file could not be parsed as a table.""" + + default_code = "E105" + + +class PlottingError(GeoSkillsError): + """An expected problem that makes figure creation or export unsafe.""" + + default_code = "E500" diff --git a/skills/geoskills/scripts/geoskills_core/export.py b/skills/geoskills/scripts/geoskills_core/export.py new file mode 100644 index 0000000..b9fe3bb --- /dev/null +++ b/skills/geoskills/scripts/geoskills_core/export.py @@ -0,0 +1,437 @@ +"""Privacy-safe and failure-clean output helpers for GeoSkills. + +``AtomicBundle`` stages every requested file beside the destination directory. +Each final file is installed with ``os.replace`` (atomic for that file on the +same filesystem). A multi-file bundle cannot be made one filesystem operation, +so the class also keeps temporary backups and rolls back a partial commit if an +installation fails. +""" + +from __future__ import annotations + +import hashlib +import os +import shutil +import tempfile +from pathlib import Path +from typing import Any, Iterable, Sequence + + +class BundleExportError(RuntimeError): + """Raised when a complete output bundle cannot be exported safely.""" + + +DEFAULT_FIGURE_FORMATS = ("svg", "pdf", "tiff", "png") +_SUPPORTED_FIGURE_FORMATS = frozenset(DEFAULT_FIGURE_FORMATS) + + +def _safe_filename(filename: str) -> str: + """Validate one flat bundle filename.""" + + name = str(filename).strip() + candidate = Path(name) + if ( + not name + or name in {".", ".."} + or candidate.name != name + or candidate.is_absolute() + ): + raise BundleExportError( + "输出文件名必须是不含目录、盘符或上级路径的单个文件名。" + ) + return name + + +def _safe_stem(stem: str) -> str: + name = _safe_filename(stem) + if Path(name).suffix: + raise BundleExportError("输出名称不能包含扩展名。") + return name + + +def sha256_file(path: Path, chunk_size: int = 1024 * 1024) -> str: + """Calculate a file digest without loading a large raster into memory.""" + + if chunk_size <= 0: + raise BundleExportError("校验分块大小必须大于 0。") + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(chunk_size), b""): + digest.update(chunk) + return digest.hexdigest() + + +def shareable_file_record( + path: Path, + *, + bundle_root: Path | None = None, + role: str | None = None, +) -> dict[str, Any]: + """Describe one output without exposing its absolute local path. + + If ``bundle_root`` is supplied, the file must be inside that directory. A + portable POSIX-style relative path is then included for nested bundles. + """ + + resolved = Path(path).resolve(strict=True) + if not resolved.is_file(): + raise BundleExportError(f"输出记录对象不是普通文件:{resolved.name}。") + + relative = Path(resolved.name) + if bundle_root is not None: + root = Path(bundle_root).resolve(strict=True) + if not root.is_dir(): + raise BundleExportError("bundle_root 必须是已存在的目录。") + try: + relative = resolved.relative_to(root) + except ValueError as exc: + raise BundleExportError("输出文件不在指定的 bundle_root 内。") from exc + + record: dict[str, Any] = { + "filename": resolved.name, + "format": resolved.suffix.lower().lstrip("."), + "bytes": resolved.stat().st_size, + "sha256": sha256_file(resolved), + } + if relative != Path(resolved.name): + record["relative_path"] = relative.as_posix() + if role is not None: + cleaned_role = str(role).strip() + if not cleaned_role: + raise BundleExportError("输出文件角色不能为空。") + record["role"] = cleaned_role + return record + + +class AtomicBundle: + """Stage and transactionally install a flat collection of output files.""" + + def __init__(self, output_dir: Path, *, overwrite: bool = False) -> None: + self.output_dir = Path(output_dir).resolve() + self.overwrite = bool(overwrite) + self._staging_dir: Path | None = None + self._staged: dict[str, Path] = {} + self._entered = False + self._committed = False + + @property + def staging_dir(self) -> Path: + """Return the active private staging directory.""" + + if self._staging_dir is None: + raise BundleExportError("输出暂存区尚未启动或已经关闭。") + return self._staging_dir + + def __enter__(self) -> "AtomicBundle": + if self._entered: + raise BundleExportError("同一个 AtomicBundle 不能重复进入。") + parent = self.output_dir.parent + parent.mkdir(parents=True, exist_ok=True) + self._staging_dir = Path( + tempfile.mkdtemp(prefix=".geoskills-stage-", dir=parent) + ).resolve() + self._entered = True + return self + + def stage_path(self, filename: str) -> Path: + """Reserve and return one path inside the private staging directory.""" + + self._require_active() + name = _safe_filename(filename) + if name not in self._staged: + self._staged[name] = self.staging_dir / name + return self._staged[name] + + def commit(self) -> list[Path]: + """Install every staged file, restoring old files after any failure.""" + + self._require_active() + if not self._staged: + raise BundleExportError("输出包中没有已登记的文件。") + + missing = [name for name, path in self._staged.items() if not path.is_file()] + if missing: + raise BundleExportError( + "以下暂存文件尚未成功写入:" + ", ".join(sorted(missing)) + "。" + ) + + targets = { + name: self.output_dir / name for name in sorted(self._staged) + } + directory_targets = [ + target.name for target in targets.values() if target.is_dir() + ] + if directory_targets: + raise BundleExportError( + "目标位置存在同名目录,不能安全覆盖:" + + ", ".join(directory_targets) + + "。" + ) + existing = [target for target in targets.values() if target.exists()] + if existing and not self.overwrite: + raise BundleExportError( + "输出文件已经存在;如需替换,请明确启用 overwrite:" + + ", ".join(path.name for path in existing) + + "。" + ) + + self.output_dir.mkdir(parents=True, exist_ok=True) + backup_dir = Path( + tempfile.mkdtemp( + prefix=".geoskills-backup-", dir=self.output_dir.parent + ) + ).resolve() + backups: list[tuple[Path, Path]] = [] + installed: list[Path] = [] + commit_succeeded = False + rollback_incomplete = False + try: + for name, target in targets.items(): + if target.exists(): + backup = backup_dir / name + os.replace(target, backup) + backups.append((backup, target)) + + for name, target in targets.items(): + os.replace(self._staged[name], target) + installed.append(target) + commit_succeeded = True + except Exception as exc: + rollback_errors: list[str] = [] + for target in reversed(installed): + try: + if target.exists(): + target.unlink() + except OSError as rollback_exc: + rollback_errors.append(f"删除 {target.name} 失败:{rollback_exc}") + for backup, target in reversed(backups): + try: + if backup.exists(): + os.replace(backup, target) + except OSError as rollback_exc: + rollback_errors.append(f"恢复 {target.name} 失败:{rollback_exc}") + + message = "输出包提交失败,已尝试恢复提交前状态。" + if rollback_errors: + rollback_incomplete = True + message += " 回滚异常:" + ";".join(rollback_errors) + message += ( + f" 未恢复的旧文件保留在恢复目录 {backup_dir.name} 中。" + ) + raise BundleExportError(message) from exc + finally: + if backup_dir.exists() and ( + commit_succeeded or not rollback_incomplete + ): + self._remove_private_directory( + backup_dir, ".geoskills-backup-" + ) + + self._committed = True + installed_paths = list(targets.values()) + self._cleanup_staging() + return installed_paths + + def abort(self) -> None: + """Discard all temporary files without changing final outputs.""" + + self._require_active() + self._cleanup_staging() + + def _require_active(self) -> None: + if ( + not self._entered + or self._committed + or self._staging_dir is None + ): + raise BundleExportError("输出包当前不处于可写状态。") + + def _cleanup_staging(self) -> None: + if self._staging_dir is not None: + self._remove_private_directory( + self._staging_dir, ".geoskills-stage-" + ) + self._staging_dir = None + self._staged.clear() + + def _remove_private_directory(self, path: Path, prefix: str) -> None: + """Remove only a verified temporary directory created beside output.""" + + resolved = path.resolve() + expected_parent = self.output_dir.parent.resolve() + if ( + resolved.parent != expected_parent + or not resolved.name.startswith(prefix) + ): + raise BundleExportError("拒绝清理未经验证的临时目录。") + if resolved.exists(): + shutil.rmtree(resolved) + + def __exit__(self, exc_type: Any, exc: Any, traceback: Any) -> bool: + if self._staging_dir is not None: + self._cleanup_staging() + return False + + +class AtomicDirectory: + """Stage a nested run directory and replace the final directory as one unit.""" + + def __init__(self, output_dir: Path, *, overwrite: bool = False) -> None: + self.output_dir = Path(output_dir).resolve() + self.overwrite = bool(overwrite) + self._staging_dir: Path | None = None + self._entered = False + self._committed = False + + @property + def staging_dir(self) -> Path: + if self._staging_dir is None: + raise BundleExportError("输出目录暂存区尚未启动或已经关闭。") + return self._staging_dir + + def __enter__(self) -> "AtomicDirectory": + if self._entered: + raise BundleExportError("同一个 AtomicDirectory 不能重复进入。") + parent = self.output_dir.parent + parent.mkdir(parents=True, exist_ok=True) + self._staging_dir = Path( + tempfile.mkdtemp(prefix=".geoskills-run-stage-", dir=parent) + ).resolve() + self._entered = True + return self + + def commit(self) -> Path: + """Promote the staged directory and restore an old bundle after failure.""" + + if not self._entered or self._committed or self._staging_dir is None: + raise BundleExportError("输出目录当前不处于可提交状态。") + if not any(self._staging_dir.iterdir()): + raise BundleExportError("输出目录暂存区为空。") + if self.output_dir.exists() and not self.output_dir.is_dir(): + raise BundleExportError("最终输出位置存在同名文件,不能安全替换。") + if self.output_dir.exists() and not self.overwrite: + raise BundleExportError( + "完整输出目录已经存在;如需替换,请明确启用 overwrite。" + ) + + backup = Path( + tempfile.mkdtemp( + prefix=".geoskills-run-backup-", + dir=self.output_dir.parent, + ) + ).resolve() + backup.rmdir() + moved_old = False + commit_succeeded = False + recovery_preserved = False + try: + if self.output_dir.exists(): + os.replace(self.output_dir, backup) + moved_old = True + os.replace(self._staging_dir, self.output_dir) + commit_succeeded = True + except Exception as exc: + if moved_old and backup.exists() and not self.output_dir.exists(): + try: + os.replace(backup, self.output_dir) + except OSError: + recovery_preserved = True + elif moved_old and backup.exists(): + recovery_preserved = True + message = "完整输出目录提交失败,已尝试恢复原输出。" + if recovery_preserved: + message += ( + f" 原输出保留在恢复目录 {backup.name} 中,请勿删除。" + ) + raise BundleExportError( + message + ) from exc + finally: + if backup.exists() and commit_succeeded: + self._remove_private_directory( + backup, ".geoskills-run-backup-" + ) + + self._staging_dir = None + self._committed = True + return self.output_dir + + def abort(self) -> None: + if not self._entered or self._committed or self._staging_dir is None: + raise BundleExportError("输出目录当前不处于可清理状态。") + self._remove_private_directory( + self._staging_dir, ".geoskills-run-stage-" + ) + self._staging_dir = None + + def _remove_private_directory(self, path: Path, prefix: str) -> None: + resolved = path.resolve() + if ( + resolved.parent != self.output_dir.parent.resolve() + or not resolved.name.startswith(prefix) + ): + raise BundleExportError("拒绝清理未经验证的目录暂存区。") + if resolved.exists(): + shutil.rmtree(resolved) + + def __exit__(self, exc_type: Any, exc: Any, traceback: Any) -> bool: + if self._staging_dir is not None: + self._remove_private_directory( + self._staging_dir, ".geoskills-run-stage-" + ) + self._staging_dir = None + return False + + +def stage_figure_outputs( + bundle: AtomicBundle, + figure: Any, + stem: str, + *, + formats: Sequence[str] = DEFAULT_FIGURE_FORMATS, + dpi: int = 600, +) -> list[Path]: + """Render vector and raster files from the same figure into a bundle.""" + + clean_stem = _safe_stem(stem) + if not 72 <= int(dpi) <= 1200: + raise BundleExportError("PNG/TIFF 分辨率必须在 72–1200 dpi 之间。") + + normalised_formats = [str(item).lower().lstrip(".") for item in formats] + if not normalised_formats or len(set(normalised_formats)) != len( + normalised_formats + ): + raise BundleExportError("图件格式不能为空或重复。") + unsupported = [ + item for item in normalised_formats if item not in _SUPPORTED_FIGURE_FORMATS + ] + if unsupported: + raise BundleExportError( + "不支持以下图件格式:" + ", ".join(unsupported) + "。" + ) + + paths: list[Path] = [] + for extension in normalised_formats: + path = bundle.stage_path(f"{clean_stem}.{extension}") + options: dict[str, Any] = {"facecolor": "white"} + if extension in {"png", "tiff"}: + options["dpi"] = int(dpi) + if extension == "tiff": + options["pil_kwargs"] = {"compression": "tiff_lzw"} + figure.savefig(path, **options) + paths.append(path) + return paths + + +def shareable_records( + paths: Iterable[Path], + *, + bundle_root: Path | None = None, + role: str | None = None, +) -> list[dict[str, Any]]: + """Create privacy-safe records for several committed output files.""" + + return [ + shareable_file_record(path, bundle_root=bundle_root, role=role) + for path in paths + ] diff --git a/skills/geoskills/scripts/geoskills_core/io.py b/skills/geoskills/scripts/geoskills_core/io.py new file mode 100644 index 0000000..de86934 --- /dev/null +++ b/skills/geoskills/scripts/geoskills_core/io.py @@ -0,0 +1,402 @@ +"""Privacy-safe table input shared by GeoSkills workflows.""" + +from __future__ import annotations + +import csv +import codecs +import hashlib +from collections.abc import Callable, Mapping +from pathlib import Path +from typing import Any, TypeAlias + +import pandas as pd + +from .analytes import ( + GROUP_NAME_ALIASES, + SAMPLE_NAME_ALIASES, + AnalyteMatch, + clean_name, + infer_unit, + match_analyte, +) +from .errors import ( + FileSizeError, + InputValidationError, + TableReadError, + TextEncodingError, + UnsupportedFormatError, + WorksheetError, +) + + +SUPPORTED_SUFFIXES = frozenset({".csv", ".txt", ".xlsx"}) +MAX_FILE_SIZE_BYTES = 20 * 1024 * 1024 +TEXT_ENCODINGS = ("utf-8-sig", "gb18030") + +TransposeResult: TypeAlias = tuple[pd.DataFrame, dict[str, Any]] +Transposer: TypeAlias = Callable[[pd.DataFrame], TransposeResult | None] +AnalyteMatcher: TypeAlias = Callable[[object], AnalyteMatch | None] + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def validate_input_file( + path: str | Path, + *, + max_file_size_bytes: int = MAX_FILE_SIZE_BYTES, +) -> Path: + """Validate a supported local input path without exposing its location.""" + input_path = Path(path) + display_name = input_path.name or "input" + if not input_path.exists() or not input_path.is_file(): + raise InputValidationError( + f"未找到输入文件:{display_name}。", + code="E106", + details={"filename": display_name}, + suggested_action=( + "确认文件位于配方旁边,或修正 input.file 后重新运行 plan。" + ), + ) + size_bytes = input_path.stat().st_size + if size_bytes > max_file_size_bytes: + raise FileSizeError( + f"Input file exceeds the {max_file_size_bytes}-byte limit.", + details={ + "filename": display_name, + "size_bytes": size_bytes, + "max_size_bytes": max_file_size_bytes, + }, + ) + suffix = input_path.suffix.lower() + if suffix not in SUPPORTED_SUFFIXES: + raise UnsupportedFormatError( + "Supported formats are .csv, .txt, and .xlsx.", + details={"filename": display_name, "format": suffix}, + ) + return input_path + + +def build_source_metadata(path: str | Path) -> dict[str, Any]: + """Build share-safe provenance: filename, content hash, and file size.""" + input_path = Path(path) + return { + "filename": input_path.name, + "file_sha256": _sha256(input_path), + "size_bytes": input_path.stat().st_size, + "format": input_path.suffix.lower(), + "sheet": None, + "sheet_names": [], + "encoding": None, + "delimiter": None, + "layout": None, + "transformation": None, + } + + +def sniff_text_format(path: str | Path) -> tuple[str, str]: + """Detect UTF-8/GB18030 and comma/tab separation.""" + input_path = Path(path) + raw = input_path.read_bytes()[:65536] + decoded: str | None = None + encoding: str | None = None + for candidate in TEXT_ENCODINGS: + try: + decoder = codecs.getincrementaldecoder(candidate)(errors="strict") + decoded = decoder.decode(raw, final=False) + encoding = candidate + break + except UnicodeDecodeError: + continue + if decoded is None or encoding is None: + raise TextEncodingError( + "Text encoding is not supported; use UTF-8 or GB18030.", + details={"filename": input_path.name}, + ) + + try: + delimiter = csv.Sniffer().sniff(decoded, delimiters=",\t").delimiter + except csv.Error: + first_line = decoded.splitlines()[0] if decoded.splitlines() else "" + delimiter = ( + "\t" if first_line.count("\t") > first_line.count(",") else "," + ) + return encoding, delimiter + + +def resolve_sheet( + sheet_names: list[str] | tuple[str, ...], + requested: str | int | None, +) -> str | None: + """Resolve one worksheet, without silently choosing among several.""" + names = list(sheet_names) + if requested is None: + return names[0] if len(names) == 1 else None + if isinstance(requested, int): + index = requested + else: + requested_text = str(requested) + if requested_text in names: + return requested_text + index = int(requested_text) if requested_text.isdigit() else -1 + if 0 <= index < len(names): + return names[index] + raise WorksheetError( + f"Worksheet {requested!r} does not exist.", + details={"requested_sheet": requested, "sheet_names": names}, + ) + + +def _coerce_match( + value: AnalyteMatch | Mapping[str, Any] | None, + source_label: object, +) -> AnalyteMatch | None: + """Accept the public match object and a small mapping compatibility form.""" + if value is None or isinstance(value, AnalyteMatch): + return value + canonical = value.get("analyte") or value.get("canonical") + if not canonical: + return None + return AnalyteMatch( + canonical=str(canonical), + kind=str(value.get("kind", "analyte")), + required_unit=str(value.get("required_unit", "unknown")), + source_label=str(source_label), + explicit_unit=str(value.get("explicit_unit", infer_unit(source_label))), + ) + + +def adapt_transposed_table( + raw: pd.DataFrame, + *, + matcher: Callable[ + [object], AnalyteMatch | Mapping[str, Any] | None + ] = match_analyte, + min_analytes: int = 3, + min_samples: int = 2, +) -> TransposeResult | None: + """Convert a common analytes-by-row table to one row per sample. + + This is deliberately a conservative skeleton. It acts only when there is + one unambiguous sample-header row and at least ``min_analytes`` exact + registry matches. Otherwise it leaves the table untouched. + """ + if ( + raw.empty + or raw.shape[0] < min_analytes + 1 + or raw.shape[1] < min_samples + 1 + ): + return None + + best_label_column: int | None = None + best_rows: list[tuple[int, AnalyteMatch]] = [] + for column_index in range(raw.shape[1]): + matches: list[tuple[int, AnalyteMatch]] = [] + seen: set[str] = set() + duplicate = False + for row_index, label in raw.iloc[:, column_index].items(): + analyte = _coerce_match(matcher(label), label) + if analyte is None: + continue + if analyte.canonical in seen: + duplicate = True + break + seen.add(analyte.canonical) + matches.append((int(row_index), analyte)) + if not duplicate and len(matches) > len(best_rows): + best_label_column = column_index + best_rows = matches + + if best_label_column is None or len(best_rows) < min_analytes: + return None + + sample_rows = [ + int(row_index) + for row_index, label in raw.iloc[:, best_label_column].items() + if clean_name(label) in SAMPLE_NAME_ALIASES + and int(raw.iloc[int(row_index)].notna().sum()) >= min_samples + 1 + ] + if len(sample_rows) != 1: + return None + sample_row = sample_rows[0] + + sample_columns = [ + column_index + for column_index in range(raw.shape[1]) + if column_index != best_label_column + and pd.notna(raw.iat[sample_row, column_index]) + and str(raw.iat[sample_row, column_index]).strip() + ] + if len(sample_columns) < min_samples: + return None + + group_row: int | None = None + for row_index in range(sample_row): + if ( + clean_name(raw.iat[row_index, best_label_column]) + in GROUP_NAME_ALIASES + ): + group_row = row_index + + matched_row_numbers = {row_index for row_index, _ in best_rows} + unit_by_row: dict[int, str] = {} + current_unit = "unknown" + for row_index in range(sample_row + 1, raw.shape[0]): + label = raw.iat[row_index, best_label_column] + explicit_unit = infer_unit(label) + if explicit_unit != "unknown" and row_index not in matched_row_numbers: + current_unit = explicit_unit + if row_index in matched_row_numbers: + unit_by_row[row_index] = ( + explicit_unit if explicit_unit != "unknown" else current_unit + ) + + converted: dict[str, list[Any]] = { + "Sample": [ + str(raw.iat[sample_row, column]).strip() + for column in sample_columns + ] + } + if group_row is not None: + group_values = pd.Series( + [raw.iat[group_row, column] for column in sample_columns], + dtype="object", + ).ffill() + if group_values.notna().any(): + converted["Group"] = [ + None if pd.isna(value) else str(value).strip() + for value in group_values.tolist() + ] + + recognized: list[dict[str, Any]] = [] + output_names = set(converted) + for row_index, analyte in best_rows: + unit = unit_by_row.get(row_index, "unknown") + output_name = ( + f"{analyte.canonical}_{unit}" + if unit != "unknown" + else analyte.canonical + ) + if output_name in output_names: + return None + output_names.add(output_name) + converted[output_name] = [ + raw.iat[row_index, column] for column in sample_columns + ] + recognized.append( + { + **analyte.to_dict(), + "source_row": row_index + 1, + "inferred_unit": unit, + "output_column": output_name, + } + ) + + transformation = { + "method": "auto_transpose_geochemical_analytes_by_row", + "sample_header_row": sample_row + 1, + "sample_identifier_label": str( + raw.iat[sample_row, best_label_column] + ), + "sample_count": len(sample_columns), + "analyte_label_column": best_label_column + 1, + "group_header_row": None if group_row is None else group_row + 1, + "recognized_analytes": recognized, + } + return pd.DataFrame(converted), transformation + + +def _read_text( + path: Path, + metadata: dict[str, Any], + transposer: Transposer | None, +) -> tuple[pd.DataFrame, dict[str, Any]]: + encoding, delimiter = sniff_text_format(path) + metadata["encoding"] = encoding + metadata["delimiter"] = "TAB" if delimiter == "\t" else delimiter + if transposer is not None: + raw = pd.read_csv( + path, + sep=delimiter, + encoding=encoding, + header=None, + ) + adapted = transposer(raw) + if adapted is not None: + frame, transformation = adapted + metadata["layout"] = "column_per_sample_transposed" + metadata["transformation"] = transformation + return frame, metadata + metadata["layout"] = "row_per_sample" + return pd.read_csv(path, sep=delimiter, encoding=encoding), metadata + + +def _read_excel( + path: Path, + metadata: dict[str, Any], + requested_sheet: str | int | None, + transposer: Transposer | None, +) -> tuple[pd.DataFrame | None, dict[str, Any]]: + with pd.ExcelFile(path) as workbook: + metadata["sheet_names"] = list(workbook.sheet_names) + selected_sheet = resolve_sheet(workbook.sheet_names, requested_sheet) + if selected_sheet is None: + return None, metadata + metadata["sheet"] = selected_sheet + if transposer is not None: + raw = pd.read_excel( + workbook, + sheet_name=selected_sheet, + header=None, + ) + adapted = transposer(raw) + if adapted is not None: + frame, transformation = adapted + metadata["layout"] = "column_per_sample_transposed" + metadata["transformation"] = transformation + return frame, metadata + metadata["layout"] = "row_per_sample" + return pd.read_excel(workbook, sheet_name=selected_sheet), metadata + + +def read_table( + path: str | Path, + requested_sheet: str | int | None = None, + *, + transposer: Transposer | None = adapt_transposed_table, + max_file_size_bytes: int = MAX_FILE_SIZE_BYTES, +) -> tuple[pd.DataFrame | None, dict[str, Any]]: + """Read CSV/TXT/XLSX and return a table plus share-safe provenance. + + A multi-sheet workbook returns ``None`` until ``requested_sheet`` is + provided. Pass ``transposer=None`` to disable conservative auto-transpose. + """ + input_path = validate_input_file( + path, + max_file_size_bytes=max_file_size_bytes, + ) + metadata = build_source_metadata(input_path) + try: + if input_path.suffix.lower() in {".csv", ".txt"}: + return _read_text(input_path, metadata, transposer) + return _read_excel( + input_path, + metadata, + requested_sheet, + transposer, + ) + except InputValidationError: + raise + except Exception as exc: + raise TableReadError( + f"Could not read table {input_path.name}.", + details={ + "filename": input_path.name, + "reason": type(exc).__name__, + }, + ) from exc diff --git a/skills/geoskills/scripts/geoskills_core/plotting.py b/skills/geoskills/scripts/geoskills_core/plotting.py new file mode 100644 index 0000000..77b1191 --- /dev/null +++ b/skills/geoskills/scripts/geoskills_core/plotting.py @@ -0,0 +1,192 @@ +"""Local, reusable Matplotlib styling for GeoSkills figures. + +The helpers in this module deliberately avoid mutating ``matplotlib.rcParams`` +at import time. A plotter opts into a controlled preset for the lifetime of a +``with`` block, and Matplotlib restores the caller's settings afterwards. +""" + +from __future__ import annotations + +import inspect +from contextlib import contextmanager +from functools import wraps +from types import MappingProxyType +from typing import Any, Callable, Iterator, Mapping, ParamSpec, Sequence, TypeVar + +import matplotlib as mpl + + +P = ParamSpec("P") +R = TypeVar("R") + + +class PlotStyleError(ValueError): + """Raised when a requested shared plotting style is invalid.""" + + +# Preserve the established v0.3 visual vocabulary while moving it out of an +# individual diagram implementation. Tuples prevent accidental in-place edits. +GROUP_COLORS = ( + "#3569A8", + "#D06B27", + "#159A80", + "#A84F7A", + "#7655A5", + "#8B6B4A", + "#4D4D4D", + "#4F9BC1", +) +MARKERS = ("o", "s", "^", "D", "v", "P", "X", "<", ">", "h", "p", "*") +LINE_STYLES = ("-", "--", "-.", ":") +FORMATS = ("svg", "pdf", "tiff", "png") + +PUBLICATION_DOUBLE_COLUMN = "publication-double-column" +REVIEW_PREVIEW = "review-preview" + +_PUBLICATION_DOUBLE_COLUMN_RCPARAMS: Mapping[str, Any] = MappingProxyType( + { + "font.family": "sans-serif", + "font.sans-serif": ["Arial", "DejaVu Sans", "Liberation Sans"], + "font.size": 7.0, + "axes.labelsize": 7.0, + "axes.linewidth": 0.7, + "axes.spines.right": False, + "axes.spines.top": False, + "xtick.labelsize": 7.0, + "ytick.labelsize": 7.0, + "xtick.major.width": 0.65, + "ytick.major.width": 0.65, + "xtick.minor.width": 0.5, + "ytick.minor.width": 0.5, + "legend.fontsize": 7.0, + "legend.frameon": False, + "savefig.facecolor": "white", + "figure.facecolor": "white", + "svg.fonttype": "none", + "pdf.fonttype": 42, + } +) + +_REVIEW_PREVIEW_RCPARAMS: Mapping[str, Any] = MappingProxyType( + { + **dict(_PUBLICATION_DOUBLE_COLUMN_RCPARAMS), + "font.size": 9.0, + "axes.labelsize": 9.0, + "xtick.labelsize": 8.0, + "ytick.labelsize": 8.0, + "legend.fontsize": 8.0, + "axes.spines.right": True, + "axes.spines.top": True, + } +) + +STYLE_PRESETS: Mapping[str, Mapping[str, Any]] = MappingProxyType( + { + PUBLICATION_DOUBLE_COLUMN: _PUBLICATION_DOUBLE_COLUMN_RCPARAMS, + REVIEW_PREVIEW: _REVIEW_PREVIEW_RCPARAMS, + } +) + + +@contextmanager +def publication_style( + preset: str = PUBLICATION_DOUBLE_COLUMN, + overrides: Mapping[str, Any] | None = None, +) -> Iterator[None]: + """Apply a controlled style locally and restore the caller's rcParams. + + Figure dimensions are intentionally absent from the presets. Every diagram + must declare its final width and height explicitly because journal layouts + and single-/double-column figures have different size contracts. + """ + + if preset not in STYLE_PRESETS: + choices = ", ".join(STYLE_PRESETS) + raise PlotStyleError(f"未知绘图预设:{preset}。可用预设:{choices}。") + + settings = dict(STYLE_PRESETS[preset]) + if overrides: + unknown = sorted(key for key in overrides if key not in mpl.rcParams) + if unknown: + raise PlotStyleError( + "Matplotlib 不识别以下样式参数:" + ", ".join(unknown) + "。" + ) + settings.update(dict(overrides)) + + # rc_context validates values and guarantees restoration even if plotting + # raises an exception inside the block. + with mpl.rc_context(rc=settings): + yield + + +def publication_styled( + preset: str = PUBLICATION_DOUBLE_COLUMN, + overrides: Mapping[str, Any] | None = None, + *, + preset_parameter: str | None = None, +) -> Callable[[Callable[P, R]], Callable[P, R]]: + """Decorate a plotter while keeping all rcParams changes local. + + ``preset_parameter`` lets a public plotting function select a registered + preset through one of its own arguments without mutating global state. + """ + + def decorator(function: Callable[P, R]) -> Callable[P, R]: + function_signature = inspect.signature(function) + + @wraps(function) + def wrapped(*args: P.args, **kwargs: P.kwargs) -> R: + selected = preset + if preset_parameter is not None: + bound = function_signature.bind_partial(*args, **kwargs) + selected = str( + bound.arguments.get(preset_parameter, preset) + ) + with publication_style(selected, overrides): + return function(*args, **kwargs) + + return wrapped + + return decorator + + +def style_for_index(index: int) -> dict[str, str]: + """Return a deterministic colour, marker, and line style for one index.""" + + if index < 0: + raise PlotStyleError("样式序号不能为负数。") + return { + "color": GROUP_COLORS[index % len(GROUP_COLORS)], + "marker": MARKERS[index % len(MARKERS)], + "linestyle": LINE_STYLES[index % len(LINE_STYLES)], + } + + +def group_style_map(groups: Sequence[str]) -> dict[str, dict[str, str]]: + """Map unique, non-empty group names to deterministic combined styles.""" + + names = [str(group).strip() for group in groups] + if any(not name for name in names): + raise PlotStyleError("分组名称不能为空。") + if len(set(names)) != len(names): + raise PlotStyleError("分组名称不能重复。") + return {name: style_for_index(index) for index, name in enumerate(names)} + + +def configure_boxed_legend(legend: Any) -> None: + """Apply the established restrained white box to an in-axes legend.""" + + legend.set_zorder(10) + frame = legend.get_frame() + frame.set_facecolor("white") + frame.set_edgecolor("#A8A8A8") + frame.set_linewidth(0.5) + frame.set_alpha(0.96) + + +def mm_to_inches(value_mm: float) -> float: + """Convert a positive physical dimension from millimetres to inches.""" + + if value_mm <= 0: + raise PlotStyleError("图件尺寸必须大于 0 mm。") + return float(value_mm) / 25.4 diff --git a/skills/geoskills/scripts/geoskills_core/recipe.py b/skills/geoskills/scripts/geoskills_core/recipe.py new file mode 100644 index 0000000..7bbb6d8 --- /dev/null +++ b/skills/geoskills/scripts/geoskills_core/recipe.py @@ -0,0 +1,1267 @@ +"""Safe, strict YAML recipe validation for the GeoSkills v0.4 workflow.""" + +from __future__ import annotations + +import json +import math +import re +from pathlib import Path, PurePosixPath, PureWindowsPath +from typing import Any, Iterable, Mapping + +import yaml + +from .analytes import ( + DEFAULT_ANALYTE_REGISTRY, + REE_ORDER, + SPIDER_ELEMENT_ORDER, +) +from .registry import BUILTIN_STYLE_PRESETS, DIAGRAMS + + +RECIPE_SCHEMA_VERSION = "geoskills.recipe/v1" +MAX_RECIPE_BYTES = 256 * 1024 +MAX_TASKS = 32 + +_ROOT_FIELDS = { + "schema_version", + "input", + "columns", + "output", + "presets", + "confirmations", + "tasks", +} +_REQUIRED_ROOT_FIELDS = _ROOT_FIELDS - {"presets"} +_INPUT_FIELDS = {"file", "sheet", "layout"} +_COLUMN_FIELDS = {"sample_id", "group", "mapping", "units"} +_OUTPUT_FIELDS = {"directory", "report_profile"} +_PRESET_FIELDS = {"extends", "style"} +_STYLE_FIELDS = {"width_mm", "height_mm", "dpi"} +_TASK_FIELDS = { + "id", + "diagram", + "stem", + "preset", + "parameters", + "confirmations", +} +_TOP_CONFIRMATIONS = ( + "input_structure_reviewed", + "column_mapping_reviewed", + "units_reviewed", + "plotted_data_export_reviewed", +) +_TAS_CONFIRMATIONS = ( + "volcanic_samples", + "composition_basis_reviewed", +) +_PROTECTED_OUTPUT_SEGMENTS = frozenset( + {".git", ".codex", "skills", "tests"} +) +_RESERVED_TASK_IDS = frozenset({"_work"}) +_URI_PATTERN = re.compile(r"^[A-Za-z][A-Za-z0-9+.-]*:") +_GLOB_CHARS = frozenset("*?[]{}") +_WINDOWS_ILLEGAL = frozenset('<>:"/\\|?*') +_WINDOWS_RESERVED = frozenset( + { + "CON", + "PRN", + "AUX", + "NUL", + *(f"COM{number}" for number in range(1, 10)), + *(f"LPT{number}" for number in range(1, 10)), + } +) +_REFERENCE_OPTIONS = { + "ree": frozenset({"chondrite-sm89"}), + "spider": frozenset({"pm-sm89", "pm-sm89-modified", "nmorb-sm89"}), +} +_SPIDER_OXIDE_ALTERNATIVES = { + "K": "K2O", + "P": "P2O5", + "Ti": "TiO2", +} +_PARAMETER_FIELDS = { + "ree": frozenset({"reference", "elements", "groups"}), + "spider": frozenset({"reference", "elements", "groups"}), + "harker": frozenset({"x", "y", "groups"}), + "tas": frozenset({"composition_basis", "groups"}), +} +_PARAMETER_REQUIRED = { + "ree": frozenset({"reference", "elements"}), + "spider": frozenset({"reference", "elements"}), + "harker": frozenset({"x", "y"}), + "tas": frozenset({"composition_basis"}), +} +_LAYOUTS = frozenset( + {"row-per-sample", "analyte-per-row", "auto"} +) +_REPORT_PROFILES = frozenset({"shareable", "local-reproducible"}) + + +def _issue( + code: str, + severity: str, + message: str, + *, + field: str | None = None, + task_id: str | None = None, + suggested_action: str | None = None, +) -> dict[str, str]: + result = {"code": code, "severity": severity, "message": message} + if field is not None: + result["field"] = field + if task_id is not None: + result["task_id"] = task_id + if suggested_action is not None: + result["suggested_action"] = suggested_action + return result + + +def _result( + *, + recipe: dict[str, Any] | None, + issues: list[dict[str, str]], +) -> dict[str, Any]: + has_error = any(item["severity"] == "error" for item in issues) + has_review = any(item["severity"] == "review" for item in issues) + status = "invalid" if has_error else ("needs_review" if has_review else "ready") + result = { + "status": status, + "recipe": None if has_error else recipe, + "issues": issues, + } + # This is an internal assertion that guards the public JSON-ready promise. + json.dumps(result, ensure_ascii=False, allow_nan=False) + return result + + +def _mapping(value: Any, field: str, issues: list[dict[str, str]]) -> dict[Any, Any]: + if not isinstance(value, Mapping): + issues.append( + _issue("E306", "error", "该字段必须是键值映射。", field=field) + ) + return {} + return dict(value) + + +def _unknown_fields( + value: Mapping[Any, Any], + allowed: Iterable[str], + field: str, + issues: list[dict[str, str]], + *, + task_id: str | None = None, +) -> None: + allowed_set = set(allowed) + for key in value: + if not isinstance(key, str) or key not in allowed_set: + issues.append( + _issue( + "E304", + "error", + f"发现未支持字段:{key!r}。", + field=f"{field}.{key}", + task_id=task_id, + suggested_action="删除该字段或按模板使用受支持的字段名。", + ) + ) + + +def _missing_fields( + value: Mapping[Any, Any], + required: Iterable[str], + field: str, + issues: list[dict[str, str]], + *, + task_id: str | None = None, +) -> None: + for name in sorted(set(required) - set(value)): + issues.append( + _issue( + "E305", + "error", + f"缺少必填字段:{name}。", + field=f"{field}.{name}", + task_id=task_id, + ) + ) + + +def _nonempty_string( + value: Any, + field: str, + issues: list[dict[str, str]], + *, + task_id: str | None = None, +) -> str | None: + if not isinstance(value, str) or not value.strip(): + issues.append( + _issue( + "E306", + "error", + "该字段必须是非空文本。", + field=field, + task_id=task_id, + ) + ) + return None + return value.strip() + + +def _safe_relative_path( + value: Any, + field: str, + issues: list[dict[str, str]], + *, + output: bool = False, +) -> str | None: + text = _nonempty_string(value, field, issues) + if text is None: + return None + windows_path = PureWindowsPath(text) + unsafe = ( + "\x00" in text + or any(ord(character) < 32 for character in text) + or "$" in text + or "%" in text + or _URI_PATTERN.match(text) is not None + or windows_path.is_absolute() + or bool(windows_path.drive) + or text.startswith(("/", "\\")) + or any(character in text for character in _GLOB_CHARS) + or any(character in text for character in '<>:"|') + ) + normalized = text.replace("\\", "/") + parts = PurePosixPath(normalized).parts + if any( + part in {"..", ""} + or part.startswith("~") + or part.endswith((" ", ".")) + or part.split(".", 1)[0].rstrip(" .").upper() + in _WINDOWS_RESERVED + for part in parts + ): + unsafe = True + if unsafe: + issues.append( + _issue( + "E307", + "error", + "路径必须是配方目录内的安全相对路径;不能使用 URI、绝对路径、环境变量、通配符或 ..。", + field=field, + ) + ) + return None + cleaned_parts = [part for part in parts if part != "."] + cleaned = "/".join(cleaned_parts) or "." + if output and ( + cleaned == "." + or any( + part.rstrip(" .").casefold() in _PROTECTED_OUTPUT_SEGMENTS + for part in cleaned_parts + ) + ): + issues.append( + _issue( + "E307", + "error", + "输出目录必须是专用子目录,且不能指向配方根目录、.git、.codex、skills 或 tests。", + field=field, + ) + ) + return None + return cleaned + + +def _safe_windows_name( + value: Any, + field: str, + issues: list[dict[str, str]], + *, + task_id: str | None = None, + allow_dot: bool = False, +) -> str | None: + text = _nonempty_string(value, field, issues, task_id=task_id) + if text is None: + return None + invalid = ( + text in {".", ".."} + or text[-1] in {" ", "."} + or any(character in _WINDOWS_ILLEGAL for character in text) + or any(ord(character) < 32 for character in text) + or (not allow_dot and Path(text).suffix != "") + or text.split(".", 1)[0].upper() in _WINDOWS_RESERVED + ) + if invalid: + issues.append( + _issue( + "E307", + "error", + "名称必须是跨 Windows/Linux 安全的单一名称,且不能包含扩展名或路径。", + field=field, + task_id=task_id, + ) + ) + return None + return text + + +def _confirmation_map( + raw: Any, + required: Iterable[str], + field: str, + issues: list[dict[str, str]], + *, + task_id: str | None = None, + optional: Iterable[str] = (), +) -> dict[str, bool]: + value = _mapping(raw, field, issues) + allowed = tuple(required) + tuple(optional) + _unknown_fields(value, allowed, field, issues, task_id=task_id) + _missing_fields(value, required, field, issues, task_id=task_id) + normalized: dict[str, bool] = {} + for name in allowed: + if name not in value: + continue + state = value[name] + if not isinstance(state, bool): + issues.append( + _issue( + "E306", + "error", + "确认项必须明确写为 true 或 false。", + field=f"{field}.{name}", + task_id=task_id, + ) + ) + continue + normalized[name] = state + if not state: + issues.append( + _issue( + "R312", + "review", + "该科学或数据确认项尚未确认。", + field=f"{field}.{name}", + task_id=task_id, + suggested_action="复核后将该项明确设为 true。", + ) + ) + return normalized + + +def _validate_input( + raw: Any, issues: list[dict[str, str]] +) -> dict[str, Any]: + value = _mapping(raw, "input", issues) + _unknown_fields(value, _INPUT_FIELDS, "input", issues) + _missing_fields(value, _INPUT_FIELDS, "input", issues) + normalized: dict[str, Any] = {} + input_file = _safe_relative_path(value.get("file"), "input.file", issues) + if input_file is not None: + normalized["file"] = input_file + sheet = value.get("sheet") + if sheet is None or ( + isinstance(sheet, str) and sheet.strip() + ) or ( + isinstance(sheet, int) + and not isinstance(sheet, bool) + and 0 <= sheet <= 65_535 + ): + # Excel permits leading/trailing spaces in worksheet names. Validate + # against an all-whitespace name, but preserve the exact spelling. + normalized["sheet"] = sheet + else: + issues.append( + _issue( + "E306", + "error", + "sheet 必须是工作表名称、0–65535 的整数或 null。", + field="input.sheet", + ) + ) + layout = value.get("layout") + if not isinstance(layout, str) or layout not in _LAYOUTS: + issues.append( + _issue( + "E306", + "error", + "layout 必须是 row-per-sample、analyte-per-row 或 auto。", + field="input.layout", + ) + ) + else: + normalized["layout"] = layout + return normalized + + +def _validate_columns( + raw: Any, issues: list[dict[str, str]] +) -> tuple[dict[str, Any], set[str]]: + value = _mapping(raw, "columns", issues) + _unknown_fields(value, _COLUMN_FIELDS, "columns", issues) + _missing_fields(value, _COLUMN_FIELDS, "columns", issues) + normalized: dict[str, Any] = {} + + sample_id = _nonempty_string(value.get("sample_id"), "columns.sample_id", issues) + if sample_id is not None: + normalized["sample_id"] = sample_id + group = value.get("group") + if group is None: + normalized["group"] = None + else: + group_name = _nonempty_string(group, "columns.group", issues) + if group_name is not None: + normalized["group"] = group_name + + mapping = _mapping(value.get("mapping"), "columns.mapping", issues) + normalized_mapping: dict[str, str] = {} + source_names: dict[str, str] = {} + for canonical, source in mapping.items(): + field = f"columns.mapping.{canonical}" + if not isinstance(canonical, str) or ( + DEFAULT_ANALYTE_REGISTRY.get(canonical) is None + ): + issues.append( + _issue( + "E313", + "error", + "mapping 的键必须是 GeoSkills 精确标准分析物名称;方向为 canonical→source。", + field=field, + ) + ) + continue + source_name = _nonempty_string(source, field, issues) + if source_name is None: + continue + source_key = source_name.casefold() + if source_key in source_names: + issues.append( + _issue( + "E308", + "error", + "同一来源列不能映射到多个分析物:" + f"{source_names[source_key]} 与 {canonical}。", + field=field, + ) + ) + continue + source_names[source_key] = canonical + normalized_mapping[canonical] = source_name + if not mapping: + issues.append( + _issue( + "E313", + "error", + "columns.mapping 至少需要一个 canonical→source 映射。", + field="columns.mapping", + ) + ) + + metadata_sources = { + name.casefold() + for name in (sample_id, normalized.get("group")) + if isinstance(name, str) + } + overlap = metadata_sources.intersection(source_names) + if overlap: + issues.append( + _issue( + "E308", + "error", + "样品编号/分组列不能同时用作分析物来源列。", + field="columns.mapping", + ) + ) + + units = _mapping(value.get("units"), "columns.units", issues) + normalized_units: dict[str, str] = {} + allowed_unit_keys = {"major_oxides", "trace_elements", *normalized_mapping} + _unknown_fields(units, allowed_unit_keys, "columns.units", issues) + for unit_key, unit_value in units.items(): + if not isinstance(unit_key, str) or unit_key not in allowed_unit_keys: + continue + expected = ( + "wt%" + if unit_key == "major_oxides" + else "ppm" if unit_key == "trace_elements" else None + ) + definition = DEFAULT_ANALYTE_REGISTRY.get(unit_key) + if definition is not None: + expected = definition.required_unit + if unit_value != expected: + issues.append( + _issue( + "E313", + "error", + f"{unit_key} 当前工作流要求明确单位 {expected}。", + field=f"columns.units.{unit_key}", + ) + ) + continue + normalized_units[unit_key] = unit_value + + for canonical in normalized_mapping: + definition = DEFAULT_ANALYTE_REGISTRY.get(canonical) + assert definition is not None + default_key = ( + "major_oxides" + if definition.kind == "major_oxide" + else "trace_elements" + ) + if canonical not in normalized_units and default_key not in normalized_units: + issues.append( + _issue( + "E313", + "error", + f"{canonical} 缺少明确单位;请设置 {default_key} 默认值或单项覆盖。", + field=f"columns.units.{canonical}", + ) + ) + + normalized["mapping"] = dict(sorted(normalized_mapping.items())) + normalized["units"] = dict(sorted(normalized_units.items())) + return normalized, set(normalized_mapping) + + +def _validate_output( + raw: Any, issues: list[dict[str, str]] +) -> dict[str, Any]: + value = _mapping(raw, "output", issues) + _unknown_fields(value, _OUTPUT_FIELDS, "output", issues) + _missing_fields(value, _OUTPUT_FIELDS, "output", issues) + normalized: dict[str, Any] = {} + directory = _safe_relative_path( + value.get("directory"), "output.directory", issues, output=True + ) + if directory is not None: + normalized["directory"] = directory + profile = value.get("report_profile") + if not isinstance(profile, str) or profile not in _REPORT_PROFILES: + issues.append( + _issue( + "E306", + "error", + "report_profile 必须是 shareable 或 local-reproducible。", + field="output.report_profile", + ) + ) + else: + normalized["report_profile"] = profile + return normalized + + +def _validate_style( + raw: Any, + field: str, + issues: list[dict[str, str]], +) -> dict[str, int | float]: + value = _mapping(raw, field, issues) + _unknown_fields(value, _STYLE_FIELDS, field, issues) + normalized: dict[str, int | float] = {} + for name in ("width_mm", "height_mm"): + if name not in value: + continue + number = value[name] + try: + converted = float(number) + except (OverflowError, TypeError, ValueError): + converted = math.nan + if ( + isinstance(number, bool) + or not isinstance(number, (int, float)) + or not math.isfinite(converted) + or not 30 <= converted <= 500 + ): + issues.append( + _issue( + "E314", + "error", + f"{name} 必须是 30–500 mm 的有限数值。", + field=f"{field}.{name}", + ) + ) + else: + normalized[name] = converted + if "dpi" in value: + dpi = value["dpi"] + if isinstance(dpi, bool) or not isinstance(dpi, int) or not 72 <= dpi <= 1200: + issues.append( + _issue( + "E314", + "error", + "dpi 必须是 72–1200 的整数。", + field=f"{field}.dpi", + ) + ) + else: + normalized["dpi"] = dpi + return normalized + + +def _validate_presets( + raw: Any, issues: list[dict[str, str]] +) -> tuple[dict[str, Any], dict[str, str]]: + if raw is None: + return {}, {name.casefold(): name for name in BUILTIN_STYLE_PRESETS} + value = _mapping(raw, "presets", issues) + normalized: dict[str, Any] = {} + names = {name.casefold(): name for name in BUILTIN_STYLE_PRESETS} + for raw_name, raw_preset in value.items(): + name = _safe_windows_name( + raw_name, f"presets.{raw_name}", issues, allow_dot=True + ) + if name is None: + continue + key = name.casefold() + if key in names: + issues.append( + _issue( + "E308", + "error", + f"预设名称与已有名称重复:{names[key]}。", + field=f"presets.{name}", + ) + ) + continue + names[key] = name + preset = _mapping(raw_preset, f"presets.{name}", issues) + _unknown_fields(preset, _PRESET_FIELDS, f"presets.{name}", issues) + _missing_fields(preset, _PRESET_FIELDS, f"presets.{name}", issues) + extends = preset.get("extends") + if extends not in BUILTIN_STYLE_PRESETS: + issues.append( + _issue( + "E314", + "error", + "extends 必须指向 publication-double-column 或 review-preview。", + field=f"presets.{name}.extends", + ) + ) + style = _validate_style( + preset.get("style"), f"presets.{name}.style", issues + ) + normalized[name] = {"extends": extends, "style": style} + return normalized, names + + +def _groups( + raw: Any, + field: str, + issues: list[dict[str, str]], + *, + task_id: str, + has_group_column: bool, +) -> str | list[str] | None: + if raw == "all": + return "all" + if not isinstance(raw, list) or not raw: + issues.append( + _issue( + "E311", + "error", + "groups 必须是 'all' 或非空文本列表。", + field=field, + task_id=task_id, + ) + ) + return None + if not has_group_column: + issues.append( + _issue( + "E311", + "error", + "选择具体 groups 前必须设置 columns.group。", + field=field, + task_id=task_id, + ) + ) + normalized: list[str] = [] + seen: set[str] = set() + for index, item in enumerate(raw): + name = _nonempty_string( + item, f"{field}[{index}]", issues, task_id=task_id + ) + if name is None: + continue + key = name.casefold() + if key in seen: + issues.append( + _issue( + "E308", + "error", + "groups 不能包含 Windows 不区分大小写的重复名称。", + field=f"{field}[{index}]", + task_id=task_id, + ) + ) + continue + seen.add(key) + normalized.append(name) + return normalized + + +def _canonical( + raw: Any, + field: str, + issues: list[dict[str, str]], + mapped: set[str], + *, + task_id: str, + allowed: set[str] | frozenset[str] | None = None, +) -> str | None: + name = _nonempty_string(raw, field, issues, task_id=task_id) + if name is None: + return None + definition = DEFAULT_ANALYTE_REGISTRY.get(name) + if definition is None or (allowed is not None and name not in allowed): + issues.append( + _issue( + "E311", + "error", + "参数必须使用该图解支持的精确标准分析物名称。", + field=field, + task_id=task_id, + ) + ) + return None + if name not in mapped: + issues.append( + _issue( + "E313", + "error", + f"{name} 未在 columns.mapping 中映射到来源列。", + field=field, + task_id=task_id, + ) + ) + return name + + +def _elements( + raw: Any, + field: str, + issues: list[dict[str, str]], + mapped: set[str], + *, + task_id: str, + diagram: str, +) -> list[str]: + minimum = 3 if diagram == "ree" else 5 + allowed = set(REE_ORDER if diagram == "ree" else SPIDER_ELEMENT_ORDER) + mapped_for_elements = set(mapped) + if diagram == "spider": + mapped_for_elements.update( + element + for element, oxide in _SPIDER_OXIDE_ALTERNATIVES.items() + if oxide in mapped + ) + if not isinstance(raw, list): + issues.append( + _issue( + "E311", + "error", + f"elements 必须是至少 {minimum} 项的列表。", + field=field, + task_id=task_id, + ) + ) + return [] + normalized: list[str] = [] + seen: set[str] = set() + for index, item in enumerate(raw): + name = _canonical( + item, + f"{field}[{index}]", + issues, + mapped_for_elements, + task_id=task_id, + allowed=allowed, + ) + if name is None: + continue + if name in seen: + issues.append( + _issue( + "E308", + "error", + "elements 不能重复。", + field=f"{field}[{index}]", + task_id=task_id, + ) + ) + continue + seen.add(name) + normalized.append(name) + if len(raw) < minimum: + issues.append( + _issue( + "E311", + "error", + f"{diagram} 图至少需要 {minimum} 个元素。", + field=field, + task_id=task_id, + ) + ) + return normalized + + +def _validate_parameters( + raw: Any, + diagram: str, + task_id: str, + mapped: set[str], + has_group_column: bool, + issues: list[dict[str, str]], +) -> dict[str, Any]: + field = f"tasks[{task_id}].parameters" + value = _mapping(raw, field, issues) + _unknown_fields(value, _PARAMETER_FIELDS[diagram], field, issues, task_id=task_id) + _missing_fields(value, _PARAMETER_REQUIRED[diagram], field, issues, task_id=task_id) + normalized: dict[str, Any] = {} + if "groups" in value: + selected = _groups( + value["groups"], + f"{field}.groups", + issues, + task_id=task_id, + has_group_column=has_group_column, + ) + if selected is not None: + normalized["groups"] = selected + + if diagram in {"ree", "spider"}: + reference = value.get("reference") + if ( + not isinstance(reference, str) + or reference not in _REFERENCE_OPTIONS[diagram] + ): + issues.append( + _issue( + "E311", + "error", + f"{diagram} reference 不在已审核的固定选项中。", + field=f"{field}.reference", + task_id=task_id, + ) + ) + else: + normalized["reference"] = reference + normalized["elements"] = _elements( + value.get("elements"), + f"{field}.elements", + issues, + mapped, + task_id=task_id, + diagram=diagram, + ) + elif diagram == "harker": + x = _canonical( + value.get("x"), + f"{field}.x", + issues, + mapped, + task_id=task_id, + ) + if x is not None: + normalized["x"] = x + raw_y = value.get("y") + y_values = raw_y if isinstance(raw_y, list) else [raw_y] + normalized_y: list[str] = [] + for index, item in enumerate(y_values): + y_name = _canonical( + item, + f"{field}.y[{index}]", + issues, + mapped, + task_id=task_id, + ) + if y_name is not None: + if y_name in normalized_y: + issues.append( + _issue( + "E308", + "error", + "Harker y 变量不能重复。", + field=f"{field}.y[{index}]", + task_id=task_id, + ) + ) + else: + normalized_y.append(y_name) + if not normalized_y: + issues.append( + _issue( + "E311", + "error", + "Harker y 至少需要一个非空标准分析物名称。", + field=f"{field}.y", + task_id=task_id, + ) + ) + if x is not None and x in normalized_y: + issues.append( + _issue( + "E311", + "error", + "Harker x 与 y 不能使用同一分析物。", + field=f"{field}.y", + task_id=task_id, + ) + ) + normalized["y"] = normalized_y + else: + basis = value.get("composition_basis") + if ( + not isinstance(basis, str) + or basis not in {"anhydrous-normalized", "as-reported"} + ): + issues.append( + _issue( + "E311", + "error", + "TAS composition_basis 必须明确为 anhydrous-normalized 或 as-reported。", + field=f"{field}.composition_basis", + task_id=task_id, + ) + ) + else: + normalized["composition_basis"] = basis + for required_analyte in ("SiO2", "Na2O", "K2O"): + if required_analyte not in mapped: + issues.append( + _issue( + "E313", + "error", + f"TAS 需要在 columns.mapping 中映射 {required_analyte}。", + field="columns.mapping", + task_id=task_id, + ) + ) + return normalized + + +def _validate_tasks( + raw: Any, + mapped: set[str], + has_group_column: bool, + preset_names: Mapping[str, str], + issues: list[dict[str, str]], +) -> list[dict[str, Any]]: + if not isinstance(raw, list): + issues.append( + _issue("E306", "error", "tasks 必须是列表。", field="tasks") + ) + return [] + if not raw: + issues.append( + _issue("E309", "error", "tasks 至少需要一个任务。", field="tasks") + ) + if len(raw) > MAX_TASKS: + issues.append( + _issue( + "E309", + "error", + f"单个配方最多允许 {MAX_TASKS} 个任务。", + field="tasks", + ) + ) + + normalized: list[dict[str, Any]] = [] + task_names: dict[str, str] = {} + stems: dict[str, str] = {} + for index, raw_task in enumerate(raw[:MAX_TASKS]): + field = f"tasks[{index}]" + value = _mapping(raw_task, field, issues) + _unknown_fields(value, _TASK_FIELDS, field, issues) + _missing_fields(value, _TASK_FIELDS, field, issues) + task_id = _safe_windows_name(value.get("id"), f"{field}.id", issues) + task_label = task_id or f"#{index + 1}" + if task_id is not None: + key = task_id.casefold() + if key in _RESERVED_TASK_IDS: + issues.append( + _issue( + "E307", + "error", + "任务 ID 与 GeoSkills 内部工作目录名称冲突。", + field=f"{field}.id", + task_id=task_id, + ) + ) + if key in task_names: + issues.append( + _issue( + "E308", + "error", + f"任务 ID 与 {task_names[key]!r} 在 Windows 下重复。", + field=f"{field}.id", + task_id=task_id, + ) + ) + else: + task_names[key] = task_id + + stem = _safe_windows_name( + value.get("stem"), f"{field}.stem", issues, task_id=task_label + ) + if stem is not None: + key = stem.casefold() + if key in stems: + issues.append( + _issue( + "E308", + "error", + f"输出 stem 与 {stems[key]!r} 在 Windows 下重复。", + field=f"{field}.stem", + task_id=task_label, + ) + ) + else: + stems[key] = stem + + diagram = value.get("diagram") + if not isinstance(diagram, str) or diagram not in DIAGRAMS: + issues.append( + _issue( + "E310", + "error", + "diagram 只能是 ree、spider、harker 或 tas。", + field=f"{field}.diagram", + task_id=task_label, + ) + ) + diagram = None + + preset = value.get("preset") + preset_name = None + if isinstance(preset, str): + preset_name = preset_names.get(preset.casefold()) + if preset_name is None: + issues.append( + _issue( + "E314", + "error", + "preset 必须引用固定预设或本配方中的自定义预设。", + field=f"{field}.preset", + task_id=task_label, + ) + ) + + parameters: dict[str, Any] = {} + confirmations: dict[str, bool] = {} + if diagram is not None: + parameters = _validate_parameters( + value.get("parameters"), + diagram, + task_label, + mapped, + has_group_column, + issues, + ) + required = _TAS_CONFIRMATIONS if diagram == "tas" else () + optional = ( + ("provisional_classification_accepted",) + if diagram == "tas" + else () + ) + confirmations = _confirmation_map( + value.get("confirmations"), + required, + f"{field}.confirmations", + issues, + task_id=task_label, + optional=optional, + ) + if ( + diagram == "tas" + and parameters.get("composition_basis") == "as-reported" + ): + accepted = confirmations.get( + "provisional_classification_accepted" + ) + if accepted is not True: + if ( + "provisional_classification_accepted" + not in confirmations + ): + issues.append( + _issue( + "E305", + "error", + "as-reported TAS 必须明确提供 " + "provisional_classification_accepted。", + field=( + f"{field}.confirmations." + "provisional_classification_accepted" + ), + task_id=task_label, + ) + ) + normalized.append( + { + "id": task_id, + "diagram": diagram, + "stem": stem, + "preset": preset_name, + "parameters": parameters, + "confirmations": confirmations, + } + ) + return normalized + + +def validate_recipe(document: Any) -> dict[str, Any]: + """Validate one parsed YAML document and return a JSON-ready result.""" + + issues: list[dict[str, str]] = [] + if not isinstance(document, Mapping): + issues.append( + _issue( + "E303", + "error", + "配方顶层必须是键值映射。", + field="$", + ) + ) + return _result(recipe=None, issues=issues) + root = dict(document) + _unknown_fields(root, _ROOT_FIELDS, "$", issues) + _missing_fields(root, _REQUIRED_ROOT_FIELDS, "$", issues) + if root.get("schema_version") != RECIPE_SCHEMA_VERSION: + issues.append( + _issue( + "E306", + "error", + f"schema_version 必须是 {RECIPE_SCHEMA_VERSION!r}。", + field="schema_version", + ) + ) + + normalized_input = _validate_input(root.get("input"), issues) + normalized_columns, mapped = _validate_columns(root.get("columns"), issues) + normalized_output = _validate_output(root.get("output"), issues) + normalized_presets, preset_names = _validate_presets( + root.get("presets"), issues + ) + normalized_confirmations = _confirmation_map( + root.get("confirmations"), + _TOP_CONFIRMATIONS, + "confirmations", + issues, + ) + normalized_tasks = _validate_tasks( + root.get("tasks"), + mapped, + normalized_columns.get("group") is not None, + preset_names, + issues, + ) + recipe = { + "schema_version": RECIPE_SCHEMA_VERSION, + "input": normalized_input, + "columns": normalized_columns, + "output": normalized_output, + "presets": normalized_presets, + "confirmations": normalized_confirmations, + "tasks": normalized_tasks, + } + return _result(recipe=recipe, issues=issues) + + +def load_recipe(path: str | Path) -> dict[str, Any]: + """Safely read and validate one UTF-8, single-document YAML recipe.""" + + recipe_path = Path(path) + issues: list[dict[str, str]] = [] + try: + size = recipe_path.stat().st_size + except OSError: + issues.append( + _issue( + "E300", + "error", + "无法读取配方文件。", + field="$", + ) + ) + return _result(recipe=None, issues=issues) + if size > MAX_RECIPE_BYTES: + issues.append( + _issue( + "E300", + "error", + f"配方文件不能超过 {MAX_RECIPE_BYTES} 字节。", + field="$", + ) + ) + return _result(recipe=None, issues=issues) + try: + text = recipe_path.read_text(encoding="utf-8-sig") + except (OSError, UnicodeError): + issues.append( + _issue( + "E301", + "error", + "配方必须是可读取的 UTF-8 YAML 文件。", + field="$", + ) + ) + return _result(recipe=None, issues=issues) + try: + document = yaml.safe_load(text) + except yaml.composer.ComposerError as exc: + multiple_documents = "expected a single document" in str(exc) + issues.append( + _issue( + "E302" if multiple_documents else "E301", + "error", + ( + "每个配方文件只能包含一个 YAML 文档。" + if multiple_documents + else "YAML 文档结构无法解析。" + ), + field="$", + ) + ) + return _result(recipe=None, issues=issues) + except (yaml.YAMLError, OverflowError, RecursionError, ValueError): + issues.append( + _issue( + "E301", + "error", + "YAML 语法无法解析。", + field="$", + ) + ) + return _result(recipe=None, issues=issues) + return validate_recipe(document) + + +def resolve_recipe_path(recipe_path: str | Path, relative_value: str) -> Path: + """Resolve a previously validated recipe member against its recipe file.""" + + issues: list[dict[str, str]] = [] + safe = _safe_relative_path(relative_value, "path", issues) + if safe is None: + raise ValueError(issues[0]["message"]) + recipe_root = Path(recipe_path).resolve().parent + resolved = (recipe_root / Path(safe)).resolve() + try: + resolved.relative_to(recipe_root) + except ValueError as exc: + raise ValueError( + "Resolved recipe path escapes the recipe directory." + ) from exc + return resolved + + +__all__ = [ + "MAX_RECIPE_BYTES", + "MAX_TASKS", + "RECIPE_SCHEMA_VERSION", + "load_recipe", + "resolve_recipe_path", + "validate_recipe", +] diff --git a/skills/geoskills/scripts/geoskills_core/registry.py b/skills/geoskills/scripts/geoskills_core/registry.py new file mode 100644 index 0000000..f508583 --- /dev/null +++ b/skills/geoskills/scripts/geoskills_core/registry.py @@ -0,0 +1,323 @@ +"""Static diagram registry for the GeoSkills v0.4 workflow. + +The registry deliberately contains only the four scientifically reviewed +v0.3 diagram families. Handler modules are imported only when a caller asks +to resolve a handler, so listing capabilities remains lightweight and cannot +trigger plotting side effects. +""" + +from __future__ import annotations + +import importlib +from dataclasses import dataclass +from types import MappingProxyType +from typing import Any, Mapping + + +DIAGRAM_API_VERSION = "geoskills.diagram/v1" +BUILTIN_STYLE_PRESETS = ( + "publication-double-column", + "review-preview", +) + + +class RegistryError(ValueError): + """Raised when a diagram or handler is not part of the public registry.""" + + +@dataclass(frozen=True) +class HandlerRef: + """An import reference which stays dormant until ``resolve`` is called.""" + + module: str + attribute: str + + @property + def reference(self) -> str: + return f"{self.module}:{self.attribute}" + + def resolve(self) -> Any: + try: + module = importlib.import_module(self.module) + except ImportError as exc: + raise RegistryError( + f"Handler module {self.module!r} could not be imported." + ) from exc + try: + handler = getattr(module, self.attribute) + except AttributeError as exc: + raise RegistryError( + f"Handler {self.reference!r} is not available." + ) from exc + if not callable(handler): + raise RegistryError( + f"Handler {self.reference!r} is not callable." + ) + return handler + + +@dataclass(frozen=True) +class DiagramSpec: + """One immutable, reviewed diagram capability.""" + + id: str + api_version: str + display_name_zh: str + display_name_en: str + operation: str + input_profile: str + adapter: str + inspector_handler: HandlerRef + runner_handler: HandlerRef + scientific_parameter_schema: Mapping[str, Any] + style_parameter_schema: Mapping[str, Any] + required_confirmations: tuple[str, ...] + required_assets: tuple[str, ...] + output_contract: Mapping[str, Any] + privacy_contract: Mapping[str, Any] + adapter_version: str + + def to_dict(self) -> dict[str, Any]: + """Return a deterministic, JSON-serializable registry record.""" + + return { + "id": self.id, + "api_version": self.api_version, + "display_name_zh": self.display_name_zh, + "display_name_en": self.display_name_en, + "operation": self.operation, + "input_profile": self.input_profile, + "adapter": self.adapter, + "inspector_handler": self.inspector_handler.reference, + "runner_handler": self.runner_handler.reference, + "scientific_parameter_schema": _plain( + self.scientific_parameter_schema + ), + "style_parameter_schema": _plain(self.style_parameter_schema), + "required_confirmations": list(self.required_confirmations), + "required_assets": list(self.required_assets), + "output_contract": _plain(self.output_contract), + "privacy_contract": _plain(self.privacy_contract), + "adapter_version": self.adapter_version, + } + + +def _plain(value: Any) -> Any: + if isinstance(value, Mapping): + return {str(key): _plain(item) for key, item in value.items()} + if isinstance(value, tuple): + return [_plain(item) for item in value] + if isinstance(value, list): + return [_plain(item) for item in value] + return value + + +def _freeze(value: Any) -> Any: + """Recursively freeze registry metadata exposed to callers.""" + + if isinstance(value, Mapping): + return MappingProxyType( + {str(key): _freeze(item) for key, item in value.items()} + ) + if isinstance(value, (list, tuple)): + return tuple(_freeze(item) for item in value) + return value + + +_STYLE_PARAMETER_SCHEMA = _freeze( + { + "allowed": ("width_mm", "height_mm", "dpi"), + "width_mm": {"type": "number", "minimum": 30, "maximum": 500}, + "height_mm": {"type": "number", "minimum": 30, "maximum": 500}, + "dpi": {"type": "integer", "minimum": 72, "maximum": 1200}, + "additional_properties": False, + } +) + +_OUTPUT_CONTRACT = _freeze( + { + "atomic_bundle": True, + "figure_formats": ("svg", "pdf", "tiff", "png"), + "includes_source_data": True, + "includes_json_report": True, + "includes_qa_summary": True, + } +) + +_PRIVACY_CONTRACT = _freeze( + { + "processing": "local-only", + "shareable_report_absolute_paths": False, + "shareable_report_source_values": False, + } +) + + +def _spec( + *, + diagram_id: str, + display_name_zh: str, + display_name_en: str, + operation: str, + input_profile: str, + inspector: tuple[str, str], + runner: tuple[str, str], + scientific_schema: Mapping[str, Any], + confirmations: tuple[str, ...] = (), + assets: tuple[str, ...] = (), +) -> DiagramSpec: + return DiagramSpec( + id=diagram_id, + api_version=DIAGRAM_API_VERSION, + display_name_zh=display_name_zh, + display_name_en=display_name_en, + operation=operation, + input_profile=input_profile, + adapter="legacy-v0.3-function", + inspector_handler=HandlerRef(*inspector), + runner_handler=HandlerRef(*runner), + scientific_parameter_schema=_freeze(scientific_schema), + style_parameter_schema=_STYLE_PARAMETER_SCHEMA, + required_confirmations=confirmations, + required_assets=assets, + output_contract=_OUTPUT_CONTRACT, + privacy_contract=_PRIVACY_CONTRACT, + adapter_version="0.3.0", + ) + + +_DIAGRAMS = { + "ree": _spec( + diagram_id="ree", + display_name_zh="稀土元素配分图", + display_name_en="REE pattern", + operation="ree_pattern_plot", + input_profile="ree-ppm", + inspector=("inspect_data", "inspect_path"), + runner=("plot_ree", "plot_path"), + scientific_schema={ + "required": ("reference", "elements"), + "allowed": ("reference", "elements", "groups"), + "reference": {"enum": ("chondrite-sm89",)}, + "elements": {"type": "array", "minimum_items": 3}, + "groups": {"type": "all-or-array"}, + "additional_properties": False, + }, + assets=("assets/normalization/chondrite-sm89.json",), + ), + "spider": _spec( + diagram_id="spider", + display_name_zh="微量元素蛛网图", + display_name_en="Trace-element spider diagram", + operation="trace_element_spider_plot", + input_profile="trace-elements-ppm", + inspector=("inspect_spider_data", "inspect_spider_path"), + runner=("plot_spider", "plot_spider_path"), + scientific_schema={ + "required": ("reference", "elements"), + "allowed": ("reference", "elements", "groups"), + "reference": { + "enum": ("pm-sm89", "pm-sm89-modified", "nmorb-sm89") + }, + "elements": {"type": "array", "minimum_items": 5}, + "groups": {"type": "all-or-array"}, + "additional_properties": False, + }, + assets=( + "assets/normalization/primitive-mantle-sm89.json", + "assets/normalization/primitive-mantle-modified-sm89.json", + "assets/normalization/nmorb-sm89.json", + ), + ), + "harker": _spec( + diagram_id="harker", + display_name_zh="哈克图解", + display_name_en="Harker diagram", + operation="harker_plot", + input_profile="mapped-geochemistry", + inspector=("inspect_major_data", "inspect_major_path"), + runner=("plot_harker", "plot_harker_path"), + scientific_schema={ + "required": ("x", "y"), + "allowed": ("x", "y", "groups"), + "x": {"type": "canonical-analyte"}, + "y": {"type": "canonical-analyte-or-array"}, + "groups": {"type": "all-or-array"}, + "additional_properties": False, + }, + ), + "tas": _spec( + diagram_id="tas", + display_name_zh="火山岩 TAS 分类图", + display_name_en="Volcanic TAS classification", + operation="tas_plot", + input_profile="major-oxides-wt-percent", + inspector=("inspect_major_data", "inspect_major_path"), + runner=("plot_tas", "plot_tas_path"), + scientific_schema={ + "required": ("composition_basis",), + "allowed": ("composition_basis", "groups"), + "composition_basis": { + "enum": ("anhydrous-normalized", "as-reported") + }, + "groups": {"type": "all-or-array"}, + "additional_properties": False, + }, + confirmations=( + "volcanic_samples", + "composition_basis_reviewed", + ), + assets=("assets/classification/tas-lemaitre-2002.json",), + ), +} + +DIAGRAMS: Mapping[str, DiagramSpec] = MappingProxyType(_DIAGRAMS) + + +def diagram_ids() -> tuple[str, ...]: + """Return the stable public diagram order.""" + + return tuple(DIAGRAMS) + + +def get_diagram(diagram_id: str) -> DiagramSpec: + """Return one reviewed diagram specification by exact ID.""" + + try: + return DIAGRAMS[diagram_id] + except (KeyError, TypeError) as exc: + available = ", ".join(diagram_ids()) + raise RegistryError( + f"Unknown diagram {diagram_id!r}; available: {available}." + ) from exc + + +def resolve_handler(diagram_id: str, kind: str) -> Any: + """Resolve an inspector or runner only when execution is requested.""" + + spec = get_diagram(diagram_id) + if kind == "inspector": + return spec.inspector_handler.resolve() + if kind == "runner": + return spec.runner_handler.resolve() + raise RegistryError("Handler kind must be 'inspector' or 'runner'.") + + +def registry_snapshot() -> list[dict[str, Any]]: + """Return all registry entries in stable, JSON-ready order.""" + + return [DIAGRAMS[diagram_id].to_dict() for diagram_id in diagram_ids()] + + +__all__ = [ + "BUILTIN_STYLE_PRESETS", + "DIAGRAMS", + "DIAGRAM_API_VERSION", + "DiagramSpec", + "HandlerRef", + "RegistryError", + "diagram_ids", + "get_diagram", + "registry_snapshot", + "resolve_handler", +] diff --git a/skills/geoskills/scripts/geoskills_core/reports.py b/skills/geoskills/scripts/geoskills_core/reports.py new file mode 100644 index 0000000..3827e07 --- /dev/null +++ b/skills/geoskills/scripts/geoskills_core/reports.py @@ -0,0 +1,406 @@ +"""Versioned, privacy-aware machine and human QA reports for GeoSkills.""" + +from __future__ import annotations + +import json +import re +from copy import deepcopy +from pathlib import Path, PurePosixPath, PureWindowsPath +from typing import Any, Mapping, Sequence + +from .export import AtomicBundle + + +class ReportError(ValueError): + """Raised when a report would be invalid or unsafe to share.""" + + +REPORT_SCHEMA_NAME = "geoskills-output-report" +REPORT_SCHEMA_VERSION = "1.0.0" +REPORT_STATUSES = frozenset( + {"ready", "needs_input", "review", "blocked", "error"} +) +ISSUE_SEVERITIES = frozenset({"info", "warning", "review", "error"}) + +# This is intentionally available without adding a jsonschema dependency. +REPORT_JSON_SCHEMA: Mapping[str, Any] = { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": f"urn:geoskills:schema:{REPORT_SCHEMA_NAME}:{REPORT_SCHEMA_VERSION}", + "title": "GeoSkills output report", + "type": "object", + "required": [ + "schema", + "status", + "operation", + "review_required", + "source", + "outputs", + "issues", + "next_actions", + "privacy", + ], + "properties": { + "schema": { + "type": "object", + "const": { + "name": REPORT_SCHEMA_NAME, + "version": REPORT_SCHEMA_VERSION, + }, + }, + "status": {"enum": sorted(REPORT_STATUSES)}, + "operation": {"type": "string", "minLength": 1}, + "review_required": {"type": "boolean"}, + "source": {"type": "object"}, + "outputs": {"type": "array"}, + "issues": {"type": "array"}, + "next_actions": { + "type": "array", + "items": {"type": "string"}, + }, + "privacy": {"type": "object"}, + }, +} + +_SHAREABLE_SOURCE_KEYS = frozenset( + { + "filename", + "file_sha256", + "size_bytes", + "format", + "sheet", + "layout", + "row_count", + "column_count", + "file_sha256", + "size_bytes", + } +) +_FORBIDDEN_SHAREABLE_KEYS = frozenset( + { + "path", + "absolute_path", + "resolved_path", + "input_path", + "output_path", + "report_file", + "raw_data", + "data", + "values", + "records", + "preview", + "examples", + "samples", + "sample_ids", + "source_values", + "data_min", + "data_max", + "raw_min", + "raw_max", + "observed_min", + "observed_max", + "source_min", + "source_max", + } +) +_WINDOWS_DRIVE = re.compile(r"^[A-Za-z]:[\\/]") + + +def _is_absolute_path_text(value: str) -> bool: + text = value.strip() + if not text: + return False + return ( + PureWindowsPath(text).is_absolute() + or PurePosixPath(text).is_absolute() + or bool(_WINDOWS_DRIVE.match(text)) + ) + + +def _assert_shareable(value: Any, trail: str = "report") -> None: + """Reject common path and raw-data leaks in a shareable report.""" + + if isinstance(value, Mapping): + for key, item in value.items(): + key_text = str(key) + if key_text.casefold() in _FORBIDDEN_SHAREABLE_KEYS: + raise ReportError( + f"可分享报告不能包含字段 {trail}.{key_text}。" + ) + _assert_shareable(item, f"{trail}.{key_text}") + elif isinstance(value, (list, tuple)): + for index, item in enumerate(value): + _assert_shareable(item, f"{trail}[{index}]") + elif isinstance(value, str) and _is_absolute_path_text(value): + raise ReportError(f"可分享报告不能包含绝对路径:{trail}。") + + +def _normalise_source( + source: Mapping[str, Any] | None, + *, + shareable: bool, +) -> dict[str, Any]: + result = dict(source or {}) + if shareable: + unsupported = sorted(set(result) - _SHAREABLE_SOURCE_KEYS) + if unsupported: + raise ReportError( + "可分享报告的 source 包含未经允许的字段:" + + ", ".join(unsupported) + + "。" + ) + _assert_shareable(result, "report.source") + return result + + +def _normalise_outputs( + outputs: Sequence[Mapping[str, Any]], + *, + shareable: bool, +) -> list[dict[str, Any]]: + records = [dict(item) for item in outputs] + for index, record in enumerate(records): + required = {"filename", "format", "bytes", "sha256"} + missing = sorted(required - set(record)) + if missing: + raise ReportError( + f"outputs[{index}] 缺少字段:" + ", ".join(missing) + "。" + ) + if shareable: + _assert_shareable(record, f"report.outputs[{index}]") + return records + + +def _normalise_issues( + issues: Sequence[Mapping[str, Any]], + *, + shareable: bool, +) -> list[dict[str, Any]]: + results: list[dict[str, Any]] = [] + for index, value in enumerate(issues): + item = dict(value) + missing = sorted({"code", "severity", "message"} - set(item)) + if missing: + raise ReportError( + f"issues[{index}] 缺少字段:" + ", ".join(missing) + "。" + ) + if item["severity"] not in ISSUE_SEVERITIES: + raise ReportError( + f"issues[{index}] 的 severity 无效:{item['severity']}。" + ) + if shareable: + _assert_shareable(item, f"report.issues[{index}]") + results.append(item) + return results + + +def build_report( + *, + operation: str, + status: str, + source: Mapping[str, Any] | None = None, + outputs: Sequence[Mapping[str, Any]] = (), + issues: Sequence[Mapping[str, Any]] = (), + qa: Mapping[str, Any] | None = None, + next_actions: Sequence[str] = (), + review_required: bool = False, + details: Mapping[str, Any] | None = None, + shareable: bool = True, +) -> dict[str, Any]: + """Build one deterministic report that follows the current schema. + + Shareable mode is the default. It rejects absolute paths, raw-data-like + fields, and source metadata beyond a small allow-list. Local diagnostics + must opt out explicitly with ``shareable=False``. + """ + + clean_operation = str(operation).strip() + if not clean_operation: + raise ReportError("operation 不能为空。") + if status not in REPORT_STATUSES: + raise ReportError( + f"status 无效:{status}。可用值:{', '.join(sorted(REPORT_STATUSES))}。" + ) + + action_list = [str(action).strip() for action in next_actions] + if any(not action for action in action_list): + raise ReportError("next_actions 不能包含空白项目。") + + report: dict[str, Any] = { + "schema": { + "name": REPORT_SCHEMA_NAME, + "version": REPORT_SCHEMA_VERSION, + }, + "status": status, + "operation": clean_operation, + "review_required": bool(review_required), + "source": _normalise_source(source, shareable=shareable), + "outputs": _normalise_outputs(outputs, shareable=shareable), + "issues": _normalise_issues(issues, shareable=shareable), + "qa": dict(qa or {}), + "next_actions": action_list, + "privacy": { + "mode": "shareable" if shareable else "local-diagnostic", + "absolute_paths_included": not shareable, + "source_values_included": not shareable, + }, + } + if details: + report["details"] = dict(details) + if shareable: + _assert_shareable(report) + validate_report(report, require_shareable=shareable) + return report + + +def validate_report( + report: Mapping[str, Any], + *, + require_shareable: bool = False, +) -> None: + """Perform lightweight validation without an optional schema library.""" + + required = set(REPORT_JSON_SCHEMA["required"]) + missing = sorted(required - set(report)) + if missing: + raise ReportError("报告缺少字段:" + ", ".join(missing) + "。") + if report.get("schema") != { + "name": REPORT_SCHEMA_NAME, + "version": REPORT_SCHEMA_VERSION, + }: + raise ReportError("报告 schema 名称或版本不受支持。") + if report.get("status") not in REPORT_STATUSES: + raise ReportError("报告 status 无效。") + if not isinstance(report.get("review_required"), bool): + raise ReportError("review_required 必须是布尔值。") + if require_shareable: + privacy = report.get("privacy", {}) + if privacy.get("mode") != "shareable": + raise ReportError("当前操作要求可分享报告。") + _assert_shareable(report) + + +def _markdown_text(value: Any) -> str: + return ( + str(value) + .replace("\\", "\\\\") + .replace("|", "\\|") + .replace("\r", " ") + .replace("\n", " ") + .strip() + ) + + +def _status_label(status: str) -> str: + return { + "ready": "已完成", + "needs_input": "需要补充信息", + "review": "等待审核", + "blocked": "已阻止输出", + "error": "运行失败", + }[status] + + +def render_qa_markdown(report: Mapping[str, Any]) -> str: + """Render a concise Chinese QA summary from a shareable report.""" + + validate_report(report, require_shareable=True) + lines = [ + "# GeoSkills 图件质量检查摘要", + "", + f"- 任务:{_markdown_text(report['operation'])}", + f"- 状态:{_status_label(str(report['status']))}", + f"- 需要人工审核:{'是' if report['review_required'] else '否'}", + ( + f"- 报告格式:{REPORT_SCHEMA_NAME} " + f"{REPORT_SCHEMA_VERSION}" + ), + "", + "## 输出文件", + "", + ] + + outputs = list(report["outputs"]) + if outputs: + lines.extend( + [ + "| 文件 | 格式 | 大小(字节) | SHA-256 |", + "|---|---:|---:|---|", + ] + ) + for record in outputs: + digest = _markdown_text(record["sha256"]) + lines.append( + "| " + + " | ".join( + [ + _markdown_text(record["filename"]), + _markdown_text(record["format"]), + _markdown_text(record["bytes"]), + digest, + ] + ) + + " |" + ) + else: + lines.append("- 尚未生成输出文件。") + + lines.extend(["", "## 检查结果", ""]) + issues = list(report["issues"]) + if not issues: + lines.append("- ✅ 未发现阻断问题。") + else: + icons = { + "info": "ℹ️", + "warning": "⚠️", + "review": "🔎", + "error": "❌", + } + for issue in issues: + lines.append( + f"- {icons[issue['severity']]} " + f"[{_markdown_text(issue['code'])}] " + f"{_markdown_text(issue['message'])}" + ) + + lines.extend(["", "## 下一步", ""]) + actions = list(report["next_actions"]) + if actions: + lines.extend(f"- {_markdown_text(action)}" for action in actions) + else: + lines.append("- 无。") + + lines.extend( + [ + "", + "## 隐私说明", + "", + "- 本摘要采用可分享模式,仅记录文件名、大小和校验值。", + "- 本摘要不包含本地绝对路径或源数据值。", + "", + ] + ) + return "\n".join(lines) + + +def write_report_files( + bundle: AtomicBundle, + stem: str, + report: Mapping[str, Any], +) -> tuple[Path, Path]: + """Write versioned JSON and Chinese Markdown reports into a staging bundle.""" + + validate_report(report, require_shareable=True) + clean_stem = str(stem).strip() + if not clean_stem or Path(clean_stem).name != clean_stem or Path( + clean_stem + ).suffix: + raise ReportError("报告名称必须是不含路径和扩展名的文件名。") + + json_path = bundle.stage_path(f"{clean_stem}.report.json") + markdown_path = bundle.stage_path(f"{clean_stem}.qa.md") + json_path.write_text( + json.dumps(deepcopy(dict(report)), ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + markdown_path.write_text(render_qa_markdown(report), encoding="utf-8") + return json_path, markdown_path diff --git a/skills/geoskills/scripts/geoskills_core/validation.py b/skills/geoskills/scripts/geoskills_core/validation.py new file mode 100644 index 0000000..27623f6 --- /dev/null +++ b/skills/geoskills/scripts/geoskills_core/validation.py @@ -0,0 +1,299 @@ +"""Reusable validation issues and explicit geochemical column mappings.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from enum import Enum +from typing import Any, Iterable, Mapping + +import pandas as pd + +from .analytes import ( + DEFAULT_ANALYTE_REGISTRY, + AnalyteRegistry, + normalize_unit, +) + + +class Severity(str, Enum): + """Stable issue severities used in JSON reports.""" + + ERROR = "error" + WARNING = "warning" + INFO = "info" + + +@dataclass(frozen=True) +class Issue: + """One machine-readable problem or review notice.""" + + code: str + severity: Severity | str + message: str + details: Mapping[str, Any] = field(default_factory=dict) + + def __post_init__(self) -> None: + object.__setattr__(self, "severity", Severity(self.severity)) + object.__setattr__(self, "details", dict(self.details)) + + @property + def blocks(self) -> bool: + return self.severity is Severity.ERROR + + def to_dict(self) -> dict[str, Any]: + result: dict[str, Any] = { + "code": self.code, + "severity": self.severity.value, + "message": self.message, + } + if self.details: + result["details"] = dict(self.details) + return result + + +def issue( + code: str, + severity: Severity | str, + message: str, + **details: Any, +) -> Issue: + """Build an issue with the same compact call style as legacy inspectors.""" + return Issue(code, severity, message, details) + + +def status_from_issues( + issues: Iterable[Issue], + *, + ready: str = "ready", + blocked: str = "needs_review", +) -> str: + return blocked if any(item.blocks for item in issues) else ready + + +@dataclass(frozen=True) +class ColumnMapping: + """One explicit source-column to canonical-analyte assignment.""" + + source_column: str + canonical_analyte: str + unit: str + origin: str = "explicit" + + def __post_init__(self) -> None: + object.__setattr__(self, "source_column", str(self.source_column)) + object.__setattr__( + self, + "canonical_analyte", + str(self.canonical_analyte), + ) + object.__setattr__(self, "unit", normalize_unit(self.unit)) + object.__setattr__(self, "origin", str(self.origin)) + + def to_dict(self) -> dict[str, str]: + return { + "source_column": self.source_column, + "canonical_analyte": self.canonical_analyte, + "unit": self.unit, + "origin": self.origin, + } + + +def automatic_column_mappings( + columns: Iterable[object], + *, + registry: AnalyteRegistry = DEFAULT_ANALYTE_REGISTRY, +) -> list[ColumnMapping]: + """Map only exact registered headers; unknown columns remain untouched.""" + mappings: list[ColumnMapping] = [] + for column in columns: + matched = registry.match(column) + if matched is None: + continue + mappings.append( + ColumnMapping( + source_column=str(column), + canonical_analyte=matched.canonical, + unit=matched.explicit_unit, + origin="header", + ) + ) + return mappings + + +def validate_column_mappings( + mappings: Iterable[ColumnMapping], + *, + available_columns: Iterable[object] | None = None, + registry: AnalyteRegistry = DEFAULT_ANALYTE_REGISTRY, +) -> list[Issue]: + """Reject ambiguous, duplicated, missing, or unitless mappings.""" + items = list(mappings) + issues: list[Issue] = [] + available = ( + None + if available_columns is None + else {str(column) for column in available_columns} + ) + + source_counts: dict[str, int] = {} + analyte_counts: dict[str, int] = {} + for mapping in items: + source_counts[mapping.source_column] = ( + source_counts.get(mapping.source_column, 0) + 1 + ) + analyte_counts[mapping.canonical_analyte] = ( + analyte_counts.get(mapping.canonical_analyte, 0) + 1 + ) + + for source, count in source_counts.items(): + if count > 1: + issues.append( + issue( + "E221", + Severity.ERROR, + "One source column cannot be assigned more than once.", + source_column=source, + assignment_count=count, + ) + ) + for analyte, count in analyte_counts.items(): + if count > 1: + issues.append( + issue( + "E222", + Severity.ERROR, + "Multiple source columns map to the same analyte.", + canonical_analyte=analyte, + source_columns=[ + item.source_column + for item in items + if item.canonical_analyte == analyte + ], + ) + ) + + for mapping in items: + if available is not None and mapping.source_column not in available: + issues.append( + issue( + "E220", + Severity.ERROR, + "Mapped source column is not present in the table.", + source_column=mapping.source_column, + ) + ) + definition = registry.get(mapping.canonical_analyte) + if definition is None: + issues.append( + issue( + "E223", + Severity.ERROR, + "Canonical analyte is not registered.", + source_column=mapping.source_column, + canonical_analyte=mapping.canonical_analyte, + ) + ) + continue + normalized_unit = normalize_unit(mapping.unit) + if normalized_unit == "unknown": + issues.append( + issue( + "E224", + Severity.ERROR, + "The analyte unit must be stated explicitly.", + source_column=mapping.source_column, + canonical_analyte=mapping.canonical_analyte, + required_unit=definition.required_unit, + ) + ) + elif normalized_unit != definition.required_unit: + issues.append( + issue( + "E225", + Severity.ERROR, + "The stated unit does not match the analyte data contract.", + source_column=mapping.source_column, + canonical_analyte=mapping.canonical_analyte, + stated_unit=normalized_unit, + required_unit=definition.required_unit, + ) + ) + return issues + + +def inspect_column_mappings( + columns: Iterable[object], + *, + mappings: Iterable[ColumnMapping] | None = None, + registry: AnalyteRegistry = DEFAULT_ANALYTE_REGISTRY, +) -> tuple[list[ColumnMapping], list[Issue]]: + """Create/validate mappings for a future guided column-mapping screen.""" + column_list = list(columns) + selected = ( + automatic_column_mappings(column_list, registry=registry) + if mappings is None + else list(mappings) + ) + return selected, validate_column_mappings( + selected, + available_columns=column_list, + registry=registry, + ) + + +def validate_table_structure(frame: pd.DataFrame) -> list[Issue]: + """Check only table-level properties shared by all diagram families.""" + issues: list[Issue] = [] + if frame.empty: + issues.append( + issue("E201", Severity.ERROR, "The input table contains no data rows.") + ) + column_names = [str(column) for column in frame.columns] + duplicates = sorted( + { + name + for name in column_names + if column_names.count(name) > 1 + } + ) + if duplicates: + issues.append( + issue( + "E202", + Severity.ERROR, + "Duplicate table headers must be resolved before mapping.", + columns=duplicates, + ) + ) + return issues + + +def problem_examples( + frame: pd.DataFrame, + mask: pd.Series, + column: object, + sample_column: object | None, + *, + limit: int = 10, +) -> list[dict[str, Any]]: + """Return short examples without exposing the source file path.""" + examples: list[dict[str, Any]] = [] + for index in frame.index[mask][:limit]: + sample_value = ( + frame.at[index, sample_column] + if sample_column is not None + else None + ) + value = frame.at[index, column] + examples.append( + { + "spreadsheet_row": ( + int(index) + 2 if isinstance(index, int) else str(index) + ), + "sample": ( + None if pd.isna(sample_value) else str(sample_value) + ), + "value": None if pd.isna(value) else str(value), + } + ) + return examples diff --git a/skills/geoskills/scripts/geoskills_core/version.py b/skills/geoskills/scripts/geoskills_core/version.py new file mode 100644 index 0000000..81a8b0e --- /dev/null +++ b/skills/geoskills/scripts/geoskills_core/version.py @@ -0,0 +1,6 @@ +"""GeoSkills workflow-core version contract.""" + +VERSION = "0.4.0-dev" +__version__ = VERSION + +__all__ = ["VERSION", "__version__"] diff --git a/skills/geoskills/scripts/geoskills_core/workflow.py b/skills/geoskills/scripts/geoskills_core/workflow.py new file mode 100644 index 0000000..7af4e87 --- /dev/null +++ b/skills/geoskills/scripts/geoskills_core/workflow.py @@ -0,0 +1,1513 @@ +"""Deterministic plan/run orchestration for GeoSkills v0.4.""" + +from __future__ import annotations + +import hashlib +import json +import os +import re +import shutil +import tempfile +import xml.etree.ElementTree as ET +from copy import deepcopy +from pathlib import Path +from typing import Any, Iterable, Mapping, Sequence + +from PIL import Image + +from .adapters import PreparedInput, inspect_task, prepare_mapped_input, run_task +from .errors import GeoSkillsError +from .export import ( + AtomicDirectory, + BundleExportError, + shareable_file_record, + sha256_file, +) +from .recipe import load_recipe, resolve_recipe_path +from .registry import DIAGRAM_API_VERSION, get_diagram +from .reports import ( + REPORT_SCHEMA_NAME, + REPORT_SCHEMA_VERSION, + ReportError, + build_report, + render_qa_markdown, + validate_report, +) +from .version import VERSION + + +PLAN_SCHEMA_VERSION = "geoskills.plan/v1" +CLI_REPORT_SCHEMA_VERSION = "geoskills.cli-report/v1" +MAX_PLAN_BYTES = 2 * 1024 * 1024 +_PLAN_ID_PATTERN = re.compile(r"^[0-9a-f]{64}$") +_PLAN_STATUSES = frozenset({"ready", "needs_confirmation", "blocked"}) + +_SKILL_ROOT = Path(__file__).resolve().parents[2] +_ASSET_PATHS = { + "chondrite-sm89": "assets/normalization/chondrite-sm89.json", + "pm-sm89": "assets/normalization/primitive-mantle-sm89.json", + "pm-sm89-modified": ( + "assets/normalization/primitive-mantle-modified-sm89.json" + ), + "nmorb-sm89": "assets/normalization/nmorb-sm89.json", + "tas-lemaitre-2002": "assets/classification/tas-lemaitre-2002.json", +} +_BUILTIN_STYLE_DEFAULTS: Mapping[str, Mapping[str, int | float]] = { + "publication-double-column": { + "width_mm": 183.0, + "height_mm": 120.0, + "dpi": 600, + }, + "review-preview": { + "width_mm": 150.0, + "height_mm": 100.0, + "dpi": 300, + }, +} +_DIAGRAM_STYLE_DEFAULTS: Mapping[str, Mapping[str, Any]] = { + "ree": { + "axes_frame": "full", + "legend_layout": "inside-auto", + "grid_style": "none", + "y_margin": 0.08, + }, + "spider": { + "axes_frame": "full", + "legend_layout": "inside-auto", + "grid_style": "none", + "y_margin": 0.08, + }, + "harker": { + "axes_frame": "full", + "margin_fraction": 0.06, + }, + "tas": {"legend_layout": "inside-auto"}, +} + + +class WorkflowError(GeoSkillsError): + """An expected plan or run failure.""" + + default_code = "E800" + + +def _issue( + code: str, + severity: str, + message: str, + **extra: Any, +) -> dict[str, Any]: + result: dict[str, Any] = { + "code": code, + "severity": severity, + "message": message, + } + result.update(extra) + return result + + +def _json_bytes(value: Any) -> bytes: + return json.dumps( + value, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + + +def _digest_json(value: Any) -> str: + return hashlib.sha256(_json_bytes(value)).hexdigest() + + +def _write_json_atomic( + path: Path, + value: Mapping[str, Any], + *, + overwrite: bool, +) -> None: + destination = Path(path) + if destination.exists(): + if not overwrite: + raise WorkflowError( + "计划文件已经存在;如需替换,请明确使用 --overwrite-plan。", + code="E801", + details={"filename": destination.name}, + ) + try: + _load_plan(destination) + except WorkflowError as exc: + raise WorkflowError( + "仅允许覆盖由 GeoSkills 创建的有效计划文件。", + code="E821", + details={"filename": destination.name}, + ) from exc + destination.parent.mkdir(parents=True, exist_ok=True) + handle = tempfile.NamedTemporaryFile( + mode="w", + encoding="utf-8", + newline="\n", + prefix=".geoskills-plan-", + suffix=".tmp", + dir=destination.parent, + delete=False, + ) + temporary = Path(handle.name) + try: + with handle: + json.dump( + value, + handle, + ensure_ascii=False, + indent=2, + sort_keys=True, + allow_nan=False, + ) + handle.write("\n") + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary, destination) + finally: + if temporary.exists(): + temporary.unlink() + + +def _load_plan(path: Path) -> dict[str, Any]: + plan_path = Path(path) + if not plan_path.exists() or not plan_path.is_file(): + raise WorkflowError( + "找不到计划文件。", + code="E802", + details={"filename": plan_path.name or "plan"}, + ) + if plan_path.stat().st_size > MAX_PLAN_BYTES: + raise WorkflowError( + "计划文件超过安全大小限制。", + code="E803", + details={"filename": plan_path.name}, + ) + try: + document = json.loads(plan_path.read_text(encoding="utf-8-sig")) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + raise WorkflowError( + "计划文件不是有效的 UTF-8 JSON。", + code="E804", + details={"filename": plan_path.name}, + ) from exc + if not isinstance(document, dict): + raise WorkflowError("计划文件顶层必须是对象。", code="E805") + if document.get("schema_version") != PLAN_SCHEMA_VERSION: + raise WorkflowError( + f"计划格式必须是 {PLAN_SCHEMA_VERSION}。", + code="E806", + ) + if not isinstance(document.get("tasks"), list): + raise WorkflowError("计划文件缺少任务列表。", code="E807") + if document.get("status") not in _PLAN_STATUSES: + raise WorkflowError("计划文件 status 无效。", code="E807") + plan_id = document.get("plan_id") + if not isinstance(plan_id, str) or _PLAN_ID_PATTERN.fullmatch(plan_id) is None: + raise WorkflowError("计划文件 plan_id 无效。", code="E807") + tasks = document["tasks"] + if not tasks: + raise WorkflowError("计划文件任务列表不能为空。", code="E807") + task_ids: list[str] = [] + for item in tasks: + if not isinstance(item, dict): + raise WorkflowError("计划文件任务必须是对象。", code="E807") + task_id = item.get("id") + if not isinstance(task_id, str) or not task_id.strip(): + raise WorkflowError("计划文件任务缺少有效 ID。", code="E807") + task_ids.append(task_id.casefold()) + if len(task_ids) != len(set(task_ids)): + raise WorkflowError("计划文件任务 ID 不能重复。", code="E807") + return document + + +def _asset_ids(task: Mapping[str, Any]) -> tuple[str, ...]: + diagram = str(task["diagram"]) + if diagram in {"ree", "spider"}: + return (str(task["parameters"]["reference"]),) + if diagram == "tas": + return ("tas-lemaitre-2002",) + return () + + +def _asset_records(task: Mapping[str, Any]) -> list[dict[str, Any]]: + records: list[dict[str, Any]] = [] + for asset_id in _asset_ids(task): + relative = _ASSET_PATHS[asset_id] + path = _SKILL_ROOT / relative + if not path.is_file(): + raise WorkflowError( + "内置科学参考文件缺失,已停止任务。", + code="E808", + details={"asset_id": asset_id, "filename": path.name}, + ) + try: + document = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + raise WorkflowError( + "内置科学参考文件无法读取,已停止任务。", + code="E818", + details={"asset_id": asset_id, "filename": path.name}, + ) from exc + source = document.get("source", {}) + record: dict[str, Any] = { + "id": asset_id, + "asset_id": str(document.get("id", asset_id)), + "display_name": str(document.get("display_name", asset_id)), + "asset_schema_version": str( + document.get("schema_version", "unknown") + ), + "filename": path.name, + "sha256": sha256_file(path), + } + if document.get("unit") is not None: + record["unit"] = str(document["unit"]) + for key in ("doi", "table", "classification_doi", "boundary_doi"): + if source.get(key) is not None: + record[key] = str(source[key]) + records.append(record) + return records + + +def _expanded_style( + task: Mapping[str, Any], + recipe: Mapping[str, Any], +) -> dict[str, Any]: + preset_name = str(task["preset"]) + custom = recipe.get("presets", {}).get(preset_name) + if custom is None: + base_name = preset_name + overrides: Mapping[str, Any] = {} + else: + base_name = str(custom["extends"]) + overrides = custom["style"] + try: + style = dict(_BUILTIN_STYLE_DEFAULTS[base_name]) + except KeyError as exc: + raise WorkflowError("绘图预设不在固定注册表中。", code="E809") from exc + style.update(dict(overrides)) + style.update(_DIAGRAM_STYLE_DEFAULTS[str(task["diagram"])]) + style["preset"] = preset_name + style["base_preset"] = base_name + return style + + +def _selected_tasks( + recipe: Mapping[str, Any], + selected_task_ids: Sequence[str] | None, +) -> list[dict[str, Any]]: + all_tasks = list(recipe["tasks"]) + if selected_task_ids is None: + return [deepcopy(task) for task in all_tasks] + requested = [str(item) for item in selected_task_ids] + if not requested: + raise WorkflowError("至少需要选择一个任务。", code="E810") + if len(set(requested)) != len(requested): + raise WorkflowError("任务选择不能重复。", code="E811") + by_id = {str(task["id"]): task for task in all_tasks} + missing = [task_id for task_id in requested if task_id not in by_id] + if missing: + raise WorkflowError( + "所选任务不存在。", + code="E812", + details={"task_ids": missing}, + ) + requested_set = set(requested) + return [ + deepcopy(task) + for task in all_tasks + if str(task["id"]) in requested_set + ] + + +def _confirmation_issues( + recipe: Mapping[str, Any], + tasks: Iterable[Mapping[str, Any]], +) -> list[dict[str, Any]]: + issues: list[dict[str, Any]] = [] + for name, confirmed in recipe["confirmations"].items(): + if confirmed is not True: + issues.append( + _issue( + "R801", + "review", + "运行前需要完成数据确认。", + field=f"confirmations.{name}", + suggested_action="核对对应内容后,将该项明确改为 true。", + ) + ) + for task in tasks: + task_id = str(task["id"]) + for name, confirmed in task.get("confirmations", {}).items(): + if confirmed is not True: + issues.append( + _issue( + "R802", + "review", + "运行前需要完成任务专属科学确认。", + task_id=task_id, + field=f"tasks[{task_id}].confirmations.{name}", + suggested_action="完成专业复核后,将该项明确改为 true。", + ) + ) + return issues + + +def _expected_outputs( + task: Mapping[str, Any], + report_profile: str, +) -> list[str]: + task_id = str(task["id"]) + stem = str(task["stem"]) + filenames = [ + *(f"{stem}.{extension}" for extension in ("svg", "pdf", "tiff", "png")), + f"{stem}.report.json", + f"{stem}.qa.md", + ] + if report_profile == "local-reproducible": + filenames.insert(4, f"{stem}.source_data.csv") + return [f"{task_id}/{filename}" for filename in filenames] + + +def _mapping_snapshot(prepared: PreparedInput) -> list[dict[str, str]]: + return [ + { + "source_column": mapping.source_column, + "canonical_analyte": mapping.canonical_analyte, + "unit": mapping.unit, + } + for mapping in prepared.mappings + ] + + +def _recipe_member_paths( + recipe_path: str | Path, + recipe: Mapping[str, Any], +) -> tuple[Path, Path, list[dict[str, Any]]]: + """Resolve source/output members and flag destructive overlap.""" + + try: + source_path = resolve_recipe_path( + recipe_path, str(recipe["input"]["file"]) + ) + output_path = resolve_recipe_path( + recipe_path, str(recipe["output"]["directory"]) + ) + except ValueError as exc: + raise WorkflowError( + "配方成员解析后超出配方目录,已停止操作。", + code="E820", + ) from exc + issues: list[dict[str, Any]] = [] + recipe_root = Path(recipe_path).resolve().parent + lexical_source = Path( + os.path.abspath( + recipe_root / Path(str(recipe["input"]["file"])) + ) + ) + lexical_output = Path( + os.path.abspath( + recipe_root / Path(str(recipe["output"]["directory"])) + ) + ) + source_in_output = False + for candidate_source, candidate_output in ( + (source_path, output_path), + (lexical_source, lexical_output), + ): + try: + candidate_source.relative_to(candidate_output) + except ValueError: + continue + source_in_output = True + break + if source_in_output: + issues.append( + _issue( + "E820", + "error", + "输出目录不能包含原始输入文件;否则覆盖运行可能删除源数据。", + field="output.directory", + suggested_action="选择与输入文件分离的专用输出子目录。", + ) + ) + return source_path, output_path, issues + + +def _finalize_plan( + *, + recipe: Mapping[str, Any], + tasks: Sequence[Mapping[str, Any]], + assets: Mapping[str, list[dict[str, Any]]], + inspections: Mapping[str, Mapping[str, Any]], + mapping: Sequence[Mapping[str, str]], + source: Mapping[str, Any], + preflight_issues: Sequence[Mapping[str, Any]], +) -> dict[str, Any]: + """Build the deterministic plan from one already-read data snapshot.""" + + confirmation_issues = _confirmation_issues(recipe, tasks) + inspection_issues = [ + issue + for task in tasks + for issue in inspections[str(task["id"])]["issues"] + ] + blocked = bool(preflight_issues) or any( + inspections[str(task["id"])]["status"] != "ready" + for task in tasks + ) + status = ( + "blocked" + if blocked + else ("needs_confirmation" if confirmation_issues else "ready") + ) + issues = [ + *[dict(item) for item in preflight_issues], + *confirmation_issues, + *inspection_issues, + ] + report_profile = str(recipe["output"]["report_profile"]) + task_records = [] + for task in tasks: + task_id = str(task["id"]) + task_records.append( + { + "id": task_id, + "diagram": task["diagram"], + "stem": task["stem"], + "preset": task["preset"], + "parameters": task["parameters"], + "confirmations": task["confirmations"], + "style": task["style"], + "registry_contract": get_diagram( + str(task["diagram"]) + ).to_dict(), + "inspection": inspections[task_id], + "assets": assets[task_id], + "expected_outputs": _expected_outputs(task, report_profile), + } + ) + + plan_basis = { + "tool_version": VERSION, + "diagram_api_version": DIAGRAM_API_VERSION, + "recipe": { + "schema_version": recipe["schema_version"], + "input": recipe["input"], + "columns": recipe["columns"], + "output": recipe["output"], + "presets": recipe["presets"], + "confirmations": recipe["confirmations"], + }, + "input": dict(source), + "column_mapping": [dict(item) for item in mapping], + "tasks": task_records, + } + plan_id = _digest_json(plan_basis) + plan = { + "schema_version": PLAN_SCHEMA_VERSION, + "plan_id": plan_id, + "status": status, + "tool": { + "name": "GeoSkills", + "version": VERSION, + "diagram_api_version": DIAGRAM_API_VERSION, + }, + "input": dict(source), + "column_mapping": [dict(item) for item in mapping], + "output": recipe["output"], + "confirmations": recipe["confirmations"], + "tasks": task_records, + "issues": issues, + } + return {"status": status, "plan": plan, "issues": issues} + + +def build_plan( + recipe_path: str | Path, + *, + selected_task_ids: Sequence[str] | None = None, +) -> dict[str, Any]: + """Validate a recipe and input, but never create final figure outputs.""" + + source_path = Path(recipe_path) + loaded = load_recipe(source_path) + if loaded["status"] == "invalid" or loaded["recipe"] is None: + return { + "status": "error", + "plan": None, + "issues": loaded["issues"], + } + recipe = loaded["recipe"] + try: + tasks = _selected_tasks(recipe, selected_task_ids) + for task in tasks: + task["style"] = _expanded_style(task, recipe) + assets = { + str(task["id"]): _asset_records(task) for task in tasks + } + except GeoSkillsError as exc: + return { + "status": "error", + "plan": None, + "issues": [exc.to_issue()], + } + + try: + _, _, preflight_issues = _recipe_member_paths(source_path, recipe) + except GeoSkillsError as exc: + return { + "status": "error", + "plan": None, + "issues": [exc.to_issue()], + } + try: + with tempfile.TemporaryDirectory(prefix="geoskills-plan-") as directory: + prepared = prepare_mapped_input( + recipe, + recipe_path=source_path, + work_dir=Path(directory), + ) + inspections = { + str(task["id"]): inspect_task(task, prepared) + for task in tasks + } + mapping = _mapping_snapshot(prepared) + source = prepared.source + except GeoSkillsError as exc: + safe_issue = exc.to_issue() + preflight_issues.append(safe_issue) + source = { + "filename": Path(str(recipe["input"]["file"])).name, + "format": Path(str(recipe["input"]["file"])).suffix.lower(), + } + mapping = [] + inspections = { + str(task["id"]): { + "task_id": str(task["id"]), + "diagram": str(task["diagram"]), + "status": "blocked", + "recognized_analytes": [], + "issues": [], + } + for task in tasks + } + except (OSError, ValueError): + return { + "status": "error", + "plan": None, + "issues": [ + _issue( + "E899", + "error", + "读取或检查配方输入时发生未预期错误。", + ) + ], + } + + return _finalize_plan( + recipe=recipe, + tasks=tasks, + assets=assets, + inspections=inspections, + mapping=mapping, + source=source, + preflight_issues=preflight_issues, + ) + + +def create_plan( + recipe_path: str | Path, + plan_path: str | Path, + *, + selected_task_ids: Sequence[str] | None = None, + overwrite: bool = False, +) -> dict[str, Any]: + """Build and atomically save a privacy-safe execution plan.""" + + result = build_plan( + recipe_path, + selected_task_ids=selected_task_ids, + ) + plan = result.get("plan") + if plan is not None: + try: + loaded = load_recipe(recipe_path) + recipe = loaded.get("recipe") + if recipe is None: + raise WorkflowError( + "写入计划前配方复核失败。", + code="E822", + ) + input_path, output_path, _ = _recipe_member_paths( + recipe_path, recipe + ) + destination_path = Path(plan_path) + destination = destination_path.resolve() + lexical_destination = Path(os.path.abspath(destination_path)) + protected = { + Path(recipe_path).resolve(), + input_path.resolve(), + Path(os.path.abspath(Path(recipe_path))), + Path(os.path.abspath(input_path)), + } + if ( + destination in protected + or lexical_destination in protected + or _path_is_within(destination_path, output_path) + ): + raise WorkflowError( + "计划文件不能覆盖配方、原始输入,也不能放在最终输出目录内。", + code="E822", + details={"filename": destination.name or "plan"}, + ) + _write_json_atomic(Path(plan_path), plan, overwrite=overwrite) + except GeoSkillsError as exc: + return { + "status": "error", + "plan": None, + "issues": [exc.to_issue()], + } + return { + "status": result["status"], + "plan_id": None if plan is None else plan["plan_id"], + "plan_file": None if plan is None else Path(plan_path).name, + "issues": result["issues"], + } + + +def _safe_run_issues(report: Mapping[str, Any]) -> list[dict[str, str]]: + results: list[dict[str, str]] = [] + for item in report.get("issues", []): + severity = str(item.get("severity", "review")) + if severity not in {"info", "warning", "review", "error"}: + severity = "review" + results.append( + { + "code": str(item.get("code", "R899")), + "severity": severity, + "message": str(item.get("message", "需要人工复核。")), + } + ) + return results + + +def _safe_plot_summary( + diagram: str, + legacy_report: Mapping[str, Any], +) -> dict[str, Any]: + plot = legacy_report.get("plot", {}) + if not isinstance(plot, Mapping): + return {} + common = { + "sample_count", + "group_count", + "palette_repeated", + "marker_repeated", + "axes_frame", + "legend_position", + } + diagram_fields = { + "ree": { + "normalization_id", + "y_limits", + "unity_line_visible", + "plotted_sample_count", + "line_style_repeated", + "grid_style", + "legend_fallback", + "legend_collision_free", + "inside_legend_strategy", + }, + "spider": { + "normalization_id", + "y_limits", + "unity_line_visible", + "plotted_sample_count", + "line_style_repeated", + "grid_style", + "legend_fallback", + "legend_collision_free", + "inside_legend_strategy", + }, + "harker": { + "panel_count", + "rows", + "columns", + "x_limits", + "legend_columns", + "legend_rows", + "legend_within_figure", + "shared_x_label", + }, + "tas": { + "classification_status_counts", + "field_counts", + "x_limits", + "y_limits", + "legend_fallback", + }, + } + allowed = common | diagram_fields.get(diagram, set()) + + def without_exact_source_extrema(value: Any) -> Any: + if isinstance(value, Mapping): + return { + str(key): without_exact_source_extrema(item) + for key, item in value.items() + if str(key).casefold() + not in { + "data_min", + "data_max", + "raw_min", + "raw_max", + "observed_min", + "observed_max", + "source_min", + "source_max", + } + } + if isinstance(value, (list, tuple)): + return [ + without_exact_source_extrema(item) for item in value + ] + return deepcopy(value) + + return { + str(key): without_exact_source_extrema(value) + for key, value in plot.items() + if key in allowed + } + + +def _safe_scientific_details( + task: Mapping[str, Any], + prepared: PreparedInput, + legacy_report: Mapping[str, Any], + *, + plan_id: str, + report_profile: str, +) -> dict[str, Any]: + diagram = str(task["diagram"]) + details: dict[str, Any] = { + "task_id": str(task["id"]), + "diagram": diagram, + "plan_id": plan_id, + "scientific_parameters": deepcopy(task["parameters"]), + "scientific_confirmations": deepcopy(task["confirmations"]), + "analyte_units": { + mapping.canonical_analyte: mapping.unit + for mapping in prepared.mappings + }, + "style": deepcopy(task["style"]), + "reference_assets": deepcopy(task["assets"]), + "report_profile": report_profile, + "plot_summary": _safe_plot_summary(diagram, legacy_report), + } + conversions = legacy_report.get("oxide_conversions") + if isinstance(conversions, list) and conversions: + details["oxide_conversions"] = deepcopy(conversions) + guidance = legacy_report.get("interpretation_guidance") + if isinstance(guidance, list): + details["interpretation_guidance"] = [ + str(item) for item in guidance + ] + caveat = legacy_report.get("scientific_caveat") + if isinstance(caveat, str) and caveat.strip(): + details["scientific_caveat"] = caveat.strip() + return details + + +def _task_directory_files(task_dir: Path, *, task_id: str) -> set[str]: + if task_dir.is_symlink() or not task_dir.is_dir(): + raise WorkflowError( + "绘图任务输出必须是普通目录。", + code="E823", + details={"task_id": task_id}, + ) + names: set[str] = set() + for entry in task_dir.iterdir(): + if entry.is_symlink() or not entry.is_file(): + raise WorkflowError( + "绘图任务生成了未经允许的目录或链接。", + code="E823", + details={"task_id": task_id, "filename": entry.name}, + ) + names.add(entry.name) + return names + + +def _verify_figure_bundle( + figure_paths: Sequence[Path], + *, + width_mm: float, + height_mm: float, + dpi: int, + task_id: str, +) -> dict[str, Any]: + """Verify that every claimed publication artifact is genuinely readable.""" + + by_suffix = {path.suffix.lower(): path for path in figure_paths} + expected_pixels = ( + round(float(width_mm) / 25.4 * int(dpi)), + round(float(height_mm) / 25.4 * int(dpi)), + ) + try: + svg_root = ET.parse(by_suffix[".svg"]).getroot() + if not svg_root.tag.casefold().endswith("svg"): + raise ValueError("not-svg") + svg_text_nodes = [ + node + for node in svg_root.iter() + if node.tag.casefold().endswith("text") + ] + if not svg_text_nodes: + raise ValueError("svg-text-not-editable") + + pdf_bytes = by_suffix[".pdf"].read_bytes() + if ( + not pdf_bytes.startswith(b"%PDF-") + or not pdf_bytes.rstrip().endswith(b"%%EOF") + or ( + b"/CIDFontType2" not in pdf_bytes + and b"/FontFile2" not in pdf_bytes + ) + ): + raise ValueError("invalid-pdf-or-font") + + raster_details: dict[str, dict[str, Any]] = {} + for suffix, expected_format in ((".png", "PNG"), (".tiff", "TIFF")): + path = by_suffix[suffix] + with Image.open(path) as image: + if image.format != expected_format: + raise ValueError(f"wrong-{expected_format.lower()}-format") + image.verify() + with Image.open(path) as image: + size = tuple(int(value) for value in image.size) + if any( + abs(actual - expected) > 3 + for actual, expected in zip(size, expected_pixels) + ): + raise ValueError(f"wrong-{expected_format.lower()}-size") + dpi_value = image.info.get("dpi", (0.0, 0.0)) + if not isinstance(dpi_value, tuple) or len(dpi_value) < 2: + raise ValueError(f"missing-{expected_format.lower()}-dpi") + dpi_pair = tuple(float(value) for value in dpi_value[:2]) + if any(abs(value - int(dpi)) > 2.0 for value in dpi_pair): + raise ValueError(f"wrong-{expected_format.lower()}-dpi") + rgba = image.convert("RGBA") + corners = [ + rgba.getpixel((0, 0)), + rgba.getpixel((size[0] - 1, 0)), + rgba.getpixel((0, size[1] - 1)), + rgba.getpixel((size[0] - 1, size[1] - 1)), + ] + if any( + pixel[3] != 255 + or any(channel < 250 for channel in pixel[:3]) + for pixel in corners + ): + raise ValueError( + f"nonwhite-{expected_format.lower()}-background" + ) + compression = None + if suffix == ".tiff": + compression = int(image.tag_v2.get(259, 0)) + if compression != 5: + raise ValueError("tiff-not-lzw") + raster_details[suffix.lstrip(".")] = { + "pixels": list(size), + "dpi": [round(value, 3) for value in dpi_pair], + **( + {"compression_tag": compression} + if compression is not None + else {} + ), + } + except ( + ET.ParseError, + OSError, + ValueError, + KeyError, + TypeError, + ) as exc: + raise WorkflowError( + "图件包未通过格式、尺寸或可编辑性验证,完整输出已取消。", + code="E824", + details={ + "task_id": task_id, + "reason_code": type(exc).__name__, + }, + ) from exc + + return { + "complete_figure_bundle": True, + "figure_formats": ["svg", "pdf", "tiff", "png"], + "final_size_mm": [float(width_mm), float(height_mm)], + "svg_text_editable": True, + "pdf_embedded_truetype_font": True, + "raster_dpi": int(dpi), + "raster_files": raster_details, + "tiff_compression": "LZW", + "white_background": True, + } + + +def _write_task_reports( + *, + task: Mapping[str, Any], + task_dir: Path, + staging_root: Path, + prepared: PreparedInput, + legacy_report: Mapping[str, Any], + plan_id: str, + report_profile: str, +) -> tuple[Path, Path, str]: + stem = str(task["stem"]) + task_id = str(task["id"]) + legacy_report_path = task_dir / f"{stem}.report.json" + source_data_path = task_dir / f"{stem}.source_data.csv" + legacy_allowed = { + *(f"{stem}.{extension}" for extension in ("svg", "pdf", "tiff", "png")), + f"{stem}.source_data.csv", + f"{stem}.report.json", + } + unexpected_legacy = sorted( + _task_directory_files(task_dir, task_id=task_id) - legacy_allowed + ) + if unexpected_legacy: + raise WorkflowError( + "绘图器生成了不在固定合同中的额外文件,完整输出已取消。", + code="E823", + details={"task_id": task_id, "filenames": unexpected_legacy}, + ) + if legacy_report_path.exists(): + legacy_report_path.unlink() + if report_profile == "shareable" and source_data_path.exists(): + source_data_path.unlink() + + figure_paths = [ + task_dir / f"{stem}.{extension}" + for extension in ("svg", "pdf", "tiff", "png") + ] + missing = [path.name for path in figure_paths if not path.is_file()] + if missing: + raise WorkflowError( + "绘图器未生成完整图件包。", + code="E813", + details={"task_id": str(task["id"]), "filenames": missing}, + ) + figure_qa = _verify_figure_bundle( + figure_paths, + width_mm=float(task["style"]["width_mm"]), + height_mm=float(task["style"]["height_mm"]), + dpi=int(task["style"]["dpi"]), + task_id=task_id, + ) + output_paths = list(figure_paths) + if report_profile == "local-reproducible": + if not source_data_path.is_file(): + raise WorkflowError( + "本地可复现模式缺少绘图源数据导出。", + code="E814", + details={"task_id": str(task["id"])}, + ) + output_paths.append(source_data_path) + output_records = [ + shareable_file_record( + path, + bundle_root=staging_root, + role=( + "plotted-source-sensitive" + if path.suffix.lower() == ".csv" + else "figure" + ), + ) + for path in output_paths + ] + issues = _safe_run_issues(legacy_report) + tas_review_codes = {"W711", "W712", "W713", "W714", "W715"} + review_required = any( + issue["severity"] in {"review", "error"} + or ( + str(task["diagram"]) == "tas" + and issue["code"] in tas_review_codes + ) + for issue in issues + ) + report = build_report( + operation=get_diagram(str(task["diagram"])).operation, + status="review" if review_required else "ready", + source=prepared.source, + outputs=output_records, + issues=issues, + qa={ + **figure_qa, + "source_export": ( + "included-sensitive" + if report_profile == "local-reproducible" + else "omitted-shareable" + ), + }, + next_actions=( + ["请完成人工科学审核后再用于论文解释。"] + if review_required + else ["请在投稿前核对图例、标签和科学解释。"] + ), + review_required=review_required, + details=_safe_scientific_details( + task, + prepared, + legacy_report, + plan_id=plan_id, + report_profile=report_profile, + ), + shareable=True, + ) + report_path = task_dir / f"{stem}.report.json" + qa_path = task_dir / f"{stem}.qa.md" + report_path.write_text( + json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True) + + "\n", + encoding="utf-8", + ) + qa_path.write_text(render_qa_markdown(report), encoding="utf-8") + expected_final = { + Path(value).name for value in task["expected_outputs"] + } + actual_final = _task_directory_files(task_dir, task_id=task_id) + if actual_final != expected_final: + raise WorkflowError( + "任务最终文件集合与已审核计划不一致,完整输出已取消。", + code="E823", + details={ + "task_id": task_id, + "missing": sorted(expected_final - actual_final), + "unexpected": sorted(actual_final - expected_final), + }, + ) + return report_path, qa_path, str(report["status"]) + + +def _verified_remove_work_dir(work_dir: Path, staging_root: Path) -> None: + resolved_work = work_dir.resolve() + resolved_stage = staging_root.resolve() + if ( + resolved_work.parent != resolved_stage + or resolved_work.name != "_work" + ): + raise WorkflowError("拒绝清理未经验证的临时目录。", code="E815") + if resolved_work.exists(): + shutil.rmtree(resolved_work) + + +def _path_is_within(path: Path, directory: Path) -> bool: + candidates = ( + (path.resolve(), directory.resolve()), + ( + Path(os.path.abspath(path)), + Path(os.path.abspath(directory)), + ), + ) + for candidate_path, candidate_directory in candidates: + try: + candidate_path.relative_to(candidate_directory) + except ValueError: + continue + return True + return False + + +def _existing_output_issue( + target: Path, + *, + overwrite: bool, +) -> dict[str, Any] | None: + if not target.exists(): + return None + if not overwrite: + return _issue( + "E825", + "review", + "完整输出目录已经存在;尚未执行任何绘图。", + suggested_action="审核旧输出后,明确使用 --overwrite。", + ) + if target.is_symlink() or not target.is_dir(): + return _issue( + "E826", + "error", + "仅允许覆盖普通的 GeoSkills 输出目录。", + ) + marker = target / "run.report.json" + try: + if ( + not marker.is_file() + or marker.is_symlink() + or marker.stat().st_size > MAX_PLAN_BYTES + ): + raise ValueError("missing-or-unsafe-marker") + document = json.loads(marker.read_text(encoding="utf-8-sig")) + if not isinstance(document, dict): + raise ValueError("invalid-marker") + validate_report(document, require_shareable=True) + if ( + document.get("schema") + != { + "name": REPORT_SCHEMA_NAME, + "version": REPORT_SCHEMA_VERSION, + } + or document.get("operation") + != "multi_task_geochemistry_workflow" + ): + raise ValueError("foreign-marker") + except ( + OSError, + UnicodeError, + json.JSONDecodeError, + ReportError, + ValueError, + ): + return _issue( + "E826", + "error", + "现有目录不是可验证的 GeoSkills 完整输出;为保护文件,拒绝覆盖。", + suggested_action="改用新的 output.directory,或先人工整理旧目录。", + ) + return None + + +def _verify_run_directory( + staging_root: Path, + *, + task_ids: Sequence[str], +) -> None: + expected_files = {"run.report.json", "run.qa.md"} + expected_directories = {str(value) for value in task_ids} + actual_files: set[str] = set() + actual_directories: set[str] = set() + for entry in staging_root.iterdir(): + if entry.is_symlink(): + raise WorkflowError( + "完整输出包含未经允许的链接,已取消提交。", + code="E823", + ) + if entry.is_file(): + actual_files.add(entry.name) + elif entry.is_dir(): + actual_directories.add(entry.name) + else: + raise WorkflowError( + "完整输出包含不受支持的文件类型,已取消提交。", + code="E823", + ) + if ( + actual_files != expected_files + or actual_directories != expected_directories + ): + raise WorkflowError( + "完整输出目录与已审核计划不一致,已取消提交。", + code="E823", + details={ + "unexpected_files": sorted(actual_files - expected_files), + "unexpected_directories": sorted( + actual_directories - expected_directories + ), + }, + ) + + +def _verify_task_assets(task: Mapping[str, Any]) -> None: + for record in task.get("assets", []): + asset_id = str(record["id"]) + relative = _ASSET_PATHS.get(asset_id) + if relative is None: + raise WorkflowError( + "计划引用了未知的内置科学参考。", + code="E828", + details={"asset_id": asset_id}, + ) + path = _SKILL_ROOT / relative + if ( + not path.is_file() + or sha256_file(path) != str(record["sha256"]) + ): + raise WorkflowError( + "内置科学参考在执行期间发生变化,完整输出已取消。", + code="E828", + details={"asset_id": asset_id}, + ) + + +def execute_plan( + recipe_path: str | Path, + plan_path: str | Path, + *, + overwrite: bool = False, +) -> dict[str, Any]: + """Revalidate a saved plan and atomically execute all selected tasks.""" + + try: + saved_plan = _load_plan(Path(plan_path)) + selected_ids = [str(task["id"]) for task in saved_plan["tasks"]] + loaded = load_recipe(recipe_path) + recipe = loaded.get("recipe") + if recipe is None: + return { + "status": "error", + "output_directory": None, + "issues": loaded["issues"], + } + tasks = _selected_tasks(recipe, selected_ids) + for task in tasks: + task["style"] = _expanded_style(task, recipe) + assets = { + str(task["id"]): _asset_records(task) for task in tasks + } + _, target, path_issues = _recipe_member_paths(recipe_path, recipe) + if path_issues: + return { + "status": "blocked", + "output_directory": None, + "issues": path_issues, + } + if _path_is_within(Path(plan_path), target): + return { + "status": "blocked", + "output_directory": None, + "issues": [ + _issue( + "E822", + "error", + "计划文件不能位于最终输出目录内。", + suggested_action="把计划文件移到输出目录外并重新生成计划。", + ) + ], + } + output_issue = _existing_output_issue(target, overwrite=overwrite) + if output_issue is not None: + return { + "status": "blocked", + "output_directory": None, + "issues": [output_issue], + } + report_profile = str(recipe["output"]["report_profile"]) + task_outputs: list[Path] = [] + task_summaries: list[dict[str, Any]] = [] + with AtomicDirectory(target, overwrite=overwrite) as transaction: + staging_root = transaction.staging_dir + work_dir = staging_root / "_work" + prepared = prepare_mapped_input( + recipe, + recipe_path=Path(recipe_path), + work_dir=work_dir, + ) + inspections = { + str(task["id"]): inspect_task(task, prepared) + for task in tasks + } + current = _finalize_plan( + recipe=recipe, + tasks=tasks, + assets=assets, + inspections=inspections, + mapping=_mapping_snapshot(prepared), + source=prepared.source, + preflight_issues=(), + ) + current_plan = current["plan"] + if current_plan["plan_id"] != saved_plan["plan_id"]: + return { + "status": "blocked", + "output_directory": None, + "issues": [ + _issue( + "R803", + "review", + "配方、输入文件或内置参考已经变化;旧计划已失效。", + suggested_action="重新运行 plan,审核新计划后再运行。", + ) + ], + } + if ( + saved_plan["status"] != "ready" + or current["status"] != "ready" + ): + return { + "status": ( + "needs_confirmation" + if current["status"] == "needs_confirmation" + else "blocked" + ), + "output_directory": None, + "issues": current["issues"], + } + for task in current_plan["tasks"]: + task_dir = staging_root / str(task["id"]) + _verify_task_assets(task) + legacy_report = run_task( + task, + prepared, + output_dir=task_dir, + ) + _verify_task_assets(task) + if legacy_report.get("status") != "ready": + raise WorkflowError( + "绘图任务未成功完成,完整输出已取消。", + code="E817", + details={ + "task_id": str(task["id"]), + "issues": _safe_run_issues(legacy_report), + }, + ) + report_path, qa_path, task_status = _write_task_reports( + task=task, + task_dir=task_dir, + staging_root=staging_root, + prepared=prepared, + legacy_report=legacy_report, + plan_id=str(current_plan["plan_id"]), + report_profile=report_profile, + ) + task_files = sorted( + path for path in task_dir.iterdir() if path.is_file() + ) + task_outputs.extend(task_files) + task_summaries.append( + { + "task_id": str(task["id"]), + "diagram": str(task["diagram"]), + "report_filename": report_path.name, + "qa_filename": qa_path.name, + "status": task_status, + } + ) + _verified_remove_work_dir(work_dir, staging_root) + + workflow_review = any( + item["status"] == "review" for item in task_summaries + ) + workflow_issues = [ + _issue( + "R804", + "review", + "该任务已生成图件,但包含必须人工复核的科学状态。", + task_id=str(item["task_id"]), + ) + for item in task_summaries + if item["status"] == "review" + ] + run_output_records = [ + shareable_file_record( + path, + bundle_root=staging_root, + role=( + "plotted-source-sensitive" + if path.suffix.lower() == ".csv" + else "task-output" + ), + ) + for path in task_outputs + ] + run_report = build_report( + operation="multi_task_geochemistry_workflow", + status="review" if workflow_review else "ready", + source=prepared.source, + outputs=run_output_records, + issues=workflow_issues, + qa={ + "atomic_directory_commit": True, + "task_count": len(task_summaries), + "all_tasks_completed": True, + }, + next_actions=[ + ( + "请先解决任务报告中的科学复核状态,再用于投稿。" + if workflow_review + else "请审核各任务的 QA 摘要和图件后再用于投稿。" + ) + ], + review_required=workflow_review, + details={ + "plan_id": str(current_plan["plan_id"]), + "report_profile": report_profile, + "data_confirmations": deepcopy( + current_plan["confirmations"] + ), + "task_summaries": task_summaries, + }, + shareable=True, + ) + run_report_path = staging_root / "run.report.json" + run_qa_path = staging_root / "run.qa.md" + run_report_path.write_text( + json.dumps( + run_report, + ensure_ascii=False, + indent=2, + sort_keys=True, + ) + + "\n", + encoding="utf-8", + ) + run_qa_path.write_text( + render_qa_markdown(run_report), + encoding="utf-8", + ) + _verify_run_directory( + staging_root, + task_ids=[ + str(task["id"]) for task in current_plan["tasks"] + ], + ) + transaction.commit() + return { + "status": "review" if workflow_review else "ready", + "plan_id": saved_plan["plan_id"], + "output_directory": str(recipe["output"]["directory"]), + "task_count": len(task_summaries), + "issues": workflow_issues, + } + except GeoSkillsError as exc: + return { + "status": "error", + "output_directory": None, + "issues": [exc.to_issue()], + } + except (BundleExportError, ReportError): + return { + "status": "error", + "output_directory": None, + "issues": [ + _issue( + "E827", + "error", + "输出事务或质量报告失败;最终输出未提交。若同目录出现 .geoskills-run-backup-* 恢复目录,请保留并人工恢复。", + ) + ], + } + except (OSError, ValueError, KeyError, TypeError): + return { + "status": "error", + "output_directory": None, + "issues": [ + _issue( + "E899", + "error", + "执行工作流时发生未预期错误;最终输出未提交。", + ) + ], + } + + +__all__ = [ + "CLI_REPORT_SCHEMA_VERSION", + "MAX_PLAN_BYTES", + "PLAN_SCHEMA_VERSION", + "WorkflowError", + "build_plan", + "create_plan", + "execute_plan", +] diff --git a/skills/geoskills/scripts/plot_geochem_common.py b/skills/geoskills/scripts/plot_geochem_common.py index 660bd84..904252b 100644 --- a/skills/geoskills/scripts/plot_geochem_common.py +++ b/skills/geoskills/scripts/plot_geochem_common.py @@ -10,7 +10,13 @@ import numpy as np import pandas as pd -from plot_ree import FORMATS, GROUP_COLORS, MARKERS, PlottingError +from geoskills_core.errors import PlottingError +from geoskills_core.plotting import ( + FORMATS, + GROUP_COLORS, + MARKERS, + publication_style, +) SUBSCRIPT_TRANSLATION = str.maketrans("0123456789", "₀₁₂₃₄₅₆₇₈₉") @@ -220,13 +226,22 @@ def save_figure_bundle( dpi: int, ) -> None: """Save vector and raster outputs from the same Matplotlib figure.""" - for path in paths: - options: dict[str, Any] = {"facecolor": "white"} - if path.suffix.lower() in {".png", ".tiff"}: - options["dpi"] = dpi - if path.suffix.lower() == ".tiff": - options["pil_kwargs"] = {"compression": "tiff_lzw"} - figure.savefig(path, **options) + with publication_style( + overrides={ + "font.sans-serif": [ + "DejaVu Sans", + "Arial", + "Liberation Sans", + ] + } + ): + for path in paths: + options: dict[str, Any] = {"facecolor": "white"} + if path.suffix.lower() in {".png", ".tiff"}: + options["dpi"] = dpi + if path.suffix.lower() == ".tiff": + options["pil_kwargs"] = {"compression": "tiff_lzw"} + figure.savefig(path, **options) def shareable_file_record(path: Path) -> dict[str, Any]: diff --git a/skills/geoskills/scripts/plot_harker.py b/skills/geoskills/scripts/plot_harker.py index 46989e7..2933e16 100644 --- a/skills/geoskills/scripts/plot_harker.py +++ b/skills/geoskills/scripts/plot_harker.py @@ -21,6 +21,13 @@ from matplotlib.lines import Line2D from matplotlib.ticker import MultipleLocator +from geoskills_core.errors import PlottingError +from geoskills_core.plotting import ( + GROUP_COLORS, + MARKERS, + PUBLICATION_DOUBLE_COLUMN, + publication_styled, +) from inspect_data import InspectionError, issue from inspect_major_data import ( inspect_major_frame, @@ -40,15 +47,6 @@ shareable_file_record, validate_export_parameters, ) -from plot_ree import GROUP_COLORS, MARKERS, PlottingError - - -plt.rcParams["font.sans-serif"] = [ - "DejaVu Sans", - "Arial", - "Liberation Sans", -] - DEFAULT_X = "SiO2" DEFAULT_Y_CANDIDATES = [ "TiO2", @@ -187,6 +185,16 @@ def add_shared_group_legend( } +@publication_styled( + overrides={ + "font.sans-serif": [ + "DejaVu Sans", + "Arial", + "Liberation Sans", + ] + }, + preset_parameter="style_preset", +) def build_harker_figure( frame: pd.DataFrame, sample_column: str, @@ -201,10 +209,28 @@ def build_harker_figure( columns: int, axes_frame: str, margin_fraction: float, + style_preset: str = PUBLICATION_DOUBLE_COLUMN, ) -> tuple[Any, dict[str, Any]]: """Build one Harker grid from validated canonical data.""" + x_numeric = pd.to_numeric(frame[x_analyte], errors="coerce") + y_numeric = { + analyte: pd.to_numeric(frame[analyte], errors="coerce") + for analyte in y_analytes + } + pair_masks = { + analyte: ( + x_numeric.notna() + & y_values.notna() + & np.isfinite(x_numeric) + & np.isfinite(y_values) + ) + for analyte, y_values in y_numeric.items() + } + x_used = pd.Series(False, index=frame.index) + for pair_mask in pair_masks.values(): + x_used |= pair_mask x_limits = clean_linear_limits( - frame[x_analyte], margin_fraction=margin_fraction + x_numeric[x_used], margin_fraction=margin_fraction ) rows = math.ceil(len(y_analytes) / columns) figure, axes = plt.subplots( @@ -220,29 +246,23 @@ def build_harker_figure( for index, analyte in enumerate(y_analytes): ax = axes.flat[index] - valid_pair_count = int( - ( - pd.to_numeric(frame[x_analyte], errors="coerce").notna() - & pd.to_numeric(frame[analyte], errors="coerce").notna() - ).sum() - ) + pair_mask = pair_masks[analyte] + valid_pair_count = int(pair_mask.sum()) if valid_pair_count < 2: raise PlottingError( f"{x_analyte}–{analyte} 只有 {valid_pair_count} 个完整数据点;" "至少需要 2 个。" ) y_limits = clean_linear_limits( - frame[analyte], margin_fraction=margin_fraction + y_numeric[analyte][pair_mask], + margin_fraction=margin_fraction, ) missing_pairs[analyte] = int(len(frame) - valid_pair_count) if group_column is None: - x_values = pd.to_numeric(frame[x_analyte], errors="coerce") - y_values = pd.to_numeric(frame[analyte], errors="coerce") - finite = x_values.notna() & y_values.notna() ax.scatter( - x_values[finite], - y_values[finite], + x_numeric[pair_mask], + y_numeric[analyte][pair_mask], s=22, c="#3569A8", marker="o", @@ -262,7 +282,12 @@ def build_harker_figure( y_values = pd.to_numeric( subset[analyte], errors="coerce" ) - finite = x_values.notna() & y_values.notna() + finite = ( + x_values.notna() + & y_values.notna() + & np.isfinite(x_values) + & np.isfinite(y_values) + ) style = styles[group] ax.scatter( x_values[finite], @@ -435,6 +460,7 @@ def plot_harker_path( axes_frame: str = "full", margin_fraction: float = 0.06, overwrite: bool = False, + style_preset: str = PUBLICATION_DOUBLE_COLUMN, ) -> dict[str, Any]: """Validate input and export a Harker figure bundle.""" figure = None @@ -546,6 +572,7 @@ def plot_harker_path( resolved_columns, axes_frame, margin_fraction, + style_preset=style_preset, ) output_dir.mkdir(parents=True, exist_ok=True) save_figure_bundle(figure, figure_paths, dpi) @@ -641,6 +668,7 @@ def plot_harker_path( "height_mm": resolved_height, "png_dpi": dpi, "tiff_dpi": dpi, + "style_preset": style_preset, }, "plot": plot_info, "outputs": [ diff --git a/skills/geoskills/scripts/plot_ree.py b/skills/geoskills/scripts/plot_ree.py index 2718ca9..5049c75 100644 --- a/skills/geoskills/scripts/plot_ree.py +++ b/skills/geoskills/scripts/plot_ree.py @@ -22,6 +22,16 @@ from matplotlib.path import Path as MatplotlibPath from matplotlib.ticker import LogFormatterMathtext, LogLocator +from geoskills_core.errors import PlottingError +from geoskills_core.plotting import ( + FORMATS, + GROUP_COLORS, + LINE_STYLES, + MARKERS, + PUBLICATION_DOUBLE_COLUMN, + configure_boxed_legend, + publication_styled, +) from inspect_data import InspectionError, inspect_frame, issue, read_table from normalize_ree import ( DEFAULT_REFERENCE_PATH, @@ -32,48 +42,12 @@ ) -# Keep text editable in vector exports and use portable sans-serif fallbacks. -plt.rcParams["font.family"] = "sans-serif" -plt.rcParams["font.sans-serif"] = ["Arial", "DejaVu Sans", "Liberation Sans"] -plt.rcParams["svg.fonttype"] = "none" -plt.rcParams["pdf.fonttype"] = 42 -plt.rcParams["font.size"] = 7 -plt.rcParams["axes.labelsize"] = 7 -plt.rcParams["axes.linewidth"] = 0.7 -plt.rcParams["xtick.labelsize"] = 7 -plt.rcParams["ytick.labelsize"] = 7 -plt.rcParams["xtick.major.width"] = 0.65 -plt.rcParams["ytick.major.width"] = 0.65 -plt.rcParams["xtick.minor.width"] = 0.5 -plt.rcParams["ytick.minor.width"] = 0.5 -plt.rcParams["axes.spines.right"] = False -plt.rcParams["axes.spines.top"] = False -plt.rcParams["legend.frameon"] = False - - -GROUP_COLORS = [ - "#3569A8", - "#D06B27", - "#159A80", - "#A84F7A", - "#7655A5", - "#8B6B4A", - "#4D4D4D", - "#4F9BC1", -] -MARKERS = ["o", "s", "^", "D", "v", "P", "X", "<", ">", "h", "p", "*"] -LINE_STYLES = ["-", "--", "-.", ":"] -FORMATS = ("svg", "pdf", "tiff", "png") DEFAULT_LOG_Y_MARGIN = 0.08 AXES_FRAMES = ("open", "full") LEGEND_LAYOUTS = ("outside", "inside-auto") GRID_STYLES = ("none", "major") -class PlottingError(Exception): - """An expected problem that makes figure creation unsafe.""" - - def adaptive_log_y_limits( values: np.ndarray, margin_fraction: float = DEFAULT_LOG_Y_MARGIN ) -> tuple[float, float, float, float, float]: @@ -137,16 +111,6 @@ def clean_log_y_limits( return lower, upper, float(lower_step), float(upper_step) -def configure_boxed_legend(legend: Any) -> None: - """Apply a restrained white legend box suitable for an in-axes key.""" - legend.set_zorder(10) - frame = legend.get_frame() - frame.set_facecolor("white") - frame.set_edgecolor("#A8A8A8") - frame.set_linewidth(0.5) - frame.set_alpha(0.96) - - def legends_overlap_data( ax: Any, data_lines: list[Line2D], @@ -156,8 +120,16 @@ def legends_overlap_data( ax.figure.canvas.draw() renderer = ax.figure.canvas.get_renderer() boxes = [legend.get_window_extent(renderer).expanded(1.02, 1.08) for legend in legends] + axes_box = ax.get_window_extent(renderer) for index, box in enumerate(boxes): + if ( + box.x0 < axes_box.x0 + or box.x1 > axes_box.x1 + or box.y0 < axes_box.y0 + or box.y1 > axes_box.y1 + ): + return True if any(box.overlaps(other) for other in boxes[index + 1 :]): return True @@ -211,6 +183,7 @@ def parse_element_selection(selection: str | None, available: list[str]) -> list return selected +@publication_styled(preset_parameter="style_preset") def build_figure( normalized: pd.DataFrame, sample_column: str, @@ -227,6 +200,7 @@ def build_figure( y_label: str = "Sample / C1 chondrite", reference_note: str | None = None, x_tick_labelsize: float | None = None, + style_preset: str = PUBLICATION_DOUBLE_COLUMN, ) -> tuple[plt.Figure, dict[str, Any]]: """Build one figure; export callers must reuse this same figure object.""" if not 50 <= width_mm <= 400 or not 50 <= height_mm <= 400: @@ -435,11 +409,22 @@ def add_outside_legends() -> None: borderaxespad=0, ) - def add_inside_legends() -> list[Any]: + def add_inside_legends( + *, + group_columns: int, + sample_columns: int, + group_wrap_width: int, + group_fontsize: float, + sample_fontsize: float, + ) -> list[Any]: legends: list[Any] = [] + sample_anchor_y = 0.985 if group_column is not None: compact_group_labels = [ - textwrap.fill(f"{group} (n={plotted_group_counts[group]})", width=22) + textwrap.fill( + f"{group} (n={plotted_group_counts[group]})", + width=group_wrap_width, + ) for group in plotted_group_order ] group_legend = ax.legend( @@ -448,11 +433,13 @@ def add_inside_legends() -> list[Any]: loc="upper right", bbox_to_anchor=(0.985, 0.985), bbox_transform=ax.transAxes, - fontsize=5.6, + fontsize=group_fontsize, title=group_title, title_fontsize=6.1, handlelength=1.65, labelspacing=0.34, + columnspacing=0.75, + ncol=min(group_columns, len(group_handles)), borderaxespad=0, frameon=True, fancybox=False, @@ -461,19 +448,25 @@ def add_inside_legends() -> list[Any]: configure_boxed_legend(group_legend) ax.add_artist(group_legend) legends.append(group_legend) + fig.canvas.draw() + renderer = fig.canvas.get_renderer() + group_box = group_legend.get_window_extent(renderer).transformed( + ax.transAxes.inverted() + ) + sample_anchor_y = float(group_box.y0) - 0.018 sample_legend = ax.legend( sample_handles, [str(sample) for sample in plotted_samples], loc="upper right", - bbox_to_anchor=(0.985, 0.66 if group_column is not None else 0.985), + bbox_to_anchor=(0.985, sample_anchor_y), bbox_transform=ax.transAxes, - fontsize=5.4, + fontsize=sample_fontsize, title="Sample ID (symbol)" if group_column is not None else "Sample ID", title_fontsize=5.9, handlelength=1.25, labelspacing=0.32, columnspacing=0.62, - ncol=min(3, len(plotted_samples)), + ncol=min(sample_columns, len(plotted_samples)), borderaxespad=0, frameon=True, fancybox=False, @@ -492,11 +485,50 @@ def add_inside_legends() -> list[Any]: ) legend_position = "outside_right" legend_fallback = False + inside_legend_strategy: str | None = None if legend_layout == "inside-auto": - inside_legends = add_inside_legends() - if legends_overlap_data(ax, data_lines, inside_legends): + candidates = [ + { + "name": "stacked", + "group_columns": 1, + "sample_columns": 3, + "group_wrap_width": 22, + "group_fontsize": 5.6, + "sample_fontsize": 5.4, + }, + { + "name": "compact-two-column", + "group_columns": 2, + "sample_columns": 4, + "group_wrap_width": 18, + "group_fontsize": 5.25, + "sample_fontsize": 5.15, + }, + { + "name": "compact-wide", + "group_columns": 2, + "sample_columns": 5, + "group_wrap_width": 16, + "group_fontsize": 5.05, + "sample_fontsize": 5.0, + }, + ] + inside_legends: list[Any] = [] + for candidate in candidates: + inside_legends = add_inside_legends( + group_columns=int(candidate["group_columns"]), + sample_columns=int(candidate["sample_columns"]), + group_wrap_width=int(candidate["group_wrap_width"]), + group_fontsize=float(candidate["group_fontsize"]), + sample_fontsize=float(candidate["sample_fontsize"]), + ) + if not legends_overlap_data(ax, data_lines, inside_legends): + inside_legend_strategy = str(candidate["name"]) + break for legend in inside_legends: legend.remove() + inside_legends = [] + if not inside_legends: fig.subplots_adjust( left=0.095, right=0.73, @@ -560,6 +592,7 @@ def add_inside_legends() -> list[Any]: "legend_layout_requested": legend_layout, "legend_position": legend_position, "legend_fallback": legend_fallback, + "inside_legend_strategy": inside_legend_strategy, "legend_collision_free": True, "inside_legend_collision_free": ( not legend_fallback if legend_layout == "inside-auto" else None @@ -611,6 +644,7 @@ def plotting_error(path: Path, message: str) -> dict[str, Any]: } +@publication_styled(preset_parameter="style_preset") def plot_path( input_path: Path, output_dir: Path, @@ -629,6 +663,7 @@ def plot_path( grid_style: str = "none", overwrite: bool = False, reference_path: Path = DEFAULT_REFERENCE_PATH, + style_preset: str = PUBLICATION_DOUBLE_COLUMN, ) -> dict[str, Any]: """Validate input and export a publication figure bundle from one figure.""" figure = None @@ -719,6 +754,7 @@ def plot_path( axes_frame=axes_frame, legend_layout=legend_layout, grid_style=grid_style, + style_preset=style_preset, ) output_dir.mkdir(parents=True, exist_ok=True) @@ -837,6 +873,7 @@ def plot_path( "png_dpi": dpi, "tiff_dpi": dpi, "formats": list(FORMATS), + "style_preset": style_preset, }, "plot": plot_info, "outputs": [file_record(path) for path in figure_paths], diff --git a/skills/geoskills/scripts/plot_spider.py b/skills/geoskills/scripts/plot_spider.py index dca4aea..4052243 100644 --- a/skills/geoskills/scripts/plot_spider.py +++ b/skills/geoskills/scripts/plot_spider.py @@ -26,6 +26,10 @@ resolve_reference, spider_reference_summary, ) +from geoskills_core.plotting import ( + PUBLICATION_DOUBLE_COLUMN, + publication_styled, +) from plot_ree import ( AXES_FRAMES, DEFAULT_LOG_Y_MARGIN, @@ -99,6 +103,7 @@ def plotting_error(path: Path, message: str) -> dict[str, Any]: } +@publication_styled(preset_parameter="style_preset") def plot_spider_path( input_path: Path, output_dir: Path, @@ -117,6 +122,7 @@ def plot_spider_path( legend_layout: str = "inside-auto", grid_style: str = "none", overwrite: bool = False, + style_preset: str = PUBLICATION_DOUBLE_COLUMN, ) -> dict[str, Any]: """Validate input and export a submission-oriented spider-plot bundle.""" figure = None @@ -243,6 +249,7 @@ def plot_spider_path( y_label=f"Sample / {axis_reference}", reference_note=note_reference, x_tick_labelsize=6.2, + style_preset=style_preset, ) output_dir.mkdir(parents=True, exist_ok=True) @@ -386,6 +393,7 @@ def plot_spider_path( "png_dpi": dpi, "tiff_dpi": dpi, "formats": list(FORMATS), + "style_preset": style_preset, }, "oxide_conversions": conversions, "plot": plot_info, diff --git a/skills/geoskills/scripts/plot_tas.py b/skills/geoskills/scripts/plot_tas.py index 55e5e7c..af9c47f 100644 --- a/skills/geoskills/scripts/plot_tas.py +++ b/skills/geoskills/scripts/plot_tas.py @@ -21,6 +21,14 @@ from matplotlib.patches import Polygon from matplotlib.path import Path as MatplotlibPath +from geoskills_core.errors import PlottingError +from geoskills_core.plotting import ( + GROUP_COLORS, + MARKERS, + PUBLICATION_DOUBLE_COLUMN, + configure_boxed_legend, + publication_styled, +) from inspect_data import InspectionError, issue from inspect_major_data import ( inspect_major_frame, @@ -36,15 +44,6 @@ shareable_file_record, validate_export_parameters, ) -from plot_ree import GROUP_COLORS, MARKERS, PlottingError, configure_boxed_legend - - -plt.rcParams["font.sans-serif"] = [ - "DejaVu Sans", - "Arial", - "Liberation Sans", -] - SKILL_DIR = Path(__file__).resolve().parents[1] DEFAULT_MODEL_PATH = ( SKILL_DIR / "assets" / "classification" / "tas-lemaitre-2002.json" @@ -295,6 +294,16 @@ def add_tas_legend( return legend, "outside_right_fallback", requested_layout == "inside-auto" +@publication_styled( + overrides={ + "font.sans-serif": [ + "DejaVu Sans", + "Arial", + "Liberation Sans", + ] + }, + preset_parameter="style_preset", +) def build_tas_figure( classified: pd.DataFrame, sample_column: str, @@ -306,6 +315,7 @@ def build_tas_figure( width_mm: float, height_mm: float, legend_layout: str, + style_preset: str = PUBLICATION_DOUBLE_COLUMN, ) -> tuple[Any, dict[str, Any]]: """Build one fixed-geometry volcanic TAS figure.""" figure, ax = plt.subplots( @@ -501,6 +511,7 @@ def plot_tas_path( legend_layout: str = "inside-auto", overwrite: bool = False, model_path: Path = DEFAULT_MODEL_PATH, + style_preset: str = PUBLICATION_DOUBLE_COLUMN, ) -> dict[str, Any]: """Validate, classify, and export a volcanic TAS figure bundle.""" figure = None @@ -625,6 +636,7 @@ def plot_tas_path( width_mm, height_mm, legend_layout, + style_preset=style_preset, ) output_dir.mkdir(parents=True, exist_ok=True) save_figure_bundle(figure, figure_paths, dpi) @@ -744,6 +756,7 @@ def plot_tas_path( "height_mm": height_mm, "png_dpi": dpi, "tiff_dpi": dpi, + "style_preset": style_preset, }, "plot": plot_info, "outputs": [ diff --git a/skills/geoskills/scripts/requirements.txt b/skills/geoskills/scripts/requirements.txt index 4bf472c..6b35754 100644 --- a/skills/geoskills/scripts/requirements.txt +++ b/skills/geoskills/scripts/requirements.txt @@ -1,4 +1,6 @@ numpy>=2.0,<3 pandas>=2.2,<4 matplotlib>=3.8,<4 +Pillow>=10,<13 openpyxl>=3.1,<4 +PyYAML>=6,<7 diff --git a/tests/test_core_io.py b/tests/test_core_io.py new file mode 100644 index 0000000..44bbf0f --- /dev/null +++ b/tests/test_core_io.py @@ -0,0 +1,229 @@ +import hashlib +import sys +from pathlib import Path + +import pandas as pd +import pytest + + +ROOT = Path(__file__).resolve().parents[1] +SCRIPTS = ROOT / "skills" / "geoskills" / "scripts" +sys.path.insert(0, str(SCRIPTS)) + +from geoskills_core import ( # noqa: E402 + ColumnMapping, + FileSizeError, + Severity, + TextEncodingError, + UnsupportedFormatError, + WorksheetError, + automatic_column_mappings, + inspect_column_mappings, + issue, + match_analyte, + read_table, + sniff_text_format, + status_from_issues, + validate_column_mappings, + validate_table_structure, +) + + +def test_csv_metadata_is_share_safe_and_content_addressed( + tmp_path: Path, +) -> None: + path = tmp_path / "private-study.csv" + content = b"Sample,La_ppm,Ce_ppm,Pr_ppm\nS1,1,2,3\n" + path.write_bytes(content) + + frame, source = read_table(path) + + assert frame is not None + assert source["filename"] == path.name + assert source["file_sha256"] == hashlib.sha256(content).hexdigest() + assert source["size_bytes"] == len(content) + assert source["format"] == ".csv" + assert source["encoding"] == "utf-8-sig" + assert source["delimiter"] == "," + assert source["layout"] == "row_per_sample" + assert "path" not in source + assert str(tmp_path) not in repr(source) + + +def test_gb18030_tab_text_is_detected_and_read(tmp_path: Path) -> None: + path = tmp_path / "published-table.txt" + path.write_bytes( + "样品\tLa_ppm\tCe_ppm\tPr_ppm\n样品一\t1\t2\t3\n".encode("gb18030") + ) + + encoding, delimiter = sniff_text_format(path) + frame, source = read_table(path) + + assert encoding == "gb18030" + assert delimiter == "\t" + assert frame is not None + assert frame.iloc[0, 0] == "样品一" + assert source["delimiter"] == "TAB" + + +def test_unsupported_encoding_and_format_are_expected_errors( + tmp_path: Path, +) -> None: + bad_text = tmp_path / "bad.csv" + bad_text.write_bytes(b"\xff\xff\xff") + with pytest.raises(TextEncodingError): + read_table(bad_text) + + unsupported = tmp_path / "table.xls" + unsupported.write_bytes(b"legacy") + with pytest.raises(UnsupportedFormatError) as captured: + read_table(unsupported) + assert "path" not in captured.value.details + assert captured.value.details["filename"] == "table.xls" + + +def test_file_size_limit_is_checked_before_parsing(tmp_path: Path) -> None: + path = tmp_path / "large.csv" + path.write_bytes(b"a,b\n1,2\n") + + with pytest.raises(FileSizeError) as captured: + read_table(path, max_file_size_bytes=2) + + assert captured.value.details["size_bytes"] == path.stat().st_size + + +def test_excel_requires_explicit_sheet_when_several_exist( + tmp_path: Path, +) -> None: + path = tmp_path / "workbook.xlsx" + frame = pd.DataFrame({"Sample": ["S1"], "SiO2_wt%": [50.0]}) + with pd.ExcelWriter(path) as writer: + frame.to_excel(writer, sheet_name="Data", index=False) + frame.to_excel(writer, sheet_name="Backup", index=False) + + pending, source = read_table(path) + selected, selected_source = read_table(path, "Data") + indexed, indexed_source = read_table(path, 1) + + assert pending is None + assert source["sheet_names"] == ["Data", "Backup"] + assert selected is not None + assert selected_source["sheet"] == "Data" + assert indexed is not None + assert indexed_source["sheet"] == "Backup" + with pytest.raises(WorksheetError): + read_table(path, "Missing") + + +def test_conservative_transpose_tracks_sections_and_group( + tmp_path: Path, +) -> None: + path = tmp_path / "transposed.xlsx" + pd.DataFrame( + [ + ["Rock type", "Suite A", None], + ["Sample No.", "S1", "S2"], + ["Major element (wt.%)", None, None], + ["SiO2", 50.0, 52.0], + ["MgO", 6.0, 4.5], + ["Trace element (ppm)", None, None], + ["Rb", 35, 48], + ["Zr", 120, 145], + ] + ).to_excel(path, index=False, header=False) + + frame, source = read_table(path) + + assert frame is not None + assert source["layout"] == "column_per_sample_transposed" + assert source["transformation"]["sample_header_row"] == 2 + assert source["transformation"]["group_header_row"] == 1 + assert list(frame["Sample"]) == ["S1", "S2"] + assert list(frame["Group"]) == ["Suite A", "Suite A"] + assert {"SiO2_wt%", "MgO_wt%", "Rb_ppm", "Zr_ppm"}.issubset( + frame.columns + ) + + +@pytest.mark.parametrize( + ("header", "canonical", "unit"), + [ + ("silica (wt.%)", "SiO2", "wt%"), + ("TFe2O3_wt%", "Fe2O3T", "wt%"), + ("Rb mg/kg", "Rb", "ppm"), + ("La_ppm", "La", "ppm"), + ("H2O+", "H2O+", "unknown"), + ("H2O-", "H2O-", "unknown"), + ], +) +def test_analyte_and_unit_aliases_are_exact( + header: str, + canonical: str, + unit: str, +) -> None: + matched = match_analyte(header) + + assert matched is not None + assert matched.canonical == canonical + assert matched.explicit_unit == unit + assert match_analyte(f"prefix-{header}-suffix") is None + + +def test_automatic_mapping_requires_explicit_units() -> None: + mappings = automatic_column_mappings( + ["Sample", "SiO2_wt%", "La_ppm", "Ce"] + ) + by_analyte = {item.canonical_analyte: item for item in mappings} + + assert by_analyte["SiO2"].unit == "wt%" + assert by_analyte["La"].unit == "ppm" + assert by_analyte["Ce"].unit == "unknown" + selected, issues = inspect_column_mappings( + ["Sample", "SiO2_wt%", "La_ppm", "Ce"] + ) + assert selected == mappings + assert {item.code for item in issues} == {"E224"} + assert status_from_issues(issues) == "needs_review" + + +def test_explicit_mapping_blocks_ambiguity_and_wrong_units() -> None: + mappings = [ + ColumnMapping("La result A", "La", "ppm"), + ColumnMapping("La result B", "La", "ppm"), + ColumnMapping("La result A", "Ce", "unknown"), + ColumnMapping("Silica", "SiO2", "ppm"), + ColumnMapping("Mystery", "NotRegistered", "ppm"), + ColumnMapping("Missing", "Nd", "ppm"), + ] + + issues = validate_column_mappings( + mappings, + available_columns=[ + "La result A", + "La result B", + "Silica", + "Mystery", + ], + ) + codes = {item.code for item in issues} + + assert {"E220", "E221", "E222", "E223", "E224", "E225"}.issubset( + codes + ) + assert all(item.blocks for item in issues) + + +def test_issue_and_table_structure_are_json_ready() -> None: + notice = issue("W001", Severity.WARNING, "Review this value.", row=3) + frame = pd.DataFrame([[1, 2]], columns=["La_ppm", "La_ppm"]) + + structural = validate_table_structure(frame) + + assert notice.to_dict() == { + "code": "W001", + "severity": "warning", + "message": "Review this value.", + "details": {"row": 3}, + } + assert notice.blocks is False + assert structural[0].code == "E202" diff --git a/tests/test_core_output.py b/tests/test_core_output.py new file mode 100644 index 0000000..06fafa9 --- /dev/null +++ b/tests/test_core_output.py @@ -0,0 +1,388 @@ +import json +import os +import sys +from pathlib import Path + +import matplotlib as mpl +import matplotlib.pyplot as plt +import pytest + + +ROOT = Path(__file__).resolve().parents[1] +SCRIPTS = ROOT / "skills" / "geoskills" / "scripts" +sys.path.insert(0, str(SCRIPTS)) + +from geoskills_core.export import ( # noqa: E402 + AtomicBundle, + AtomicDirectory, + BundleExportError, + shareable_file_record, + stage_figure_outputs, +) +from geoskills_core.plotting import ( # noqa: E402 + GROUP_COLORS, + MARKERS, + PUBLICATION_DOUBLE_COLUMN, + REVIEW_PREVIEW, + STYLE_PRESETS, + group_style_map, + publication_style, + publication_styled, +) +from geoskills_core.reports import ( # noqa: E402 + REPORT_SCHEMA_VERSION, + ReportError, + build_report, + render_qa_markdown, + write_report_files, +) + + +def test_style_presets_are_local_and_deterministic() -> None: + original_font_size = mpl.rcParams["font.size"] + original_right_spine = mpl.rcParams["axes.spines.right"] + + with publication_style(PUBLICATION_DOUBLE_COLUMN): + assert mpl.rcParams["font.size"] == 7.0 + assert mpl.rcParams["axes.spines.right"] is False + with publication_style(REVIEW_PREVIEW, {"font.size": 10.0}): + assert mpl.rcParams["font.size"] == 10.0 + assert mpl.rcParams["axes.spines.right"] is True + assert mpl.rcParams["font.size"] == 7.0 + + assert mpl.rcParams["font.size"] == original_font_size + assert mpl.rcParams["axes.spines.right"] == original_right_spine + assert set(STYLE_PRESETS) == { + "publication-double-column", + "review-preview", + } + styles = group_style_map(["Granite", "Basalt"]) + assert styles["Granite"]["color"] == GROUP_COLORS[0] + assert styles["Basalt"]["marker"] == MARKERS[1] + + +def test_decorated_plotter_can_select_a_registered_preset() -> None: + original_font_size = mpl.rcParams["font.size"] + + @publication_styled(preset_parameter="style_preset") + def current_font_size( + style_preset: str = PUBLICATION_DOUBLE_COLUMN, + ) -> float: + return float(mpl.rcParams["font.size"]) + + assert current_font_size() == 7.0 + assert current_font_size(style_preset=REVIEW_PREVIEW) == 9.0 + assert mpl.rcParams["font.size"] == original_font_size + + +def test_shareable_file_record_never_exposes_absolute_path( + tmp_path: Path, +) -> None: + output_dir = tmp_path / "bundle" + output_dir.mkdir() + output = output_dir / "figure.svg" + output.write_text("", encoding="utf-8") + + record = shareable_file_record( + output, bundle_root=output_dir, role="figure" + ) + + assert record["filename"] == "figure.svg" + assert record["format"] == "svg" + assert record["role"] == "figure" + assert record["bytes"] == len(b"") + assert len(record["sha256"]) == 64 + assert "path" not in record + assert str(tmp_path) not in json.dumps(record) + + outside = tmp_path / "outside.svg" + outside.write_text("", encoding="utf-8") + with pytest.raises(BundleExportError, match="bundle_root"): + shareable_file_record(outside, bundle_root=output_dir) + + +def test_atomic_bundle_commits_all_files_and_cleans_staging( + tmp_path: Path, +) -> None: + output_dir = tmp_path / "outputs" + with AtomicBundle(output_dir) as bundle: + first = bundle.stage_path("figure.svg") + second = bundle.stage_path("figure.report.json") + staging_dir = bundle.staging_dir + first.write_text("", encoding="utf-8") + second.write_text("{}\n", encoding="utf-8") + installed = bundle.commit() + + assert {path.name for path in installed} == { + "figure.svg", + "figure.report.json", + } + assert (output_dir / "figure.svg").read_text(encoding="utf-8") == "" + assert not staging_dir.exists() + assert not list(tmp_path.glob(".geoskills-*")) + + +def test_atomic_bundle_aborts_cleanly_after_writer_failure( + tmp_path: Path, +) -> None: + output_dir = tmp_path / "outputs" + staging_dir: Path | None = None + + with pytest.raises(RuntimeError, match="simulated"): + with AtomicBundle(output_dir) as bundle: + staged = bundle.stage_path("figure.svg") + staging_dir = bundle.staging_dir + staged.write_text("partial", encoding="utf-8") + raise RuntimeError("simulated writer failure") + + assert not (output_dir / "figure.svg").exists() + assert staging_dir is not None and not staging_dir.exists() + assert not list(tmp_path.glob(".geoskills-*")) + + +def test_atomic_directory_commits_nested_outputs(tmp_path: Path) -> None: + target = tmp_path / "complete_run" + with AtomicDirectory(target) as transaction: + task = transaction.staging_dir / "ree_main" + task.mkdir() + (task / "figure.svg").write_text("", encoding="utf-8") + committed = transaction.commit() + + assert committed == target.resolve() + assert (target / "ree_main" / "figure.svg").read_text(encoding="utf-8") == ( + "" + ) + + +def test_atomic_directory_keeps_old_bundle_after_writer_failure( + tmp_path: Path, +) -> None: + target = tmp_path / "complete_run" + target.mkdir() + (target / "old.txt").write_text("old", encoding="utf-8") + + with pytest.raises(RuntimeError, match="simulated"): + with AtomicDirectory(target, overwrite=True) as transaction: + (transaction.staging_dir / "new.txt").write_text( + "new", encoding="utf-8" + ) + raise RuntimeError("simulated task failure") + + assert (target / "old.txt").read_text(encoding="utf-8") == "old" + assert not (target / "new.txt").exists() + + +def test_atomic_bundle_rolls_back_overwrite_after_partial_commit( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + from geoskills_core import export + + output_dir = tmp_path / "outputs" + output_dir.mkdir() + first_target = output_dir / "a.txt" + second_target = output_dir / "b.txt" + first_target.write_text("old-a", encoding="utf-8") + second_target.write_text("old-b", encoding="utf-8") + real_replace = os.replace + + with AtomicBundle(output_dir, overwrite=True) as bundle: + bundle.stage_path("a.txt").write_text("new-a", encoding="utf-8") + bundle.stage_path("b.txt").write_text("new-b", encoding="utf-8") + + def fail_on_second_install(source: object, target: object) -> None: + source_path = Path(source) + target_path = Path(target) + if ( + source_path.parent == bundle.staging_dir + and target_path.name == "b.txt" + ): + raise OSError("simulated commit failure") + real_replace(source, target) + + monkeypatch.setattr(export.os, "replace", fail_on_second_install) + with pytest.raises(BundleExportError, match="已尝试恢复"): + bundle.commit() + + assert first_target.read_text(encoding="utf-8") == "old-a" + assert second_target.read_text(encoding="utf-8") == "old-b" + assert not list(tmp_path.glob(".geoskills-*")) + + +def test_atomic_bundle_preserves_backup_when_restore_fails( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + from geoskills_core import export + + output_dir = tmp_path / "outputs" + output_dir.mkdir() + target = output_dir / "a.txt" + target.write_text("old-a", encoding="utf-8") + real_replace = os.replace + + with pytest.raises(BundleExportError, match="恢复目录"): + with AtomicBundle(output_dir, overwrite=True) as bundle: + bundle.stage_path("a.txt").write_text( + "new-a", encoding="utf-8" + ) + + def fail_install_and_restore( + source: object, + destination: object, + ) -> None: + source_path = Path(source) + destination_path = Path(destination) + if ( + source_path.parent == bundle.staging_dir + and destination_path == target + ): + raise OSError("synthetic install failure") + if ( + source_path.parent.name.startswith( + ".geoskills-backup-" + ) + and destination_path == target + ): + raise OSError("synthetic restore failure") + real_replace(source, destination) + + monkeypatch.setattr( + export.os, "replace", fail_install_and_restore + ) + bundle.commit() + + backups = list(tmp_path.glob(".geoskills-backup-*")) + assert len(backups) == 1 + assert (backups[0] / "a.txt").read_text(encoding="utf-8") == "old-a" + assert not target.exists() + + +def test_figure_and_reports_commit_as_one_shareable_bundle( + tmp_path: Path, +) -> None: + output_dir = tmp_path / "outputs" + figure, axis = plt.subplots(figsize=(2, 1.5)) + axis.plot([1, 2], [3, 4]) + + try: + with AtomicBundle(output_dir) as bundle: + stage_figure_outputs( + bundle, + figure, + "test_plot", + formats=("svg", "png"), + dpi=100, + ) + report = build_report( + operation="harker_plot", + status="review", + source={ + "filename": "published_example.csv", + "format": "csv", + "row_count": 2, + "column_count": 2, + }, + outputs=(), + issues=[ + { + "code": "R001", + "severity": "review", + "message": "请复核图例位置。", + } + ], + next_actions=["确认图例未遮挡数据。"], + review_required=True, + ) + write_report_files(bundle, "test_plot", report) + bundle.commit() + finally: + plt.close(figure) + + assert (output_dir / "test_plot.svg").is_file() + assert (output_dir / "test_plot.png").is_file() + stored_report = json.loads( + (output_dir / "test_plot.report.json").read_text(encoding="utf-8") + ) + summary = (output_dir / "test_plot.qa.md").read_text(encoding="utf-8") + assert stored_report["schema"]["version"] == REPORT_SCHEMA_VERSION + assert stored_report["review_required"] is True + assert stored_report["next_actions"] == ["确认图例未遮挡数据。"] + assert "# GeoSkills 图件质量检查摘要" in summary + assert "请复核图例位置" in summary + assert "不包含本地绝对路径或源数据值" in summary + assert str(tmp_path) not in summary + + +def test_shareable_report_rejects_paths_and_source_values( + tmp_path: Path, +) -> None: + with pytest.raises(ReportError, match="未经允许"): + build_report( + operation="tas_plot", + status="ready", + source={"path": str(tmp_path / "private.csv")}, + ) + + with pytest.raises(ReportError, match="不能包含字段"): + build_report( + operation="tas_plot", + status="ready", + details={"values": [50.1, 3.2]}, + ) + + +@pytest.mark.parametrize( + "key", + [ + "data_min", + "data_max", + "raw_min", + "raw_max", + "observed_min", + "observed_max", + "source_min", + "source_max", + ], +) +def test_shareable_report_rejects_exact_extrema_fields(key: str) -> None: + with pytest.raises(ReportError): + build_report( + operation="harker_plot", + status="ready", + details={"plot_summary": {"x_limits": {key: 48.4986}}}, + ) + + +def test_shareable_report_accepts_input_hash_and_size() -> None: + report = build_report( + operation="workflow_plan", + status="ready", + source={ + "filename": "published.csv", + "file_sha256": "a" * 64, + "size_bytes": 123, + "format": ".csv", + }, + ) + + assert report["source"]["file_sha256"] == "a" * 64 + assert report["source"]["size_bytes"] == 123 + + +def test_markdown_escapes_table_control_characters() -> None: + report = build_report( + operation="test", + status="ready", + outputs=[ + { + "filename": "a|b.svg", + "format": "svg", + "bytes": 10, + "sha256": "a" * 64, + } + ], + ) + + markdown = render_qa_markdown(report) + assert "a\\|b.svg" in markdown diff --git a/tests/test_environment_check.py b/tests/test_environment_check.py index 7a63d45..3e6403f 100644 --- a/tests/test_environment_check.py +++ b/tests/test_environment_check.py @@ -18,15 +18,53 @@ ) -def test_development_environment_is_ready() -> None: +def run_check(*options: str) -> tuple[subprocess.CompletedProcess[str], dict[str, object]]: result = subprocess.run( - [sys.executable, str(CHECK_SCRIPT)], + [sys.executable, str(CHECK_SCRIPT), *options], capture_output=True, check=False, text=True, ) - report = json.loads(result.stdout) + return result, json.loads(result.stdout) + + +def test_runtime_environment_is_ready_without_local_paths() -> None: + result, report = run_check() assert result.returncode == 0 + assert report["check_mode"] == "runtime" assert report["ready"] is True assert report["missing_modules"] == [] + assert "pytest" not in report["modules"] + assert "yaml" in report["modules"] + assert report["modules"]["PIL"]["version"] + assert "python_executable" not in report + + +def test_development_check_includes_pytest() -> None: + result, report = run_check("--dev") + + assert result.returncode == 0 + assert report["check_mode"] == "development" + assert report["ready"] is True + assert "pytest" in report["modules"] + + +def test_paths_are_only_included_when_explicitly_requested() -> None: + result, report = run_check("--include-paths") + + assert result.returncode == 0 + assert report["python_executable"] == sys.executable + + +def test_build_report_is_reusable_by_the_unified_cli() -> None: + sys.path.insert(0, str(CHECK_SCRIPT.parent)) + try: + from check_environment import build_report + + report = build_report() + finally: + sys.path.remove(str(CHECK_SCRIPT.parent)) + + assert report["ready"] is True + assert report["check_mode"] == "runtime" diff --git a/tests/test_plot_core_integration.py b/tests/test_plot_core_integration.py new file mode 100644 index 0000000..0e59b21 --- /dev/null +++ b/tests/test_plot_core_integration.py @@ -0,0 +1,48 @@ +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +SCRIPTS = ROOT / "skills" / "geoskills" / "scripts" +sys.path.insert(0, str(SCRIPTS)) + + +def test_plot_ree_reexports_shared_plot_contract() -> None: + from geoskills_core.errors import PlottingError as CorePlottingError + from geoskills_core.plotting import ( + FORMATS as CORE_FORMATS, + GROUP_COLORS as CORE_GROUP_COLORS, + MARKERS as CORE_MARKERS, + configure_boxed_legend as core_configure_boxed_legend, + ) + from plot_ree import ( + FORMATS, + GROUP_COLORS, + MARKERS, + PlottingError, + configure_boxed_legend, + ) + + assert PlottingError is CorePlottingError + assert FORMATS is CORE_FORMATS + assert GROUP_COLORS is CORE_GROUP_COLORS + assert MARKERS is CORE_MARKERS + assert configure_boxed_legend is core_configure_boxed_legend + + +def test_major_plot_modules_do_not_import_plot_ree() -> None: + for filename in ("plot_geochem_common.py", "plot_harker.py", "plot_tas.py"): + source = (SCRIPTS / filename).read_text(encoding="utf-8") + assert "from plot_ree import" not in source + assert "import plot_ree" not in source + + +def test_plot_modules_do_not_mutate_global_rcparams_at_import() -> None: + for filename in ( + "plot_ree.py", + "plot_spider.py", + "plot_harker.py", + "plot_tas.py", + ): + source = (SCRIPTS / filename).read_text(encoding="utf-8") + assert "plt.rcParams[" not in source diff --git a/tests/test_plot_harker.py b/tests/test_plot_harker.py index d40adb1..6f40949 100644 --- a/tests/test_plot_harker.py +++ b/tests/test_plot_harker.py @@ -143,6 +143,36 @@ def test_missing_pair_is_reported_not_imputed(tmp_path: Path) -> None: assert any(item["code"] == "W651" for item in report["issues"]) +def test_panel_limits_use_only_complete_xy_pairs(tmp_path: Path) -> None: + input_path = tmp_path / "unpaired_outlier.csv" + output = tmp_path / "unpaired_outlier" + frame = pd.read_csv(EXAMPLE) + frame.loc[0, "SiO2_wt%"] = None + frame.loc[0, "MgO_wt%"] = 1000.0 + frame.to_csv(input_path, index=False) + + result, report = run_plotter( + input_path, + "--output-dir", + output, + "--stem", + "unpaired_outlier", + "--y", + "MgO,CaO", + "--dpi", + "90", + ) + + mg_panel = next( + panel + for panel in report["plot"]["panels"] + if panel["y"] == "MgO" + ) + assert result.returncode == 0 + assert mg_panel["complete_pairs"] == len(frame) - 1 + assert mg_panel["y_limits"]["upper"] < 20 + + def test_existing_bundle_requires_explicit_overwrite( tmp_path: Path, ) -> None: diff --git a/tests/test_plot_ree.py b/tests/test_plot_ree.py index a0637b8..e1bbea3 100644 --- a/tests/test_plot_ree.py +++ b/tests/test_plot_ree.py @@ -279,6 +279,101 @@ def test_full_frame_inside_auto_legend_avoids_data() -> None: sys.path.remove(scripts) +def test_inside_auto_compacts_long_group_and_sample_legends() -> None: + scripts = str(SKILL / "scripts") + sys.path.insert(0, scripts) + try: + from plot_ree import build_figure + + elements = [ + "Rb", + "Ba", + "Th", + "U", + "Nb", + "Ta", + "K", + "La", + "Ce", + "Pb", + "Pr", + "Sr", + "P", + "Nd", + "Sm", + "Zr", + "Hf", + "Eu", + "Ti", + "Gd", + "Tb", + "Dy", + "Y", + "Ho", + "Er", + "Tm", + "Yb", + "Lu", + ] + sample_count = 11 + normalized = pd.DataFrame( + { + "Sample": [ + f"SYN-{index + 1:02d}" for index in range(sample_count) + ], + "Group": [ + f"Long synthetic group {index % 4 + 1} " + + "X" * 32 + for index in range(sample_count) + ], + **{ + f"{element}_N": [ + 300.0 + * np.exp(-0.10 * element_index) + * (1.0 + 0.10 * sample_index) + * ( + 1.0 + + 0.30 + * np.sin( + (element_index + sample_index) / 2.5 + ) + ) + for sample_index in range(sample_count) + ] + for element_index, element in enumerate(elements) + }, + } + ) + + figure, plot_info = build_figure( + normalized, + "Sample", + "Group", + elements, + "PM_synthetic", + width_mm=183, + height_mm=120, + axes_frame="full", + legend_layout="inside-auto", + reference_note="Synthetic normalization for layout testing", + ) + + assert plot_info["legend_position"] == "inside_upper_right" + assert plot_info["legend_fallback"] is False + assert plot_info["inside_legend_strategy"] in { + "compact-two-column", + "compact-wide", + } + assert plot_info["inside_legend_collision_free"] is True + axes = figure.axes[0] + assert axes.get_legend() is not None + assert len(axes.artists) == 1 + finally: + if "plot_ree" in sys.modules: + sys.modules["plot_ree"].plt.close("all") + sys.path.remove(scripts) + + def test_subunity_to_enriched_pattern_exports_successfully(tmp_path: Path) -> None: input_path = tmp_path / "cross_unity.csv" output_dir = tmp_path / "figures" @@ -385,13 +480,14 @@ def test_inside_auto_legend_falls_back_when_data_fill_the_axes() -> None: from plot_ree import build_figure elements = ["La", "Ce", "Pr", "Nd", "Sm", "Eu", "Gd"] + values = np.geomspace(1.0, 1000.0, 8) normalized = pd.DataFrame( { - "Sample": ["S1", "S2", "S3", "S4"], - "Group": ["A", "A", "B", "B"], + "Sample": [f"S{index + 1}" for index in range(8)], + "Group": [f"G{index % 4 + 1}" for index in range(8)], **{ - f"{element}_N": [1000.0, 930.0, 860.0, 790.0] - for element in elements + f"{element}_N": np.roll(values, index) + for index, element in enumerate(elements) }, } ) diff --git a/tests/test_recipe_registry.py b/tests/test_recipe_registry.py new file mode 100644 index 0000000..78f7584 --- /dev/null +++ b/tests/test_recipe_registry.py @@ -0,0 +1,555 @@ +from __future__ import annotations + +import importlib +import json +import sys +from pathlib import Path + +import pytest +import yaml + + +ROOT = Path(__file__).resolve().parents[1] +SCRIPTS = ROOT / "skills" / "geoskills" / "scripts" +sys.path.insert(0, str(SCRIPTS)) + +from geoskills_core.recipe import ( # noqa: E402 + MAX_RECIPE_BYTES, + MAX_TASKS, + RECIPE_SCHEMA_VERSION, + load_recipe, + validate_recipe, +) +from geoskills_core.registry import ( # noqa: E402 + BUILTIN_STYLE_PRESETS, + DIAGRAMS, + RegistryError, + diagram_ids, + get_diagram, + registry_snapshot, + resolve_handler, +) +from geoskills_core.version import VERSION # noqa: E402 + + +def valid_recipe() -> dict: + elements = ["La", "Ce", "Pr", "Nd", "Sm"] + return { + "schema_version": RECIPE_SCHEMA_VERSION, + "input": { + "file": "localdata/published.csv", + "sheet": None, + "layout": "auto", + }, + "columns": { + "sample_id": "Sample", + "group": "Group", + "mapping": { + **{element: f"{element}_ppm" for element in elements}, + "SiO2": "SiO2_wt%", + "Na2O": "Na2O_wt%", + "K2O": "K2O_wt%", + "MgO": "MgO_wt%", + }, + "units": { + "major_oxides": "wt%", + "trace_elements": "ppm", + }, + }, + "output": { + "directory": "outputs/submission", + "report_profile": "shareable", + }, + "presets": { + "journal-main": { + "extends": "publication-double-column", + "style": {"width_mm": 183, "height_mm": 120, "dpi": 600}, + } + }, + "confirmations": { + "input_structure_reviewed": True, + "column_mapping_reviewed": True, + "units_reviewed": True, + "plotted_data_export_reviewed": True, + }, + "tasks": [ + { + "id": "ree-main", + "diagram": "ree", + "stem": "figure-ree", + "preset": "journal-main", + "parameters": { + "reference": "chondrite-sm89", + "elements": elements, + "groups": "all", + }, + "confirmations": {}, + }, + { + "id": "harker-main", + "diagram": "harker", + "stem": "figure-harker", + "preset": "publication-double-column", + "parameters": { + "x": "SiO2", + "y": ["MgO", "K2O"], + "groups": ["Suite A", "Suite B"], + }, + "confirmations": {}, + }, + { + "id": "tas-main", + "diagram": "tas", + "stem": "figure-tas", + "preset": "review-preview", + "parameters": { + "composition_basis": "anhydrous-normalized", + "groups": "all", + }, + "confirmations": { + "volcanic_samples": True, + "composition_basis_reviewed": True, + }, + }, + ], + } + + +def issue_codes(result: dict) -> set[str]: + return {item["code"] for item in result["issues"]} + + +def test_version_and_registry_are_fixed_and_json_ready() -> None: + assert VERSION == "0.4.0-dev" + assert diagram_ids() == ("ree", "spider", "harker", "tas") + assert set(DIAGRAMS) == {"ree", "spider", "harker", "tas"} + assert BUILTIN_STYLE_PRESETS == ( + "publication-double-column", + "review-preview", + ) + + snapshot = registry_snapshot() + json.dumps(snapshot, ensure_ascii=False) + required_fields = { + "id", + "api_version", + "display_name_zh", + "display_name_en", + "operation", + "input_profile", + "adapter", + "inspector_handler", + "runner_handler", + "scientific_parameter_schema", + "style_parameter_schema", + "required_confirmations", + "required_assets", + "output_contract", + "privacy_contract", + "adapter_version", + } + assert all(set(record) == required_fields for record in snapshot) + assert get_diagram("tas").required_confirmations == ( + "volcanic_samples", + "composition_basis_reviewed", + ) + with pytest.raises(RegistryError): + get_diagram("sr-nd") + + +def test_registry_handlers_are_imported_only_when_resolved( + monkeypatch: pytest.MonkeyPatch, +) -> None: + imported: list[str] = [] + + class FakeModule: + @staticmethod + def plot_path() -> None: + return None + + def fake_import(name: str): + imported.append(name) + return FakeModule + + monkeypatch.setattr(importlib, "import_module", fake_import) + spec = get_diagram("ree") + assert imported == [] + handler = resolve_handler("ree", "runner") + assert handler is FakeModule.plot_path + assert imported == ["plot_ree"] + with pytest.raises(RegistryError): + resolve_handler("ree", "normalizer") + + def missing_import(name: str): + raise ModuleNotFoundError(name) + + monkeypatch.setattr(importlib, "import_module", missing_import) + with pytest.raises(RegistryError, match="could not be imported"): + resolve_handler("ree", "runner") + + +def test_registry_metadata_is_deeply_immutable() -> None: + schema = get_diagram("ree").scientific_parameter_schema + + with pytest.raises(TypeError): + schema["new"] = True + with pytest.raises(TypeError): + schema["reference"]["enum"] = ("unreviewed",) + + +def test_valid_recipe_normalizes_to_json_ready_structure() -> None: + result = validate_recipe(valid_recipe()) + + assert result["status"] == "ready" + assert result["issues"] == [] + assert result["recipe"]["tasks"][1]["parameters"]["y"] == [ + "MgO", + "K2O", + ] + assert result["recipe"]["columns"]["mapping"]["La"] == "La_ppm" + json.dumps(result, ensure_ascii=False, allow_nan=False) + + +def test_excel_sheet_name_preserves_significant_edge_spaces() -> None: + recipe = valid_recipe() + recipe["input"]["sheet"] = "Table S1 " + + result = validate_recipe(recipe) + + assert result["status"] == "ready" + assert result["recipe"]["input"]["sheet"] == "Table S1 " + + +@pytest.mark.parametrize( + "filename", + [ + "geoskills_ree_workflow.yaml", + "geoskills_spider_workflow.yaml", + "geoskills_major_workflow.yaml", + ], +) +def test_bundled_workflow_recipes_remain_valid(filename: str) -> None: + path = ROOT / "skills" / "geoskills" / "examples" / filename + + result = load_recipe(path) + + assert result["status"] == "ready" + + +def test_yaml_loader_is_safe_single_document_and_size_limited( + tmp_path: Path, +) -> None: + recipe_path = tmp_path / "recipe.yaml" + recipe_path.write_text( + yaml.safe_dump(valid_recipe(), allow_unicode=True, sort_keys=False), + encoding="utf-8", + ) + assert load_recipe(recipe_path)["status"] == "ready" + + multiple = tmp_path / "multiple.yaml" + multiple.write_text("---\na: 1\n---\nb: 2\n", encoding="utf-8") + assert issue_codes(load_recipe(multiple)) == {"E302"} + + unsafe = tmp_path / "unsafe.yaml" + unsafe.write_text( + "!!python/object/apply:os.system ['echo unsafe']\n", + encoding="utf-8", + ) + assert issue_codes(load_recipe(unsafe)) == {"E301"} + + oversized = tmp_path / "oversized.yaml" + oversized.write_bytes(b" " * (MAX_RECIPE_BYTES + 1)) + assert issue_codes(load_recipe(oversized)) == {"E300"} + + +@pytest.mark.parametrize( + "bad_path", + [ + "https://example.com/data.csv", + "C:/private/data.csv", + r"\\server\share\data.csv", + "~/data.csv", + "$env:DATA/data.csv", + "data/$env:DATA/data.csv", + "../data.csv", + "data/*.csv", + ], +) +def test_recipe_rejects_nonlocal_or_unsafe_member_paths( + bad_path: str, +) -> None: + recipe = valid_recipe() + recipe["input"]["file"] = bad_path + result = validate_recipe(recipe) + + assert result["status"] == "invalid" + assert "E307" in issue_codes(result) + + +@pytest.mark.parametrize( + "bad_output", + [".git/plots", ".codex/plots", "skills/output", "results/tests"], +) +def test_recipe_rejects_protected_output_directories( + bad_output: str, +) -> None: + recipe = valid_recipe() + recipe["output"]["directory"] = bad_output + + assert "E307" in issue_codes(validate_recipe(recipe)) + + +@pytest.mark.parametrize( + "bad_output", + [ + ".git./plots", + ".codex /plots", + "results/tests.", + "NUL", + "safe/COM1.txt", + ], +) +def test_windows_equivalent_protected_or_device_paths_are_rejected( + bad_output: str, +) -> None: + recipe = valid_recipe() + recipe["output"]["directory"] = bad_output + + assert "E307" in issue_codes(validate_recipe(recipe)) + + +def test_unknown_fields_and_wrong_mapping_direction_are_errors() -> None: + recipe = valid_recipe() + recipe["surprise"] = True + recipe["tasks"][0]["parameters"]["secret"] = "guess" + recipe["columns"]["mapping"] = {"La_ppm": "La"} + + result = validate_recipe(recipe) + + assert result["status"] == "invalid" + assert {"E304", "E313"}.issubset(issue_codes(result)) + assert result["recipe"] is None + + +def test_units_must_cover_every_canonical_mapping() -> None: + recipe = valid_recipe() + recipe["columns"]["units"] = {"major_oxides": "wt%"} + + result = validate_recipe(recipe) + + assert result["status"] == "invalid" + assert "E313" in issue_codes(result) + assert any( + item.get("field") == "columns.units.La" + for item in result["issues"] + ) + + +def test_windows_casefold_deduplicates_task_ids_and_stems() -> None: + recipe = valid_recipe() + duplicate = dict(recipe["tasks"][0]) + duplicate["id"] = "REE-MAIN" + duplicate["stem"] = "FIGURE-REE" + duplicate["parameters"] = dict(recipe["tasks"][0]["parameters"]) + duplicate["confirmations"] = {} + recipe["tasks"].append(duplicate) + + result = validate_recipe(recipe) + + assert result["status"] == "invalid" + assert "E308" in issue_codes(result) + duplicate_messages = [ + item["message"] for item in result["issues"] if item["code"] == "E308" + ] + assert any("任务 ID" in message for message in duplicate_messages) + assert any("stem" in message for message in duplicate_messages) + + +def test_recipe_allows_at_most_32_tasks() -> None: + recipe = valid_recipe() + base = recipe["tasks"][0] + recipe["tasks"] = [] + for index in range(MAX_TASKS + 1): + task = { + **base, + "id": f"task-{index}", + "stem": f"figure-{index}", + "parameters": dict(base["parameters"]), + "confirmations": {}, + } + recipe["tasks"].append(task) + + result = validate_recipe(recipe) + + assert result["status"] == "invalid" + assert "E309" in issue_codes(result) + + +def test_scientific_parameters_are_diagram_specific_and_explicit() -> None: + recipe = valid_recipe() + recipe["tasks"][0]["parameters"]["reference"] = "user-guessed" + recipe["tasks"][0]["parameters"]["elements"] = ["La", "Ce"] + recipe["tasks"][1]["parameters"]["x"] = "" + + result = validate_recipe(recipe) + + assert result["status"] == "invalid" + assert "E311" in issue_codes(result) + + +def spider_only_recipe(elements: list[str]) -> dict: + recipe = valid_recipe() + recipe["tasks"] = [ + { + "id": "spider-main", + "diagram": "spider", + "stem": "figure-spider", + "preset": "journal-main", + "parameters": { + "reference": "pm-sm89-modified", + "elements": elements, + "groups": "all", + }, + "confirmations": {}, + } + ] + return recipe + + +def test_spider_accepts_only_three_reviewed_oxide_alternatives() -> None: + recipe = spider_only_recipe(["K", "P", "Ti", "La", "Ce"]) + mapping = recipe["columns"]["mapping"] + mapping.update( + { + "P2O5": "P2O5_wt%", + "TiO2": "TiO2_wt%", + } + ) + + result = validate_recipe(recipe) + + assert result["status"] == "ready" + assert result["recipe"]["tasks"][0]["parameters"]["elements"] == [ + "K", + "P", + "Ti", + "La", + "Ce", + ] + + +def test_spider_still_accepts_direct_k_p_ti_ppm_mappings() -> None: + recipe = spider_only_recipe(["K", "P", "Ti", "La", "Ce"]) + mapping = recipe["columns"]["mapping"] + mapping.pop("K2O") + mapping.update({"K": "K_ppm", "P": "P_ppm", "Ti": "Ti_ppm"}) + + result = validate_recipe(recipe) + + assert result["status"] == "ready" + + +def test_spider_oxide_alternatives_do_not_hide_other_missing_elements() -> None: + recipe = spider_only_recipe(["K", "P", "Ti", "La", "Nb"]) + recipe["columns"]["mapping"].update( + { + "P2O5": "P2O5_wt%", + "TiO2": "TiO2_wt%", + } + ) + + result = validate_recipe(recipe) + + assert result["status"] == "invalid" + missing = [ + item + for item in result["issues"] + if item["code"] == "E313" + and item.get("field", "").endswith("elements[4]") + ] + assert len(missing) == 1 + assert "Nb" in missing[0]["message"] + + +@pytest.mark.parametrize( + ("field_path", "bad_value"), + [ + (("input", "layout"), ["auto"]), + (("output", "report_profile"), ["shareable"]), + (("tasks", 0, "diagram"), ["ree"]), + (("tasks", 0, "parameters", "reference"), ["chondrite-sm89"]), + ], +) +def test_unhashable_enum_values_return_issues_instead_of_crashing( + field_path: tuple[object, ...], + bad_value: object, +) -> None: + recipe = valid_recipe() + target = recipe + for part in field_path[:-1]: + target = target[part] + target[field_path[-1]] = bad_value + + result = validate_recipe(recipe) + + assert result["status"] == "invalid" + assert result["issues"] + + +def test_extreme_numbers_return_structured_errors( + tmp_path: Path, +) -> None: + recipe = valid_recipe() + recipe["presets"]["journal-main"]["style"]["width_mm"] = 10**10_000 + direct = validate_recipe(recipe) + assert direct["status"] == "invalid" + assert "E314" in issue_codes(direct) + + yaml_path = tmp_path / "huge-integer.yaml" + yaml_path.write_text( + "schema_version: " + ("9" * 5000) + "\n", + encoding="utf-8", + ) + loaded = load_recipe(yaml_path) + assert loaded["status"] == "invalid" + assert "E301" in issue_codes(loaded) + + +def test_false_top_level_confirmation_returns_needs_review() -> None: + recipe = valid_recipe() + recipe["confirmations"]["units_reviewed"] = False + + result = validate_recipe(recipe) + + assert result["status"] == "needs_review" + assert issue_codes(result) == {"R312"} + assert result["recipe"] is not None + + +def test_tas_requires_volcanic_and_composition_confirmations() -> None: + recipe = valid_recipe() + recipe["tasks"][2]["confirmations"]["volcanic_samples"] = False + + result = validate_recipe(recipe) + + assert result["status"] == "needs_review" + assert "R312" in issue_codes(result) + + +def test_as_reported_tas_requires_provisional_acceptance() -> None: + recipe = valid_recipe() + task = recipe["tasks"][2] + task["parameters"]["composition_basis"] = "as-reported" + + missing = validate_recipe(recipe) + assert missing["status"] == "invalid" + assert "E305" in issue_codes(missing) + + task["confirmations"]["provisional_classification_accepted"] = False + pending = validate_recipe(recipe) + assert pending["status"] == "needs_review" + assert "R312" in issue_codes(pending) + + task["confirmations"]["provisional_classification_accepted"] = True + ready = validate_recipe(recipe) + assert ready["status"] == "ready" diff --git a/tests/test_v03_cli_contract.py b/tests/test_v03_cli_contract.py new file mode 100644 index 0000000..9d4b75a --- /dev/null +++ b/tests/test_v03_cli_contract.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +SCRIPTS = ROOT / "skills" / "geoskills" / "scripts" + + +LEGACY_CLI_CONTRACT = { + "inspect_data.py": ("--sheet", "--sample-column", "--group-column"), + "inspect_spider_data.py": ("--sheet", "--sample-column", "--group-column"), + "inspect_major_data.py": ("--sheet", "--sample-column", "--group-column"), + "normalize_ree.py": ("--output", "--overwrite"), + "normalize_spider.py": ("--output", "--reference", "--overwrite"), + "plot_ree.py": ( + "--output-dir", + "--axes-frame", + "--legend-layout", + "--overwrite", + ), + "plot_spider.py": ( + "--output-dir", + "--reference", + "--elements", + "--overwrite", + ), + "plot_harker.py": ("--output-dir", "--x", "--y", "--overwrite"), + "plot_tas.py": ( + "--output-dir", + "--confirm-volcanic", + "--composition-basis", + "--overwrite", + ), +} + + +def test_v03_command_names_and_options_remain_available() -> None: + """Keep the released v0.3 command surface while v0.4 adds one entry point.""" + for script_name, required_options in LEGACY_CLI_CONTRACT.items(): + result = subprocess.run( + [sys.executable, str(SCRIPTS / script_name), "--help"], + cwd=ROOT, + text=True, + encoding="utf-8", + capture_output=True, + check=False, + ) + assert result.returncode == 0, result.stderr + for option in required_options: + assert option in result.stdout diff --git a/tests/test_workflow_cli.py b/tests/test_workflow_cli.py new file mode 100644 index 0000000..ae06dce --- /dev/null +++ b/tests/test_workflow_cli.py @@ -0,0 +1,514 @@ +from __future__ import annotations + +import json +import shutil +import subprocess +import sys +from pathlib import Path + +import pytest +import yaml + + +ROOT = Path(__file__).resolve().parents[1] +SCRIPTS = ROOT / "skills" / "geoskills" / "scripts" +CLI = SCRIPTS / "geoskills.py" +EXAMPLES = ROOT / "skills" / "geoskills" / "examples" +sys.path.insert(0, str(SCRIPTS)) + +from geoskills_core.workflow import create_plan, execute_plan # noqa: E402 + + +REE_ELEMENTS = ["La", "Ce", "Pr", "Nd", "Sm"] + + +def ree_recipe( + *, + report_profile: str = "shareable", + confirmed: bool = True, + tasks: int = 1, +) -> dict: + return { + "schema_version": "geoskills.recipe/v1", + "input": { + "file": "input.csv", + "sheet": None, + "layout": "row-per-sample", + }, + "columns": { + "sample_id": "Sample", + "group": "Group", + "mapping": { + element: f"{element}_ppm" for element in REE_ELEMENTS + }, + "units": { + "major_oxides": "wt%", + "trace_elements": "ppm", + }, + }, + "output": { + "directory": "bundle", + "report_profile": report_profile, + }, + "presets": {}, + "confirmations": { + "input_structure_reviewed": confirmed, + "column_mapping_reviewed": True, + "units_reviewed": True, + "plotted_data_export_reviewed": True, + }, + "tasks": [ + { + "id": f"ree-{index + 1}", + "diagram": "ree", + "stem": f"figure-ree-{index + 1}", + "preset": "review-preview", + "parameters": { + "reference": "chondrite-sm89", + "elements": REE_ELEMENTS, + "groups": "all", + }, + "confirmations": {}, + } + for index in range(tasks) + ], + } + + +def prepare_recipe( + tmp_path: Path, + *, + report_profile: str = "shareable", + confirmed: bool = True, + tasks: int = 1, +) -> Path: + shutil.copyfile( + EXAMPLES / "synthetic_ree_data.csv", + tmp_path / "input.csv", + ) + path = tmp_path / "recipe.yaml" + path.write_text( + yaml.safe_dump( + ree_recipe( + report_profile=report_profile, + confirmed=confirmed, + tasks=tasks, + ), + allow_unicode=True, + sort_keys=False, + ), + encoding="utf-8", + ) + return path + + +def prepare_spider_recipe(tmp_path: Path) -> Path: + shutil.copyfile( + EXAMPLES / "synthetic_spider_data.csv", + tmp_path / "input.csv", + ) + elements = ["Rb", "Ba", "Th", "U", "Nb", "Ta", "K", "P", "Ti"] + recipe = { + "schema_version": "geoskills.recipe/v1", + "input": { + "file": "input.csv", + "sheet": None, + "layout": "row-per-sample", + }, + "columns": { + "sample_id": "Sample", + "group": "Group", + "mapping": { + "Rb": "Rb_ppm", + "Ba": "Ba_ppm", + "Th": "Th_ppm", + "U": "U_ppm", + "Nb": "Nb_ppm", + "Ta": "Ta_ppm", + "K2O": "K2O_wt%", + "P2O5": "P2O5_wt%", + "TiO2": "TiO2_wt%", + }, + "units": { + "major_oxides": "wt%", + "trace_elements": "ppm", + }, + }, + "output": { + "directory": "bundle", + "report_profile": "shareable", + }, + "presets": {}, + "confirmations": { + "input_structure_reviewed": True, + "column_mapping_reviewed": True, + "units_reviewed": True, + "plotted_data_export_reviewed": True, + }, + "tasks": [ + { + "id": "spider-main", + "diagram": "spider", + "stem": "figure-spider", + "preset": "review-preview", + "parameters": { + "reference": "pm-sm89-modified", + "elements": elements, + "groups": "all", + }, + "confirmations": {}, + } + ], + } + path = tmp_path / "recipe.yaml" + path.write_text( + yaml.safe_dump(recipe, allow_unicode=True, sort_keys=False), + encoding="utf-8", + ) + return path + + +def prepare_major_multitask_recipe(tmp_path: Path) -> Path: + shutil.copyfile( + EXAMPLES / "synthetic_major_element_data.csv", + tmp_path / "synthetic_major_element_data.csv", + ) + recipe = yaml.safe_load( + (EXAMPLES / "geoskills_major_workflow.yaml").read_text( + encoding="utf-8" + ) + ) + recipe["output"]["directory"] = "bundle" + recipe["presets"]["journal-main"]["style"]["dpi"] = 300 + path = tmp_path / "recipe.yaml" + path.write_text( + yaml.safe_dump(recipe, allow_unicode=True, sort_keys=False), + encoding="utf-8", + ) + return path + + +def run_cli(*arguments: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, str(CLI), *arguments], + cwd=ROOT, + text=True, + encoding="utf-8", + capture_output=True, + check=False, + ) + + +def parse_stdout(result: subprocess.CompletedProcess[str]) -> dict: + lines = result.stdout.splitlines() + assert len(lines) == 1 + return json.loads(lines[0]) + + +def test_version_works_without_site_packages() -> None: + result = subprocess.run( + [sys.executable, "-S", str(CLI), "version"], + cwd=ROOT, + text=True, + encoding="utf-8", + capture_output=True, + check=False, + ) + + report = parse_stdout(result) + assert result.returncode == 0 + assert report["status"] == "ready" + assert report["result"]["tool_version"] == "0.4.0-dev" + + +def test_self_check_is_machine_readable_and_private_by_default() -> None: + result = run_cli("self-check") + + report = parse_stdout(result) + assert result.returncode == 0 + assert report["status"] == "ready" + assert "python_executable" not in report["result"] + assert str(ROOT.resolve()) not in result.stdout + + +def test_plan_then_run_shareable_bundle(tmp_path: Path) -> None: + recipe_path = prepare_recipe(tmp_path) + plan_path = tmp_path / "plan.json" + + planned = run_cli( + "plan", + str(recipe_path), + "--output", + str(plan_path), + ) + plan_report = parse_stdout(planned) + + assert planned.returncode == 0 + assert plan_report["status"] == "ready" + assert plan_path.is_file() + assert not (tmp_path / "bundle").exists() + plan_text = plan_path.read_text(encoding="utf-8") + assert str(tmp_path.resolve()) not in plan_text + assert "SYN-01" not in plan_text + + executed = run_cli( + "run", + str(recipe_path), + "--plan", + str(plan_path), + ) + run_report = parse_stdout(executed) + + assert executed.returncode == 0 + assert run_report["status"] == "ready" + task_dir = tmp_path / "bundle" / "ree-1" + for suffix in ("svg", "pdf", "tiff", "png", "report.json", "qa.md"): + assert (task_dir / f"figure-ree-1.{suffix}").is_file() + assert not (task_dir / "figure-ree-1.source_data.csv").exists() + report_text = (tmp_path / "bundle" / "run.report.json").read_text( + encoding="utf-8" + ) + task_report_text = ( + task_dir / "figure-ree-1.report.json" + ).read_text(encoding="utf-8") + assert str(tmp_path.resolve()) not in report_text + assert "SYN-01" not in report_text + assert '"data_min"' not in task_report_text + assert '"data_max"' not in task_report_text + + +def test_missing_confirmation_returns_exit_two_and_no_figures( + tmp_path: Path, +) -> None: + recipe_path = prepare_recipe(tmp_path, confirmed=False) + plan_path = tmp_path / "plan.json" + + result = run_cli( + "plan", + str(recipe_path), + "--output", + str(plan_path), + ) + report = parse_stdout(result) + + assert result.returncode == 2 + assert report["status"] == "needs_confirmation" + assert plan_path.is_file() + assert not (tmp_path / "bundle").exists() + + +def test_blocked_input_still_writes_reviewable_plan(tmp_path: Path) -> None: + recipe_path = prepare_recipe(tmp_path) + (tmp_path / "input.csv").unlink() + plan_path = tmp_path / "plan.json" + + result = create_plan(recipe_path, plan_path) + + assert result["status"] == "blocked" + assert plan_path.is_file() + plan = json.loads(plan_path.read_text(encoding="utf-8")) + assert plan["status"] == "blocked" + assert plan["input"]["filename"] == "input.csv" + issue = next(item for item in result["issues"] if item["code"] == "E106") + assert issue["message"] == "未找到输入文件:input.csv。" + assert "input.file" in issue["suggested_action"] + assert str(tmp_path.resolve()) not in plan_path.read_text(encoding="utf-8") + + +def test_same_recipe_and_input_produce_identical_plan(tmp_path: Path) -> None: + recipe_path = prepare_recipe(tmp_path) + first_path = tmp_path / "first-plan.json" + second_path = tmp_path / "second-plan.json" + + first = create_plan(recipe_path, first_path) + second = create_plan(recipe_path, second_path) + + assert first["plan_id"] == second["plan_id"] + assert first_path.read_bytes() == second_path.read_bytes() + + +def test_stale_input_blocks_run_without_creating_output( + tmp_path: Path, +) -> None: + recipe_path = prepare_recipe(tmp_path) + plan_path = tmp_path / "plan.json" + assert create_plan(recipe_path, plan_path)["status"] == "ready" + with (tmp_path / "input.csv").open("a", encoding="utf-8") as handle: + handle.write("\n") + + result = execute_plan(recipe_path, plan_path) + + assert result["status"] == "blocked" + assert result["issues"][0]["code"] == "R803" + assert not (tmp_path / "bundle").exists() + + +def test_local_reproducible_keeps_sensitive_plotted_source( + tmp_path: Path, +) -> None: + recipe_path = prepare_recipe( + tmp_path, + report_profile="local-reproducible", + ) + plan_path = tmp_path / "plan.json" + assert create_plan(recipe_path, plan_path)["status"] == "ready" + + result = execute_plan(recipe_path, plan_path) + + assert result["status"] == "ready" + source = ( + tmp_path + / "bundle" + / "ree-1" + / "figure-ree-1.source_data.csv" + ) + assert source.is_file() + report_text = (tmp_path / "bundle" / "run.report.json").read_text( + encoding="utf-8" + ) + assert "plotted-source-sensitive" in report_text + assert str(tmp_path.resolve()) not in report_text + + +def test_spider_recipe_preserves_three_reviewed_oxide_conversions( + tmp_path: Path, +) -> None: + recipe_path = prepare_spider_recipe(tmp_path) + plan_path = tmp_path / "plan.json" + + assert create_plan(recipe_path, plan_path)["status"] == "ready" + result = execute_plan(recipe_path, plan_path) + + assert result["status"] == "ready" + task_dir = tmp_path / "bundle" / "spider-main" + assert (task_dir / "figure-spider.svg").is_file() + report = json.loads( + (task_dir / "figure-spider.report.json").read_text(encoding="utf-8") + ) + assert report["details"]["diagram"] == "spider" + assert { + item["element"] for item in report["details"]["oxide_conversions"] + } == {"K", "P", "Ti"} + assert ( + report["details"]["reference_assets"][0]["doi"] + == "10.1144/GSL.SP.1989.042.01.19" + ) + assert report["details"]["plot_summary"]["inside_legend_strategy"] in { + "stacked", + "compact-two-column", + "compact-wide", + None, + } + report_text = json.dumps(report, ensure_ascii=False, sort_keys=True) + assert '"data_min"' not in report_text + assert '"data_max"' not in report_text + assert report["qa"]["svg_text_editable"] is True + assert not (task_dir / "figure-spider.source_data.csv").exists() + + +def test_major_example_runs_harker_and_tas_as_one_transaction( + tmp_path: Path, +) -> None: + recipe_path = prepare_major_multitask_recipe(tmp_path) + plan_path = tmp_path / "plan.json" + + planned = create_plan(recipe_path, plan_path) + executed = execute_plan(recipe_path, plan_path) + + assert planned["status"] == "ready" + assert executed["status"] == "ready" + assert executed["task_count"] == 2 + assert (tmp_path / "bundle" / "harker-main" / "figure-harker.pdf").is_file() + assert (tmp_path / "bundle" / "tas-main" / "figure-tas.pdf").is_file() + run_report = json.loads( + (tmp_path / "bundle" / "run.report.json").read_text(encoding="utf-8") + ) + harker_report_text = ( + tmp_path + / "bundle" + / "harker-main" + / "figure-harker.report.json" + ).read_text(encoding="utf-8") + assert run_report["qa"]["all_tasks_completed"] is True + assert run_report["qa"]["task_count"] == 2 + assert '"data_min"' not in harker_report_text + assert '"data_max"' not in harker_report_text + assert "48.4986" not in harker_report_text + assert "74.9823" not in harker_report_text + + +def test_provisional_tas_commits_outputs_but_returns_review( + tmp_path: Path, +) -> None: + recipe_path = prepare_major_multitask_recipe(tmp_path) + recipe = yaml.safe_load(recipe_path.read_text(encoding="utf-8")) + tas_task = recipe["tasks"][1] + tas_task["parameters"]["composition_basis"] = "as-reported" + tas_task["confirmations"]["provisional_classification_accepted"] = True + recipe["tasks"] = [tas_task] + recipe_path.write_text( + yaml.safe_dump(recipe, allow_unicode=True, sort_keys=False), + encoding="utf-8", + ) + plan_path = tmp_path / "plan.json" + + assert create_plan(recipe_path, plan_path)["status"] == "ready" + result = execute_plan(recipe_path, plan_path) + + assert result["status"] == "review" + assert (tmp_path / "bundle" / "tas-main" / "figure-tas.svg").is_file() + run_report = json.loads( + (tmp_path / "bundle" / "run.report.json").read_text(encoding="utf-8") + ) + assert run_report["status"] == "review" + assert run_report["review_required"] is True + + +def test_multitask_failure_preserves_previous_complete_directory( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + from geoskills_core import workflow + + recipe_path = prepare_recipe(tmp_path, tasks=2) + plan_path = tmp_path / "plan.json" + assert create_plan(recipe_path, plan_path)["status"] == "ready" + assert execute_plan(recipe_path, plan_path)["status"] == "ready" + target = tmp_path / "bundle" + previous_figure = ( + target / "ree-1" / "figure-ree-1.svg" + ).read_bytes() + (target / "previous.txt").write_text("keep", encoding="utf-8") + + calls = 0 + real_run_task = workflow.run_task + + def fake_run_task(task, prepared, *, output_dir): + nonlocal calls + calls += 1 + if calls == 2: + return { + "status": "error", + "issues": [ + { + "code": "E-test", + "severity": "error", + "message": "synthetic failure", + } + ], + } + return real_run_task( + task, + prepared, + output_dir=output_dir, + ) + + monkeypatch.setattr(workflow, "run_task", fake_run_task) + + result = execute_plan(recipe_path, plan_path, overwrite=True) + + assert result["status"] == "error" + assert (target / "previous.txt").read_text(encoding="utf-8") == "keep" + assert ( + target / "ree-1" / "figure-ree-1.svg" + ).read_bytes() == previous_figure diff --git a/tests/test_workflow_science.py b/tests/test_workflow_science.py new file mode 100644 index 0000000..b783c70 --- /dev/null +++ b/tests/test_workflow_science.py @@ -0,0 +1,611 @@ +from __future__ import annotations + +import json +import shutil +import sys +from pathlib import Path +from typing import Any + +import pandas as pd +import pytest +import yaml + + +ROOT = Path(__file__).resolve().parents[1] +SCRIPTS = ROOT / "skills" / "geoskills" / "scripts" +EXAMPLES = ROOT / "skills" / "geoskills" / "examples" +LOCAL_RECIPE = ROOT / "local_data" / "v04_published_regression.yaml" +sys.path.insert(0, str(SCRIPTS)) + +from geoskills_core.workflow import ( # noqa: E402 + build_plan, + create_plan, + execute_plan, +) + + +TOP_CONFIRMATIONS = { + "input_structure_reviewed": True, + "column_mapping_reviewed": True, + "units_reviewed": True, + "plotted_data_export_reviewed": True, +} +REE_ELEMENTS = ["La", "Ce", "Pr"] +SPIDER_ELEMENTS = ["Rb", "Ba", "Th", "U", "Nb"] + + +def _task( + task_id: str, + diagram: str, + parameters: dict[str, Any], + *, + confirmations: dict[str, bool] | None = None, +) -> dict[str, Any]: + return { + "id": task_id, + "diagram": diagram, + "stem": f"figure-{task_id}", + "preset": "test-small", + "parameters": parameters, + "confirmations": confirmations or {}, + } + + +def _write_recipe( + directory: Path, + frame: pd.DataFrame, + *, + mapping: dict[str, str], + units: dict[str, str], + tasks: list[dict[str, Any]], + report_profile: str = "shareable", +) -> Path: + directory.mkdir(parents=True, exist_ok=True) + frame.to_csv(directory / "input.csv", index=False) + recipe = { + "schema_version": "geoskills.recipe/v1", + "input": { + "file": "input.csv", + "sheet": None, + "layout": "row-per-sample", + }, + "columns": { + "sample_id": "Sample", + "group": "Group" if "Group" in frame.columns else None, + "mapping": mapping, + "units": units, + }, + "output": { + "directory": "bundle", + "report_profile": report_profile, + }, + "presets": { + "test-small": { + "extends": "review-preview", + "style": { + "width_mm": 100, + "height_mm": 70, + "dpi": 72, + }, + } + }, + "confirmations": TOP_CONFIRMATIONS, + "tasks": tasks, + } + path = directory / "recipe.yaml" + path.write_text( + yaml.safe_dump(recipe, allow_unicode=True, sort_keys=False), + encoding="utf-8", + ) + return path + + +def _issue_codes(value: object) -> set[str]: + codes: set[str] = set() + if isinstance(value, dict): + code = value.get("code") + if isinstance(code, str): + codes.add(code) + for child in value.values(): + codes.update(_issue_codes(child)) + elif isinstance(value, list): + for child in value: + codes.update(_issue_codes(child)) + return codes + + +@pytest.mark.parametrize( + ("case_name", "frame", "mapping"), + [ + ( + "unit-conflict", + pd.DataFrame( + { + "Sample": ["S1", "S2"], + "Group": ["A", "A"], + "La_ppb": [20.0, 22.0], + "Ce_ppm": [40.0, 44.0], + "Pr_ppm": [5.0, 5.5], + } + ), + {"La": "La_ppb", "Ce": "Ce_ppm", "Pr": "Pr_ppm"}, + ), + ( + "analyte-conflict", + pd.DataFrame( + { + "Sample": ["S1", "S2"], + "Group": ["A", "A"], + "La_ppm": [20.0, 22.0], + "Ce_ppm": [40.0, 44.0], + "Pr_ppm": [5.0, 5.5], + } + ), + {"La": "Ce_ppm", "Ce": "La_ppm", "Pr": "Pr_ppm"}, + ), + ], +) +def test_e415_rejects_explicit_header_conflicts( + tmp_path: Path, + case_name: str, + frame: pd.DataFrame, + mapping: dict[str, str], +) -> None: + recipe_path = _write_recipe( + tmp_path / case_name, + frame, + mapping=mapping, + units={"trace_elements": "ppm"}, + tasks=[ + _task( + "ree-main", + "ree", + { + "reference": "chondrite-sm89", + "elements": REE_ELEMENTS, + "groups": "all", + }, + ) + ], + ) + + result = build_plan(recipe_path) + + assert result["status"] == "blocked" + assert "E415" in _issue_codes(result) + + +def test_ree_and_spider_group_filters_reach_exported_source( + tmp_path: Path, +) -> None: + elements = [*REE_ELEMENTS, *SPIDER_ELEMENTS] + frame = pd.DataFrame( + { + "Sample": ["S1", "S2", "S3"], + "Group": ["A", "A", "B"], + "La_ppm": [20.0, 22.0, 35.0], + "Ce_ppm": [40.0, 44.0, 70.0], + "Pr_ppm": [5.0, 5.5, 8.0], + "Rb_ppm": [30.0, 32.0, 60.0], + "Ba_ppm": [300.0, 320.0, 600.0], + "Th_ppm": [5.0, 5.5, 10.0], + "U_ppm": [1.2, 1.3, 2.5], + "Nb_ppm": [15.0, 16.0, 30.0], + } + ) + recipe_path = _write_recipe( + tmp_path / "filtered-patterns", + frame, + mapping={element: f"{element}_ppm" for element in elements}, + units={"trace_elements": "ppm"}, + tasks=[ + _task( + "ree-main", + "ree", + { + "reference": "chondrite-sm89", + "elements": REE_ELEMENTS, + "groups": ["A"], + }, + ), + _task( + "spider-main", + "spider", + { + "reference": "pm-sm89", + "elements": SPIDER_ELEMENTS, + "groups": ["A"], + }, + ), + ], + report_profile="local-reproducible", + ) + plan_path = recipe_path.parent / "plan.json" + + assert create_plan(recipe_path, plan_path)["status"] == "ready" + executed = execute_plan(recipe_path, plan_path) + + assert executed["status"] == "ready" + for task_id in ("ree-main", "spider-main"): + source = pd.read_csv( + recipe_path.parent + / "bundle" + / task_id + / f"figure-{task_id}.source_data.csv" + ) + assert len(source) == 2 + assert set(source["Group"].astype(str).str.strip()) == {"A"} + + +def test_missing_group_blocks_with_e412(tmp_path: Path) -> None: + frame = pd.DataFrame( + { + "Sample": ["S1", "S2"], + "Group": ["A", "B"], + "La_ppm": [20.0, 30.0], + "Ce_ppm": [40.0, 60.0], + "Pr_ppm": [5.0, 7.5], + } + ) + recipe_path = _write_recipe( + tmp_path / "missing-group", + frame, + mapping={element: f"{element}_ppm" for element in REE_ELEMENTS}, + units={"trace_elements": "ppm"}, + tasks=[ + _task( + "ree-main", + "ree", + { + "reference": "chondrite-sm89", + "elements": REE_ELEMENTS, + "groups": ["not-present"], + }, + ) + ], + ) + + result = build_plan(recipe_path) + + assert result["status"] == "blocked" + assert "E412" in _issue_codes(result) + + +def test_spider_zero_positive_selection_blocks_with_e419( + tmp_path: Path, +) -> None: + frame = pd.DataFrame( + { + "Sample": ["S1", "S2", "S3", "S4"], + "Group": ["A", "A", "B", "B"], + **{ + f"{element}_ppm": [10.0, 12.0, None, None] + for element in SPIDER_ELEMENTS + }, + } + ) + recipe_path = _write_recipe( + tmp_path / "no-positive-spider-data", + frame, + mapping={ + element: f"{element}_ppm" for element in SPIDER_ELEMENTS + }, + units={"trace_elements": "ppm"}, + tasks=[ + _task( + "spider-main", + "spider", + { + "reference": "pm-sm89", + "elements": SPIDER_ELEMENTS, + "groups": ["B"], + }, + ) + ], + ) + + result = build_plan(recipe_path) + + assert result["status"] == "blocked" + assert "E419" in _issue_codes(result) + + +def test_ree_zero_positive_selection_blocks_with_e419( + tmp_path: Path, +) -> None: + frame = pd.DataFrame( + { + "Sample": ["S1", "S2", "S3", "S4"], + "Group": ["A", "A", "B", "B"], + **{ + f"{element}_ppm": [10.0, 12.0, None, None] + for element in REE_ELEMENTS + }, + } + ) + recipe_path = _write_recipe( + tmp_path / "no-positive-ree-data", + frame, + mapping={ + element: f"{element}_ppm" for element in REE_ELEMENTS + }, + units={"trace_elements": "ppm"}, + tasks=[ + _task( + "ree-main", + "ree", + { + "reference": "chondrite-sm89", + "elements": REE_ELEMENTS, + "groups": ["B"], + }, + ) + ], + ) + + result = build_plan(recipe_path) + + assert result["status"] == "blocked" + assert "E419" in _issue_codes(result) + + +def test_spider_direct_k_is_preferred_before_oxide_fallback( + tmp_path: Path, +) -> None: + frame = pd.DataFrame( + { + "Sample": ["S1", "S2"], + "Group": ["A", "A"], + "Rb_ppm": [None, None], + "Ba_ppm": [None, None], + "Th_ppm": [None, None], + "U_ppm": [None, None], + "K_ppm": [25000.0, 27000.0], + } + ) + elements = ["Rb", "Ba", "Th", "U", "K"] + recipe_path = _write_recipe( + tmp_path / "direct-k", + frame, + mapping={element: f"{element}_ppm" for element in elements}, + units={"trace_elements": "ppm"}, + tasks=[ + _task( + "spider-main", + "spider", + { + "reference": "pm-sm89", + "elements": elements, + "groups": "all", + }, + ) + ], + ) + + result = build_plan(recipe_path) + + assert result["status"] == "ready" + assert "E419" not in _issue_codes(result) + + +def test_harker_limits_panels_with_e416(tmp_path: Path) -> None: + frame = pd.read_csv(EXAMPLES / "synthetic_major_element_data.csv") + mapping = { + "SiO2": "SiO2_wt%", + "TiO2": "TiO2_wt%", + "Al2O3": "Al2O3_wt%", + "Fe2O3T": "Fe2O3T_wt%", + "MnO": "MnO_wt%", + "MgO": "MgO_wt%", + "CaO": "CaO_wt%", + "Na2O": "Na2O_wt%", + "K2O": "K2O_wt%", + "P2O5": "P2O5_wt%", + "Rb": "Rb_ppm", + } + recipe_path = _write_recipe( + tmp_path / "too-many-harker-panels", + frame, + mapping=mapping, + units={"major_oxides": "wt%", "trace_elements": "ppm"}, + tasks=[ + _task( + "harker-main", + "harker", + { + "x": "SiO2", + "y": [ + "TiO2", + "Al2O3", + "Fe2O3T", + "MnO", + "MgO", + "CaO", + "Na2O", + "K2O", + "P2O5", + "Rb", + ], + "groups": "all", + }, + ) + ], + ) + + result = build_plan(recipe_path) + + assert result["status"] == "blocked" + assert "E416" in _issue_codes(result) + + +def test_harker_selected_subset_needs_two_complete_pairs( + tmp_path: Path, +) -> None: + frame = pd.DataFrame( + { + "Sample": ["S1", "S2"], + "Group": ["A", "B"], + "SiO2_wt%": [50.0, 55.0], + "MgO_wt%": [7.0, 4.0], + } + ) + recipe_path = _write_recipe( + tmp_path / "short-harker-subset", + frame, + mapping={"SiO2": "SiO2_wt%", "MgO": "MgO_wt%"}, + units={"major_oxides": "wt%"}, + tasks=[ + _task( + "harker-main", + "harker", + { + "x": "SiO2", + "y": ["MgO"], + "groups": ["B"], + }, + ) + ], + ) + + result = build_plan(recipe_path) + + assert result["status"] == "blocked" + assert "E417" in _issue_codes(result) + + +def test_tas_selected_subset_needs_a_complete_coordinate( + tmp_path: Path, +) -> None: + frame = pd.DataFrame( + { + "Sample": ["S1", "S2", "S3"], + "Group": ["A", "B", "B"], + "SiO2_wt%": [50.0, None, None], + "Na2O_wt%": [3.0, None, None], + "K2O_wt%": [1.0, None, None], + } + ) + recipe_path = _write_recipe( + tmp_path / "empty-tas-subset", + frame, + mapping={ + "SiO2": "SiO2_wt%", + "Na2O": "Na2O_wt%", + "K2O": "K2O_wt%", + }, + units={"major_oxides": "wt%"}, + tasks=[ + _task( + "tas-main", + "tas", + { + "composition_basis": "anhydrous-normalized", + "groups": ["B"], + }, + confirmations={ + "volcanic_samples": True, + "composition_basis_reviewed": True, + }, + ) + ], + ) + + result = build_plan(recipe_path) + + assert result["status"] == "blocked" + assert "E418" in _issue_codes(result) + + +def test_major_example_basis_and_scientific_audit_reports( + tmp_path: Path, +) -> None: + source_data = EXAMPLES / "synthetic_major_element_data.csv" + frame = pd.read_csv(source_data) + major_columns = [ + "SiO2_wt%", + "TiO2_wt%", + "Al2O3_wt%", + "Fe2O3T_wt%", + "MnO_wt%", + "MgO_wt%", + "CaO_wt%", + "Na2O_wt%", + "K2O_wt%", + "P2O5_wt%", + ] + totals = frame[major_columns].sum(axis=1) + assert ((totals - 100.0).abs() <= 0.001).all() + + case = tmp_path / "major-example" + case.mkdir() + shutil.copyfile( + source_data, + case / "synthetic_major_element_data.csv", + ) + recipe = yaml.safe_load( + (EXAMPLES / "geoskills_major_workflow.yaml").read_text( + encoding="utf-8" + ) + ) + recipe["output"]["directory"] = "bundle" + recipe["presets"]["journal-main"]["style"]["dpi"] = 300 + recipe_path = case / "recipe.yaml" + recipe_path.write_text( + yaml.safe_dump(recipe, allow_unicode=True, sort_keys=False), + encoding="utf-8", + ) + plan_path = case / "plan.json" + + planned = create_plan(recipe_path, plan_path) + executed = execute_plan(recipe_path, plan_path) + + assert planned["status"] == "ready" + assert executed["status"] == "ready" + harker_report = json.loads( + ( + case + / "bundle" + / "harker-main" + / "figure-harker.report.json" + ).read_text(encoding="utf-8") + ) + tas_report = json.loads( + ( + case + / "bundle" + / "tas-main" + / "figure-tas.report.json" + ).read_text(encoding="utf-8") + ) + run_report = json.loads( + (case / "bundle" / "run.report.json").read_text(encoding="utf-8") + ) + analyte_units = harker_report["details"]["analyte_units"] + assert analyte_units["SiO2"] == "wt%" + assert analyte_units["Rb"] == "ppm" + assert tas_report["details"]["scientific_confirmations"] == { + "volcanic_samples": True, + "composition_basis_reviewed": True, + } + assert run_report["details"]["data_confirmations"] == TOP_CONFIRMATIONS + + +def test_local_published_recipe_build_plan_is_ready() -> None: + if not LOCAL_RECIPE.is_file(): + pytest.skip("local published regression recipe is not available") + recipe = yaml.safe_load(LOCAL_RECIPE.read_text(encoding="utf-8")) + input_path = LOCAL_RECIPE.parent / str(recipe["input"]["file"]) + if not input_path.is_file(): + pytest.skip("local published regression input is not available") + + result = build_plan(LOCAL_RECIPE) + + assert result["status"] == "ready" + assert result["plan"] is not None + tasks = result["plan"]["tasks"] + assert {task["diagram"] for task in tasks} == { + "ree", + "spider", + "harker", + } + assert all(task["inspection"]["status"] == "ready" for task in tasks) diff --git a/tests/test_workflow_security.py b/tests/test_workflow_security.py new file mode 100644 index 0000000..3768990 --- /dev/null +++ b/tests/test_workflow_security.py @@ -0,0 +1,517 @@ +from __future__ import annotations + +import json +import os +import shutil +import sys +from pathlib import Path +from typing import Any + +import pytest +import yaml + + +ROOT = Path(__file__).resolve().parents[1] +SCRIPTS = ROOT / "skills" / "geoskills" / "scripts" +EXAMPLES = ROOT / "skills" / "geoskills" / "examples" +sys.path.insert(0, str(SCRIPTS)) + +from geoskills_core import adapters, export, workflow # noqa: E402 +from geoskills_core.export import ( # noqa: E402 + AtomicDirectory, + BundleExportError, +) +from geoskills_core.workflow import create_plan, execute_plan # noqa: E402 + + +REE_ELEMENTS = ["La", "Ce", "Pr", "Nd", "Sm"] + + +def _ree_recipe( + *, + input_file: str = "input.csv", + output_directory: str = "bundle", + task_count: int = 1, +) -> dict[str, Any]: + return { + "schema_version": "geoskills.recipe/v1", + "input": { + "file": input_file, + "sheet": None, + "layout": "row-per-sample", + }, + "columns": { + "sample_id": "Sample", + "group": "Group", + "mapping": { + element: f"{element}_ppm" for element in REE_ELEMENTS + }, + "units": { + "major_oxides": "wt%", + "trace_elements": "ppm", + }, + }, + "output": { + "directory": output_directory, + "report_profile": "shareable", + }, + "presets": {}, + "confirmations": { + "input_structure_reviewed": True, + "column_mapping_reviewed": True, + "units_reviewed": True, + "plotted_data_export_reviewed": True, + }, + "tasks": [ + { + "id": f"ree-{index + 1}", + "diagram": "ree", + "stem": f"figure-ree-{index + 1}", + "preset": "review-preview", + "parameters": { + "reference": "chondrite-sm89", + "elements": REE_ELEMENTS, + "groups": "all", + }, + "confirmations": {}, + } + for index in range(task_count) + ], + } + + +def _write_recipe(tmp_path: Path, recipe: dict[str, Any]) -> Path: + path = tmp_path / "recipe.yaml" + path.write_text( + yaml.safe_dump(recipe, allow_unicode=True, sort_keys=False), + encoding="utf-8", + ) + return path + + +def _prepare_ree_recipe( + tmp_path: Path, + *, + output_directory: str = "bundle", + task_count: int = 1, +) -> Path: + shutil.copyfile(EXAMPLES / "synthetic_ree_data.csv", tmp_path / "input.csv") + return _write_recipe( + tmp_path, + _ree_recipe( + output_directory=output_directory, + task_count=task_count, + ), + ) + + +def _plan_ready(tmp_path: Path) -> tuple[Path, Path]: + recipe_path = _prepare_ree_recipe(tmp_path) + plan_path = tmp_path / "plan.json" + assert create_plan(recipe_path, plan_path)["status"] == "ready" + return recipe_path, plan_path + + +def _issue_codes(result: dict[str, Any]) -> set[str]: + return {str(item["code"]) for item in result.get("issues", [])} + + +def test_output_directory_cannot_be_recipe_root(tmp_path: Path) -> None: + shutil.copyfile(EXAMPLES / "synthetic_ree_data.csv", tmp_path / "input.csv") + recipe_path = _write_recipe( + tmp_path, + _ree_recipe(output_directory="."), + ) + plan_path = tmp_path / "plan.json" + + result = create_plan(recipe_path, plan_path) + + assert result["status"] == "error" + assert "E307" in _issue_codes(result) + assert not plan_path.exists() + assert (tmp_path / "input.csv").is_file() + + +def test_input_inside_output_is_blocked_without_touching_source( + tmp_path: Path, +) -> None: + source_dir = tmp_path / "bundle" + source_dir.mkdir() + source_path = source_dir / "input.csv" + shutil.copyfile(EXAMPLES / "synthetic_ree_data.csv", source_path) + original = source_path.read_bytes() + recipe_path = _write_recipe( + tmp_path, + _ree_recipe( + input_file="bundle/input.csv", + output_directory="bundle", + ), + ) + plan_path = tmp_path / "plan.json" + + planned = create_plan(recipe_path, plan_path) + + assert planned["status"] == "blocked" + assert "E820" in _issue_codes(planned) + assert source_path.read_bytes() == original + assert plan_path.is_file() + + +def test_plan_cannot_replace_recipe_even_with_overwrite( + tmp_path: Path, +) -> None: + recipe_path = _prepare_ree_recipe(tmp_path) + original = recipe_path.read_bytes() + + result = create_plan(recipe_path, recipe_path, overwrite=True) + + assert result["status"] == "error" + assert "E822" in _issue_codes(result) + assert recipe_path.read_bytes() == original + + +def test_plan_cannot_be_written_inside_final_output(tmp_path: Path) -> None: + recipe_path = _prepare_ree_recipe(tmp_path) + plan_path = tmp_path / "bundle" / "plan.json" + + result = create_plan(recipe_path, plan_path) + + assert result["status"] == "error" + assert "E822" in _issue_codes(result) + assert not plan_path.exists() + + +def test_non_geoskills_file_cannot_be_overwritten_as_plan( + tmp_path: Path, +) -> None: + recipe_path = _prepare_ree_recipe(tmp_path) + plan_path = tmp_path / "plan.json" + sentinel = b"not a GeoSkills plan\n" + plan_path.write_bytes(sentinel) + + result = create_plan(recipe_path, plan_path, overwrite=True) + + assert result["status"] == "error" + assert "E821" in _issue_codes(result) + assert plan_path.read_bytes() == sentinel + + +def test_ordinary_directory_cannot_be_overwritten( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + recipe_path, plan_path = _plan_ready(tmp_path) + target = tmp_path / "bundle" + target.mkdir() + sentinel = target / "user-file.txt" + sentinel.write_text("keep", encoding="utf-8") + + def forbidden_runner(*args: Any, **kwargs: Any) -> dict[str, Any]: + raise AssertionError("runner must not start for an unowned directory") + + monkeypatch.setattr(workflow, "run_task", forbidden_runner) + + result = execute_plan(recipe_path, plan_path, overwrite=True) + + assert result["status"] == "blocked" + assert "E826" in _issue_codes(result) + assert sentinel.read_text(encoding="utf-8") == "keep" + + +def test_existing_output_without_overwrite_stops_before_runner( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + recipe_path, plan_path = _plan_ready(tmp_path) + target = tmp_path / "bundle" + target.mkdir() + sentinel = target / "user-file.txt" + sentinel.write_text("keep", encoding="utf-8") + + def forbidden_runner(*args: Any, **kwargs: Any) -> dict[str, Any]: + raise AssertionError("runner must not start without overwrite approval") + + monkeypatch.setattr(workflow, "run_task", forbidden_runner) + + result = execute_plan(recipe_path, plan_path) + + assert result["status"] == "blocked" + assert "E825" in _issue_codes(result) + assert sentinel.read_text(encoding="utf-8") == "keep" + + +def test_failed_overwrite_preserves_valid_previous_bundle( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + recipe_path, plan_path = _plan_ready(tmp_path) + assert execute_plan(recipe_path, plan_path)["status"] == "ready" + target = tmp_path / "bundle" + previous_svg = target / "ree-1" / "figure-ree-1.svg" + previous_hash = workflow.sha256_file(previous_svg) + sentinel = target / "previous.txt" + sentinel.write_text("keep", encoding="utf-8") + + def failed_runner(*args: Any, **kwargs: Any) -> dict[str, Any]: + return { + "status": "error", + "issues": [ + { + "code": "E-test", + "severity": "error", + "message": "synthetic failure", + } + ], + } + + monkeypatch.setattr(workflow, "run_task", failed_runner) + + result = execute_plan(recipe_path, plan_path, overwrite=True) + + assert result["status"] == "error" + assert sentinel.read_text(encoding="utf-8") == "keep" + assert workflow.sha256_file(previous_svg) == previous_hash + + +def test_atomic_directory_keeps_backup_when_restore_also_fails( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + target = tmp_path / "bundle" + target.mkdir() + (target / "old.txt").write_text("recover me", encoding="utf-8") + real_replace = os.replace + + with pytest.raises(BundleExportError): + with AtomicDirectory(target, overwrite=True) as transaction: + staging = transaction.staging_dir + (staging / "new.txt").write_text("new", encoding="utf-8") + + def fail_promotion_and_restore( + source: str | os.PathLike[str], + destination: str | os.PathLike[str], + ) -> None: + source_path = Path(source).resolve() + destination_path = Path(destination).resolve() + if ( + source_path == staging + and destination_path == target.resolve() + ): + raise OSError("synthetic promotion failure") + if ( + source_path.name.startswith(".geoskills-run-backup-") + and destination_path == target.resolve() + ): + raise OSError("synthetic restore failure") + real_replace(source, destination) + + monkeypatch.setattr(export.os, "replace", fail_promotion_and_restore) + transaction.commit() + + backups = list(tmp_path.glob(".geoskills-run-backup-*")) + assert len(backups) == 1 + assert (backups[0] / "old.txt").read_text(encoding="utf-8") == "recover me" + assert not target.exists() + + +@pytest.mark.parametrize( + "document", + [ + { + "schema_version": "geoskills.plan/v1", + "plan_id": "not-a-sha256", + "status": "ready", + "tasks": [{"id": "ree-1"}], + }, + { + "schema_version": "geoskills.plan/v1", + "plan_id": "0" * 64, + "status": "ready", + "tasks": [{"id": "Ree-1"}, {"id": "ree-1"}], + }, + { + "schema_version": "geoskills.plan/v1", + "plan_id": "0" * 64, + "status": "not-a-status", + "tasks": [{"id": "ree-1"}], + }, + ], + ids=[ + "invalid-plan-id", + "duplicate-casefold-task-id", + "invalid-status", + ], +) +def test_malformed_plan_returns_structured_error( + tmp_path: Path, + document: dict[str, Any], +) -> None: + recipe_path = _prepare_ree_recipe(tmp_path) + plan_path = tmp_path / "malformed-plan.json" + plan_path.write_text(json.dumps(document), encoding="utf-8") + + result = execute_plan(recipe_path, plan_path) + + assert result["status"] == "error" + assert _issue_codes(result) == {"E807"} + assert not (tmp_path / "bundle").exists() + + +def test_unexpected_runner_file_rejects_whole_bundle_without_path_leak( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + recipe_path, plan_path = _plan_ready(tmp_path) + secret_value = "PRIVATE-SAMPLE-VALUE" + + def runner_with_extra_file( + task: dict[str, Any], + prepared: Any, + *, + output_dir: Path, + ) -> dict[str, Any]: + output_dir.mkdir(parents=True) + (output_dir / "secret-debug.csv").write_text( + secret_value, + encoding="utf-8", + ) + return {"status": "ready", "issues": [], "plot": {}} + + monkeypatch.setattr(workflow, "run_task", runner_with_extra_file) + + result = execute_plan(recipe_path, plan_path) + payload = json.dumps(result, ensure_ascii=False) + + assert result["status"] == "error" + assert "E823" in _issue_codes(result) + assert not (tmp_path / "bundle").exists() + assert secret_value not in payload + assert str(tmp_path.resolve()) not in payload + assert tmp_path.name not in payload + + +def test_invalid_figure_is_rejected_without_exposing_temporary_path( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + recipe_path, plan_path = _plan_ready(tmp_path) + + def runner_with_fake_figures( + task: dict[str, Any], + prepared: Any, + *, + output_dir: Path, + ) -> dict[str, Any]: + output_dir.mkdir(parents=True) + stem = str(task["stem"]) + (output_dir / f"{stem}.svg").write_text( + 'x', + encoding="utf-8", + ) + (output_dir / f"{stem}.pdf").write_bytes( + b"%PDF-1.4\n/CIDFontType2\n%%EOF\n" + ) + (output_dir / f"{stem}.png").write_text( + "not a png", + encoding="utf-8", + ) + (output_dir / f"{stem}.tiff").write_text( + "not a tiff", + encoding="utf-8", + ) + return {"status": "ready", "issues": [], "plot": {}} + + monkeypatch.setattr(workflow, "run_task", runner_with_fake_figures) + + result = execute_plan(recipe_path, plan_path) + payload = json.dumps(result, ensure_ascii=False) + + assert result["status"] == "error" + assert "E824" in _issue_codes(result) + assert not (tmp_path / "bundle").exists() + assert str(tmp_path.resolve()) not in payload + assert tmp_path.name not in payload + assert ".geoskills-run-stage-" not in payload + + +def test_review_result_includes_r804_in_api_and_saved_report( + tmp_path: Path, +) -> None: + shutil.copyfile( + EXAMPLES / "synthetic_major_element_data.csv", + tmp_path / "synthetic_major_element_data.csv", + ) + recipe = yaml.safe_load( + (EXAMPLES / "geoskills_major_workflow.yaml").read_text( + encoding="utf-8" + ) + ) + recipe["output"]["directory"] = "bundle" + recipe["presets"]["journal-main"]["style"]["dpi"] = 300 + tas_task = recipe["tasks"][1] + tas_task["parameters"]["composition_basis"] = "as-reported" + tas_task["confirmations"]["provisional_classification_accepted"] = True + recipe["tasks"] = [tas_task] + recipe_path = _write_recipe(tmp_path, recipe) + plan_path = tmp_path / "plan.json" + assert create_plan(recipe_path, plan_path)["status"] == "ready" + + result = execute_plan(recipe_path, plan_path) + report = json.loads( + (tmp_path / "bundle" / "run.report.json").read_text(encoding="utf-8") + ) + + assert result["status"] == "review" + assert "R804" in _issue_codes(result) + assert "R804" in _issue_codes(report) + + +def test_execute_uses_one_snapshot_when_source_changes_after_copy( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + recipe_path, plan_path = _plan_ready(tmp_path) + source_path = tmp_path / "input.csv" + marker = "TOCTOU-MUTATION" + real_copyfile = shutil.copyfile + mutation_count = 0 + observed_canonical = "" + real_run_task = workflow.run_task + + def copy_then_mutate( + source: str | os.PathLike[str], + destination: str | os.PathLike[str], + *args: Any, + **kwargs: Any, + ) -> str: + nonlocal mutation_count + result = real_copyfile(source, destination, *args, **kwargs) + if ( + mutation_count == 0 + and Path(destination).name.startswith("source_snapshot") + ): + mutation_count += 1 + with Path(source).open("a", encoding="utf-8") as handle: + handle.write(f"\n{marker}\n") + return result + + def observing_runner( + task: dict[str, Any], + prepared: Any, + *, + output_dir: Path, + ) -> dict[str, Any]: + nonlocal observed_canonical + observed_canonical = prepared.path.read_text(encoding="utf-8") + return real_run_task(task, prepared, output_dir=output_dir) + + monkeypatch.setattr(adapters.shutil, "copyfile", copy_then_mutate) + monkeypatch.setattr(workflow, "run_task", observing_runner) + + result = execute_plan(recipe_path, plan_path) + + assert mutation_count == 1 + assert marker in source_path.read_text(encoding="utf-8") + assert marker not in observed_canonical + assert result["status"] == "ready" + assert (tmp_path / "bundle" / "ree-1" / "figure-ree-1.svg").is_file()