diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS deleted file mode 100644 index 62933d7..0000000 --- a/.github/CODEOWNERS +++ /dev/null @@ -1,2 +0,0 @@ -# Replace or extend this list as repository maintainers change. -* @Austin243 diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml deleted file mode 100644 index 3ba13e0..0000000 --- a/.github/ISSUE_TEMPLATE/config.yml +++ /dev/null @@ -1 +0,0 @@ -blank_issues_enabled: false diff --git a/.github/ISSUE_TEMPLATE/feature.yml b/.github/ISSUE_TEMPLATE/feature.yml deleted file mode 100644 index de99649..0000000 --- a/.github/ISSUE_TEMPLATE/feature.yml +++ /dev/null @@ -1,47 +0,0 @@ -name: Feature proposal -description: Propose a scoped software, format, adapter, or documentation improvement. -title: "[Feature]: " -labels: [enhancement, triage] -body: - - type: textarea - id: problem - attributes: - label: Problem - description: What concrete workflow or research question is currently blocked? - validations: - required: true - - type: textarea - id: proposal - attributes: - label: Proposed behavior - description: Describe inputs, outputs, failure behavior, and the smallest useful scope. - validations: - required: true - - type: dropdown - id: milestone - attributes: - label: Likely scope - options: - - ElecTrace 0.1 ordered VASP path - - ElecTrace 0.2 electronic-feature lineage - - ElecTrace 1.0 sparse trajectories - - Documentation or project infrastructure - validations: - required: true - - type: textarea - id: science - attributes: - label: Scientific assumptions and alternatives - description: Identify method dependence, related tools, and simpler alternatives. - - type: textarea - id: compatibility - attributes: - label: Reproducibility and compatibility - description: Note schema, CLI, configuration, licensing, data, and migration concerns. - - type: checkboxes - id: terms - attributes: - label: Scope confirmation - options: - - label: I have separated implemented needs from speculative scientific claims. - required: true diff --git a/.github/ISSUE_TEMPLATE/scientific-validation.yml b/.github/ISSUE_TEMPLATE/scientific-validation.yml deleted file mode 100644 index 0b1c869..0000000 --- a/.github/ISSUE_TEMPLATE/scientific-validation.yml +++ /dev/null @@ -1,52 +0,0 @@ -name: Scientific validation -description: Report a method-dependent discrepancy, unstable event, or validation case. -title: "[Validation]: " -labels: [scientific-validation, triage] -body: - - type: markdown - attributes: - value: >- - This form is for scientific behavior, not sensitive security reports. Share only - data and tool output you are authorized to redistribute. - - type: textarea - id: claim - attributes: - label: Observation or disputed claim - description: State the machine-readable event and the interpretation separately. - validations: - required: true - - type: textarea - id: provenance - attributes: - label: Calculation and analysis provenance - description: >- - Include code/tool versions, functional, potentials, cutoffs, k-points, grids, - spin/charge, path spacing, plugin settings, and ElecTrace revision. - validations: - required: true - - type: textarea - id: convergence - attributes: - label: Convergence and sensitivity checks - description: Describe grid, image-density, electronic, threshold, and matching tests. - validations: - required: true - - type: textarea - id: evidence - attributes: - label: Supporting, negative, and conflicting evidence - description: Include all relevant descriptors and warnings, not only supporting ones. - validations: - required: true - - type: textarea - id: fixture - attributes: - label: Redistributable reproduction - description: Link or describe the smallest authorized fixture and its SHA-256 digest. - - type: checkboxes - id: terms - attributes: - label: Data confirmation - options: - - label: I am authorized to share every attached input and output. - required: true diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md deleted file mode 100644 index 24891c4..0000000 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ /dev/null @@ -1,31 +0,0 @@ -## Summary - - - -## Scientific and user impact - - - -## Validation - - - -- [ ] `ruff check .` -- [ ] `ruff format --check .` -- [ ] `mypy src` -- [ ] `pytest -ra --strict-config --strict-markers` - -## Reproducibility and compatibility - -- [ ] Tests cover success, ambiguity, and meaningful failure modes. -- [ ] No credentials, private data, licensed pseudopotentials, or proprietary binaries - are included. -- [ ] User-visible changes are documented in `CHANGELOG.md` and relevant guides. -- [ ] CLI/config/schema compatibility is unchanged, or migration notes are included. -- [ ] Generated results record inputs, versions, settings, and warnings. -- [ ] Scientific claims distinguish evidence from interpretation and disclose negative - or conflicting evidence. - -## Related issues - - diff --git a/.github/dependabot.yml b/.github/dependabot.yml deleted file mode 100644 index c2231cf..0000000 --- a/.github/dependabot.yml +++ /dev/null @@ -1,27 +0,0 @@ -version: 2 -updates: - - package-ecosystem: pip - directory: "/" - schedule: - interval: weekly - day: monday - time: "09:00" - timezone: America/New_York - open-pull-requests-limit: 5 - labels: [dependencies, python] - groups: - development-tools: - dependency-type: development - - - package-ecosystem: github-actions - directory: "/" - schedule: - interval: weekly - day: monday - time: "09:30" - timezone: America/New_York - open-pull-requests-limit: 5 - labels: [dependencies, github-actions] - groups: - actions: - patterns: ["*"] diff --git a/.github/release.yml b/.github/release.yml deleted file mode 100644 index 643312d..0000000 --- a/.github/release.yml +++ /dev/null @@ -1,19 +0,0 @@ -changelog: - exclude: - labels: - - skip-changelog - authors: - - dependabot - categories: - - title: Breaking changes - labels: [breaking-change] - - title: Scientific features - labels: [scientific-feature] - - title: Features - labels: [enhancement] - - title: Fixes - labels: [bug] - - title: Documentation and maintenance - labels: [documentation, dependencies] - - title: Other changes - labels: ["*"] diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d7a35c9..c440468 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,11 +47,6 @@ jobs: run: ruff format --check . - name: Type check run: mypy src - - name: Repository hygiene hooks - env: - SKIP: ruff-check,ruff-format - run: pre-commit run --all-files --show-diff-on-failure - test: name: Test Python ${{ matrix.python-version }} runs-on: ubuntu-latest @@ -115,6 +110,5 @@ jobs: python -m electrace --version electrace --version electrace --help - electrace benchmark --output .ci-benchmark.json electrace demo --output .ci-demo electrace inspect .ci-demo/electrace-results diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 389932f..8c43539 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -53,8 +53,3 @@ jobs: electrace demo --output .ci-demo python -m json.tool .ci-demo/electrace-results/analysis.json > /dev/null python -c 'import json; from jsonschema import Draft202012Validator as V; s=json.load(open("schemas/analysis-v0.1.schema.json")); d=json.load(open(".ci-demo/electrace-results/analysis.json")); V.check_schema(s); V(s).validate(d)' - - name: Generate and validate synthetic benchmark - run: | - electrace benchmark --output .ci-benchmark.json - python -m json.tool .ci-benchmark.json > /dev/null - python -c 'import json; from jsonschema import Draft202012Validator as V; s=json.load(open("schemas/benchmark-v0.1.schema.json")); d=json.load(open(".ci-benchmark.json")); V.check_schema(s); V(s).validate(d)' diff --git a/.gitignore b/.gitignore index 959ce6a..b422031 100644 --- a/.gitignore +++ b/.gitignore @@ -42,9 +42,7 @@ Thumbs.db # Local ElecTrace products .electrace-demo/ .electrace-work/ -.ci-benchmark.json .ci-demo/ -electrace-benchmark.json electrace-results/ # Local logs and scratch diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml deleted file mode 100644 index 5552bd9..0000000 --- a/.pre-commit-config.yaml +++ /dev/null @@ -1,27 +0,0 @@ -minimum_pre_commit_version: "3.7.0" -default_install_hook_types: [pre-commit, pre-push] - -repos: - - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v6.0.0 - hooks: - - id: check-added-large-files - args: [--maxkb=1024] - - id: check-case-conflict - - id: check-json - - id: check-merge-conflict - - id: check-toml - - id: check-yaml - - id: debug-statements - - id: detect-private-key - - id: end-of-file-fixer - - id: mixed-line-ending - args: [--fix=lf] - - id: trailing-whitespace - - - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.16.0 - hooks: - - id: ruff-check - args: [--fix] - - id: ruff-format diff --git a/AGENTS.md b/AGENTS.md index cf8bca3..8ad20a2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -7,7 +7,7 @@ chemical truth. ## Before changing code -- Read `README.md`, `docs/scientific-scope.md`, and `docs/architecture.md`. +- Read `README.md`, `docs/workflows.md`, and `docs/scientific-scope.md`. - Preserve the row-vector convention `cartesian = fractional @ cell`. - Treat frame-local feature identifiers as local only; persistent identity belongs to the matching engine. @@ -28,25 +28,22 @@ mypy src/electrace pytest --cov=electrace --cov-report=term-missing mkdocs build --strict python -m build -electrace benchmark --output electrace-benchmark.json electrace demo --output .electrace-demo ``` -The deterministic demo and synthetic benchmark are regression fixtures, not -scientific validation. +The deterministic demo is a regression fixture, not scientific validation. ## Layout - `src/electrace/models.py`: serializable scientific domain models. - `src/electrace/geometry.py`: periodic and deformation-aware geometry. - `src/electrace/matching.py`: atom/feature assignment and split/merge candidates. -- `src/electrace/events.py`: event evidence, persistence, and hysteresis. +- `src/electrace/events.py`: event evidence and consecutive-frame persistence. - `src/electrace/io/`: VASP discovery, parsing, descriptor normalization. - `src/electrace/plugins/`: capability checks and non-executing command plans. - `src/electrace/workflow/`: manifests, staging, explicit execution, and collection. - `src/electrace/pipeline.py`: end-to-end analysis and artifact bundle. - `src/electrace/report.py`: offline self-contained HTML report. -- `src/electrace/benchmark.py`: versioned synthetic lineage regression suite. - `schemas/`: versioned machine-readable contracts. ## Scientific change requirements @@ -55,7 +52,7 @@ Any change to matching costs, thresholds, conservation logic, confidence, or eve semantics must include: 1. focused unit tests; -2. an adversarial or ambiguous case; +2. an adversarial or boundary case; 3. an explanation of the scientific assumption; 4. unchanged or intentionally versioned output schemas; 5. a changelog entry when behavior visible to users changes. diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d6d2a2..fec71aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,85 +1,29 @@ # Changelog -All notable changes to this project will be documented here. The format follows -[Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and released versions will -follow [Semantic Versioning](https://semver.org/) after the initial development phase. - -## [Unreleased] - -### Changed - -- Standardized opt-in machine output on the versioned `electrace.cli.v1` success and - error envelope, including parse-time CLI failures and stable exit semantics; the - Typer minimum is now 0.20, the oldest validated release for this command surface. -- Published the source under the BSD 3-Clause License with clean public package and - citation metadata. -- Consolidated the distribution, Python package, command, configuration filename, - schema identifiers, demo paths, result directories, and provenance fields under the - single ElecTrace/`electrace` identity for version `0.1.0a2`. -- This is an intentional breaking pre-release namespace change. No compatibility - aliases are provided because no earlier tag or release was published; development - environments and generated artifacts should be reinstalled or migrated. -- Added explicit local, external, and merged descriptor-source policies with duplicate - feature rejection, path-wide configured diagnostics, and deterministic batch command - planning. -- Wired configured descriptor scales and event thresholds, sustained property-state - persistence, hysteresis, validated high/medium confidence cutoffs, and a validated - minimum event-confidence filter into matching and event classification. -- Require at least one complete, semantically compatible, tolerance-passing charge or - volume channel before promoting split/merge hyperedges. Proximity-only, partial, - incompatible, and failed-channel candidates now remain unpromoted with structured - rejection diagnostics. -- Detect atom and feature ambiguities by verifying complete alternative assignments - within a configurable cost gap. Strict analysis now fails before publication; - permissive analysis records warnings and suppresses affected mapping, lineage, - birth/death, property-change, and topology claims. Hyperedge-equivalent backbone - ties are retained as resolved evidence rather than false ambiguity failures. -- Keep plugin planning and top-level analysis non-executing while adding a separate, - explicit workflow boundary: local execution and Slurm submission now require - `--yes`, and preparing Slurm scripts remains non-executing. -- Keep the default installation lightweight by using the bundled assignment solver; - SciPy is now an optional `accelerate` extra, and unused ASE, mkdocstrings, and - setuptools-stub dependencies have been removed from the package extras. -- Stream plain VASP volumetric grids, bound descriptor and workflow-record reads - during I/O, require valid UTF-8 evidence, and publish generated text atomically - without following destination symlinks. -- Reuse periodic-cell factorizations and indexed feature lookups across matching and - event tracking, substantially reducing repeated geometry work without changing - scientific thresholds, event semantics, or result schemas. -- Make workflow restaging transactional, validate serialized jobs against the hashed - manifest before execution, and use bounded heap-based DAG scheduling and batch - Slurm preflight to avoid partial or quadratic large-workflow behavior. -- Report only the tested CPython 3.11–3.13 range as supported in `electrace doctor`. +## Unreleased ### Added -- Added strictly read-only `electrace workflow validate --json` planning for resolved - frames, commands, inputs, outputs, dependencies, scheduler settings, and warnings. -- Added a compact coding-agent guide and installable ElecTrace skill for routing - existing-output analysis and approval-gated local or Slurm workflows through the - public CLI. -- Added a deterministic `electrace benchmark` command, versioned benchmark JSON - schema, and 15 labeled synthetic cases spanning periodic wrapping, coordinate and - cell perturbations, ordering, direct events, conservation-backed split/merge, - negative controls, semantic incompatibility, and ambiguity handling. The report - explicitly identifies itself as a regression suite rather than scientific - validation. -- Added the versioned `electrace.workflow.v1` YAML contract, public JSON Schema and - example, canonical index/slice frame selection, marker-protected source-isolated - staging, dependency-aware local execution, reviewed Slurm script preparation and - submission, filesystem status, validated descriptor collection, and analysis of - the isolated collected series. -- Added `electrace volume` for strict, read-only summaries and periodic sampling of - plain VASP scalar grids. -- Initial ElecTrace foundation with a focused 0.1 ordered-path workflow. -- Ordered VASP path models, plugin adapter boundary, and feature normalization. -- Periodic matching and evidence-oriented event-analysis foundations. -- Command-line workflows for environment diagnosis, demos, initialization, analysis, - and result inspection. -- Scientific-scope, protocol, reproducibility, governance, and contributor - documentation. -- Python 3.11–3.13 CI, repository templates, pre-commit hooks, and dependency updates. -- Documentation CI validation of the feature-schema example and the deterministic - demo's generated analysis bundle. +- Public BSD-3-Clause packaging under the ElecTrace/`electrace` name. +- Ordered VASP path discovery, periodic atom and feature matching, event evidence, + provenance, and JSON/CSV/HTML reports. +- Normalization for canonical feature JSON and primary BaderKit, critic2, LOBSTER, + Bader, and Chargemol outputs. +- YAML workflows for selected frames, source-isolated staging, explicit local or + Slurm execution, output collection, and analysis. +- Read-only `electrace workflow validate --json`, a compact coding-agent guide, and + an installable ElecTrace skill. +- Streaming plain VASP grid summaries and periodic sampling through `electrace volume`. + +### Changed -[Unreleased]: https://github.com/Austin243/ElecTrace/commits/main +- Machine-readable command results use the versioned `electrace.cli.v1` envelope when + `--json` is supported and requested. +- External execution requires explicit `--yes`; plugin planning remains non-executing. +- Split/merge candidates require compatible charge or volume conservation evidence. +- Periodic geometry and assignment reuse reduce repeated matching work; SciPy remains + an optional accelerator. +- The lean pre-release surface omits legacy parser aliases, the synthetic benchmark, + HDF5-wrapped JSON, exhaustive alternative-assignment enumeration, and topology- + specific hysteresis. +- Supported and tested Python versions are 3.11–3.13. diff --git a/CITATION.cff b/CITATION.cff index cfb6399..a7e839e 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -7,7 +7,7 @@ type: software authors: - family-names: Ellis given-names: Austin -version: 0.1.0a4 +version: 0.1.0a5 license: BSD-3-Clause abstract: >- ElecTrace is a research-software framework for tracking electronic-topology diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0a2b010..dfd7a54 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,19 +1,11 @@ # Contributing to ElecTrace -Thank you for helping build ElecTrace. The project welcomes software engineering, -documentation, test fixtures, benchmarks, and careful scientific validation. +Focused fixes, parser fixtures, tests, and documentation improvements are welcome. +Open an issue before introducing a new data model, external-tool integration, or +matching algorithm. Report security issues through the process in +[SECURITY.md](SECURITY.md). -## Before you start - -- Search existing issues and pull requests. -- Use a feature request for a small change or open a design issue before a new data - model, external-tool integration, or matching/event algorithm. -- Use the scientific-validation template for a disputed interpretation, convergence - failure, or method-dependent discrepancy. -- Report security-sensitive findings privately as described in [SECURITY.md](SECURITY.md). -- Confirm that you may redistribute every fixture or output you contribute. - -## Development setup +## Development ElecTrace supports CPython 3.11–3.13. @@ -22,79 +14,26 @@ python -m venv .venv source .venv/bin/activate python -m pip install --upgrade pip python -m pip install -e ".[dev]" -pre-commit install -``` - -Run the local checks before opening a pull request: - -```bash ruff check . ruff format --check . mypy src -pytest -ra --strict-config --strict-markers -pre-commit run --all-files +pytest ``` -Use `ruff format .` and `ruff check --fix .` for safe automated formatting and lint -fixes. Do not weaken a check simply to make a pull request green; document a narrow -exception when the code genuinely requires one. - -## Pull-request expectations - -Keep changes focused. A pull request should explain the problem, approach, scientific -impact, validation, and any compatibility or reproducibility consequences. - -Every user-visible change should include: - -- tests for expected behavior and meaningful failures; -- documentation and a changelog entry; -- deterministic fixtures small enough for the repository; -- migration notes for CLI, configuration, or schema changes; -- explicit disclosure of manual curation, thresholds, and unsupported cases. - -Code review is not a substitute for scientific validation. Changes to feature -matching, event semantics, confidence labels, or physical interpretation need a -domain-aware reviewer and evidence across both positive and negative cases. - -## Testing external adapters +Keep pull requests focused. Include tests for changed behavior, update user-facing +documentation when needed, and mention compatibility or scientific consequences. +Matching, event, and confidence changes need both a positive case and a meaningful +counterexample. CI must not require network access, licensed software, a cluster, or a DFT run. -Adapter tests should use inspectable command plans and the smallest legally -redistributable parser fixtures. Tests must verify at least: - -- argument boundaries and path handling; -- required and expected file declarations; -- executable/version provenance; -- malformed, missing, partial, and non-finite output behavior; -- conversion to the versioned normalized schema. - -Never commit credentials, licensed pseudopotentials, proprietary executables, or -unpublished scientific inputs without documented authorization. - -## Data and schema changes - -`electrace.features.v1` is a contract, not an incidental serialization. Contract -changes require: - -1. a design rationale and compatibility analysis; -2. updated JSON Schema, examples, and protocol documentation; -3. reader and writer tests; -4. a migration path or a new schema identifier; -5. a changelog entry. - -Unknown major versions must be rejected. Do not reinterpret an existing field in -place. - -## Commit and review hygiene +Use the smallest redistributable fixtures needed to test external-tool adapters, and +never commit credentials, pseudopotentials, proprietary executables, or unpublished +scientific inputs without authorization. -- Write imperative, specific commit subjects. -- Avoid drive-by formatting or unrelated generated files. -- Keep raw generated reports out of source control unless they are intentional test - snapshots. -- Link the issue or scientific question being addressed. -- Respond to review with code or a documented rationale; resolve threads only after - the concern is addressed. +The `electrace.features.v1` format is a public contract. Changes require an updated +schema, example, reader/writer tests, and either a compatible migration or a new +schema identifier. -All contributors must follow the [Code of Conduct](CODE_OF_CONDUCT.md). By submitting -a contribution, you confirm that you have the right to submit it and agree that it -will be licensed under the repository's [BSD 3-Clause License](LICENSE). +By contributing, you agree that your work is licensed under the repository's +[BSD 3-Clause License](LICENSE) and that you have the right to submit it. All +contributors must follow the [Code of Conduct](CODE_OF_CONDUCT.md). diff --git a/README.md b/README.md index e6ff2bf..d32e2ef 100644 --- a/README.md +++ b/README.md @@ -16,11 +16,10 @@ workflow. - Reads `POSCAR`/`CONTCAR` structures and supported plain `ELFCAR`/`CHGCAR`-like grids. - Ingests supported Bader/BaderKit, critic2, LOBSTER, and Chargemol results. - Matches atoms and electronic features across periodic and changing cells. -- Builds candidate feature correspondences and reports births, deaths, split/merge - candidates, descriptor changes, and ambiguities. +- Builds feature correspondences and reports births, deaths, conservation-backed + split/merge candidates, and descriptor changes. - Stages VASP, Bader/BaderKit, critic2, and LOBSTER steps from a YAML workflow. -- Writes JSON, CSV, HTML, and optional HDF5 results with available source and tool - provenance. +- Writes JSON, CSV, and HTML results with available source and tool provenance. ## Install @@ -34,9 +33,8 @@ python -m pip install . ``` The base installation includes a deterministic assignment solver. Use -`python -m pip install ".[accelerate]"` for the SciPy solver, -`python -m pip install ".[hdf5]"` for HDF5 export, or -`python -m pip install ".[all]"` for both. +`python -m pip install ".[accelerate]"` for the SciPy solver, or +`python -m pip install ".[all]"` for all optional features. ## Try the demo @@ -83,8 +81,8 @@ electrace workflow stage electrace-workflow.yaml ``` Validation resolves the selected frames, commands, inputs, outputs, dependencies, and -scheduler settings without creating a workspace. Commands also accept `--json` for a -common `electrace.cli.v1` response envelope suitable for scripts and coding agents. +scheduler settings without creating a workspace. Use `--json` when a script or coding +agent needs machine-readable output. Run locally and analyze the collected outputs: @@ -106,8 +104,7 @@ same path analysis. ## Learn more -See the [quick start](docs/quickstart.md), [workflow guide](docs/workflows.md), -[coding-agent guide](docs/agents.md), and [scientific scope](docs/scientific-scope.md) -for detailed configuration and supported data formats. +See [Using ElecTrace](docs/workflows.md), the [coding-agent reference](docs/agents.md), +and [scientific scope](docs/scientific-scope.md). ElecTrace is available under the [BSD 3-Clause License](LICENSE). diff --git a/SECURITY.md b/SECURITY.md index c1c1c4e..04de7e5 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -1,62 +1,25 @@ # Security policy -## Supported versions +Security fixes target the latest default-branch revision; this pre-release does not +promise long-term support for older versions. -ElecTrace is pre-release software. Security fixes are applied to the latest commit on -the default branch; no released version is currently promised long-term support. +## Reporting -| Version | Supported | -| --- | --- | -| Default branch | Yes | -| Older commits or forks | No | +Do not open a public issue or attach sensitive data. Use GitHub's **Report a +vulnerability** flow, or ask a maintainer through an established private channel for +a secure reporting route. Include the affected revision, a minimal reproduction, +impact, and any known workaround when it is safe to do so. -## Reporting a vulnerability +## Scope -Do not open a public issue for a suspected vulnerability or attach sensitive input -data to an issue. +ElecTrace reads external calculation files and can launch explicitly authorized +executables. Planning does not execute commands; the workflow `--yes` boundary does. +ElecTrace is not a sandbox, so do not run unknown binaries or process hostile data +with valuable credentials or write access available. -Use GitHub's **Report a vulnerability** / private security advisory flow for this -repository. If that feature is unavailable, contact a repository maintainer through -an already established private channel and ask for a secure reporting route. Do not -send exploit details through a new public channel. +Relevant issues include path traversal, symlink escapes, unsafe overwrites, command +argument handling, executable substitution, denial of service from malformed inputs, +and sensitive data leaking into provenance or reports. -Include, when safe: - -- affected revision and operating system; -- minimal reproduction and impact; -- whether untrusted files, paths, plugin output, or HTML reports are involved; -- any known workaround; -- your preferred disclosure and credit handling. - -Maintainers will acknowledge a complete report as soon as practical, assess scope, -coordinate a fix, and agree on disclosure timing. Because this is a research project -without a guaranteed response-time service level, do not rely on it for isolation of -hostile workloads. - -## Security model - -ElecTrace reads calculation directories and external-tool output that may be -untrusted. Its adapters construct command plans containing an argument vector, -working directory, standard input, required inputs, and expected outputs. Constructing -a plan does not execute it. Callers that execute a plan remain responsible for -sandboxing and access control. - -Security-sensitive areas include: - -- shell metacharacters, path traversal, symlink escapes, and output overwrites; -- malicious or malformed VASP/plugin files causing denial of service; -- external executable substitution through `PATH` or configuration; -- credentials or private paths leaking into provenance and reports; -- HTML/script injection from frame labels or tool output; -- decompression bombs or unexpectedly large volumetric files; -- non-finite values and parser differentials at trust boundaries. - -ElecTrace is not a sandbox. Do not execute plugins from an untrusted checkout, run -unknown external binaries, or analyze hostile data with credentials or valuable -write access available. - -## Scientific integrity is different - -Incorrect results, unstable matching, and misleading scientific interpretations are -important bugs. Use the scientific-validation issue template unless the behavior can -also cross a security/privacy boundary or enable malicious code execution. +Incorrect scientific results are important bugs, but they are security issues only +when they also cross a confidentiality, integrity, or execution boundary. diff --git a/docs/agents.md b/docs/agents.md index 3486122..eb316ad 100644 --- a/docs/agents.md +++ b/docs/agents.md @@ -1,74 +1,33 @@ -# Use ElecTrace with coding agents +# Coding-agent reference -ElecTrace gives coding agents a reusable command-line workflow for Bader charge trends -across NEB frames, ELF/QTAIM path analysis, and LOBSTER ICOHP/ICOBI changes across an -ordered VASP structure series. It replaces one-off frame discovery, Slurm generation, -output parsing, periodic correspondence, and report code with a tested interface. +Install ElecTrace from GitHub, inspect `electrace --help`, and prefer its CLI over +reimplementing VASP discovery, descriptor parsing, periodic matching, or workflow +staging. -## Install the CLI - -ElecTrace supports CPython 3.11–3.13. Install the current public source in a virtual -environment: - -```bash -python -m pip install "git+https://github.com/Austin243/ElecTrace.git" -electrace --version -electrace doctor --json -``` - -The built-in demo requires no external scientific program: +For existing outputs: ```bash -electrace demo --output .electrace-demo --json -electrace inspect .electrace-demo/electrace-results --json -``` - -## Install the Codex skill - -The repository ships an -[`electrace` skill](https://github.com/Austin243/ElecTrace/tree/main/skills/electrace) -that teaches Codex when to analyze existing descriptor outputs, when to prepare a -calculation workflow, which commands are non-executing, and which actions require -explicit user authorization. - -Ask Codex: - -```text -Use $skill-installer to install the ElecTrace skill from -https://github.com/Austin243/ElecTrace/tree/main/skills/electrace +electrace init electrace.toml --images /absolute/path/to/frames +electrace doctor --config electrace.toml --json +electrace analyze electrace.toml --json ``` -The skill becomes available on the next Codex turn. Invoke it explicitly with -`$electrace`, or let its task description trigger when working with Bader, BadELF, -ELF/QTAIM, LOBSTER, DDEC, VASP NEB frames, or ordered periodic structure series. - -## Agent workflow - -An agent should start with the installed help and a read-only environment check: +For missing calculations: ```bash -electrace --help -electrace doctor --json +electrace workflow init electrace-workflow.yaml --images /absolute/path/to/frames +electrace workflow validate electrace-workflow.yaml --json +electrace workflow stage electrace-workflow.yaml --json ``` -Every operational subcommand accepts `--json`. Successful and handled-error responses -use one compact, unstyled `electrace.cli.v1` document with the fields `schema_version`, -`ok`, `command`, `data` or `error`, `warnings`, and `artifacts`. Help and version output -remain human-readable; request them without `--json`. Exit code `0` means the operation -succeeded, `1` means a completed benchmark or external execution reported failure, and -`2` means the request or a prerequisite was invalid. The public contract is the -[`electrace.cli.v1` schema](https://github.com/Austin243/ElecTrace/blob/main/schemas/electrace.cli.v1.schema.json). -Resolved workflow plans in `data` follow -[`electrace.workflow-validation.v1`](https://github.com/Austin243/ElecTrace/blob/main/schemas/electrace.workflow-validation.v1.schema.json). +Staging and validation do not execute scientific programs. Add `--yes` to +`workflow run` or `workflow submit` only after the user authorizes the resolved +commands. Prepare Slurm scripts without `--yes`, review them, then submit with +`--yes --force` so the reviewed generated scripts may be replaced. -If supported descriptor results already exist, it should create and review -`electrace.toml`, run `electrace analyze --json`, and consume `analysis.json` as the -canonical result. If calculations are missing, it should create and review -`electrace-workflow.yaml`, run -`electrace workflow validate electrace-workflow.yaml --json`, then stage the selected -frames and inspect workflow status with `--json`. -Local execution and Slurm submission remain explicit actions; the skill does not add -`--yes` unless the user authorizes the reviewed job scope. +After outputs exist, run `electrace workflow analyze MANIFEST --json`. Surface +warnings and missing evidence. Never treat Bader/DDEC values as definitive oxidation +states, ICOHP as a transferable bond energy, or a filesystem-complete job as proof of +scientific convergence. -See the [workflow guide](workflows.md) for complete manifest options and the -[scientific scope](scientific-scope.md) for interpretation boundaries. +The installable skill is in [`skills/electrace`](https://github.com/Austin243/ElecTrace/tree/main/skills/electrace). diff --git a/docs/architecture.md b/docs/architecture.md deleted file mode 100644 index bc2199b..0000000 --- a/docs/architecture.md +++ /dev/null @@ -1,188 +0,0 @@ -# Architecture - -ElecTrace separates scientific correspondence logic from external analysis tools. -That boundary keeps provenance explicit, makes the core testable without licensed -software, and avoids silently treating one electronic descriptor as truth. - -## Data flow - -```mermaid -flowchart TB - subgraph Inputs - V["Ordered VASP image directories"] - Y["Workflow YAML + reviewed templates"] - N["Normalized feature JSON"] - end - - subgraph Orchestration - S["Source-isolated staging"] - X["Explicit local run / Slurm submit"] - C["Validated collection"] - end - - subgraph Adapters - P["Plugin configuration"] - Q["Validated command plan"] - R["Tool-specific output parser"] - end - - subgraph Core - I["Frame discovery and canonicalization"] - M["Periodic atom / feature correspondence"] - E["Evidence-vector and change-point logic"] - end - - subgraph Products - J["Versioned JSON"] - T["Event tables"] - H["HTML pathway report"] - end - - V --> I --> M - P --> Q --> S - V --> S - Y --> S --> X --> C --> R --> N --> M - M --> E - E --> J - E --> T - E --> H -``` - -The plugin layer describes a subprocess as an argument vector, working directory, -standard input, required inputs, and expected outputs. Constructing that plan does -not execute it. ElecTrace implements a separate orchestration boundary that stages -jobs without execution, requires `--yes` for a local launch or Slurm submission, and -collects supported outputs into an isolated analysis tree. This boundary makes a -command auditable; it does not prove convergence or scientific validity. - -## Layers - -### 1. Discovery and canonical frames - -Path discovery must be deterministic. Each frame receives an integer index, a stable -label, a cell, periodic boundary flags, an ordered atom table, and hashes of the -selected source files. Numeric image directory names are sorted numerically rather -than lexicographically. - -Atom identity is never inferred from display labels alone. The 0.1 baseline expects a -consistent atom list and uses periodic geometry to validate or recover -correspondence. Ambiguity is surfaced rather than silently resolved. - -### 2. Plugin boundary - -Built-in adapters target BaderKit, critic2, LOBSTER, and Chargemol. They translate -project configuration into inspectable command plans and translate tool-specific -outputs into the [`electrace.features.v1`](spec/plugin-protocol-v1.md) envelope. -The CLI can produce one plan or a deterministic path-wide batch, but planning never -executes a tool. critic2 integration requests `critic2.features.json` through -`CPREPORT`; raw `.cro` output is not parsed as normalized evidence. - -Adapters do not redefine the scientific meaning of external tools, redistribute -them, or bypass their licenses. Raw tool outputs remain important provenance and -should be retained alongside normalized records. - -### 3. Explicit workflow orchestration - -A versioned YAML manifest selects canonical frame indices, executable argument -vectors, reviewed template directories, scheduler policy, and an ordered dependency -DAG. Staging copies the selected structure and templates into a workspace outside -the source path. The source calculation directories remain read-only. - -The workspace carries an ElecTrace marker and a hash of the source manifest. -Replacement is permitted only for a correctly marked, non-symlinked workspace; a -manifest change requires restaging. Local commands use argument arrays with -`shell=False`, dedicated working directories, separate output/error logs, required -input checks, and dependency failure propagation. Slurm preparation emits one -dependency-ordered script per selected frame. Preparing a script is non-executing; -submission is a second, explicit action. - -Step `resources` values are advisory metadata in workflow v1. Local concurrency is -controlled by `--max-parallel`, while actual Slurm allocation comes from validated -`scheduler.directives`. - -Collection is a separate success boundary. It requires nonempty, regular, -non-symlinked expected descriptor outputs, parses them against the associated frame, -and atomically copies supported evidence and selected structures under -`workspace/analysis`. Neither a zero exit status nor the existence of an expected -filename establishes DFT convergence, projection quality, or a valid scientific -interpretation. See the [external workflow guide](workflows.md) for the operational -contract. - -### 4. Normalized feature model - -Every frame contains zero or more frame-local electronic features. A feature has a -stable local identifier, a normalized `kind`, a fractional position, optional charge -and volume, zero-based neighboring atom indices, optional voxel support, and -extensible typed attributes. - -Frame-local plugin identifiers are not cross-frame identities. The core engine owns -correspondence and persistent track identifiers so a parser cannot accidentally -assert temporal continuity. - -### 5. Correspondence and evidence - -ElecTrace 0.1 implements periodic atom matching and an experimental feature matcher -that combines deformation-aware displacement, periodic voxel overlap, charge, -volume, and neighboring-atom fingerprints. Hyperedges represent candidate splits, -merges, and rearrangements only when at least one complete, semantically compatible -charge or volume channel is conserved across every participant. Every complete -channel must pass the configured tolerance; a passing channel cannot average away a -failure in another. Proximity-only and partial-evidence groupings remain one-to-one -matches plus births/deaths, with structured rejection diagnostics in the transition -record. - -Assignment ambiguity is evaluated against complete alternative assignments, not a -local pair-cost tie alone. A selected atom or feature decision is ambiguous when -forbidding it yields another complete assignment within the configured -`matching.ambiguity_tolerance` total-cost gap. Locally tied pairs that are resolved by -the rest of the assignment are not flagged. Likewise, equivalent one-to-one backbone -choices are treated as resolved when a conservation-backed hyperedge contains every -affected participant; the resolved alternatives remain in that edge's evidence. - -That is an algorithmic implementation, not a validated general solution. Scientific -lineage claims still require benchmark data, grid-refinement studies, ambiguity -audits, and domain calibration. No 0.1 result should imply that basin identity or the -general split/merge problem is uniquely resolved. - -### 6. Events and products - -Event records should preserve an evidence vector: which descriptors changed, in -which direction, by how much, under what thresholds, with what convergence checks, -and with which warnings. Human-readable labels are views over that record. - -Machine-readable JSON is the source product. CSV tables and HTML reports are derived -views and must retain links to the source frame, normalized record, configuration, -and software versions. - -CSV is a spreadsheet-safe derived view, not the lossless interchange format. Any -string cell or untrusted column name whose first non-whitespace character is `=`, -`+`, `-`, or `@` is prefixed with an apostrophe to prevent formula execution in -spreadsheet software. Ordinary strings are unchanged, structured cells use strict -JSON, and the canonical analysis JSON retains the original values. - -## Determinism and failure policy - -- Stable sorting and stable tie-breaking are required. -- Non-finite JSON numbers are forbidden. -- Unknown schema major versions are rejected. -- Missing optional evidence is represented as missing, never as zero. -- A partial plugin result is visibly partial and cannot silently become a complete - event vector. -- In strict mode, inconsistent atom counts/order, malformed cells, hash mismatches, - and complete alternative mappings within the configured ambiguity tolerance fail - the run before output publication. -- In permissive mode, the same conditions remain in transition diagnostics and - warnings. Ambiguous atom mappings are excluded from neighbor evidence; ambiguous - feature edges, births/deaths, lineage propagation, and property/topology events are - suppressed. - -## Trust boundaries - -Input directories, templates, staged records, collected outputs, and external -executables are untrusted. Paths must be normalized; command plans use argument -arrays rather than shell strings; adapters must not embed credentials; selected -outputs must not escape through symlinks; HTML output must escape source-provided -text; and no input calculation is modified in place. Licensed pseudopotentials are -user-supplied and are never shipped by ElecTrace. See the -[security policy](https://github.com/Austin243/ElecTrace/blob/main/SECURITY.md) for -reporting concerns. diff --git a/docs/benchmarking.md b/docs/benchmarking.md deleted file mode 100644 index 2f30dd5..0000000 --- a/docs/benchmarking.md +++ /dev/null @@ -1,77 +0,0 @@ -# Synthetic benchmark - -`electrace benchmark` runs a deterministic, offline regression suite through the -same atom matcher, feature matcher, and transition event detector used by -`electrace analyze`. It provides machine-readable evidence that controlled software -behaviors have not changed unexpectedly. - -```bash -electrace benchmark --output electrace-benchmark.json -``` - -The command returns a non-zero status if any labeled case fails. It refuses to -replace an existing report unless `--force` is supplied. The output conforms to the -[`benchmark-v0.1` JSON Schema](https://github.com/Austin243/ElecTrace/blob/main/schemas/benchmark-v0.1.schema.json). - -!!! warning "Regression benchmark, not scientific validation" - - The cases are constructed inputs. A perfect score shows agreement with explicit - synthetic expectations; it does not estimate accuracy on experimental data, - independently computed basins, or a physical domain. Event confidence values - remain deterministic evidence scores rather than calibrated probabilities. - -## Versioned contract - -Every report records three independent identities: - -- `schema_version` identifies the JSON shape; -- `suite.version` identifies the case definitions and expected labels; -- `software_version` identifies the ElecTrace revision that produced the result. - -The report intentionally has no timestamp, hostname, or random identifier, so two -runs with the same software and suite are byte-for-byte reproducible. Change the -suite version whenever a case, threshold, or expected label changes. Change the -schema version only when the report contract changes. - -## Current coverage - -The v0.1 suite contains 15 two-frame cases: - -| Behavior | Controlled perturbations | -| --- | --- | -| Periodic matching | Atom and feature wrapping across a cell face | -| Robustness | Coordinate jitter, anisotropic cell deformation, reversed feature order | -| Direct events | Birth, death, and threshold-crossing ICOHP change | -| Hyperedges | Volume-conserving split and merge | -| Negative controls | Missing conservation evidence and failed volume conservation | -| Ambiguity | Exact and near-optimal assignment ties, plus a tie resolved by a conserving hyperedge | -| Semantic gates | Explicitly incompatible charge conventions | - -Each case records the expected and observed event identities, atom mapping, feature -mapping, ambiguity decisions, final match relations, and structured hyperedge -rejections. Event scoring uses the exact multiset of `(type, source_ids, -target_ids)` labels. It reports true positives, false positives, false negatives, -precision, recall, and F1. Mapping and ambiguity scores are exact assertion counts. - -The built-in baseline is expected to have zero failures. A new failure should be -handled in one of three ways: - -1. fix an unintended implementation regression; -2. document and review an intentional scientific-behavior change, then update the - expected label and suite version; -3. add a more precise case when the old construction was under-specified. - -Do not weaken or remove a negative control solely to recover a perfect score. - -## What should come next - -The synthetic suite is the first validation layer. The next layer must use -redistributable, independently generated electronic-structure fixtures with labeled -expectations and grid-refinement studies. Those fixtures should report performance by -feature family and perturbation regime, not combine every domain into one headline -score. Until that exists, ElecTrace should continue to describe basin lineage as -experimental. - -ElecTrace currently has no GPU compute backend. Running this command on a GPU node is -useful as an installation and environment compatibility check, but it does not -exercise GPU kernels or demonstrate GPU acceleration. diff --git a/docs/governance.md b/docs/governance.md deleted file mode 100644 index 4070541..0000000 --- a/docs/governance.md +++ /dev/null @@ -1,68 +0,0 @@ -# Governance - -ElecTrace is maintainer-led research software. Governance is lightweight during the -pre-1.0 phase and is intended to become more formal if the contributor community -grows. - -## Roles - -- **Maintainers** set scope, merge changes, manage releases and security reports, - and are accountable for scientific claims made by the project. -- **Contributors** propose code, tests, documentation, benchmarks, or scientific - validation through reviewed pull requests. -- **Scientific reviewers** may review domain assumptions, fixtures, and - interpretations without taking on general maintenance duties. - -Repository permissions, not a hard-coded list in this document, are the source of -truth for current maintainers. - -## Decision process - -Routine changes are decided in pull-request review. Material changes require a short -design note in the pull request or `docs/` covering alternatives and migration: - -- feature-matching or event semantics; -- a versioned JSON contract; -- support for a new external tool or file format; -- backward-incompatible CLI/configuration changes; -- changes to the scientific domain of validity; -- collection or publication of benchmark data. - -Maintainers seek rough consensus. If consensus is not possible, a maintainer records -the decision, rationale, dissent, and conditions for revisiting it. Scientific -uncertainty is documented rather than decided by vote. - -## Releases - -- Pre-1.0 releases may change quickly but require changelog entries and migration - notes for user-visible breaks. -- Plugin/data schemas use explicit identifiers and evolve independently of package - versions. -- A release requires passing CI, a reviewed changelog, an offline demo, and an audit - of documentation against implemented capabilities. -- Only maintainers publish tags or packages. - -## Scientific integrity - -Contributors must disclose provenance, manual curation, excluded data, and known -negative evidence. Benchmarks should separate algorithm development cases from -held-out validation cases. Results must not be described as calibrated probabilities -without an explicit calibration study. - -Substantive corrections are documented in the changelog and affected reports or -releases are annotated when feasible. - -## Conduct and conflicts - -Participation is governed by the -[Code of Conduct](https://github.com/Austin243/ElecTrace/blob/main/CODE_OF_CONDUCT.md). -Contributors should disclose financial, institutional, or software-vendor conflicts -that could reasonably affect a scientific or integration decision. - -## Security - -Security reports follow the -[security policy](https://github.com/Austin243/ElecTrace/blob/main/SECURITY.md). -Maintainers may embargo a fix until affected users can update. Scientific -disagreements that do not expose a security or privacy risk belong in the normal -issue process. diff --git a/docs/index.md b/docs/index.md index 4a6fd6f..5f0a438 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,50 +1,15 @@ -# ElecTrace documentation +# ElecTrace -ElecTrace is one research-software project with a deliberately focused 0.1 release -and a broader scientific roadmap. This documentation distinguishes implemented -capabilities from planned or scientifically unvalidated work. +ElecTrace compares electronic-structure evidence across ordered periodic structures. +It can analyze existing Bader, BadELF, critic2, LOBSTER, Chargemol, and normalized +feature records, or stage user-configured VASP and descriptor calculations locally or +through Slurm. -## Start here +- [Using ElecTrace](workflows.md): install, analyze existing results, select frames, + stage calculations, run locally, submit through Slurm, and collect outputs. +- [Coding-agent reference](agents.md): compact command routing and authorization rules. +- [Scientific scope and limitations](scientific-scope.md): interpretation boundaries. -1. [Quick start](quickstart.md) — install, run the deterministic demo, and initialize - an ordered VASP path. -2. [External calculation workflows](workflows.md) — stage, explicitly execute or - submit, collect, and analyze a VASP→descriptor workflow. -3. [Coding agents](agents.md) — install the ElecTrace skill and use the CLI through a - compact, approval-aware agent workflow. -4. [Scientific scope and limitations](scientific-scope.md) — understand what a - ElecTrace event does and does not mean. -5. [Synthetic benchmark](benchmarking.md) — run and interpret the versioned offline - regression cases without mistaking them for physical validation. -6. [Reproducibility](reproducibility.md) — preserve enough information to audit and - repeat an analysis. -7. [Pre-release namespace migration](migration.md) — update development environments, - scripts, configurations, and serialized artifacts for `0.1.0a2`. - -## Design and integration - -- [Architecture](architecture.md) -- [Plugin protocol v1](spec/plugin-protocol-v1.md) -- [`electrace.features.v1` JSON Schema](https://github.com/Austin243/ElecTrace/blob/main/schemas/electrace.features.v1.schema.json) -- [`electrace.workflow.v1` JSON Schema](https://github.com/Austin243/ElecTrace/blob/main/schemas/electrace.workflow.v1.schema.json) -- [`electrace.cli.v1` JSON Schema](https://github.com/Austin243/ElecTrace/blob/main/schemas/electrace.cli.v1.schema.json) -- [`electrace.workflow-validation.v1` JSON Schema](https://github.com/Austin243/ElecTrace/blob/main/schemas/electrace.workflow-validation.v1.schema.json) -- [Public workflow YAML example](https://github.com/Austin243/ElecTrace/blob/main/examples/electrace.workflow.yaml) -- [`electrace.benchmark.v0.1` JSON Schema](https://github.com/Austin243/ElecTrace/blob/main/schemas/benchmark-v0.1.schema.json) -- [Example normalized feature document](examples/electrace.features.v1.example.json) - -## Project direction - -- [Roadmap](roadmap.md) -- [Governance](governance.md) -- [Prior-art context and acknowledgements](sources.md) -- [Contributing guide](https://github.com/Austin243/ElecTrace/blob/main/CONTRIBUTING.md) -- [Changelog](https://github.com/Austin243/ElecTrace/blob/main/CHANGELOG.md) - -## Versioning policy - -Code follows semantic versioning after the first tagged release. Data and plugin -contracts are versioned independently through their `schema_version` field. A reader -must reject an unknown schema identifier rather than guessing. Extensions belong in -the designated `attributes` and `metadata` maps; changing the closed envelope or the -meaning or type of an existing field requires a new contract identifier. +Machine-readable contracts and examples live in the repository's +[`schemas`](https://github.com/Austin243/ElecTrace/tree/main/schemas) and +[`examples`](https://github.com/Austin243/ElecTrace/tree/main/examples) directories. diff --git a/docs/migration.md b/docs/migration.md deleted file mode 100644 index 5aef192..0000000 --- a/docs/migration.md +++ /dev/null @@ -1,67 +0,0 @@ -# Pre-release namespace migration - -ElecTrace `0.1.0a2` establishes one product and namespace across the repository. The -project had no earlier tag or published release, so this is a clean pre-release break: -there are no compatibility aliases for development-only names or serialized -identifiers. - -## Canonical surface - -| Surface | ElecTrace `0.1.0a2` value | -| --- | --- | -| Distribution | `electrace` | -| Python imports | `electrace` | -| Configuration API | `electrace.config.ElecTraceConfig` | -| Command and module entry point | `electrace`; `python -m electrace` | -| Project configuration | `electrace.toml` | -| Example configuration | `examples/electrace.example.toml` | -| Normalized feature contract | `electrace.features.v1` | -| Feature schema | `schemas/electrace.features.v1.schema.json` | -| Analysis schema | `schemas/analysis-v0.1.schema.json`; version `0.1` | -| Bundle manifest contract | `electrace.bundle.v1` | -| HDF5 schema label | `electrace.analysis` | -| Default result directory | `electrace-results/` | -| Default demo directory | `.electrace-demo/` | -| Version field from `doctor --json` | `data.electrace_version` in `electrace.cli.v1` | -| Built-in demo producer | `electrace-demo` | - -Existing TOML table names and previously supported keys are unchanged. New checkouts -also expose `inputs.descriptor_source` to select local, external, or merged descriptor -discovery explicitly; newly generated configurations select `auto`. The canonical -filename, generated defaults, program identifiers, and namespaces changed. - -## Machine-readable CLI output in 0.1.0a4 - -Explicit `--json` output now uses the `electrace.cli.v1` envelope on every operational -subcommand. Consumers of the earlier pre-release shapes must read the former -`doctor --json` and `volume --json` payloads below `data`. `plan` and `workflow status` -retain their original raw JSON when `--json` is omitted; adding the flag requests the -new envelope. Help and version output remain human-readable and should be requested -without `--json`. - -## Migrate a development checkout - -Use a fresh virtual environment so an older editable installation cannot shadow the -renamed package: - -```bash -python -m venv .venv -source .venv/bin/activate -python -m pip install --upgrade pip -python -m pip install . -electrace --version -electrace doctor -``` - -Rename the project configuration to `electrace.toml`, update its `[outputs].directory` -to `electrace-results`, and update scripts to invoke `electrace`. The configuration -contents otherwise retain the same tables and keys. - -Normalized descriptor JSON must declare `electrace.features.v1` exactly. Validate -migrated records against `schemas/electrace.features.v1.schema.json`; the reader -intentionally rejects every other schema identifier rather than guessing. - -Regenerate result bundles when source inputs remain available. If an existing -development bundle must be retained, keep it as an immutable historical artifact and -do not mix its files with an `electrace.bundle.v1` manifest. ElecTrace does not provide -an automatic artifact-migration command in this pre-release phase. diff --git a/docs/quickstart.md b/docs/quickstart.md deleted file mode 100644 index 4031971..0000000 --- a/docs/quickstart.md +++ /dev/null @@ -1,266 +0,0 @@ -# Quick start - -This guide exercises ElecTrace safely before you point it at scientific data. Commands -describe the intended 0.1 interface; `electrace COMMAND --help` is authoritative for -the installed revision while the project remains pre-release. - -## 1. Install from a checkout - -ElecTrace supports CPython 3.11–3.13. - -```bash -git clone https://github.com/Austin243/ElecTrace.git -cd ElecTrace -python -m venv .venv -source .venv/bin/activate -python -m pip install --upgrade pip -python -m pip install . -``` - -This installs the runtime package without development tooling. Contributors should -instead follow the editable setup in `CONTRIBUTING.md`. - -The base package stays lightweight by using ElecTrace's bundled deterministic -assignment solver. Install `electrace[accelerate]` to prefer SciPy's compiled solver -for larger analyses, `electrace[hdf5]` to write HDF5 output, or `electrace[all]` for -both optional runtime features. ElecTrace's VASP readers do not require ASE. - -## 2. Diagnose the environment - -```bash -electrace doctor -``` - -`doctor` should distinguish required Python capabilities from optional external -adapters. A missing critic2, BaderKit, LOBSTER, or Chargemol executable is expected -for users who only want the built-in demo or who are importing already-normalized -records. Without a configuration it checks the runtime and adapter executables; after -initialization, `electrace doctor --config electrace.toml` deterministically checks -every discovered frame and its required plugin inputs. - -With `--json`, unconfigured executable checks appear in the `data.plugins` array. A -configured path check uses `data.frames`, whose entries identify each frame and contain -that frame's plugin reports; consumers should not mistake an empty `data.plugins` array -for a skipped configured-path check. - -## 3. Run the deterministic demo - -```bash -electrace demo --output .electrace-demo -electrace inspect .electrace-demo/electrace-results -``` - -The demo is a software smoke test, not a scientific benchmark. It should use small -synthetic or redistributable fixtures and should not require licensed software, -network access, or a DFT calculation. - -Run the more adversarial, labeled regression suite separately: - -```bash -electrace benchmark --output electrace-benchmark.json -``` - -This exercises periodic wrapping, controlled coordinate and cell perturbations, -event labels, conservation negative controls, semantic gates, and ambiguity handling. -It is still synthetic and therefore does not establish scientific accuracy. See the -[benchmark guide](benchmarking.md) for the exact scoring contract and limitations. - -Record the exact command and installed revision when filing a bug: - -```bash -python --version -electrace --version -electrace doctor -git rev-parse HEAD -``` - -## 4. Initialize a VASP path - -The image root should contain deterministically ordered directories such as `00`, -`01`, …, each with a consistent atom list and a readable `CONTCAR` or `POSCAR`. - -```text -neb-images/ -├── 00/ -│ └── CONTCAR -├── 01/ -│ └── CONTCAR -└── 02/ - └── CONTCAR -``` - -Create a project configuration: - -```bash -electrace init electrace.toml --images /absolute/path/to/neb-images -``` - -Review the generated file. A representative configuration is: - -```toml -[project] -name = "Example NEB pathway" -series_kind = "neb" - -[inputs] -root = "/absolute/path/to/neb-images" -descriptor_source = "auto" -structure_name = "CONTCAR" -fallback_structure_name = "POSCAR" -descriptor_name = "features.json" -prefer_contcar = true - -[plugins.critic2] -enabled = true -executable = "critic2" - -[plugins.baderkit] -enabled = false -executable = "baderkit" - -[plugins.lobster] -enabled = false -executable = "lobster" -``` - -`inputs.descriptor_source` makes descriptor discovery explicit: - -- `auto` is compatibility behavior for configurations that omit an explicit policy: - it uses frame-local descriptors without `descriptor_dir` and external descriptors - when that directory is present; -- `local` reads only each frame directory and ignores `descriptor_dir`; -- `external` requires `descriptor_dir` and reads only that tree; -- `merge` requires `descriptor_dir`, combines local and external records, and rejects - duplicate feature IDs rather than silently choosing a source. - -The structure and descriptor filenames must be basenames, not paths. The loader -recognizes canonical `features.json` and `electrace.features.json` records as well as -supported native-tool outputs. A plugin's `enabled` setting affects only diagnostics -and command planning; it never hides evidence that has already been produced. - -Use absolute input paths for long-lived analyses, or record the project-root-relative -base unambiguously. Do not edit, reorder, or overwrite source calculations during an -analysis. - -## 5. Inspect a plugin command plan - -ElecTrace does not implicitly execute external scientific programs. Ask it for a -shell-free plan for one plugin, frame, and capability: - -```bash -electrace plan critic2 /absolute/path/to/neb-images/03 \ - --capability qtaim_critical_points --config electrace.toml --json -``` - -Generate a deterministic plan for every discovered frame by passing the path root: - -```bash -electrace plan critic2 /absolute/path/to/neb-images --all-frames \ - --capability qtaim_critical_points --config electrace.toml --json -``` - -With `--json`, the one-frame or batch plan is returned under `data` in an -`electrace.cli.v1` envelope. Batch `data.plans` remains in deterministic frame order. -Each element contains a canonical `frame` record and a `command` that follows the same -command-plan contract as the one-frame form. - -The command prints JSON containing the argument vector, working directory, standard -input, required inputs, and expected outputs. Review it before an authorized workflow -executes anything. A plan is an audit record, not evidence that the tool ran or that -its output is valid. - -critic2 plans request a standardized `critic2.features.json` through `CPREPORT`. -ElecTrace does not treat the human-readable raw `.cro` stream as normalized evidence. - -## 6. Optional: orchestrate external calculations - -`electrace plan` remains non-executing. If you instead want ElecTrace to stage and -explicitly run an authorized VASP→descriptor job graph, create a separate workflow -manifest: - -```bash -electrace workflow init electrace-workflow.yaml \ - --images /absolute/path/to/neb-images -# Edit and review the YAML, executable paths, templates, frames, and scheduler. -electrace workflow validate electrace-workflow.yaml --json -electrace workflow stage electrace-workflow.yaml -electrace workflow status electrace-workflow.yaml --json -``` - -`workflow validate` resolves the complete selected job graph without creating the -workspace or writing any file. - -For `scheduler.kind: local`, execution requires an explicit gate: - -```bash -electrace workflow run electrace-workflow.yaml --yes --max-parallel 2 -``` - -For `scheduler.kind: slurm`, first prepare and review scripts without submitting, -then repeat with authorization: - -```bash -electrace workflow submit electrace-workflow.yaml --job-name electrace -electrace workflow submit electrace-workflow.yaml --job-name electrace --yes --force -``` - -After the expected descriptor outputs exist, isolate and analyze them: - -```bash -electrace workflow collect electrace-workflow.yaml -electrace workflow analyze electrace-workflow.yaml -electrace inspect .electrace-work/analysis/results -``` - -`workflow analyze` performs collection when needed and produces a dedicated analysis -configuration and report below the marker-protected workspace. It is different from -top-level `electrace analyze electrace.toml`, which reads an existing TOML project -and never launches an external scientific program. - -Read the [external workflow guide](workflows.md) before using `--yes`. It provides a -complete YAML manifest, frame-slice syntax, template rules, collection layout, -BadELF grid requirements, LOBSTER prerequisites, and the exact boundary between -collected evidence and scientific validation. - -## 7. Analyze an existing project and inspect it - -```bash -electrace analyze electrace.toml -electrace inspect electrace-results -``` - -With the default output settings, a successful run writes: - -```text -electrace-results/ -├── analysis.json -├── events.csv -├── feature_tracks.csv -├── manifest.json -└── report.html -``` - -`analysis.json` is the canonical analysis product. The tables and report are derived -views; `manifest.json` records the size and SHA-256 digest of each product. - -Before interpreting results, confirm that: - -- every expected frame was discovered in the intended order; -- atom counts, element ordering, and cells passed validation; -- the selected structure file is correct for every image; -- every enabled adapter reports its executable version and input hashes; -- warnings and partial plugin results have been reviewed; -- matching, persistence, descriptor, and minimum-confidence thresholds have been - reviewed for the intended domain; -- conclusions survive appropriate grid, image-spacing, and DFT convergence checks. - -## Troubleshooting - -Run `electrace doctor` first. If the problem is reproducible with the built-in demo, -use the bug-report issue template and attach only non-sensitive logs. For a scientific -discrepancy, use the validation issue template and include the method, grid, -pseudopotentials, cell convention, and a minimal redistributable fixture when -licensing permits. - -Never upload licensed pseudopotentials, proprietary executables, credentials, or -unpublished structures without authorization. diff --git a/docs/reproducibility.md b/docs/reproducibility.md deleted file mode 100644 index 55807ee..0000000 --- a/docs/reproducibility.md +++ /dev/null @@ -1,102 +0,0 @@ -# Reproducibility - -An ElecTrace result is reproducible only when another researcher can reconstruct the -ordered frames, external-tool outputs, normalized feature records, configuration, -and software environment that produced it. - -## Archive with every analysis - -### Source identity - -- ordered frame index and label; -- selected `POSCAR` or `CONTCAR` path and SHA-256 digest; -- cell, periodic boundary convention, atom symbols, and atom order; -- hashes of every external-tool input consumed; -- repository commit or release tag. - -### Electronic-structure provenance - -- code and version; -- exchange-correlation functional; -- pseudopotential/PAW dataset identifiers and permitted checksums; -- plane-wave and augmentation cutoffs; -- k-point sampling, smearing, charge, and spin settings; -- SCF and ionic convergence thresholds; -- NEB spring/optimizer settings and image energies; -- density/ELF grid dimensions and any interpolation. - -Do not redistribute licensed pseudopotentials. Record identifiers and hashes only -when redistribution is prohibited. - -### Adapter provenance - -- adapter name and ElecTrace version; -- external executable name, resolved path, and version; -- full argument vector (redacted only for secrets, which should not be present); -- working-directory policy and standard-input digest; -- required input and expected output paths; -- raw output hashes, parser warnings, and normalized schema version. - -### Analysis provenance - -- complete configuration file; -- all default values materialized or the exact code revision defining them; -- matching costs, descriptor scales, event and confidence thresholds, hysteresis, - sustained-state persistence, and tie-breaking rules; -- deterministic random seed, if a future algorithm requires one; -- warnings, suppressed claims, and manual overrides. - -## Recommended bundle layout - -```text -analysis-bundle/ -├── README.txt -├── electrace.toml -├── manifest.json -├── environment/ -│ ├── python.txt -│ ├── packages.txt -│ └── external-tools.json -├── frames/ -│ └── frame-index.json -├── normalized/ -│ └── *.features.json -├── results/ -│ ├── events.json -│ └── events.csv -└── report/ - └── index.html -``` - -Raw DFT data can be referenced by content hash and repository/archive identifier when -it is too large or cannot legally be redistributed. - -## Convergence ladder - -For each central scientific claim, rerun a small targeted ladder: - -1. baseline calculation and plugin settings; -2. finer density/ELF grid; -3. tighter electronic convergence; -4. denser path images around the proposed event; -5. perturbed matching and event thresholds; -6. an independent descriptor or analysis method where meaningful. - -Record outcomes even when the event disappears. A result that survives this ladder -has stronger numerical support; it is still method dependent. - -## Determinism check - -Run the same configuration twice in clean output directories. Machine-readable -outputs should be byte-identical after excluding explicitly volatile provenance such -as wall-clock timestamps and absolute scratch paths. If they are not, identify and -record the source of nondeterminism. - -## Data sharing checklist - -- Remove credentials, usernames, private hostnames, and proprietary path fragments. -- Check licenses before sharing external-tool outputs or pseudopotentials. -- Prefer open, documented fixtures for tests and tutorials. -- Publish cryptographic hashes for data hosted separately. -- Include `CITATION.cff`, the ElecTrace version, and citations for every external - scientific tool used. diff --git a/docs/roadmap.md b/docs/roadmap.md deleted file mode 100644 index 1f4d22d..0000000 --- a/docs/roadmap.md +++ /dev/null @@ -1,67 +0,0 @@ -# Roadmap - -The roadmap is capability-gated. Dates are intentionally omitted: scientific -validation, fixtures, and reproducibility criteria matter more than calendar labels. - -## ElecTrace 0.1 — ordered VASP paths - -Goal: a useful, auditable foundation for NEB and static pressure series. - -- [x] deterministic VASP image discovery and validation; -- [x] canonical periodic frame and atom models; -- [x] inspectable command plans for BaderKit, critic2, LOBSTER, and Chargemol; -- [x] native/normalized evidence parsers and `electrace.features.v1` validation; -- [x] periodic atom/feature correspondence with complete-assignment ambiguity - diagnostics and strict/permissive failure policy; -- [x] experimental deformation-aware feature costs and voxel overlap; -- [x] birth/death candidates and conservation-gated split/merge hyperedges; -- [x] descriptor events, persistent track records, JSON/CSV, and an HTML report; -- [x] `doctor`, `benchmark`, `demo`, `init`, `plan`, `analyze`, and `inspect` CLI - workflows; -- [x] Python 3.11–3.13 CI, typing, tests, and reproducibility documentation. - -Exit criteria: a clean checkout can run the offline demo; at least one redistributable -ordered-path fixture has expected normalized records; malformed and ambiguous inputs -fail safely; repeat runs are deterministic; scientific limitations appear in the -report. - -## ElecTrace 0.2 — validated electronic-feature lineage - -Goal: establish where the experimental lineage algorithms are scientifically -reliable beyond atom-centered scalar series. - -- [x] versioned synthetic lineage regression harness with exact event, mapping, - ambiguity, conservation, and negative-control expectations; -- variable-cell stress tests, including large strain and reconstruction; -- curated basin and non-nuclear-attractor lineage benchmarks; -- grid-shape/origin interoperability and voxel-overlap convergence; -- false-event characterization for one-to-many and many-to-one assignments; -- persistence, hysteresis, and automated grid-refinement evidence; -- feature lifetimes, trajectories, and uncertainty flags; -- pressure-series visualizations; -- exploratory Quantum ESPRESSO and CP2K import contracts. - -Exit criteria: curated synthetic cases and independent scientific fixtures cover -birth/death/split/merge events; false event rates under grid refinement are reported; -ambiguous lineage remains inspectable. - -## ElecTrace 1.0 — sparse trajectory analysis - -Goal: apply validated correspondence to selected AIMD and MLIP frames. - -- sparse frame selection with explicit selection bias; -- pocket lifetimes, diffusion, splitting, and recombination statistics; -- automated DFT-frame audit of MLIP trajectories; -- calibrated uncertainty on supported benchmark domains; -- pressure–temperature onset maps; -- interactive electronic-autopsy reports; -- export of candidate endpoints or path segments for downstream NEB work. - -Exit criteria: versioned public benchmarks, stable data contracts, migration tooling, -performance characterization, and a documented domain of validity. - -## Non-goals - -ElecTrace will not become a DFT engine, replace upstream electronic-topology tools, -or claim a universal definition of bonding. Integration should remain modular and -scientific conclusions should remain evidence based. diff --git a/docs/scientific-scope.md b/docs/scientific-scope.md index c70c574..25faddc 100644 --- a/docs/scientific-scope.md +++ b/docs/scientific-scope.md @@ -1,119 +1,62 @@ # Scientific scope and limitations -ElecTrace is designed to organize evidence across a path. It is not a replacement -for electronic-structure theory, convergence testing, or scientific judgment. +ElecTrace organizes evidence across an ordered path. It does not replace electronic- +structure theory, convergence testing, or scientific judgment. -## ElecTrace 0.1 scope +## Current scope -The first release focuses on: +ElecTrace supports VASP-style NEB and static-series directories with consistent atom +content. It records frame provenance, matches atoms and normalized electronic +features under periodic boundary conditions, proposes birth/death and conservation- +backed split/merge events, tracks descriptor changes, and produces an auditable +report. Inputs can include BaderKit, critic2, LOBSTER, Bader, Chargemol/DDEC, and +canonical ElecTrace feature records. -- VASP-style ordered NEB image directories and static pressure-series directories; -- fixed or closely related periodic cells with consistent atom content; -- deterministic frame discovery and structure provenance; -- periodic atom correspondence and validation; -- normalized BaderKit and critic2 feature records; -- optional LOBSTER bonding descriptors; -- optional Chargemol/DDEC atomic-charge records; -- experimental deformation-aware feature costs and periodic voxel-overlap evidence; -- birth/death candidates and conservation-gated split/merge hyperedges; -- descriptor change points and an automated, auditable report; -- a non-executing plugin-planning boundary plus a separate, confirmation-gated local - and Slurm workflow orchestrator. +The separate workflow layer can stage and explicitly launch user-configured VASP and +analysis programs locally or through Slurm. It does not create pseudopotentials, +choose electronic-structure settings, or establish convergence. -Chargemol planning and conservative DDEC charge parsing are implemented, but external -DDEC execution is never implicit and cross-method charge fusion remains uncalibrated. +The current release does not claim: -## Explicitly outside 0.1 claims - -- Scientifically validated feature identity across large or reconstructive cell - changes. -- Calibrated basin-lineage classification or known false-event rates. -- Automated grid-refinement auditing of birth/death/split/merge candidates. -- AIMD/MLIP-scale sparse-frame selection and temporal statistics. -- Calibrated probabilistic confidence or universal event thresholds. -- Automated construction of a reaction mechanism or oxidation-state assignment. -- ElecTrace-generated electronic-structure physics, pseudopotential management, or - convergence automation. The workflow layer can launch a user-reviewed VASP job; - it does not make that job scientifically appropriate or converged. +- validated feature identity through reconstruction or large cell changes; +- calibrated event probabilities, false-event rates, or universal thresholds; +- automatic grid-refinement or electronic-convergence audits; +- a unique oxidation state, bond definition, reaction mechanism, or causal event; +- AIMD-scale temporal statistics; or - Quantum ESPRESSO or CP2K ingestion. -These are research goals in the [roadmap](roadmap.md), not implied features. - -## What an event means - -An ElecTrace event is a structured observation that selected descriptors changed -between ordered frames under recorded thresholds. For example, a report may combine -the appearance of a critical point, a change in ELF evidence, stronger ICOHP, and a -charge shift. Agreement makes the interpretation better supported; it does not make -any descriptor uniquely correct or establish causality. - -In particular: - -- A Bader or DDEC charge is partitioning-scheme dependent and is not automatically - an oxidation state. ElecTrace preserves BaderKit's explicitly supplied - `oxidation_states` values when present but does not derive or validate a unique - formal oxidation-state assignment. -- A bond critical point or bond path is a QTAIM topological feature, not by itself a - universal definition of a chemical bond. -- ELF basin topology depends on the underlying wavefunction/density, numerical grid, - and analysis settings. -- COHP/COBI values depend on basis/projection choices and the energy integration - convention. -- A NEB path is a discretized model path; event location is bounded by image spacing - unless additional images or interpolation evidence are supplied. - -## Numerical and methodological limitations - -### Grid sensitivity - -Small basins, non-nuclear attractors, and near-degenerate critical points may appear -or disappear with grid resolution. Important events require grid-refinement checks. -Persistence across adjacent frames can reduce isolated noise but cannot replace a -convergence study. - -### Periodic geometry - -Fractional coordinates remove trivial boundary crossings, and the core includes -deformation-aware metrics for changing cells. Those metrics do not make large cell -deformation, reconstruction, or atom permutation uniquely solvable. Near-equal -complete-assignment costs are an ambiguity, not a license to choose an attractive -story. ElecTrace records the selected and alternative assignments and suppresses -affected claims in permissive mode; the configured tolerance remains an experimental, -uncalibrated cost threshold. - -### Path sampling - -Change-point timing depends on the sampled images. The absence of a detected event -between widely spaced images is weak evidence. Conversely, repeated nearly identical -images can make persistence appear stronger than the underlying path justifies. -Events too near the final frame to satisfy the configured persistence window are -reported as right-censored rather than silently classified as confirmed or transient. - -### Method heterogeneity - -Comparing frames generated with different functionals, pseudopotentials, charge -states, spin treatments, smearing, grids, or projection settings can create apparent -events. ElecTrace records provenance where available but cannot make an inconsistent -series physically comparable. - -### Confidence language - -Until confidence models are calibrated against labeled benchmarks, terms such as -“low,” “medium,” and “high” are deterministic rule labels, not statistical -probabilities. Configured hysteresis and sustained-state persistence can suppress -transient labels, and `minimum_event_confidence` can suppress low-scoring events, but -neither operation calibrates confidence. Reports must expose the underlying evidence -and thresholds. - -## Minimum scientific review - -Before publishing a conclusion: - -1. inspect the source calculations and verify path ordering; -2. confirm electronic-structure and plugin settings are consistent; -3. repeat important observations under tighter grid and DFT convergence; -4. vary event thresholds and matching tolerances; -5. inspect ambiguous mappings and missing/partial plugin results; -6. compare at least two independent descriptors when the conclusion permits; -7. report negative and conflicting evidence, not only supporting features; -8. archive the configuration, normalized records, hashes, and software versions. +## Interpreting results + +An event means selected, method-dependent descriptors changed between sampled frames +under recorded thresholds. Agreement among charge, ELF/QTAIM, and ICOHP/ICOBI +evidence can strengthen an interpretation but does not make it unique. + +- Bader and DDEC charges depend on the partitioning scheme and are not automatically + formal oxidation states. +- A bond critical point or bond path is a QTAIM feature, not a universal bond test. +- ELF topology depends on the wavefunction or density, grid, and analysis settings. +- COHP/COBI depends on basis, projection, and integration choices; ICOHP is not a + universal bond energy. +- Event location is limited by the spacing of the supplied frames. + +## Numerical limitations + +Small basins and near-degenerate critical points may appear or disappear with grid +resolution. Consecutive-frame persistence filters isolated labels but cannot replace +a convergence study. For property changes, persistence tracks the matched lineage, +not whether the numerical change remains above its threshold in every later frame. + +Fractional coordinates handle boundary crossings, and deformation-aware distances +support related cells. Large reconstruction or atom permutation is not uniquely +solvable. The matcher reports one deterministic best assignment and does not +enumerate near-optimal alternatives, so close mappings require manual review. + +Inconsistent functionals, pseudopotentials, charge states, spin treatments, smearing, +grids, or projection settings can create artificial changes. Provenance is retained +when available but cannot make inconsistent calculations comparable. Confidence +labels are deterministic thresholds, not calibrated probabilities. + +Before drawing a conclusion, verify path order and calculation settings, repeat +important observations with tighter convergence and grid settings, vary matching and +event thresholds, inspect surprising mappings or missing evidence, and archive the +configuration, normalized records, hashes, and software versions. diff --git a/docs/sources.md b/docs/sources.md deleted file mode 100644 index 0698334..0000000 --- a/docs/sources.md +++ /dev/null @@ -1,55 +0,0 @@ -# Prior-art context and acknowledgements - -ElecTrace is intended to compose with—not replace—the electronic-structure and -trajectory-analysis ecosystem. This page records the sources that shaped the initial -concept survey in August 2026. It is context, not an exhaustive or legally conclusive -prior-art review; unpublished and domain-specific workflows may overlap. - -## Electronic-structure analysis - -- [BaderKit](https://sweav02.github.io/baderkit/) provides workflows around Bader, - ELF, critical-point, and electride analysis. -- [critic2](https://aoterodelaroza.github.io/critic2/) analyzes scalar fields in - molecules and crystals, including QTAIM-related topology. -- [LobsterPy](https://github.com/JaGeo/LobsterPy) automates analysis and visualization - of LOBSTER bonding outputs such as COHP/COBI descriptors. -- [baderVis](https://github.com/koerper/baderVis) motivated comparison with existing - Bader-charge visualization along VASP NEB paths. - -ElecTrace adapters do not include these programs. Users are responsible for their -installation, citation, licensing, and scientifically appropriate configuration. - -## Trajectories and pathways - -- [ReacNetGenerator](https://docs.deepmodeling.com/projects/reacnetgenerator/en/latest/) - and [ChemTraYzer2](https://www.scm.com/doc/Workflows/ChemTraYzer2/ChemTraYzer2.html) - provide important reference points for deriving reaction information from - coordinate or bond-order trajectories. -- [ASE NEB/AutoNEB](https://docs.ase-lib.org/ase/neb.html) and - [TSASE SSNEB](https://theory.cm.utexas.edu/tsase/ssneb.html) provide pathway - optimization context. ElecTrace analyzes an ordered path; it does not optimize one. - -## Motivating application - -The initial concept was motivated in part by a high-pressure electride study whose -methodology analyzes selected AIMD snapshots and aggregates changing non-nuclear -features. See the [study preprint](https://arxiv.org/html/2510.01583v1). The proposed -long-term ElecTrace contribution is to test whether persistent pocket identities, -lifetimes, diffusion, splitting, and recombination can be recovered instead of -reducing the ensemble to an average. - -That application remains a research target. Its mention is not validation of the -software and does not imply endorsement by the study authors. - -## Citation practice - -A paper or dataset produced with ElecTrace should cite: - -1. the archived ElecTrace release and its `CITATION.cff` metadata; -2. the electronic-structure code and methodology; -3. every external analysis tool actually used; -4. underlying methods for Bader/DDEC, ELF/QTAIM, COHP/COBI, or other descriptors as - required by those tools' documentation; -5. the archived input/output dataset or its content-addressed manifest. - -Please report missing related software or attribution through a documentation issue. diff --git a/docs/spec/plugin-protocol-v1.md b/docs/spec/plugin-protocol-v1.md deleted file mode 100644 index 47f1818..0000000 --- a/docs/spec/plugin-protocol-v1.md +++ /dev/null @@ -1,228 +0,0 @@ -# Plugin protocol v1 - -Status: **pre-release contract for ElecTrace 0.1** - -This specification defines the normalized JSON boundary between an ElecTrace adapter -and the correspondence engine. The normative schema identifier is -`electrace.features.v1`. - -The key words **MUST**, **MUST NOT**, **SHOULD**, and **MAY** are to be interpreted as -requirements for interoperable implementations. - -## Design goals - -- Keep external scientific tools outside the core dependency graph. -- Preserve enough provenance to audit every normalized value. -- Prevent a frame-local parser identifier from becoming an accidental cross-frame - identity. -- Use one periodic coordinate and atom-index convention. -- Permit tool-specific evidence without changing the core envelope. -- Make command execution an explicit policy decision, not a parser side effect. - -## Adapter lifecycle - -An adapter has two independent responsibilities: - -1. **Plan:** inspect configuration and return a command plan. -2. **Parse:** read declared outputs and return a validated - `electrace.features.v1` document. - -Planning MUST NOT launch an executable or modify a source calculation. A command plan -contains: - -| Field | Meaning | -| --- | --- | -| `plugin` | Registered adapter name. | -| `capability` | Requested scientific result family. | -| `argv` | Non-empty argument vector. It is never a shell command string. | -| `cwd` | Explicit working directory for an authorized executor. | -| `stdin` | Exact standard-input content or an explicit absence. | -| `stdout_path` | Optional explicit output-capture path; never a shell redirection. | -| `required_inputs` | Files that MUST exist before execution. | -| `expected_outputs` | Files the parser expects after successful execution. | -| `environment` | Explicit environment additions; secrets MUST NOT be embedded. | -| `notes` | Human-readable, non-authoritative plan warnings or instructions. | - -The serialized audit form also includes `missing_inputs`, computed from the declared -requirements at inspection time. It is diagnostic state, not part of the executable -instruction. - -`electrace plan PLUGIN FRAME` serializes one plan. With `--all-frames`, `FRAME` is an -ordered VASP path root and the CLI serializes a deterministic batch containing one -independent plan per discovered frame. Batch planning does not change the plugin -contract and still performs no execution. Its envelope contains `plugin`, `path`, and -a `plans` array; `plans` preserves discovered frame order, and each element pairs a -canonical `frame` record with its `command` plan. - -The executor—not the adapter API—decides whether to run a plan. An executor SHOULD -resolve the executable, apply a sandbox appropriate to the data, capture versions and -exit status, verify the required inputs, and avoid overwriting source files. A parser -MUST NOT search arbitrary undeclared locations to make a failed plan appear -successful. - -## Normalized document - -The complete JSON Schema is -[`electrace.features.v1.schema.json`](https://github.com/Austin243/ElecTrace/blob/main/schemas/electrace.features.v1.schema.json). -A valid document has this shape: - -```json -{ - "schema_version": "electrace.features.v1", - "producer": { - "name": "critic2-adapter", - "version": "0.1.0" - }, - "frame": { - "index": 5, - "label": "05" - }, - "features": [ - { - "id": "bcp-12", - "kind": "bond_critical_point", - "fractional_position": [0.125, 0.5, 0.875], - "neighbor_atom_indices": [3, 8], - "attributes": { - "rho_e_per_angstrom3": 0.42 - } - } - ], - "metadata": {} -} -``` - -Only `schema_version`, `producer`, and `features` are required at the root. `frame` -and `metadata` are optional. Producers SHOULD include `frame` and provenance metadata -for any record persisted outside a parent analysis manifest. - -### `schema_version` - -The value MUST be exactly `electrace.features.v1`. Readers MUST reject any other -identifier unless they implement an explicit migration. They MUST NOT guess based on -field similarity. - -### `producer` - -`producer.name` identifies the adapter or writer, not merely the upstream executable. -`producer.version` SHOULD be present when known. The upstream executable, its version, -and the structured argument vector belong in `metadata.provenance`. - -### `frame` - -`frame.index`, when present, is the zero-based position in the canonical ordered -path. `frame.label`, when present, is a display label such as an image directory name. -Neither establishes identity outside the parent analysis manifest. At least an index -or label SHOULD be included in records written to disk. - -### `features` - -Each feature MUST contain: - -- `id`: a non-empty identifier unique within this document; -- `kind`: a non-empty feature kind normalized to lower case; -- `fractional_position`: exactly three finite fractional-coordinate values. - -Writers SHOULD wrap periodic fractional coordinates into the half-open interval -`[0, 1)`. Readers accept finite unwrapped values because a legitimate upstream -representation may preserve image offsets. - -Optional `neighbor_atom_indices` are unique, zero-based indices into the canonical -atom order. Optional `attributes` are a JSON object for descriptor-specific evidence. - -`id` is frame-local. Only the correspondence engine may assign a persistent track -identifier spanning frames. - -The recommended vocabulary is `atomic_basin`, `elf_basin`, -`non_nuclear_attractor`, `bond_critical_point`, `ring_critical_point`, -`cage_critical_point`, `bond_path`, `bond_descriptor`, `atomic_charge`, and `other`. -The core accepts additional normalized kinds for third-party tools. Writers SHOULD -reuse the recommended spelling and MUST document genuinely new kinds in `metadata`. - -Optional standard scalars are: - -- `charge`: a finite value in elementary-charge units; -- `volume`: a non-negative finite value in cubic ångström. - -Because upstream tools expose populations and charges with different sign and -reference conventions, any document containing `charge` MUST record an unambiguous -`metadata.charge_convention`. Values with incompatible conventions MUST NOT be fused -or differenced as though they were equivalent. - -Optional `voxel_indices` provides unique integer grid triplets for features with -voxel support. The grid shape, origin, ordering, and periodic convention MUST be -recorded in `attributes` or frame metadata; a voxel index has no portable meaning -without them. - -Attribute keys SHOULD encode their quantity and unit, for example -`rho_e_per_angstrom3` or `icohp_ev`. A producer-specific key SHOULD be namespaced when -its semantics are not common, for example `critic2:laplacian`. Missing evidence is -omitted or represented as JSON `null` where the schema permits it; it MUST NOT be -encoded as zero, `NaN`, `Infinity`, or a string sentinel. - -### `metadata` - -`metadata` is the extension and provenance object. It MAY contain arbitrary valid -JSON, but producers SHOULD include: - -```json -{ - "provenance": { - "external_tool": {"name": "critic2", "version": "..."}, - "source_sha256": {"CONTCAR": "..."}, - "argv": ["critic2", "critic2.in"], - "input_sha256": "...", - "output_sha256": "..." - }, - "charge_convention": "description of reference and sign", - "warnings": [] -} -``` - -Paths in portable records SHOULD be project-relative or paired with content hashes. -Credentials and sensitive host paths MUST NOT be recorded. - -## JSON rules - -- Encoding MUST be UTF-8. -- The root MUST be a single JSON object. -- Numbers MUST be finite; JSON extensions for `NaN` and infinity are forbidden. -- Duplicate object keys are forbidden. -- Array order is significant for `features` and MUST be deterministic. -- Producers SHOULD serialize with stable key ordering for reproducible diffs. -- The top-level and feature envelopes are closed. Tool-specific data SHOULD live in - `attributes` or `metadata` rather than creating a near-synonym for a standard field. - -## Validation and failure behavior - -Writers MUST validate before publishing a normalized document. Readers MUST validate -before scientific use. - -A malformed envelope, empty kind, duplicate feature ID, invalid neighbor index, or -non-finite value is an error. Extension kinds are accepted. A missing optional -scientific quantity is not an error, but downstream event evidence that requires it -remains missing. - -Parser warnings belong in `metadata.warnings` and must survive into reports. A parser -MUST NOT fabricate an empty successful document after an external-tool or parse -failure. The orchestration layer records failed/partial runs separately and -suppresses dependent scientific claims. - -## Compatibility - -The core fields and their meanings are frozen for v1. New tool-specific attributes or -metadata are compatible when old readers can ignore them safely. Adding or changing -an envelope field, changing a unit/convention, or changing requiredness requires a new -schema identifier and migration documentation. - -## Security notes - -- Pass `argv` directly to a subprocess API with shell evaluation disabled. -- Resolve and validate `cwd`, inputs, outputs, and symlinks against the authorized - project boundary. -- Do not trust frame labels, attribute keys, warnings, or upstream output in HTML. -- Impose size and resource limits on volumetric and JSON input. -- Do not allow a plugin document to select or execute an executable. - -See the [security policy](https://github.com/Austin243/ElecTrace/blob/main/SECURITY.md) -for the reporting process and broader threat model. diff --git a/docs/workflows.md b/docs/workflows.md index 1e215d3..313e5a5 100644 --- a/docs/workflows.md +++ b/docs/workflows.md @@ -1,438 +1,124 @@ -# External calculation workflows - -ElecTrace can stage an ordered VASP path, explicitly run or submit selected -external calculations, collect supported descriptor outputs, and analyze the -collected series. This is an opt-in orchestration layer. It does not design a -scientifically converged VASP, BaderKit, critic2, or LOBSTER calculation for you. - -The plugin command-plan API remains non-executing. Only `electrace workflow run ---yes` and `electrace workflow submit --yes` cross the execution boundary. Review -every generated job or Slurm script and follow the policies of the machine on which -it will run. - -## Safety and directory boundary - -Keep the workflow manifest, source frames, templates, and generated workspace -separate: - -```text -path-study/ -├── electrace-workflow.yaml -├── source-frames/ -│ ├── 00/CONTCAR -│ ├── 01/CONTCAR -│ └── 02/CONTCAR -└── templates/ - ├── vasp-static/ - │ ├── INCAR - │ ├── KPOINTS - │ └── POTCAR # user-supplied; never commit it - └── lobster/ - └── lobsterin -``` +# Using ElecTrace -The important boundaries are: - -- `project.root` is read-only. ElecTrace copies the selected structure but never - writes into a source frame. -- `project.workspace` must be outside `project.root`. Staging records - `.electrace-workspace.json` there and `--force` refuses to replace an unmarked or - symlinked directory. -- Changing the YAML after staging invalidates the recorded manifest hash. Stage - again rather than running a stale job graph. -- Template directories are copied into isolated job directories. They must contain - only reviewed regular files, without nested directories or symlinks. -- ElecTrace does not install external programs, acquire licenses, download - pseudopotentials, or provide a `POTCAR`. If your license permits it, place your own - `POTCAR` in the private VASP template; staging copies that user-supplied template - into the private workspace. Never commit or redistribute the file. - -Relative `project.root`, `project.workspace`, `template_dir`, and executable paths -that contain a slash are resolved from the YAML file's directory. A bare executable -name such as `baderkit` is resolved through the execution environment's `PATH`. -`argv_prefix` is an argument array, not a shell fragment; for example, `["srun"]` -adds exactly one token. Shell expansion, pipes, redirection, and command -substitution are never inferred. - -## Create and review the manifest - -Generate a starter beside your project: +ElecTrace can analyze descriptor files that already exist or stage the calculations +needed for selected frames. External programs and `POTCAR` files remain user-supplied. -```bash -electrace workflow init electrace-workflow.yaml \ - --images /absolute/path/to/source-frames -``` +## Install and inspect -Then edit it. The public contract is -[`electrace.workflow.v1`](https://github.com/Austin243/ElecTrace/blob/main/schemas/electrace.workflow.v1.schema.json), -and the repository also contains a compact -[public example](https://github.com/Austin243/ElecTrace/blob/main/examples/electrace.workflow.yaml). -The workflow YAML is distinct from the `electrace.toml` analysis configuration: -the YAML describes external jobs, while the TOML describes correspondence, event -detection, and result exports. - -The following complete manifest uses only staging option names implemented in v1. -It runs one static VASP step, then branches to BaderKit BadELF basins and LOBSTER -ICOHP output. Replace every executable and template with settings validated for your -system before staging. - -```yaml -schema_version: electrace.workflow.v1 - -project: - root: source-frames - series_kind: neb - workspace: .electrace-work - structure: CONTCAR - fallback: POSCAR - -# Canonical zero-based frame indices, not directory names. -frames: "all" - -executables: - vasp: - path: /opt/vasp/bin/vasp_std - baderkit: - path: /opt/baderkit/bin/baderkit - lobster: - path: /opt/lobster/bin/lobster - -scheduler: - kind: local - directives: {} - -steps: - - id: vasp_static - kind: vasp - executable: vasp - template_dir: templates/vasp-static - options: - required_inputs: - - INCAR - - KPOINTS - - POSCAR - - POTCAR - expected_outputs: - - CHGCAR - - ELFCAR - - WAVECAR - incar_overrides: - LCHARG: true - LELF: true - LWAVE: true - NSW: 0 - ISYM: -1 - - - id: badelf_basins - kind: baderkit - executable: baderkit - capability: elf_basins - depends_on: - - vasp_static - options: - charge_file: CHGCAR - elf_file: ELFCAR - - - id: lobster_icohp - kind: lobster - executable: lobster - capability: cohp - template_dir: templates/lobster - depends_on: - - vasp_static - options: - input_file: lobsterin - wavecar: WAVECAR - structure_file: POSCAR - potcar: POTCAR +```bash +python -m pip install "git+https://github.com/Austin243/ElecTrace.git" +electrace --help +electrace doctor ``` -BadELF output already carries BaderKit atomic population or oxidation-state arrays -when the external tool supplies them. Do not add a simultaneous BaderKit -`bader_charges` step for the same frame: `bader.json` and `badelf.json` both normalize -their atomic records as `baderkit:atom:*`, and analysis correctly rejects those -duplicate feature IDs. For a Bader-only workflow, replace `badelf_basins` with this -step and remove `ELFCAR` from the VASP expected outputs if no other step needs it: - -```yaml - - id: bader_populations - kind: baderkit - executable: baderkit - capability: bader_charges - depends_on: - - vasp_static - options: - charge_file: CHGCAR -``` +Create a TOML configuration for an ordered VASP path and analyze existing evidence: -Dependencies must refer to earlier step IDs. Staging links declared dependency -outputs into each downstream job directory, while the runner still refuses to start -a child after a failed dependency. The VASP `required_inputs`, `expected_outputs`, -and `incar_overrides` keys above are orchestration checks, not proof that the -calculation is converged. - -An optional step-level `resources` mapping is retained as advisory metadata in v1; -it does not control local concurrency or Slurm allocation. Use `--max-parallel` for -local concurrency and `scheduler.directives` for actual Slurm requests. - -## Select frames - -`frames` addresses the canonical zero-based order produced by deterministic frame -discovery. It does not address a directory merely because that directory is named -with the same number. - -```yaml -frames: "all" # every discovered frame -frames: "0,3,7" # three explicit canonical indices -frames: "1:8" # indices 1 through 7 -frames: "0:10:2" # every second frame among indices 0 through 9 -frames: "::3" # every third discovered frame -frames: "0,2:9:2" # an explicit endpoint plus a slice +```bash +electrace init electrace.toml --images /absolute/path/to/frames +# Set descriptor_source and descriptor_dir in electrace.toml when needed. +electrace analyze electrace.toml +electrace inspect electrace-results ``` -Slices follow Python's clipping and negative-index rules, including negative steps. -Comma-separated items are evaluated left to right. An out-of-range explicit index, -an empty result, a zero slice step, or selecting one frame twice is an error. Quote -selectors containing `:` so YAML cannot reinterpret them. +Supported inputs include normalized ElecTrace feature JSON, Bader/BaderKit output, +critic2 `CPREPORT` JSON, LOBSTER integrated bond lists, and Chargemol/DDEC charges. +`analysis.json` is the canonical result; CSV and HTML files are derived views. -## Validate without writing +## Select frames and stage calculations -Resolve the manifest, frame selector, templates, executable vectors, scheduler, -dependencies, required inputs, and expected outputs before creating a workspace: +Create and edit a workflow manifest: ```bash +electrace workflow init electrace-workflow.yaml --images /absolute/path/to/frames electrace workflow validate electrace-workflow.yaml --json +electrace workflow stage electrace-workflow.yaml --json ``` -Validation is strictly read-only. It does not create the configured workspace, copy a -template, render a Slurm script, or launch a command. Missing executables are reported -as warnings because they may be supplied later by a compute-node module environment; -invalid templates, unresolved inputs, unsafe paths, and invalid job graphs are errors. -The resolved plan is returned inside an `electrace.cli.v1` response envelope and -follows the public -[`electrace.workflow-validation.v1` schema](https://github.com/Austin243/ElecTrace/blob/main/schemas/electrace.workflow-validation.v1.schema.json). +The manifest supplies executable paths, template directories, dependencies, +scheduler settings, and a frame selector. Selectors accept `all`, explicit indices, +or Python-style slices: -## Stage and inspect without executing +| Selector | Result | +| --- | --- | +| `all` | Every discovered frame | +| `0,3,7` | Frames 0, 3, and 7 | +| `1:8` | Frames 1 through 7 | +| `0:10:2` | Every second frame from 0 through 8 | +| `0,2:9:2` | Frame 0, then 2, 4, 6, and 8 | -Staging discovers the selected frames, copies templates and structures, applies -`INCAR` overrides, constructs the dependency DAG, and writes auditable job records. -It does not launch a program: +Staging writes an isolated workspace and does not execute a program. Review the +generated commands and inputs before authorizing work. -```bash -electrace workflow stage electrace-workflow.yaml -electrace workflow status electrace-workflow.yaml --json -``` +## Supported calculation steps -Use `workflow status` as a filesystem snapshot only. It reports required inputs and -expected outputs that currently exist; it does not query Slurm, parse convergence, -or establish that a scientific calculation succeeded. +| Goal | Step | Capability | Collected output | +| --- | --- | --- | --- | +| VASP static calculation | `vasp` | user-defined | manifest `expected_outputs` | +| Bader charge analysis | `bader` | `bader_charges` | `ACF.dat` | +| BaderKit charge analysis | `baderkit` | `bader_charges` | `bader.json` | +| BadELF or NNA analysis | `baderkit` | `elf_basins` or `non_nuclear_attractors` | `badelf.json` | +| QTAIM/ELF critical points | `critic2` | `qtaim_critical_points` or `non_nuclear_attractors` | `critic2.features.json` | +| Integrated COHP/COBI | `lobster` | `cohp` or `cobi` | `ICOHPLIST.lobster` or `ICOBILIST.lobster` | -If you intentionally changed a template or manifest, restage the marker-protected -workspace: +Chargemol can be planned and its existing results can be parsed, but workflow v1 does +not stage a Chargemol step. -```bash -electrace workflow stage electrace-workflow.yaml --force -``` +Each step points to an executable entry and may declare a template directory, +dependencies, expected outputs, environment variables, and plugin options. VASP +templates normally provide `INCAR`, `KPOINTS`, and `POTCAR`; ElecTrace places the +selected structure at `POSCAR`. Descriptor steps can consume files from dependency +steps with `options.input_links`. ## Run locally -Local execution requires `scheduler.kind: local` and explicit confirmation: +Set `scheduler.kind: local`, stage the workflow, and authorize the reviewed commands: ```bash electrace workflow run electrace-workflow.yaml --yes --max-parallel 2 -electrace workflow status electrace-workflow.yaml -``` - -Commands are launched with `shell=False` in their dedicated job directories. -Standard output and standard error are captured in separate files, required inputs -are checked before launch, and expected outputs are checked afterward. The default -is one job at a time. `--keep-going` permits new independent jobs to start after a -failure; dependents of a failed job still do not run. - -Without `--yes`, `workflow run` refuses to execute anything. - -## Prepare or submit Slurm jobs - -For Slurm, replace the local scheduler block with site-reviewed directives: - -```yaml -scheduler: - kind: slurm - directives: - partition: compute - time: "00:30:00" - nodes: 1 - ntasks: 8 - cpus-per-task: 1 - exclude: "node[1-8]" +electrace workflow status electrace-workflow.yaml --json ``` -If the site requires a launcher, add it as tokens to the relevant executable rather -than embedding a shell command: +Independent jobs may run concurrently. A failed dependency blocks its dependents. +The command verifies declared inputs and expected outputs but cannot determine DFT or +descriptor convergence. -```yaml -executables: - vasp: - path: /opt/vasp/bin/vasp_std - argv_prefix: - - srun -``` +## Submit through Slurm -First prepare scripts without submitting them: +Set `scheduler.kind: slurm` and put ordinary `#SBATCH` options under +`scheduler.directives`. First generate scripts without submitting: ```bash -electrace workflow stage electrace-workflow.yaml electrace workflow submit electrace-workflow.yaml --job-name electrace ``` -This creates one dependency-ordered script per selected frame. Review the scripts, -their paths, requested resources, environment, and job name. Submit only after that -review: +Review the scripts, then authorize submission: ```bash -electrace workflow submit electrace-workflow.yaml \ - --job-name electrace \ - --yes \ - --force +electrace workflow submit electrace-workflow.yaml --job-name electrace --yes --force ``` -`--force` here replaces generated ElecTrace Slurm scripts; it is not the execution -confirmation. `--yes` invokes `sbatch` (or the executable supplied with -`--submit-command`). Submission returns a scheduler job ID, not a successful -calculation. Use your site's scheduler tools and then `workflow status` to review the -jobs and files. - -ElecTrace journals every accepted ID in `slurm/submissions.json`. If an authorized -submission exits `1`, inspect `error.details.partial_submissions`, `record_path`, and -that journal before retrying. A prior journal blocks another submission even with -`--force`; archive or remove it only after reconciling every accepted ID. Never -resubmit an already accepted script blindly. - -## Collect, analyze, and report - -After every selected descriptor job has produced its declared output, collection -validates parser-supported files and publishes an isolated analysis input tree: +Use a manifest scheduler directive such as `exclude: "node[01-08]"` when particular +nodes must not be used. ElecTrace does not query Slurm availability or choose resources. -```bash -electrace workflow collect electrace-workflow.yaml -``` - -Collection rejects missing, empty, or symlinked descriptor outputs. It copies the -selected structures and recognized native or canonical descriptors without changing -the source frames or staged job directories. The managed tree is: - -```text -.electrace-work/analysis/ -├── .electrace-collection.json -├── input/ -│ └── /CONTCAR -├── descriptors/ -│ └── /{bader.json,badelf.json,ICOHPLIST.lobster,...} -├── electrace.toml -└── results/ -``` +## Collect and analyze -The generated TOML points the core analyzer at this isolated, ordered series. Run -collection and analysis together with: +After the declared outputs exist: ```bash -electrace workflow analyze electrace-workflow.yaml -electrace inspect .electrace-work/analysis/results -``` - -`workflow analyze` creates a collection when one is absent and reuses a valid -existing collection, so `workflow collect` followed by `workflow analyze` does not -need `--force`. Strict analysis is the default and fails on incomplete discovery or -ambiguous correspondence. Use `--permissive` only after reviewing that policy: it -records warnings and suppresses affected claims rather than resolving ambiguity. - -Collection and analysis protect managed output. Re-collecting into an existing -collection, or replacing nonempty analysis results on a later analyze run, requires -an intentional `--force`: - -```bash -electrace workflow analyze electrace-workflow.yaml --force +electrace workflow status electrace-workflow.yaml --json +electrace workflow analyze electrace-workflow.yaml --json ``` -The result directory contains canonical `analysis.json`, a hash manifest, derived -CSV tables, and the offline `report.html` when enabled. JSON is the lossless source; -the report and tables are views. A clean artifact bundle does not mean that VASP, -BaderKit, LOBSTER, or the scientific interpretation was converged. - -## Scientific checks before interpreting the report - -### Bader and BadELF evidence - -Bader atomic values are partition-dependent integrated electron populations, not -formal oxidation states. ElecTrace labels BaderKit `atom_charges` and basin charges -as positive electron populations. When a BaderKit document instead supplies its -`oxidation_states` array, ElecTrace preserves those signed values with the explicit -`bader_oxidation_state` definition; it does not derive, validate, or promote them as -a unique formal oxidation-state assignment. - -BadELF requires the charge and ELF grids to be congruent. BaderKit's -[BadELF VASP guide](https://sweav02.github.io/baderkit/tutorials/vasp/badelf/) -warns that VASP normally writes `CHGCAR` and `ELFCAR` on different FFT meshes and -recommends matching `NGX/NGY/NGZ` with `NGXF/NGYF/NGZF`. Select and converge those -values for the material; the generic YAML above cannot supply universal numbers. -Grid agreement at one resolution is necessary but is not a grid-refinement study. - -### critic2 boundary - -The current critic2 adapter runs a conservative `AUTO` critical-point search and -collects native `CPREPORT` JSON as `critic2.features.json`. It normalizes critical -point topology; it does not parse `critic2.cro`, request attraction-basin -integration, or treat critical-point properties as integrated basin populations. -Selecting the critic2 `elf_basins` capability changes the analyzed scalar field to -`ELFCAR`, but the current handoff still reports ELF critical-point topology rather -than integrated ELF basin volumes or populations. Use the supported BaderKit BadELF -path when integrated BadELF basin records are required. -critic2 itself supports several -[basin-integration methods](https://aoterodelaroza.github.io/critic2/manual/integrate/), -but those methods are outside the current ElecTrace workflow handoff. - -### LOBSTER and ICOHP evidence - -Run LOBSTER from a converged VASP static calculation on the intended geometry. The -[LobsterPy input guide](https://jageo.github.io/LobsterPy/tutorial/commandlineinterface.html) -identifies the critical VASP controls: `NSW = 0`, `LWAVE = .TRUE.`, a LOBSTER-compatible -`ISYM` setting (`-1` for the complete mesh or `0` for the time-reversal half mesh), -and enough `NBANDS` for the chosen local projection basis. Choose and converge the -`KPOINTS` mesh rather than inheriting an unrelated relaxation mesh. - -ElecTrace currently collects `ICOHPLIST.lobster` for `capability: cohp` and uses its -integrated values as bond-descriptor evidence. The ElecTrace 0.1 native-list contract -supports simple, non-spin rows with one terminal integrated value. Spin-resolved -LOBSTER list ingestion is not supported, and users must not interpret a parsed value -as channel-resolved evidence. The current parser does not guarantee detection of -every spin-resolved native variant. - -ElecTrace does not parse `lobsterout`, verify LOBSTER convergence, judge the basis, -or enforce a spilling threshold. Retain and manually inspect the full LOBSTER output, -absolute/total charge spilling, band overlaps where applicable, basis choice, energy -range, k-point convergence, and VASP convergence. The LOBSTER authors emphasize that -[lower spilling indicates a more faithful projection](https://schmeling.ac.rwth-aachen.de/cohp/index.php?menuID=202), -and that an integrated COHP only -[hints at bond strength](https://schmeling.ac.rwth-aachen.de/cohp/index.php?menuID=3); -it is not a bond energy or a universal bond definition. - -## Inspect a volumetric file directly - -`electrace volume` is an independent, read-only diagnostic for a plain VASP scalar -grid: +Analysis collects supported outputs, normalizes them, matches atoms and electronic +features across frames, and writes the same result bundle used for existing evidence. -```bash -electrace volume /path/to/ELFCAR -electrace volume /path/to/CHGCAR --json -electrace volume /path/to/ELFCAR --dataset 0 --sample 0.5,0.5,0.5 --json -``` +## Scientific boundaries -It reports the grid shape, point count, cell volume, dataset count, and simple value -statistics, and can periodically sample one fractional coordinate. Compare the -reported `shape` for `CHGCAR` and `ELFCAR` when reviewing BadELF inputs. +- Bader and DDEC populations are not unique formal oxidation states. +- ICOHP/ICOBI values depend on the projection basis and integration convention. +- critic2 support normalizes critical-point topology; it does not integrate ELF basins. +- Native spin-resolved LOBSTER list ingestion is unsupported. +- Filesystem completion is not proof of electronic, grid, or projection convergence. +- ElecTrace never supplies licensed pseudopotentials or chooses universal VASP settings. -The reader preserves the raw numbers exactly as stored; it does not assign physical -units or normalize CHGCAR values. It supports plain same-shape scalar datasets and -explicitly rejects augmentation blocks or ambiguous tails rather than guessing how -to skip them. It is a format and grid diagnostic, not a Bader/ELF integration or a -convergence test. +See [Scientific scope and limitations](scientific-scope.md) before interpreting events. diff --git a/examples/electrace.example.toml b/examples/electrace.example.toml index 42bb3a0..35a4627 100644 --- a/examples/electrace.example.toml +++ b/examples/electrace.example.toml @@ -23,7 +23,6 @@ overlap_weight = 1.0 descriptor_weight = 0.35 match_threshold = 3.0 # Maximum total-cost gap for a complete alternative assignment. -ambiguity_tolerance = 0.05 conservation_tolerance = 0.35 split_merge_threshold = 0.55 @@ -33,7 +32,6 @@ icobi = 0.1 [events] minimum_persistence = 2 -hysteresis_frames = 1 minimum_event_confidence = 0.10 charge_change_threshold = 0.10 volume_change_fraction = 0.20 @@ -51,7 +49,6 @@ directory = "electrace-results" html_report = true json = true csv = true -hdf5 = false # Plugins are inspected and planned by ElecTrace, but never executed implicitly. [plugins.baderkit] diff --git a/mkdocs.yml b/mkdocs.yml index d0814ce..112db0d 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -16,19 +16,9 @@ theme: accent: cyan nav: - Overview: index.md - - Quickstart: quickstart.md - - External workflows: workflows.md + - Using ElecTrace: workflows.md - Coding agents: agents.md - Scientific scope: scientific-scope.md - - Architecture: architecture.md - - Integration: - - Plugin protocol v1: spec/plugin-protocol-v1.md - - Synthetic benchmark: benchmarking.md - - Reproducibility: reproducibility.md - - Pre-release migration: migration.md - - Roadmap: roadmap.md - - Governance: governance.md - - Sources: sources.md markdown_extensions: - admonition - attr_list diff --git a/pyproject.toml b/pyproject.toml index af1d4a9..f2948a8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "electrace" -version = "0.1.0a4" +version = "0.1.0a5" description = "Persistent electronic-topology tracking across periodic reaction paths and structure series" readme = "README.md" requires-python = ">=3.11,<3.14" @@ -42,7 +42,6 @@ dependencies = [ [project.optional-dependencies] accelerate = ["scipy>=1.11"] -hdf5 = ["h5py>=3.10"] docs = [ "jsonschema>=4.23,<5", "mkdocs>=1.6", @@ -55,13 +54,11 @@ test = [ dev = [ "build>=1.2", "mypy>=1.11", - "pre-commit>=3.7", "pytest>=8.2", "pytest-cov>=5.0", "ruff>=0.6", ] all = [ - "h5py>=3.10", "scipy>=1.11", ] @@ -78,7 +75,6 @@ packages = ["src/electrace"] [tool.hatch.build.targets.sdist] exclude = [ "/.github", - "/.pre-commit-config.yaml", "/AGENTS.md", ] @@ -120,5 +116,5 @@ warn_unreachable = true show_error_codes = true [[tool.mypy.overrides]] -module = ["scipy.*", "h5py.*"] +module = ["scipy.*"] ignore_missing_imports = true diff --git a/schemas/benchmark-v0.1.schema.json b/schemas/benchmark-v0.1.schema.json deleted file mode 100644 index cf57071..0000000 --- a/schemas/benchmark-v0.1.schema.json +++ /dev/null @@ -1,231 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://github.com/Austin243/ElecTrace/schemas/benchmark-v0.1.schema.json", - "title": "ElecTrace synthetic lineage benchmark", - "type": "object", - "required": ["schema_version", "software_version", "suite", "summary", "cases"], - "properties": { - "schema_version": {"const": "electrace.benchmark.v0.1"}, - "software_version": {"type": "string", "minLength": 1}, - "suite": { - "type": "object", - "required": [ - "name", - "version", - "synthetic", - "scientific_validation", - "description", - "limitations" - ], - "properties": { - "name": {"const": "electrace-synthetic-lineage"}, - "version": {"type": "string", "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"}, - "synthetic": {"const": true}, - "scientific_validation": {"const": false}, - "description": {"type": "string", "minLength": 1}, - "limitations": { - "type": "array", - "minItems": 1, - "items": {"type": "string", "minLength": 1} - } - }, - "additionalProperties": false - }, - "summary": {"$ref": "#/$defs/summary"}, - "cases": { - "type": "array", - "minItems": 1, - "items": {"$ref": "#/$defs/case"} - } - }, - "additionalProperties": false, - "$defs": { - "identifier": { - "oneOf": [{"type": "integer"}, {"type": "string"}] - }, - "mappingRecord": { - "type": "object", - "required": ["source_id", "target_id"], - "properties": { - "source_id": {"$ref": "#/$defs/identifier"}, - "target_id": {"$ref": "#/$defs/identifier"} - }, - "additionalProperties": false - }, - "eventLabel": { - "type": "object", - "required": ["type", "source_ids", "target_ids"], - "properties": { - "type": { - "type": "string", - "enum": ["birth", "death", "split", "merge", "rearrangement", "property_change"] - }, - "source_ids": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, - "target_ids": {"type": "array", "items": {"type": "string"}, "uniqueItems": true} - }, - "additionalProperties": false - }, - "observedEvent": { - "type": "object", - "required": ["type", "source_ids", "target_ids", "confidence"], - "properties": { - "type": { - "type": "string", - "enum": ["birth", "death", "split", "merge", "rearrangement", "property_change"] - }, - "source_ids": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, - "target_ids": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, - "confidence": {"type": "number", "minimum": 0, "maximum": 1} - }, - "additionalProperties": false - }, - "expectation": { - "type": "object", - "required": [ - "events", - "atom_mapping", - "feature_mapping", - "atom_ambiguous", - "feature_ambiguous" - ], - "properties": { - "events": {"type": "array", "items": {"$ref": "#/$defs/eventLabel"}}, - "atom_mapping": {"type": "array", "items": {"$ref": "#/$defs/mappingRecord"}}, - "feature_mapping": {"type": "array", "items": {"$ref": "#/$defs/mappingRecord"}}, - "atom_ambiguous": {"type": "boolean"}, - "feature_ambiguous": {"type": "boolean"} - }, - "additionalProperties": false - }, - "relation": { - "type": "object", - "required": ["relation", "source_ids", "target_ids", "confidence", "ambiguous"], - "properties": { - "relation": { - "type": "string", - "enum": ["one_to_one", "split", "merge", "many_to_many"] - }, - "source_ids": {"type": "array", "items": {"type": "string"}, "minItems": 1}, - "target_ids": {"type": "array", "items": {"type": "string"}, "minItems": 1}, - "confidence": {"type": "number", "minimum": 0, "maximum": 1}, - "ambiguous": {"type": "boolean"} - }, - "additionalProperties": false - }, - "observation": { - "type": "object", - "required": [ - "events", - "atom_mapping", - "feature_mapping", - "atom_ambiguous", - "feature_ambiguous", - "atom_ambiguity_count", - "feature_ambiguity_count", - "relations", - "hyperedge_rejections" - ], - "properties": { - "events": {"type": "array", "items": {"$ref": "#/$defs/observedEvent"}}, - "atom_mapping": {"type": "array", "items": {"$ref": "#/$defs/mappingRecord"}}, - "feature_mapping": {"type": "array", "items": {"$ref": "#/$defs/mappingRecord"}}, - "atom_ambiguous": {"type": "boolean"}, - "feature_ambiguous": {"type": "boolean"}, - "atom_ambiguity_count": {"type": "integer", "minimum": 0}, - "feature_ambiguity_count": {"type": "integer", "minimum": 0}, - "relations": {"type": "array", "items": {"$ref": "#/$defs/relation"}}, - "hyperedge_rejections": { - "type": "array", - "items": {"type": "object", "additionalProperties": true} - } - }, - "additionalProperties": false - }, - "metrics": { - "type": "object", - "required": [ - "event_true_positives", - "event_false_positives", - "event_false_negatives", - "mapping_assertions_correct", - "mapping_assertions_total", - "ambiguity_assertions_correct", - "ambiguity_assertions_total" - ], - "properties": { - "event_true_positives": {"type": "integer", "minimum": 0}, - "event_false_positives": {"type": "integer", "minimum": 0}, - "event_false_negatives": {"type": "integer", "minimum": 0}, - "mapping_assertions_correct": {"type": "integer", "minimum": 0}, - "mapping_assertions_total": {"type": "integer", "minimum": 0}, - "ambiguity_assertions_correct": {"type": "integer", "minimum": 0}, - "ambiguity_assertions_total": {"type": "integer", "minimum": 0} - }, - "additionalProperties": false - }, - "case": { - "type": "object", - "required": [ - "id", - "category", - "description", - "perturbation", - "passed", - "failures", - "expected", - "observed", - "metrics" - ], - "properties": { - "id": {"type": "string", "pattern": "^[a-z0-9]+(?:-[a-z0-9]+)*$"}, - "category": {"type": "string", "minLength": 1}, - "description": {"type": "string", "minLength": 1}, - "perturbation": {"type": "string", "minLength": 1}, - "passed": {"type": "boolean"}, - "failures": {"type": "array", "items": {"type": "string", "minLength": 1}}, - "expected": {"$ref": "#/$defs/expectation"}, - "observed": {"$ref": "#/$defs/observation"}, - "metrics": {"$ref": "#/$defs/metrics"} - }, - "additionalProperties": false - }, - "summary": { - "type": "object", - "required": [ - "cases", - "passed", - "failed", - "event_true_positives", - "event_false_positives", - "event_false_negatives", - "event_precision", - "event_recall", - "event_f1", - "mapping_assertions_correct", - "mapping_assertions_total", - "mapping_accuracy", - "ambiguity_assertions_correct", - "ambiguity_assertions_total", - "ambiguity_accuracy" - ], - "properties": { - "cases": {"type": "integer", "minimum": 1}, - "passed": {"type": "integer", "minimum": 0}, - "failed": {"type": "integer", "minimum": 0}, - "event_true_positives": {"type": "integer", "minimum": 0}, - "event_false_positives": {"type": "integer", "minimum": 0}, - "event_false_negatives": {"type": "integer", "minimum": 0}, - "event_precision": {"type": "number", "minimum": 0, "maximum": 1}, - "event_recall": {"type": "number", "minimum": 0, "maximum": 1}, - "event_f1": {"type": "number", "minimum": 0, "maximum": 1}, - "mapping_assertions_correct": {"type": "integer", "minimum": 0}, - "mapping_assertions_total": {"type": "integer", "minimum": 1}, - "mapping_accuracy": {"type": "number", "minimum": 0, "maximum": 1}, - "ambiguity_assertions_correct": {"type": "integer", "minimum": 0}, - "ambiguity_assertions_total": {"type": "integer", "minimum": 1}, - "ambiguity_accuracy": {"type": "number", "minimum": 0, "maximum": 1} - }, - "additionalProperties": false - } - } -} diff --git a/schemas/electrace.cli.v1.schema.json b/schemas/electrace.cli.v1.schema.json index bb133d2..439be47 100644 --- a/schemas/electrace.cli.v1.schema.json +++ b/schemas/electrace.cli.v1.schema.json @@ -1,8 +1,7 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://github.com/Austin243/ElecTrace/schemas/electrace.cli.v1.schema.json", - "title": "ElecTrace CLI response envelope", - "description": "The common machine-readable response emitted by ElecTrace commands when --json is requested.", + "title": "ElecTrace CLI response", "oneOf": [ { "$ref": "#/$defs/success" @@ -12,39 +11,25 @@ } ], "$defs": { - "base": { + "common": { "type": "object", "required": [ "schema_version", "ok", "command", - "warnings", - "artifacts" + "warnings" ], "properties": { "schema_version": { "const": "electrace.cli.v1" }, - "ok": { - "type": "boolean" - }, "command": { - "type": "string", - "minLength": 1, - "pattern": "\\S" + "type": "string" }, "warnings": { "type": "array", "items": { - "type": "string", - "minLength": 1, - "pattern": "\\S" - } - }, - "artifacts": { - "type": "array", - "items": { - "$ref": "#/$defs/jsonValue" + "type": "string" } } } @@ -52,67 +37,33 @@ "success": { "allOf": [ { - "$ref": "#/$defs/base" + "$ref": "#/$defs/common" }, { - "type": "object", "required": [ - "schema_version", - "ok", - "command", - "data", - "warnings", - "artifacts" + "data" ], "properties": { - "schema_version": { - "const": "electrace.cli.v1" - }, "ok": { "const": true - }, - "command": { - "type": "string" - }, - "data": { - "$ref": "#/$defs/jsonValue" - }, - "warnings": { - "type": "array" - }, - "artifacts": { - "type": "array" } - }, - "additionalProperties": false + } } ] }, "failure": { "allOf": [ { - "$ref": "#/$defs/base" + "$ref": "#/$defs/common" }, { - "type": "object", "required": [ - "schema_version", - "ok", - "command", - "error", - "warnings", - "artifacts" + "error" ], "properties": { - "schema_version": { - "const": "electrace.cli.v1" - }, "ok": { "const": false }, - "command": { - "type": "string" - }, "error": { "type": "object", "required": [ @@ -122,51 +73,14 @@ ], "properties": { "code": { - "type": "string", - "pattern": "^[a-z][a-z0-9_]*$" + "type": "string" }, "message": { - "type": "string", - "minLength": 1, - "pattern": "\\S" + "type": "string" }, - "details": { - "$ref": "#/$defs/jsonValue" - } - }, - "additionalProperties": false - }, - "warnings": { - "type": "array" - }, - "artifacts": { - "type": "array" + "details": {} + } } - }, - "additionalProperties": false - } - ] - }, - "jsonValue": { - "oneOf": [ - { - "type": [ - "null", - "boolean", - "number", - "string" - ] - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/jsonValue" - } - }, - { - "type": "object", - "additionalProperties": { - "$ref": "#/$defs/jsonValue" } } ] diff --git a/schemas/electrace.workflow-validation.v1.schema.json b/schemas/electrace.workflow-validation.v1.schema.json index 6ced849..e31f84b 100644 --- a/schemas/electrace.workflow-validation.v1.schema.json +++ b/schemas/electrace.workflow-validation.v1.schema.json @@ -1,221 +1,64 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://github.com/Austin243/ElecTrace/schemas/electrace.workflow-validation.v1.schema.json", - "title": "ElecTrace resolved workflow validation", - "description": "The deterministic read-only plan returned by electrace workflow validate.", + "title": "ElecTrace resolved workflow plan", "type": "object", - "required": [ - "schema_version", - "manifest", - "workspace", - "selected_frames", - "resolved_jobs", - "scheduler", - "warnings" - ], + "required": ["schema_version", "manifest", "workspace", "selected_frames", "resolved_jobs", "scheduler", "warnings"], "properties": { - "schema_version": { - "const": "electrace.workflow-validation.v1" - }, - "manifest": { - "$ref": "#/$defs/path" - }, - "workspace": { - "$ref": "#/$defs/path" - }, - "selected_frames": { - "type": "array", - "minItems": 1, - "items": { - "$ref": "#/$defs/frame" - } - }, - "resolved_jobs": { - "type": "array", - "minItems": 1, - "items": { - "$ref": "#/$defs/job" - } - }, - "scheduler": { - "$ref": "#/$defs/scheduler" - }, - "warnings": { - "type": "array", - "items": { - "type": "string", - "minLength": 1, - "pattern": "\\S" - } - } + "schema_version": {"const": "electrace.workflow-validation.v1"}, + "manifest": {"$ref": "#/$defs/path"}, + "workspace": {"$ref": "#/$defs/path"}, + "selected_frames": {"type": "array", "items": {"$ref": "#/$defs/frame"}}, + "resolved_jobs": {"type": "array", "items": {"$ref": "#/$defs/job"}}, + "scheduler": {"$ref": "#/$defs/scheduler"}, + "warnings": {"$ref": "#/$defs/strings"} }, - "additionalProperties": false, "$defs": { - "path": { - "type": "string", - "minLength": 1, - "pattern": "\\S" - }, - "identifier": { - "type": "string", - "pattern": "^[A-Za-z0-9][A-Za-z0-9_.:-]*$" - }, + "path": {"type": "string", "minLength": 1}, + "strings": {"type": "array", "items": {"type": "string"}}, "frame": { "type": "object", - "required": [ - "index", - "label", - "analysis_label", - "source_directory", - "structure", - "workspace_directory", - "energy_ev" - ], + "required": ["index", "label", "analysis_label", "source_directory", "structure", "workspace_directory", "energy_ev"], "properties": { - "index": { - "type": "integer", - "minimum": 0 - }, - "label": { - "type": "string", - "minLength": 1 - }, - "analysis_label": { - "type": "string", - "minLength": 1 - }, - "source_directory": { - "$ref": "#/$defs/path" - }, - "structure": { - "$ref": "#/$defs/path" - }, - "workspace_directory": { - "$ref": "#/$defs/path" - }, - "energy_ev": { - "type": [ - "number", - "null" - ] - } + "index": {"type": "integer", "minimum": 0}, + "label": {"type": "string"}, + "analysis_label": {"type": "string"}, + "source_directory": {"$ref": "#/$defs/path"}, + "structure": {"$ref": "#/$defs/path"}, + "workspace_directory": {"$ref": "#/$defs/path"}, + "energy_ev": {"type": ["number", "null"]} }, "additionalProperties": false }, "job": { "type": "object", - "required": [ - "id", - "frame_index", - "frame_label", - "task", - "cwd", - "argv", - "stdin", - "stdout_path", - "stderr_path", - "required_inputs", - "expected_outputs", - "dependencies", - "environment" - ], + "required": ["id", "frame_index", "frame_label", "task", "cwd", "argv", "stdin", "stdout_path", "stderr_path", "required_inputs", "expected_outputs", "dependencies", "environment"], "properties": { - "id": { - "$ref": "#/$defs/identifier" - }, - "frame_index": { - "type": "integer", - "minimum": 0 - }, - "frame_label": { - "type": "string", - "minLength": 1 - }, - "task": { - "type": "string", - "minLength": 1 - }, - "cwd": { - "$ref": "#/$defs/path" - }, - "argv": { - "type": "array", - "minItems": 1, - "items": { - "type": "string", - "minLength": 1 - } - }, - "stdin": { - "type": [ - "string", - "null" - ] - }, - "stdout_path": { - "$ref": "#/$defs/path" - }, - "stderr_path": { - "$ref": "#/$defs/path" - }, - "required_inputs": { - "$ref": "#/$defs/pathArray" - }, - "expected_outputs": { - "$ref": "#/$defs/pathArray" - }, - "dependencies": { - "type": "array", - "uniqueItems": true, - "items": { - "$ref": "#/$defs/identifier" - } - }, - "environment": { - "type": "object", - "propertyNames": { - "pattern": "^[A-Za-z_][A-Za-z0-9_]*$" - }, - "additionalProperties": { - "type": "string" - } - } + "id": {"type": "string", "minLength": 1}, + "frame_index": {"type": "integer", "minimum": 0}, + "frame_label": {"type": "string"}, + "task": {"type": "string", "minLength": 1}, + "cwd": {"$ref": "#/$defs/path"}, + "argv": {"type": "array", "minItems": 1, "items": {"type": "string"}}, + "stdin": {"type": ["string", "null"]}, + "stdout_path": {"$ref": "#/$defs/path"}, + "stderr_path": {"$ref": "#/$defs/path"}, + "required_inputs": {"$ref": "#/$defs/strings"}, + "expected_outputs": {"$ref": "#/$defs/strings"}, + "dependencies": {"$ref": "#/$defs/strings"}, + "environment": {"type": "object", "additionalProperties": {"type": "string"}} }, "additionalProperties": false }, - "pathArray": { - "type": "array", - "uniqueItems": true, - "items": { - "$ref": "#/$defs/path" - } - }, "scheduler": { "type": "object", - "required": [ - "kind", - "directives" - ], + "required": ["kind", "directives"], "properties": { - "kind": { - "enum": [ - "local", - "slurm" - ] - }, - "directives": { - "type": "object", - "additionalProperties": { - "type": [ - "null", - "boolean", - "number", - "string" - ] - } - } + "kind": {"enum": ["local", "slurm"]}, + "directives": {"type": "object", "additionalProperties": {"type": ["string", "number", "boolean", "null"]}} }, "additionalProperties": false } - } + }, + "additionalProperties": false } diff --git a/schemas/electrace.workflow.v1.schema.json b/schemas/electrace.workflow.v1.schema.json index f831793..fc5750a 100644 --- a/schemas/electrace.workflow.v1.schema.json +++ b/schemas/electrace.workflow.v1.schema.json @@ -2,319 +2,62 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://github.com/Austin243/ElecTrace/schemas/electrace.workflow.v1.schema.json", "title": "ElecTrace external workflow manifest", - "description": "A versioned, shell-free plan for selecting periodic frames and ordering external scientific-tool steps.", - "$comment": "WorkflowManifest additionally enforces unique step ids, executable references, dependency references, acyclic dependencies, and dependencies that name earlier steps. Those cross-record constraints are not expressible in portable JSON Schema draft 2020-12.", + "description": "The stable shape of a shell-free external calculation workflow. Runtime validation enforces selector syntax, safe directives and options, references, and dependency order.", "type": "object", - "required": [ - "schema_version", - "project", - "frames", - "executables", - "scheduler", - "steps" - ], + "required": ["schema_version", "project", "frames", "executables", "scheduler", "steps"], "properties": { - "schema_version": { - "const": "electrace.workflow.v1" - }, - "project": { - "$ref": "#/$defs/project" - }, - "frames": { - "description": "Canonical zero-based frame indices: 'all', an explicit index, a Python-style slice, or a comma-separated mixture.", - "allOf": [ - { - "$ref": "#/$defs/trimmedString" - }, - { - "oneOf": [ - { - "const": "all" - }, - { - "pattern": "^(?:(?:\\+?[0-9]+|-0+)|(?:(?:[+-]?[0-9]+)?\\s*:\\s*(?:[+-]?[0-9]+)?(?:\\s*:\\s*(?:[+-]?[0-9]+)?)?))(?:\\s*,\\s*(?:(?:\\+?[0-9]+|-0+)|(?:(?:[+-]?[0-9]+)?\\s*:\\s*(?:[+-]?[0-9]+)?(?:\\s*:\\s*(?:[+-]?[0-9]+)?)?)))*$" - } - ] - }, - { - "not": { - "pattern": "(?:^|,)\\s*(?:[+-]?[0-9]+)?\\s*:\\s*(?:[+-]?[0-9]+)?\\s*:\\s*[+-]?0+\\s*(?:,|$)" - } - } - ] - }, + "schema_version": {"const": "electrace.workflow.v1"}, + "project": {"$ref": "#/$defs/project"}, + "frames": {"type": "string", "minLength": 1}, "executables": { "type": "object", "minProperties": 1, - "propertyNames": { - "$ref": "#/$defs/identifier" - }, - "additionalProperties": { - "$ref": "#/$defs/executable" - } - }, - "scheduler": { - "$ref": "#/$defs/scheduler" + "propertyNames": {"$ref": "#/$defs/identifier"}, + "additionalProperties": {"$ref": "#/$defs/executable"} }, - "steps": { - "type": "array", - "minItems": 1, - "items": { - "$ref": "#/$defs/step" - } - } + "scheduler": {"$ref": "#/$defs/scheduler"}, + "steps": {"type": "array", "minItems": 1, "items": {"$ref": "#/$defs/step"}} }, "$defs": { - "controlFreeString": { - "type": "string", - "pattern": "^(?![\\s\\S]*[\\u0000-\\u001F\\u007F])[\\s\\S]*$" - }, - "trimmedString": { - "type": "string", - "minLength": 1, - "pattern": "^(?![\\s\\S]*[\\u0000-\\u001F\\u007F])(?!\\s)(?:[\\s\\S]*\\S)?$" - }, - "identifier": { - "type": "string", - "pattern": "^[a-z][a-z0-9_-]*$" - }, - "capability": { - "type": "string", - "pattern": "^[A-Za-z0-9][A-Za-z0-9_.:-]*$" - }, - "basename": { - "allOf": [ - { - "$ref": "#/$defs/trimmedString" - }, - { - "pattern": "^[^/\\\\]+$" - }, - { - "not": { - "enum": [ - ".", - ".." - ] - } - } - ] - }, + "nonempty": {"type": "string", "minLength": 1}, + "identifier": {"type": "string", "pattern": "^[a-z][a-z0-9_-]*$"}, + "basename": {"allOf": [ + {"$ref": "#/$defs/nonempty"}, + {"pattern": "^[^/\\\\]+$"}, + {"not": {"enum": [".", ".."]}} + ]}, "project": { "type": "object", - "required": [ - "root", - "series_kind", - "workspace", - "structure", - "fallback" - ], - "properties": { - "root": { - "$ref": "#/$defs/trimmedString" - }, - "series_kind": { - "enum": [ - "auto", - "neb", - "ordered", - "pressure", - "static" - ] - }, - "workspace": { - "$ref": "#/$defs/trimmedString" - }, - "structure": { - "$ref": "#/$defs/basename" - }, - "fallback": { - "$ref": "#/$defs/basename" - } - }, - "additionalProperties": false - }, - "executable": { - "type": "object", - "required": [ - "path" - ], - "properties": { - "path": { - "$ref": "#/$defs/trimmedString" - }, - "argv_prefix": { - "type": "array", - "items": { - "$ref": "#/$defs/trimmedString" - } - } - }, - "additionalProperties": false - }, - "jsonScalar": { - "oneOf": [ - { - "type": "boolean" - }, - { - "type": "number" - }, - { - "$ref": "#/$defs/controlFreeString" - }, - { - "type": "null" - } - ] - }, - "jsonValue": { - "oneOf": [ - { - "$ref": "#/$defs/jsonScalar" - }, - { - "type": "array", - "items": { - "$ref": "#/$defs/jsonValue" - } - }, - { - "$ref": "#/$defs/jsonObject" - } - ] - }, - "jsonObject": { - "type": "object", - "propertyNames": { - "$ref": "#/$defs/controlFreeString" - }, - "additionalProperties": { - "$ref": "#/$defs/jsonValue" - } - }, - "schedulerDirectiveName": { - "allOf": [ - { - "type": "string", - "pattern": "^[a-z][a-z0-9-]*$" - }, - { - "not": { - "enum": [ - "command", - "job-name", - "script", - "wrap" - ] - } - } - ] - }, - "schedulerDirectiveValue": { - "oneOf": [ - { - "type": "boolean" - }, - { - "type": "number" - }, - { - "type": "string", - "pattern": "^[A-Za-z0-9_./,:@%+=\\[\\]*?{}~-]+$" - }, - { - "type": "null" - } - ] - }, - "scheduler": { - "type": "object", - "required": [ - "kind" - ], + "required": ["root", "series_kind", "workspace", "structure", "fallback"], "properties": { - "kind": { - "enum": [ - "local", - "slurm" - ] - }, - "directives": { - "type": "object", - "propertyNames": { - "$ref": "#/$defs/schedulerDirectiveName" - }, - "additionalProperties": { - "$ref": "#/$defs/schedulerDirectiveValue" - } - } + "root": {"$ref": "#/$defs/nonempty"}, + "series_kind": {"enum": ["auto", "neb", "ordered", "pressure", "static"]}, + "workspace": {"$ref": "#/$defs/nonempty"}, + "structure": {"$ref": "#/$defs/basename"}, + "fallback": {"$ref": "#/$defs/basename"} }, - "allOf": [ - { - "if": { - "required": [ - "kind" - ], - "properties": { - "kind": { - "const": "local" - } - } - }, - "then": { - "properties": { - "directives": { - "maxProperties": 0 - } - } - } - } - ], "additionalProperties": false }, + "executable": {"type": "object", "required": ["path"], "properties": { + "path": {"$ref": "#/$defs/nonempty"}, + "argv_prefix": {"type": "array", "items": {"$ref": "#/$defs/nonempty"}} + }, "additionalProperties": false}, + "scheduler": {"type": "object", "required": ["kind"], "properties": { + "kind": {"enum": ["local", "slurm"]}, + "directives": {"type": "object"} + }, "additionalProperties": false}, "step": { "type": "object", - "required": [ - "id", - "kind", - "executable" - ], + "required": ["id", "kind", "executable"], "properties": { - "id": { - "$ref": "#/$defs/identifier" - }, - "kind": { - "enum": [ - "bader", - "baderkit", - "critic2", - "lobster", - "vasp" - ] - }, - "executable": { - "$ref": "#/$defs/identifier" - }, - "capability": { - "$ref": "#/$defs/capability" - }, - "template_dir": { - "$ref": "#/$defs/trimmedString" - }, - "depends_on": { - "type": "array", - "items": { - "$ref": "#/$defs/identifier" - }, - "uniqueItems": true - }, - "options": { - "$ref": "#/$defs/jsonObject" - }, - "resources": { - "$ref": "#/$defs/jsonObject" - } + "id": {"$ref": "#/$defs/identifier"}, + "kind": {"enum": ["bader", "baderkit", "critic2", "lobster", "vasp"]}, + "executable": {"$ref": "#/$defs/identifier"}, + "capability": {"type": "string", "pattern": "^[A-Za-z0-9][A-Za-z0-9_.:-]*$"}, + "template_dir": {"$ref": "#/$defs/nonempty"}, + "depends_on": {"type": "array", "items": {"$ref": "#/$defs/identifier"}, "uniqueItems": true}, + "options": {"type": "object"}, + "resources": {"type": "object"} }, "additionalProperties": false } diff --git a/skills/electrace/SKILL.md b/skills/electrace/SKILL.md index a1f05ec..28bc31b 100644 --- a/skills/electrace/SKILL.md +++ b/skills/electrace/SKILL.md @@ -1,132 +1,57 @@ --- name: electrace -description: Track Bader, BadELF, ELF/QTAIM, LOBSTER ICOHP/ICOBI, or DDEC evidence across ordered VASP NEB images and structure series with ElecTrace. Use for selecting frames, analyzing existing outputs, inspecting ELFCAR/CHGCAR grids, staging local or Slurm calculations, collecting results, and reporting changes across a path. Do not use as a scientific-convergence or formal-oxidation-state oracle. +description: Analyze Bader, BadELF, QTAIM/critic2, LOBSTER, DDEC, and VASP grid evidence across ordered periodic frames, including VASP NEB paths. Use ElecTrace to select frames, stage local or Slurm calculations, parse outputs, track changes, and generate reports. Do not use it as a convergence or formal-oxidation-state oracle. --- # Use ElecTrace -Use the installed `electrace` CLI as the deterministic execution layer. Do not recreate -its parsers, periodic matching, workflow staging, or report generation in ad hoc code. +Use the `electrace` CLI instead of recreating its parsers, periodic matching, or +workflow staging. Confirm the installed interface with `electrace --help`. If it is +missing, offer installation from `git+https://github.com/Austin243/ElecTrace.git` in +an environment authorized by the user. -## Start safely +Read [references/workflows.md](references/workflows.md) for frame selectors and the +capability map. -1. Work in the user's existing environment or create a project-local virtual environment. -2. Check the installed interface before constructing commands: - - ```bash - electrace --version - electrace --help - electrace doctor --json - ``` - -3. If the command is missing, offer this public-source installation and install only in - an environment the user has authorized: - - ```bash - python -m pip install "git+https://github.com/Austin243/ElecTrace.git" - ``` - -4. Determine whether supported descriptor outputs already exist or must be calculated. -5. Inspect first. Use planning and staging before any external execution. - -Read [references/workflows.md](references/workflows.md) when selecting capabilities, -constructing a workflow manifest, or checking supported scientific boundaries. - -## Analyze existing evidence - -Use this route when the ordered frames already contain supported Bader/BaderKit, -critic2, LOBSTER, Chargemol, or normalized ElecTrace descriptor files. +## Existing outputs ```bash -electrace init electrace.toml --images /absolute/path/to/frames --json -# Review electrace.toml and configure descriptor_source/descriptor_dir. +electrace init electrace.toml --images /absolute/path/to/frames +# Review descriptor_source and descriptor_dir. electrace doctor --config electrace.toml --json electrace analyze electrace.toml --json -electrace inspect electrace-results --json ``` -Treat the generated `analysis.json` as the canonical result. Treat CSV and HTML as -views. Report warnings, missing evidence, ambiguities, frame coverage, and provenance -alongside event summaries. Do not silently switch from strict to permissive analysis. - -Top-level analysis uses all frames discovered by its TOML configuration. Use a workflow -manifest when the user asks for an explicit subset or every nth frame. - -## Prepare missing calculations +Use `analysis.json` as the canonical result. Report warnings, missing frames, and +provenance. Do not silently add `--permissive` to relax frame discovery. -Use this route when VASP or descriptor calculations must be staged across selected -frames. +## Missing calculations ```bash -electrace workflow init electrace-workflow.yaml --images /absolute/path/to/frames --json -# Review executables, templates, dependencies, resources, scheduler, and frames. +electrace workflow init electrace-workflow.yaml --images /absolute/path/to/frames +# Review frames, executables, templates, dependencies, and scheduler settings. electrace workflow validate electrace-workflow.yaml --json electrace workflow stage electrace-workflow.yaml --json -electrace workflow status electrace-workflow.yaml --json -``` - -Validation resolves the complete plan without writing anything. Staging then writes an -isolated, marker-protected workspace but runs no external program. Inspect the resolved -commands, generated job records, and inputs before execution. - -For local execution, add `--yes` only when the user explicitly authorizes all staged -commands: - -```bash -electrace workflow run electrace-workflow.yaml --yes --max-parallel 2 --json -``` - -For Slurm, prepare scripts without submitting, review them, then submit only with -explicit authorization: - -```bash -electrace workflow submit electrace-workflow.yaml --job-name electrace --json -# Review scripts and scheduler directives. -electrace workflow submit electrace-workflow.yaml --job-name electrace --yes --force --json ``` -If authorized submission exits `1`, inspect `error.details.partial_submissions` and -`record_path`, then read `slurm/submissions.json`. Do not blindly resubmit scripts whose -job IDs were already accepted. ElecTrace blocks a new submission while that journal -exists, including with `--force`; remove or archive it only after reconciliation. +Validation and staging do not execute scientific programs. For local execution, use +`electrace workflow run MANIFEST --yes` only after the user authorizes every staged +command. For Slurm, generate scripts without `--yes`, review them, and submit with +`--yes --force` only after authorization; `--force` replaces the reviewed scripts. -After expected outputs exist: +After the declared outputs exist: ```bash electrace workflow status electrace-workflow.yaml --json electrace workflow analyze electrace-workflow.yaml --json ``` -Do not assume filesystem completion proves electronic, projection, grid, or scientific -convergence. Inspect external-tool outputs and surface failures. - -## Plan and inspect without execution - -- Use `electrace plan PLUGIN FRAME --capability CAPABILITY --json` for one command plan. -- Add `--all-frames` to plan across a discovered path. -- Use `electrace volume ELFCAR --json` to summarize a plain VASP scalar grid. -- Use `electrace volume ELFCAR --sample X,Y,Z --json` for periodic fractional sampling. -- Use `electrace workflow validate MANIFEST --json` for a strictly read-only resolved plan. -- Use `electrace workflow status MANIFEST --json` for machine-readable filesystem status. - -With `--json`, consume the single `electrace.cli.v1` envelope. Check `ok` and the process -exit code before reading `data`; on failure, read `error.code` and `error.message`. Treat -`warnings` and `artifacts` as first-class outputs rather than scraping terminal prose. - -Planning never executes a tool. `workflow submit` without `--yes` writes Slurm scripts -but does not call `sbatch`. - -## Preserve scientific and execution boundaries +## Boundaries -- Treat Bader and DDEC values as charge/electron-population evidence, not definitive - formal oxidation states. -- Treat ICOHP/ICOBI and topology events as method-dependent evidence, not unique bond - energies, mechanisms, or identities. -- Never invent or redistribute a `POTCAR`, executable, credential, or private input. +- Never invent a `POTCAR`, executable, credential, or private input. - Never add `--yes`, `--force`, or `--permissive` merely to make a command succeed. -- Never invent a `chargemol` workflow step; ElecTrace can plan Chargemol and ingest - existing DDEC results, but `electrace.workflow.v1` does not stage it. -- Do not combine BaderKit Bader and BadELF atomic records for the same frame unless the - duplicate normalized atom-ID conflict has been resolved deliberately. -- Preserve negative and missing evidence. Stop on schema, provenance, semantic-family, - or ambiguity errors instead of fabricating a correspondence. +- Treat Bader/DDEC values, ELF/QTAIM topology, and ICOHP/ICOBI as method-dependent + evidence, not unique oxidation states, bonds, mechanisms, or energies. +- Treat missing evidence as missing and surface external-tool failures. +- Filesystem completion does not establish electronic, grid, projection, or + scientific convergence. diff --git a/skills/electrace/references/workflows.md b/skills/electrace/references/workflows.md index 208da9d..731cef3 100644 --- a/skills/electrace/references/workflows.md +++ b/skills/electrace/references/workflows.md @@ -1,104 +1,26 @@ # ElecTrace workflow reference -Read this reference only when choosing a descriptor capability, building a workflow, -or interpreting a current support boundary. Confirm exact flags with -`electrace COMMAND --help` because ElecTrace is pre-1.0. +## Frame selectors -## Frame selection +Use `all`, comma-separated indices, or Python-style slices: `0,3,7`, `1:8`, +`0:10:2`, or `0,2:9:2`. Selections preserve expression order and reject duplicates. -The workflow manifest accepts `all` or comma-separated indices and Python-style slices: +## Capabilities -| Selection | Frames | -| --- | --- | -| `all` | Every discovered frame | -| `0,3,7` | Exact indices 0, 3, and 7 | -| `1:8` | Indices 1 through 7 | -| `0:10:2` | Every second frame from 0 through 8 | -| `::3` | Every third discovered frame | -| `0,2:9:2` | Index 0, then 2, 4, 6, and 8 | - -Selections preserve expression order and reject duplicate or empty resolution. - -## Capability map - -| Goal | Plugin or workflow step | Capability | Required scientific output | +| Goal | Plugin/step | Capability | Output | | --- | --- | --- | --- | | Bader populations | `bader` | `bader_charges` | `ACF.dat` | | BaderKit populations | `baderkit` | `bader_charges` | `bader.json` | | BadELF/NNA evidence | `baderkit` | `elf_basins` or `non_nuclear_attractors` | `badelf.json` | -| QTAIM critical points | `critic2` | `qtaim_critical_points` | `critic2.features.json` | -| ELF topology/NNA | `critic2` | `non_nuclear_attractors` | `critic2.features.json` | -| Integrated COHP | `lobster` | `cohp` | `ICOHPLIST.lobster` | -| Integrated COBI | `lobster` | `cobi` | `ICOBILIST.lobster` | -| DDEC charges | `chargemol` planning or existing results only | `ddec_charges` | supported DDEC charge output | - -Use `electrace doctor --config CONFIG --json` to check configured executables and -required inputs across discovered frames. Use `electrace plan PLUGIN FRAME ---capability CAPABILITY --json` to inspect the exact argument vector, standard input, -required inputs, expected outputs, and working directory without execution. - -## Existing-output route - -```bash -electrace init electrace.toml --images /absolute/path/to/frames --json -# Set series_kind, descriptor_source, and descriptor_dir as required. -electrace doctor --config electrace.toml --json -electrace analyze electrace.toml --json -electrace inspect electrace-results --json -``` - -Use this route for existing Chargemol/DDEC results. Do not add `chargemol` to a -workflow manifest. - -## Calculation route - -```bash -electrace workflow init electrace-workflow.yaml --images /absolute/path/to/frames --json -# Configure frames, executables, templates, steps, dependencies, and scheduler. -electrace workflow validate electrace-workflow.yaml --json -electrace workflow stage electrace-workflow.yaml --json -electrace workflow status electrace-workflow.yaml --json -``` - -For local work, run only after authorization: - -```bash -electrace workflow run electrace-workflow.yaml --yes --max-parallel 2 --json -``` - -For Slurm, prepare first and submit second: - -```bash -electrace workflow submit electrace-workflow.yaml --job-name electrace --json -electrace workflow submit electrace-workflow.yaml --job-name electrace --yes --force --json -``` - -On exit `1`, use `error.details.partial_submissions` and `record_path`, and inspect the -`slurm/submissions.json` journal before retrying. Never duplicate an already accepted -submission. A prior journal blocks another submission even with `--force`; remove or -archive it only after reconciling accepted IDs. - -Then collect and analyze: - -```bash -electrace workflow status electrace-workflow.yaml --json -electrace workflow analyze electrace-workflow.yaml --json -``` - -Every `--json` response uses the versioned `electrace.cli.v1` envelope. Check `ok`, -`error`, `warnings`, and `artifacts`; do not scrape human-formatted output. Workflow -validation resolves the plan without creating the workspace or any generated file. - -## Current boundaries - -- External VASP, Bader/BaderKit, critic2, LOBSTER, and Chargemol installations remain - user-supplied. ElecTrace does not select convergence settings or supply `POTCAR`. -- BaderKit `bader.json` and `badelf.json` both normalize atomic records to - `baderkit:atom:*`; do not attach both for one frame without an explicit resolution. -- critic2 support tracks critical-point topology and NNAs, not integrated ELF basin - populations. -- Native spin-resolved LOBSTER list ingestion is unsupported. -- `workflow status` reports staged files and expected outputs, not scheduler state or - scientific convergence. -- Bader/DDEC populations are not definitive formal oxidation states, and ICOHP is not - a transferable bond energy. +| QTAIM/ELF critical points | `critic2` | `qtaim_critical_points` or `non_nuclear_attractors` | `critic2.features.json` | +| Integrated COHP/COBI | `lobster` | `cohp` or `cobi` | `ICOHPLIST.lobster` or `ICOBILIST.lobster` | +| DDEC charges | existing output or planning only | `ddec_charges` | DDEC charge output | + +Use `electrace plan PLUGIN FRAME --capability CAPABILITY --json` to inspect a command +without execution. Add `--all-frames` for a path-wide plan. Use `electrace volume +ELFCAR --json` to inspect a VASP scalar grid. + +Current boundaries: external programs and pseudopotentials are user-supplied; +critic2 integration reports critical-point topology rather than integrated ELF basin +populations; spin-resolved native LOBSTER lists are unsupported; and workflow status +reports files rather than scheduler or scientific convergence. diff --git a/src/electrace/__init__.py b/src/electrace/__init__.py index 7d300c2..5474427 100644 --- a/src/electrace/__init__.py +++ b/src/electrace/__init__.py @@ -6,6 +6,6 @@ from __future__ import annotations -__version__ = "0.1.0a4" +__version__ = "0.1.0a5" __all__ = ["__version__"] diff --git a/src/electrace/benchmark.py b/src/electrace/benchmark.py deleted file mode 100644 index e008253..0000000 --- a/src/electrace/benchmark.py +++ /dev/null @@ -1,599 +0,0 @@ -"""Deterministic synthetic regression benchmark for ElecTrace correspondence. - -This suite exercises the production matchers and transition event detector on -small, labeled constructions. It is a software/scientific-assumption regression -benchmark, not validation against independent electronic-structure calculations. -""" - -from __future__ import annotations - -import math -from collections import Counter -from collections.abc import Mapping, Sequence -from dataclasses import dataclass, field -from pathlib import Path -from typing import Any - -from . import __version__ -from .events import EventDetectionConfig, detect_transition_events -from .export import write_json -from .matching import AtomMatchConfig, FeatureMatchConfig, match_atoms, match_features -from .models import Atom, Event, Feature, Frame, Identifier, MatchResult, Matrix3 - -BENCHMARK_SCHEMA_VERSION = "electrace.benchmark.v0.1" -BENCHMARK_SUITE_VERSION = "0.1.0" - -_CELL: Matrix3 = ((10.0, 0.0, 0.0), (0.0, 10.0, 0.0), (0.0, 0.0, 10.0)) -_DEFORMED_CELL: Matrix3 = ((12.0, 0.0, 0.0), (0.0, 8.0, 0.0), (0.0, 0.0, 10.0)) - - -@dataclass(frozen=True, slots=True) -class EventLabel: - """Identity-only event label used for exact synthetic scoring.""" - - type: str - source_ids: tuple[str, ...] = () - target_ids: tuple[str, ...] = () - - def key(self) -> tuple[str, tuple[str, ...], tuple[str, ...]]: - return (self.type, tuple(sorted(self.source_ids)), tuple(sorted(self.target_ids))) - - def to_dict(self) -> dict[str, Any]: - event_type, source_ids, target_ids = self.key() - return { - "type": event_type, - "source_ids": list(source_ids), - "target_ids": list(target_ids), - } - - -@dataclass(frozen=True, slots=True) -class SyntheticCase: - """One controlled two-frame perturbation and its exact expected outcome.""" - - id: str - category: str - description: str - perturbation: str - previous: Frame - current: Frame - events: tuple[EventLabel, ...] = () - atom_mapping: tuple[tuple[int, int], ...] = () - feature_mapping: tuple[tuple[str, str], ...] = () - atom_ambiguous: bool = False - feature_ambiguous: bool = False - atom_config: AtomMatchConfig = field(default_factory=AtomMatchConfig) - feature_config: FeatureMatchConfig = field(default_factory=FeatureMatchConfig) - event_config: EventDetectionConfig = field(default_factory=EventDetectionConfig) - - -def _feature( - identifier: str, - position: tuple[float, float, float], - *, - kind: str = "elf_basin", - charge: float | None = None, - volume: float | None = None, - attributes: Mapping[str, Any] | None = None, -) -> Feature: - return Feature( - identifier, - kind, - position, - charge=charge, - volume=volume, - attributes=dict(attributes or {}), - ) - - -def _frame( - index: int, - *, - atoms: Sequence[Atom] = (), - features: Sequence[Feature] = (), - cell: Matrix3 = _CELL, -) -> Frame: - return Frame( - index=index, - label=str(index), - cell=cell, - atoms=tuple(atoms), - features=tuple(features), - ) - - -def _synthetic_cases() -> tuple[SyntheticCase, ...]: - descriptor_config = FeatureMatchConfig(descriptor_scales={"icohp": 1.0, "icobi": 1.0}) - property_config = FeatureMatchConfig(descriptor_scales={"icohp": 2.0}) - hp = {"descriptor_family": "icohp", "icohp": -1.0} - bi = {"descriptor_family": "icobi", "icobi": 0.5} - return ( - SyntheticCase( - id="atom-periodic-wrap", - category="periodic_boundary", - description="An atom crossing a periodic face retains its lineage.", - perturbation="fractional x coordinate 0.99 to 0.01", - previous=_frame(0, atoms=(Atom(7, "C", (0.99, 0.5, 0.5)),)), - current=_frame(1, atoms=(Atom(12, "C", (0.01, 0.5, 0.5)),)), - atom_mapping=((7, 12),), - ), - SyntheticCase( - id="feature-periodic-wrap", - category="periodic_boundary", - description="A feature crossing a periodic face retains its lineage.", - perturbation="fractional x coordinate 0.99 to 0.01", - previous=_frame(0, features=(_feature("p", (0.99, 0.5, 0.5)),)), - current=_frame(1, features=(_feature("q", (0.01, 0.5, 0.5)),)), - feature_mapping=(("p", "q"),), - ), - SyntheticCase( - id="feature-position-jitter", - category="coordinate_noise", - description="Small coordinate noise does not create a false event.", - perturbation="fractional displacement (0.002, -0.001, 0.001)", - previous=_frame(0, features=(_feature("p", (0.4, 0.4, 0.4)),)), - current=_frame(1, features=(_feature("q", (0.402, 0.399, 0.401)),)), - feature_mapping=(("p", "q"),), - ), - SyntheticCase( - id="variable-cell-stability", - category="cell_deformation", - description="A stable fractional feature survives an anisotropic cell change.", - perturbation="cell axes change from (10, 10, 10) to (12, 8, 10) angstrom", - previous=_frame(0, features=(_feature("p", (0.3, 0.4, 0.5)),)), - current=_frame( - 1, - features=(_feature("q", (0.3, 0.4, 0.5)),), - cell=_DEFORMED_CELL, - ), - feature_mapping=(("p", "q"),), - ), - SyntheticCase( - id="descriptor-order-reversal", - category="input_order", - description="Semantic descriptor families prevent order-dependent lineage swaps.", - perturbation="target feature array is reversed", - previous=_frame( - 0, - features=( - _feature("p-hp", (0.5, 0.5, 0.5), kind="bond_descriptor", attributes=hp), - _feature("p-bi", (0.5, 0.5, 0.5), kind="bond_descriptor", attributes=bi), - ), - ), - current=_frame( - 1, - features=( - _feature("q-bi", (0.5, 0.5, 0.5), kind="bond_descriptor", attributes=bi), - _feature("q-hp", (0.5, 0.5, 0.5), kind="bond_descriptor", attributes=hp), - ), - ), - feature_mapping=(("p-bi", "q-bi"), ("p-hp", "q-hp")), - feature_config=descriptor_config, - ), - SyntheticCase( - id="feature-birth-and-death", - category="topology_event", - description="Incompatible disappearing and appearing kinds produce death and birth labels.", - perturbation="one stable feature plus one removed and one added feature", - previous=_frame( - 0, - features=( - _feature("p-stable", (0.5, 0.5, 0.5)), - _feature("p-dead", (0.2, 0.2, 0.2), kind="bond_critical_point"), - ), - ), - current=_frame( - 1, - features=( - _feature("q-stable", (0.501, 0.5, 0.5)), - _feature("q-born", (0.2, 0.2, 0.2), kind="non_nuclear_attractor"), - ), - ), - events=( - EventLabel("birth", target_ids=("q-born",)), - EventLabel("death", source_ids=("p-dead",)), - ), - feature_mapping=(("p-stable", "q-stable"),), - ), - SyntheticCase( - id="descriptor-property-change", - category="property_event", - description="A threshold-crossing ICOHP change produces a property-change label.", - perturbation="ICOHP changes from -0.2 to -1.2 eV", - previous=_frame( - 0, - features=( - _feature( - "p", - (0.5, 0.5, 0.5), - kind="bond_descriptor", - attributes={"descriptor_family": "icohp", "icohp": -0.2}, - ), - ), - ), - current=_frame( - 1, - features=( - _feature( - "q", - (0.501, 0.5, 0.5), - kind="bond_descriptor", - attributes={"descriptor_family": "icohp", "icohp": -1.2}, - ), - ), - ), - events=(EventLabel("property_change", ("p",), ("q",)),), - feature_mapping=(("p", "q"),), - feature_config=property_config, - ), - SyntheticCase( - id="conserving-split", - category="topology_event", - description="A volume-conserving one-to-two transition produces a split.", - perturbation="volume 10 becomes volumes 4 and 6", - previous=_frame( - 0, - features=(_feature("p", (0.5, 0.5, 0.5), volume=10.0),), - ), - current=_frame( - 1, - features=( - _feature("q1", (0.48, 0.5, 0.5), volume=4.0), - _feature("q2", (0.54, 0.5, 0.5), volume=6.0), - ), - ), - events=(EventLabel("split", ("p",), ("q1", "q2")),), - ), - SyntheticCase( - id="conserving-merge", - category="topology_event", - description="A volume-conserving two-to-one transition produces a merge.", - perturbation="volumes 4 and 6 become volume 10", - previous=_frame( - 0, - features=( - _feature("p1", (0.48, 0.5, 0.5), volume=4.0), - _feature("p2", (0.54, 0.5, 0.5), volume=6.0), - ), - ), - current=_frame( - 1, - features=(_feature("q", (0.5, 0.5, 0.5), volume=10.0),), - ), - events=(EventLabel("merge", ("p1", "p2"), ("q",)),), - ), - SyntheticCase( - id="proximity-without-conservation", - category="negative_control", - description="Nearby unquantified children are not promoted to a split.", - perturbation="one unquantified feature becomes two nearby unquantified features", - previous=_frame(0, features=(_feature("p", (0.5, 0.5, 0.5)),)), - current=_frame( - 1, - features=( - _feature("q1", (0.48, 0.5, 0.5)), - _feature("q2", (0.54, 0.5, 0.5)), - ), - ), - events=(EventLabel("birth", target_ids=("q2",)),), - feature_mapping=(("p", "q1"),), - ), - SyntheticCase( - id="failed-conservation", - category="negative_control", - description="A one-to-two candidate that violates volume conservation is not a split.", - perturbation="volume 10 becomes volumes 9 and 9", - previous=_frame( - 0, - features=(_feature("p", (0.5, 0.5, 0.5), volume=10.0),), - ), - current=_frame( - 1, - features=( - _feature("q1", (0.48, 0.5, 0.5), volume=9.0), - _feature("q2", (0.54, 0.5, 0.5), volume=9.0), - ), - ), - events=(EventLabel("birth", target_ids=("q2",)),), - feature_mapping=(("p", "q1"),), - ), - SyntheticCase( - id="exact-assignment-tie", - category="ambiguity", - description="An exact target tie is surfaced and affected event claims are suppressed.", - perturbation="two targets have identical positions and missing descriptors", - previous=_frame(0, features=(_feature("p", (0.5, 0.5, 0.5)),)), - current=_frame( - 1, - features=( - _feature("q1", (0.5, 0.5, 0.5)), - _feature("q2", (0.5, 0.5, 0.5)), - ), - ), - feature_ambiguous=True, - ), - SyntheticCase( - id="near-assignment-tie", - category="ambiguity", - description="A near-optimal target tie within tolerance is surfaced.", - perturbation="alternative target is displaced by 0.001 angstrom", - previous=_frame(0, features=(_feature("p", (0.5, 0.5, 0.5)),)), - current=_frame( - 1, - features=( - _feature("q1", (0.5, 0.5, 0.5)), - _feature("q2", (0.5001, 0.5, 0.5)), - ), - ), - feature_ambiguous=True, - ), - SyntheticCase( - id="incompatible-charge-convention", - category="semantic_gate", - description="Explicitly incompatible charge conventions cannot share a lineage.", - perturbation="charge convention changes from convention-a to convention-b", - previous=_frame( - 0, - features=( - _feature( - "p", - (0.5, 0.5, 0.5), - charge=0.5, - attributes={"charge_convention": "convention-a"}, - ), - ), - ), - current=_frame( - 1, - features=( - _feature( - "q", - (0.5, 0.5, 0.5), - charge=0.5, - attributes={"charge_convention": "convention-b"}, - ), - ), - ), - events=( - EventLabel("birth", target_ids=("q",)), - EventLabel("death", source_ids=("p",)), - ), - ), - SyntheticCase( - id="hyperedge-resolves-backbone-tie", - category="ambiguity_resolution", - description="A conserving split resolves an arbitrary one-to-one backbone tie.", - perturbation="symmetric children each carry half the parent volume", - previous=_frame( - 0, - features=(_feature("p", (0.5, 0.5, 0.5), volume=10.0),), - ), - current=_frame( - 1, - features=( - _feature("q1", (0.49, 0.5, 0.5), volume=5.0), - _feature("q2", (0.51, 0.5, 0.5), volume=5.0), - ), - ), - events=(EventLabel("split", ("p",), ("q1", "q2")),), - ), - ) - - -def _event_label(event: Event) -> EventLabel: - return EventLabel( - event.event_type.value, - tuple(str(value) for value in event.source_ids), - tuple(str(value) for value in event.target_ids), - ) - - -def _event_scores( - expected: Sequence[EventLabel], observed: Sequence[Event] -) -> tuple[int, int, int]: - expected_counts = Counter(label.key() for label in expected) - observed_counts = Counter(_event_label(event).key() for event in observed) - true_positives = sum((expected_counts & observed_counts).values()) - false_positives = sum((observed_counts - expected_counts).values()) - false_negatives = sum((expected_counts - observed_counts).values()) - return true_positives, false_positives, false_negatives - - -def _mapping_pairs( - mapping: Mapping[Identifier, Identifier], -) -> tuple[tuple[Identifier, Identifier], ...]: - return tuple(sorted(mapping.items(), key=lambda item: (str(item[0]), str(item[1])))) - - -def _mapping_records( - pairs: Sequence[tuple[Identifier, Identifier]], -) -> list[dict[str, Identifier]]: - return [ - {"source_id": source, "target_id": target} - for source, target in sorted(pairs, key=lambda item: (str(item[0]), str(item[1]))) - ] - - -def _relations(result: MatchResult) -> list[dict[str, Any]]: - return [ - { - "relation": edge.relation.value, - "source_ids": sorted(str(value) for value in edge.source_ids), - "target_ids": sorted(str(value) for value in edge.target_ids), - "confidence": edge.confidence, - "ambiguous": edge.ambiguous, - } - for edge in sorted( - result.edges, - key=lambda item: ( - item.relation.value, - tuple(str(value) for value in item.source_ids), - tuple(str(value) for value in item.target_ids), - ), - ) - ] - - -def _rate(numerator: int | float, denominator: int | float, *, empty: float = 1.0) -> float: - return numerator / denominator if denominator else empty - - -def _evaluate_case(case: SyntheticCase) -> dict[str, Any]: - atom_result = match_atoms(case.previous, case.current, case.atom_config) - feature_result = match_features( - case.previous, - case.current, - atom_result, - case.feature_config, - ) - events = detect_transition_events( - case.previous, - case.current, - feature_result, - case.event_config, - ) - event_tp, event_fp, event_fn = _event_scores(case.events, events) - observed_atom_mapping = _mapping_pairs(atom_result.mapping) - observed_feature_mapping = _mapping_pairs(feature_result.mapping) - expected_atom_mapping = _mapping_pairs(dict(case.atom_mapping)) - expected_feature_mapping = _mapping_pairs(dict(case.feature_mapping)) - observed_atom_ambiguous = bool(atom_result.diagnostics.get("ambiguous")) - observed_feature_ambiguous = bool(feature_result.diagnostics.get("ambiguous")) - atom_mapping_correct = observed_atom_mapping == expected_atom_mapping - feature_mapping_correct = observed_feature_mapping == expected_feature_mapping - atom_ambiguity_correct = observed_atom_ambiguous is case.atom_ambiguous - feature_ambiguity_correct = observed_feature_ambiguous is case.feature_ambiguous - failures: list[str] = [] - if event_fp or event_fn: - failures.append("event labels differ") - if not atom_mapping_correct: - failures.append("atom mapping differs") - if not feature_mapping_correct: - failures.append("feature mapping differs") - if not atom_ambiguity_correct: - failures.append("atom ambiguity decision differs") - if not feature_ambiguity_correct: - failures.append("feature ambiguity decision differs") - return { - "id": case.id, - "category": case.category, - "description": case.description, - "perturbation": case.perturbation, - "passed": not failures, - "failures": failures, - "expected": { - "events": [label.to_dict() for label in sorted(case.events, key=EventLabel.key)], - "atom_mapping": _mapping_records(expected_atom_mapping), - "feature_mapping": _mapping_records(expected_feature_mapping), - "atom_ambiguous": case.atom_ambiguous, - "feature_ambiguous": case.feature_ambiguous, - }, - "observed": { - "events": [ - {**_event_label(event).to_dict(), "confidence": event.confidence} - for event in events - ], - "atom_mapping": _mapping_records(observed_atom_mapping), - "feature_mapping": _mapping_records(observed_feature_mapping), - "atom_ambiguous": observed_atom_ambiguous, - "feature_ambiguous": observed_feature_ambiguous, - "atom_ambiguity_count": len(atom_result.diagnostics.get("ambiguities", [])), - "feature_ambiguity_count": len(feature_result.diagnostics.get("ambiguities", [])), - "relations": _relations(feature_result), - "hyperedge_rejections": feature_result.diagnostics.get("hyperedge_rejections", []), - }, - "metrics": { - "event_true_positives": event_tp, - "event_false_positives": event_fp, - "event_false_negatives": event_fn, - "mapping_assertions_correct": int(atom_mapping_correct) + int(feature_mapping_correct), - "mapping_assertions_total": 2, - "ambiguity_assertions_correct": int(atom_ambiguity_correct) - + int(feature_ambiguity_correct), - "ambiguity_assertions_total": 2, - }, - } - - -def run_synthetic_benchmark() -> dict[str, Any]: - """Run the versioned offline suite and return a deterministic JSON-ready report.""" - - cases = [_evaluate_case(case) for case in _synthetic_cases()] - event_tp = sum(int(case["metrics"]["event_true_positives"]) for case in cases) - event_fp = sum(int(case["metrics"]["event_false_positives"]) for case in cases) - event_fn = sum(int(case["metrics"]["event_false_negatives"]) for case in cases) - mapping_correct = sum(int(case["metrics"]["mapping_assertions_correct"]) for case in cases) - mapping_total = sum(int(case["metrics"]["mapping_assertions_total"]) for case in cases) - ambiguity_correct = sum(int(case["metrics"]["ambiguity_assertions_correct"]) for case in cases) - ambiguity_total = sum(int(case["metrics"]["ambiguity_assertions_total"]) for case in cases) - passed = sum(bool(case["passed"]) for case in cases) - precision = _rate(event_tp, event_tp + event_fp) - recall = _rate(event_tp, event_tp + event_fn) - f1 = _rate(2 * precision * recall, precision + recall) - return { - "schema_version": BENCHMARK_SCHEMA_VERSION, - "software_version": __version__, - "suite": { - "name": "electrace-synthetic-lineage", - "version": BENCHMARK_SUITE_VERSION, - "synthetic": True, - "scientific_validation": False, - "description": ( - "Deterministic regression cases for periodic matching, semantic gates, " - "conservation-backed hyperedges, event labels, and ambiguity handling." - ), - "limitations": [ - "Cases are constructed inputs, not independent electronic-structure results.", - "Perfect synthetic scores do not establish accuracy on a physical domain.", - "Confidence values are deterministic evidence scores, not calibrated probabilities.", - ], - }, - "summary": { - "cases": len(cases), - "passed": passed, - "failed": len(cases) - passed, - "event_true_positives": event_tp, - "event_false_positives": event_fp, - "event_false_negatives": event_fn, - "event_precision": precision, - "event_recall": recall, - "event_f1": f1, - "mapping_assertions_correct": mapping_correct, - "mapping_assertions_total": mapping_total, - "mapping_accuracy": _rate(mapping_correct, mapping_total), - "ambiguity_assertions_correct": ambiguity_correct, - "ambiguity_assertions_total": ambiguity_total, - "ambiguity_accuracy": _rate(ambiguity_correct, ambiguity_total), - }, - "cases": cases, - } - - -def write_synthetic_benchmark( - path: str | Path = "electrace-benchmark.json", *, overwrite: bool = False -) -> Path: - """Run and atomically write the synthetic benchmark report.""" - - output = Path(path) - if (output.exists() or output.is_symlink()) and not overwrite: - raise FileExistsError( - f"benchmark output already exists: {output}; use --force to replace it" - ) - return write_json(run_synthetic_benchmark(), output) - - -def benchmark_passed(report: Mapping[str, Any]) -> bool: - """Return whether a benchmark report has no labeled-case failures.""" - - summary = report.get("summary") - return ( - isinstance(summary, Mapping) - and summary.get("failed") == 0 - and summary.get("cases") == summary.get("passed") - and isinstance(summary.get("cases"), int) - and int(summary["cases"]) > 0 - and all( - math.isfinite(float(summary.get(key, math.nan))) - for key in ("event_precision", "event_recall", "event_f1") - ) - ) diff --git a/src/electrace/cli.py b/src/electrace/cli.py index 722241d..3ef848e 100644 --- a/src/electrace/cli.py +++ b/src/electrace/cli.py @@ -7,21 +7,17 @@ import sys from collections.abc import Sequence from pathlib import Path -from typing import Any, Never, Protocol, cast +from typing import Any, Never, cast import typer -import typer.core as typer_core from rich.console import Console from rich.markup import escape from rich.table import Table -from typer.core import TyperGroup from electrace import __version__ -from electrace.benchmark import benchmark_passed, run_synthetic_benchmark -from electrace.cli_output import failure, normalize_error_code, serialize, success -from electrace.config import ElecTraceConfig, load_config, write_default_config +from electrace.cli_output import failure, serialize, success +from electrace.config import load_config, write_default_config from electrace.demo import generate_demo -from electrace.export import ExportDependencyError, write_json from electrace.io import load_sequence from electrace.pipeline import load_and_analyze, run_project from electrace.plugins import ( @@ -48,97 +44,8 @@ ) from electrace.workflow.staging import stage_workflow -_CLICK_RUNTIME = cast(Any, typer_core.__dict__.get("_click") or typer_core.__dict__["click"]) -_CLICK_EXCEPTION = cast(type[BaseException], _CLICK_RUNTIME.exceptions.ClickException) -_CLICK_EXIT = cast(type[BaseException], _CLICK_RUNTIME.exceptions.Exit) - - -class _ClickExceptionLike(Protocol): - exit_code: int - - def format_message(self) -> str: ... - - -def _command_from_args(args: Sequence[str]) -> str: - """Return a stable command identifier from a raw CLI argument vector.""" - - tokens = [token for token in args if not token.startswith("-")] - if not tokens: - return "electrace" - if tokens[0] == "workflow": - return f"workflow.{tokens[1]}" if len(tokens) > 1 else "workflow" - return tokens[0] - - -def _click_error_code(exc: BaseException) -> str: - if type(exc).__name__ == "BadParameter" and "does not exist" in str(exc).casefold(): - return "not_found" - return { - "NoSuchOption": "invalid_option", - "MissingParameter": "missing_parameter", - "BadParameter": "invalid_input", - "UsageError": "invalid_usage", - }.get(type(exc).__name__, "invalid_input") - - -class _JsonAwareGroup(TyperGroup): - """Render parse-time Click failures as JSON whenever ``--json`` is present.""" - - def main( - self, - args: Sequence[str] | None = None, - prog_name: str | None = None, - complete_var: str | None = None, - standalone_mode: bool = True, - windows_expand_args: bool = True, - **extra: Any, - ) -> Any: - arguments = list(sys.argv[1:] if args is None else args) - json_requested = any( - argument == "--json" or argument.startswith("--json=") for argument in arguments - ) - if not json_requested: - return super().main( - args=arguments, - prog_name=prog_name, - complete_var=complete_var, - standalone_mode=standalone_mode, - windows_expand_args=windows_expand_args, - **extra, - ) - try: - result = super().main( - args=arguments, - prog_name=prog_name, - complete_var=complete_var, - standalone_mode=False, - windows_expand_args=windows_expand_args, - **extra, - ) - except _CLICK_EXCEPTION as raw_exc: - exc = cast(_ClickExceptionLike, raw_exc) - sys.stdout.write( - serialize( - failure( - _command_from_args(arguments), - _click_error_code(raw_exc), - exc.format_message(), - ) - ) - ) - if standalone_mode: - raise SystemExit(exc.exit_code) from raw_exc - raise _CLICK_EXIT(exc.exit_code) from raw_exc - if isinstance(result, int) and result != 0: - if standalone_mode: - raise SystemExit(result) - raise _CLICK_EXIT(result) - return result - - app = typer.Typer( name="electrace", - cls=_JsonAwareGroup, help="Track electronic features across periodic VASP paths.", no_args_is_help=True, rich_markup_mode="markdown", @@ -187,28 +94,22 @@ def main( def _emit_json( command: str, - data: object | None = None, + data: object, *, warnings: Sequence[str] = (), - artifacts: Sequence[object] = (), ) -> None: - """Write one versioned JSON success envelope without terminal styling.""" - - sys.stdout.write( - serialize( - success( - command, - data, - warnings=warnings, - artifacts=artifacts, - ) - ) - ) + sys.stdout.write(serialize(success(command, data, warnings=warnings))) -def _exception_code(exc: BaseException) -> str: - """Map handled CLI exceptions to stable machine-readable error codes.""" +def _finish(command: str, data: object, as_json: bool, *messages: str) -> None: + if as_json: + _emit_json(command, data) + else: + for message in messages: + console.print(message) + +def _exception_code(exc: BaseException) -> str: if isinstance(exc, PermissionError): message = str(exc).casefold() return ( @@ -216,13 +117,19 @@ def _exception_code(exc: BaseException) -> str: if "confirmed=true" in message or "--yes" in message else "permission_denied" ) - if isinstance(exc, ExportDependencyError): - return "dependency_unavailable" if isinstance(exc, KeyError): return "unknown_resource" if isinstance(exc, RuntimeError): return "execution_error" - return normalize_error_code(exc) + if isinstance(exc, FileNotFoundError): + return "not_found" + if isinstance(exc, FileExistsError): + return "already_exists" + if isinstance(exc, (IsADirectoryError, NotADirectoryError)): + return "invalid_path" + if isinstance(exc, OSError): + return "io_error" + return "invalid_input" def _fail( @@ -233,20 +140,9 @@ def _fail( command: str = "electrace", error_code: str = "invalid_input", details: object | None = None, - artifacts: Sequence[object] = (), ) -> Never: if as_json: - sys.stdout.write( - serialize( - failure( - command, - error_code, - message, - details=details, - artifacts=artifacts, - ) - ) - ) + sys.stdout.write(serialize(failure(command, error_code, message, details=details))) raise typer.Exit(code=code) error_console.print(f"[bold red]Error:[/bold red] {escape(message)}") raise typer.Exit(code=code) @@ -269,63 +165,40 @@ def _fail_exception( ) -def _load_optional_config(path: Path | None) -> ElecTraceConfig | None: - if path is None: - return None - return load_config(path) - - -def _artifact(path: Path, role: str) -> dict[str, str]: - return {"path": str(path.expanduser().resolve()), "role": role} - - def _analysis_summary(analysis: dict[str, Any], source: Path) -> dict[str, Any]: - project = analysis.get("project", {}) - if not isinstance(project, dict): - raise ValueError("analysis project must be an object") - collections: dict[str, list[Any]] = {} - for name in ("frames", "tracks", "events", "warnings"): - value = analysis.get(name, []) - if not isinstance(value, list): - raise ValueError(f"analysis {name} must be an array") - collections[name] = value return { "source": str(source.expanduser().resolve()), - "project": project, + "project": analysis.get("project", {}), "counts": { - "frames": len(collections["frames"]), - "feature_observations": len(collections["tracks"]), - "events": len(collections["events"]), - "warnings": len(collections["warnings"]), + "frames": len(analysis.get("frames", [])), + "feature_observations": len(analysis.get("tracks", [])), + "events": len(analysis.get("events", [])), + "warnings": len(analysis.get("warnings", [])), }, } def _analysis_warnings(analysis: dict[str, Any]) -> tuple[str, ...]: - value = analysis.get("warnings", []) - if not isinstance(value, list): - raise ValueError("analysis warnings must be an array") - return tuple(str(item) for item in value) + return tuple(str(item) for item in analysis.get("warnings", [])) + + +def _doctor_row(report: DoctorReport) -> tuple[str, str]: + status = ( + "[green]ready[/green]" + if report.ready + else "[yellow]installed; inputs missing[/yellow]" + if report.installed + else "[dim]optional; not found[/dim]" + ) + return status, escape("; ".join(report.messages) or str(report.executable_path)) @app.command() def init( - config: Path = typer.Argument( - Path("electrace.toml"), - help="Configuration file to create.", - ), - images: Path = typer.Option( - ..., - "--images", - "-i", - exists=True, - file_okay=False, - dir_okay=True, - resolve_path=True, - help="Ordered VASP image or static-series root.", - ), - force: bool = typer.Option(False, "--force", help="Replace an existing configuration."), - as_json: bool = typer.Option(False, "--json", help="Emit a versioned JSON envelope."), + config: Path = typer.Argument(Path("electrace.toml")), + images: Path = typer.Option(..., "--images", "-i"), + force: bool = typer.Option(False, "--force"), + as_json: bool = typer.Option(False, "--json"), ) -> None: """Create a reviewed starter configuration for an existing VASP path.""" @@ -333,34 +206,24 @@ def init( output = write_default_config(config, input_root=images, overwrite=force) except (OSError, TypeError, ValueError) as exc: _fail_exception(exc, command="init", as_json=as_json) - if as_json: - _emit_json( - "init", - {"configuration": str(output.resolve()), "images": str(images.resolve())}, - artifacts=(_artifact(output, "configuration"),), - ) - return - console.print(f"[green]Created[/green] {output}") - console.print("Review matching thresholds and plugin settings before analysis.") + _finish( + "init", + {"configuration": str(output.resolve()), "images": str(images.resolve())}, + as_json, + f"[green]Created[/green] {output}", + "Review matching thresholds and plugin settings before analysis.", + ) @app.command() def doctor( - config: Path | None = typer.Option( - None, - "--config", - "-c", - exists=True, - dir_okay=False, - resolve_path=True, - help="Optional ElecTrace TOML configuration.", - ), - as_json: bool = typer.Option(False, "--json", help="Emit machine-readable JSON."), + config: Path | None = typer.Option(None, "--config", "-c"), + as_json: bool = typer.Option(False, "--json"), ) -> None: """Check the Python runtime, plugins, and configured frame inputs.""" try: - loaded = _load_optional_config(config) + loaded = load_config(config) if config is not None else None except (OSError, TypeError, ValueError) as exc: _fail_exception(exc, command="doctor", as_json=as_json) plugin_config = loaded.plugins if loaded is not None else None @@ -412,34 +275,10 @@ def doctor( elif configured_root: table.add_row("Configured path", "[green]readable[/green]", str(configured_root)) for report in reports: - status = ( - "[green]ready[/green]" - if report.ready - else ( - "[yellow]installed; inputs missing[/yellow]" - if report.installed - else "[dim]optional; not found[/dim]" - ) - ) - details = "; ".join(report.messages) or str(report.executable_path) - table.add_row(report.plugin, status, escape(details)) + table.add_row(report.plugin, *_doctor_row(report)) for frame_report in frame_reports: for report in frame_report.plugins: - status = ( - "[green]ready[/green]" - if report.ready - else ( - "[yellow]installed; inputs missing[/yellow]" - if report.installed - else "[dim]optional; not found[/dim]" - ) - ) - details = "; ".join(report.messages) or str(report.executable_path) - table.add_row( - f"{report.plugin} [{frame_report.label}]", - status, - escape(details), - ) + table.add_row(f"{report.plugin} [{frame_report.label}]", *_doctor_row(report)) console.print(table) if not reports and not any(report.plugins for report in frame_reports): console.print("No plugins are enabled in the selected configuration.") @@ -447,29 +286,19 @@ def doctor( @app.command() def demo( - output: Path = typer.Option( - Path(".electrace-demo"), - "--output", - "-o", - help="Directory for the synthetic demonstration.", - ), - force: bool = typer.Option(False, "--force", help="Replace known demo artifacts."), - analyze: bool = typer.Option( - True, - "--analyze/--no-analyze", - help="Run the generated demonstration through the full pipeline.", - ), - as_json: bool = typer.Option(False, "--json", help="Emit a versioned JSON envelope."), + output: Path = typer.Option(Path(".electrace-demo"), "--output", "-o"), + force: bool = typer.Option(False, "--force"), + analyze: bool = typer.Option(True, "--analyze/--no-analyze"), + as_json: bool = typer.Option(False, "--json"), ) -> None: """Generate a deterministic synthetic path and optionally analyze it.""" try: config_path = generate_demo(output, overwrite=force) artifacts = run_project(config_path, overwrite=force) if analyze else None - except (OSError, TypeError, ValueError, ExportDependencyError) as exc: + except (OSError, TypeError, ValueError) as exc: _fail_exception(exc, command="demo", as_json=as_json) if as_json: - written = [_artifact(config_path, "configuration")] data: dict[str, Any] = { "output_directory": str(output.resolve()), "configuration": str(config_path.resolve()), @@ -477,13 +306,12 @@ def demo( } warnings: Sequence[str] = () if artifacts is not None: - written.extend(_artifact(path, "analysis_artifact") for path in artifacts.files) data["analysis"] = _analysis_summary( artifacts.analysis, artifacts.output_directory, ) warnings = tuple(str(item) for item in artifacts.analysis.get("warnings", [])) - _emit_json("demo", data, warnings=warnings, artifacts=written) + _emit_json("demo", data, warnings=warnings) return console.print(f"[green]Generated demo[/green] at {output.resolve()}") if artifacts is not None: @@ -493,83 +321,13 @@ def demo( ) -@app.command() -def benchmark( - output: Path = typer.Option( - Path("electrace-benchmark.json"), - "--output", - "-o", - help="Path for the deterministic JSON benchmark report.", - ), - force: bool = typer.Option(False, "--force", help="Replace an existing report."), - as_json: bool = typer.Option(False, "--json", help="Emit a versioned JSON envelope."), -) -> None: - """Run the offline synthetic matching and event regression benchmark.""" - - try: - if (output.exists() or output.is_symlink()) and not force: - raise FileExistsError( - f"benchmark output already exists: {output}; use --force to replace it" - ) - report = run_synthetic_benchmark() - write_json(report, output) - except (OSError, TypeError, ValueError) as exc: - _fail_exception(exc, command="benchmark", as_json=as_json) - summary = cast(dict[str, Any], report["summary"]) - passed = benchmark_passed(report) - status = "passed" if passed else "failed" - color = "green" if passed else "red" - if as_json: - artifact = _artifact(output.resolve(), "benchmark_report") - if passed: - _emit_json( - "benchmark", - {"passed": True, "summary": summary}, - artifacts=(artifact,), - ) - return - _fail( - "synthetic benchmark failed", - code=1, - as_json=True, - command="benchmark", - error_code="benchmark_failed", - details={"passed": False, "summary": summary}, - artifacts=(artifact,), - ) - console.print( - f"[{color}]Synthetic benchmark {status}[/{color}]: " - f"{summary['passed']}/{summary['cases']} cases · " - f"event precision {summary['event_precision']:.3f} · " - f"event recall {summary['event_recall']:.3f}" - ) - console.print(f"Report: {output.resolve()}") - if not passed: - raise typer.Exit(code=1) - - @app.command() def analyze( - config: Path = typer.Argument( - ..., - exists=True, - dir_okay=False, - resolve_path=True, - help="ElecTrace TOML configuration.", - ), - output: Path | None = typer.Option( - None, - "--output", - "-o", - help="Override the configured output directory.", - ), - strict: bool = typer.Option( - True, - "--strict/--permissive", - help="Fail on incomplete discovery or ambiguous atom/feature correspondence.", - ), - force: bool = typer.Option(False, "--force", help="Replace known result artifacts."), - as_json: bool = typer.Option(False, "--json", help="Emit a versioned JSON envelope."), + config: Path = typer.Argument(...), + output: Path | None = typer.Option(None, "--output", "-o"), + strict: bool = typer.Option(True, "--strict/--permissive"), + force: bool = typer.Option(False, "--force"), + as_json: bool = typer.Option(False, "--json"), ) -> None: """Run periodic correspondence, event detection, and report generation.""" @@ -580,7 +338,7 @@ def analyze( strict=strict, overwrite=force, ) - except (OSError, TypeError, ValueError, ExportDependencyError) as exc: + except (OSError, TypeError, ValueError) as exc: _fail_exception(exc, command="analyze", as_json=as_json) analysis = artifacts.analysis if as_json: @@ -589,7 +347,6 @@ def analyze( "analyze", _analysis_summary(analysis, artifacts.output_directory), warnings=warnings, - artifacts=tuple(_artifact(path, "analysis_artifact") for path in artifacts.files), ) return console.print(f"[green]Analysis complete[/green]: {artifacts.output_directory}") @@ -630,50 +387,29 @@ def _print_analysis_summary(analysis: dict[str, Any]) -> None: @app.command() def inspect( - target: Path = typer.Argument( - ..., exists=True, resolve_path=True, help="Config, input path, or result bundle." - ), - as_json: bool = typer.Option(False, "--json", help="Emit a versioned JSON envelope."), + target: Path = typer.Argument(...), + as_json: bool = typer.Option(False, "--json"), ) -> None: """Summarize a configuration, source series, or analysis bundle read-only.""" + kind = "analysis" try: analysis = _read_analysis(target) + if analysis is None and target.is_file() and target.suffix.lower() == ".toml": + analysis, _ = load_and_analyze(target) + kind = "configuration" except (OSError, TypeError, ValueError) as exc: _fail_exception(exc, command="inspect", as_json=as_json) if analysis is not None: try: summary = _analysis_summary(analysis, target) warnings = _analysis_warnings(analysis) - if as_json: - _emit_json( - "inspect", - {"kind": "analysis", **summary}, - warnings=warnings, - ) - return - _print_analysis_summary(analysis) - except (OSError, TypeError, ValueError) as exc: - _fail_exception(exc, command="inspect", as_json=as_json) - return - if target.is_file() and target.suffix.lower() == ".toml": - try: - configured_analysis, _ = load_and_analyze(target) - except (OSError, TypeError, ValueError) as exc: - _fail_exception(exc, command="inspect", as_json=as_json) - try: - summary = _analysis_summary(configured_analysis, target) - warnings = _analysis_warnings(configured_analysis) - if as_json: - _emit_json( - "inspect", - {"kind": "configuration", **summary}, - warnings=warnings, - ) - return - _print_analysis_summary(configured_analysis) except (OSError, TypeError, ValueError) as exc: _fail_exception(exc, command="inspect", as_json=as_json) + if as_json: + _emit_json("inspect", {"kind": kind, **summary}, warnings=warnings) + else: + _print_analysis_summary(analysis) return if not target.is_dir(): _fail( @@ -724,36 +460,16 @@ def inspect( @app.command() def plan( plugin: str = typer.Argument(..., help=f"Plugin name: {', '.join(DEFAULT_REGISTRY.names())}."), - frame: Path = typer.Argument( - ..., - exists=True, - file_okay=False, - resolve_path=True, - help="One VASP frame directory, or a path root with --all-frames.", - ), + frame: Path = typer.Argument(...), capability: str | None = typer.Option(None, "--capability", "-c"), - config: Path | None = typer.Option( - None, - "--config", - exists=True, - dir_okay=False, - resolve_path=True, - ), - all_frames: bool = typer.Option( - False, - "--all-frames", - help="Plan deterministically for every frame discovered below FRAME.", - ), - as_json: bool = typer.Option( - False, - "--json", - help="Wrap the command plan in the versioned CLI JSON envelope.", - ), + config: Path | None = typer.Option(None, "--config"), + all_frames: bool = typer.Option(False, "--all-frames"), + as_json: bool = typer.Option(False, "--json"), ) -> None: """Print an external-tool command plan without executing it.""" try: - loaded = _load_optional_config(config) + loaded = load_config(config) if config is not None else None except (OSError, TypeError, ValueError) as exc: _fail_exception(exc, command="plan", as_json=as_json) plugin_name = plugin.strip().lower() @@ -777,26 +493,22 @@ def plan( "path": str(frame), "plans": [command.to_dict() for command in commands], } - if as_json: - _emit_json("plan", payload) - else: - console.print_json(data=payload) - return - if loaded is not None and plugin_name == "critic2": - plugin_config = dict(plugin_config or {}) - plugin_config.setdefault("structure_file", loaded.inputs.structure_name) - command = plan_plugin( - plugin_name, - frame, - capability=capability, - configuration=plugin_config, - ) + else: + if loaded is not None and plugin_name == "critic2": + plugin_config = dict(plugin_config or {}) + plugin_config.setdefault("structure_file", loaded.inputs.structure_name) + payload = plan_plugin( + plugin_name, + frame, + capability=capability, + configuration=plugin_config, + ).to_dict() except (KeyError, OSError, ValueError, TypeError) as exc: _fail_exception(exc, command="plan", as_json=as_json) if as_json: - _emit_json("plan", command.to_dict()) + _emit_json("plan", payload) else: - console.print_json(data=command.to_dict()) + console.print_json(data=payload) def _fractional_coordinate(value: str) -> tuple[float, float, float]: @@ -811,20 +523,10 @@ def _fractional_coordinate(value: str) -> tuple[float, float, float]: @app.command() def volume( - path: Path = typer.Argument( - ..., - exists=True, - dir_okay=False, - resolve_path=True, - help="Plain VASP ELFCAR/CHGCAR-like volumetric file.", - ), - dataset: int = typer.Option(0, "--dataset", min=0, help="Zero-based scalar dataset."), - sample: str | None = typer.Option( - None, - "--sample", - help="Periodically sample at fractional x,y,z.", - ), - as_json: bool = typer.Option(False, "--json", help="Emit machine-readable JSON."), + path: Path = typer.Argument(...), + dataset: int = typer.Option(0, "--dataset", min=0), + sample: str | None = typer.Option(None, "--sample"), + as_json: bool = typer.Option(False, "--json"), ) -> None: """Read a VASP volumetric grid directly, summarize it, and optionally sample it.""" @@ -864,21 +566,10 @@ def volume( @workflow_app.command("init") def workflow_init( - manifest: Path = typer.Argument( - Path("electrace-workflow.yaml"), - help="Workflow YAML file to create.", - ), - images: Path = typer.Option( - ..., - "--images", - "-i", - exists=True, - file_okay=False, - resolve_path=True, - help="Ordered VASP NEB or static-series root.", - ), - force: bool = typer.Option(False, "--force", help="Replace an existing manifest."), - as_json: bool = typer.Option(False, "--json", help="Emit a versioned JSON envelope."), + manifest: Path = typer.Argument(Path("electrace-workflow.yaml")), + images: Path = typer.Option(..., "--images", "-i"), + force: bool = typer.Option(False, "--force"), + as_json: bool = typer.Option(False, "--json"), ) -> None: """Create a reviewed YAML workflow with explicit executable settings.""" @@ -886,27 +577,19 @@ def workflow_init( output = dump_starter_manifest(manifest, root=str(images), overwrite=force) except (OSError, TypeError, ValueError) as exc: _fail_exception(exc, command="workflow.init", as_json=as_json) - if as_json: - _emit_json( - "workflow.init", - {"manifest": str(output.resolve()), "images": str(images)}, - artifacts=(_artifact(output.resolve(), "workflow_manifest"),), - ) - return - console.print(f"[green]Created workflow[/green] {output.resolve()}") - console.print("Set executable paths and provide reviewed VASP/template inputs before staging.") + _finish( + "workflow.init", + {"manifest": str(output.resolve()), "images": str(images)}, + as_json, + f"[green]Created workflow[/green] {output.resolve()}", + "Set executable paths and provide reviewed VASP/template inputs before staging.", + ) @workflow_app.command("validate") def workflow_validate( - manifest: Path = typer.Argument( - ..., - exists=True, - dir_okay=False, - resolve_path=True, - help="ElecTrace workflow YAML.", - ), - as_json: bool = typer.Option(False, "--json", help="Emit a versioned JSON envelope."), + manifest: Path = typer.Argument(...), + as_json: bool = typer.Option(False, "--json"), ) -> None: """Resolve and validate the complete workflow plan without writing anything.""" @@ -939,19 +622,9 @@ def workflow_validate( @workflow_app.command("stage") def workflow_stage( - manifest: Path = typer.Argument( - ..., - exists=True, - dir_okay=False, - resolve_path=True, - help="ElecTrace workflow YAML.", - ), - force: bool = typer.Option( - False, - "--force", - help="Replace only an existing marker-protected ElecTrace workspace.", - ), - as_json: bool = typer.Option(False, "--json", help="Emit a versioned JSON envelope."), + manifest: Path = typer.Argument(...), + force: bool = typer.Option(False, "--force"), + as_json: bool = typer.Option(False, "--json"), ) -> None: """Select frames and stage auditable jobs without executing anything.""" @@ -959,43 +632,22 @@ def workflow_stage( staged = stage_workflow(manifest, overwrite=force) except (OSError, TypeError, ValueError, KeyError) as exc: _fail_exception(exc, command="workflow.stage", as_json=as_json) - if as_json: - _emit_json( - "workflow.stage", - staged.to_dict(), - artifacts=(_artifact(staged.record_path, "workspace_record"),), - ) - return - console.print(f"[green]Staged[/green] {len(staged.jobs)} jobs in {staged.workspace}") - console.print(f"Selected frames: {', '.join(frame.label for frame in staged.frames)}") + _finish( + "workflow.stage", + staged.to_dict(), + as_json, + f"[green]Staged[/green] {len(staged.jobs)} jobs in {staged.workspace}", + f"Selected frames: {', '.join(frame.label for frame in staged.frames)}", + ) @workflow_app.command("run") def workflow_run( - manifest: Path = typer.Argument( - ..., - exists=True, - dir_okay=False, - resolve_path=True, - help="Staged local workflow YAML.", - ), - confirmed: bool = typer.Option( - False, - "--yes", - help="Explicitly authorize execution of all staged external commands.", - ), - max_parallel: int = typer.Option( - 1, - "--max-parallel", - min=1, - help="Maximum independent local jobs to run concurrently.", - ), - keep_going: bool = typer.Option( - False, - "--keep-going", - help="Continue starting independent jobs after a failure.", - ), - as_json: bool = typer.Option(False, "--json", help="Emit a versioned JSON envelope."), + manifest: Path = typer.Argument(...), + confirmed: bool = typer.Option(False, "--yes"), + max_parallel: int = typer.Option(1, "--max-parallel", min=1), + keep_going: bool = typer.Option(False, "--keep-going"), + as_json: bool = typer.Option(False, "--json"), ) -> None: """Execute a staged local DAG; external commands require ``--yes``.""" @@ -1041,30 +693,12 @@ def workflow_run( @workflow_app.command("submit") def workflow_submit( - manifest: Path = typer.Argument( - ..., - exists=True, - dir_okay=False, - resolve_path=True, - help="Staged Slurm workflow YAML.", - ), - job_name: str = typer.Option( - "electrace", - "--job-name", - help="Exact Slurm job name used for every selected frame.", - ), - confirmed: bool = typer.Option( - False, - "--yes", - help="Actually invoke sbatch; without this flag only scripts are prepared.", - ), - submit_command: str = typer.Option( - "sbatch", - "--submit-command", - help="sbatch-compatible executable or wrapper path.", - ), - force: bool = typer.Option(False, "--force", help="Replace generated Slurm scripts."), - as_json: bool = typer.Option(False, "--json", help="Emit a versioned JSON envelope."), + manifest: Path = typer.Argument(...), + job_name: str = typer.Option("electrace", "--job-name"), + confirmed: bool = typer.Option(False, "--yes"), + submit_command: str = typer.Option("sbatch", "--submit-command"), + force: bool = typer.Option(False, "--force"), + as_json: bool = typer.Option(False, "--json"), ) -> None: """Prepare Slurm scripts and optionally submit one job per selected frame.""" @@ -1079,7 +713,6 @@ def workflow_submit( "job_name": job_name, "scripts": [str(path) for path in scripts], }, - artifacts=tuple(_artifact(path, "slurm_script") for path in scripts), ) return console.print(f"[green]Prepared[/green] {len(scripts)} Slurm scripts") @@ -1102,9 +735,6 @@ def workflow_submit( "submission_complete": exc.submission_complete, "record_path": str(exc.record_path), } - partial_artifacts = tuple( - _artifact(Path(path), "submitted_slurm_script") for path in exc.partial_submissions - ) _fail( str(exc), code=1, @@ -1112,10 +742,6 @@ def workflow_submit( command="workflow.submit", error_code="slurm_submission_failed", details=details, - artifacts=( - *partial_artifacts, - _artifact(exc.record_path, "slurm_submission_record"), - ), ) except (OSError, RuntimeError, TypeError, ValueError) as exc: _fail_exception(exc, command="workflow.submit", as_json=as_json) @@ -1129,10 +755,6 @@ def workflow_submit( "submissions": submissions, "submission_record": str(record_path), }, - artifacts=( - *(_artifact(Path(path), "submitted_slurm_script") for path in submissions), - _artifact(record_path, "slurm_submission_record"), - ), ) return console.print(f"[green]Submitted[/green] {len(submissions)} Slurm jobs") @@ -1142,18 +764,8 @@ def workflow_submit( @workflow_app.command("status") def workflow_status_command( - manifest: Path = typer.Argument( - ..., - exists=True, - dir_okay=False, - resolve_path=True, - help="Staged workflow YAML.", - ), - as_json: bool = typer.Option( - False, - "--json", - help="Wrap status in the versioned CLI JSON envelope.", - ), + manifest: Path = typer.Argument(...), + as_json: bool = typer.Option(False, "--json"), ) -> None: """Report which staged inputs and expected outputs currently exist.""" @@ -1163,25 +775,15 @@ def workflow_status_command( _fail_exception(exc, command="workflow.status", as_json=as_json) if as_json: _emit_json("workflow.status", payload) - else: - console.print_json(data=payload) + return + console.print_json(data=payload) @workflow_app.command("collect") def workflow_collect( - manifest: Path = typer.Argument( - ..., - exists=True, - dir_okay=False, - resolve_path=True, - help="Completed staged workflow YAML.", - ), - force: bool = typer.Option( - False, - "--force", - help="Replace only the marker-protected collected analysis tree.", - ), - as_json: bool = typer.Option(False, "--json", help="Emit a versioned JSON envelope."), + manifest: Path = typer.Argument(...), + force: bool = typer.Option(False, "--force"), + as_json: bool = typer.Option(False, "--json"), ) -> None: """Validate and collect every selected descriptor output for analysis.""" @@ -1189,42 +791,26 @@ def workflow_collect( collected = collect_workflow_outputs(manifest, overwrite=force) except (OSError, TypeError, ValueError) as exc: _fail_exception(exc, command="workflow.collect", as_json=as_json) - if as_json: - _emit_json( - "workflow.collect", - { - "configuration": str(collected.config_path), - "input_root": str(collected.input_root), - "descriptor_root": str(collected.descriptor_root), - "files": [str(path) for path in collected.files], - }, - artifacts=tuple(_artifact(path, "collected_artifact") for path in collected.files), - ) - return - console.print(f"[green]Collected[/green] workflow evidence at {collected.config_path.parent}") - console.print(f"Analysis configuration: {collected.config_path}") + _finish( + "workflow.collect", + { + "configuration": str(collected.config_path), + "input_root": str(collected.input_root), + "descriptor_root": str(collected.descriptor_root), + "files": [str(path) for path in collected.files], + }, + as_json, + f"[green]Collected[/green] workflow evidence at {collected.config_path.parent}", + f"Analysis configuration: {collected.config_path}", + ) @workflow_app.command("analyze") def workflow_analyze( - manifest: Path = typer.Argument( - ..., - exists=True, - dir_okay=False, - resolve_path=True, - help="Completed staged workflow YAML.", - ), - strict: bool = typer.Option( - True, - "--strict/--permissive", - help="Fail on ambiguous atom or feature correspondence.", - ), - force: bool = typer.Option( - False, - "--force", - help="Replace marker-protected collected evidence and known result artifacts.", - ), - as_json: bool = typer.Option(False, "--json", help="Emit a versioned JSON envelope."), + manifest: Path = typer.Argument(...), + strict: bool = typer.Option(True, "--strict/--permissive"), + force: bool = typer.Option(False, "--force"), + as_json: bool = typer.Option(False, "--json"), ) -> None: """Collect completed outputs, track changes, and generate the result bundle.""" @@ -1234,7 +820,7 @@ def workflow_analyze( strict=strict, overwrite=force, ) - except (OSError, TypeError, ValueError, ExportDependencyError) as exc: + except (OSError, TypeError, ValueError) as exc: _fail_exception(exc, command="workflow.analyze", as_json=as_json) analysis = artifacts.analysis if as_json: @@ -1243,7 +829,6 @@ def workflow_analyze( "workflow.analyze", _analysis_summary(analysis, artifacts.output_directory), warnings=warnings, - artifacts=tuple(_artifact(path, "analysis_artifact") for path in artifacts.files), ) return console.print(f"[green]Workflow analysis complete[/green]: {artifacts.output_directory}") diff --git a/src/electrace/cli_output.py b/src/electrace/cli_output.py index 711f9f9..a1b282d 100644 --- a/src/electrace/cli_output.py +++ b/src/electrace/cli_output.py @@ -1,225 +1,56 @@ -"""Strict, versioned JSON envelopes for the ElecTrace command line.""" +"""Small, versioned JSON envelopes for the ElecTrace command line.""" from __future__ import annotations import json -import math -import re from collections.abc import Mapping, Sequence -from typing import Final, Literal, TypeAlias, TypedDict +from typing import Any, Final CLI_SCHEMA_VERSION: Final = "electrace.cli.v1" -JsonValue: TypeAlias = bool | int | float | str | list["JsonValue"] | dict[str, "JsonValue"] | None - - -class CliError(TypedDict): - """Machine-readable error information carried by a failed response.""" - - code: str - message: str - details: JsonValue - - -class SuccessEnvelope(TypedDict): - """Successful CLI response.""" - - schema_version: Literal["electrace.cli.v1"] - ok: Literal[True] - command: str - data: JsonValue - warnings: list[str] - artifacts: list[JsonValue] - - -class ErrorEnvelope(TypedDict): - """Failed CLI response.""" - - schema_version: Literal["electrace.cli.v1"] - ok: Literal[False] - command: str - error: CliError - warnings: list[str] - artifacts: list[JsonValue] - - -ResponseEnvelope: TypeAlias = SuccessEnvelope | ErrorEnvelope - -_ERROR_CODE_ALIASES: Final = { - "eacces": "permission_denied", - "eexist": "already_exists", - "enoent": "not_found", - "eperm": "permission_denied", - "file_exists": "already_exists", - "file_exists_error": "already_exists", - "file_not_found": "not_found", - "file_not_found_error": "not_found", - "ioerror": "io_error", - "is_adirectory_error": "invalid_path", - "is_a_directory": "invalid_path", - "is_a_directory_error": "invalid_path", - "not_adirectory_error": "invalid_path", - "not_a_directory": "invalid_path", - "not_a_directory_error": "invalid_path", - "oserror": "io_error", - "os_error": "io_error", - "permission_error": "permission_denied", - "timed_out": "timeout", - "timeout_error": "timeout", - "type_error": "invalid_input", - "value_error": "invalid_input", -} -_CAMEL_CASE_BOUNDARY = re.compile(r"(?<=[a-z0-9])(?=[A-Z])") -_ERROR_CODE_SEPARATOR = re.compile(r"[^a-z0-9]+") - - -def normalize_error_code(code: str | BaseException) -> str: - """Return a stable ``snake_case`` code for a label or built-in exception.""" - - if isinstance(code, BaseException): - if isinstance(code, FileNotFoundError): - return "not_found" - if isinstance(code, FileExistsError): - return "already_exists" - if isinstance(code, PermissionError): - return "permission_denied" - if isinstance(code, (IsADirectoryError, NotADirectoryError)): - return "invalid_path" - if isinstance(code, TimeoutError): - return "timeout" - if isinstance(code, (TypeError, ValueError)): - return "invalid_input" - if isinstance(code, OSError): - return "io_error" - if isinstance(code, KeyboardInterrupt): - return "interrupted" - return "internal_error" - if not isinstance(code, str): - raise TypeError("error code must be a string or exception") - words = _CAMEL_CASE_BOUNDARY.sub("_", code.strip()).casefold() - normalized = _ERROR_CODE_SEPARATOR.sub("_", words).strip("_") - if not normalized: - raise ValueError("error code must not be empty") - if normalized[0].isdigit(): - normalized = f"error_{normalized}" - return _ERROR_CODE_ALIASES.get(normalized, normalized) - - -def _required_text(value: object, *, name: str) -> str: - if not isinstance(value, str): - raise TypeError(f"{name} must be a string") - if not value.strip(): - raise ValueError(f"{name} must not be empty") - return value - - -def _json_value(value: object, *, context: str, active: set[int] | None = None) -> JsonValue: - """Copy *value* into the strict JSON value domain.""" - - if value is None or isinstance(value, (bool, str)): - return value - if isinstance(value, int): - return int(value) - if isinstance(value, float): - if not math.isfinite(value): - raise ValueError(f"{context} contains a non-finite number") - return float(value) - - stack = active if active is not None else set() - if isinstance(value, Mapping): - marker = id(value) - if marker in stack: - raise ValueError(f"{context} contains a circular reference") - stack.add(marker) - try: - result: dict[str, JsonValue] = {} - for key, item in value.items(): - if not isinstance(key, str): - raise TypeError(f"{context} contains a non-string object key: {key!r}") - result[key] = _json_value(item, context=f"{context}.{key}", active=stack) - return result - finally: - stack.remove(marker) - - if isinstance(value, (list, tuple)): - marker = id(value) - if marker in stack: - raise ValueError(f"{context} contains a circular reference") - stack.add(marker) - try: - return [ - _json_value(item, context=f"{context}[{index}]", active=stack) - for index, item in enumerate(value) - ] - finally: - stack.remove(marker) - - raise TypeError(f"{context} contains a non-JSON value: {type(value).__name__}") - - -def _warnings(values: Sequence[str]) -> list[str]: - if isinstance(values, (str, bytes)): - raise TypeError("warnings must be a sequence of strings") - return [_required_text(value, name="warning") for value in values] - - -def _artifacts(values: Sequence[object]) -> list[JsonValue]: - if isinstance(values, (str, bytes)): - raise TypeError("artifacts must be a sequence of JSON values") - return [_json_value(value, context=f"artifact[{index}]") for index, value in enumerate(values)] - def success( command: str, data: object = None, *, warnings: Sequence[str] = (), - artifacts: Sequence[object] = (), -) -> SuccessEnvelope: - """Build a successful, JSON-safe CLI response envelope.""" +) -> dict[str, Any]: + """Return the common successful response shape.""" return { "schema_version": CLI_SCHEMA_VERSION, "ok": True, - "command": _required_text(command, name="command"), - "data": _json_value(data, context="data"), - "warnings": _warnings(warnings), - "artifacts": _artifacts(artifacts), + "command": command, + "data": data, + "warnings": list(warnings), } def failure( command: str, - code: str | BaseException, + code: str, message: str, *, details: object = None, warnings: Sequence[str] = (), - artifacts: Sequence[object] = (), -) -> ErrorEnvelope: - """Build a failed, JSON-safe CLI response envelope.""" +) -> dict[str, Any]: + """Return the common failed response shape.""" return { "schema_version": CLI_SCHEMA_VERSION, "ok": False, - "command": _required_text(command, name="command"), - "error": { - "code": normalize_error_code(code), - "message": _required_text(message, name="error message"), - "details": _json_value(details, context="error details"), - }, - "warnings": _warnings(warnings), - "artifacts": _artifacts(artifacts), + "command": command, + "error": {"code": code, "message": message, "details": details}, + "warnings": list(warnings), } -def serialize(envelope: ResponseEnvelope) -> str: - """Serialize one response as deterministic, compact JSON followed by a newline.""" +def serialize(envelope: Mapping[str, object]) -> str: + """Serialize one compact JSON document followed by a newline.""" - validated = _json_value(envelope, context="CLI response") return ( json.dumps( - validated, + envelope, allow_nan=False, ensure_ascii=True, separators=(",", ":"), @@ -229,14 +60,4 @@ def serialize(envelope: ResponseEnvelope) -> str: ) -__all__ = [ - "CLI_SCHEMA_VERSION", - "ErrorEnvelope", - "JsonValue", - "ResponseEnvelope", - "SuccessEnvelope", - "failure", - "normalize_error_code", - "serialize", - "success", -] +__all__ = ["CLI_SCHEMA_VERSION", "failure", "serialize", "success"] diff --git a/src/electrace/config.py b/src/electrace/config.py index c33d2fc..f0d2b87 100644 --- a/src/electrace/config.py +++ b/src/electrace/config.py @@ -11,7 +11,6 @@ import json import math import tomllib -from collections.abc import Mapping from dataclasses import asdict, dataclass, field from pathlib import Path from typing import Any @@ -38,14 +37,7 @@ def _string(value: Any, name: str, *, normalize: bool = False) -> str: stripped = value.strip() if not stripped or any(ord(character) < 32 or ord(character) == 127 for character in value): raise ValueError(f"{name} must be a non-empty string without control characters") - return stripped.casefold() if normalize else value - - -def _table_key(value: Any, name: str) -> str: - key = _string(value, name) - if key != key.strip(): - raise ValueError(f"{name} must not contain surrounding whitespace") - return key + return stripped.casefold() if normalize else stripped def _number( @@ -78,32 +70,11 @@ def _integer(value: Any, name: str, *, minimum: int) -> int: return result -def _validate_json_compatible(value: Any, *, path: str) -> None: - """Reject values that cannot appear in strict canonical JSON. - - TOML admits dates, times, and non-finite floats that JSON does not. The - configuration is embedded verbatim in every analysis artifact, so reject - those values at construction instead of stringifying or emitting the - non-standard ``NaN``/``Infinity`` tokens. - """ - - if value is None or isinstance(value, (str, bool, int)): - return - if isinstance(value, float): - if not math.isfinite(value): - raise ValueError(f"{path} must be finite for JSON serialization") - return - if isinstance(value, Mapping): - for key, item in value.items(): - if not isinstance(key, str): - raise ValueError(f"{path} contains non-string key {key!r}") - _validate_json_compatible(item, path=f"{path}.{key}") - return - if isinstance(value, (list, tuple)): - for index, item in enumerate(value): - _validate_json_compatible(item, path=f"{path}[{index}]") - return - raise ValueError(f"{path} has non-JSON-compatible value of type {type(value).__name__}") +def _validate_json(value: Any) -> None: + try: + json.dumps(value, allow_nan=False) + except (TypeError, ValueError, RecursionError) as exc: + raise ValueError("configuration must contain finite JSON-compatible values") from exc def _input_filename(value: str, field_name: str) -> str: @@ -174,7 +145,6 @@ class MatchingSettings: descriptor_weight: float = 0.35 descriptor_scales: dict[str, float] = field(default_factory=dict) match_threshold: float = 3.0 - ambiguity_tolerance: float = 0.05 conservation_tolerance: float = 0.35 split_merge_threshold: float = 0.55 @@ -210,7 +180,7 @@ def __post_init__(self) -> None: if not isinstance(self.descriptor_scales, dict): raise TypeError("matching.descriptor_scales must be a table") self.descriptor_scales = { - _table_key(key, "matching.descriptor_scales key"): _number( + _string(key, "matching.descriptor_scales key"): _number( value, f"matching.descriptor_scales.{key}", minimum=0.0, @@ -230,11 +200,6 @@ def __post_init__(self) -> None: minimum=0.0, maximum=1.0, ) - self.ambiguity_tolerance = _number( - self.ambiguity_tolerance, - "matching.ambiguity_tolerance", - minimum=0.0, - ) @dataclass(slots=True) @@ -242,7 +207,6 @@ class EventSettings: """Temporal filters used to suppress one-frame numerical noise.""" minimum_persistence: int = 2 - hysteresis_frames: int = 1 minimum_event_confidence: float = 0.10 charge_change_threshold: float = 0.10 volume_change_fraction: float = 0.20 @@ -262,9 +226,6 @@ def __post_init__(self) -> None: self.minimum_persistence = _integer( self.minimum_persistence, "events.minimum_persistence", minimum=1 ) - self.hysteresis_frames = _integer( - self.hysteresis_frames, "events.hysteresis_frames", minimum=0 - ) self.minimum_event_confidence = _number( self.minimum_event_confidence, "events.minimum_event_confidence", @@ -286,7 +247,7 @@ def __post_init__(self) -> None: if not isinstance(self.descriptor_thresholds, dict): raise TypeError("events.descriptor_thresholds must be a table") self.descriptor_thresholds = { - _table_key(key, "events.descriptor_thresholds key"): _number( + _string(key, "events.descriptor_thresholds key"): _number( value, f"events.descriptor_thresholds.{key}", minimum=0.0, @@ -321,11 +282,10 @@ class OutputConfig: html_report: bool = True json: bool = True csv: bool = True - hdf5: bool = False def __post_init__(self) -> None: self.directory = _string(self.directory, "outputs.directory") - for name in ("html_report", "json", "csv", "hdf5"): + for name in ("html_report", "json", "csv"): setattr(self, name, _boolean(getattr(self, name), f"outputs.{name}")) @@ -356,40 +316,28 @@ def __post_init__(self) -> None: ): if not isinstance(value, expected): raise TypeError(f"configuration.{name} must be a {expected.__name__}") - if not isinstance(self.plugins, dict): - raise TypeError("[plugins] must be a TOML table") plugins: dict[str, dict[str, Any]] = {} for name, options in self.plugins.items(): - plugin_name = _string(name, "plugin name") - normalized_name = plugin_name.strip().casefold() - if normalized_name in plugins: - raise ValueError(f"duplicate normalized plugin name {normalized_name!r}") - if plugin_name != normalized_name: - raise ValueError( - f"plugin table name {plugin_name!r} must be lowercase without surrounding whitespace" - ) + normalized_name = _string(name, "plugin name", normalize=True) if not isinstance(options, dict): raise TypeError(f"[plugins.{normalized_name}] must be a TOML table") - settings = dict(options) - if "enabled" in settings: - _boolean(settings["enabled"], f"plugins.{normalized_name}.enabled") - for reserved in ("executable", "capability"): - if reserved in settings: - _string(settings[reserved], f"plugins.{normalized_name}.{reserved}") - plugins[normalized_name] = settings + if "enabled" in options: + _boolean(options["enabled"], f"plugins.{normalized_name}.enabled") + for key in ("executable", "capability"): + if key in options: + _string(options[key], f"plugins.{normalized_name}.{key}") + plugins[normalized_name] = dict(options) self.plugins = plugins if not isinstance(self.metadata, dict): raise TypeError("[metadata] must be a TOML table") self.metadata = dict(self.metadata) - _validate_json_compatible(asdict(self), path="configuration") + _validate_json(asdict(self)) def to_dict(self) -> dict[str, Any]: """Return a JSON-serializable nested mapping.""" result = asdict(self) - # Metadata/plugin mappings remain mutable after construction. Keep - # the public contract true even if a caller mutates one in place. - _validate_json_compatible(result, path="configuration") + _validate_json(result) return result @@ -400,7 +348,7 @@ def _section(data: dict[str, Any], name: str) -> dict[str, Any]: return value -def _reject_unknown_keys(data: Mapping[str, Any], allowed: frozenset[str], context: str) -> None: +def _reject_unknown_keys(data: dict[str, Any], allowed: frozenset[str], context: str) -> None: unknown = sorted(set(data) - allowed) if unknown: raise ValueError(f"{context} contains unknown keys: {', '.join(unknown)}") @@ -451,8 +399,6 @@ def load_config(path: str | Path) -> ElecTraceConfig: overlap_weight = 1.0 descriptor_weight = 0.35 match_threshold = 3.0 -# Maximum total-cost gap for a complete alternative assignment. -ambiguity_tolerance = 0.05 conservation_tolerance = 0.35 split_merge_threshold = 0.55 @@ -464,7 +410,6 @@ def load_config(path: str | Path) -> ElecTraceConfig: [events] minimum_persistence = 2 -hysteresis_frames = 1 minimum_event_confidence = 0.10 charge_change_threshold = 0.10 volume_change_fraction = 0.20 @@ -483,8 +428,6 @@ def load_config(path: str | Path) -> ElecTraceConfig: html_report = true json = true csv = true -hdf5 = false - [plugins.critic2] enabled = true executable = "critic2" diff --git a/src/electrace/events.py b/src/electrace/events.py index ddfb0fb..f2bf759 100644 --- a/src/electrace/events.py +++ b/src/electrace/events.py @@ -1,10 +1,9 @@ -"""Electronic-topology event detection, persistence, and hysteresis. +"""Electronic-topology event detection and sequence persistence. ElecTrace reports an evidence vector rather than treating any single charge, ELF, or bond-order definition as ground truth. Transition-level events are therefore useful on their own, while sequence-level tracking annotates them -with persistence and right-censoring information before confidence is used -for scientific interpretation. +with a compact consecutive-lineage persistence classification. """ from __future__ import annotations @@ -17,7 +16,6 @@ import numpy as np -from .geometry import deformation_aware_distance from .matching import ( AtomMatchConfig, FeatureMatchConfig, @@ -40,7 +38,7 @@ @dataclass(slots=True) class EventDetectionConfig: - """Thresholds for topology, descriptor-change, and persistence evidence.""" + """Thresholds for event detection and consecutive-frame persistence.""" charge_change_threshold: float = 0.15 volume_relative_threshold: float = 0.25 @@ -54,17 +52,12 @@ class EventDetectionConfig: ) min_event_confidence: float = 0.1 min_persistence: int = 2 - hysteresis_frames: int = 1 include_transient: bool = True - hysteresis_distance: float = 0.75 - transient_confidence_exponent: float = 1.0 def __post_init__(self) -> None: for name in ( "charge_change_threshold", "volume_relative_threshold", - "hysteresis_distance", - "transient_confidence_exponent", ): value = float(getattr(self, name)) if not math.isfinite(value) or value <= 0: @@ -79,9 +72,6 @@ def __post_init__(self) -> None: self.min_persistence = int(self.min_persistence) if self.min_persistence < 1: raise ValueError("min_persistence must be at least one") - self.hysteresis_frames = int(self.hysteresis_frames) - if self.hysteresis_frames < 0: - raise ValueError("hysteresis_frames must be non-negative") self.descriptor_thresholds = { str(key): _positive(value, f"descriptor threshold {key!r}") for key, value in self.descriptor_thresholds.items() @@ -147,21 +137,12 @@ def _property_change_event( config: EventDetectionConfig, ) -> Event | None: evidence: dict[str, Any] = {} - signals: dict[str, dict[str, Any]] = {} severities: list[float] = [] if source.charge is not None and target.charge is not None: delta = target.charge - source.charge if abs(delta) >= config.charge_change_threshold: evidence["charge_delta"] = delta evidence["charge_threshold"] = config.charge_change_threshold - signals["charge"] = { - "location": "charge", - "baseline": source.charge, - "onset": target.charge, - "threshold": config.charge_change_threshold, - "mode": "absolute", - "direction": 1 if delta > 0 else -1, - } severities.append(abs(delta) / config.charge_change_threshold) if source.volume is not None and target.volume is not None: relative = abs(target.volume - source.volume) / max( @@ -170,15 +151,6 @@ def _property_change_event( if relative >= config.volume_relative_threshold: evidence["volume_relative_change"] = relative evidence["volume_threshold"] = config.volume_relative_threshold - volume_delta = target.volume - source.volume - signals["volume"] = { - "location": "volume", - "baseline": source.volume, - "onset": target.volume, - "threshold": config.volume_relative_threshold, - "mode": "relative", - "direction": 1 if volume_delta > 0 else -1, - } severities.append(relative / config.volume_relative_threshold) descriptor_changes: dict[str, dict[str, float]] = {} for key, threshold in config.descriptor_thresholds.items(): @@ -191,15 +163,6 @@ def _property_change_event( delta = right_value - left_value if abs(delta) >= threshold: descriptor_changes[key] = {"delta": delta, "threshold": threshold} - signals[f"attribute:{key}"] = { - "location": "attribute", - "attribute": key, - "baseline": left_value, - "onset": right_value, - "threshold": threshold, - "mode": "absolute", - "direction": 1 if delta > 0 else -1, - } severities.append(abs(delta) / threshold) if descriptor_changes: evidence["descriptor_changes"] = descriptor_changes @@ -217,7 +180,6 @@ def _property_change_event( { "effect_size": severity, "match_confidence": edge.confidence, - "property_signals": signals, "quality_notes": notes, } ) @@ -257,23 +219,6 @@ def detect_transition_events( ): raise ValueError("feature match result does not correspond to the supplied frames") - raw_ambiguities = feature_matches.diagnostics.get("ambiguities", []) - ambiguities = raw_ambiguities if isinstance(raw_ambiguities, list) else [] - ambiguous_unmatched_previous = { - str(item.get("selected", {}).get("source_id")) - for item in ambiguities - if isinstance(item, dict) - and item.get("decision") == "unmatched_previous" - and isinstance(item.get("selected"), dict) - } - ambiguous_unmatched_current = { - str(item.get("selected", {}).get("target_id")) - for item in ambiguities - if isinstance(item, dict) - and item.get("decision") == "unmatched_current" - and isinstance(item.get("selected"), dict) - } - previous_features = ( {feature.id: feature for feature in previous.features} if feature_matches.edges or feature_matches.unmatched_previous @@ -286,8 +231,6 @@ def detect_transition_events( ) events: list[Event] = [] for edge in feature_matches.edges: - if edge.ambiguous: - continue topology_type = _edge_event_type(edge) if topology_type is not None: sources = tuple(previous_features[str(item)] for item in edge.source_ids) @@ -324,8 +267,6 @@ def detect_transition_events( events.append(changed) for source_id in feature_matches.unmatched_previous: - if str(source_id) in ambiguous_unmatched_previous: - continue source = previous_features[str(source_id)] quality, notes = _feature_quality((source,)) confidence = 0.75 * quality @@ -346,8 +287,6 @@ def detect_transition_events( ) ) for target_id in feature_matches.unmatched_current: - if str(target_id) in ambiguous_unmatched_current: - continue target = current_features[str(target_id)] quality, notes = _feature_quality((target,)) confidence = 0.75 * quality @@ -378,234 +317,43 @@ def detect_transition_events( ) -def _successors(ids: set[str], result: MatchResult) -> set[str]: - successors: set[str] = set() - for edge in result.edges: - if edge.ambiguous: - continue - if any(str(item) in ids for item in edge.source_ids): - successors.update(str(item) for item in edge.target_ids) - return successors - - -def _stable_one_to_one_successors( - ids: set[str], result: MatchResult -) -> tuple[set[str] | None, str | None]: - """Advance every participant through a distinct one-to-one lineage. - - Split, merge, and rearrangement persistence describes the lifetime of the - topology created at onset, not merely survival of any descendant. An - outgoing hyperedge therefore ends that state even when descendants still - exist. - """ - - outgoing: dict[str, list[MatchEdge]] = {} - for edge in result.edges: - for source_id in edge.source_ids: - outgoing.setdefault(str(source_id), []).append(edge) +def _next_lineage(ids: set[str], result: MatchResult) -> set[str]: + """Advance a lineage only when every current participant has a match.""" successors: set[str] = set() - for source_id in sorted(ids): - source_edges = outgoing.get(source_id, []) - if not source_edges: - return None, "participant_lost" - if len(source_edges) != 1: - return None, "ambiguous_outgoing_lineage" - edge = source_edges[0] - if edge.ambiguous: - return None, "ambiguous_outgoing_lineage" - if edge.relation is MatchRelation.SPLIT: - return None, "participant_expanded" - if edge.relation is MatchRelation.MERGE: - return None, "participants_coalesced" - if edge.relation is MatchRelation.MANY_TO_MANY: - return None, "participant_rearranged" - successor = str(edge.target_id) - if successor in successors: - return None, "participants_coalesced" - successors.add(successor) - if len(successors) != len(ids): - return None, "cardinality_changed" - return successors, None - - -def _topology_persistence( - event: Event, - transition_index: int, - matches: TypingSequence[MatchResult], -) -> tuple[int, int, dict[str, Any]]: - live = {str(item) for item in event.target_ids} - end_position = transition_index + 1 - if not live: - return 1, end_position, {"persistence_termination": "no_target_lineage"} - persistence = 1 - exact_state = event.event_type in { - EventType.SPLIT, - EventType.MERGE, - EventType.REARRANGEMENT, - } - termination = "sequence_end" - for result_index, result in enumerate( - matches[transition_index + 1 :], start=transition_index + 1 - ): - if exact_state: - next_live, reason = _stable_one_to_one_successors(live, result) - if next_live is None: - termination = f"{event.event_type.value}_{reason}" - break - live = next_live - else: - live = _successors(live, result) - if not live: - termination = "lineage_ended" - break - persistence += 1 - end_position = result_index + 1 - return ( - persistence, - end_position, - { - "persistence_termination": termination, - "persistence_cardinality_criterion": ( - "all_target_participants_distinct_one_to_one" - if exact_state - else "any_descendant_lineage" - ), - }, - ) - - -def _one_to_one_successor(source_id: str, result: MatchResult) -> str | None: + matched_sources: set[str] = set() for edge in result.edges: - if edge.relation is not MatchRelation.ONE_TO_ONE or edge.ambiguous: - continue - if str(edge.source_id) == source_id: - return str(edge.target_id) - return None - - -def _property_signal_value(feature: Feature, signal: dict[str, Any]) -> float | None: - location = signal.get("location") - if location == "charge": - return feature.charge - if location == "volume": - return feature.volume - if location == "attribute": - return _as_finite_float(feature.attributes.get(str(signal.get("attribute", "")))) - return None + sources = {str(item) for item in edge.source_ids} + overlap = ids & sources + if overlap: + matched_sources.update(overlap) + successors.update(str(item) for item in edge.target_ids) + return successors if matched_sources == ids else set() -def _property_signal_state(feature: Feature, signal: dict[str, Any]) -> str: - current = _property_signal_value(feature, signal) - baseline = _as_finite_float(signal.get("baseline")) - threshold = _as_finite_float(signal.get("threshold")) - direction = int(signal.get("direction", 0)) - if current is None or baseline is None or threshold is None or direction not in (-1, 1): - return "missing" - delta = current - baseline - if delta * direction < 0: - return "reversed" - if signal.get("mode") == "relative": - magnitude = abs(delta) / max(abs(current), abs(baseline), 1.0e-12) - else: - magnitude = abs(delta) - return "supported" if magnitude >= threshold else "below_threshold" - - -def _property_change_persistence( +def _consecutive_persistence( event: Event, transition_index: int, frames: TypingSequence[Frame], matches: TypingSequence[MatchResult], - config: EventDetectionConfig, -) -> tuple[int, int, dict[str, Any]]: - raw_signals = event.evidence.get("property_signals", {}) - if not isinstance(raw_signals, dict) or not raw_signals: - return 1, transition_index + 1, {"persistence_termination": "missing_property_signal"} - signals = {str(key): value for key, value in raw_signals.items() if isinstance(value, dict)} - if not signals or len(event.target_ids) != 1: - return 1, transition_index + 1, {"persistence_termination": "ambiguous_property_lineage"} - - current_id = str(event.target_ids[0]) - persistence = 1 - end_position = transition_index + 1 - consecutive_gaps = 0 - gap_frames: list[int] = [] - termination = "sequence_end" - for result_index in range(transition_index + 1, len(matches)): - successor = _one_to_one_successor(current_id, matches[result_index]) - if successor is None: - termination = "lineage_changed" - break - target_position = result_index + 1 - feature = frames[target_position].feature_by_id(successor) - states = {key: _property_signal_state(feature, signal) for key, signal in signals.items()} - if "reversed" in states.values(): - termination = "property_reversed" - break - if all(state == "supported" for state in states.values()): - persistence += 1 - end_position = target_position - consecutive_gaps = 0 - else: - consecutive_gaps += 1 - gap_frames.append(frames[target_position].index) - if consecutive_gaps > config.hysteresis_frames: - termination = "hysteresis_exceeded" - break - current_id = successor - return ( - persistence, - end_position, - { - "persistence_termination": termination, - "hysteresis_gap_frames": gap_frames, - "sustained_signal_ids": sorted(signals), - }, - ) - +) -> tuple[int, int]: + """Count consecutive frames with a surviving target or absent source lineage.""" -def _death_persistence( - event: Event, - transition_index: int, - frames: TypingSequence[Frame], - config: EventDetectionConfig, -) -> tuple[int, int, dict[str, Any]]: - source_frame = frames[transition_index] - source = source_frame.feature_by_id(str(event.source_ids[0])) persistence = 1 end_position = transition_index + 1 - # Begin after the immediate current frame, whose absence generated death. - for frame_position, frame in enumerate( - frames[transition_index + 2 :], start=transition_index + 2 - ): - reappeared = False - for feature in frame.features: - if feature.kind != source.kind: - continue - distance = float( - deformation_aware_distance( - source.fractional_position, - feature.fractional_position, - source_frame.cell, - frame.cell, - ) - ) - if distance <= config.hysteresis_distance: - reappeared = True + live = {str(item) for item in event.target_ids} + if live: + for result_index in range(transition_index + 1, len(matches)): + live = _next_lineage(live, matches[result_index]) + if not live: break - if reappeared: - return ( - persistence, - end_position, - { - "persistence_termination": "nearby_feature_reappeared", - "reappearance_frame": frame.index, - }, - ) - persistence += 1 - end_position = frame_position - return persistence, end_position, {"persistence_termination": "sequence_end"} + persistence += 1 + end_position = result_index + 1 + return persistence, end_position + + # Feature identifiers are frame-local, so a later reuse of the same label is + # not evidence that the ended lineage reappeared. + return len(frames) - transition_index - 1, len(frames) - 1 def _annotate_persistence( @@ -621,48 +369,25 @@ def _annotate_persistence( annotated: list[Event] = [] for event in events: transition_index = transition_by_frames[event.frame_before, event.frame_after] - if event.event_type is EventType.PROPERTY_CHANGE: - persistence, end_position, persistence_evidence = _property_change_persistence( - event, transition_index, frames, matches, config - ) - elif event.event_type is EventType.DEATH: - persistence, end_position, persistence_evidence = _death_persistence( - event, transition_index, frames, config - ) - else: - persistence, end_position, persistence_evidence = _topology_persistence( - event, transition_index, matches - ) + persistence, end_position = _consecutive_persistence( + event, transition_index, frames, matches + ) maximum_observable = len(frames) - transition_index - 1 metadata = dict(event.metadata) evidence = dict(event.evidence) evidence["minimum_persistence"] = config.min_persistence - evidence["hysteresis_frames"] = config.hysteresis_frames - evidence.update(persistence_evidence) - terminated_early = persistence_evidence.get("persistence_termination") != "sequence_end" - hysteresis_rejected = terminated_early and persistence <= config.hysteresis_frames - if hysteresis_rejected: - status = "transient" - denominator = max(config.min_persistence, config.hysteresis_frames + 1) - confidence = event.confidence * persistence / denominator - elif terminated_early and persistence < config.min_persistence: - status = "transient" - fraction = persistence / config.min_persistence - confidence = event.confidence * (fraction**config.transient_confidence_exponent) - elif persistence >= config.min_persistence: + evidence["maximum_observable_persistence"] = maximum_observable + if persistence >= config.min_persistence: status = "confirmed" confidence = event.confidence - elif not terminated_early and maximum_observable < config.min_persistence: + elif end_position == len(frames) - 1 and maximum_observable < config.min_persistence: status = "right_censored" - # Do not call an event noise merely because the sequence ended. confidence = event.confidence * 0.85 else: status = "transient" - fraction = persistence / config.min_persistence - confidence = event.confidence * fraction**config.transient_confidence_exponent + confidence = event.confidence * persistence / config.min_persistence metadata["persistence_status"] = status metadata["persistence_end_frame"] = frames[end_position].index - evidence["maximum_observable_persistence"] = maximum_observable revised = replace( event, persistence=persistence, @@ -711,12 +436,6 @@ def track_sequence( "raw_events": len(raw_events), "reported_events": len(events), "minimum_persistence": settings.min_persistence, - "ambiguous_atom_decisions": sum( - len(result.diagnostics.get("ambiguities", [])) for result in atom_results - ), - "ambiguous_feature_decisions": sum( - len(result.diagnostics.get("ambiguities", [])) for result in feature_results - ), }, ) diff --git a/src/electrace/export.py b/src/electrace/export.py index f807e71..e3209ab 100644 --- a/src/electrace/export.py +++ b/src/electrace/export.py @@ -18,10 +18,6 @@ from electrace._files import atomic_output_path, atomic_write_text -class ExportDependencyError(RuntimeError): - """Raised when a requested export needs an unavailable optional package.""" - - def _strict_json_dumps(value: Any, *, context: str, **kwargs: Any) -> str: """Serialize *value* as standards-compliant JSON with an actionable error.""" @@ -128,33 +124,3 @@ def write_tracks_csv(tracks: Iterable[Mapping[str, Any]], path: str | Path) -> P fields = preferred + extras _write_csv(rows, fields, output) return output - - -def write_hdf5(data: Mapping[str, Any], path: str | Path) -> Path: - """Write the canonical JSON into an HDF5 container. - - Keeping the canonical document as a UTF-8 dataset provides a lossless, - versionable interchange layer while leaving room for optimized numerical - datasets in later schema versions. - """ - - payload = _strict_json_dumps( - data, - context="HDF5 analysis JSON", - separators=(",", ":"), - sort_keys=True, - ) - try: - import h5py - except ImportError as exc: # pragma: no cover - optional dependency - raise ExportDependencyError("HDF5 export requires `pip install electrace[hdf5]`") from exc - - output = Path(path) - with ( - atomic_output_path(output) as temporary, - h5py.File(temporary, "w") as handle, - ): - handle.attrs["schema"] = "electrace.analysis" - handle.attrs["schema_version"] = str(data.get("schema_version", "0.1")) - handle.create_dataset("analysis_json", data=payload) - return output diff --git a/src/electrace/geometry.py b/src/electrace/geometry.py index 01efabc..5a8943c 100644 --- a/src/electrace/geometry.py +++ b/src/electrace/geometry.py @@ -70,48 +70,6 @@ def cell_volume(cell: ArrayLike) -> float: return float(abs(np.linalg.det(as_cell(cell)))) -def fractional_to_cartesian(fractional: ArrayLike, cell: ArrayLike) -> FloatArray: - """Convert one or more fractional row vectors to Cartesian coordinates.""" - - points = np.asarray(fractional, dtype=float) - if points.shape == () or points.shape[-1] != 3: - raise ValueError("fractional coordinates must end in an axis of length 3") - if not np.all(np.isfinite(points)): - raise ValueError("coordinates must contain only finite values") - return points @ as_cell(cell) - - -def cartesian_to_fractional(cartesian: ArrayLike, cell: ArrayLike) -> FloatArray: - """Convert one or more Cartesian row vectors to fractional coordinates.""" - - points = np.asarray(cartesian, dtype=float) - if points.shape == () or points.shape[-1] != 3: - raise ValueError("cartesian coordinates must end in an axis of length 3") - if not np.all(np.isfinite(points)): - raise ValueError("coordinates must contain only finite values") - # Solving cell.T @ frac.T = cart.T is more stable than explicit inversion. - return np.linalg.solve(as_cell(cell).T, points.reshape(-1, 3).T).T.reshape(points.shape) - - -def wrap_fractional( - fractional: ArrayLike, pbc: Sequence[bool] | bool = (True, True, True) -) -> FloatArray: - """Wrap periodic fractional components into the half-open interval [0, 1).""" - - result = np.array(fractional, dtype=float, copy=True) - if result.shape == () or result.shape[-1] != 3: - raise ValueError("fractional coordinates must end in an axis of length 3") - if not np.all(np.isfinite(result)): - raise ValueError("coordinates must contain only finite values") - periodic = as_pbc(pbc) - for axis, enabled in enumerate(periodic): - if enabled: - result[..., axis] -= np.floor(result[..., axis]) - # Avoid returning 1.0 due to roundoff near a cell boundary. - result[..., axis] %= 1.0 - return result - - def metric_average_cell(cell_a: ArrayLike, cell_b: ArrayLike) -> FloatArray: """Construct a cell realizing the arithmetic mean lattice metric. @@ -127,44 +85,6 @@ def metric_average_cell(cell_a: ArrayLike, cell_b: ArrayLike) -> FloatArray: return result -def interpolate_cell(cell_a: ArrayLike, cell_b: ArrayLike, fraction: float) -> FloatArray: - """Linearly interpolate lattice vectors (appropriate for nearby images).""" - - amount = float(fraction) - if not math.isfinite(amount) or not 0.0 <= amount <= 1.0: - raise ValueError("fraction must lie in [0, 1]") - result = (1.0 - amount) * as_cell(cell_a) + amount * as_cell(cell_b) - return as_cell(result) - - -def deformation_gradient(cell_a: ArrayLike, cell_b: ArrayLike) -> FloatArray: - """Return the Cartesian deformation gradient mapping cell A onto cell B. - - With column Cartesian vectors, ``x_b = F @ x_a`` for a fixed fractional - coordinate and ``F = cell_b.T @ inv(cell_a.T)``. - """ - - first = as_cell(cell_a) - second = as_cell(cell_b) - result = np.asarray(np.linalg.solve(first, second).T, dtype=np.float64) - return result - - -def green_lagrange_strain(cell_a: ArrayLike, cell_b: ArrayLike) -> FloatArray: - """Return the Green--Lagrange strain tensor for A -> B.""" - - gradient = deformation_gradient(cell_a, cell_b) - return 0.5 * (gradient.T @ gradient - np.eye(3)) - - -def map_cartesian_between_cells( - cartesian: ArrayLike, source_cell: ArrayLike, target_cell: ArrayLike -) -> FloatArray: - """Map Cartesian points between cells while preserving fractional position.""" - - return fractional_to_cartesian(cartesian_to_fractional(cartesian, source_cell), target_cell) - - def _nearest_image_single( delta: FloatArray, cell: FloatArray, @@ -263,23 +183,6 @@ def _minimum_image_fractional_displacement( return cast(FloatArray, result.reshape(first.shape)) -def minimum_image_fractional_displacement( - fractional_a: ArrayLike, - fractional_b: ArrayLike, - cell: ArrayLike, - pbc: Sequence[bool] | bool = (True, True, True), -) -> FloatArray: - """Return nearest-image fractional displacement ``b - a``. - - Leading dimensions are broadcast following NumPy rules. - """ - - first, second = _fractional_coordinate_pair(fractional_a, fractional_b) - lattice, sigma_min = _validated_cell(cell) - periodic = as_pbc(pbc) - return _minimum_image_fractional_displacement(first, second, lattice, periodic, sigma_min) - - def minimum_image_displacement( fractional_a: ArrayLike, fractional_b: ArrayLike, @@ -297,20 +200,6 @@ def minimum_image_displacement( return displacement @ lattice -def periodic_distance( - fractional_a: ArrayLike, - fractional_b: ArrayLike, - cell: ArrayLike, - pbc: Sequence[bool] | bool = (True, True, True), -) -> float | FloatArray: - """Return nearest-image distance(s) between fractional positions.""" - - result = np.linalg.norm( - minimum_image_displacement(fractional_a, fractional_b, cell, pbc), axis=-1 - ) - return float(result) if result.ndim == 0 else result - - def deformation_aware_displacement( fractional_a: ArrayLike, fractional_b: ArrayLike, @@ -355,58 +244,6 @@ def deformation_aware_distance( return float(result) if result.ndim == 0 else result -def pairwise_periodic_distances( - fractional_a: ArrayLike, - fractional_b: ArrayLike, - cell: ArrayLike, - pbc: Sequence[bool] | bool = (True, True, True), -) -> FloatArray: - """Return an ``(len(a), len(b))`` nearest-image distance matrix.""" - - first = np.asarray(fractional_a, dtype=float) - second = np.asarray(fractional_b, dtype=float) - if first.ndim != 2 or first.shape[1] != 3: - raise ValueError("fractional_a must have shape (n, 3)") - if second.ndim != 2 or second.shape[1] != 3: - raise ValueError("fractional_b must have shape (m, 3)") - if not np.all(np.isfinite(first)) or not np.all(np.isfinite(second)): - raise ValueError("coordinates must contain only finite values") - lattice, sigma_min = _validated_cell(cell) - periodic = as_pbc(pbc) - result = np.empty((len(first), len(second)), dtype=float) - # Keep peak working memory O(len(second)) instead of materializing a full - # (n, m, 3) displacement tensor. Cell factorization remains outside the - # row loop, so this does not restore the former O(n*m) SVD overhead. - for row, point in enumerate(first): - row_points = np.broadcast_to(point, second.shape) - fractional = _minimum_image_fractional_displacement( - row_points, - second, - lattice, - periodic, - sigma_min, - ) - result[row] = np.linalg.norm(fractional @ lattice, axis=-1) - return result - - -def pairwise_deformation_aware_distances( - fractional_a: ArrayLike, - fractional_b: ArrayLike, - cell_a: ArrayLike, - cell_b: ArrayLike, - pbc: Sequence[bool] | bool = (True, True, True), -) -> FloatArray: - """Pairwise distances in the symmetric mean metric of two cells.""" - - return pairwise_periodic_distances( - fractional_a, - fractional_b, - metric_average_cell(cell_a, cell_b), - pbc, - ) - - def periodic_voxel_overlap( voxels_a: Iterable[Sequence[int]], voxels_b: Iterable[Sequence[int]], @@ -458,45 +295,3 @@ def normalize( if denominator == 0: return 1.0 if not first and not second else 0.0 return float(intersection / denominator) - - -def periodic_fractional_centroid( - fractional: ArrayLike, - weights: ArrayLike | None = None, - pbc: Sequence[bool] | bool = (True, True, True), -) -> FloatArray: - """Return a boundary-safe centroid using circular means on periodic axes.""" - - points = np.asarray(fractional, dtype=float) - if points.ndim != 2 or points.shape[1] != 3 or len(points) == 0: - raise ValueError("fractional coordinates must have nonempty shape (n, 3)") - if not np.all(np.isfinite(points)): - raise ValueError("coordinates must contain only finite values") - if weights is None: - weight = np.ones(len(points), dtype=float) - else: - weight = np.asarray(weights, dtype=float) - if weight.shape != (len(points),): - raise ValueError("weights must have shape (n,)") - if np.any(weight < 0) or not np.all(np.isfinite(weight)): - raise ValueError("weights must be finite and non-negative") - total = float(weight.sum()) - if total <= 0: - raise ValueError("at least one weight must be positive") - periodic = as_pbc(pbc) - centroid = np.empty(3, dtype=float) - for axis, enabled in enumerate(periodic): - if not enabled: - centroid[axis] = float(np.average(points[:, axis], weights=weight)) - continue - angles = 2.0 * np.pi * points[:, axis] - vector = np.sum(weight * np.exp(1j * angles)) - if abs(vector) <= 1.0e-12 * total: - # The circular mean is undefined for a symmetric distribution. - # Use a deterministic unwrapped mean around the first point. - delta = points[:, axis] - points[0, axis] - delta -= np.rint(delta) - centroid[axis] = (points[0, axis] + np.average(delta, weights=weight)) % 1.0 - else: - centroid[axis] = (np.angle(vector) / (2.0 * np.pi)) % 1.0 - return centroid diff --git a/src/electrace/io/__init__.py b/src/electrace/io/__init__.py index 48c6101..77b38ba 100644 --- a/src/electrace/io/__init__.py +++ b/src/electrace/io/__init__.py @@ -17,18 +17,15 @@ SeriesKind, discover_neb, discover_path, - discover_pressure_series, discover_static_series, - iter_structure_directories, pressure_from_label, pressure_from_outcar, ) -from .loader import DescriptorSource, load_path, load_sequence +from .loader import load_path, load_sequence from .vasp import ( PoscarData, VaspParseError, parse_poscar, - poscar_to_frame, read_poscar, read_vasp_energy, ) @@ -38,7 +35,6 @@ "SCHEMA_VERSION", "DescriptorDocument", "DescriptorError", - "DescriptorSource", "DiscoveredPath", "DiscoveryError", "FrameSource", @@ -49,15 +45,12 @@ "descriptor_from_dict", "discover_neb", "discover_path", - "discover_pressure_series", "discover_static_series", "dump_descriptor", - "iter_structure_directories", "load_descriptor", "load_path", "load_sequence", "parse_poscar", - "poscar_to_frame", "pressure_from_label", "pressure_from_outcar", "read_poscar", diff --git a/src/electrace/io/descriptors.py b/src/electrace/io/descriptors.py index d3ce80a..08de604 100644 --- a/src/electrace/io/descriptors.py +++ b/src/electrace/io/descriptors.py @@ -44,20 +44,6 @@ SCHEMA_VERSION = "electrace.features.v1" MAX_DESCRIPTOR_BYTES = 64 * 1024 * 1024 -_ROOT_FIELDS = {"schema_version", "producer", "frame", "features", "metadata"} -_PRODUCER_FIELDS = {"name", "version"} -_FRAME_FIELDS = {"index", "label"} -_FEATURE_FIELDS = { - "id", - "kind", - "fractional_position", - "charge", - "volume", - "neighbor_atom_indices", - "voxel_indices", - "attributes", -} - class DescriptorError(ValueError): """Raised when a plugin descriptor is malformed or ambiguous.""" @@ -88,15 +74,6 @@ def _read_descriptor_text(path: Path) -> str: ) from exc -def _object_without_duplicates(pairs: list[tuple[str, Any]]) -> dict[str, Any]: - result: dict[str, Any] = {} - for key, value in pairs: - if key in result: - raise DescriptorError(f"descriptor JSON contains duplicate object key {key!r}") - result[key] = value - return result - - @dataclass(frozen=True, slots=True) class DescriptorDocument: """Validated normalized features from one producer for one optional frame.""" @@ -119,11 +96,8 @@ def __post_init__(self) -> None: identifiers = [feature.id for feature in self.features] if len(identifiers) != len(set(identifiers)): raise DescriptorError("feature ids must be unique within a descriptor document") - for feature in self.features: - _ensure_json(feature.attributes, f"feature {feature.id!r} attributes") if self.frame_index is not None and self.frame_index < 0: raise DescriptorError("descriptor frame index must be non-negative") - _ensure_json(self.metadata, "descriptor metadata") convention = self.metadata.get("charge_convention") if convention is not None and (not isinstance(convention, str) or not convention.strip()): raise DescriptorError("metadata.charge_convention must be a non-empty string") @@ -131,12 +105,6 @@ def __post_init__(self) -> None: raise DescriptorError( "descriptor documents containing charge must declare metadata.charge_convention" ) - warnings = self.metadata.get("warnings") - if warnings is not None and ( - not isinstance(warnings, list) - or any(not isinstance(warning, str) for warning in warnings) - ): - raise DescriptorError("metadata.warnings must be an array of strings") def to_dict(self) -> dict[str, Any]: producer: dict[str, Any] = {"name": self.producer} @@ -150,7 +118,7 @@ def to_dict(self) -> dict[str, Any]: result: dict[str, Any] = { "schema_version": self.schema_version, "producer": producer, - "features": [_feature_to_dict(feature) for feature in self.features], + "features": [feature.to_dict() for feature in self.features], "metadata": self.metadata, } if frame: @@ -158,13 +126,6 @@ def to_dict(self) -> dict[str, Any]: return result -def _ensure_json(value: Any, context: str) -> None: - try: - json.dumps(value, allow_nan=False) - except (TypeError, ValueError, RecursionError) as exc: - raise DescriptorError(f"{context} must contain only finite JSON values") from exc - - def _finite(value: Any, context: str) -> float: if isinstance(value, bool): raise DescriptorError(f"{context} must be a number, not boolean") @@ -198,78 +159,17 @@ def _neighbors(value: Any, context: str) -> tuple[int, ...]: return tuple(result) -def _voxels(value: Any, context: str) -> tuple[tuple[int, int, int], ...] | None: - if value is None: - return None - if not isinstance(value, Sequence) or isinstance(value, (str, bytes)): - raise DescriptorError(f"{context} must be an array of integer triplets") - result: list[tuple[int, int, int]] = [] - for index, item in enumerate(value): - if not isinstance(item, Sequence) or isinstance(item, (str, bytes)) or len(item) != 3: - raise DescriptorError(f"{context}[{index}] must contain exactly three integers") - if any(isinstance(component, bool) or not isinstance(component, int) for component in item): - raise DescriptorError(f"{context}[{index}] must contain exactly three integers") - result.append(tuple(item)) - if len(result) != len(set(result)): - raise DescriptorError(f"{context} must not contain duplicates") - return tuple(result) - - -def _feature_to_dict(feature: Feature) -> dict[str, Any]: - result: dict[str, Any] = { - "id": feature.id, - "kind": feature.kind.value if hasattr(feature.kind, "value") else str(feature.kind), - "fractional_position": list(feature.fractional_position), - "charge": feature.charge, - "volume": feature.volume, - "neighbor_atom_indices": list(feature.neighbor_atom_indices), - "attributes": dict(feature.attributes), - } - if feature.voxel_indices is not None: - result["voxel_indices"] = [list(voxel) for voxel in feature.voxel_indices] - return result - - def _feature_from_mapping(item: Any, index: int) -> Feature: if not isinstance(item, Mapping): raise DescriptorError(f"features[{index}] must be an object") - unknown = set(item) - _FEATURE_FIELDS - if unknown: - raise DescriptorError( - f"features[{index}] contains unknown fields: {', '.join(sorted(unknown))}" - ) - identifier = item.get("id") - kind = item.get("kind") - if not isinstance(identifier, str) or not identifier.strip(): + if not isinstance(item.get("id"), str) or not str(item["id"]).strip(): raise DescriptorError(f"features[{index}].id must be a non-empty string") - if not isinstance(kind, str) or not kind.strip(): + if not isinstance(item.get("kind"), str) or not str(item["kind"]).strip(): raise DescriptorError(f"features[{index}].kind must be a non-empty string") - if "fractional_position" not in item: - raise DescriptorError(f"features[{index}].fractional_position is required") - charge = item.get("charge") - volume = item.get("volume") - parsed_charge = None if charge is None else _finite(charge, f"features[{index}].charge") - parsed_volume = None if volume is None else _finite(volume, f"features[{index}].volume") - if parsed_volume is not None and parsed_volume < 0: - raise DescriptorError(f"features[{index}].volume must be non-negative") - attributes = item.get("attributes", {}) - if not isinstance(attributes, Mapping): - raise DescriptorError(f"features[{index}].attributes must be an object") - _ensure_json(attributes, f"features[{index}].attributes") - return Feature( - id=identifier, - kind=kind, - fractional_position=_position( - item["fractional_position"], f"features[{index}].fractional_position" - ), - charge=parsed_charge, - volume=parsed_volume, - neighbor_atom_indices=_neighbors( - item.get("neighbor_atom_indices", ()), f"features[{index}].neighbor_atom_indices" - ), - attributes=dict(attributes), - voxel_indices=_voxels(item.get("voxel_indices"), f"features[{index}].voxel_indices"), - ) + try: + return Feature.from_dict(item) + except (KeyError, TypeError, ValueError) as exc: + raise DescriptorError(f"invalid features[{index}]: {exc}") from exc def descriptor_from_dict(payload: Any) -> DescriptorDocument: @@ -277,39 +177,24 @@ def descriptor_from_dict(payload: Any) -> DescriptorDocument: if not isinstance(payload, Mapping): raise DescriptorError("descriptor root must be a JSON object") - unknown = set(payload) - _ROOT_FIELDS - if unknown: - raise DescriptorError( - f"descriptor contains unknown root fields: {', '.join(sorted(unknown))}" - ) - schema = payload.get("schema_version") - if schema != SCHEMA_VERSION: + if payload.get("schema_version") != SCHEMA_VERSION: raise DescriptorError(f"schema_version must be exactly {SCHEMA_VERSION!r}") producer = payload.get("producer") - if not isinstance(producer, Mapping) or not isinstance(producer.get("name"), str): + if ( + not isinstance(producer, Mapping) + or not isinstance(producer.get("name"), str) + or not producer["name"].strip() + ): raise DescriptorError("producer must be an object with a non-empty name") - unknown_producer_fields = set(producer) - _PRODUCER_FIELDS - if unknown_producer_fields: - raise DescriptorError( - "producer contains unknown fields: " + ", ".join(sorted(unknown_producer_fields)) - ) features_payload = payload.get("features") if not isinstance(features_payload, list): raise DescriptorError("features must be an array") features = tuple( _feature_from_mapping(item, index) for index, item in enumerate(features_payload) ) - identifiers = [feature.id for feature in features] - if len(identifiers) != len(set(identifiers)): - raise DescriptorError("feature ids must be unique") frame = payload.get("frame", {}) if not isinstance(frame, Mapping): raise DescriptorError("frame must be an object when present") - unknown_frame_fields = set(frame) - _FRAME_FIELDS - if unknown_frame_fields: - raise DescriptorError( - f"frame contains unknown fields: {', '.join(sorted(unknown_frame_fields))}" - ) frame_index = frame.get("index") if frame_index is not None and ( isinstance(frame_index, bool) or not isinstance(frame_index, int) @@ -334,26 +219,6 @@ def descriptor_from_dict(payload: Any) -> DescriptorDocument: ) -def _first_mapping_value(payload: Mapping[str, Any], names: Iterable[str]) -> Any: - for name in names: - if name in payload: - return payload[name] - for container_name in ( - "maxima_results", - "basin_results", - "atom_results", - "nna_results", - "results", - "data", - ): - nested = payload.get(container_name) - if isinstance(nested, Mapping): - for name in names: - if name in nested: - return nested[name] - return None - - def _frame_position( frame: Frame | None, atom_index: int, context: str ) -> tuple[float, float, float]: @@ -392,7 +257,7 @@ def _baderkit_structure_positions(payload: Mapping[str, Any]) -> Sequence[Any] | for site in sites: if not isinstance(site, Mapping): return None - position = site.get("abc", site.get("frac_coords")) + position = site.get("abc") if position is None: return None positions.append(position) @@ -402,17 +267,17 @@ def _baderkit_structure_positions(payload: Mapping[str, Any]) -> Sequence[Any] | def _parse_baderkit( payload: Mapping[str, Any], frame: Frame | None, *, native_format: str = "bader.json" ) -> DescriptorDocument: - positions = _first_mapping_value(payload, ("maxima_frac", "basin_frac", "attractor_frac")) - charges = _first_mapping_value(payload, ("basin_charges", "maxima_charges")) - volumes = _first_mapping_value(payload, ("basin_volumes", "maxima_volumes")) - basin_atoms = _first_mapping_value(payload, ("basin_atoms", "atom_indices")) + atom_results = payload.get("atom_results", {}) + maxima_results = payload.get("maxima_results", {}) + if not isinstance(atom_results, Mapping) or not isinstance(maxima_results, Mapping): + raise DescriptorError("BaderKit JSON atom_results and maxima_results must be objects") + positions = maxima_results.get("maxima_frac") + charges = maxima_results.get("basin_charges") + volumes = maxima_results.get("basin_volumes") + basin_atoms = atom_results.get("basin_atoms") known_atom_indices = {atom.index for atom in frame.atoms} if frame is not None else None features: list[Feature] = [] if isinstance(positions, Sequence) and not isinstance(positions, (str, bytes)): - is_elf = ( - native_format.lower() == "badelf.json" - or "elf" in str(payload.get("method", payload.get("analysis_type", ""))).lower() - ) for index, raw_position in enumerate(positions): assigned = _series_item(basin_atoms, index) atom_index = ( @@ -423,9 +288,7 @@ def _parse_baderkit( is_nna = atom_index is None or ( known_atom_indices is not None and atom_index not in known_atom_indices ) - kind = ( - "non_nuclear_attractor" if is_nna else ("elf_basin" if is_elf else "atomic_basin") - ) + kind = "non_nuclear_attractor" if is_nna else "atomic_basin" charge = _series_item(charges, index) volume = _series_item(volumes, index) features.append( @@ -451,18 +314,16 @@ def _parse_baderkit( ) ) - oxidation_states = _first_mapping_value(payload, ("oxidation_states",)) - atom_charges = _first_mapping_value(payload, ("atom_charges",)) + oxidation_states = atom_results.get("oxidation_states") + atom_charges = atom_results.get("atom_charges") charge_definition = "bader_electron_population" if atom_charges is None: atom_charges = oxidation_states charge_definition = "bader_oxidation_state" - atom_volumes = _first_mapping_value(payload, ("atom_volumes",)) - species = _first_mapping_value(payload, ("species",)) - maxima_elf_values = _first_mapping_value(payload, ("maxima_elf_values",)) - is_badelf = native_format.lower() == "badelf.json" or isinstance( - payload.get("nna_results"), Mapping - ) + atom_volumes = atom_results.get("atom_volumes") + species = atom_results.get("species") + maxima_elf_values = atom_results.get("maxima_elf_values") + is_badelf = native_format.lower() == "badelf.json" structure_positions = _baderkit_structure_positions(payload) if is_badelf else None atom_count = len(frame.atoms) if frame is not None else None if isinstance(atom_charges, Sequence) and not isinstance(atom_charges, (str, bytes)): @@ -542,15 +403,6 @@ def _parse_baderkit( def _critic2_kind(item: Mapping[str, Any]) -> str: - name = str(item.get("kind", item.get("type", item.get("name", "")))).lower().replace("-", "_") - if "bond" in name or name in {"bcp", "(3,_1)", "3,-1"}: - return "bond_critical_point" - if "ring" in name or name in {"rcp", "(3,+1)", "3,1"}: - return "ring_critical_point" - if "cage" in name or name in {"ccp", "(3,+3)", "3,3"}: - return "cage_critical_point" - if "non_nuclear" in name or name in {"nna", "nnm"}: - return "non_nuclear_attractor" signature = item.get("signature") if signature == -3 and item.get("is_nucleus") is False: return "non_nuclear_attractor" @@ -565,7 +417,7 @@ def _critic2_kind(item: Mapping[str, Any]) -> str: def _critic2_cpreport_records( payload: Mapping[str, Any], -) -> tuple[Sequence[Any], Mapping[int, Mapping[str, Any]], int | None] | None: +) -> tuple[Sequence[Any], Mapping[int, Mapping[str, Any]], int | None]: """Return records and joins from critic2's native ``CPREPORT *.json``. The upstream format nests ``cell_cps`` and ``nonequivalent_cps`` below a @@ -576,16 +428,11 @@ def _critic2_cpreport_records( critical_points = payload.get("critical_points") if not isinstance(critical_points, Mapping): - return None + raise DescriptorError("critic2 CPREPORT JSON requires a critical_points object") cell_records = critical_points.get("cell_cps") nonequivalent = critical_points.get("nonequivalent_cps") if not isinstance(cell_records, Sequence) or isinstance(cell_records, (str, bytes)): - if not isinstance(nonequivalent, Sequence) or isinstance(nonequivalent, (str, bytes)): - raise DescriptorError( - "critic2 CPREPORT JSON must contain critical_points.cell_cps or " - "critical_points.nonequivalent_cps" - ) - cell_records = nonequivalent + raise DescriptorError("critic2 CPREPORT JSON requires critical_points.cell_cps") joined: dict[int, Mapping[str, Any]] = {} if isinstance(nonequivalent, Sequence) and not isinstance(nonequivalent, (str, bytes)): for index, record in enumerate(nonequivalent): @@ -656,19 +503,7 @@ def _critic2_hessian_trace(value: Any) -> float | None: def _parse_critic2(payload: Mapping[str, Any]) -> DescriptorDocument: - cpreport = _critic2_cpreport_records(payload) - joined: Mapping[int, Mapping[str, Any]] = {} - atom_count: int | None = None - if cpreport is None: - records = _first_mapping_value( - payload, ("critical_points", "criticalpoints", "cps", "attractors") - ) - native_format = "json" - else: - records, joined, atom_count = cpreport - native_format = "cpreport.json" - if not isinstance(records, Sequence) or isinstance(records, (str, bytes)): - raise DescriptorError("critic2 JSON must contain a critical_points array") + records, joined, atom_count = _critic2_cpreport_records(payload) features: list[Feature] = [] for index, record in enumerate(records): if not isinstance(record, Mapping): @@ -683,55 +518,32 @@ def _parse_critic2(payload: Mapping[str, Any]) -> DescriptorDocument: # definition. Explicit cell-record values are more specific and # intentionally override the shared definition when both are present. combined = {**detail, **record} - raw_position = record.get( - "fractional_position", - record.get( - "fractional", - record.get( - "frac", - record.get("fractional_coordinates", detail.get("fractional_coordinates")), - ), - ), - ) + raw_position = record.get("fractional_coordinates", detail.get("fractional_coordinates")) if raw_position is None: raise DescriptorError(f"critic2 critical point {index} has no fractional coordinates") if combined.get("signature") == -3 and combined.get("is_nucleus") is True: continue - neighbors = record.get("neighbor_atom_indices", record.get("atoms")) - if neighbors is None: - neighbors = _critic2_attractor_atoms(record, atom_count) + neighbors = _critic2_attractor_atoms(record, atom_count) attributes = { str(key): value for key, value in combined.items() if key not in { "id", - "kind", - "type", "name", - "fractional_position", - "fractional", - "frac", "fractional_coordinates", "cartesian_coordinates", "charge", "volume", - "neighbor_atom_indices", - "atoms", } } if "laplacian" not in attributes: laplacian = _critic2_hessian_trace(attributes.get("hessian")) if laplacian is not None: attributes["laplacian"] = laplacian - _ensure_json(attributes, f"critic2 critical point {index} attributes") features.append( Feature( - id=( - f"critic2:cp:{record['id']}" - if cpreport is not None and "id" in record - else str(record.get("id", f"critic2:cp:{index}")) - ), + id=f"critic2:cp:{record.get('id', index + 1)}", kind=_critic2_kind(combined), fractional_position=_position( raw_position, f"critic2 critical point {index} position" @@ -753,7 +565,7 @@ def _parse_critic2(payload: Mapping[str, Any]) -> DescriptorDocument: ) ) metadata: dict[str, Any] = { - "native_format": native_format, + "native_format": "cpreport.json", **({"units": payload["units"]} if isinstance(payload.get("units"), str) else {}), } payload_metadata = payload.get("metadata") @@ -774,7 +586,7 @@ def _parse_critic2(payload: Mapping[str, Any]) -> DescriptorDocument: _INTEGER_FIELD = re.compile(r"[+-]?\d+\Z") -def _atom_index(value: Any, context: str, *, index_base: int = 1) -> int: +def _atom_index(value: Any, context: str) -> int: if isinstance(value, bool): raise DescriptorError(f"{context} is not an atom index") if isinstance(value, int): @@ -786,9 +598,9 @@ def _atom_index(value: Any, context: str, *, index_base: int = 1) -> int: raw = int(match.group(1)) else: raise DescriptorError(f"cannot read atom index from {value!r} in {context}") - result = raw - index_base + result = raw - 1 if result < 0: - raise DescriptorError(f"{context} must use {index_base}-based positive atom labels") + raise DescriptorError(f"{context} must use one-based positive atom labels") return result @@ -815,204 +627,11 @@ def _periodic_midpoint( return tuple((left[axis] + 0.5 * displacement[axis]) % 1.0 for axis in range(3)) # type: ignore[return-value] -def _lobster_json_translation( - record: Mapping[str, Any], context: str -) -> tuple[int, int, int] | None: - translations: list[tuple[int, int, int]] = [] - for field_name in ("translation", "cell_translation"): - if field_name not in record: - continue - raw = record[field_name] - if ( - not isinstance(raw, Sequence) - or isinstance(raw, (str, bytes)) - or len(raw) != 3 - or any(type(component) is not int for component in raw) - ): - raise DescriptorError(f"{context} {field_name} must contain exactly three integers") - translations.append((int(raw[0]), int(raw[1]), int(raw[2]))) - if len(translations) == 2 and translations[0] != translations[1]: - raise DescriptorError(f"{context} has conflicting translation and cell_translation") - return translations[0] if translations else None - - -def _parse_lobster_json(payload: Mapping[str, Any], frame: Frame | None) -> DescriptorDocument: - records = _first_mapping_value(payload, ("bonds", "icohp", "icohps", "icobi", "icobis")) - if not isinstance(records, Sequence) or isinstance(records, (str, bytes)): - raise DescriptorError("LOBSTER JSON must contain a bonds/ICOHP/ICOBI array") - if frame is None: - raise DescriptorError( - "LOBSTER bond descriptors require a Frame for periodic bond midpoints" - ) - raw_index_base = payload.get("atom_index_base", 1) - if isinstance(raw_index_base, bool) or raw_index_base not in {0, 1}: - raise DescriptorError("LOBSTER atom_index_base must be 0 or 1") - index_base = int(raw_index_base) - features: list[Feature] = [] - for index, record in enumerate(records): - if not isinstance(record, Mapping): - raise DescriptorError(f"LOBSTER bond {index} must be an object") - first = _atom_index( - record.get("atom1", record.get("from")), - f"LOBSTER bond {index} atom1", - index_base=index_base, - ) - second = _atom_index( - record.get("atom2", record.get("to")), - f"LOBSTER bond {index} atom2", - index_base=index_base, - ) - attribute_aliases = { - "icohp": "icohp_ev", - "icohp_ev": "icohp_ev", - "icobi": "icobi", - "icoop": "icoop", - "distance": "bond_length_angstrom", - "bond_length_angstrom": "bond_length_angstrom", - } - attributes: dict[str, Any] = { - normalized: _finite(record[source], f"LOBSTER bond {index} {source}") - for source, normalized in attribute_aliases.items() - if source in record - } - if "icohp" in record: - attributes["icohp"] = _finite(record["icohp"], f"LOBSTER bond {index} icohp") - if "distance" in record: - attributes["distance"] = _finite(record["distance"], f"LOBSTER bond {index} distance") - families = tuple( - family - for family, keys in ( - ("icohp", ("icohp", "icohp_ev")), - ("icobi", ("icobi",)), - ("icoop", ("icoop",)), - ) - if any(key in record for key in keys) - ) - if not families: - raise DescriptorError( - f"LOBSTER bond {index} has no recognized ICOHP, ICOBI, or ICOOP value" - ) - attributes["descriptor_family"] = "+".join(families) - lattice_translation = _lobster_json_translation(record, f"LOBSTER bond {index}") - if lattice_translation is not None: - attributes["lattice_translation"] = list(lattice_translation) - attributes["midpoint_convention"] = "explicit_atom2_lattice_translation" - features.append( - Feature( - id=str(record.get("id", f"lobster:bond:{index}")), - kind="bond_descriptor", - fractional_position=_periodic_midpoint( - frame, - first, - second, - lattice_translation=lattice_translation, - ), - neighbor_atom_indices=(first, second), - attributes=attributes, - ) - ) - return DescriptorDocument( - producer="lobster", features=tuple(features), metadata={"native_format": "json"} - ) - - -def _parse_charge_json( - payload: Mapping[str, Any], frame: Frame | None, producer: str -) -> DescriptorDocument: - records = _first_mapping_value(payload, ("atoms", "charges", "atomic_charges")) - if not isinstance(records, Sequence) or isinstance(records, (str, bytes)): - raise DescriptorError(f"{producer} JSON must contain an atoms/charges array") - raw_index_base = payload.get("atom_index_base", 0) - if isinstance(raw_index_base, bool) or raw_index_base not in {0, 1}: - raise DescriptorError(f"{producer} atom_index_base must be 0 or 1") - index_base = int(raw_index_base) - features: list[Feature] = [] - for index, record in enumerate(records): - if isinstance(record, Mapping): - raw_atom_index = record.get("index", index + index_base) - if isinstance(raw_atom_index, bool) or not isinstance(raw_atom_index, int): - raise DescriptorError(f"{producer} atom {index} index must be an integer") - atom_index = raw_atom_index - index_base - if atom_index < 0: - raise DescriptorError(f"{producer} atom {index} index is below atom_index_base") - raw_charge = record.get("charge", record.get("net_charge", record.get("ddec_charge"))) - raw_position = record.get("fractional_position") - attributes = { - str(key): value - for key, value in record.items() - if key - not in {"index", "charge", "net_charge", "ddec_charge", "fractional_position"} - } - else: - atom_index = index - raw_charge = record - raw_position = None - attributes = {} - if raw_charge is None: - raise DescriptorError(f"{producer} atom {index} has no charge") - position = ( - _position(raw_position, f"{producer} atom {index} position") - if raw_position is not None - else _frame_position(frame, atom_index, f"{producer} atomic charges") - ) - features.append( - Feature( - id=f"{producer}:atom:{atom_index}", - kind="atomic_charge", - fractional_position=position, - charge=_finite(raw_charge, f"{producer} atom {index} charge"), - neighbor_atom_indices=(atom_index,), - attributes={**attributes, "charge_definition": "ddec_net_atomic_charge"}, - ) - ) - return DescriptorDocument( - producer=producer, - features=tuple(features), - metadata={ - "native_format": "json", - "charge_convention": ( - "DDEC net atomic charge in elementary-charge units; positive values indicate electron deficiency." - if producer == "chargemol" - else f"{producer} atomic charge; see feature attributes for convention." - ), - }, - ) - - -def _infer_json_producer(payload: Mapping[str, Any], path: Path) -> str | None: - producer = payload.get("producer") - if isinstance(producer, str): - return producer.lower() - if isinstance(producer, Mapping): - producer_name = producer.get("name") - if isinstance(producer_name, str): - return producer_name.lower() - name = path.name.lower() - if ( - name in {"bader.json", "badelf.json"} - or any(key in payload for key in ("maxima_frac", "basin_charges", "atom_volumes")) - or ( - "atom_results" in payload - and any(key in payload for key in ("maxima_results", "nna_results")) - ) - ): - return "baderkit" - if "critic" in name or any( - key in payload for key in ("critical_points", "criticalpoints", "cps") - ): - return "critic2" - if "lobster" in name or any(key in payload for key in ("bonds", "icohp", "icobi")): - return "lobster" - if "ddec" in name or "chargemol" in name: - return "chargemol" - return None - - def _parse_lobster_list(path: Path, frame: Frame | None) -> DescriptorDocument: if frame is None: raise DescriptorError("LOBSTER list files require a Frame for periodic bond midpoints") lower_name = path.name.lower() - family = "icobi" if "icobi" in lower_name else "icoop" if "icoop" in lower_name else "icohp" + family = "icobi" if "icobi" in lower_name else "icohp" descriptor = "icohp_ev" if family == "icohp" else family features: list[Feature] = [] for line_number, line in enumerate(_read_descriptor_text(path).splitlines(), start=1): @@ -1158,20 +777,17 @@ def load_descriptor( ) -> DescriptorDocument: """Load a canonical or supported native plugin descriptor. - JSON carrying ``schema_version=electrace.features.v1`` is validated - strictly. Native JSON dispatch can be made deterministic with ``producer`` - (``baderkit``, ``critic2``, ``lobster``, or ``chargemol``). LOBSTER lists, - ACF.dat, and Chargemol charge XYZ files are recognized by filename. All - supported formats must be UTF-8 and are byte-limited while being read. + JSON carrying ``schema_version=electrace.features.v1`` is normalized as the + canonical format. Official BaderKit ``bader.json``/``badelf.json`` and + critic2 CPREPORT JSON are also accepted. LOBSTER lists, ACF.dat, and + Chargemol charge XYZ files are recognized by filename. """ source = Path(path).expanduser().resolve() if not source.is_file(): raise DescriptorError(f"descriptor file does not exist: {source}") - if source.stat().st_size > MAX_DESCRIPTOR_BYTES: - raise _descriptor_size_error(source) lower_name = source.name.lower() - if any(name in lower_name for name in ("icohplist", "icobilist", "icooplist")): + if any(name in lower_name for name in ("icohplist", "icobilist")): return _parse_lobster_list(source, frame) if source.name.upper() == "ACF.DAT": return _parse_atomic_charge_text(source, frame, "bader") @@ -1180,26 +796,26 @@ def load_descriptor( if source.suffix.lower() != ".json": raise DescriptorError(f"unsupported descriptor format: {source.name}") try: - payload = json.loads( - _read_descriptor_text(source), object_pairs_hook=_object_without_duplicates - ) + payload = json.loads(_read_descriptor_text(source)) except (json.JSONDecodeError, RecursionError) as exc: raise DescriptorError(f"cannot parse descriptor JSON {source}: {exc}") from exc if not isinstance(payload, Mapping): raise DescriptorError("descriptor JSON root must be an object") if "schema_version" in payload: return descriptor_from_dict(payload) - selected = (producer or _infer_json_producer(payload, source) or "").lower() - if selected in {"baderkit", "bader"}: + selected = (producer or "").lower() + if not selected: + if lower_name in {"bader.json", "badelf.json"}: + selected = "baderkit" + elif "critic" in lower_name or isinstance(payload.get("critical_points"), Mapping): + selected = "critic2" + if selected == "baderkit": return _parse_baderkit(payload, frame, native_format=source.name) if selected == "critic2": return _parse_critic2(payload) - if selected in {"lobster", "lobsterpy"}: - return _parse_lobster_json(payload, frame) - if selected in {"chargemol", "ddec", "ddec6"}: - return _parse_charge_json(payload, frame, "chargemol") raise DescriptorError( - f"cannot infer native JSON producer for {source.name}; pass producer='baderkit', 'critic2', 'lobster', or 'chargemol'" + f"unsupported native JSON descriptor {source.name}; expected bader.json, " + "badelf.json, or critic2 CPREPORT JSON" ) @@ -1210,11 +826,9 @@ def dump_descriptor( indent: int = 2, overwrite: bool = False, ) -> Path: - """Atomically write descriptor JSON without following a leaf symlink.""" + """Write canonical descriptor JSON.""" destination = Path(path) - if destination.is_symlink(): - raise OSError(f"refusing to write descriptor through a symbolic link: {destination}") if destination.exists() and not overwrite: raise FileExistsError(f"refusing to replace existing descriptor: {destination}") payload = json.dumps(document.to_dict(), indent=indent, allow_nan=False) + "\n" @@ -1295,9 +909,7 @@ def attach_descriptors(frame: Frame, documents: Iterable[DescriptorDocument]) -> { "schema_version": document.schema_version, "producer": producer, - "metadata": json.loads( - json.dumps(document.metadata, allow_nan=False, sort_keys=True) - ), + "metadata": dict(document.metadata), } ) if sources: diff --git a/src/electrace/io/discovery.py b/src/electrace/io/discovery.py index 0ad2921..cf40cd2 100644 --- a/src/electrace/io/discovery.py +++ b/src/electrace/io/discovery.py @@ -9,7 +9,6 @@ import math import re -from collections.abc import Iterable from dataclasses import dataclass, field from enum import StrEnum from pathlib import Path @@ -342,24 +341,6 @@ def pressure_key(item: tuple[Path, Path, float | None]) -> tuple[float, object]: ) -def discover_pressure_series( - root: str | Path, - *, - prefer_contcar: bool = True, - structure_name: str = "CONTCAR", - fallback_structure_name: str = "POSCAR", -) -> DiscoveredPath: - """Discover a static series for which every frame has a unique pressure.""" - - return discover_static_series( - root, - prefer_contcar=prefer_contcar, - structure_name=structure_name, - fallback_structure_name=fallback_structure_name, - require_pressure=True, - ) - - def discover_path( root: str | Path, *, @@ -418,25 +399,3 @@ def discover_path( structure_name=structure_name, fallback_structure_name=fallback_structure_name, ) - - -def iter_structure_directories( - root: str | Path, - *, - prefer_contcar: bool = True, - structure_name: str = "CONTCAR", - fallback_structure_name: str = "POSCAR", -) -> Iterable[Path]: - """Yield immediate child directories containing a usable VASP structure.""" - - base = Path(root).expanduser().resolve() - if not base.is_dir(): - return - for child in sorted( - (item for item in base.iterdir() if item.is_dir()), key=lambda item: _natural_key(item.name) - ): - if ( - _structure_in(child, prefer_contcar, structure_name, fallback_structure_name) - is not None - ): - yield child diff --git a/src/electrace/io/loader.py b/src/electrace/io/loader.py index 882456e..4eef268 100644 --- a/src/electrace/io/loader.py +++ b/src/electrace/io/loader.py @@ -4,7 +4,6 @@ from collections.abc import Iterable from dataclasses import replace -from enum import StrEnum from pathlib import Path from electrace.models import Frame, Sequence @@ -20,40 +19,19 @@ "critic2.features.json", "ICOHPLIST.lobster", "ICOBILIST.lobster", - "ICOOPLIST.lobster", "DDEC6_even_tempered_net_atomic_charges.xyz", "ACF.dat", ) -class DescriptorSource(StrEnum): - """Where frame descriptors are selected during path loading. - - ``AUTO`` preserves the original API behavior: local frame directories are - searched when ``descriptor_dir`` is absent, otherwise only the external - descriptor location is searched. ``MERGE`` loads local descriptors first - and external descriptors second; duplicate feature IDs remain an error, - so this policy never silently overwrites evidence. - """ - - AUTO = "auto" - LOCAL = "local" - EXTERNAL = "external" - MERGE = "merge" +_DESCRIPTOR_SOURCES = {"auto", "local", "external", "merge"} def _descriptor_filename(value: str) -> str: if not isinstance(value, str): raise TypeError("descriptor_name must be a string") name = value.strip() - if ( - not name - or name in {".", ".."} - or any(ord(character) < 32 or ord(character) == 127 for character in name) - or "/" in name - or "\\" in name - or Path(name).is_absolute() - ): + if not name or Path(name).name != name or name in {".", ".."}: raise DescriptorError(f"descriptor_name must be a basename, not a path: {value!r}") return name @@ -68,26 +46,17 @@ def _descriptor_filenames(descriptor_name: str) -> tuple[str, ...]: def _descriptor_source( - value: str | DescriptorSource, + value: str, descriptor_root: Path | None, -) -> DescriptorSource: - try: - selected = ( - value - if isinstance(value, DescriptorSource) - else DescriptorSource(str(value).strip().lower()) - ) - except ValueError as exc: - choices = ", ".join(item.value for item in DescriptorSource) - raise DescriptorError( - f"unknown descriptor_source {value!r}; expected one of: {choices}" - ) from exc - if selected is DescriptorSource.AUTO: - return DescriptorSource.LOCAL if descriptor_root is None else DescriptorSource.EXTERNAL - if selected in {DescriptorSource.EXTERNAL, DescriptorSource.MERGE} and descriptor_root is None: - raise DescriptorError( - f"descriptor_dir is required for descriptor_source={selected.value!r}" - ) +) -> str: + selected = str(value).strip().lower() + if selected not in _DESCRIPTOR_SOURCES: + choices = ", ".join(sorted(_DESCRIPTOR_SOURCES)) + raise DescriptorError(f"unknown descriptor_source {value!r}; expected one of: {choices}") + if selected == "auto": + return "local" if descriptor_root is None else "external" + if selected in {"external", "merge"} and descriptor_root is None: + raise DescriptorError(f"descriptor_dir is required for descriptor_source={selected!r}") return selected @@ -110,16 +79,14 @@ def _descriptor_candidates( *, frame_count: int, descriptor_name: str, - descriptor_source: str | DescriptorSource, + descriptor_source: str, ) -> tuple[Path, ...]: filenames = _descriptor_filenames(descriptor_name) selected = _descriptor_source(descriptor_source, descriptor_root) local = ( - [source.directory / name for name in filenames] - if selected in {DescriptorSource.LOCAL, DescriptorSource.MERGE} - else [] + [source.directory / name for name in filenames] if selected in {"local", "merge"} else [] ) - if selected is DescriptorSource.LOCAL: + if selected == "local": return _unique_paths(local) assert descriptor_root is not None if descriptor_root.is_file(): @@ -133,13 +100,9 @@ def _descriptor_candidates( raise DescriptorError(f"descriptor_dir is not a directory: {descriptor_root}") candidates: list[Path] = [] - # Flat descriptor stores: descriptors/00.json or descriptors/0.json. - for stem in dict.fromkeys((source.label, str(source.index), f"{source.index:02d}")): - candidates.append(descriptor_root / f"{stem}.json") - # Mirrored frame stores: descriptors/00/features.json (or native output). - for frame_name in dict.fromkeys((source.label, str(source.index), f"{source.index:02d}")): - directory = descriptor_root / frame_name - candidates.extend(directory / name for name in filenames) + candidates.append(descriptor_root / f"{source.label}.json") + directory = descriptor_root / source.label + candidates.extend(directory / name for name in filenames) return _unique_paths((*local, *candidates)) @@ -148,7 +111,7 @@ def _load_discovered( *, descriptor_dir: str | Path | None, descriptor_name: str, - descriptor_source: str | DescriptorSource, + descriptor_source: str, ) -> list[Frame]: descriptor_root = ( Path(descriptor_dir).expanduser().resolve() if descriptor_dir is not None else None @@ -160,7 +123,7 @@ def _load_discovered( "path_root": str(discovered.root), "frame_directory": str(source.directory), "series_kind": discovered.kind.value, - "descriptor_source_policy": selected_source.value, + "descriptor_source_policy": selected_source, "descriptor_sources": [], **source.metadata, } @@ -202,7 +165,7 @@ def load_path( structure_name: str = "CONTCAR", fallback_structure_name: str = "POSCAR", descriptor_name: str = "features.json", - descriptor_source: str | DescriptorSource = DescriptorSource.AUTO, + descriptor_source: str = "auto", ) -> list[Frame]: """Discover and load a VASP path into ordered core ``Frame`` objects. @@ -248,7 +211,7 @@ def load_sequence( structure_name: str = "CONTCAR", fallback_structure_name: str = "POSCAR", descriptor_name: str = "features.json", - descriptor_source: str | DescriptorSource = DescriptorSource.AUTO, + descriptor_source: str = "auto", ) -> Sequence: """Load a path and wrap it in ElecTrace's core ``Sequence`` model.""" diff --git a/src/electrace/io/vasp.py b/src/electrace/io/vasp.py index a0b4a83..71aadc7 100644 --- a/src/electrace/io/vasp.py +++ b/src/electrace/io/vasp.py @@ -236,54 +236,6 @@ def parse_poscar(path: str | Path) -> PoscarData: ) -def poscar_to_frame( - data: PoscarData, - *, - index: int = 0, - label: str = "", - energy: float | None = None, - metadata: dict[str, Any] | None = None, -) -> Frame: - """Convert parsed POSCAR data into ElecTrace's core ``Frame`` model.""" - - atoms = [] - for atom_index, (element, position, flags) in enumerate( - zip( - data.elements, - data.fractional_positions, - data.selective_dynamics, - strict=True, - ) - ): - attributes: dict[str, Any] = {} - if flags is not None: - attributes["selective_dynamics"] = flags - atoms.append( - Atom( - index=atom_index, - element=element, - fractional_position=position, - attributes=attributes, - ) - ) - frame_metadata: dict[str, Any] = { - "comment": data.comment, - "coordinate_mode": data.coordinate_mode, - "symbols_inferred": data.symbols_inferred, - } - if metadata: - frame_metadata.update(metadata) - return Frame( - index=index, - label=label or str(index), - cell=data.cell, - atoms=tuple(atoms), - features=(), - energy=energy, - metadata=frame_metadata, - ) - - def read_poscar( path: str | Path, *, @@ -295,13 +247,31 @@ def read_poscar( """Read a POSCAR/CONTCAR directly into an ElecTrace ``Frame``.""" source = Path(path) - frame_metadata = {"structure_source": str(source)} + data = parse_poscar(source) + frame_metadata: dict[str, Any] = { + "structure_source": str(source), + "comment": data.comment, + "coordinate_mode": data.coordinate_mode, + "symbols_inferred": data.symbols_inferred, + } if metadata: frame_metadata.update(metadata) - return poscar_to_frame( - parse_poscar(source), + atoms = tuple( + Atom( + index=atom_index, + element=element, + fractional_position=position, + attributes={"selective_dynamics": flags} if flags is not None else {}, + ) + for atom_index, (element, position, flags) in enumerate( + zip(data.elements, data.fractional_positions, data.selective_dynamics, strict=True) + ) + ) + return Frame( index=index, label=label or source.parent.name or source.name, + cell=data.cell, + atoms=atoms, energy=energy, metadata=frame_metadata, ) @@ -319,30 +289,15 @@ def read_vasp_energy(path: str | Path) -> float | None: source = Path(path) if not source.is_file(): return None - toten: float | None = None - e0: float | None = None - free: float | None = None + patterns = (_OSZICAR_E0, _OSZICAR_F) if source.name.upper() == "OSZICAR" else (_TOTEN,) + values: list[float | None] = [None] * len(patterns) try: with source.open("r", encoding="utf-8", errors="replace") as handle: for line in handle: - for pattern, target in ( - (_TOTEN, "toten"), - (_OSZICAR_E0, "e0"), - (_OSZICAR_F, "free"), - ): + for index, pattern in enumerate(patterns): match = pattern.search(line) - if match is None: - continue - value = _float(match.group(1), context="VASP energy") - if target == "toten": - toten = value - elif target == "e0": - e0 = value - else: - free = value + if match is not None: + values[index] = _float(match.group(1), context="VASP energy") except OSError: return None - if source.name.upper() == "OSZICAR": - return e0 if e0 is not None else free - # Be permissive when a caller passes content with a nonstandard filename. - return toten if toten is not None else (e0 if e0 is not None else free) + return next((value for value in values if value is not None), None) diff --git a/src/electrace/matching.py b/src/electrace/matching.py index 83d2a64..03860c4 100644 --- a/src/electrace/matching.py +++ b/src/electrace/matching.py @@ -14,7 +14,7 @@ from collections import Counter from collections.abc import Iterable, Mapping, Sequence from dataclasses import dataclass, field -from itertools import combinations, islice +from itertools import islice from typing import Any import numpy as np @@ -25,7 +25,7 @@ deformation_aware_distance, periodic_voxel_overlap, ) -from .models import Atom, Feature, Frame, MatchEdge, MatchRelation, MatchResult +from .models import Atom, Feature, Frame, MatchEdge, MatchResult FloatArray = npt.NDArray[np.float64] IntArray = npt.NDArray[np.int64] @@ -42,7 +42,6 @@ class AtomMatchConfig: element_mismatch_cost: float = 4.0 attribute_scales: dict[str, float] = field(default_factory=dict) attribute_weight: float = 0.25 - ambiguity_tolerance: float = 0.05 pbc: tuple[bool, bool, bool] = (True, True, True) def __post_init__(self) -> None: @@ -53,7 +52,6 @@ def __post_init__(self) -> None: self.element_mismatch_cost, "element_mismatch_cost" ) self.attribute_weight = _nonnegative(self.attribute_weight, "attribute_weight") - self.ambiguity_tolerance = _nonnegative(self.ambiguity_tolerance, "ambiguity_tolerance") self.attribute_scales = { str(key): _positive(value, f"attribute scale {key!r}") for key, value in self.attribute_scales.items() @@ -94,7 +92,6 @@ class FeatureMatchConfig: conservation_tolerance: float = 0.35 max_group_size: int = 6 min_split_merge_confidence: float = 0.15 - ambiguity_tolerance: float = 0.05 pbc: tuple[bool, bool, bool] = (True, True, True) def __post_init__(self) -> None: @@ -119,7 +116,6 @@ def __post_init__(self) -> None: "kind_mismatch_cost", "conservation_tolerance", "min_split_merge_confidence", - "ambiguity_tolerance", ): setattr(self, name, _nonnegative(getattr(self, name), name)) if self.min_split_merge_confidence > 1.0: @@ -316,181 +312,6 @@ def _augmented_unmatched_cost(pair_cost: FloatArray, unmatched_cost: float) -> F return augmented -def _assignment_snapshot( - assignment: set[tuple[int, int]], - pair_cost: FloatArray, - source_ids: Sequence[Any], - target_ids: Sequence[Any], -) -> dict[str, Any]: - rows, columns = pair_cost.shape - matches = sorted( - ( - { - "source_id": source_ids[row], - "target_id": target_ids[column], - } - for row, column in assignment - if row < rows and column < columns and math.isfinite(pair_cost[row, column]) - ), - key=lambda item: (str(item["source_id"]), str(item["target_id"])), - ) - matched_sources = { - row - for row, column in assignment - if row < rows and column < columns and math.isfinite(pair_cost[row, column]) - } - matched_targets = { - column - for row, column in assignment - if row < rows and column < columns and math.isfinite(pair_cost[row, column]) - } - return { - "matches": matches, - "unmatched_previous": [ - source_ids[row] - for row in sorted( - set(range(rows)) - matched_sources, key=lambda item: str(source_ids[item]) - ) - ], - "unmatched_current": [ - target_ids[column] - for column in sorted( - set(range(columns)) - matched_targets, - key=lambda item: str(target_ids[item]), - ) - ], - } - - -def _assignment_ambiguities( - pair_cost: FloatArray, - unmatched_cost: float, - matched: Sequence[tuple[int, int]], - unmatched_rows: set[int], - unmatched_columns: set[int], - optimal_total: float, - source_ids: Sequence[Any], - target_ids: Sequence[Any], - tolerance: float, -) -> tuple[list[dict[str, Any]], dict[tuple[int, int], dict[str, Any]]]: - """Find selected decisions with a complete near-optimal alternative assignment. - - Cheap local gaps screen the common, clearly separated case. Every reported - ambiguity is then verified by resolving the complete assignment with one - selected scientific decision forbidden. This avoids treating a local tie - as ambiguous when the rest of the global assignment resolves it. - """ - - rows, columns = pair_cost.shape - if rows == 0 or columns == 0: - return [], {} - augmented = _augmented_unmatched_cost(pair_cost, unmatched_cost) - decisions: dict[tuple[int, int], tuple[str, int | None, int | None]] = {} - candidates: set[tuple[int, int]] = set() - for row, column in matched: - coordinate = (row, column) - decisions[coordinate] = ("match", row, column) - local_gap = _ambiguity_gap(pair_cost, row, column) - unmatched_gap = 2.0 * unmatched_cost - float(pair_cost[row, column]) - if min(local_gap, unmatched_gap) <= tolerance: - candidates.add(coordinate) - for row in unmatched_rows: - coordinate = (row, columns + row) - decisions[coordinate] = ("unmatched_previous", row, None) - if np.isfinite(pair_cost[row]).any(): - candidates.add(coordinate) - for column in unmatched_columns: - coordinate = (rows + column, column) - decisions[coordinate] = ("unmatched_current", None, column) - if np.isfinite(pair_cost[:, column]).any(): - candidates.add(coordinate) - - records: dict[tuple[int, int], dict[str, Any]] = {} - unresolved = set(candidates) - while unresolved: - selected_coordinate = min(unresolved) - unresolved.remove(selected_coordinate) - alternative_cost = augmented.copy() - alternative_cost[selected_coordinate] = math.inf - try: - alternative_rows, alternative_columns = solve_assignment(alternative_cost) - except ValueError: - continue - alternative_assignment = { - (int(row), int(column)) - for row, column in zip( - alternative_rows, - alternative_columns, - strict=True, - ) - } - chosen_costs = [augmented[row, column] for row, column in alternative_assignment] - if any(not math.isfinite(float(value)) for value in chosen_costs): - continue - alternative_total = float(sum(float(value) for value in chosen_costs)) - gap = max(0.0, alternative_total - optimal_total) - if gap > tolerance + 1.0e-12: - continue - snapshot = _assignment_snapshot( - alternative_assignment, - pair_cost, - source_ids, - target_ids, - ) - changed = set(decisions) - alternative_assignment - affected_previous_set: set[Any] = set() - affected_current_set: set[Any] = set() - for changed_coordinate in changed: - _, affected_row, affected_column = decisions[changed_coordinate] - if affected_row is not None: - affected_previous_set.add(source_ids[affected_row]) - if affected_column is not None: - affected_current_set.add(target_ids[affected_column]) - affected_previous = sorted(affected_previous_set, key=str) - affected_current = sorted(affected_current_set, key=str) - for coordinate in sorted(changed): - decision, decision_row, decision_column = decisions[coordinate] - selected: dict[str, Any] = {} - if decision_row is not None: - selected["source_id"] = source_ids[decision_row] - if decision_column is not None: - selected["target_id"] = target_ids[decision_column] - record = { - "decision": decision, - "selected": selected, - "reason": "alternative_assignment_within_tolerance", - "assignment_cost_gap": gap, - "selected_assignment_cost": optimal_total, - "alternative_assignment_cost": alternative_total, - "tolerance": tolerance, - "affected_previous": affected_previous, - "affected_current": affected_current, - "alternative": snapshot, - } - previous_record = records.get(coordinate) - if previous_record is None or gap < float(previous_record["assignment_cost_gap"]): - records[coordinate] = record - unresolved.discard(coordinate) - - ordered = [ - records[coordinate] - for coordinate in sorted( - records, - key=lambda coordinate: ( - decisions[coordinate][0], - str(records[coordinate]["selected"].get("source_id", "")), - str(records[coordinate]["selected"].get("target_id", "")), - ), - ) - ] - by_match = { - (row, column): records[coordinate] - for coordinate, (decision, row, column) in decisions.items() - if decision == "match" and row is not None and column is not None and coordinate in records - } - return ordered, by_match - - def _weighted_cost(components: Mapping[str, tuple[float, float]]) -> float: active = [(value, weight) for value, weight in components.values() if weight > 0] if not active: @@ -499,31 +320,8 @@ def _weighted_cost(components: Mapping[str, tuple[float, float]]) -> float: return float(sum(value * weight for value, weight in active) / denominator) -def _pair_confidence(cost: float, ambiguity_gap: float = math.inf) -> float: - base = math.exp(-max(0.0, cost)) - if math.isfinite(ambiguity_gap): - # A tie retains half the base confidence rather than becoming exactly - # zero; a clearly separated optimum approaches full base confidence. - base *= 0.5 + 0.5 * (1.0 - math.exp(-max(0.0, ambiguity_gap))) - return float(min(1.0, max(0.0, base))) - - -def _ambiguity_gap(cost: FloatArray, row: int, column: int) -> float: - alternatives = [ - *( - float(value) - for index, value in enumerate(cost[row]) - if index != column and math.isfinite(float(value)) - ), - *( - float(value) - for index, value in enumerate(cost[:, column]) - if index != row and math.isfinite(float(value)) - ), - ] - if not alternatives: - return math.inf - return max(0.0, min(alternatives) - float(cost[row, column])) +def _pair_confidence(cost: float) -> float: + return float(math.exp(-max(0.0, cost))) def _atom_pair( @@ -602,32 +400,15 @@ def match_atoms( matched, unmatched_rows, unmatched_columns, total = _assign_with_unmatched( costs, settings.unmatched_cost ) - ambiguities, ambiguous_pairs = _assignment_ambiguities( - costs, - settings.unmatched_cost, - matched, - unmatched_rows, - unmatched_columns, - total, - [atom.index for atom in previous.atoms], - [atom.index for atom in current.atoms], - settings.ambiguity_tolerance, - ) edges: list[MatchEdge] = [] for row, column in sorted(matched): - pair_evidence = dict(evidence[row, column]) - ambiguity = ambiguous_pairs.get((row, column)) - if ambiguity is not None: - pair_evidence["assignment_ambiguity"] = ambiguity edges.append( MatchEdge( source_ids=(previous.atoms[row].index,), target_ids=(current.atoms[column].index,), cost=float(costs[row, column]), - confidence=_pair_confidence( - float(costs[row, column]), _ambiguity_gap(costs, row, column) - ), - evidence=pair_evidence, + confidence=_pair_confidence(float(costs[row, column])), + evidence=evidence[row, column], ) ) return MatchResult( @@ -642,9 +423,6 @@ def match_atoms( diagnostics={ "entity": "atom", "candidate_pairs": int(np.isfinite(costs).sum()), - "ambiguous": bool(ambiguities), - "ambiguity_tolerance": settings.ambiguity_tolerance, - "ambiguities": ambiguities, "cell_volume_ratio": _cell_volume_ratio(previous, current), }, ) @@ -791,39 +569,11 @@ def _declared_descriptor_families(feature: Feature) -> frozenset[str]: ) -def _configured_descriptor_families(feature: Feature, config: FeatureMatchConfig) -> frozenset[str]: - """Infer evidence families only from configured, finite numeric values. - - This is a defensive fallback for canonical/plugin features predating the - explicit ``descriptor_family`` tag. A key absent on one side remains - optional; only two non-empty, disjoint sets are incompatible. - """ - - families: set[str] = set() - for key in config.descriptor_scales: - if _as_finite_float(feature.attributes.get(key)) is None: - continue - family = _descriptor_family_token(key) - if family is not None: - families.add(family) - return frozenset(families) - - -def _semantic_compatibility( - source: Feature, target: Feature, config: FeatureMatchConfig -) -> tuple[bool, dict[str, Any]]: - """Conservatively gate pairs with explicitly contradictory semantics. - - Producer identity is also conservative by default because cross-tool - feature identity has not been calibrated. Conversely, an absent producer, - family, or charge-definition declaration is treated as genuinely missing - optional evidence and does not reject a candidate. - """ +def _semantic_compatibility(source: Feature, target: Feature) -> tuple[bool, dict[str, Any]]: + """Reject pairs with explicitly contradictory descriptor semantics.""" source_declared = _declared_descriptor_families(source) target_declared = _declared_descriptor_families(target) - source_configured = _configured_descriptor_families(source, config) - target_configured = _configured_descriptor_families(target, config) source_charge = _semantic_token(source.attributes.get("charge_definition")) target_charge = _semantic_token(target.attributes.get("charge_definition")) source_convention = _semantic_token(source.attributes.get("charge_convention")) @@ -849,24 +599,7 @@ def _semantic_compatibility( and source_producer != target_producer ): reasons.append("incompatible_descriptor_producers") - if source_configured and target_configured and source_configured.isdisjoint(target_configured): - reasons.append("disjoint_configured_descriptor_families") - - evidence: dict[str, Any] = { - "compatible": not reasons, - "rejection_reasons": reasons, - "source_descriptor_families": sorted(source_declared), - "target_descriptor_families": sorted(target_declared), - "source_configured_descriptor_families": sorted(source_configured), - "target_configured_descriptor_families": sorted(target_configured), - "source_charge_definition": source_charge, - "target_charge_definition": target_charge, - "source_charge_convention": source_convention, - "target_charge_convention": target_convention, - "source_descriptor_producer": source_producer, - "target_descriptor_producer": target_producer, - } - return not reasons, evidence + return not reasons, {"compatible": not reasons, "rejection_reasons": reasons} def _grid_shape(feature: Feature, frame: Frame) -> tuple[int, int, int] | None: @@ -918,7 +651,7 @@ def _feature_pair( kind_match = source.kind == target.kind if not _compatible_kind(source, target, config): return math.inf, {"kind_match": False} - semantic_match, semantic_evidence = _semantic_compatibility(source, target, config) + semantic_match, semantic_evidence = _semantic_compatibility(source, target) if not semantic_match: return math.inf, { "kind_match": kind_match, @@ -1117,44 +850,33 @@ def _best_group( best_evidence: dict[str, Any] = {} best_objective = math.inf best_rejection: dict[str, Any] | None = None - best_rejection_objective = math.inf - # Exhaustive subset selection is cheap at max_group_size <= 6 and avoids - # greedy failures when two children together conserve volume. + # Test only cumulative nearest-neighbor groups. This keeps the common + # conserved split/merge case without exponential subset enumeration. for size in range(1, len(limited) + 1): - for subset in combinations(limited, size): - if split: - sources = tuple(fixed) - targets = (*base, *(item[0] for item in subset)) - else: - sources = (*base, *(item[0] for item in subset)) - targets = tuple(fixed) - error, evidence, rejection_reason = _conservation_assessment( - sources, - targets, - config.conservation_tolerance, - ) - mean_pair_cost = float(np.mean([item[1] for item in subset])) - # Conservation dominates when available; a modest complexity term - # avoids swallowing unrelated nearby births/deaths. - objective = mean_pair_cost + 0.08 * len(subset) - if error is not None: - objective += 2.0 * error - if rejection_reason is not None: - if objective < best_rejection_objective: - best_rejection_objective = objective - best_rejection = { - "relation": "split" if split else "merge", - "source_ids": [feature.id for feature in sources], - "target_ids": [feature.id for feature in targets], - "reason": rejection_reason, - "candidate_member_cost": mean_pair_cost, - "conservation": evidence, - } - continue - if objective < best_objective: - best_objective = objective - best = list(subset) - best_evidence = evidence + subset = limited[:size] + if split: + sources = tuple(fixed) + targets = (*base, *(item[0] for item in subset)) + else: + sources = (*base, *(item[0] for item in subset)) + targets = tuple(fixed) + error, evidence, rejection_reason = _conservation_assessment( + sources, targets, config.conservation_tolerance + ) + mean_cost = float(np.mean([item[1] for item in subset])) + objective = mean_cost + 2.0 * (error or 0.0) + if rejection_reason is None and objective < best_objective: + best_objective = objective + best = list(subset) + best_evidence = evidence + elif best_rejection is None: + best_rejection = { + "relation": "split" if split else "merge", + "source_ids": [feature.id for feature in sources], + "target_ids": [feature.id for feature in targets], + "reason": rejection_reason, + "conservation": evidence, + } return best, best_evidence, best_rejection @@ -1165,7 +887,6 @@ def _augment_hyperedges( costs: FloatArray, distances: FloatArray, pair_evidence: Mapping[tuple[int, int], dict[str, Any]], - ambiguous_pairs: Mapping[tuple[int, int], dict[str, Any]], previous: Frame, current: Frame, atom_mapping: Mapping[int, int], @@ -1173,15 +894,7 @@ def _augment_hyperedges( ) -> tuple[list[MatchEdge], set[int], set[int], list[dict[str, Any]]]: if not config.detect_splits_merges or not direct: direct_edges = [ - _direct_feature_edge( - row, - column, - costs, - pair_evidence, - previous, - current, - assignment_ambiguity=ambiguous_pairs.get((row, column)), - ) + _direct_feature_edge(row, column, costs, pair_evidence, previous, current) for row, column in direct ] return direct_edges, unmatched_rows, unmatched_columns, [] @@ -1240,7 +953,6 @@ def _augment_hyperedges( for row, column in sorted(direct): source = previous.features[row] target = current.features[column] - assignment_ambiguity = ambiguous_pairs.get((row, column)) split_options = [ (current.features[index], cost, evidence) for index, cost, evidence in split_candidates.get(row, []) @@ -1357,17 +1069,7 @@ def _augment_hyperedges( } ) rejections.append(rejection) - edges.append( - _direct_feature_edge( - row, - column, - costs, - pair_evidence, - previous, - current, - assignment_ambiguity=assignment_ambiguity, - ) - ) + edges.append(_direct_feature_edge(row, column, costs, pair_evidence, previous, current)) rejections.sort( key=lambda item: ( str(item["relation"]), @@ -1384,42 +1086,6 @@ def _augment_hyperedges( ) -def _resolve_hyperedge_ambiguities( - ambiguities: Sequence[dict[str, Any]], - edges: Sequence[MatchEdge], -) -> list[dict[str, Any]]: - """Retain only assignment ties that alter final scientific membership. - - A split or merge may have multiple equivalent one-to-one backbones while - still producing the same conservation-backed participant set. Such a tie - is evidence about an implementation detail, not about the final hyperedge. - """ - - unresolved: list[dict[str, Any]] = [] - for ambiguity in ambiguities: - affected_previous = {str(value) for value in ambiguity.get("affected_previous", [])} - affected_current = {str(value) for value in ambiguity.get("affected_current", [])} - resolving_edge = next( - ( - edge - for edge in edges - if edge.relation is not MatchRelation.ONE_TO_ONE - and affected_previous - and affected_current - and affected_previous.issubset({str(value) for value in edge.source_ids}) - and affected_current.issubset({str(value) for value in edge.target_ids}) - ), - None, - ) - if resolving_edge is None: - unresolved.append(ambiguity) - continue - resolved = resolving_edge.evidence.setdefault("resolved_assignment_ambiguities", []) - if isinstance(resolved, list): - resolved.append(ambiguity) - return unresolved - - def _direct_feature_edge( row: int, column: int, @@ -1427,22 +1093,17 @@ def _direct_feature_edge( pair_evidence: Mapping[tuple[int, int], dict[str, Any]], previous: Frame, current: Frame, - *, - assignment_ambiguity: Mapping[str, Any] | None = None, ) -> MatchEdge: source = previous.features[row] target = current.features[column] quality, quality_evidence = _quality_factor((source, target)) evidence = dict(pair_evidence[row, column]) evidence.update(quality_evidence) - if assignment_ambiguity is not None: - evidence["assignment_ambiguity"] = dict(assignment_ambiguity) return MatchEdge( source_ids=(source.id,), target_ids=(target.id,), cost=float(costs[row, column]), - confidence=_pair_confidence(float(costs[row, column]), _ambiguity_gap(costs, row, column)) - * quality, + confidence=_pair_confidence(float(costs[row, column])) * quality, evidence=evidence, ) @@ -1465,7 +1126,7 @@ def match_features( for row, source in enumerate(previous.features) for column, target in enumerate(current.features) if _compatible_kind(source, target, settings) - and _semantic_compatibility(source, target, settings)[0] + and _semantic_compatibility(source, target)[0] ), previous.cell, current.cell, @@ -1490,20 +1151,9 @@ def match_features( costs[row, column] = cost pair_evidence[row, column] = evidence - direct, unmatched_rows, unmatched_columns, assignment_total = _assign_with_unmatched( + direct, unmatched_rows, unmatched_columns, _ = _assign_with_unmatched( costs, settings.unmatched_cost ) - ambiguities, ambiguous_pairs = _assignment_ambiguities( - costs, - settings.unmatched_cost, - direct, - unmatched_rows, - unmatched_columns, - assignment_total, - [feature.id for feature in previous.features], - [feature.id for feature in current.features], - settings.ambiguity_tolerance, - ) edges, unmatched_rows, unmatched_columns, hyperedge_rejections = _augment_hyperedges( direct, unmatched_rows, @@ -1511,13 +1161,11 @@ def match_features( costs, distances, pair_evidence, - ambiguous_pairs, previous, current, mapping, settings, ) - ambiguities = _resolve_hyperedge_ambiguities(ambiguities, edges) total = sum(edge.cost for edge in edges) + settings.unmatched_cost * ( len(unmatched_rows) + len(unmatched_columns) ) @@ -1538,9 +1186,6 @@ def match_features( for pair in pair_evidence.values() ), "one_to_one_backbone": len(direct), - "ambiguous": bool(ambiguities), - "ambiguity_tolerance": settings.ambiguity_tolerance, - "ambiguities": ambiguities, "hyperedges": sum( len(edge.source_ids) != 1 or len(edge.target_ids) != 1 for edge in edges ), diff --git a/src/electrace/models.py b/src/electrace/models.py index 098fc5e..0e3df52 100644 --- a/src/electrace/models.py +++ b/src/electrace/models.py @@ -1,10 +1,4 @@ -"""Serializable domain models used by the ElecTrace tracking engine. - -Coordinates are fractional and cells use the crystallographic row-vector -convention: ``cartesian = fractional @ cell``. Keeping this convention in -the model layer makes data produced by ASE, pymatgen, and common electronic -structure codes straightforward to adapt at the I/O boundary. -""" +"""Domain records for periodic structures, matches, and events.""" from __future__ import annotations @@ -13,7 +7,7 @@ from collections.abc import Sequence as TypingSequence from dataclasses import dataclass, field from enum import StrEnum -from itertools import islice, pairwise +from itertools import pairwise from typing import Any, TypeAlias Vec3: TypeAlias = tuple[float, float, float] @@ -22,13 +16,6 @@ class FeatureKind(StrEnum): - """Well-known feature kinds. - - Parsers may also use arbitrary strings: ElecTrace intentionally does not - reject unknown feature kinds so third-party topology tools can extend the - vocabulary without changing the core package. - """ - ATOMIC_BASIN = "atomic_basin" ELF_BASIN = "elf_basin" NON_NUCLEAR_ATTRACTOR = "non_nuclear_attractor" @@ -42,8 +29,6 @@ class FeatureKind(StrEnum): class MatchRelation(StrEnum): - """Cardinality of a correspondence edge.""" - ONE_TO_ONE = "one_to_one" SPLIT = "split" MERGE = "merge" @@ -51,8 +36,6 @@ class MatchRelation(StrEnum): class EventType(StrEnum): - """Electronic-topology changes reported between frames.""" - BIRTH = "birth" DEATH = "death" SPLIT = "split" @@ -61,74 +44,59 @@ class EventType(StrEnum): PROPERTY_CHANGE = "property_change" -def _vec3(value: Iterable[float], *, name: str) -> Vec3: - # Consume at most one item beyond the accepted shape. Inputs cross an - # untrusted parser boundary and may be generators; materializing an - # accidentally unbounded iterable just to reject its shape is avoidable. - values = tuple(float(component) for component in islice(value, 4)) - if len(values) != 3: - raise ValueError(f"{name} must contain exactly three values") - if not all(math.isfinite(component) for component in values): - raise ValueError(f"{name} must contain only finite values") - return values +def _vec3(value: Iterable[float], name: str = "coordinate") -> Vec3: + result = tuple(float(component) for component in value) + if len(result) != 3 or not all(math.isfinite(component) for component in result): + raise ValueError(f"{name} must contain three finite values") + return result -def _matrix3(value: Iterable[Iterable[float]], *, name: str = "cell") -> Matrix3: - rows = tuple(_vec3(row, name=f"{name} row") for row in islice(value, 4)) +def _matrix3(value: Iterable[Iterable[float]]) -> Matrix3: + rows = tuple(_vec3(row, "cell row") for row in value) if len(rows) != 3: - raise ValueError(f"{name} must contain exactly three row vectors") - # Determinant in pure Python so importing the data model never requires - # importing NumPy (useful for metadata-only inspection). + raise ValueError("cell must contain three rows") a, b, c = rows determinant = ( a[0] * (b[1] * c[2] - b[2] * c[1]) - a[1] * (b[0] * c[2] - b[2] * c[0]) + a[2] * (b[0] * c[1] - b[1] * c[0]) ) - scale = max(abs(component) for row in rows for component in row) - if abs(determinant) <= max(1.0, scale**3) * 1.0e-12: - raise ValueError(f"{name} must be nonsingular") + if abs(determinant) < 1e-12: + raise ValueError("cell must be nonsingular") return rows -def _finite_optional(value: float | None, *, name: str) -> float | None: +def _finite(value: float | None, name: str) -> float | None: if value is None: return None result = float(value) if not math.isfinite(result): - raise ValueError(f"{name} must be finite when provided") + raise ValueError(f"{name} must be finite") return result def _json_value(value: Any) -> Any: - """Convert common scientific-Python values to JSON-compatible objects.""" - if isinstance(value, StrEnum): return value.value if value is None or isinstance(value, (str, int, bool)): return value if isinstance(value, float): if not math.isfinite(value): - raise TypeError("non-finite floats are not JSON serializable") + raise TypeError("non-finite value is not JSON serializable") return value if isinstance(value, Mapping): return {str(key): _json_value(item) for key, item in value.items()} if isinstance(value, (tuple, list, set, frozenset)): return [_json_value(item) for item in value] - if hasattr(value, "item"): - try: - return _json_value(value.item()) - except (TypeError, ValueError): - pass if hasattr(value, "tolist"): return _json_value(value.tolist()) - raise TypeError(f"value of type {type(value).__name__} is not JSON serializable") + if hasattr(value, "item"): + return _json_value(value.item()) + raise TypeError(f"{type(value).__name__} is not JSON serializable") @dataclass(slots=True) class Atom: - """An atom in one frame, identified by its frame-local integer index.""" - index: int element: str fractional_position: Vec3 @@ -136,12 +104,10 @@ class Atom: def __post_init__(self) -> None: self.index = int(self.index) - if self.index < 0: - raise ValueError("atom index must be non-negative") self.element = str(self.element).strip() - if not self.element: - raise ValueError("atom element must not be empty") - self.fractional_position = _vec3(self.fractional_position, name="fractional_position") + if self.index < 0 or not self.element: + raise ValueError("atom requires a non-negative index and element") + self.fractional_position = _vec3(self.fractional_position) self.attributes = dict(self.attributes) def to_dict(self) -> dict[str, Any]: @@ -155,24 +121,15 @@ def to_dict(self) -> dict[str, Any]: @classmethod def from_dict(cls, data: Mapping[str, Any]) -> Atom: return cls( - index=int(data["index"]), - element=str(data["element"]), - fractional_position=data["fractional_position"], - attributes=dict(data.get("attributes", {})), + int(data["index"]), + str(data["element"]), + data["fractional_position"], + dict(data.get("attributes", {})), ) @dataclass(slots=True) class Feature: - """A non-atomic electronic feature in one frame. - - ``charge`` and ``volume`` are treated as extensive properties by the - split/merge logic. ``neighbor_atom_indices`` is a frame-local atom - fingerprint. ``voxel_indices`` is optional because point-like critical - points do not have a voxel support; when present it should refer to the - grid shape stored in ``attributes['grid_shape']`` or frame metadata. - """ - id: str kind: str | FeatureKind fractional_position: Vec3 @@ -184,34 +141,30 @@ class Feature: def __post_init__(self) -> None: self.id = str(self.id) - if not self.id: - raise ValueError("feature id must not be empty") self.kind = str(self.kind).strip().lower() - if not self.kind: - raise ValueError("feature kind must not be empty") - self.fractional_position = _vec3(self.fractional_position, name="fractional_position") - self.charge = _finite_optional(self.charge, name="feature charge") - self.volume = _finite_optional(self.volume, name="feature volume") + if not self.id or not self.kind: + raise ValueError("feature requires an id and kind") + self.fractional_position = _vec3(self.fractional_position) + self.charge = _finite(self.charge, "feature charge") + self.volume = _finite(self.volume, "feature volume") if self.volume is not None and self.volume < 0: raise ValueError("feature volume must be non-negative") - neighbors = tuple(int(index) for index in self.neighbor_atom_indices) - if any(index < 0 for index in neighbors): - raise ValueError("neighbor atom indices must be non-negative") - if len(set(neighbors)) != len(neighbors): - raise ValueError("neighbor atom indices must be unique") - self.neighbor_atom_indices = tuple(sorted(neighbors)) + neighbors = tuple(sorted(int(index) for index in self.neighbor_atom_indices)) + if len(neighbors) != len(set(neighbors)) or any(index < 0 for index in neighbors): + raise ValueError("neighbor atom indices must be unique and non-negative") + self.neighbor_atom_indices = neighbors self.attributes = dict(self.attributes) if self.voxel_indices is not None: voxels: set[tuple[int, int, int]] = set() for voxel in self.voxel_indices: item = tuple(int(component) for component in voxel) if len(item) != 3: - raise ValueError("every voxel index must have three components") - voxels.add(item) + raise ValueError("voxel indices must have three components") + voxels.add((item[0], item[1], item[2])) self.voxel_indices = tuple(sorted(voxels)) def to_dict(self) -> dict[str, Any]: - result: dict[str, Any] = { + result = { "id": self.id, "kind": self.kind, "fractional_position": list(self.fractional_position), @@ -226,6 +179,7 @@ def to_dict(self) -> dict[str, Any]: @classmethod def from_dict(cls, data: Mapping[str, Any]) -> Feature: + voxels = data.get("voxel_indices") return cls( id=str(data["id"]), kind=str(data["kind"]), @@ -234,18 +188,12 @@ def from_dict(cls, data: Mapping[str, Any]) -> Feature: volume=data.get("volume"), neighbor_atom_indices=tuple(data.get("neighbor_atom_indices", ())), attributes=dict(data.get("attributes", {})), - voxel_indices=( - tuple(tuple(voxel) for voxel in data["voxel_indices"]) - if data.get("voxel_indices") is not None - else None - ), + voxel_indices=tuple(tuple(item) for item in voxels) if voxels is not None else None, ) @dataclass(slots=True) class Frame: - """Atoms and electronic features sampled at one ordered path image.""" - index: int label: str cell: Matrix3 @@ -259,39 +207,32 @@ def __post_init__(self) -> None: self.label = str(self.label) self.cell = _matrix3(self.cell) self.atoms = tuple( - atom if isinstance(atom, Atom) else Atom.from_dict(atom) for atom in self.atoms + item if isinstance(item, Atom) else Atom.from_dict(item) for item in self.atoms ) self.features = tuple( - feature if isinstance(feature, Feature) else Feature.from_dict(feature) - for feature in self.features + item if isinstance(item, Feature) else Feature.from_dict(item) for item in self.features ) - atom_indices = [atom.index for atom in self.atoms] - if len(set(atom_indices)) != len(atom_indices): - raise ValueError("atom indices must be unique within a frame") - feature_ids = [feature.id for feature in self.features] - if len(set(feature_ids)) != len(feature_ids): - raise ValueError("feature ids must be unique within a frame") - known_atoms = set(atom_indices) - for feature in self.features: - unknown = set(feature.neighbor_atom_indices) - known_atoms - if unknown: - raise ValueError( - f"feature {feature.id!r} references unknown atom indices {sorted(unknown)}" - ) - self.energy = _finite_optional(self.energy, name="frame energy") + atom_ids = {atom.index for atom in self.atoms} + if len(atom_ids) != len(self.atoms) or len({item.id for item in self.features}) != len( + self.features + ): + raise ValueError("atom indices and feature ids must be unique within a frame") + if any(set(item.neighbor_atom_indices) - atom_ids for item in self.features): + raise ValueError("feature references an unknown atom") + self.energy = _finite(self.energy, "frame energy") self.metadata = dict(self.metadata) def atom_by_index(self, index: int) -> Atom: - for atom in self.atoms: - if atom.index == index: - return atom - raise KeyError(index) + try: + return next(atom for atom in self.atoms if atom.index == index) + except StopIteration: + raise KeyError(index) from None def feature_by_id(self, feature_id: str) -> Feature: - for feature in self.features: - if feature.id == feature_id: - return feature - raise KeyError(feature_id) + try: + return next(feature for feature in self.features if feature.id == feature_id) + except StopIteration: + raise KeyError(feature_id) from None def to_dict(self) -> dict[str, Any]: return { @@ -308,56 +249,50 @@ def to_dict(self) -> dict[str, Any]: def from_dict(cls, data: Mapping[str, Any]) -> Frame: index = int(data.get("index", 0)) return cls( - index=index, - label=str(data.get("label", index)), - cell=data["cell"], - atoms=tuple(Atom.from_dict(atom) for atom in data.get("atoms", ())), - features=tuple(Feature.from_dict(feature) for feature in data.get("features", ())), - energy=data.get("energy"), - metadata=dict(data.get("metadata", {})), + index, + str(data.get("label", index)), + data["cell"], + tuple(Atom.from_dict(item) for item in data.get("atoms", ())), + tuple(Feature.from_dict(item) for item in data.get("features", ())), + data.get("energy"), + dict(data.get("metadata", {})), ) @dataclass(slots=True) class Sequence: - """An ordered NEB, pressure, strain, or trajectory sequence.""" - frames: tuple[Frame, ...] label: str = "" metadata: dict[str, Any] = field(default_factory=dict) def __post_init__(self) -> None: self.frames = tuple( - frame if isinstance(frame, Frame) else Frame.from_dict(frame) for frame in self.frames + item if isinstance(item, Frame) else Frame.from_dict(item) for item in self.frames ) indices = [frame.index for frame in self.frames] - if len(set(indices)) != len(indices): - raise ValueError("frame indices must be unique") if any(right <= left for left, right in pairwise(indices)): - raise ValueError("frames must be strictly ordered by increasing index") + raise ValueError("frames must have increasing unique indices") self.label = str(self.label) self.metadata = dict(self.metadata) def to_dict(self) -> dict[str, Any]: return { "label": self.label, - "frames": [frame.to_dict() for frame in self.frames], + "frames": [item.to_dict() for item in self.frames], "metadata": _json_value(self.metadata), } @classmethod def from_dict(cls, data: Mapping[str, Any]) -> Sequence: return cls( - frames=tuple(Frame.from_dict(frame) for frame in data.get("frames", ())), - label=str(data.get("label", "")), - metadata=dict(data.get("metadata", {})), + tuple(Frame.from_dict(item) for item in data.get("frames", ())), + str(data.get("label", "")), + dict(data.get("metadata", {})), ) @dataclass(slots=True) class MatchEdge: - """A one-to-one or hyperedge correspondence between adjacent frames.""" - source_ids: tuple[Identifier, ...] target_ids: tuple[Identifier, ...] cost: float @@ -365,21 +300,16 @@ class MatchEdge: evidence: dict[str, Any] = field(default_factory=dict) def __post_init__(self) -> None: - self.source_ids = tuple(self.source_ids) - self.target_ids = tuple(self.target_ids) - if not self.source_ids or not self.target_ids: - raise ValueError("a match edge needs at least one source and one target") - if len(set(self.source_ids)) != len(self.source_ids): - raise ValueError("source ids in a match edge must be unique") - if len(set(self.target_ids)) != len(self.target_ids): - raise ValueError("target ids in a match edge must be unique") - self.cost = float(self.cost) - self.confidence = float(self.confidence) - if not math.isfinite(self.cost) or self.cost < 0: - raise ValueError("match cost must be finite and non-negative") - if not math.isfinite(self.confidence) or not 0.0 <= self.confidence <= 1.0: - raise ValueError("match confidence must lie in [0, 1]") + self.source_ids, self.target_ids = tuple(self.source_ids), tuple(self.target_ids) + self.cost, self.confidence = float(self.cost), float(self.confidence) self.evidence = dict(self.evidence) + if ( + not self.source_ids + or not self.target_ids + or self.cost < 0 + or not 0 <= self.confidence <= 1 + ): + raise ValueError("invalid match edge") @property def relation(self) -> MatchRelation: @@ -394,31 +324,21 @@ def relation(self) -> MatchRelation: @property def source_id(self) -> Identifier: if len(self.source_ids) != 1: - raise AttributeError("hyperedge does not have a single source_id") + raise AttributeError("hyperedge has multiple sources") return self.source_ids[0] @property def target_id(self) -> Identifier: if len(self.target_ids) != 1: - raise AttributeError("hyperedge does not have a single target_id") + raise AttributeError("hyperedge has multiple targets") return self.target_ids[0] - @property - def score(self) -> float: - """Alias retained for consumers that call confidence a match score.""" - - return self.confidence - - @property - def ambiguous(self) -> bool: - """Whether a complete near-optimal assignment changes this decision.""" - - return isinstance(self.evidence.get("assignment_ambiguity"), Mapping) + score = property(lambda self: self.confidence) def to_dict(self) -> dict[str, Any]: return { - "source_ids": _json_value(self.source_ids), - "target_ids": _json_value(self.target_ids), + "source_ids": list(self.source_ids), + "target_ids": list(self.target_ids), "relation": self.relation.value, "cost": self.cost, "confidence": self.confidence, @@ -428,18 +348,16 @@ def to_dict(self) -> dict[str, Any]: @classmethod def from_dict(cls, data: Mapping[str, Any]) -> MatchEdge: return cls( - source_ids=tuple(data["source_ids"]), - target_ids=tuple(data["target_ids"]), - cost=float(data["cost"]), - confidence=float(data["confidence"]), - evidence=dict(data.get("evidence", {})), + tuple(data["source_ids"]), + tuple(data["target_ids"]), + float(data["cost"]), + float(data["confidence"]), + dict(data.get("evidence", {})), ) @dataclass(slots=True) class MatchResult: - """Complete correspondence result for a pair of adjacent frames.""" - previous_frame_index: int current_frame_index: int edges: tuple[MatchEdge, ...] = () @@ -448,32 +366,16 @@ class MatchResult: total_cost: float = 0.0 diagnostics: dict[str, Any] = field(default_factory=dict) - def __post_init__(self) -> None: - self.previous_frame_index = int(self.previous_frame_index) - self.current_frame_index = int(self.current_frame_index) - self.edges = tuple( - edge if isinstance(edge, MatchEdge) else MatchEdge.from_dict(edge) - for edge in self.edges - ) - self.unmatched_previous = tuple(self.unmatched_previous) - self.unmatched_current = tuple(self.unmatched_current) - self.total_cost = float(self.total_cost) - if not math.isfinite(self.total_cost) or self.total_cost < 0: - raise ValueError("total match cost must be finite and non-negative") - self.diagnostics = dict(self.diagnostics) - @property def matches(self) -> tuple[MatchEdge, ...]: return self.edges @property def mapping(self) -> dict[Identifier, Identifier]: - """Return only unambiguous one-to-one portions of the mapping.""" - return { edge.source_id: edge.target_id for edge in self.edges - if edge.relation is MatchRelation.ONE_TO_ONE and not edge.ambiguous + if edge.relation is MatchRelation.ONE_TO_ONE } def to_dict(self) -> dict[str, Any]: @@ -481,8 +383,8 @@ def to_dict(self) -> dict[str, Any]: "previous_frame_index": self.previous_frame_index, "current_frame_index": self.current_frame_index, "edges": [edge.to_dict() for edge in self.edges], - "unmatched_previous": _json_value(self.unmatched_previous), - "unmatched_current": _json_value(self.unmatched_current), + "unmatched_previous": list(self.unmatched_previous), + "unmatched_current": list(self.unmatched_current), "total_cost": self.total_cost, "diagnostics": _json_value(self.diagnostics), } @@ -490,20 +392,18 @@ def to_dict(self) -> dict[str, Any]: @classmethod def from_dict(cls, data: Mapping[str, Any]) -> MatchResult: return cls( - previous_frame_index=int(data["previous_frame_index"]), - current_frame_index=int(data["current_frame_index"]), - edges=tuple(MatchEdge.from_dict(edge) for edge in data.get("edges", ())), - unmatched_previous=tuple(data.get("unmatched_previous", ())), - unmatched_current=tuple(data.get("unmatched_current", ())), - total_cost=float(data.get("total_cost", 0.0)), - diagnostics=dict(data.get("diagnostics", {})), + int(data["previous_frame_index"]), + int(data["current_frame_index"]), + tuple(MatchEdge.from_dict(item) for item in data.get("edges", ())), + tuple(data.get("unmatched_previous", ())), + tuple(data.get("unmatched_current", ())), + float(data.get("total_cost", 0)), + dict(data.get("diagnostics", {})), ) @dataclass(slots=True) class Event: - """A topology event with an explicit, machine-readable evidence vector.""" - type: EventType | str frame_before: int frame_after: int @@ -517,19 +417,11 @@ class Event: def __post_init__(self) -> None: self.type = EventType(self.type) - self.frame_before = int(self.frame_before) - self.frame_after = int(self.frame_after) - self.source_ids = tuple(self.source_ids) - self.target_ids = tuple(self.target_ids) - self.confidence = float(self.confidence) - if not math.isfinite(self.confidence) or not 0.0 <= self.confidence <= 1.0: - raise ValueError("event confidence must lie in [0, 1]") - self.persistence = int(self.persistence) - if self.persistence < 1: - raise ValueError("event persistence must be at least one frame") - self.evidence = dict(self.evidence) - self.description = str(self.description) - self.metadata = dict(self.metadata) + self.source_ids, self.target_ids = tuple(self.source_ids), tuple(self.target_ids) + self.confidence, self.persistence = float(self.confidence), int(self.persistence) + self.evidence, self.metadata = dict(self.evidence), dict(self.metadata) + if not 0 <= self.confidence <= 1 or self.persistence < 1: + raise ValueError("invalid event confidence or persistence") @property def event_type(self) -> EventType: @@ -540,8 +432,8 @@ def to_dict(self) -> dict[str, Any]: "type": self.event_type.value, "frame_before": self.frame_before, "frame_after": self.frame_after, - "source_ids": _json_value(self.source_ids), - "target_ids": _json_value(self.target_ids), + "source_ids": list(self.source_ids), + "target_ids": list(self.target_ids), "confidence": self.confidence, "persistence": self.persistence, "evidence": _json_value(self.evidence), @@ -552,23 +444,21 @@ def to_dict(self) -> dict[str, Any]: @classmethod def from_dict(cls, data: Mapping[str, Any]) -> Event: return cls( - type=str(data["type"]), - frame_before=int(data["frame_before"]), - frame_after=int(data["frame_after"]), - source_ids=tuple(data.get("source_ids", ())), - target_ids=tuple(data.get("target_ids", ())), - confidence=float(data.get("confidence", 0.0)), - persistence=int(data.get("persistence", 1)), - evidence=dict(data.get("evidence", {})), - description=str(data.get("description", "")), - metadata=dict(data.get("metadata", {})), + str(data["type"]), + int(data["frame_before"]), + int(data["frame_after"]), + tuple(data.get("source_ids", ())), + tuple(data.get("target_ids", ())), + float(data.get("confidence", 0)), + int(data.get("persistence", 1)), + dict(data.get("evidence", {})), + str(data.get("description", "")), + dict(data.get("metadata", {})), ) @dataclass(slots=True) class TrackingResult: - """Matching and event results for a complete sequence.""" - atom_matches: tuple[MatchResult, ...] feature_matches: tuple[MatchResult, ...] events: tuple[Event, ...] @@ -576,29 +466,21 @@ class TrackingResult: def to_dict(self) -> dict[str, Any]: return { - "atom_matches": [result.to_dict() for result in self.atom_matches], - "feature_matches": [result.to_dict() for result in self.feature_matches], - "events": [event.to_dict() for event in self.events], + "atom_matches": [item.to_dict() for item in self.atom_matches], + "feature_matches": [item.to_dict() for item in self.feature_matches], + "events": [item.to_dict() for item in self.events], "diagnostics": _json_value(self.diagnostics), } @classmethod def from_dict(cls, data: Mapping[str, Any]) -> TrackingResult: return cls( - atom_matches=tuple( - MatchResult.from_dict(result) for result in data.get("atom_matches", ()) - ), - feature_matches=tuple( - MatchResult.from_dict(result) for result in data.get("feature_matches", ()) - ), - events=tuple(Event.from_dict(event) for event in data.get("events", ())), - diagnostics=dict(data.get("diagnostics", {})), + tuple(MatchResult.from_dict(item) for item in data.get("atom_matches", ())), + tuple(MatchResult.from_dict(item) for item in data.get("feature_matches", ())), + tuple(Event.from_dict(item) for item in data.get("events", ())), + dict(data.get("diagnostics", {})), ) def ensure_sequence(value: Sequence | TypingSequence[Frame]) -> Sequence: - """Coerce a frame iterable to :class:`Sequence` for public APIs.""" - - if isinstance(value, Sequence): - return value - return Sequence(tuple(value)) + return value if isinstance(value, Sequence) else Sequence(tuple(value)) diff --git a/src/electrace/pipeline.py b/src/electrace/pipeline.py index 16d185f..3115daa 100644 --- a/src/electrace/pipeline.py +++ b/src/electrace/pipeline.py @@ -8,7 +8,6 @@ from dataclasses import dataclass from datetime import UTC, datetime from pathlib import Path -from tempfile import TemporaryDirectory from typing import Any import numpy as np @@ -19,13 +18,12 @@ from electrace.events import EventDetectionConfig, track_sequence from electrace.export import ( write_events_csv, - write_hdf5, write_json, write_tracks_csv, ) from electrace.io import load_sequence from electrace.matching import AtomMatchConfig, FeatureMatchConfig -from electrace.models import Feature, Frame, MatchRelation, Sequence, TrackingResult +from electrace.models import Feature, Frame, MatchRelation, Sequence from electrace.provenance import environment_record, fingerprint_files, sha256_file from electrace.report import render_html_report @@ -48,10 +46,7 @@ def _resolve(base: Path, value: str | None) -> Path | None: def _matching_configs(config: ElecTraceConfig) -> tuple[AtomMatchConfig, FeatureMatchConfig]: settings = config.matching - atom = AtomMatchConfig( - max_distance=settings.max_atom_displacement, - ambiguity_tolerance=settings.ambiguity_tolerance, - ) + atom = AtomMatchConfig(max_distance=settings.max_atom_displacement) feature = FeatureMatchConfig( max_distance=settings.max_feature_displacement, max_cost=settings.match_threshold, @@ -62,7 +57,6 @@ def _matching_configs(config: ElecTraceConfig) -> tuple[AtomMatchConfig, Feature voxel_weight=settings.overlap_weight, descriptor_weight=settings.descriptor_weight, descriptor_scales=settings.descriptor_scales, - ambiguity_tolerance=settings.ambiguity_tolerance, conservation_tolerance=settings.conservation_tolerance, min_split_merge_confidence=settings.split_merge_threshold, ) @@ -77,7 +71,6 @@ def _event_config(config: ElecTraceConfig) -> EventDetectionConfig: descriptor_thresholds=settings.descriptor_thresholds, min_event_confidence=settings.minimum_event_confidence, min_persistence=settings.minimum_persistence, - hysteresis_frames=settings.hysteresis_frames, include_transient=True, ) @@ -211,8 +204,6 @@ def _build_tracks( candidates: dict[str, list[tuple[float, str, tuple[str, ...]]]] = {} for edge in result.edges: - if edge.ambiguous: - continue parent_tracks = tuple( sorted({active[str(source)] for source in edge.source_ids if str(source) in active}) ) @@ -303,38 +294,11 @@ def _normalize_events( return normalized -def _matching_ambiguity_warnings(tracking: TrackingResult) -> list[str]: - warnings: list[str] = [] - for entity, results in ( - ("atom", tracking.atom_matches), - ("feature", tracking.feature_matches), - ): - for result in results: - raw = result.diagnostics.get("ambiguities", []) - ambiguities = raw if isinstance(raw, list) else [] - if not ambiguities: - continue - tolerance = float(result.diagnostics.get("ambiguity_tolerance", 0.0)) - consequence = ( - "ambiguous atom mappings were excluded from feature-neighbor evidence" - if entity == "atom" - else "affected lineage and event claims were suppressed" - ) - warnings.append( - f"Ambiguous {entity} correspondence for transition " - f"{result.previous_frame_index}→{result.current_frame_index}: " - f"{len(ambiguities)} selected decision(s) have a complete alternative " - f"assignment within {tolerance:g} cost units; {consequence}." - ) - return warnings - - def analyze_sequence( sequence: Sequence, config: ElecTraceConfig, *, config_path: str | Path | None = None, - strict: bool = True, ) -> dict[str, Any]: """Run matching, event detection, lineage construction, and provenance.""" @@ -343,17 +307,10 @@ def analyze_sequence( atom_config, feature_config = _matching_configs(config) event_config = _event_config(config) tracking = track_sequence(sequence, atom_config, feature_config, event_config) - ambiguity_warnings = _matching_ambiguity_warnings(tracking) - if strict and ambiguity_warnings: - raise ValueError( - "Ambiguous correspondence in strict mode. " - + " ".join(ambiguity_warnings) - + " Re-run with --permissive to retain diagnostics while suppressing affected claims." - ) tracks, track_warnings = _build_tracks(sequence, tracking.feature_matches) descriptor_documents, descriptor_warnings = _descriptor_provenance(sequence) - warnings = [*ambiguity_warnings, *track_warnings, *descriptor_warnings] + warnings = [*track_warnings, *descriptor_warnings] missing_features = [frame.label for frame in sequence.frames if not frame.features] if missing_features: warnings.append( @@ -451,7 +408,7 @@ def load_and_analyze( descriptor_name=config.inputs.descriptor_name, descriptor_source=config.inputs.descriptor_source, ) - return analyze_sequence(sequence, config, config_path=path, strict=strict), config + return analyze_sequence(sequence, config, config_path=path), config def _prepare_output(directory: Path, overwrite: bool) -> None: @@ -467,54 +424,12 @@ def _prepare_output(directory: Path, overwrite: bool) -> None: "analysis.json", "events.csv", "feature_tracks.csv", - "analysis.h5", "report.html", "manifest.json", } ) -def _selected_bundle_artifacts(config: ElecTraceConfig) -> set[str]: - selected = {"manifest.json"} - if config.outputs.json: - selected.add("analysis.json") - if config.outputs.csv: - selected.update(("events.csv", "feature_tracks.csv")) - if config.outputs.hdf5: - selected.add("analysis.h5") - if config.outputs.html_report: - selected.add("report.html") - return selected - - -def _remove_unselected_bundle_artifacts(directory: Path, selected: set[str]) -> None: - """Remove only reserved ElecTrace files disabled for this overwrite.""" - - for name in sorted(_KNOWN_BUNDLE_ARTIFACTS - selected): - artifact = directory / name - if artifact.is_file() or artifact.is_symlink(): - artifact.unlink() - - -def _publish_staged_bundle(staging: Path, directory: Path, selected: set[str]) -> None: - """Publish a complete staged bundle without following reserved symlinks.""" - - for name in sorted(selected): - artifact = directory / name - if artifact.is_dir() and not artifact.is_symlink(): - raise IsADirectoryError(f"Reserved bundle artifact path is a directory: {artifact}") - # Writers never create symlinks. A reserved symlink is therefore not a - # valid prior artifact; unlink only its directory entry, never its target. - for name in sorted(_KNOWN_BUNDLE_ARTIFACTS): - artifact = directory / name - if artifact.is_symlink(): - artifact.unlink() - _remove_unselected_bundle_artifacts(directory, selected) - for name in sorted(selected - {"manifest.json"}): - (staging / name).replace(directory / name) - (staging / "manifest.json").replace(directory / "manifest.json") - - def write_analysis_bundle( analysis: Mapping[str, Any], config: ElecTraceConfig, @@ -522,49 +437,42 @@ def write_analysis_bundle( *, overwrite: bool = False, ) -> AnalysisArtifacts: - """Write selected JSON/CSV/HDF5/HTML products and an integrity manifest.""" + """Write selected JSON, CSV, and HTML products plus an integrity manifest.""" directory = Path(output_directory).expanduser().resolve() _prepare_output(directory, overwrite) - selected = _selected_bundle_artifacts(config) - written_names: list[str] - with TemporaryDirectory(prefix=".electrace-stage-", dir=directory) as temporary: - staging = Path(temporary) - staged: list[Path] = [] - if config.outputs.json: - staged.append(write_json(analysis, staging / "analysis.json")) - if config.outputs.csv: - staged.append(write_events_csv(analysis.get("events", []), staging / "events.csv")) - staged.append( - write_tracks_csv(analysis.get("tracks", []), staging / "feature_tracks.csv") - ) - if config.outputs.hdf5: - staged.append(write_hdf5(analysis, staging / "analysis.h5")) - if config.outputs.html_report: - staged.append(render_html_report(analysis, staging / "report.html")) - - manifest = { - "schema_version": "electrace.bundle.v1", - "generated_at": datetime.now(UTC).isoformat(), - "files": [ - { - "path": item.name, - "size_bytes": item.stat().st_size, - "sha256": sha256_file(item), - } - for item in staged - ], - } - manifest_path = staging / "manifest.json" - atomic_write_text( - manifest_path, - json.dumps(manifest, indent=2, sort_keys=True, allow_nan=False) + "\n", + written: list[Path] = [] + if config.outputs.json: + written.append(write_json(analysis, directory / "analysis.json")) + if config.outputs.csv: + written.append(write_events_csv(analysis.get("events", []), directory / "events.csv")) + written.append( + write_tracks_csv(analysis.get("tracks", []), directory / "feature_tracks.csv") ) - written_names = [item.name for item in staged] - _publish_staged_bundle(staging, directory, selected) - - written = tuple(directory / name for name in (*written_names, "manifest.json")) - return AnalysisArtifacts(dict(analysis), directory, written) + if config.outputs.html_report: + written.append(render_html_report(analysis, directory / "report.html")) + + selected = {item.name for item in written} | {"manifest.json"} + if overwrite: + for name in _KNOWN_BUNDLE_ARTIFACTS - selected: + artifact = directory / name + if artifact.is_file() or artifact.is_symlink(): + artifact.unlink() + manifest = { + "schema_version": "electrace.bundle.v1", + "generated_at": datetime.now(UTC).isoformat(), + "files": [ + {"path": item.name, "size_bytes": item.stat().st_size, "sha256": sha256_file(item)} + for item in written + ], + } + manifest_path = directory / "manifest.json" + atomic_write_text( + manifest_path, json.dumps(manifest, indent=2, sort_keys=True, allow_nan=False) + "\n" + ) + written.append(manifest_path) + files = tuple(written) + return AnalysisArtifacts(dict(analysis), directory, files) def run_project( diff --git a/src/electrace/plugins/base.py b/src/electrace/plugins/base.py index 564cc68..4bacadf 100644 --- a/src/electrace/plugins/base.py +++ b/src/electrace/plugins/base.py @@ -1,4 +1,4 @@ -"""Non-executing plugin contracts for external electronic-analysis tools.""" +"""Non-executing plans for external electronic-analysis tools.""" from __future__ import annotations @@ -8,13 +8,10 @@ from dataclasses import dataclass, field from enum import StrEnum from pathlib import Path -from types import MappingProxyType from typing import Any, Protocol class Capability(StrEnum): - """Scientific result families a plugin can produce.""" - BADER_CHARGES = "bader_charges" ELF_BASINS = "elf_basins" NON_NUCLEAR_ATTRACTORS = "non_nuclear_attractors" @@ -26,14 +23,7 @@ class Capability(StrEnum): @dataclass(frozen=True, slots=True) class CommandPlan: - """A serializable, shell-free plan for a command that has *not* run. - - ``argv`` is intentionally a token tuple instead of a shell command. A - workflow layer may display, serialize, or explicitly approve and execute a - plan later; plugin planning itself never invokes external science tools. - ``stdout_path`` expresses safe output capture without embedding ``>`` in a - shell string. - """ + """A shell-free command description; planning never executes it.""" plugin: str capability: Capability @@ -47,80 +37,34 @@ class CommandPlan: notes: tuple[str, ...] = () def __post_init__(self) -> None: - plugin = str(self.plugin).strip().lower() - if not plugin: - raise ValueError("plugin name must not be empty") - object.__setattr__(self, "plugin", plugin) - try: - capability = ( - self.capability - if isinstance(self.capability, Capability) - else Capability(self.capability) - ) - except ValueError as exc: - raise ValueError(f"unknown command-plan capability {self.capability!r}") from exc - object.__setattr__(self, "capability", capability) - argv = tuple(self.argv) - if not argv or any(not isinstance(token, str) or not token for token in argv): - raise ValueError("argv must contain non-empty string tokens") - if any("\x00" in token for token in argv): - raise ValueError("argv tokens must not contain NUL bytes") - object.__setattr__(self, "argv", argv) - cwd = Path(self.cwd) - if not cwd.is_absolute(): - raise ValueError("command-plan cwd must be absolute") - object.__setattr__(self, "cwd", cwd) - for field_name in ("required_inputs", "expected_outputs"): - paths = tuple(Path(path) for path in getattr(self, field_name)) - if any(not path.is_absolute() for path in paths): - raise ValueError(f"command-plan {field_name} must contain only absolute paths") - object.__setattr__(self, field_name, paths) - if self.stdout_path is not None: - stdout_path = Path(self.stdout_path) - if not stdout_path.is_absolute(): - raise ValueError("command-plan stdout_path must be absolute") - object.__setattr__(self, "stdout_path", stdout_path) - if self.stdin is not None and "\x00" in self.stdin: - raise ValueError("stdin must not contain NUL bytes") - environment = dict(self.environment) - if any( - not isinstance(key, str) - or not isinstance(value, str) - or "\x00" in key - or "\x00" in value - for key, value in environment.items() + if not self.plugin.strip() or not self.argv: + raise ValueError("command plan requires a plugin and argv") + object.__setattr__(self, "plugin", self.plugin.strip().lower()) + object.__setattr__(self, "capability", Capability(self.capability)) + object.__setattr__(self, "argv", tuple(self.argv)) + object.__setattr__(self, "cwd", Path(self.cwd)) + object.__setattr__(self, "required_inputs", tuple(map(Path, self.required_inputs))) + object.__setattr__(self, "expected_outputs", tuple(map(Path, self.expected_outputs))) + object.__setattr__(self, "environment", dict(self.environment)) + object.__setattr__(self, "notes", tuple(self.notes)) + if not self.cwd.is_absolute() or any( + not path.is_absolute() for path in (*self.required_inputs, *self.expected_outputs) ): - raise ValueError("command-plan environment must contain NUL-free string pairs") - object.__setattr__(self, "environment", MappingProxyType(environment)) - notes = tuple(self.notes) - if any(not isinstance(note, str) for note in notes): - raise ValueError("command-plan notes must contain strings") - object.__setattr__(self, "notes", notes) + raise ValueError("command-plan paths must be absolute") @property def missing_inputs(self) -> tuple[Path, ...]: - """Required inputs that do not currently exist.""" - return tuple(path for path in self.required_inputs if not path.is_file()) @property def ready(self) -> bool: - """Whether the executable and all declared inputs are available.""" - - executable = self.argv[0] - command_path = Path(executable).expanduser() - if command_path.is_absolute(): - executable_available = command_path.is_file() and os.access(command_path, os.X_OK) - elif len(command_path.parts) > 1: - candidate = self.cwd / command_path - executable_available = candidate.is_file() and os.access(candidate, os.X_OK) - else: - executable_available = shutil.which(executable) is not None - return self.cwd.is_dir() and executable_available and not self.missing_inputs + return ( + self.cwd.is_dir() + and resolve_executable(self.argv[0]) is not None + and not self.missing_inputs + ) def to_dict(self) -> dict[str, Any]: - """Return a JSON-serializable audit record.""" - return { "plugin": self.plugin, "capability": self.capability.value, @@ -138,8 +82,6 @@ def to_dict(self) -> dict[str, Any]: @dataclass(frozen=True, slots=True) class DoctorReport: - """Availability and input-readiness report for one plugin.""" - plugin: str executable: str executable_path: Path | None @@ -162,7 +104,7 @@ def to_dict(self) -> dict[str, Any]: "executable_path": str(self.executable_path) if self.executable_path else None, "installed": self.installed, "ready": self.ready, - "capabilities": [capability.value for capability in self.capabilities], + "capabilities": [item.value for item in self.capabilities], "missing_inputs": [str(path) for path in self.missing_inputs], "messages": list(self.messages), } @@ -170,75 +112,38 @@ def to_dict(self) -> dict[str, Any]: @dataclass(frozen=True, slots=True) class FrameCommandPlan: - """A command plan associated with one discovered path frame.""" - index: int label: str directory: Path command: CommandPlan - def __post_init__(self) -> None: - if self.index < 0: - raise ValueError("frame index must be non-negative") - if not self.label or not self.label.strip(): - raise ValueError("frame label must not be empty") - directory = Path(self.directory) - if not directory.is_absolute(): - raise ValueError("frame directory must be absolute") - if self.command.cwd != directory: - raise ValueError("frame directory must match command-plan cwd") - object.__setattr__(self, "directory", directory) - def to_dict(self) -> dict[str, Any]: return { - "frame": { - "index": self.index, - "label": self.label, - "directory": str(self.directory), - }, + "frame": {"index": self.index, "label": self.label, "directory": str(self.directory)}, "command": self.command.to_dict(), } @dataclass(frozen=True, slots=True) class FrameDoctorReport: - """Read-only plugin checks associated with one discovered path frame.""" - index: int label: str directory: Path plugins: tuple[DoctorReport, ...] - def __post_init__(self) -> None: - if self.index < 0: - raise ValueError("frame index must be non-negative") - if not self.label or not self.label.strip(): - raise ValueError("frame label must not be empty") - directory = Path(self.directory) - if not directory.is_absolute(): - raise ValueError("frame directory must be absolute") - object.__setattr__(self, "directory", directory) - object.__setattr__(self, "plugins", tuple(self.plugins)) - @property def ready(self) -> bool: return all(report.ready for report in self.plugins) def to_dict(self) -> dict[str, Any]: return { - "frame": { - "index": self.index, - "label": self.label, - "directory": str(self.directory), - }, + "frame": {"index": self.index, "label": self.label, "directory": str(self.directory)}, "ready": self.ready, "plugins": [report.to_dict() for report in self.plugins], } def resolve_executable(executable: str) -> Path | None: - """Resolve an explicit path or PATH executable without launching it.""" - candidate = Path(executable).expanduser() if candidate.is_absolute() or len(candidate.parts) > 1: return ( @@ -249,8 +154,6 @@ def resolve_executable(executable: str) -> Path | None: class Plugin(Protocol): - """Structural interface implemented by built-in and third-party plugins.""" - name: str default_executable: str capabilities: tuple[Capability, ...] diff --git a/src/electrace/plugins/builtin.py b/src/electrace/plugins/builtin.py index 82b9fa4..86f519d 100644 --- a/src/electrace/plugins/builtin.py +++ b/src/electrace/plugins/builtin.py @@ -65,7 +65,6 @@ class _BuiltinPlugin: name = "" default_executable = "" capabilities: tuple[Capability, ...] = () - doctor_inputs: tuple[str, ...] = () def plan( self, @@ -124,7 +123,6 @@ class BaderKitPlugin(_BuiltinPlugin): Capability.ELF_BASINS, Capability.NON_NUCLEAR_ATTRACTORS, ) - doctor_inputs = ("CHGCAR",) def plan( self, @@ -177,7 +175,6 @@ class BaderPlugin(_BuiltinPlugin): name = "bader" default_executable = "bader" capabilities: tuple[Capability, ...] = (Capability.BADER_CHARGES,) - doctor_inputs = ("CHGCAR",) def plan( self, @@ -222,7 +219,6 @@ class Critic2Plugin(_BuiltinPlugin): Capability.QTAIM_CRITICAL_POINTS, Capability.NON_NUCLEAR_ATTRACTORS, ) - doctor_inputs = ("CHGCAR",) def plan( self, @@ -291,7 +287,6 @@ class LobsterPlugin(_BuiltinPlugin): name = "lobster" default_executable = "lobster" capabilities: tuple[Capability, ...] = (Capability.COHP, Capability.COBI) - doctor_inputs = ("lobsterin", "WAVECAR", "POSCAR", "POTCAR") def plan( self, @@ -333,7 +328,6 @@ class ChargemolPlugin(_BuiltinPlugin): name = "chargemol" default_executable = "chargemol" capabilities: tuple[Capability, ...] = (Capability.DDEC_CHARGES,) - doctor_inputs = ("job_control.txt",) def plan( self, diff --git a/src/electrace/plugins/registry.py b/src/electrace/plugins/registry.py index 5b4f6a7..5dc374d 100644 --- a/src/electrace/plugins/registry.py +++ b/src/electrace/plugins/registry.py @@ -1,4 +1,4 @@ -"""Plugin registration, lookup, capability filtering, and doctor checks.""" +"""Registry and convenience functions for built-in command planners.""" from __future__ import annotations @@ -6,22 +6,13 @@ from pathlib import Path from typing import Any, cast -from electrace.io.discovery import SeriesKind, discover_path +from electrace.io.discovery import discover_path -from .base import ( - Capability, - CommandPlan, - DoctorReport, - FrameCommandPlan, - FrameDoctorReport, - Plugin, -) +from .base import Capability, CommandPlan, DoctorReport, FrameCommandPlan, FrameDoctorReport, Plugin from .builtin import BUILTIN_PLUGINS class PluginRegistry: - """A small explicit registry; importing a module never runs a plugin.""" - def __init__(self, plugins: Iterable[Plugin] = ()) -> None: self._plugins: dict[str, Plugin] = {} for plugin in plugins: @@ -29,27 +20,25 @@ def __init__(self, plugins: Iterable[Plugin] = ()) -> None: def register(self, plugin: Plugin, *, replace: bool = False) -> None: name = plugin.name.strip().lower() - if not name: - raise ValueError("plugin name must not be empty") + if not name or not plugin.capabilities: + raise ValueError("plugin requires a name and capabilities") if name in self._plugins and not replace: raise ValueError(f"plugin {name!r} is already registered") - if not plugin.capabilities: - raise ValueError(f"plugin {name!r} declares no capabilities") self._plugins[name] = plugin def get(self, name: str) -> Plugin: - normalized = name.strip().lower() try: - return self._plugins[normalized] + return self._plugins[name.strip().lower()] except KeyError as exc: - choices = ", ".join(self.names()) or "none" - raise KeyError(f"unknown plugin {name!r}; registered plugins: {choices}") from exc + raise KeyError( + f"unknown plugin {name!r}; choose from {', '.join(self.names())}" + ) from exc def names(self) -> tuple[str, ...]: return tuple(sorted(self._plugins)) def for_capability(self, capability: Capability | str) -> tuple[Plugin, ...]: - selected = capability if isinstance(capability, Capability) else Capability(str(capability)) + selected = Capability(capability) return tuple(plugin for plugin in self._plugins.values() if selected in plugin.capabilities) def doctor( @@ -58,62 +47,21 @@ def doctor( *, configuration: dict[str, dict[str, Any]] | None = None, ) -> tuple[DoctorReport, ...]: - """Inspect executable and input availability without running commands.""" - - configuration = configuration or {} reports = [] for name in self.names(): - options = dict(configuration.get(name, {})) - if options.pop("enabled", True) is False: + options = dict((configuration or {}).get(name, {})) + if not options.pop("enabled", True): continue - executable = options.pop("executable", None) - capability = options.pop("capability", None) reports.append( self._plugins[name].doctor( frame_directory, - capability=capability, - executable=executable, + executable=options.pop("executable", None), + capability=options.pop("capability", None), **options, ) ) return tuple(reports) - def doctor_path( - self, - path: str | Path, - *, - configuration: dict[str, dict[str, Any]] | None = None, - mode: str | SeriesKind = "auto", - prefer_contcar: bool = True, - structure_name: str = "CONTCAR", - fallback_structure_name: str = "POSCAR", - strict: bool = True, - ) -> tuple[FrameDoctorReport, ...]: - """Inspect every discovered frame in deterministic path order.""" - - discovered = discover_path( - path, - mode=mode, - prefer_contcar=prefer_contcar, - structure_name=structure_name, - fallback_structure_name=fallback_structure_name, - strict=strict, - ) - return tuple( - FrameDoctorReport( - index=frame.index, - label=frame.label, - directory=frame.directory, - plugins=self.doctor( - frame.directory, - configuration=_configuration_for_frame( - configuration, critic2_structure_name=frame.structure.name - ), - ), - ) - for frame in discovered.frames - ) - def plan( self, name: str, @@ -123,19 +71,31 @@ def plan( configuration: dict[str, Any] | None = None, **options: Any, ) -> CommandPlan: - plugin = self.get(name) - configured = dict(configuration or {}) - if configured.pop("enabled", True) is False: + settings = dict(configuration or {}) + if not settings.pop("enabled", True): raise ValueError(f"plugin {name!r} is disabled") - executable = configured.pop("executable", None) - configured_capability = configured.pop("capability", None) - selected_capability = capability if capability is not None else configured_capability - configured.update(options) - return plugin.plan( + settings.update(options) + return self.get(name).plan( frame_directory, - capability=selected_capability, - executable=executable, - **configured, + capability=capability or settings.pop("capability", None), + executable=settings.pop("executable", None), + **settings, + ) + + def doctor_path(self, path: str | Path, **options: Any) -> tuple[FrameDoctorReport, ...]: + configuration = options.pop("configuration", None) + discovered = discover_path(path, **options) + return tuple( + FrameDoctorReport( + frame.index, + frame.label, + frame.directory, + self.doctor( + frame.directory, + configuration=_frame_configuration(configuration, frame.structure.name), + ), + ) + for frame in discovered.frames ) def plan_path( @@ -145,41 +105,26 @@ def plan_path( *, capability: Capability | str | None = None, configuration: dict[str, Any] | None = None, - mode: str | SeriesKind = "auto", - prefer_contcar: bool = True, - structure_name: str = "CONTCAR", - fallback_structure_name: str = "POSCAR", - strict: bool = True, **options: Any, ) -> tuple[FrameCommandPlan, ...]: - """Build one non-executing plan per discovered frame, in path order.""" - - discovered = discover_path( - path, - mode=mode, - prefer_contcar=prefer_contcar, - structure_name=structure_name, - fallback_structure_name=fallback_structure_name, - strict=strict, - ) + discovery_keys = { + key: options.pop(key) + for key in tuple(options) + if key + in {"mode", "prefer_contcar", "structure_name", "fallback_structure_name", "strict"} + } + discovered = discover_path(path, **discovery_keys) return tuple( FrameCommandPlan( - index=frame.index, - label=frame.label, - directory=frame.directory, - command=self.plan( + frame.index, + frame.label, + frame.directory, + self.plan( name, frame.directory, capability=capability, - configuration=( - { - **dict(configuration or {}), - "structure_file": frame.structure.name, - } - if name.strip().lower() == "critic2" - and "structure_file" not in (configuration or {}) - and "structure_file" not in options - else configuration + configuration=_single_frame_configuration( + name, configuration, frame.structure.name, options ), **options, ), @@ -188,16 +133,21 @@ def plan_path( ) -def _configuration_for_frame( - configuration: dict[str, dict[str, Any]] | None, - *, - critic2_structure_name: str, +def _frame_configuration( + configuration: dict[str, dict[str, Any]] | None, structure: str ) -> dict[str, dict[str, Any]]: - """Copy plugin configuration and bind critic2 to the discovered structure.""" + result = {name: dict(values) for name, values in (configuration or {}).items()} + result.setdefault("critic2", {}).setdefault("structure_file", structure) + return result + - result = {name: dict(options) for name, options in (configuration or {}).items()} - critic2 = result.setdefault("critic2", {}) - critic2.setdefault("structure_file", critic2_structure_name) +def _single_frame_configuration( + name: str, configuration: dict[str, Any] | None, structure: str, options: dict[str, Any] +) -> dict[str, Any] | None: + if name.strip().lower() != "critic2" or "structure_file" in options: + return configuration + result = dict(configuration or {}) + result.setdefault("structure_file", structure) return result @@ -205,81 +155,18 @@ def _configuration_for_frame( def doctor_plugins( - frame_directory: str | Path | None = None, - *, - configuration: dict[str, dict[str, Any]] | None = None, + frame_directory: str | Path | None = None, **options: Any ) -> tuple[DoctorReport, ...]: - """Run read-only availability checks for enabled built-in plugins.""" - - return DEFAULT_REGISTRY.doctor(frame_directory, configuration=configuration) - - -def doctor_plugins_path( - path: str | Path, - *, - configuration: dict[str, dict[str, Any]] | None = None, - mode: str | SeriesKind = "auto", - prefer_contcar: bool = True, - structure_name: str = "CONTCAR", - fallback_structure_name: str = "POSCAR", - strict: bool = True, -) -> tuple[FrameDoctorReport, ...]: - """Run read-only built-in plugin checks for every discovered frame.""" - - return DEFAULT_REGISTRY.doctor_path( - path, - configuration=configuration, - mode=mode, - prefer_contcar=prefer_contcar, - structure_name=structure_name, - fallback_structure_name=fallback_structure_name, - strict=strict, - ) + return DEFAULT_REGISTRY.doctor(frame_directory, **options) -def plan_plugin( - name: str, - frame_directory: str | Path, - *, - capability: Capability | str | None = None, - configuration: dict[str, Any] | None = None, - **options: Any, -) -> CommandPlan: - """Build, but do not execute, a command plan for a built-in plugin.""" +def doctor_plugins_path(path: str | Path, **options: Any) -> tuple[FrameDoctorReport, ...]: + return DEFAULT_REGISTRY.doctor_path(path, **options) - return DEFAULT_REGISTRY.plan( - name, - frame_directory, - capability=capability, - configuration=configuration, - **options, - ) +def plan_plugin(name: str, frame_directory: str | Path, **options: Any) -> CommandPlan: + return DEFAULT_REGISTRY.plan(name, frame_directory, **options) -def plan_plugin_path( - name: str, - path: str | Path, - *, - capability: Capability | str | None = None, - configuration: dict[str, Any] | None = None, - mode: str | SeriesKind = "auto", - prefer_contcar: bool = True, - structure_name: str = "CONTCAR", - fallback_structure_name: str = "POSCAR", - strict: bool = True, - **options: Any, -) -> tuple[FrameCommandPlan, ...]: - """Build deterministic non-executing plans for a complete VASP path.""" - return DEFAULT_REGISTRY.plan_path( - name, - path, - capability=capability, - configuration=configuration, - mode=mode, - prefer_contcar=prefer_contcar, - structure_name=structure_name, - fallback_structure_name=fallback_structure_name, - strict=strict, - **options, - ) +def plan_plugin_path(name: str, path: str | Path, **options: Any) -> tuple[FrameCommandPlan, ...]: + return DEFAULT_REGISTRY.plan_path(name, path, **options) diff --git a/src/electrace/provenance.py b/src/electrace/provenance.py index a794a9e..df626d1 100644 --- a/src/electrace/provenance.py +++ b/src/electrace/provenance.py @@ -72,7 +72,7 @@ def environment_record(*, project_directory: str | Path | None = None) -> dict[s root = Path(project_directory or Path.cwd()).resolve() packages = { name: value - for name in ("electrace", "numpy", "scipy", "ase", "h5py", "typer", "rich") + for name in ("electrace", "numpy", "scipy", "ase", "typer", "rich") if (value := _version(name)) is not None } return { diff --git a/src/electrace/volume.py b/src/electrace/volume.py index 374c89a..fd8f850 100644 --- a/src/electrace/volume.py +++ b/src/electrace/volume.py @@ -1,19 +1,12 @@ -"""Strict, dependency-light reading of plain VASP volumetric grids. - -The reader preserves raw values exactly as written by VASP. In particular, it -does not assign physical units or normalize CHGCAR values: those semantics -depend on the source file and calculation. Array indices follow -``data[ix, iy, iz]`` and VASP's x-fastest on-disk ordering. -""" +"""Streaming reader for plain VASP ELFCAR/CHGCAR-style scalar grids.""" from __future__ import annotations import math -import re from collections.abc import Iterator, Sequence from dataclasses import dataclass from pathlib import Path -from typing import Literal, TypeAlias +from typing import Literal, TypeAlias, cast import numpy as np import numpy.typing as npt @@ -25,58 +18,14 @@ FloatGrid: TypeAlias = npt.NDArray[np.float64] DEFAULT_MAX_POINTS = 100_000_000 -_INTEGER = re.compile(r"[+-]?\d+") class VaspGridError(ValueError): - """Raised when a VASP volumetric file is malformed or unsupported.""" - - -def _determinant(matrix: Matrix3) -> float: - a, b, c = matrix - return ( - a[0] * (b[1] * c[2] - b[2] * c[1]) - - a[1] * (b[0] * c[2] - b[2] * c[0]) - + a[2] * (b[0] * c[1] - b[1] * c[0]) - ) - - -def _inverse(matrix: Matrix3) -> Matrix3: - a, b, c = matrix - determinant = _determinant(matrix) - if abs(determinant) < 1.0e-14: - raise VaspGridError("VASP lattice is singular") - return ( - ( - (b[1] * c[2] - b[2] * c[1]) / determinant, - (a[2] * c[1] - a[1] * c[2]) / determinant, - (a[1] * b[2] - a[2] * b[1]) / determinant, - ), - ( - (b[2] * c[0] - b[0] * c[2]) / determinant, - (a[0] * c[2] - a[2] * c[0]) / determinant, - (a[2] * b[0] - a[0] * b[2]) / determinant, - ), - ( - (b[0] * c[1] - b[1] * c[0]) / determinant, - (a[1] * c[0] - a[0] * c[1]) / determinant, - (a[0] * b[1] - a[1] * b[0]) / determinant, - ), - ) - - -def _row_vector_times_matrix(vector: Vector3, matrix: Matrix3) -> Vector3: - return ( - sum(vector[row] * matrix[row][0] for row in range(3)), - sum(vector[row] * matrix[row][1] for row in range(3)), - sum(vector[row] * matrix[row][2] for row in range(3)), - ) + pass @dataclass(frozen=True, slots=True) class VaspGridMetadata: - """Structure and dataset metadata attached to one VASP scalar grid.""" - source: Path comment: str cell: Matrix3 @@ -90,158 +39,84 @@ class VaspGridMetadata: dataset_index: int dataset_count: int - def __post_init__(self) -> None: - object.__setattr__(self, "source", Path(self.source)) - if len(self.cell) != 3 or any(len(vector) != 3 for vector in self.cell): - raise ValueError("VASP grid cell must be a 3x3 row-vector matrix") - if not all(math.isfinite(component) for vector in self.cell for component in vector): - raise ValueError("VASP grid cell must contain only finite values") - if abs(_determinant(self.cell)) < 1.0e-14: - raise ValueError("VASP grid cell must be nonsingular") - if not self.species or len(self.species) != len(self.atom_counts): - raise ValueError("VASP grid species and atom-count lengths must match") - if any(not species for species in self.species): - raise ValueError("VASP grid species names must not be empty") - if any(count < 0 for count in self.atom_counts) or sum(self.atom_counts) == 0: - raise ValueError("VASP grid atom counts must be non-negative with a positive total") - atom_count = sum(self.atom_counts) - if len(self.fractional_positions) != atom_count: - raise ValueError("VASP grid position count does not match its atom counts") - if len(self.selective_dynamics) != atom_count: - raise ValueError("VASP grid selective-dynamics count does not match its atom counts") - if any( - not math.isfinite(component) - for position in self.fractional_positions - for component in position - ): - raise ValueError("VASP grid atom positions must contain only finite values") - if self.coordinate_mode not in {"direct", "cartesian"}: - raise ValueError("VASP grid coordinate mode must be direct or cartesian") - if len(self.shape) != 3 or any(size <= 0 for size in self.shape): - raise ValueError("VASP grid shape must contain three positive integers") - if self.dataset_count < 1: - raise ValueError("VASP grid dataset count must be positive") - if not 0 <= self.dataset_index < self.dataset_count: - raise ValueError("VASP grid dataset index is outside the available range") - @property def elements(self) -> tuple[str, ...]: - """Return one element label per atom, in POSCAR order.""" - return tuple( - species - for species, count in zip(self.species, self.atom_counts, strict=True) + symbol + for symbol, count in zip(self.species, self.atom_counts, strict=True) for _ in range(count) ) @property def point_count(self) -> int: - """Return the number of scalar samples in the selected dataset.""" - return math.prod(self.shape) @property def cell_volume(self) -> float: - """Return the positive cell volume in the header's length units cubed.""" - - return abs(_determinant(self.cell)) + return abs(float(np.linalg.det(self.cell))) @dataclass(frozen=True, slots=True) class VaspGrid: - """One immutable scalar dataset from a VASP ELFCAR/CHGCAR-like file.""" - metadata: VaspGridMetadata data: FloatGrid def __post_init__(self) -> None: - array = np.array(self.data, dtype=np.float64, copy=True, order="C") - if array.shape != self.metadata.shape: - raise ValueError( - f"VASP grid data shape {array.shape!r} does not match {self.metadata.shape!r}" - ) - if not bool(np.isfinite(array).all()): - raise ValueError("VASP grid data must contain only finite values") + array = np.asarray(self.data, dtype=np.float64).copy(order="C") + if array.shape != self.metadata.shape or not np.isfinite(array).all(): + raise ValueError("grid data must match its shape and contain finite values") array.setflags(write=False) object.__setattr__(self, "data", array) @property def shape(self) -> GridShape: - """Return the grid dimensions as ``(nx, ny, nz)``.""" - return self.metadata.shape def summary(self) -> dict[str, object]: - """Return a deterministic, JSON-compatible summary of this dataset.""" - return { "source": str(self.metadata.source), "dataset_index": self.metadata.dataset_index, "dataset_count": self.metadata.dataset_count, - "shape": list(self.metadata.shape), + "shape": list(self.shape), "point_count": self.metadata.point_count, "cell_volume": self.metadata.cell_volume, - "minimum": float(np.min(self.data)), - "maximum": float(np.max(self.data)), - "mean": float(np.mean(self.data)), + "minimum": float(self.data.min()), + "maximum": float(self.data.max()), + "mean": float(self.data.mean()), } def sample(self, fractional: Sequence[float]) -> float: - """Periodically trilinearly interpolate at a fractional coordinate. - - VASP samples are located at ``(ix/nx, iy/ny, iz/nz)`` and do not repeat - the periodic endpoint. Coordinates outside ``[0, 1)`` are wrapped. - """ - - if isinstance(fractional, (str, bytes)): - raise ValueError("fractional coordinate must contain three finite numbers") try: - coordinates = tuple(fractional) - except TypeError as exc: + coordinate = np.asarray(tuple(fractional), dtype=float) + except (TypeError, ValueError) as exc: raise ValueError("fractional coordinate must contain three finite numbers") from exc - if len(coordinates) != 3: + if coordinate.shape != (3,) or not np.isfinite(coordinate).all(): raise ValueError("fractional coordinate must contain three finite numbers") - - axes: list[tuple[int, int, float]] = [] - for coordinate, size in zip(coordinates, self.shape, strict=True): - if isinstance(coordinate, bool): - raise ValueError("fractional coordinate must contain three finite numbers") - try: - value = float(coordinate) - except (TypeError, ValueError) as exc: - raise ValueError("fractional coordinate must contain three finite numbers") from exc - if not math.isfinite(value): - raise ValueError("fractional coordinate must contain three finite numbers") - scaled = (value % 1.0) * size - floor = math.floor(scaled) - lower = floor % size - axes.append((lower, (lower + 1) % size, scaled - floor)) - + scaled = (coordinate % 1.0) * np.asarray(self.shape) + lower = np.floor(scaled).astype(int) + weight = scaled - lower result = 0.0 - for x_choice in range(2): - ix = axes[0][1] if x_choice else axes[0][0] - wx = axes[0][2] if x_choice else 1.0 - axes[0][2] - for y_choice in range(2): - iy = axes[1][1] if y_choice else axes[1][0] - wy = axes[1][2] if y_choice else 1.0 - axes[1][2] - for z_choice in range(2): - iz = axes[2][1] if z_choice else axes[2][0] - wz = axes[2][2] if z_choice else 1.0 - axes[2][2] - result += wx * wy * wz * float(self.data[ix, iy, iz]) + for mask in range(8): + indices = tuple( + int((lower[axis] + ((mask >> axis) & 1)) % self.shape[axis]) for axis in range(3) + ) + factor = math.prod( + weight[axis] if (mask >> axis) & 1 else 1.0 - weight[axis] for axis in range(3) + ) + result += factor * float(self.data[indices]) return result @dataclass(slots=True) -class _LineCursor: +class _Cursor: lines: Iterator[str] - line_number: int = 0 - - def next_nonblank(self, context: str, *, required: bool = True) -> tuple[str, int] | None: - for raw_line in self.lines: - self.line_number += 1 - line = raw_line.rstrip("\r\n") - if line.strip(): - return line, self.line_number + number: int = 0 + + def next(self, context: str, required: bool = True) -> str | None: + for raw in self.lines: + self.number += 1 + if raw.strip(): + return raw.strip() if required: raise VaspGridError(f"VASP volumetric file is missing {context}") return None @@ -253,284 +128,172 @@ def _float(token: str, context: str) -> float: except ValueError as exc: raise VaspGridError(f"invalid number {token!r} in {context}") from exc if not math.isfinite(value): - raise VaspGridError(f"non-finite number {token!r} in {context}") + raise VaspGridError(f"non-finite number in {context}") return value def _vector(line: str, context: str) -> Vector3: fields = line.split() if len(fields) < 3: - raise VaspGridError(f"expected three numbers in {context}, got {line!r}") - return ( - _float(fields[0], context), - _float(fields[1], context), - _float(fields[2], context), - ) - - -def _integer_tokens(line: str, context: str) -> tuple[int, ...]: - fields = line.split() - if not fields or any(_INTEGER.fullmatch(token) is None for token in fields): - raise VaspGridError(f"{context} must contain integers") - return tuple(int(token) for token in fields) - - -def _shape(line: str, context: str) -> GridShape: - dimensions = _integer_tokens(line, context) - if len(dimensions) != 3 or any(size <= 0 for size in dimensions): - raise VaspGridError(f"{context} must contain exactly three positive integers") - return dimensions + raise VaspGridError(f"expected three numbers in {context}") + return cast(Vector3, tuple(_float(token, context) for token in fields[:3])) -def _augmentation_line(line: str) -> bool: - return line.lstrip().casefold().startswith("augmentation") - - -def _read_values( - cursor: _LineCursor, - *, - point_count: int, - dataset_index: int, - keep: bool, -) -> FloatGrid | None: - values = np.empty(point_count, dtype=np.float64) if keep else None - consumed = 0 - while consumed < point_count: - item = cursor.next_nonblank(f"values for dataset {dataset_index}") - assert item is not None - line, line_number = item - if _augmentation_line(line): - raise VaspGridError( - "VASP augmentation blocks are not supported; provide a plain volumetric file" - ) - fields = line.split() - remaining = point_count - consumed - if len(fields) > remaining: - raise VaspGridError( - f"dataset {dataset_index} has extra values on line {line_number}; " - "dataset boundaries must occur between lines" - ) - for offset, token in enumerate(fields): - parsed = _float(token, f"dataset {dataset_index} line {line_number}") - if values is not None: - values[consumed + offset] = parsed - consumed += len(fields) +def _integers(line: str, context: str) -> tuple[int, ...]: + try: + values = tuple(int(token) for token in line.split()) + except ValueError as exc: + raise VaspGridError(f"{context} must contain integers") from exc + if not values: + raise VaspGridError(f"{context} is empty") return values -def _validate_request(dataset: int, max_points: int) -> None: - if isinstance(dataset, bool) or not isinstance(dataset, int) or dataset < 0: - raise ValueError("dataset must be a non-negative integer") - if isinstance(max_points, bool) or not isinstance(max_points, int) or max_points < 1: - raise ValueError("max_points must be a positive integer") +def _shape(line: str) -> GridShape: + values = _integers(line, "grid dimensions") + if len(values) != 3 or any(value <= 0 for value in values): + raise VaspGridError("grid dimensions must be three positive integers") + return values -def _parse_vasp_grid( - *, - source: Path, - comment: str, - cursor: _LineCursor, - dataset: int, - max_points: int, -) -> VaspGrid: - scale_item = cursor.next_nonblank("scale factor") - assert scale_item is not None - scale_fields = scale_item[0].split() - if len(scale_fields) not in {1, 3}: - raise VaspGridError("VASP scale line must contain one or three numbers") - scales = tuple(_float(token, "VASP scale") for token in scale_fields) - - raw_cell_rows: list[Vector3] = [] - for vector_index in range(3): - item = cursor.next_nonblank(f"lattice vector {vector_index + 1}") - assert item is not None - raw_cell_rows.append(_vector(item[0], f"lattice vector {vector_index + 1}")) - raw_cell: Matrix3 = tuple(raw_cell_rows) # type: ignore[assignment] - raw_volume = abs(_determinant(raw_cell)) - if raw_volume < 1.0e-14: +def _values(cursor: _Cursor, count: int, index: int, keep: bool) -> FloatGrid | None: + result = np.empty(count) if keep else None + consumed = 0 + while consumed < count: + line = cursor.next(f"values for dataset {index}") + assert line is not None + if line.casefold().startswith("augmentation"): + raise VaspGridError("VASP augmentation blocks are not supported") + tokens = line.split() + if consumed + len(tokens) > count: + raise VaspGridError(f"dataset {index} has extra values") + for offset, token in enumerate(tokens): + value = _float(token, f"dataset {index}") + if result is not None: + result[consumed + offset] = value + consumed += len(tokens) + return result + + +def _parse(source: Path, comment: str, cursor: _Cursor, dataset: int, max_points: int) -> VaspGrid: + scale_line = cursor.next("scale factor") + assert scale_line is not None + scales = tuple(_float(token, "scale") for token in scale_line.split()) + if len(scales) not in {1, 3}: + raise VaspGridError("scale line must contain one or three numbers") + raw_cell = np.asarray( + [_vector(cast(str, cursor.next("lattice vector")), "lattice vector") for _ in range(3)] + ) + raw_volume = abs(float(np.linalg.det(raw_cell))) + if raw_volume < 1e-14: raise VaspGridError("VASP lattice is singular") - - coordinate_scale: Vector3 if len(scales) == 1: scale = scales[0] - if scale == 0.0: - raise VaspGridError("VASP scale factor cannot be zero") - if scale < 0.0: - scale = (-scale / raw_volume) ** (1.0 / 3.0) - cell = tuple(tuple(component * scale for component in vector) for vector in raw_cell) - coordinate_scale = (scale, scale, scale) + if scale == 0: + raise VaspGridError("scale factor cannot be zero") + scale = (-scale / raw_volume) ** (1 / 3) if scale < 0 else scale + cell_array = raw_cell * scale + coordinate_scale = np.full(3, scale) else: - if any(scale <= 0.0 for scale in scales): - raise VaspGridError("three VASP scale factors must all be positive") - cell = tuple( - tuple(raw_cell[row][axis] * scales[axis] for axis in range(3)) for row in range(3) - ) - coordinate_scale = scales # type: ignore[assignment] - typed_cell: Matrix3 = cell # type: ignore[assignment] - - symbols_item = cursor.next_nonblank("element symbols or atom counts") - assert symbols_item is not None - symbol_tokens = symbols_item[0].split() - symbols_inferred = all(_INTEGER.fullmatch(token) is not None for token in symbol_tokens) - if symbols_inferred: - atom_counts = _integer_tokens(symbols_item[0], "VASP atom counts") - species = tuple(f"X{index + 1}" for index in range(len(atom_counts))) + if any(value <= 0 for value in scales): + raise VaspGridError("three scale factors must be positive") + coordinate_scale = np.asarray(scales) + cell_array = raw_cell * coordinate_scale + cell = cast(Matrix3, tuple(map(tuple, cell_array.tolist()))) + + symbol_line = cast(str, cursor.next("element symbols or atom counts")) + symbols = symbol_line.split() + inferred = all(token.lstrip("+-").isdigit() for token in symbols) + if inferred: + counts = _integers(symbol_line, "atom counts") + species = tuple(f"X{index + 1}" for index in range(len(counts))) else: - species = tuple(symbol_tokens) - counts_item = cursor.next_nonblank("atom counts") - assert counts_item is not None - atom_counts = _integer_tokens(counts_item[0], "VASP atom counts") - if len(species) != len(atom_counts): - raise VaspGridError("VASP element symbol and atom-count lengths differ") - if any(count < 0 for count in atom_counts) or sum(atom_counts) == 0: - raise VaspGridError("VASP atom counts must be non-negative with a positive total") - - mode_item = cursor.next_nonblank("coordinate mode") - assert mode_item is not None - has_selective = mode_item[0].strip().casefold().startswith("s") - if has_selective: - mode_item = cursor.next_nonblank("coordinate mode after Selective dynamics") - assert mode_item is not None - mode_token = mode_item[0].strip().casefold() - if mode_token.startswith("d"): - coordinate_mode: Literal["direct", "cartesian"] = "direct" - elif mode_token.startswith(("c", "k")): - coordinate_mode = "cartesian" + species = tuple(symbols) + counts = _integers(cast(str, cursor.next("atom counts")), "atom counts") + if len(species) != len(counts) or any(count < 0 for count in counts) or not sum(counts): + raise VaspGridError("invalid element symbols or atom counts") + + mode_line = cast(str, cursor.next("coordinate mode")) + selective = mode_line.casefold().startswith("s") + if selective: + mode_line = cast(str, cursor.next("coordinate mode")) + if mode_line.casefold().startswith("d"): + mode: Literal["direct", "cartesian"] = "direct" + elif mode_line.casefold().startswith(("c", "k")): + mode = "cartesian" else: - raise VaspGridError(f"unknown VASP coordinate mode {mode_item[0]!r}") - - inverse_cell = _inverse(typed_cell) - fractional_positions: list[Vector3] = [] - selective_dynamics: list[SelectiveFlags | None] = [] - for atom_index in range(sum(atom_counts)): - position_item = cursor.next_nonblank(f"coordinates for atom {atom_index + 1}") - assert position_item is not None - fields = position_item[0].split() - position = _vector(position_item[0], f"atom {atom_index + 1} coordinates") - if coordinate_mode == "cartesian": - cartesian: Vector3 = tuple(position[axis] * coordinate_scale[axis] for axis in range(3)) # type: ignore[assignment] - position = _row_vector_times_matrix(cartesian, inverse_cell) - fractional_positions.append(position) - if has_selective: - if len(fields) < 6 or any(flag.casefold() not in {"t", "f"} for flag in fields[3:6]): - raise VaspGridError( - f"atom {atom_index + 1} must have three T/F selective-dynamics flags" - ) - selective_dynamics.append( - tuple(flag.casefold() == "t" for flag in fields[3:6]) # type: ignore[arg-type] + raise VaspGridError(f"unknown coordinate mode {mode_line!r}") + + positions: list[Vector3] = [] + flags: list[SelectiveFlags | None] = [] + inverse = np.linalg.inv(cell_array) + for atom in range(sum(counts)): + line = cast(str, cursor.next(f"coordinates for atom {atom + 1}")) + fields = line.split() + position = np.asarray(_vector(line, "atom coordinates")) + if mode == "cartesian": + position = (position * coordinate_scale) @ inverse + positions.append(cast(Vector3, tuple(position.tolist()))) + if selective: + if len(fields) < 6 or any(value.casefold() not in {"t", "f"} for value in fields[3:6]): + raise VaspGridError("invalid selective-dynamics flags") + flags.append( + cast(SelectiveFlags, tuple(value.casefold() == "t" for value in fields[3:6])) ) else: - selective_dynamics.append(None) + flags.append(None) - header_item = cursor.next_nonblank("grid dimensions") - assert header_item is not None - grid_shape = _shape(header_item[0], "VASP grid dimensions") - point_count = math.prod(grid_shape) + shape = _shape(cast(str, cursor.next("grid dimensions"))) + point_count = math.prod(shape) if point_count > max_points: - raise VaspGridError( - f"VASP grid has {point_count} points, exceeding max_points={max_points}" - ) - + raise VaspGridError(f"grid exceeds max_points={max_points}") + selected = _values(cursor, point_count, 0, dataset == 0) dataset_count = 1 - selected_values = _read_values( - cursor, - point_count=point_count, - dataset_index=0, - keep=dataset == 0, - ) - while True: - next_item = cursor.next_nonblank("another dataset", required=False) - if next_item is None: - break - line, _line_number = next_item - if _augmentation_line(line): - raise VaspGridError( - "VASP augmentation blocks are not supported; provide a plain volumetric file" - ) - try: - next_shape = _shape(line, f"dataset {dataset_count} grid dimensions") - except VaspGridError as exc: - raise VaspGridError( - "unsupported content follows the plain VASP grid; augmentation and " - "unlabeled tails are not supported" - ) from exc - if next_shape != grid_shape: - raise VaspGridError( - f"dataset {dataset_count} shape {next_shape!r} differs from first dataset " - f"shape {grid_shape!r}" - ) - values = _read_values( - cursor, - point_count=point_count, - dataset_index=dataset_count, - keep=dataset == dataset_count, - ) - if values is not None: - selected_values = values + while (next_dataset := cursor.next("another dataset", required=False)) is not None: + if next_dataset.casefold().startswith("augmentation"): + raise VaspGridError("VASP augmentation blocks are not supported") + if _shape(next_dataset) != shape: + raise VaspGridError("all datasets must use the same shape") + values = _values(cursor, point_count, dataset_count, dataset == dataset_count) + selected = values if values is not None else selected dataset_count += 1 - - if dataset >= dataset_count or selected_values is None: - raise VaspGridError( - f"dataset {dataset} is unavailable; file contains {dataset_count} dataset(s)" - ) - + if selected is None: + raise VaspGridError(f"dataset {dataset} is unavailable") metadata = VaspGridMetadata( - source=source, - comment=comment, - cell=typed_cell, - species=species, - atom_counts=atom_counts, - fractional_positions=tuple(fractional_positions), - selective_dynamics=tuple(selective_dynamics), - coordinate_mode=coordinate_mode, - symbols_inferred=symbols_inferred, - shape=grid_shape, - dataset_index=dataset, - dataset_count=dataset_count, + source, + comment, + cell, + species, + counts, + tuple(positions), + tuple(flags), + mode, + inferred, + shape, + dataset, + dataset_count, ) - data = np.reshape(selected_values, grid_shape, order="F") - return VaspGrid(metadata=metadata, data=data) + return VaspGrid(metadata, np.reshape(selected, shape, order="F")) def read_vasp_grid( - path: str | Path, - dataset: int = 0, - max_points: int = DEFAULT_MAX_POINTS, + path: str | Path, dataset: int = 0, max_points: int = DEFAULT_MAX_POINTS ) -> VaspGrid: - """Read one plain scalar dataset from a VASP volumetric file. - - VASP 4 and VASP 5 headers, optional selective dynamics, Direct and - Cartesian coordinates, blank separator lines, and repeated same-shape - plain datasets are supported. Augmentation occupancies and any other tail - format are rejected explicitly rather than skipped or guessed. The file - is streamed so memory use scales with the selected numeric dataset, not - with the size of its text representation. - """ - - _validate_request(dataset, max_points) + if isinstance(dataset, bool) or not isinstance(dataset, int) or dataset < 0: + raise ValueError("dataset must be a non-negative integer") + if isinstance(max_points, bool) or not isinstance(max_points, int) or max_points < 1: + raise ValueError("max_points must be positive") source = Path(path).expanduser().resolve() try: with source.open("r", encoding="utf-8") as stream: - comment_line = stream.readline() - if not comment_line: + comment = stream.readline() + if not comment: raise VaspGridError(f"VASP volumetric file is empty: {source}") - return _parse_vasp_grid( - source=source, - comment=comment_line.strip(), - cursor=_LineCursor(iter(stream), line_number=1), - dataset=dataset, - max_points=max_points, - ) + return _parse(source, comment.strip(), _Cursor(iter(stream), 1), dataset, max_points) except VaspGridError: raise except (OSError, UnicodeError) as exc: raise VaspGridError(f"cannot read VASP volumetric file {source}: {exc}") from exc -__all__ = [ - "DEFAULT_MAX_POINTS", - "VaspGrid", - "VaspGridError", - "VaspGridMetadata", - "read_vasp_grid", -] +__all__ = ["DEFAULT_MAX_POINTS", "VaspGrid", "VaspGridError", "VaspGridMetadata", "read_vasp_grid"] diff --git a/src/electrace/workflow/_files.py b/src/electrace/workflow/_files.py index 9df834c..6c406d3 100644 --- a/src/electrace/workflow/_files.py +++ b/src/electrace/workflow/_files.py @@ -21,21 +21,6 @@ def symlink_component(path: Path) -> Path | None: return next((component for component in (path, *path.parents) if component.is_symlink()), None) -def read_limited_text(path: Path, *, limit: int, context: str) -> str: - """Read UTF-8 text while enforcing a byte limit during the read.""" - - if limit < 1: - raise ValueError("text read limit must be positive") - with path.open("rb") as handle: - payload = handle.read(limit + 1) - if len(payload) > limit: - raise ValueError(f"{context} exceeds {limit} bytes: {path}") - try: - return payload.decode("utf-8") - except UnicodeDecodeError as exc: - raise ValueError(f"{context} is not valid UTF-8: {path}") from exc - - def atomic_write_json(path: Path, payload: object) -> Path: """Atomically publish deterministic, standards-compliant JSON.""" diff --git a/src/electrace/workflow/collection.py b/src/electrace/workflow/collection.py index 21be68a..9be8552 100644 --- a/src/electrace/workflow/collection.py +++ b/src/electrace/workflow/collection.py @@ -1,18 +1,10 @@ -"""Collect completed workflow evidence into an isolated ElecTrace analysis. - -Collection is deliberately a copy boundary. Original structures and external -tool outputs remain read-only; analysis consumes only a marker-owned directory -assembled beside the staged workspace and published after complete validation. -""" +"""Collect completed workflow outputs into a canonical analysis tree.""" from __future__ import annotations import json import math -import os -import re import shutil -import stat import tempfile from dataclasses import dataclass from pathlib import Path @@ -28,154 +20,33 @@ ) from electrace.io.descriptors import load_descriptor from electrace.io.vasp import read_poscar -from electrace.models import Frame from electrace.pipeline import AnalysisArtifacts, run_project -from electrace.provenance import sha256_file -from ._files import absolute_lexical, read_limited_text, symlink_component from .manifest import load_workflow_manifest from .models import WorkflowManifest -from .staging import StagedFrame, StagedWorkflow, descriptor_outputs, load_staged_workflow +from .staging import StagedWorkflow, descriptor_outputs, load_staged_workflow COLLECTION_SCHEMA_VERSION = "electrace.collection.v1" COLLECTION_DIRECTORY = "analysis" COLLECTION_RECORD = ".electrace-collection.json" -MAX_COLLECTION_RECORD_BYTES = 16_000_000 -_ANALYSIS_LABEL = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$") @dataclass(frozen=True, slots=True) class CollectionArtifacts: - """Paths published by a successful workflow collection.""" - config_path: Path input_root: Path descriptor_root: Path files: tuple[Path, ...] -@dataclass(frozen=True, slots=True) -class _FilePlan: - relative_path: Path - role: str - frame: StagedFrame | None = None - source: Path | None = None - content: str | None = None - job_id: str | None = None - - def __post_init__(self) -> None: - if self.relative_path.is_absolute() or ".." in self.relative_path.parts: - raise ValueError(f"collection destination must be relative: {self.relative_path}") - if (self.source is None) == (self.content is None): - raise ValueError("collection file plan must have exactly one source or content") - - -def _reject_symlink_path(path: Path, context: str) -> None: - component = symlink_component(path) - if component is not None: - raise ValueError(f"{context} must not traverse a symlink: {component}") - - -def _within( - path: Path, - root: Path, - context: str, - *, - allow_root: bool = False, -) -> Path: - if not path.is_absolute(): - raise ValueError(f"{context} must be an absolute path: {path}") - normalized = Path(os.path.normpath(path)) - normalized_root = Path(os.path.normpath(root)) - if (normalized == normalized_root and not allow_root) or ( - normalized != normalized_root and normalized_root not in normalized.parents - ): - raise ValueError(f"{context} escapes its managed root {normalized_root}: {normalized}") - return normalized - - -def _require_directory(path: Path, context: str) -> None: - _reject_symlink_path(path, context) - if not path.is_dir(): - raise FileNotFoundError(f"{context} is missing or not a directory: {path}") - - -def _require_regular_file(path: Path, context: str, *, nonempty: bool = True) -> os.stat_result: - _reject_symlink_path(path, context) - try: - status = path.lstat() - except FileNotFoundError as exc: - raise FileNotFoundError(f"{context} is missing: {path}") from exc - if not stat.S_ISREG(status.st_mode): - raise ValueError(f"{context} is not a regular file: {path}") - if nonempty and status.st_size == 0: - raise ValueError(f"{context} is empty: {path}") - return status - - -def _safe_analysis_label(value: object, order: int) -> str: - if not isinstance(value, str) or _ANALYSIS_LABEL.fullmatch(value) is None: - raise ValueError(f"unsafe staged analysis_label: {value!r}") - if not value.startswith(f"{order:06d}-"): - raise ValueError( - f"staged analysis_label {value!r} does not preserve selected-frame order {order}" - ) - return value - - -def _copy_regular_file(source: Path, destination: Path, context: str) -> None: - """Copy through a no-follow descriptor and reject concurrent replacement.""" - - before = _require_regular_file(source, context) - flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) - descriptor = os.open(source, flags) - destination.parent.mkdir(parents=True, exist_ok=True) - try: - opened = os.fstat(descriptor) - if (opened.st_dev, opened.st_ino) != (before.st_dev, before.st_ino): - raise ValueError(f"{context} changed while collection was starting: {source}") - with ( - os.fdopen(descriptor, "rb", closefd=False) as source_handle, - destination.open("xb") as destination_handle, - ): - shutil.copyfileobj(source_handle, destination_handle) - after_open = os.fstat(descriptor) - if (after_open.st_size, after_open.st_mtime_ns) != ( - opened.st_size, - opened.st_mtime_ns, - ): - raise ValueError(f"{context} changed while it was being collected: {source}") - finally: - os.close(descriptor) - after_path = _require_regular_file(source, context) - if (after_path.st_dev, after_path.st_ino) != (before.st_dev, before.st_ino): - raise ValueError(f"{context} was replaced while it was being collected: {source}") - - -def _write_text(path: Path, content: str) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - with path.open("x", encoding="utf-8", newline="\n") as handle: - handle.write(content) - - -def _write_json_exclusive(path: Path, payload: object) -> None: - _write_text(path, json.dumps(payload, indent=2, sort_keys=True, allow_nan=False) + "\n") - - -def _toml_string(value: str) -> str: - return json.dumps(value, ensure_ascii=False) - - def _toml_value(value: object) -> str: if isinstance(value, str): - return _toml_string(value) + return json.dumps(value, ensure_ascii=False) if type(value) is bool: return "true" if value else "false" if type(value) is int: return str(value) - if type(value) is float: - if not math.isfinite(value): - raise ValueError("generated TOML values must be finite") + if type(value) is float and math.isfinite(value): return repr(value) if isinstance(value, list): return "[" + ", ".join(_toml_value(item) for item in value) + "]" @@ -183,17 +54,6 @@ def _toml_value(value: object) -> str: def _collection_config(staged: StagedWorkflow, final_root: Path) -> ElecTraceConfig: - metadata: dict[str, Any] = { - "collection_schema_version": COLLECTION_SCHEMA_VERSION, - "collection_record": str(final_root / COLLECTION_RECORD), - "workflow_manifest": str(staged.manifest_path), - "workflow_manifest_sha256": staged.manifest_sha256, - "workflow_workspace_record": str(staged.record_path), - "workflow_source_root": str(staged.source_root), - "workflow_frame_indices": [frame.index for frame in staged.frames], - "workflow_frame_labels": [frame.label for frame in staged.frames], - "workflow_analysis_labels": [frame.analysis_label for frame in staged.frames], - } return ElecTraceConfig( project_name="Collected ElecTrace workflow", series_kind="ordered", @@ -209,28 +69,39 @@ def _collection_config(staged: StagedWorkflow, final_root: Path) -> ElecTraceCon matching=MatchingSettings(), events=EventSettings(), outputs=OutputConfig(directory="results"), - metadata=metadata, + metadata={ + "collection_schema_version": COLLECTION_SCHEMA_VERSION, + "collection_record": str(final_root / COLLECTION_RECORD), + "workflow_manifest": str(staged.manifest_path), + "workflow_manifest_sha256": staged.manifest_sha256, + "workflow_workspace_record": str(staged.record_path), + "workflow_source_root": str(staged.source_root), + "workflow_frame_indices": [frame.index for frame in staged.frames], + "workflow_frame_labels": [frame.label for frame in staged.frames], + "workflow_analysis_labels": [frame.analysis_label for frame in staged.frames], + }, ) def _render_config(config: ElecTraceConfig) -> str: - matching = config.matching - events = config.events - output = config.outputs - inputs = config.inputs + inputs, matching, events, output = ( + config.inputs, + config.matching, + config.events, + config.outputs, + ) lines = [ - "# Generated by ElecTrace workflow collection. Original inputs remain read-only.", "[project]", - f"name = {_toml_string(config.project_name)}", - f"series_kind = {_toml_string(config.series_kind)}", + f"name = {_toml_value(config.project_name)}", + f"series_kind = {_toml_value(config.series_kind)}", "", "[inputs]", - f"root = {_toml_string(inputs.root)}", - f"descriptor_dir = {_toml_string(inputs.descriptor_dir or '')}", - f"structure_name = {_toml_string(inputs.structure_name)}", - f"fallback_structure_name = {_toml_string(inputs.fallback_structure_name)}", - f"descriptor_name = {_toml_string(inputs.descriptor_name)}", - f"descriptor_source = {_toml_string(inputs.descriptor_source)}", + f"root = {_toml_value(inputs.root)}", + f"descriptor_dir = {_toml_value(inputs.descriptor_dir or '')}", + f"structure_name = {_toml_value(inputs.structure_name)}", + f"fallback_structure_name = {_toml_value(inputs.fallback_structure_name)}", + f"descriptor_name = {_toml_value(inputs.descriptor_name)}", + f"descriptor_source = {_toml_value(inputs.descriptor_source)}", f"prefer_contcar = {_toml_value(inputs.prefer_contcar)}", "", "[matching]", @@ -245,7 +116,6 @@ def _render_config(config: ElecTraceConfig) -> str: "overlap_weight", "descriptor_weight", "match_threshold", - "ambiguity_tolerance", "conservation_tolerance", "split_merge_threshold", ): @@ -253,7 +123,6 @@ def _render_config(config: ElecTraceConfig) -> str: lines.extend(("", "[events]")) for name in ( "minimum_persistence", - "hysteresis_frames", "minimum_event_confidence", "charge_change_threshold", "volume_change_fraction", @@ -270,11 +139,10 @@ def _render_config(config: ElecTraceConfig) -> str: ( "", "[outputs]", - f"directory = {_toml_string(output.directory)}", + f"directory = {_toml_value(output.directory)}", f"html_report = {_toml_value(output.html_report)}", f"json = {_toml_value(output.json)}", f"csv = {_toml_value(output.csv)}", - f"hdf5 = {_toml_value(output.hdf5)}", "", "[metadata]", ) @@ -283,141 +151,20 @@ def _render_config(config: ElecTraceConfig) -> str: return "\n".join(lines) + "\n" -def _expected_descriptor_job_ids( - staged: StagedWorkflow, manifest: WorkflowManifest -) -> tuple[str, ...]: - descriptor_steps = tuple(step for step in manifest.steps if step.kind != "vasp") - if not descriptor_steps: +def _expected_descriptor_ids(staged: StagedWorkflow, manifest: WorkflowManifest) -> tuple[str, ...]: + steps = tuple(step for step in manifest.steps if step.kind != "vasp") + if not steps: raise ValueError("workflow has no descriptor steps to collect") - return tuple( - f"f{frame.index:06d}.{step.id}" for frame in staged.frames for step in descriptor_steps - ) - - -def _collection_plans(staged: StagedWorkflow, manifest: WorkflowManifest) -> tuple[_FilePlan, ...]: - _require_directory(staged.workspace, "staged workflow workspace") - _require_directory(staged.source_root, "workflow source root") - _require_regular_file(staged.record_path, "staged workflow record") - if not staged.frames: - raise ValueError("staged workflow has no selected frames") - - labels: set[str] = set() - frames_by_index: dict[int, StagedFrame] = {} - parsed_frames: dict[int, Frame] = {} - plans: list[_FilePlan] = [] - destinations: dict[Path, str] = {} - for order, frame in enumerate(staged.frames): - label = _safe_analysis_label(frame.analysis_label, order) - if label in labels: - raise ValueError(f"duplicate staged analysis_label: {label!r}") - labels.add(label) - if frame.index in frames_by_index: - raise ValueError(f"duplicate staged frame index: {frame.index}") - frames_by_index[frame.index] = frame - source_directory = _within( - frame.source_directory, - staged.source_root, - "staged frame source directory", - allow_root=True, - ) - _require_directory(source_directory, "staged frame source directory") - structure = _within(frame.structure, source_directory, "selected structure") - _require_regular_file(structure, "selected structure") - parsed_frames[frame.index] = read_poscar( - structure, - index=frame.index, - label=frame.label, - energy=frame.energy_ev, - ) - frame_workspace = _within( - frame.workspace_directory, staged.workspace, "staged frame workspace" - ) - _require_directory(frame_workspace, "staged frame workspace") - structure_destination = Path("input") / label / "CONTCAR" - destinations[structure_destination] = "selected structure" - plans.append( - _FilePlan( - relative_path=structure_destination, - role="structure", - frame=frame, - source=structure, - ) - ) - if frame.energy_ev is not None: - if not math.isfinite(frame.energy_ev): - raise ValueError(f"recorded energy for frame {frame.label!r} must be finite") - energy_destination = Path("input") / label / "OSZICAR" - destinations[energy_destination] = "recorded energy" - energy = f" 1 F= {frame.energy_ev:.16g} E0= {frame.energy_ev:.16g} d E = 0.0\n" - plans.append( - _FilePlan( - relative_path=energy_destination, - role="synthetic_energy", - frame=frame, - content=energy, - ) - ) - - expected_ids = _expected_descriptor_job_ids(staged, manifest) - if staged.descriptor_job_ids != expected_ids: - raise ValueError("staged descriptor job ids do not match the workflow manifest") - jobs = {job.id: job for job in staged.jobs} - recognized = descriptor_outputs(staged) - for job_id in expected_ids: - job = jobs.get(job_id) - if job is None: - raise ValueError(f"staged workflow is missing descriptor job {job_id!r}") - descriptor_frame = frames_by_index.get(job.frame_index) - if descriptor_frame is None: - raise ValueError(f"descriptor job {job_id!r} refers to an unselected frame") - if job.frame_label != descriptor_frame.label: - raise ValueError(f"descriptor job {job_id!r} has inconsistent frame provenance") - cwd = _within(job.cwd, staged.workspace, f"descriptor job {job_id!r} directory") - _require_directory(cwd, f"descriptor job {job_id!r} directory") - outputs = recognized.get(job_id, ()) - if not outputs: - raise ValueError( - f"descriptor job {job_id!r} declares no parser-supported output; " - "use a canonical output filename" - ) - for source in outputs: - source = _within(source, cwd, f"parser-supported descriptor output for job {job_id!r}") - _require_regular_file(source, f"parser-supported descriptor output for job {job_id!r}") - load_descriptor(source, frame=parsed_frames[job.frame_index]) - destination = Path("descriptors") / descriptor_frame.analysis_label / source.name - previous = destinations.get(destination) - if previous is not None: - raise ValueError( - f"descriptor output collision at {destination}: {previous} and {job_id}" - ) - destinations[destination] = job_id - plans.append( - _FilePlan( - relative_path=destination, - role="descriptor", - frame=descriptor_frame, - source=source, - job_id=job_id, - ) - ) - return tuple(plans) + return tuple(f"f{frame.index:06d}.{step.id}" for frame in staged.frames for step in steps) -def _collection_marker(path: Path, staged: StagedWorkflow) -> dict[str, Any] | None: - if path.is_symlink() or not path.is_dir(): - return None +def _managed(path: Path, staged: StagedWorkflow) -> dict[str, Any] | None: marker = path / COLLECTION_RECORD - if marker.is_symlink() or not marker.is_file(): + if path.is_symlink() or marker.is_symlink() or not marker.is_file(): return None try: - payload = json.loads( - read_limited_text( - marker, - limit=MAX_COLLECTION_RECORD_BYTES, - context="collection marker", - ) - ) - except (OSError, RecursionError, ValueError, json.JSONDecodeError): + payload = json.loads(marker.read_text(encoding="utf-8")) + except (OSError, ValueError, json.JSONDecodeError): return None if not ( isinstance(payload, dict) @@ -430,70 +177,14 @@ def _collection_marker(path: Path, staged: StagedWorkflow) -> dict[str, Any] | N return payload -def _collection_is_managed(path: Path, staged: StagedWorkflow) -> bool: - return _collection_marker(path, staged) is not None - - -def _existing_collection_artifacts(path: Path, staged: StagedWorkflow) -> CollectionArtifacts: - _require_directory(staged.workspace, "staged workflow workspace") - payload = _collection_marker(path, staged) - if payload is None: - raise ValueError(f"existing collection is not marker-bound to this workflow: {path}") - input_root = path / "input" - descriptor_root = path / "descriptors" - config_path = path / "electrace.toml" - results = path / "results" - _require_directory(input_root, "collected input root") - _require_directory(descriptor_root, "collected descriptor root") - _require_directory(results, "collected results directory") - _require_regular_file(config_path, "collected ElecTrace configuration") - - raw_records = payload.get("files") - if not isinstance(raw_records, list): - raise ValueError("collection marker files must be an array") - files: list[Path] = [] - seen: set[Path] = set() - for index, raw_record in enumerate(raw_records): - if not isinstance(raw_record, dict): - raise ValueError(f"collection marker files[{index}] must be an object") - raw_relative = raw_record.get("path") - if not isinstance(raw_relative, str): - raise ValueError(f"collection marker files[{index}].path must be a string") - relative = Path(raw_relative) - if relative.is_absolute() or ".." in relative.parts or relative in seen: - raise ValueError(f"unsafe or duplicate collection marker path: {raw_relative!r}") - seen.add(relative) - file_path = _within(path / relative, path, "collected file") - status = _require_regular_file(file_path, "collected file") - if raw_record.get("size_bytes") != status.st_size: - raise ValueError(f"collected file size does not match marker: {file_path}") - if raw_record.get("sha256") != sha256_file(file_path): - raise ValueError(f"collected file hash does not match marker: {file_path}") - files.append(file_path) - if Path("electrace.toml") not in seen: - raise ValueError("collection marker does not include electrace.toml") - files.append(path / COLLECTION_RECORD) - return CollectionArtifacts( - config_path=config_path, - input_root=input_root, - descriptor_root=descriptor_root, - files=tuple(sorted(files, key=lambda item: item.relative_to(path).as_posix())), - ) - - -def _publish_collection( - temporary: Path, target: Path, staged: StagedWorkflow, *, overwrite: bool -) -> None: - if not (target.exists() or target.is_symlink()): +def _publish(temporary: Path, target: Path, staged: StagedWorkflow, overwrite: bool) -> None: + if not target.exists(): temporary.replace(target) return if not overwrite: - raise FileExistsError( - f"collected analysis already exists: {target}; use overwrite=True to replace it" - ) - if not _collection_is_managed(target, staged): - raise ValueError(f"refusing to replace unmarked or symlinked collection: {target}") - + raise FileExistsError(f"collected analysis already exists: {target}") + if _managed(target, staged) is None: + raise ValueError(f"refusing to replace unmarked collection: {target}") backup = Path(tempfile.mkdtemp(prefix=".analysis.previous-", dir=target.parent)) backup.rmdir() target.replace(backup) @@ -505,60 +196,20 @@ def _publish_collection( shutil.rmtree(backup) -def _marker_payload( - staged: StagedWorkflow, - final_root: Path, - temporary: Path, - plans: tuple[_FilePlan, ...], -) -> dict[str, Any]: - provenance = {plan.relative_path: plan for plan in plans} - records: list[dict[str, Any]] = [] - for path in sorted( - (item for item in temporary.rglob("*") if item.is_file()), - key=lambda item: item.relative_to(temporary).as_posix(), - ): - relative = path.relative_to(temporary) - if relative == Path(COLLECTION_RECORD): - continue - plan = provenance.get(relative) - if relative == Path("electrace.toml"): - role = "configuration" - elif plan is not None: - role = plan.role - else: # pragma: no cover - internal staging invariant - raise RuntimeError(f"unplanned collection file: {relative}") - record: dict[str, Any] = { - "path": relative.as_posix(), - "size_bytes": path.stat().st_size, - "sha256": sha256_file(path), - "role": role, - } - if plan is not None and plan.source is not None: - record["source_path"] = str(plan.source) - # The collected digest is the exact source snapshot copied through - # the no-follow descriptor, even if the live source changes later. - record["source_sha256"] = record["sha256"] - if plan is not None and plan.frame is not None: - record["frame"] = { - "index": plan.frame.index, - "label": plan.frame.label, - "analysis_label": plan.frame.analysis_label, - } - if plan is not None and plan.job_id is not None: - record["job_id"] = plan.job_id - records.append(record) - return { - "schema_version": COLLECTION_SCHEMA_VERSION, - "analysis_root": str(final_root), - "workflow_manifest": str(staged.manifest_path), - "workflow_manifest_sha256": staged.manifest_sha256, - "workflow_workspace_record": str(staged.record_path), - "input_root": str(final_root / "input"), - "descriptor_root": str(final_root / "descriptors"), - "config_path": str(final_root / "electrace.toml"), - "results_directory": str(final_root / "results"), - "files": records, - } +def _existing(path: Path, staged: StagedWorkflow) -> CollectionArtifacts: + payload = _managed(path, staged) + if payload is None: + raise ValueError(f"existing collection is not owned by this workflow: {path}") + required = (path / "electrace.toml", path / "input", path / "descriptors", path / "results") + if not required[0].is_file() or any(not item.is_dir() for item in required[1:]): + raise ValueError(f"existing collection is incomplete: {path}") + raw_files = payload.get("files") + if not isinstance(raw_files, list) or any(not isinstance(item, str) for item in raw_files): + raise ValueError("collection marker has invalid files") + files = tuple(path / item for item in raw_files) + if any(not item.is_file() for item in files): + raise ValueError("collection marker names a missing file") + return CollectionArtifacts(required[0], required[1], required[2], files) def collect_workflow_outputs( @@ -566,62 +217,100 @@ def collect_workflow_outputs( *, overwrite: bool = False, ) -> CollectionArtifacts: - """Validate and atomically collect all staged descriptor outputs.""" + """Collect selected structures and parser-supported descriptor outputs.""" - requested_manifest = absolute_lexical(Path(manifest_path)) - _reject_symlink_path(requested_manifest, "workflow manifest") - staged = load_staged_workflow(requested_manifest) + staged = load_staged_workflow(manifest_path) final_root = staged.workspace / COLLECTION_DIRECTORY if final_root.exists() or final_root.is_symlink(): if not overwrite: - raise FileExistsError( - f"collected analysis already exists: {final_root}; use overwrite=True to replace it" - ) - if not _collection_is_managed(final_root, staged): - raise ValueError(f"refusing to replace unmarked or symlinked collection: {final_root}") - + raise FileExistsError(f"collected analysis already exists: {final_root}") + if _managed(final_root, staged) is None: + raise ValueError(f"refusing to replace unmarked collection: {final_root}") manifest = load_workflow_manifest(staged.manifest_path) - plans = _collection_plans(staged, manifest) - + expected_ids = _expected_descriptor_ids(staged, manifest) + if staged.descriptor_job_ids != expected_ids: + raise ValueError("staged descriptor jobs do not match the manifest") + jobs = {job.id: job for job in staged.jobs} + recognized = descriptor_outputs(staged) temporary = Path(tempfile.mkdtemp(prefix=".analysis.collect-", dir=staged.workspace)) - relative_files: tuple[Path, ...] = () + written: list[Path] = [] try: - (temporary / "input").mkdir() - (temporary / "descriptors").mkdir() - (temporary / "results").mkdir() - for plan in plans: - destination = temporary / plan.relative_path - if plan.source is not None: - _copy_regular_file(plan.source, destination, plan.role) - else: - assert plan.content is not None - _write_text(destination, plan.content) + for name in ("input", "descriptors", "results"): + (temporary / name).mkdir() + frames = {} + for frame in staged.frames: + if frame.structure.is_symlink() or not frame.structure.is_file(): + raise FileNotFoundError( + f"selected structure is not a regular file: {frame.structure}" + ) + destination = Path("input") / frame.analysis_label / "CONTCAR" + (temporary / destination).parent.mkdir() + shutil.copy2(frame.structure, temporary / destination) + written.append(destination) + frames[frame.index] = read_poscar( + frame.structure, + index=frame.index, + label=frame.label, + energy=frame.energy_ev, + ) + if frame.energy_ev is not None: + energy = Path("input") / frame.analysis_label / "OSZICAR" + (temporary / energy).write_text( + f" 1 F= {frame.energy_ev:.16g} E0= {frame.energy_ev:.16g} d E = 0.0\n", + encoding="utf-8", + ) + written.append(energy) + + destinations: set[Path] = set() + for job_id in expected_ids: + job = jobs.get(job_id) + outputs = recognized.get(job_id, ()) + if job is None or not outputs: + raise ValueError(f"descriptor job {job_id!r} has no supported output") + for source in outputs: + if source.is_symlink() or not source.is_file() or source.stat().st_size == 0: + raise FileNotFoundError(f"descriptor output is missing or invalid: {source}") + load_descriptor(source, frame=frames[job.frame_index]) + label = next( + frame.analysis_label + for frame in staged.frames + if frame.index == job.frame_index + ) + destination = Path("descriptors") / label / source.name + if destination in destinations: + raise ValueError(f"descriptor output collision at {destination}") + destinations.add(destination) + (temporary / destination).parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(source, temporary / destination) + written.append(destination) - config = _collection_config(staged, final_root) config_path = temporary / "electrace.toml" - _write_text(config_path, _render_config(config)) - # Defense in depth: the generated text must round-trip through the same - # strict configuration loader used by analysis. + config_path.write_text( + _render_config(_collection_config(staged, final_root)), encoding="utf-8", newline="\n" + ) load_config(config_path) - marker = _marker_payload(staged, final_root, temporary, plans) - _write_json_exclusive(temporary / COLLECTION_RECORD, marker) - relative_files = tuple( - sorted( - (path.relative_to(temporary) for path in temporary.rglob("*") if path.is_file()), - key=Path.as_posix, - ) + written.append(Path("electrace.toml")) + marker = { + "schema_version": COLLECTION_SCHEMA_VERSION, + "analysis_root": str(final_root), + "workflow_workspace_record": str(staged.record_path), + "workflow_manifest_sha256": staged.manifest_sha256, + "files": [item.as_posix() for item in sorted(written, key=Path.as_posix)], + } + (temporary / COLLECTION_RECORD).write_text( + json.dumps(marker, indent=2, sort_keys=True) + "\n", encoding="utf-8" ) - _publish_collection(temporary, final_root, staged, overwrite=overwrite) + written.append(Path(COLLECTION_RECORD)) + _publish(temporary, final_root, staged, overwrite) except BaseException: - if temporary.exists() and not temporary.is_symlink(): + if temporary.exists(): shutil.rmtree(temporary, ignore_errors=True) raise - return CollectionArtifacts( - config_path=final_root / "electrace.toml", - input_root=final_root / "input", - descriptor_root=final_root / "descriptors", - files=tuple(final_root / relative for relative in relative_files), + final_root / "electrace.toml", + final_root / "input", + final_root / "descriptors", + tuple(final_root / item for item in written), ) @@ -631,25 +320,15 @@ def analyze_collected_workflow( strict: bool = True, overwrite: bool = False, ) -> AnalysisArtifacts: - """Collect a staged workflow and run the normal ElecTrace analysis pipeline.""" - if overwrite: collected = collect_workflow_outputs(manifest_path, overwrite=True) else: - requested_manifest = absolute_lexical(Path(manifest_path)) - _reject_symlink_path(requested_manifest, "workflow manifest") - staged = load_staged_workflow(requested_manifest) - final_root = staged.workspace / COLLECTION_DIRECTORY + staged = load_staged_workflow(manifest_path) + path = staged.workspace / COLLECTION_DIRECTORY collected = ( - _existing_collection_artifacts(final_root, staged) - if final_root.exists() or final_root.is_symlink() - else collect_workflow_outputs(requested_manifest) + _existing(path, staged) if path.exists() else collect_workflow_outputs(manifest_path) ) - return run_project( - collected.config_path, - strict=strict, - overwrite=overwrite, - ) + return run_project(collected.config_path, strict=strict, overwrite=overwrite) __all__ = [ diff --git a/src/electrace/workflow/execution.py b/src/electrace/workflow/execution.py index 5cd0bfa..5c8dca7 100644 --- a/src/electrace/workflow/execution.py +++ b/src/electrace/workflow/execution.py @@ -1,10 +1,4 @@ -"""Explicit, confirmation-gated execution of external workflow jobs. - -The plugin layer remains non-executing. This module is the separate execution -boundary: callers must construct auditable :class:`JobSpec` records and opt in -with ``confirmed=True`` before a local process or Slurm submission can occur. -Commands are always passed as argument vectors with ``shell=False``. -""" +"""Execution primitives for local and Slurm workflow jobs.""" from __future__ import annotations @@ -13,7 +7,6 @@ import re import shlex import subprocess -import tempfile from collections import defaultdict from collections.abc import Iterable, Mapping, Sequence from concurrent.futures import FIRST_COMPLETED, Future, ThreadPoolExecutor, wait @@ -24,7 +17,7 @@ from types import MappingProxyType from typing import Any, Literal -from ._files import atomic_write_text, symlink_component +from ._files import atomic_write_text JobStatus = Literal["success", "failed", "blocked"] DirectiveValue = str | int | float | bool | None @@ -37,36 +30,24 @@ _UNSAFE_SLURM_DIRECTIVES = frozenset({"command", "script", "wrap"}) -def _safe_text(value: object, field_name: str, *, identifier: bool = False) -> str: - if not isinstance(value, str) or not value: - raise ValueError(f"{field_name} must be a non-empty string") - if any(character in value for character in ("\x00", "\r", "\n")): - raise ValueError(f"{field_name} must not contain NUL or newline characters") +def _text(value: object, field_name: str, *, identifier: bool = False) -> str: + if not isinstance(value, str) or not value or any(c in value for c in "\x00\r\n"): + raise ValueError(f"{field_name} must be non-empty text without control characters") if identifier and _ID_PATTERN.fullmatch(value) is None: - raise ValueError( - f"{field_name} must start with an alphanumeric character and contain only " - "letters, digits, '.', '_', ':', or '-'" - ) + raise ValueError(f"invalid {field_name}: {value!r}") return value -def _absolute_path(value: str | Path, field_name: str) -> Path: +def _absolute(value: str | Path, field_name: str) -> Path: path = Path(value).expanduser() - if not path.is_absolute(): + if not path.is_absolute() or "\x00" in str(path): raise ValueError(f"{field_name} must be an absolute path: {path}") - if "\x00" in str(path): - raise ValueError(f"{field_name} must not contain NUL characters") return Path(os.path.normpath(path)) @dataclass(frozen=True, slots=True) class JobSpec: - """A self-contained external task specification. - - All paths are absolute and commands are tokenized. ``stdout_path`` is - captured by the execution layer; stderr is written beside it as - ``.stderr.log``. - """ + """One external command and its declared inputs, outputs, and dependencies.""" id: str frame_index: int @@ -82,69 +63,44 @@ class JobSpec: environment: Mapping[str, str] = field(default_factory=dict) def __post_init__(self) -> None: - object.__setattr__(self, "id", _safe_text(self.id, "job id", identifier=True)) - if isinstance(self.frame_index, bool) or not isinstance(self.frame_index, int): - raise TypeError("frame_index must be an integer") - if self.frame_index < 0: - raise ValueError("frame_index must be non-negative") - object.__setattr__(self, "frame_label", _safe_text(self.frame_label, "frame_label")) - object.__setattr__(self, "task", _safe_text(self.task, "task")) - object.__setattr__(self, "cwd", _absolute_path(self.cwd, "cwd")) - + object.__setattr__(self, "id", _text(self.id, "job id", identifier=True)) + if type(self.frame_index) is not int or self.frame_index < 0: + raise ValueError("frame_index must be a non-negative integer") + object.__setattr__(self, "frame_label", _text(self.frame_label, "frame_label")) + object.__setattr__(self, "task", _text(self.task, "task")) + object.__setattr__(self, "cwd", _absolute(self.cwd, "cwd")) argv = tuple(self.argv) if not argv: raise ValueError("argv must contain at least one token") for index, token in enumerate(argv): - _safe_text(token, f"argv[{index}]") + _text(token, f"argv[{index}]") object.__setattr__(self, "argv", argv) - - if self.stdin is not None: - if not isinstance(self.stdin, str): - raise TypeError("stdin must be a string or None") - if "\x00" in self.stdin: - raise ValueError("stdin must not contain NUL characters") - object.__setattr__( - self, - "stdout_path", - _absolute_path(self.stdout_path, "stdout_path"), - ) - for field_name in ("required_inputs", "expected_outputs"): - paths = tuple( - _absolute_path(path, f"{field_name}[{index}]") - for index, path in enumerate(getattr(self, field_name)) - ) + if self.stdin is not None and (not isinstance(self.stdin, str) or "\x00" in self.stdin): + raise ValueError("stdin must be NUL-free text or None") + object.__setattr__(self, "stdout_path", _absolute(self.stdout_path, "stdout_path")) + for name in ("required_inputs", "expected_outputs"): + paths = tuple(_absolute(path, name) for path in getattr(self, name)) if len(paths) != len(set(paths)): - raise ValueError(f"{field_name} must not contain duplicate paths") - object.__setattr__(self, field_name, paths) - + raise ValueError(f"{name} must not contain duplicates") + object.__setattr__(self, name, paths) dependencies = tuple(self.dependencies) - for index, dependency in enumerate(dependencies): - _safe_text(dependency, f"dependencies[{index}]", identifier=True) if len(dependencies) != len(set(dependencies)): - raise ValueError("dependencies must not contain duplicate job ids") + raise ValueError("dependencies must not contain duplicates") + for dependency in dependencies: + _text(dependency, "dependency", identifier=True) object.__setattr__(self, "dependencies", dependencies) - environment = dict(self.environment) for key, value in environment.items(): - if not isinstance(key, str) or _ENVIRONMENT_KEY_PATTERN.fullmatch(key) is None: + if _ENVIRONMENT_KEY_PATTERN.fullmatch(key) is None: raise ValueError(f"invalid environment variable name: {key!r}") - if not isinstance(value, str): - raise TypeError(f"environment value for {key!r} must be a string") - if any(character in value for character in ("\x00", "\r", "\n")): - raise ValueError( - f"environment value for {key!r} must not contain NUL or newline characters" - ) + _text(value, f"environment value for {key!r}") object.__setattr__(self, "environment", MappingProxyType(environment)) @property def stderr_path(self) -> Path: - """Return the dedicated stderr log beside ``stdout_path``.""" - return self.stdout_path.with_name(f"{self.stdout_path.name}.stderr.log") def to_dict(self) -> dict[str, Any]: - """Return a JSON-ready execution specification.""" - return { "id": self.id, "frame_index": self.frame_index, @@ -164,8 +120,6 @@ def to_dict(self) -> dict[str, Any]: @dataclass(frozen=True, slots=True) class JobResult: - """JSON-serializable outcome of one local job.""" - id: str frame_index: int frame_label: str @@ -186,8 +140,6 @@ def succeeded(self) -> bool: return self.status == "success" def to_dict(self) -> dict[str, Any]: - """Return a JSON-ready result record.""" - return { "id": self.id, "frame_index": self.frame_index, @@ -206,21 +158,19 @@ def to_dict(self) -> dict[str, Any]: } -def _declared_outputs(job: JobSpec) -> tuple[Path, ...]: - """Return every job-owned output once, in deterministic declaration order.""" - +def _outputs(job: JobSpec) -> tuple[Path, ...]: return tuple(dict.fromkeys((job.stdout_path, job.stderr_path, *job.expected_outputs))) def validate_jobs(jobs: Iterable[JobSpec]) -> tuple[JobSpec, ...]: - """Validate ids, outputs, and dependencies; return stable topological order.""" + """Return jobs in stable dependency order after validating the DAG.""" specs = tuple(jobs) if not specs: raise ValueError("at least one job is required") by_id: dict[str, JobSpec] = {} position: dict[str, int] = {} - output_owners: dict[Path, str] = {} + owners: dict[Path, str] = {} for index, job in enumerate(specs): if not isinstance(job, JobSpec): raise TypeError(f"jobs[{index}] must be a JobSpec") @@ -228,107 +178,48 @@ def validate_jobs(jobs: Iterable[JobSpec]) -> tuple[JobSpec, ...]: raise ValueError(f"duplicate job id: {job.id!r}") by_id[job.id] = job position[job.id] = index - for output in _declared_outputs(job): - owner = output_owners.get(output) - if owner is not None and owner != job.id: - raise ValueError( - f"jobs {owner!r} and {job.id!r} declare the same output path: {output}" - ) - output_owners[output] = job.id + for output in _outputs(job): + if output in owners: + raise ValueError(f"jobs {owners[output]!r} and {job.id!r} share output {output}") + owners[output] = job.id dependents: dict[str, list[str]] = defaultdict(list) indegree: dict[str, int] = {} for job in specs: - if job.id in job.dependencies: - raise ValueError(f"job {job.id!r} cannot depend on itself") - unknown = [dependency for dependency in job.dependencies if dependency not in by_id] - if unknown: - raise ValueError( - f"job {job.id!r} has unknown dependencies: {', '.join(sorted(unknown))}" - ) + unknown = [item for item in job.dependencies if item not in by_id] + if job.id in job.dependencies or unknown: + detail = "itself" if job.id in job.dependencies else ", ".join(unknown) + raise ValueError(f"job {job.id!r} has invalid dependencies: {detail}") indegree[job.id] = len(job.dependencies) for dependency in job.dependencies: dependents[dependency].append(job.id) ready = [(position[job.id], job.id) for job in specs if indegree[job.id] == 0] heapify(ready) - ordered_ids: list[str] = [] + ordered: list[str] = [] while ready: _, job_id = heappop(ready) - ordered_ids.append(job_id) + ordered.append(job_id) for dependent in dependents[job_id]: indegree[dependent] -= 1 if indegree[dependent] == 0: heappush(ready, (position[dependent], dependent)) - if len(ordered_ids) != len(specs): - cyclic = sorted( - (job_id for job_id, degree in indegree.items() if degree > 0), - key=position.__getitem__, - ) - raise ValueError("job dependency graph contains a cycle involving: " + ", ".join(cyclic)) - return tuple(by_id[job_id] for job_id in ordered_ids) + if len(ordered) != len(specs): + raise ValueError("job dependency graph contains a cycle") + return tuple(by_id[job_id] for job_id in ordered) def _timestamp() -> str: return datetime.now(UTC).isoformat() -def _allocate_capture(path: Path) -> tuple[int, Path]: - path.parent.mkdir(parents=True, exist_ok=True) - descriptor, name = tempfile.mkstemp( - prefix=f".{path.name}.", - suffix=".tmp", - dir=path.parent, - ) - return descriptor, Path(name) - - -def _allocate_job_captures(job: JobSpec) -> tuple[int, Path, int, Path]: - """Allocate both log files without leaking the first if the second fails.""" - - stdout_descriptor, stdout_temporary = _allocate_capture(job.stdout_path) - try: - stderr_descriptor, stderr_temporary = _allocate_capture(job.stderr_path) - except BaseException: - os.close(stdout_descriptor) - stdout_temporary.unlink(missing_ok=True) - raise - return stdout_descriptor, stdout_temporary, stderr_descriptor, stderr_temporary - - -def _unsafe_output(path: Path) -> str | None: - if (component := symlink_component(path)) is not None: - return f"refusing to follow output symlink component {component} for {path}" - if path.exists() and not path.is_file(): - return f"output path exists but is not a regular file: {path}" - return None - - -def _diagnostic_log(job: JobSpec, message: str) -> None: - """Best-effort atomic diagnostic without following a stderr symlink.""" - - if _unsafe_output(job.stderr_path) is not None: - return - temporary: Path | None = None - try: - descriptor, temporary = _allocate_capture(job.stderr_path) - with os.fdopen(descriptor, "w", encoding="utf-8") as handle: - handle.write(message.rstrip() + "\n") - os.replace(temporary, job.stderr_path) - except OSError: - return - finally: - if temporary is not None: - temporary.unlink(missing_ok=True) - - def _result( job: JobSpec, - *, status: JobStatus, - returncode: int | None, - started_at: str | None, - error: str | None, + *, + returncode: int | None = None, + started_at: str | None = None, + error: str | None = None, missing: tuple[Path, ...] = (), ) -> JobResult: return JobResult( @@ -350,133 +241,62 @@ def _result( def _execute_job(job: JobSpec) -> JobResult: - started_at = _timestamp() + started = _timestamp() if not job.cwd.is_dir(): - message = f"working directory does not exist: {job.cwd}" - _diagnostic_log(job, message) - return _result( - job, - status="failed", - returncode=None, - started_at=started_at, - error=message, - ) + return _result(job, "failed", started_at=started, error=f"missing directory: {job.cwd}") missing_inputs = tuple(path for path in job.required_inputs if not path.is_file()) if missing_inputs: - message = "required inputs are missing: " + ", ".join(str(path) for path in missing_inputs) - _diagnostic_log(job, message) return _result( job, - status="failed", - returncode=None, - started_at=started_at, - error=message, + "failed", + started_at=started, + error="required inputs are missing: " + ", ".join(map(str, missing_inputs)), ) - for output in _declared_outputs(job): - if (output_error := _unsafe_output(output)) is not None: - _diagnostic_log(job, output_error) - return _result( - job, - status="failed", - returncode=None, - started_at=started_at, - error=output_error, - ) + for output in _outputs(job): + if output.is_symlink() or (output.exists() and not output.is_file()): + return _result(job, "failed", started_at=started, error=f"unsafe output: {output}") + output.parent.mkdir(parents=True, exist_ok=True) - try: - ( - stdout_descriptor, - stdout_temporary, - stderr_descriptor, - stderr_temporary, - ) = _allocate_job_captures(job) - except OSError as exc: - message = f"could not allocate job logs: {exc}" - _diagnostic_log(job, message) - return _result( - job, - status="failed", - returncode=None, - started_at=started_at, - error=message, - ) returncode: int | None = None - execution_error: str | None = None + error: str | None = None try: - with ( - os.fdopen(stdout_descriptor, "wb") as stdout_handle, - os.fdopen(stderr_descriptor, "wb") as stderr_handle, - ): - kwargs: dict[str, Any] = { + with job.stdout_path.open("wb") as stdout, job.stderr_path.open("wb") as stderr: + options: dict[str, Any] = { "cwd": job.cwd, "env": {**os.environ, **dict(job.environment)}, - "stdout": stdout_handle, - "stderr": stderr_handle, + "stdout": stdout, + "stderr": stderr, "shell": False, "check": False, } if job.stdin is None: - kwargs["stdin"] = subprocess.DEVNULL + options["stdin"] = subprocess.DEVNULL else: - kwargs.update({"input": job.stdin, "text": True, "encoding": "utf-8"}) - try: - completed = subprocess.run(list(job.argv), **kwargs) - returncode = int(completed.returncode) - except OSError as exc: - execution_error = f"could not execute {job.argv[0]!r}: {exc}" - stderr_handle.write((execution_error + "\n").encode("utf-8", errors="replace")) - # os.replace atomically replaces a leaf symlink entry rather than - # following its target, closing the leaf-level preflight race. - os.replace(stdout_temporary, job.stdout_path) - os.replace(stderr_temporary, job.stderr_path) + options.update({"input": job.stdin, "text": True, "encoding": "utf-8"}) + completed = subprocess.run(list(job.argv), **options) + returncode = int(completed.returncode) except OSError as exc: - execution_error = execution_error or f"could not publish job logs atomically: {exc}" - finally: - stdout_temporary.unlink(missing_ok=True) - stderr_temporary.unlink(missing_ok=True) - - expected_symlinks = {path: symlink_component(path) for path in job.expected_outputs} - unsafe_expected = tuple( - path for path, component in expected_symlinks.items() if component is not None - ) - missing_expected = tuple( - path - for path in job.expected_outputs - if expected_symlinks[path] is not None or not path.is_file() - ) - errors: list[str] = [] - if execution_error is not None: - errors.append(execution_error) - if returncode not in {0, None}: + error = f"could not execute {job.argv[0]!r}: {exc}" + + missing = tuple(path for path in job.expected_outputs if not path.is_file()) + errors = [message for message in (error,) if message] + if returncode not in (0, None): errors.append(f"command exited with status {returncode}") - if unsafe_expected: - errors.append( - "expected outputs are unsafe symlinks: " - + ", ".join(str(path) for path in unsafe_expected) - ) - if missing_expected: - errors.append( - "expected outputs are missing: " + ", ".join(str(path) for path in missing_expected) - ) + if missing: + errors.append("expected outputs are missing: " + ", ".join(map(str, missing))) status: JobStatus = "success" if returncode == 0 and not errors else "failed" return _result( job, - status=status, + status, returncode=returncode, - started_at=started_at, + started_at=started, error="; ".join(errors) or None, - missing=missing_expected, + missing=missing, ) -def _blocked_result(job: JobSpec, message: str) -> JobResult: - return _result( - job, - status="blocked", - returncode=None, - started_at=None, - error=message, - ) +def _blocked(job: JobSpec, message: str) -> JobResult: + return _result(job, "blocked", error=message) def run_jobs( @@ -486,127 +306,87 @@ def run_jobs( max_parallel: int = 1, stop_on_error: bool = True, ) -> tuple[JobResult, ...]: - """Execute a dependency DAG locally after explicit confirmation. - - Independent jobs may run concurrently. A failed dependency always blocks - its dependents. With ``stop_on_error=True``, no new jobs are started after - the first observed failure; already-running jobs are allowed to finish. - """ + """Execute a dependency DAG locally with bounded parallelism.""" if not confirmed: raise PermissionError("external job execution requires confirmed=True") - if isinstance(max_parallel, bool) or not isinstance(max_parallel, int) or max_parallel < 1: + if type(max_parallel) is not int or max_parallel < 1: raise ValueError("max_parallel must be a positive integer") ordered = validate_jobs(jobs) by_id = {job.id: job for job in ordered} position = {job.id: index for index, job in enumerate(ordered)} dependents: dict[str, list[str]] = defaultdict(list) - remaining_dependencies: dict[str, int] = {} + remaining = {job.id: len(job.dependencies) for job in ordered} for job in ordered: - remaining_dependencies[job.id] = len(job.dependencies) for dependency in job.dependencies: dependents[dependency].append(job.id) - - ready = [(position[job.id], job.id) for job in ordered if remaining_dependencies[job.id] == 0] + ready = [(position[job.id], job.id) for job in ordered if not job.dependencies] heapify(ready) pending = set(by_id) results: dict[str, JobResult] = {} futures: dict[Future[JobResult], JobSpec] = {} - stop_requested = False - - def release_dependents(job_id: str) -> None: - """Promote or transitively block jobs whose last dependency completed.""" + stopped = False + def release(job_id: str) -> None: resolved = [job_id] while resolved: for dependent_id in dependents[resolved.pop()]: if dependent_id not in pending: continue - remaining_dependencies[dependent_id] -= 1 - if remaining_dependencies[dependent_id] != 0: + remaining[dependent_id] -= 1 + if remaining[dependent_id]: continue - dependent = by_id[dependent_id] - failed = [ - dependency - for dependency in dependent.dependencies - if not results[dependency].succeeded - ] + job = by_id[dependent_id] + failed = [item for item in job.dependencies if not results[item].succeeded] if failed: pending.remove(dependent_id) - results[dependent_id] = _blocked_result( - dependent, - "dependency did not succeed: " + ", ".join(failed), + results[dependent_id] = _blocked( + job, "dependency did not succeed: " + ", ".join(failed) ) resolved.append(dependent_id) else: heappush(ready, (position[dependent_id], dependent_id)) - worker_count = min(max_parallel, len(ordered)) - with ThreadPoolExecutor(max_workers=worker_count) as executor: + with ThreadPoolExecutor(max_workers=min(max_parallel, len(ordered))) as executor: while pending or futures: - if stop_requested: - for job in ordered: - if job.id not in pending: - continue - results[job.id] = _blocked_result( - job, - "not started because stop_on_error was triggered", - ) - pending.remove(job.id) + if stopped: + for job_id in tuple(pending): + results[job_id] = _blocked(by_id[job_id], "not started after a failure") + pending.remove(job_id) ready.clear() else: - while ready and len(futures) < worker_count: + while ready and len(futures) < max_parallel: _, job_id = heappop(ready) - if job_id not in pending: - continue - job = by_id[job_id] - futures[executor.submit(_execute_job, job)] = job - pending.remove(job_id) - + if job_id in pending: + pending.remove(job_id) + job = by_id[job_id] + futures[executor.submit(_execute_job, job)] = job if not futures: - if pending: # pragma: no cover - validate_jobs prevents deadlock - raise RuntimeError("validated job graph could not make execution progress") continue - completed_futures, _ = wait(tuple(futures), return_when=FIRST_COMPLETED) - for future in sorted(completed_futures, key=lambda item: position[futures[item].id]): + completed, _ = wait(tuple(futures), return_when=FIRST_COMPLETED) + for future in sorted(completed, key=lambda item: position[futures[item].id]): job = futures.pop(future) - try: - outcome = future.result() - except Exception as exc: # pragma: no cover - defensive worker boundary - outcome = _result( - job, - status="failed", - returncode=None, - started_at=None, - error=f"unexpected execution failure: {exc}", - ) + outcome = future.result() results[job.id] = outcome - release_dependents(job.id) - if stop_on_error and not outcome.succeeded: - stop_requested = True - + release(job.id) + stopped = stopped or (stop_on_error and not outcome.succeeded) return tuple(results[job.id] for job in ordered) def _slurm_directives(directives: Mapping[str, DirectiveValue], job_name: str) -> list[str]: - if not isinstance(job_name, str) or _SLURM_NAME_PATTERN.fullmatch(job_name) is None: - raise ValueError( - "job_name must be 1-128 characters using only letters, digits, '.', '_', or '-'" - ) + if _SLURM_NAME_PATTERN.fullmatch(job_name) is None: + raise ValueError("invalid Slurm job name") lines = [f"#SBATCH --job-name={job_name}"] normalized: dict[str, DirectiveValue] = {} for raw_key, value in directives.items(): - if not isinstance(raw_key, str): - raise TypeError("Slurm directive names must be strings") key = raw_key.removeprefix("--") - if _SLURM_DIRECTIVE_PATTERN.fullmatch(key) is None: - raise ValueError(f"unsafe Slurm directive name: {raw_key!r}") - if key == "job-name": - raise ValueError("pass the Slurm job name through job_name, not directives") - if key in _UNSAFE_SLURM_DIRECTIVES: - raise ValueError(f"Slurm directive {key!r} may not embed executable content") - if key in normalized: - raise ValueError(f"duplicate normalized Slurm directive: {key!r}") + if ( + _SLURM_DIRECTIVE_PATTERN.fullmatch(key) is None + or key in _UNSAFE_SLURM_DIRECTIVES + or key == "job-name" + or key in normalized + ): + raise ValueError(f"unsafe or duplicate Slurm directive: {raw_key!r}") normalized[key] = value for key in sorted(normalized): value = normalized[key] @@ -616,11 +396,9 @@ def _slurm_directives(directives: Mapping[str, DirectiveValue], job_name: str) - lines.append(f"#SBATCH --{key}") continue if isinstance(value, bool) or not isinstance(value, (str, int, float)): - raise TypeError( - f"Slurm directive {key!r} must be a string, finite number, boolean, or None" - ) + raise TypeError(f"invalid Slurm directive value for {key!r}") if isinstance(value, float) and not math.isfinite(value): - raise ValueError(f"Slurm directive {key!r} must be finite") + raise ValueError(f"invalid Slurm directive value for {key!r}") text = str(value) if _SLURM_VALUE_PATTERN.fullmatch(text) is None: raise ValueError(f"unsafe Slurm directive value for {key!r}: {text!r}") @@ -628,111 +406,45 @@ def _slurm_directives(directives: Mapping[str, DirectiveValue], job_name: str) - return lines -def _shell_failure(message: str) -> str: - return f"printf '%s\\n' {shlex.quote(message)} >&2; exit 1" - - -def _shell_test_regular(path: Path, message: str) -> str: - quoted = shlex.quote(str(path)) - return f"[ -f {quoted} ] || {{ {_shell_failure(message)}; }}" - - -def _shell_test_not_symlink(path: Path) -> str: - quoted = shlex.quote(str(path)) - return f"[ ! -L {quoted} ] || {{ {_shell_failure(f'refusing output symlink: {path}')}; }}" - - -def _shell_output_guards(path: Path) -> list[str]: - """Render guards for the output leaf and every lexical parent.""" - - return [_shell_test_not_symlink(component) for component in (path, *path.parents[:-1])] - - def render_slurm_script( - jobs: Iterable[JobSpec], - directives: Mapping[str, DirectiveValue], - job_name: str, + jobs: Iterable[JobSpec], directives: Mapping[str, DirectiveValue], job_name: str ) -> str: - """Render a strict, topologically ordered Slurm batch script.""" + """Render a quoted, dependency-ordered Slurm script.""" - ordered = validate_jobs(jobs) lines = ["#!/bin/sh", *_slurm_directives(directives, job_name), "", "set -eu", ""] - for index, job in enumerate(ordered): - lines.append( - f"# ElecTrace job {job.id}: frame {job.frame_index} ({job.frame_label}) - {job.task}" - ) - lines.append( - f"[ -d {shlex.quote(str(job.cwd))} ] || " - f"{{ {_shell_failure(f'working directory does not exist: {job.cwd}')}; }}" - ) + for job in validate_jobs(jobs): + lines.append(f"# ElecTrace job {job.id}: frame {job.frame_index} - {job.task}") + lines.append(f"cd {shlex.quote(str(job.cwd))}") for required in job.required_inputs: - lines.append(_shell_test_regular(required, f"required input is missing: {required}")) - for output in _declared_outputs(job): - lines.extend(_shell_output_guards(output)) - for parent in sorted({job.stdout_path.parent, job.stderr_path.parent}, key=str): + path = shlex.quote(str(required)) + lines.append(f"[ -f {path} ] || {{ echo 'missing required input' >&2; exit 1; }}") + for parent in {job.stdout_path.parent, job.stderr_path.parent}: lines.append(f"mkdir -p -- {shlex.quote(str(parent))}") - - stdout_variable = f"electrace_stdout_{index}" - stderr_variable = f"electrace_stderr_{index}" - returncode_variable = f"electrace_returncode_{index}" - stdout_template = job.stdout_path.parent / f".{job.stdout_path.name}.XXXXXX.tmp" - stderr_template = job.stderr_path.parent / f".{job.stderr_path.name}.XXXXXX.tmp" - lines.extend( - ( - f"{stdout_variable}=$(mktemp {shlex.quote(str(stdout_template))})", - f"{stderr_variable}=$(mktemp {shlex.quote(str(stderr_template))})", - f'trap \'rm -f -- "${{{stdout_variable}}}" "${{{stderr_variable}}}"\' EXIT HUP INT TERM', - ) - ) environment = " ".join( shlex.quote(f"{key}={value}") for key, value in sorted(job.environment.items()) ) command = shlex.join(job.argv) if environment: command = f"env {environment} {command}" - if job.stdin is None: - command = f"{command} < /dev/null" - else: - command = f"printf %s {shlex.quote(job.stdin)} | {command}" - lines.extend( - ( - f"if (cd {shlex.quote(str(job.cwd))} && {command}) " - f'>"${{{stdout_variable}}}" 2>"${{{stderr_variable}}}"; then', - f" {returncode_variable}=0", - "else", - f" {returncode_variable}=$?", - "fi", - *_shell_output_guards(job.stdout_path), - *_shell_output_guards(job.stderr_path), - f'mv -f -- "${{{stdout_variable}}}" {shlex.quote(str(job.stdout_path))}', - f'mv -f -- "${{{stderr_variable}}}" {shlex.quote(str(job.stderr_path))}', - "trap - EXIT HUP INT TERM", - f'[ "${{{returncode_variable}}}" -eq 0 ] || ' - f"{{ {_shell_failure(f'job {job.id} failed')}; }}", - ) + command = ( + f"{command} < /dev/null" + if job.stdin is None + else f"printf %s {shlex.quote(job.stdin)} | {command}" + ) + lines.append( + f"{command} >{shlex.quote(str(job.stdout_path))} 2>{shlex.quote(str(job.stderr_path))}" ) for expected in job.expected_outputs: - lines.extend(_shell_output_guards(expected)) - lines.append(_shell_test_regular(expected, f"expected output is missing: {expected}")) + path = shlex.quote(str(expected)) + lines.append(f"[ -f {path} ] || {{ echo 'missing expected output' >&2; exit 1; }}") lines.append("") return "\n".join(lines).rstrip() + "\n" -def write_slurm_script( - script: str, - path: str | Path, - *, - overwrite: bool = False, -) -> Path: - """Atomically write an executable Slurm script without following a symlink.""" - - if not isinstance(script, str) or "\x00" in script: - raise ValueError("script must be NUL-free text") - output = _absolute_path(path, "Slurm script path") - if (output.exists() or output.is_symlink()) and not overwrite: +def write_slurm_script(script: str, path: str | Path, *, overwrite: bool = False) -> Path: + output = _absolute(path, "Slurm script path") + if output.is_symlink() or (output.exists() and not overwrite): raise FileExistsError(f"refusing to replace existing Slurm script: {output}") - if (component := symlink_component(output.parent)) is not None: - raise ValueError(f"refusing to follow Slurm script parent symlink: {component}") return atomic_write_text(output, script, mode=0o700) @@ -742,20 +454,16 @@ def submit_slurm_script( *, confirmed: bool = False, ) -> str: - """Submit a batch script with ``shell=False`` and return its Slurm job id.""" - if not confirmed: raise PermissionError("Slurm submission requires confirmed=True") - script_path = _absolute_path(script, "Slurm script") - if symlink_component(script_path) is not None or not script_path.is_file(): + script_path = _absolute(script, "Slurm script") + if script_path.is_symlink() or not script_path.is_file(): raise FileNotFoundError(f"Slurm script is not a regular file: {script_path}") - command: tuple[str, ...] = ( - (submit_command,) if isinstance(submit_command, str) else tuple(submit_command) - ) + command = (submit_command,) if isinstance(submit_command, str) else tuple(submit_command) if not command: raise ValueError("submit_command must contain at least one token") - for index, token in enumerate(command): - _safe_text(token, f"submit_command[{index}]") + for token in command: + _text(token, "submit command") completed = subprocess.run( [*command, str(script_path)], stdin=subprocess.DEVNULL, @@ -765,11 +473,12 @@ def submit_slurm_script( shell=False, check=False, ) - if completed.returncode != 0: + if completed.returncode: detail = completed.stderr.strip() or completed.stdout.strip() or "no diagnostic output" raise RuntimeError(f"Slurm submission failed with status {completed.returncode}: {detail}") - output = completed.stdout.strip() - match = re.fullmatch(r"(?:Submitted\s+batch\s+job\s+)?([0-9]+)(?:;[^\s;]+)?", output) + match = re.fullmatch( + r"(?:Submitted\s+batch\s+job\s+)?([0-9]+)(?:;[^\s;]+)?", completed.stdout.strip() + ) if match is None: - raise ValueError(f"could not parse Slurm job id from submission output: {output!r}") + raise ValueError(f"could not parse Slurm job id from {completed.stdout.strip()!r}") return match.group(1) diff --git a/src/electrace/workflow/manifest.py b/src/electrace/workflow/manifest.py index a2752a5..eb7818b 100644 --- a/src/electrace/workflow/manifest.py +++ b/src/electrace/workflow/manifest.py @@ -1,16 +1,14 @@ -"""Safe YAML loading and starter generation for ElecTrace workflows.""" +"""Load and create ElecTrace workflow YAML manifests.""" from __future__ import annotations -import os -import tempfile -from collections.abc import Hashable from pathlib import Path -from typing import Any +from typing import Any, cast import yaml # type: ignore[import-untyped] -from ._files import read_limited_text +from electrace._files import atomic_write_text + from .models import ( WORKFLOW_SCHEMA_VERSION, ExecutableSpec, @@ -24,63 +22,34 @@ class WorkflowManifestError(ValueError): - """Raised when a workflow YAML document cannot be loaded safely.""" + """A workflow YAML document could not be loaded.""" class _UniqueKeyLoader(yaml.SafeLoader): # type: ignore[misc] - """SafeLoader variant that refuses PyYAML's last-key-wins behavior.""" - - def compose_node(self, parent: Any, index: Any) -> Any: - if self.check_event(yaml.events.AliasEvent): - raise WorkflowManifestError("workflow YAML aliases are not allowed") - return super().compose_node(parent, index) - - -def _construct_unique_mapping( - loader: _UniqueKeyLoader, node: Any, deep: bool = False -) -> dict[Any, Any]: - mapping: dict[Any, Any] = {} - for key_node, value_node in node.value: - key = loader.construct_object(key_node, deep=deep) - if not isinstance(key, Hashable): - raise WorkflowManifestError("workflow YAML contains a non-hashable mapping key") - if key in mapping: - raise WorkflowManifestError(f"workflow YAML contains duplicate mapping key {key!r}") - mapping[key] = loader.construct_object(value_node, deep=deep) - return mapping - - -_UniqueKeyLoader.add_constructor( - yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, _construct_unique_mapping -) + def construct_mapping(self, node: Any, deep: bool = False) -> dict[object, object]: + self.flatten_mapping(node) + seen: set[object] = set() + for key_node, _ in node.value: + key = self.construct_object(key_node, deep=deep) + if key in seen: + raise WorkflowManifestError(f"duplicate workflow YAML key: {key!r}") + seen.add(key) + return cast(dict[object, object], super().construct_mapping(node, deep=deep)) def load_workflow_manifest(path: str | Path) -> WorkflowManifest: - """Load a strict ``electrace.workflow.v1`` YAML manifest. - - Paths remain raw manifest values. Resolution belongs to the orchestration - boundary, which has both the manifest location and an explicit workspace - policy; loading therefore has no current-working-directory side effects. - """ - source = Path(path) try: - text = read_limited_text( - source, - limit=MAX_WORKFLOW_MANIFEST_BYTES, - context="workflow manifest", - ) - except ValueError as exc: - raise WorkflowManifestError(str(exc)) from exc - try: - data = yaml.load(text, Loader=_UniqueKeyLoader) - except (RecursionError, yaml.YAMLError) as exc: + if source.stat().st_size > MAX_WORKFLOW_MANIFEST_BYTES: + raise WorkflowManifestError("workflow manifest exceeds 1 MB") + data = yaml.load(source.read_text(encoding="utf-8"), Loader=_UniqueKeyLoader) + except (OSError, RecursionError, UnicodeError, yaml.YAMLError) as exc: raise WorkflowManifestError(f"invalid workflow YAML in {source}: {exc}") from exc return WorkflowManifest.from_dict(data) def starter_workflow_manifest(*, root: str = ".") -> WorkflowManifest: - """Return a local VASP-to-BadELF starter that is executable after review.""" + """Return a local VASP-to-BadELF starter manifest.""" return WorkflowManifest( schema_version=WORKFLOW_SCHEMA_VERSION, @@ -121,44 +90,15 @@ def starter_workflow_manifest(*, root: str = ".") -> WorkflowManifest: def dump_starter_manifest(path: str | Path, *, root: str = ".", overwrite: bool = False) -> Path: - """Write a readable starter manifest and return its path. - - Existing files are preserved unless ``overwrite=True`` is explicit. - Parent directories are created because the requested path itself is the - authorization boundary for this local scaffolding operation. - """ - target = Path(path) - target.parent.mkdir(parents=True, exist_ok=True) - manifest = starter_workflow_manifest(root=root) - document = yaml.safe_dump( - manifest.to_dict(), - allow_unicode=True, - default_flow_style=False, - sort_keys=False, - ) - if not overwrite: - with target.open("x", encoding="utf-8", newline="\n") as handle: - handle.write(document) - return target if target.is_symlink(): raise ValueError(f"refusing to replace symlinked workflow manifest: {target}") - - descriptor, temporary_name = tempfile.mkstemp( - prefix=f".{target.name}.", suffix=".tmp", dir=target.parent + if target.exists() and not overwrite: + raise FileExistsError(f"workflow manifest already exists: {target}") + document = yaml.safe_dump( + starter_workflow_manifest(root=root).to_dict(), sort_keys=False, allow_unicode=True ) - temporary = Path(temporary_name) - try: - with os.fdopen(descriptor, "w", encoding="utf-8", newline="\n") as handle: - handle.write(document) - # Recheck for a symlink to give a clear diagnostic. os.replace itself - # replaces a raced leaf symlink rather than following its target. - if target.is_symlink(): - raise ValueError(f"refusing to replace symlinked workflow manifest: {target}") - os.replace(temporary, target) - finally: - temporary.unlink(missing_ok=True) - return target + return atomic_write_text(target, document) __all__ = [ diff --git a/src/electrace/workflow/models.py b/src/electrace/workflow/models.py index eeb7476..c157fbb 100644 --- a/src/electrace/workflow/models.py +++ b/src/electrace/workflow/models.py @@ -1,15 +1,10 @@ -"""Typed, JSON-ready models for versioned ElecTrace workflow manifests. - -The workflow contract describes command *planning*. It deliberately contains -no shell snippets: executable prefixes are argument vectors, scheduler settings -are scalar directives, and multiline input belongs in reviewed template files. -""" +"""Typed models for ``electrace.workflow.v1`` manifests.""" from __future__ import annotations +import json import math import re -from collections import Counter from dataclasses import dataclass, field from pathlib import Path from typing import Any, TypeAlias, cast @@ -19,178 +14,65 @@ JSONScalar: TypeAlias = bool | int | float | str | None JSONValue: TypeAlias = JSONScalar | list["JSONValue"] | dict[str, "JSONValue"] -_SERIES_KINDS = frozenset({"auto", "neb", "ordered", "pressure", "static"}) -_STEP_KINDS = frozenset({"bader", "baderkit", "critic2", "lobster", "vasp"}) -_SCHEDULER_KINDS = frozenset({"local", "slurm"}) -_IDENTIFIER = re.compile(r"^[a-z][a-z0-9_-]*$") -_CAPABILITY = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.:-]*$") -_DIRECTIVE = re.compile(r"^[a-z][a-z0-9-]*$") -_DIRECTIVE_VALUE = re.compile(r"^[A-Za-z0-9_./,:@%+=\[\]*?{}~-]+$") -_UNSAFE_SLURM_DIRECTIVES = frozenset({"command", "job-name", "script", "wrap"}) - - -def _has_control_characters(value: str) -> bool: - return any( - ord(character) < 32 or 127 <= ord(character) <= 159 or character in {"\u2028", "\u2029"} - for character in value - ) - - -def _required_string(value: object, path: str) -> str: - if type(value) is not str: - raise TypeError(f"{path} must be a string") - result = value - if not result or result != result.strip() or _has_control_characters(result): - raise ValueError( - f"{path} must be a non-empty string without surrounding whitespace " - "or control characters" - ) - return result - +_SERIES_KINDS = {"auto", "neb", "ordered", "pressure", "static"} +_STEP_KINDS = {"bader", "baderkit", "critic2", "lobster", "vasp"} +_IDENTIFIER = re.compile(r"[a-z][a-z0-9_-]*\Z") +_CAPABILITY = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.:-]*\Z") +_DIRECTIVE = re.compile(r"[a-z][a-z0-9-]*\Z") +_DIRECTIVE_VALUE = re.compile(r"[A-Za-z0-9_./,:@%+=\[\]*?{}~-]+\Z") +_UNSAFE_DIRECTIVES = {"command", "job-name", "script", "wrap"} -def _path_string(value: object, path: str) -> str: - """Validate a path-shaped value without resolving it or requiring existence.""" - return _required_string(value, path) - - -def _basename(value: object, path: str) -> str: - result = _required_string(value, path) - if result in {".", ".."} or "/" in result or "\\" in result or Path(result).is_absolute(): - raise ValueError(f"{path} must be a basename, not a path: {result!r}") - return result +def _text(value: object, path: str) -> str: + if not isinstance(value, str) or not value.strip() or "\x00" in value: + raise ValueError(f"{path} must be non-empty text") + return value.strip() def _identifier(value: object, path: str) -> str: - result = _required_string(value, path) + result = _text(value, path) if _IDENTIFIER.fullmatch(result) is None: - raise ValueError( - f"{path} must start with a lowercase letter and contain only " - "lowercase letters, digits, '-' or '_'" - ) + raise ValueError(f"{path} must use lowercase letters, digits, '-' or '_'") return result -def _capability(value: object, path: str) -> str: - result = _required_string(value, path) - if _CAPABILITY.fullmatch(result) is None: - raise ValueError(f"{path} contains unsafe characters") +def _basename(value: object, path: str) -> str: + result = _text(value, path) + if result in {".", ".."} or Path(result).name != result or "\\" in result: + raise ValueError(f"{path} must be a basename") return result -def _json_copy( - value: object, - *, - path: str, - _active: set[int] | None = None, - _depth: int = 0, -) -> JSONValue: - """Return a strict JSON copy, rejecting YAML-only and unsafe values.""" - - if _depth > 64: - raise ValueError(f"{path} exceeds the maximum JSON nesting depth of 64") - if value is None or type(value) is bool or type(value) is int: - return cast(JSONScalar, value) - if type(value) is float: - number = value - if not math.isfinite(number): - raise ValueError(f"{path} must be finite for JSON serialization") - return number - if type(value) is str: - text = value - if _has_control_characters(text): - raise ValueError(f"{path} must not contain control characters or newlines") - return text - if type(value) is list: - sequence = cast(list[object], value) - active = set() if _active is None else _active - identity = id(sequence) - if identity in active: - raise ValueError(f"{path} contains a cyclic sequence") - active.add(identity) - try: - return [ - _json_copy( - item, - path=f"{path}[{index}]", - _active=active, - _depth=_depth + 1, - ) - for index, item in enumerate(sequence) - ] - finally: - active.remove(identity) - if type(value) is dict: - mapping = cast(dict[object, object], value) - active = set() if _active is None else _active - identity = id(mapping) - if identity in active: - raise ValueError(f"{path} contains a cyclic mapping") - active.add(identity) - result: dict[str, JSONValue] = {} - try: - for key, item in mapping.items(): - if type(key) is not str: - raise ValueError(f"{path} contains non-string key {key!r}") - string_key = key - if _has_control_characters(string_key): - raise ValueError(f"{path} contains an unsafe key {string_key!r}") - result[string_key] = _json_copy( - item, - path=f"{path}.{string_key}", - _active=active, - _depth=_depth + 1, - ) - return result - finally: - active.remove(identity) - raise ValueError(f"{path} has non-JSON-compatible value of type {type(value).__name__}") - - -def _table(value: object, path: str) -> dict[str, object]: - if type(value) is not dict: - raise TypeError(f"{path} must be a mapping") - table = cast(dict[object, object], value) - result: dict[str, object] = {} - for key, item in table.items(): - if type(key) is not str: - raise ValueError(f"{path} contains non-string key {key!r}") - result[key] = item - return result +def _mapping(value: object, path: str) -> dict[str, Any]: + if not isinstance(value, dict) or any(not isinstance(key, str) for key in value): + raise TypeError(f"{path} must be a string-keyed mapping") + return dict(value) -def _list(value: object, path: str) -> list[object]: - if type(value) is not list: +def _sequence(value: object, path: str) -> list[Any]: + if not isinstance(value, list): raise TypeError(f"{path} must be a sequence") - return cast(list[object], value) + return value -def _check_keys( - data: dict[str, object], - *, - allowed: frozenset[str], - required: frozenset[str], - path: str, -) -> None: - unknown = sorted(set(data) - allowed) - if unknown: - raise ValueError(f"{path} contains unknown keys: {', '.join(unknown)}") - missing = sorted(required - set(data)) - if missing: - raise ValueError(f"{path} is missing required keys: {', '.join(missing)}") +def _fields(data: dict[str, Any], required: set[str], allowed: set[str], path: str) -> None: + if missing := required - data.keys(): + raise ValueError(f"{path} is missing required keys: {', '.join(sorted(missing))}") + if unknown := data.keys() - allowed: + raise ValueError(f"{path} contains unknown keys: {', '.join(sorted(unknown))}") def _json_object(value: object, path: str) -> dict[str, JSONValue]: - copied = _json_copy(value, path=path) - if type(copied) is not dict: - raise TypeError(f"{path} must be a mapping") - return copied + result = _mapping(value, path) + try: + json.dumps(result, allow_nan=False) + except (TypeError, ValueError, RecursionError) as exc: + raise ValueError(f"{path} must contain finite JSON values") from exc + return cast(dict[str, JSONValue], result) @dataclass(frozen=True, slots=True) class WorkflowProject: - """Discovery and workspace settings shared by every workflow step.""" - root: str series_kind: str workspace: str @@ -198,34 +80,23 @@ class WorkflowProject: fallback: str def __post_init__(self) -> None: - object.__setattr__(self, "root", _path_string(self.root, "project.root")) - series_kind = _required_string(self.series_kind, "project.series_kind") - if series_kind not in _SERIES_KINDS: - choices = ", ".join(sorted(_SERIES_KINDS)) - raise ValueError(f"project.series_kind must be one of {choices}") - object.__setattr__(self, "series_kind", series_kind) - object.__setattr__(self, "workspace", _path_string(self.workspace, "project.workspace")) + object.__setattr__(self, "root", _text(self.root, "project.root")) + kind = _text(self.series_kind, "project.series_kind") + if kind not in _SERIES_KINDS: + raise ValueError(f"unsupported project.series_kind: {kind}") + object.__setattr__(self, "series_kind", kind) + object.__setattr__(self, "workspace", _text(self.workspace, "project.workspace")) object.__setattr__(self, "structure", _basename(self.structure, "project.structure")) object.__setattr__(self, "fallback", _basename(self.fallback, "project.fallback")) @classmethod def from_dict(cls, value: object) -> WorkflowProject: - """Construct a project section from a strict manifest mapping.""" - - data = _table(value, "project") - keys = frozenset({"fallback", "root", "series_kind", "structure", "workspace"}) - _check_keys(data, allowed=keys, required=keys, path="project") - return cls( - root=_required_string(data["root"], "project.root"), - series_kind=_required_string(data["series_kind"], "project.series_kind"), - workspace=_required_string(data["workspace"], "project.workspace"), - structure=_required_string(data["structure"], "project.structure"), - fallback=_required_string(data["fallback"], "project.fallback"), - ) - - def to_dict(self) -> dict[str, Any]: - """Return a deterministic JSON-ready project mapping.""" + data = _mapping(value, "project") + keys = {"root", "series_kind", "workspace", "structure", "fallback"} + _fields(data, keys, keys, "project") + return cls(**{key: _text(data[key], f"project.{key}") for key in keys}) + def to_dict(self) -> dict[str, str]: return { "root": self.root, "series_kind": self.series_kind, @@ -237,42 +108,25 @@ def to_dict(self) -> dict[str, Any]: @dataclass(frozen=True, slots=True) class ExecutableSpec: - """An executable path and optional argument-vector prefix.""" - path: str argv_prefix: tuple[str, ...] = () def __post_init__(self) -> None: - object.__setattr__(self, "path", _path_string(self.path, "executable.path")) - if type(self.argv_prefix) is not tuple: - raise TypeError("executable.argv_prefix must be a sequence") - prefix = tuple( - _required_string(token, f"executable.argv_prefix[{index}]") - for index, token in enumerate(self.argv_prefix) + object.__setattr__(self, "path", _text(self.path, "executable.path")) + object.__setattr__( + self, + "argv_prefix", + tuple(_text(token, "executable.argv_prefix") for token in self.argv_prefix), ) - object.__setattr__(self, "argv_prefix", prefix) @classmethod def from_dict(cls, value: object, *, path: str = "executable") -> ExecutableSpec: - """Construct an executable entry from a strict manifest mapping.""" - - data = _table(value, path) - _check_keys( - data, - allowed=frozenset({"argv_prefix", "path"}), - required=frozenset({"path"}), - path=path, - ) - raw_prefix = data.get("argv_prefix", []) - prefix = tuple( - _required_string(token, f"{path}.argv_prefix[{index}]") - for index, token in enumerate(_list(raw_prefix, f"{path}.argv_prefix")) - ) - return cls(path=_required_string(data["path"], f"{path}.path"), argv_prefix=prefix) + data = _mapping(value, path) + _fields(data, {"path"}, {"path", "argv_prefix"}, path) + prefix = _sequence(data.get("argv_prefix", []), f"{path}.argv_prefix") + return cls(_text(data["path"], f"{path}.path"), tuple(prefix)) def to_dict(self) -> dict[str, Any]: - """Return a JSON-ready executable mapping.""" - result: dict[str, Any] = {"path": self.path} if self.argv_prefix: result["argv_prefix"] = list(self.argv_prefix) @@ -281,74 +135,44 @@ def to_dict(self) -> dict[str, Any]: @dataclass(frozen=True, slots=True) class SchedulerSpec: - """Local or Slurm scheduling settings without embedded commands.""" - kind: str directives: dict[str, JSONScalar] = field(default_factory=dict) def __post_init__(self) -> None: - kind = _required_string(self.kind, "scheduler.kind") - if kind not in _SCHEDULER_KINDS: - raise ValueError("scheduler.kind must be one of local, slurm") - object.__setattr__(self, "kind", kind) - if type(self.directives) is not dict: - raise TypeError("scheduler.directives must be a mapping") - + kind = _text(self.kind, "scheduler.kind") + if kind not in {"local", "slurm"}: + raise ValueError("scheduler.kind must be local or slurm") directives: dict[str, JSONScalar] = {} - for raw_key, raw_value in self.directives.items(): - key = _required_string(raw_key, "scheduler directive key") - if _DIRECTIVE.fullmatch(key) is None: - raise ValueError(f"scheduler directive {key!r} contains unsafe characters") - if key == "job-name": - raise ValueError( - "scheduler directive 'job-name' is reserved; pass the job name " - "through workflow submit --job-name" - ) - if key in _UNSAFE_SLURM_DIRECTIVES: - raise ValueError( - f"scheduler directive {key!r} is not allowed to embed executable content" - ) - value = _json_copy(raw_value, path=f"scheduler.directives.{key}") - if isinstance(value, (dict, list)): + for key, value in self.directives.items(): + if _DIRECTIVE.fullmatch(key) is None or key in _UNSAFE_DIRECTIVES: + raise ValueError(f"unsafe Slurm directive: {key!r}") + if not isinstance(value, (str, int, float, bool, type(None))): raise TypeError(f"scheduler.directives.{key} must be a scalar") + if isinstance(value, float) and not math.isfinite(value): + raise ValueError(f"scheduler.directives.{key} must be finite") if isinstance(value, str) and _DIRECTIVE_VALUE.fullmatch(value) is None: - raise ValueError(f"scheduler directive {key!r} has unsafe string value {value!r}") + raise ValueError(f"unsafe Slurm directive value for {key!r}") directives[key] = value if kind == "local" and directives: - raise ValueError("scheduler.directives must be empty when scheduler.kind is 'local'") + raise ValueError("local scheduler does not accept Slurm directives") + object.__setattr__(self, "kind", kind) object.__setattr__(self, "directives", directives) @classmethod def from_dict(cls, value: object) -> SchedulerSpec: - """Construct a scheduler section from a strict manifest mapping.""" - - data = _table(value, "scheduler") - _check_keys( - data, - allowed=frozenset({"directives", "kind"}), - required=frozenset({"kind"}), - path="scheduler", + data = _mapping(value, "scheduler") + _fields(data, {"kind"}, {"kind", "directives"}, "scheduler") + return cls( + _text(data["kind"], "scheduler.kind"), + _mapping(data.get("directives", {}), "scheduler.directives"), ) - raw_directives = _table(data.get("directives", {}), "scheduler.directives") - directives: dict[str, JSONScalar] = {} - for key, value_item in raw_directives.items(): - copied = _json_copy(value_item, path=f"scheduler.directives.{key}") - if isinstance(copied, (dict, list)): - raise TypeError(f"scheduler.directives.{key} must be a scalar") - directives[key] = copied - return cls(kind=_required_string(data["kind"], "scheduler.kind"), directives=directives) def to_dict(self) -> dict[str, Any]: - """Return a JSON-ready scheduler mapping.""" - - validated = SchedulerSpec(kind=self.kind, directives=self.directives) - return {"kind": validated.kind, "directives": dict(validated.directives)} + return {"kind": self.kind, "directives": dict(self.directives)} @dataclass(frozen=True, slots=True) class WorkflowStep: - """One ordered, dependency-aware external-tool planning step.""" - id: str kind: str executable: str @@ -359,114 +183,72 @@ class WorkflowStep: resources: dict[str, JSONValue] = field(default_factory=dict) def __post_init__(self) -> None: - object.__setattr__(self, "id", _identifier(self.id, "step.id")) - kind = _required_string(self.kind, f"step {self.id!r}.kind") + identifier = _identifier(self.id, "step.id") + kind = _text(self.kind, f"step {identifier}.kind") if kind not in _STEP_KINDS: - choices = ", ".join(sorted(_STEP_KINDS)) - raise ValueError(f"step {self.id!r}.kind must be one of {choices}") + raise ValueError(f"unsupported workflow step kind: {kind}") + object.__setattr__(self, "id", identifier) object.__setattr__(self, "kind", kind) - object.__setattr__( - self, - "executable", - _identifier(self.executable, f"step {self.id!r}.executable"), - ) + object.__setattr__(self, "executable", _identifier(self.executable, "step.executable")) if self.capability is not None: - object.__setattr__( - self, - "capability", - _capability(self.capability, f"step {self.id!r}.capability"), - ) + capability = _text(self.capability, "step.capability") + if _CAPABILITY.fullmatch(capability) is None: + raise ValueError("step.capability contains unsafe characters") + object.__setattr__(self, "capability", capability) if self.template_dir is not None: - object.__setattr__( - self, - "template_dir", - _path_string(self.template_dir, f"step {self.id!r}.template_dir"), - ) - if type(self.depends_on) is not tuple: - raise TypeError(f"step {self.id!r}.depends_on must be a sequence") - dependencies = tuple( - _identifier(dependency, f"step {self.id!r}.depends_on[{index}]") - for index, dependency in enumerate(self.depends_on) - ) + object.__setattr__(self, "template_dir", _text(self.template_dir, "step.template_dir")) + dependencies = tuple(_identifier(item, "step.depends_on") for item in self.depends_on) if len(dependencies) != len(set(dependencies)): - raise ValueError(f"step {self.id!r}.depends_on must not contain duplicates") + raise ValueError("step.depends_on contains duplicates") object.__setattr__(self, "depends_on", dependencies) - object.__setattr__(self, "options", _json_object(self.options, f"step {self.id!r}.options")) - object.__setattr__( - self, - "resources", - _json_object(self.resources, f"step {self.id!r}.resources"), - ) + object.__setattr__(self, "options", _json_object(self.options, "step.options")) + object.__setattr__(self, "resources", _json_object(self.resources, "step.resources")) @classmethod def from_dict(cls, value: object, *, index: int) -> WorkflowStep: - """Construct a step from one strict manifest sequence entry.""" - path = f"steps[{index}]" - data = _table(value, path) - allowed = frozenset( - { - "capability", - "depends_on", - "executable", - "id", - "kind", - "options", - "resources", - "template_dir", - } - ) - required = frozenset({"executable", "id", "kind"}) - _check_keys(data, allowed=allowed, required=required, path=path) - raw_capability = data.get("capability") - capability = ( - None - if raw_capability is None - else _required_string(raw_capability, f"{path}.capability") - ) - raw_template = data.get("template_dir") - template_dir = ( - None if raw_template is None else _required_string(raw_template, f"{path}.template_dir") - ) - dependencies = tuple( - _required_string(dependency, f"{path}.depends_on[{dependency_index}]") - for dependency_index, dependency in enumerate( - _list(data.get("depends_on", []), f"{path}.depends_on") - ) - ) + data = _mapping(value, path) + allowed = { + "id", + "kind", + "executable", + "capability", + "template_dir", + "depends_on", + "options", + "resources", + } + _fields(data, {"id", "kind", "executable"}, allowed, path) + dependencies = _sequence(data.get("depends_on", []), f"{path}.depends_on") return cls( - id=_required_string(data["id"], f"{path}.id"), - kind=_required_string(data["kind"], f"{path}.kind"), - executable=_required_string(data["executable"], f"{path}.executable"), - capability=capability, - template_dir=template_dir, - depends_on=dependencies, + id=_text(data["id"], f"{path}.id"), + kind=_text(data["kind"], f"{path}.kind"), + executable=_text(data["executable"], f"{path}.executable"), + capability=data.get("capability"), + template_dir=data.get("template_dir"), + depends_on=tuple(dependencies), options=_json_object(data.get("options", {}), f"{path}.options"), resources=_json_object(data.get("resources", {}), f"{path}.resources"), ) def to_dict(self) -> dict[str, Any]: - """Return a deterministic JSON-ready step mapping.""" - result: dict[str, Any] = { "id": self.id, "kind": self.kind, "executable": self.executable, + "depends_on": list(self.depends_on), + "options": dict(self.options), + "resources": dict(self.resources), } if self.capability is not None: result["capability"] = self.capability if self.template_dir is not None: result["template_dir"] = self.template_dir - result["depends_on"] = list(self.depends_on) - result["options"] = _json_object(self.options, f"step {self.id!r}.options") - result["resources"] = _json_object(self.resources, f"step {self.id!r}.resources") return result @dataclass(frozen=True, slots=True) class WorkflowManifest: - """Complete v1 workflow manifest with validated cross-references.""" - schema_version: str project: WorkflowProject frames: str @@ -475,143 +257,54 @@ class WorkflowManifest: steps: tuple[WorkflowStep, ...] def __post_init__(self) -> None: - version = _required_string(self.schema_version, "schema_version") - if version != WORKFLOW_SCHEMA_VERSION: - raise ValueError(f"schema_version must be {WORKFLOW_SCHEMA_VERSION!r}; got {version!r}") - object.__setattr__(self, "schema_version", version) - if not isinstance(self.project, WorkflowProject): - raise TypeError("project must be a WorkflowProject") - frames = _required_string(self.frames, "frames") - # Imported lazily so selection can depend on the discovery model without - # introducing a module import cycle. + if self.schema_version != WORKFLOW_SCHEMA_VERSION: + raise ValueError(f"schema_version must be {WORKFLOW_SCHEMA_VERSION!r}") from .selection import validate_frame_selection - validate_frame_selection(frames) - object.__setattr__(self, "frames", frames) - - if type(self.executables) is not dict: - raise TypeError("executables must be a mapping") - if not self.executables: - raise ValueError("executables must contain at least one entry") - executables: dict[str, ExecutableSpec] = {} - for raw_name, spec in self.executables.items(): - name = _identifier(raw_name, "executable name") - if not isinstance(spec, ExecutableSpec): - raise TypeError(f"executables.{name} must be an ExecutableSpec") - executables[name] = spec - object.__setattr__(self, "executables", executables) - - if not isinstance(self.scheduler, SchedulerSpec): - raise TypeError("scheduler must be a SchedulerSpec") - if type(self.steps) is not tuple: - raise TypeError("steps must be a sequence") - if not self.steps: - raise ValueError("steps must contain at least one entry") - if any(not isinstance(step, WorkflowStep) for step in self.steps): - raise TypeError("every steps entry must be a WorkflowStep") - - identifiers = [step.id for step in self.steps] - if len(identifiers) != len(set(identifiers)): - duplicates = sorted( - identifier for identifier, count in Counter(identifiers).items() if count > 1 - ) - raise ValueError(f"steps contain duplicate ids: {', '.join(duplicates)}") - step_by_id = {step.id: step for step in self.steps} + validate_frame_selection(self.frames) + if not self.executables or not self.steps: + raise ValueError("workflow requires executables and steps") + ids: set[str] = set() for step in self.steps: - if step.executable not in executables: - raise ValueError( - f"step {step.id!r} references unknown executable {step.executable!r}" - ) - unknown = sorted(set(step.depends_on) - set(step_by_id)) + if step.id in ids: + raise ValueError(f"duplicate workflow step id: {step.id}") + if step.executable not in self.executables: + raise ValueError(f"step {step.id!r} references an unknown executable") + unknown = set(step.depends_on) - ids if unknown: - raise ValueError( - f"step {step.id!r} references unknown dependencies: {', '.join(unknown)}" - ) - - self._reject_dependency_cycles(step_by_id) - positions = {step.id: index for index, step in enumerate(self.steps)} - for step in self.steps: - forward = [ - dependency - for dependency in step.depends_on - if positions[dependency] >= positions[step.id] - ] - if forward: - raise ValueError( - f"step {step.id!r} has forward dependencies: {', '.join(forward)}; " - "dependencies must name earlier steps" - ) - - @staticmethod - def _reject_dependency_cycles(step_by_id: dict[str, WorkflowStep]) -> None: - state: dict[str, int] = {} - stack: list[str] = [] - - def visit(identifier: str) -> None: - state[identifier] = 1 - stack.append(identifier) - for dependency in step_by_id[identifier].depends_on: - dependency_state = state.get(dependency, 0) - if dependency_state == 1: - start = stack.index(dependency) - cycle = [*stack[start:], dependency] - raise ValueError(f"workflow dependency cycle: {' -> '.join(cycle)}") - if dependency_state == 0: - visit(dependency) - stack.pop() - state[identifier] = 2 - - for identifier in step_by_id: - if state.get(identifier, 0) == 0: - visit(identifier) + raise ValueError(f"step {step.id!r} dependencies must name earlier steps") + ids.add(step.id) @classmethod def from_dict(cls, value: object) -> WorkflowManifest: - """Construct and validate a manifest from parsed YAML data.""" - - copied = _json_copy(value, path="workflow manifest") - data = _table(copied, "workflow manifest") - keys = frozenset( - {"executables", "frames", "project", "scheduler", "schema_version", "steps"} - ) - _check_keys(data, allowed=keys, required=keys, path="workflow manifest") - - executable_data = _table(data["executables"], "executables") - executables = { - _identifier(name, "executable name"): ExecutableSpec.from_dict( - spec, path=f"executables.{name}" - ) - for name, spec in executable_data.items() - } - raw_steps = _list(data["steps"], "steps") + data = _mapping(value, "workflow manifest") + keys = {"schema_version", "project", "frames", "executables", "scheduler", "steps"} + _fields(data, keys, keys, "workflow manifest") + executable_data = _mapping(data["executables"], "executables") + steps = _sequence(data["steps"], "steps") return cls( - schema_version=_required_string(data["schema_version"], "schema_version"), + schema_version=_text(data["schema_version"], "schema_version"), project=WorkflowProject.from_dict(data["project"]), - frames=_required_string(data["frames"], "frames"), - executables=executables, + frames=_text(data["frames"], "frames"), + executables={ + name: ExecutableSpec.from_dict(spec, path=f"executables.{name}") + for name, spec in executable_data.items() + }, scheduler=SchedulerSpec.from_dict(data["scheduler"]), steps=tuple( - WorkflowStep.from_dict(step, index=index) for index, step in enumerate(raw_steps) + WorkflowStep.from_dict(step, index=index) for index, step in enumerate(steps) ), ) def to_dict(self) -> dict[str, Any]: - """Return a deterministic mapping accepted by strict JSON encoders.""" - - result: dict[str, Any] = { + return { "schema_version": self.schema_version, "project": self.project.to_dict(), "frames": self.frames, - "executables": { - name: executable.to_dict() for name, executable in self.executables.items() - }, + "executables": {name: value.to_dict() for name, value in self.executables.items()}, "scheduler": self.scheduler.to_dict(), "steps": [step.to_dict() for step in self.steps], } - # Defend the public contract if a caller mutated one of the nested - # extension mappings after construction. - copied = _json_copy(result, path="workflow manifest") - return cast(dict[str, Any], copied) __all__ = [ diff --git a/src/electrace/workflow/operations.py b/src/electrace/workflow/operations.py index c51fd6e..049cb51 100644 --- a/src/electrace/workflow/operations.py +++ b/src/electrace/workflow/operations.py @@ -1,10 +1,8 @@ -"""Local and Slurm operations over a staged ElecTrace workflow.""" +"""Local and Slurm operations over a staged workflow.""" from __future__ import annotations import json -import os -import stat from collections import defaultdict from collections.abc import Mapping, Sequence from pathlib import Path @@ -23,19 +21,11 @@ from .manifest import load_workflow_manifest from .staging import StagedWorkflow, load_staged_workflow -_SubmissionStatus = Literal["submitting", "failed", "complete"] _SubmissionFailurePhase = Literal["submission", "recording"] class SlurmSubmissionError(RuntimeError): - """An interrupted Slurm submission sequence with safe-recovery context. - - ``partial_submissions`` contains every known accepted job id. For a - ``submission`` failure those ids have already been journaled at - ``record_path``. For a ``recording`` failure the exception is authoritative - because that journal may lag the accepted ids. Callers should report both - before permitting a retry. - """ + """A failed submission with every scheduler id known to this process.""" def __init__( self, @@ -46,118 +36,47 @@ def __init__( record_path: Path, submission_complete: bool = False, ) -> None: - if phase == "submission" and failed_script is None: - raise ValueError("failed_script is required for a submission failure") - if phase == "recording" and failed_script is not None: - raise ValueError("failed_script must be None for a recording failure") - if phase == "submission" and submission_complete: - raise ValueError("a failed submission cannot complete the submission sequence") self.phase = phase self.failed_script = failed_script self.partial_submissions = MappingProxyType(dict(partial_submissions)) self.record_path = record_path self.submission_complete = submission_complete - count = len(self.partial_submissions) - if phase == "submission": - message = ( - f"Slurm submission failed for {failed_script} after {count} successful " - f"submission{'s' if count != 1 else ''}" - ) - elif submission_complete: - message = ( - f"all {count} Slurm jobs were accepted, but marking the record complete failed" - ) - else: - message = ( - f"{count} Slurm submission{'s were' if count != 1 else ' was'} accepted, " - "but updating the record failed" - ) - super().__init__(f"{message}; inspect {record_path} before retrying") - - -def _write_submission_record( - path: Path, - *, - job_name: str, - manifest_sha256: str, - submissions: Mapping[str, str], - status: _SubmissionStatus, - failed_script: Path | None = None, -) -> None: - atomic_write_json( - path, - _submission_record( - job_name=job_name, - manifest_sha256=manifest_sha256, - submissions=submissions, - status=status, - failed_script=failed_script, - ), - ) + detail = f" for {failed_script}" if failed_script is not None else "" + super().__init__( + f"Slurm {phase} failed{detail} after {len(partial_submissions)} accepted job(s); " + f"inspect {record_path} before retrying" + ) def _submission_record( - *, + staged: StagedWorkflow, job_name: str, - manifest_sha256: str, submissions: Mapping[str, str], - status: _SubmissionStatus, + status: str, failed_script: Path | None = None, ) -> dict[str, Any]: return { "schema_version": "electrace.slurm-submissions.v1", "job_name": job_name, - "manifest_sha256": manifest_sha256, + "manifest_sha256": staged.manifest_sha256, "status": status, - "failed_script": str(failed_script) if failed_script is not None else None, + "failed_script": str(failed_script) if failed_script else None, "submissions": dict(submissions), } -def _reserve_submission_record( +def _write_submission_record( path: Path, - *, + staged: StagedWorkflow, job_name: str, - manifest_sha256: str, -) -> tuple[int, int]: - """Exclusively reserve a journal before any scheduler call is made.""" - - payload = _submission_record( - job_name=job_name, - manifest_sha256=manifest_sha256, - submissions={}, - status="submitting", + submissions: Mapping[str, str], + status: str, + failed_script: Path | None = None, +) -> None: + atomic_write_json( + path, + _submission_record(staged, job_name, submissions, status, failed_script), ) - document = json.dumps(payload, indent=2, sort_keys=True, allow_nan=False) + "\n" - path.parent.mkdir(parents=True, exist_ok=True) - flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0) - try: - descriptor = os.open(path, flags, 0o600) - except FileExistsError as exc: - raise FileExistsError( - f"refusing to submit while a prior Slurm submission record exists: {path}" - ) from exc - identity = os.fstat(descriptor) - try: - with os.fdopen(descriptor, "w", encoding="utf-8", newline="\n") as handle: - handle.write(document) - handle.flush() - os.fsync(handle.fileno()) - except BaseException: - _remove_reserved_submission_record(path, (identity.st_dev, identity.st_ino)) - raise - return identity.st_dev, identity.st_ino - - -def _remove_reserved_submission_record(path: Path, identity: tuple[int, int]) -> None: - """Remove only the exact empty-attempt journal reserved by this process.""" - - try: - current = path.stat(follow_symlinks=False) - except FileNotFoundError: - return - if stat.S_ISREG(current.st_mode) and (current.st_dev, current.st_ino) == identity: - path.unlink() def run_staged_workflow( @@ -167,8 +86,6 @@ def run_staged_workflow( max_parallel: int = 1, stop_on_error: bool = True, ) -> tuple[JobResult, ...]: - """Execute a local staged workflow after explicit confirmation.""" - manifest = load_workflow_manifest(manifest_path) if manifest.scheduler.kind != "local": raise ValueError("workflow run requires scheduler.kind: local") @@ -192,12 +109,9 @@ def run_staged_workflow( def _jobs_by_frame(staged: StagedWorkflow) -> tuple[tuple[JobSpec, ...], ...]: grouped: dict[int, list[JobSpec]] = defaultdict(list) - order: list[int] = [] for job in staged.jobs: - if job.frame_index not in grouped: - order.append(job.frame_index) grouped[job.frame_index].append(job) - return tuple(tuple(grouped[index]) for index in order) + return tuple(tuple(grouped[frame.index]) for frame in staged.frames) def prepare_slurm_scripts( @@ -206,28 +120,25 @@ def prepare_slurm_scripts( job_name: str = "electrace", overwrite: bool = False, ) -> tuple[Path, ...]: - """Render one dependency-ordered Slurm script per selected frame.""" - manifest = load_workflow_manifest(manifest_path) if manifest.scheduler.kind != "slurm": raise ValueError("workflow submit requires scheduler.kind: slurm") staged = load_staged_workflow(manifest_path) script_directory = staged.workspace / "slurm" - directives = manifest.scheduler.directives - planned: list[tuple[str, Path]] = [] - for jobs in _jobs_by_frame(staged): - first = jobs[0] - document = render_slurm_script(jobs, directives, job_name) - path = script_directory / f"frame-{first.frame_index:06d}.slurm" - planned.append((document, path)) - if not overwrite: - existing = next( - (path for _, path in planned if path.exists() or path.is_symlink()), - None, + planned = tuple( + ( + render_slurm_script(jobs, manifest.scheduler.directives, job_name), + script_directory / f"frame-{jobs[0].frame_index:06d}.slurm", ) + for jobs in _jobs_by_frame(staged) + ) + if not overwrite: + existing = next((path for _, path in planned if path.exists() or path.is_symlink()), None) if existing is not None: raise FileExistsError(f"refusing to replace existing Slurm script: {existing}") - paths = [write_slurm_script(document, path, overwrite=overwrite) for document, path in planned] + paths = tuple( + write_slurm_script(document, path, overwrite=overwrite) for document, path in planned + ) atomic_write_json( script_directory / "scripts.json", { @@ -237,7 +148,7 @@ def prepare_slurm_scripts( "scripts": [str(path) for path in paths], }, ) - return tuple(paths) + return paths def submit_staged_workflow( @@ -248,61 +159,66 @@ def submit_staged_workflow( confirmed: bool = False, overwrite_scripts: bool = False, ) -> dict[str, str]: - """Render and explicitly submit one Slurm job per selected frame.""" + """Submit one script per frame and persist each accepted scheduler id.""" if not confirmed: raise PermissionError("Slurm submission requires confirmed=True") staged = load_staged_workflow(manifest_path) record_path = staged.workspace / "slurm" / "submissions.json" - submissions: dict[str, str] = {} - reservation = _reserve_submission_record( - record_path, + if record_path.exists() or record_path.is_symlink(): + raise FileExistsError( + f"refusing to submit while a prior Slurm submission record exists: {record_path}" + ) + scripts = prepare_slurm_scripts( + manifest_path, job_name=job_name, - manifest_sha256=staged.manifest_sha256, + overwrite=overwrite_scripts, ) - try: - scripts = prepare_slurm_scripts( - manifest_path, - job_name=job_name, - overwrite=overwrite_scripts, + record_path.parent.mkdir(parents=True, exist_ok=True) + initial = ( + json.dumps( + _submission_record(staged, job_name, {}, "submitting"), + indent=2, + sort_keys=True, + allow_nan=False, ) - except BaseException: - _remove_reserved_submission_record(record_path, reservation) - raise + + "\n" + ) + with record_path.open("x", encoding="utf-8", newline="\n") as handle: + handle.write(initial) + + submissions: dict[str, str] = {} for script in scripts: try: - job_id = submit_slurm_script( + submissions[str(script)] = submit_slurm_script( script, submit_command=submit_command, confirmed=True, ) except Exception as exc: - error = SlurmSubmissionError( - phase="submission", - failed_script=script, - partial_submissions=submissions, - record_path=record_path, - ) try: _write_submission_record( record_path, - job_name=job_name, - manifest_sha256=staged.manifest_sha256, - submissions=submissions, - status="failed", - failed_script=script, + staged, + job_name, + submissions, + "failed", + script, ) - except Exception as record_error: - error.add_note(f"could not mark the submission record failed: {record_error}") - raise error from exc - submissions[str(script)] = job_id + finally: + raise SlurmSubmissionError( + phase="submission", + failed_script=script, + partial_submissions=submissions, + record_path=record_path, + ) from exc try: _write_submission_record( record_path, - job_name=job_name, - manifest_sha256=staged.manifest_sha256, - submissions=submissions, - status="submitting", + staged, + job_name, + submissions, + "submitting", ) except Exception as exc: raise SlurmSubmissionError( @@ -312,13 +228,7 @@ def submit_staged_workflow( record_path=record_path, ) from exc try: - _write_submission_record( - record_path, - job_name=job_name, - manifest_sha256=staged.manifest_sha256, - submissions=submissions, - status="complete", - ) + _write_submission_record(record_path, staged, job_name, submissions, "complete") except Exception as exc: raise SlurmSubmissionError( phase="recording", @@ -331,8 +241,6 @@ def submit_staged_workflow( def workflow_status(manifest_path: str | Path) -> dict[str, Any]: - """Return a read-only status snapshot for every staged job.""" - staged = load_staged_workflow(manifest_path) jobs = [] for job in staged.jobs: diff --git a/src/electrace/workflow/selection.py b/src/electrace/workflow/selection.py index 34c6a3a..802051f 100644 --- a/src/electrace/workflow/selection.py +++ b/src/electrace/workflow/selection.py @@ -1,150 +1,87 @@ -"""Canonical frame-index selection for discovered ElecTrace paths.""" +"""Frame selection for discovered workflow paths.""" from __future__ import annotations -import re from collections.abc import Sequence from electrace.io.discovery import FrameSource -_INTEGER = re.compile(r"^[+-]?\d+$") - class FrameSelectionError(ValueError): - """Raised when a workflow frame selector is malformed or cannot resolve.""" - - -def _selection_tokens(selection: str) -> tuple[int | slice, ...]: - if type(selection) is not str: - raise TypeError("frame selection must be a string") - if not selection or any( - ord(character) < 32 or ord(character) == 127 for character in selection - ): - raise FrameSelectionError( - "frame selection must be non-empty and contain no control characters" - ) - - stripped = selection.strip() - if not stripped: - raise FrameSelectionError("frame selection must not be blank") - if stripped == "all": - return (slice(None, None, None),) - if "all" in {part.strip() for part in stripped.split(",")}: - raise FrameSelectionError("'all' cannot be combined with other frame selectors") - - selectors: list[int | slice] = [] - for token_index, raw_token in enumerate(stripped.split(",")): - token = raw_token.strip() + """A frame selector is malformed or resolves to no frames.""" + + +def _selectors(selection: str) -> tuple[int | slice, ...]: + if not isinstance(selection, str) or not selection.strip(): + raise FrameSelectionError("frame selection must be non-empty text") + selection = selection.strip() + if selection == "all": + return (slice(None),) + if "all" in {part.strip() for part in selection.split(",")}: + raise FrameSelectionError("'all' cannot be combined with other selectors") + + result: list[int | slice] = [] + for token in (part.strip() for part in selection.split(",")): if not token: - raise FrameSelectionError(f"frame selector token {token_index} is empty") + raise FrameSelectionError("frame selection contains an empty token") if ":" not in token: - if _INTEGER.fullmatch(token) is None: - raise FrameSelectionError(f"invalid canonical frame index {token!r}") - index = int(token) + try: + index = int(token) + except ValueError as exc: + raise FrameSelectionError(f"invalid frame index: {token!r}") from exc if index < 0: - raise FrameSelectionError( - f"explicit canonical frame index must be non-negative; got {index}" - ) - selectors.append(index) + raise FrameSelectionError("explicit frame indices must be non-negative") + result.append(index) continue - - components = token.split(":") - if len(components) not in {2, 3}: - raise FrameSelectionError( - f"frame slice {token!r} must use start:stop or start:stop:step" - ) - parsed: list[int | None] = [] - for raw_component in components: - component = raw_component.strip() - if not component: - parsed.append(None) - continue - if _INTEGER.fullmatch(component) is None: - raise FrameSelectionError( - f"frame slice {token!r} has invalid component {component!r}" - ) - parsed.append(int(component)) - start, stop = parsed[0], parsed[1] - step = parsed[2] if len(parsed) == 3 else None - if step == 0: - raise FrameSelectionError(f"frame slice {token!r} has a zero step") - selectors.append(slice(start, stop, step)) - return tuple(selectors) + parts = token.split(":") + if len(parts) not in {2, 3}: + raise FrameSelectionError(f"invalid frame slice: {token!r}") + try: + values = [int(part) if part.strip() else None for part in parts] + except ValueError as exc: + raise FrameSelectionError(f"invalid frame slice: {token!r}") from exc + if len(values) == 2: + values.append(None) + if values[2] == 0: + raise FrameSelectionError("frame slice step must not be zero") + result.append(slice(*values)) + return tuple(result) def validate_frame_selection(selection: str) -> None: - """Validate selector syntax without requiring a discovered frame count.""" - - _selection_tokens(selection) + _selectors(selection) def parse_frame_selection(selection: str, frame_count: int) -> tuple[int, ...]: - """Resolve a selector to canonical indices for a sequence of known length. - - Individual indices are zero-based canonical frame indices. Slice entries - follow Python's clipping and negative-index rules, including negative steps. - Comma-separated entries are evaluated left to right. Duplicate selections - are rejected because running a frame twice is almost always a workflow error. - """ - - if type(frame_count) is not int: - raise TypeError("frame_count must be an integer") - if frame_count < 0: - raise ValueError("frame_count must be non-negative") - + if not isinstance(frame_count, int) or isinstance(frame_count, bool) or frame_count < 0: + raise ValueError("frame_count must be a non-negative integer") resolved: list[int] = [] - seen: set[int] = set() - for selector in _selection_tokens(selection): - indices: Sequence[int] + for selector in _selectors(selection): if isinstance(selector, int): if selector >= frame_count: - raise FrameSelectionError( - f"canonical frame index {selector} is outside 0:{frame_count}" - ) - indices = (selector,) + raise FrameSelectionError(f"frame index {selector} is outside 0:{frame_count}") + resolved.append(selector) else: - indices = range(*selector.indices(frame_count)) - for index in indices: - if index in seen: - raise FrameSelectionError( - f"canonical frame index {index} is selected more than once" - ) - seen.add(index) - resolved.append(index) - + resolved.extend(range(*selector.indices(frame_count))) if not resolved: raise FrameSelectionError("frame selection resolved to no frames") + if len(resolved) != len(set(resolved)): + raise FrameSelectionError("frame selection contains duplicates") return tuple(resolved) def resolve_frame_selection( selection: str, frames: Sequence[FrameSource] ) -> tuple[FrameSource, ...]: - """Return selected discovered records in selector evaluation order. - - Discovery assigns canonical indices as consecutive zero-based positions. - Requiring that invariant here prevents a slice position from silently being - mistaken for a source-directory number or a caller-provided sparse index. - """ - records = tuple(frames) - if any(not isinstance(frame, FrameSource) for frame in records): - raise TypeError("frames must contain only FrameSource records") - canonical_indices = tuple(frame.index for frame in records) - expected = tuple(range(len(records))) - if canonical_indices != expected: - raise FrameSelectionError( - "FrameSource records must be in canonical order with consecutive zero-based indices" - ) - indices = parse_frame_selection(selection, len(records)) - return tuple(records[index] for index in indices) + if tuple(frame.index for frame in records) != tuple(range(len(records))): + raise FrameSelectionError("frames must use consecutive zero-based canonical indices") + return tuple(records[index] for index in parse_frame_selection(selection, len(records))) def resolve_selected_frames( frames: Sequence[FrameSource], selection: str ) -> tuple[FrameSource, ...]: - """Sequence-first convenience wrapper around :func:`resolve_frame_selection`.""" - return resolve_frame_selection(selection, frames) diff --git a/src/electrace/workflow/staging.py b/src/electrace/workflow/staging.py index 29a69d2..7293938 100644 --- a/src/electrace/workflow/staging.py +++ b/src/electrace/workflow/staging.py @@ -1,15 +1,8 @@ -"""Safe staging of executable ElecTrace workflows. - -The source VASP path is always read-only. Every executable runs in a fresh, -marker-protected workspace populated from reviewed template directories and -explicit dependency links. The resulting :class:`JobSpec` records are the -only objects accepted by the separate execution boundary. -""" +"""Stage workflow inputs in an isolated, marker-owned workspace.""" from __future__ import annotations import json -import math import os import re import shutil @@ -22,13 +15,7 @@ from electrace.plugins import plan_plugin from electrace.provenance import sha256_file -from ._files import ( - absolute_lexical, - atomic_write_json, - atomic_write_text, - read_limited_text, - symlink_component, -) +from ._files import absolute_lexical, atomic_write_json, atomic_write_text, symlink_component from .execution import JobSpec, validate_jobs from .manifest import load_workflow_manifest from .models import ExecutableSpec, JSONValue, WorkflowManifest, WorkflowStep @@ -36,53 +23,39 @@ WORKSPACE_SCHEMA_VERSION = "electrace.workspace.v1" WORKSPACE_RECORD = ".electrace-workspace.json" -MAX_WORKSPACE_RECORD_BYTES = 16_000_000 _INCAR_KEY = re.compile(r"^[A-Za-z][A-Za-z0-9_]*$") _SAFE_LABEL = re.compile(r"[^A-Za-z0-9_.-]+") _LOBSTER_BOND_REQUEST = re.compile(r"(?im)^\s*(?:cohpgenerator|cohpbetween)\b") _LOBSTER_SKIP_COBI = re.compile(r"(?im)^\s*skipcobi\b") _DESCRIPTOR_FILENAMES = frozenset( - { - "ACF.dat", - "DDEC6_even_tempered_net_atomic_charges.xyz", - "ICOBILIST.lobster", - "ICOHPLIST.lobster", - "ICOOPLIST.lobster", - "badelf.json", - "bader.json", - "critic2.features.json", - "electrace.features.json", - "features.json", - } + """ACF.dat DDEC6_even_tempered_net_atomic_charges.xyz ICOBILIST.lobster + ICOHPLIST.lobster badelf.json bader.json critic2.features.json + electrace.features.json features.json""".split() # noqa: SIM905 ) def _resolve_from(base: Path, value: str) -> Path: - candidate = Path(value).expanduser() - return absolute_lexical(candidate if candidate.is_absolute() else base / candidate) + path = Path(value).expanduser() + return absolute_lexical(path if path.is_absolute() else base / path) def _resolve_command(base: Path, value: str) -> str: - candidate = Path(value).expanduser() - if candidate.is_absolute() or "/" in value or "\\" in value: - return str(_resolve_from(base, value)) - return value + return str(_resolve_from(base, value)) if Path(value).is_absolute() or "/" in value else value def _basename(value: object, path: str) -> str: if not isinstance(value, str): raise TypeError(f"{path} must be a string") - name = value.strip() if ( - not name - or name in {".", ".."} - or name != value - or "/" in name - or "\\" in name - or any(ord(character) < 32 or ord(character) == 127 for character in name) + not value + or value != value.strip() + or value in {".", ".."} + or "/" in value + or "\\" in value + or any(ord(character) < 32 for character in value) ): - raise ValueError(f"{path} must be a safe basename, not a path: {value!r}") - return name + raise ValueError(f"{path} must be a safe basename: {value!r}") + return value def _string_list(value: JSONValue | None, path: str) -> tuple[str, ...]: @@ -90,20 +63,17 @@ def _string_list(value: JSONValue | None, path: str) -> tuple[str, ...]: return () if not isinstance(value, list): raise TypeError(f"{path} must be a list") - return tuple(_basename(item, f"{path}[{index}]") for index, item in enumerate(value)) + return tuple(_basename(item, path) for item in value) def _string_mapping(value: JSONValue | None, path: str) -> dict[str, str]: if value is None: return {} - if not isinstance(value, dict): - raise TypeError(f"{path} must be a mapping") - result: dict[str, str] = {} - for key, item in value.items(): - if not isinstance(key, str) or not isinstance(item, str): - raise TypeError(f"{path} must contain string keys and values") - result[key] = item - return result + if not isinstance(value, dict) or any( + not isinstance(key, str) or not isinstance(item, str) for key, item in value.items() + ): + raise TypeError(f"{path} must contain string keys and values") + return cast(dict[str, str], value) def _safe_label(index: int, label: str) -> str: @@ -111,31 +81,8 @@ def _safe_label(index: int, label: str) -> str: return f"{index:06d}-{suffix[:80]}" -def _record_string(data: dict[str, object], key: str, context: str) -> str: - value = data.get(key) - if not isinstance(value, str): - raise TypeError(f"{context} {key} must be a string") - return value - - -def _record_list(data: dict[str, object], key: str, context: str) -> list[object]: - value = data.get(key) - if not isinstance(value, list): - raise TypeError(f"{context} {key} must be an array") - return cast(list[object], value) - - -def _record_string_list(data: dict[str, object], key: str, context: str) -> tuple[str, ...]: - values = _record_list(data, key, context) - if any(not isinstance(value, str) for value in values): - raise TypeError(f"{context} {key} must contain only strings") - return tuple(cast(list[str], values)) - - @dataclass(frozen=True, slots=True) class StagedFrame: - """One selected source frame and its dedicated workspace directory.""" - index: int label: str analysis_label: str @@ -144,17 +91,6 @@ class StagedFrame: workspace_directory: Path energy_ev: float | None = None - def __post_init__(self) -> None: - if isinstance(self.index, bool) or not isinstance(self.index, int): - raise TypeError("staged frame index must be an integer") - if self.index < 0: - raise ValueError("staged frame index must be non-negative") - if self.energy_ev is not None: - if isinstance(self.energy_ev, bool) or not isinstance(self.energy_ev, (int, float)): - raise TypeError("staged frame energy must be numeric or null") - if not math.isfinite(self.energy_ev): - raise ValueError("staged frame energy must be finite") - def to_dict(self) -> dict[str, Any]: return { "index": self.index, @@ -166,36 +102,9 @@ def to_dict(self) -> dict[str, Any]: "energy_ev": self.energy_ev, } - @classmethod - def from_dict(cls, payload: object) -> StagedFrame: - if not isinstance(payload, dict): - raise TypeError("workspace frame record must be an object") - data = cast(dict[str, object], payload) - raw_index = data.get("index") - raw_energy = data.get("energy_ev") - if type(raw_index) is not int: - raise TypeError("workspace frame index must be an integer") - if raw_energy is not None and ( - isinstance(raw_energy, bool) or not isinstance(raw_energy, (int, float)) - ): - raise TypeError("workspace frame energy must be numeric or null") - return cls( - index=raw_index, - label=_record_string(data, "label", "workspace frame"), - analysis_label=_record_string(data, "analysis_label", "workspace frame"), - source_directory=Path(_record_string(data, "source_directory", "workspace frame")), - structure=Path(_record_string(data, "structure", "workspace frame")), - workspace_directory=Path( - _record_string(data, "workspace_directory", "workspace frame") - ), - energy_ev=None if raw_energy is None else float(raw_energy), - ) - @dataclass(frozen=True, slots=True) class StagedWorkflow: - """Auditable output of workflow staging.""" - manifest_path: Path manifest_sha256: str source_root: Path @@ -221,20 +130,14 @@ def to_dict(self) -> dict[str, Any]: } -def _copy_template(source: Path, destination: Path) -> None: - if not source.is_dir(): - raise FileNotFoundError(f"workflow template directory does not exist: {source}") - if symlink_component(source) is not None: - raise ValueError(f"workflow template directory must not traverse a symlink: {source}") - for item in sorted(source.iterdir(), key=lambda path: path.name): - if item.is_symlink(): - raise ValueError(f"workflow templates must not contain symlinks: {item}") - if not item.is_file(): - raise ValueError(f"workflow templates must contain only regular files: {item}") +def _copy_template(files: tuple[Path, ...], destination: Path) -> None: + for item in files: shutil.copy2(item, destination / _basename(item.name, "template filename")) def _replace_file(source: Path, destination: Path) -> None: + if source.is_symlink() or not source.is_file(): + raise FileNotFoundError(f"workflow source is not a regular file: {source}") if destination.exists() or destination.is_symlink(): destination.unlink() shutil.copy2(source, destination) @@ -243,94 +146,150 @@ def _replace_file(source: Path, destination: Path) -> None: def _apply_incar_overrides(path: Path, raw: JSONValue | None) -> None: if raw is None: return - if not isinstance(raw, dict): - raise TypeError("vasp options.incar_overrides must be a mapping") - if not path.is_file(): - raise FileNotFoundError("vasp options.incar_overrides requires an INCAR template") + if not isinstance(raw, dict) or not path.is_file(): + raise ValueError("vasp incar_overrides requires a mapping and an INCAR template") overrides: dict[str, str] = {} - for raw_key, value in raw.items(): - if _INCAR_KEY.fullmatch(raw_key) is None: - raise ValueError(f"invalid INCAR tag in workflow: {raw_key!r}") - if isinstance(value, bool): - encoded = ".TRUE." if value else ".FALSE." - elif isinstance(value, (str, int, float)) and not isinstance(value, bool): - encoded = str(value) - else: - raise TypeError(f"INCAR override {raw_key!r} must be a scalar") - overrides[raw_key.upper()] = encoded - - lines = path.read_text(encoding="utf-8").splitlines() - written: set[str] = set() + for key, value in raw.items(): + if _INCAR_KEY.fullmatch(key) is None or not isinstance(value, (str, int, float, bool)): + raise ValueError(f"invalid INCAR override: {key!r}") + overrides[key.upper()] = ( + ".TRUE." if value is True else ".FALSE." if value is False else str(value) + ) result: list[str] = [] - for line in lines: + for line in path.read_text(encoding="utf-8").splitlines(): content = line.split("!", 1)[0].split("#", 1)[0] - candidate = content.split("=", 1)[0].strip().upper() if "=" in content else "" - if candidate in overrides: - if candidate not in written: - result.append(f"{candidate} = {overrides[candidate]}") - written.add(candidate) - continue - result.append(line) - for key, value in overrides.items(): - if key not in written: - result.append(f"{key} = {value}") + key = content.split("=", 1)[0].strip().upper() if "=" in content else "" + result.append(f"{key} = {overrides.pop(key)}" if key in overrides else line) + result.extend(f"{key} = {value}" for key, value in overrides.items()) atomic_write_text(path, "\n".join(result).rstrip() + "\n") +def _plugin_options(step: WorkflowStep) -> dict[str, JSONValue]: + reserved = {"environment", "input_links", "plugin_options"} + result = {key: value for key, value in step.options.items() if key not in reserved} + nested = step.options.get("plugin_options") + if nested is not None: + if not isinstance(nested, dict) or set(result) & set(nested): + raise ValueError(f"step {step.id} has invalid or duplicate plugin_options") + result.update(nested) + return result + + def _step_outputs(step: WorkflowStep) -> tuple[str, ...]: if step.kind == "vasp": - raw_outputs = step.options.get("outputs", step.options.get("expected_outputs")) - outputs = _string_list(raw_outputs, f"step {step.id}.options.expected_outputs") + outputs = _string_list( + step.options.get("outputs", step.options.get("expected_outputs")), + f"step {step.id}.options.expected_outputs", + ) if not outputs: - raise ValueError(f"vasp step {step.id!r} must declare options.expected_outputs") + raise ValueError(f"vasp step {step.id!r} must declare expected outputs") return outputs - capability = step.capability or "" if step.kind == "bader": return ("ACF.dat",) if step.kind == "baderkit": return ( ("badelf.json",) - if capability in {"elf_basins", "non_nuclear_attractors"} + if step.capability in {"elf_basins", "non_nuclear_attractors"} else ("bader.json",) ) if step.kind == "critic2": - report = str(_plugin_options(step).get("report_file", "critic2.features.json")) - report = _basename(report, f"step {step.id}.report_file") + report = _basename( + _plugin_options(step).get("report_file", "critic2.features.json"), "report_file" + ) if report != "critic2.features.json": - raise ValueError( - "workflow critic2 report_file must be 'critic2.features.json' so collection " - "can discover it deterministically" - ) + raise ValueError("critic2 report_file must be critic2.features.json") return ("critic2.cro", report) if step.kind == "lobster": - return ("ICOBILIST.lobster",) if capability == "cobi" else ("ICOHPLIST.lobster",) + return ("ICOBILIST.lobster",) if step.capability == "cobi" else ("ICOHPLIST.lobster",) raise ValueError(f"unsupported workflow step kind: {step.kind}") +def _vasp_required_inputs(step: WorkflowStep) -> tuple[str, ...]: + return _string_list(step.options.get("required_inputs"), "required_inputs") or ( + "INCAR", + "KPOINTS", + "POSCAR", + "POTCAR", + ) + + +def _template_files(step: WorkflowStep, manifest_base: Path) -> tuple[Path, ...]: + if step.template_dir is None: + files: tuple[Path, ...] = () + else: + source = _resolve_from(manifest_base, step.template_dir) + if source.is_symlink() or not source.is_dir(): + raise FileNotFoundError(f"workflow template directory is invalid: {source}") + files = tuple(source.iterdir()) + if any(item.is_symlink() or not item.is_file() for item in files): + raise ValueError(f"workflow templates must contain regular files: {source}") + names = {_basename(item.name, "template filename") for item in files} + if step.kind != "vasp" and (collision := names.intersection(_step_outputs(step))): + raise ValueError( + f"step {step.id!r} template files collide with expected outputs: " + + ", ".join(sorted(collision)) + ) + if step.kind == "vasp" and ( + missing := set(_vasp_required_inputs(step)).difference(names, {"POSCAR"}) + ): + raise FileNotFoundError( + f"vasp step {step.id!r} required inputs are missing from its template: " + + ", ".join(sorted(missing)) + ) + return files + + +def _plugin_required_names(step: WorkflowStep) -> tuple[str, ...]: + options = _plugin_options(step) + if step.kind in {"bader", "baderkit"}: + names = [str(options.get("charge_file", "CHGCAR"))] + if step.kind == "baderkit" and step.capability in {"elf_basins", "non_nuclear_attractors"}: + names.append(str(options.get("elf_file", "ELFCAR"))) + if options.get("reference_file") is not None: + names.append(str(options["reference_file"])) + return tuple(names) + if step.kind == "critic2": + default = ( + "ELFCAR" if step.capability in {"elf_basins", "non_nuclear_attractors"} else "CHGCAR" + ) + names = [ + str(options.get("structure_file", "CONTCAR")), + str(options.get("field_file", default)), + ] + if options.get("input_file") is not None: + names.append(str(options["input_file"])) + return tuple(names) + if step.kind == "lobster": + return tuple( + str(options.get(key, default)) + for key, default in ( + ("input_file", "lobsterin"), + ("wavecar", "WAVECAR"), + ("structure_file", "POSCAR"), + ("potcar", "POTCAR"), + ) + ) + return () + + def _dependency_source( - target: str, - dependencies: tuple[WorkflowStep, ...], - directories: dict[str, Path], + name: str, dependencies: tuple[WorkflowStep, ...], directories: dict[str, Path] ) -> Path | None: for dependency in reversed(dependencies): - directory = directories[dependency.id] - candidate = directory / target - if candidate.exists() or target in _step_outputs(dependency): + candidate = directories[dependency.id] / name + if candidate.exists() or name in _step_outputs(dependency): return candidate return None def _relative_link(source: Path, destination: Path) -> None: if destination.exists() or destination.is_symlink(): - raise FileExistsError(f"workflow input target already exists: {destination}") - destination.symlink_to(os.path.relpath(source, start=destination.parent)) + raise FileExistsError(f"workflow input already exists: {destination}") + destination.symlink_to(os.path.relpath(source, destination.parent)) def _explicit_input_links( - step: WorkflowStep, - source: FrameSource, - destination: Path, - directories: dict[str, Path], + step: WorkflowStep, source: FrameSource, destination: Path, directories: dict[str, Path] ) -> None: raw = step.options.get("input_links") if raw is None: @@ -338,58 +297,17 @@ def _explicit_input_links( if not isinstance(raw, dict): raise TypeError(f"step {step.id}.options.input_links must be a mapping") for raw_target, specification in raw.items(): - target = _basename(raw_target, f"step {step.id}.options.input_links key") + target = _basename(raw_target, "input link target") if specification == "source:structure": _replace_file(source.structure, destination / target) continue if not isinstance(specification, dict): - raise TypeError( - f"step {step.id}.options.input_links.{target} must be an object or source:structure" - ) + raise TypeError(f"invalid input link for {target}") dependency = specification.get("step") - filename = specification.get("file", target) if not isinstance(dependency, str) or dependency not in step.depends_on: - raise ValueError( - f"step {step.id}.options.input_links.{target}.step must be a declared dependency" - ) - source_name = _basename(filename, f"step {step.id}.options.input_links.{target}.file") - _relative_link(directories[dependency] / source_name, destination / target) - - -def _plugin_required_names(step: WorkflowStep) -> tuple[str, ...]: - options = _plugin_options(step) - if step.kind == "bader": - result = [str(options.get("charge_file", "CHGCAR"))] - if options.get("reference_file") is not None: - result.append(str(options["reference_file"])) - return tuple(result) - if step.kind == "baderkit": - result = [str(options.get("charge_file", "CHGCAR"))] - if step.capability in {"elf_basins", "non_nuclear_attractors"}: - result.append(str(options.get("elf_file", "ELFCAR"))) - if options.get("reference_file") is not None: - result.append(str(options["reference_file"])) - return tuple(result) - if step.kind == "critic2": - structure = str(options.get("structure_file", "CONTCAR")) - default_field = ( - "ELFCAR" if step.capability in {"elf_basins", "non_nuclear_attractors"} else "CHGCAR" - ) - result = [structure, str(options.get("field_file", default_field))] - if options.get("input_file") is not None: - result.append(str(options["input_file"])) - return tuple(result) - if step.kind == "lobster": - return tuple( - str(options.get(key, default)) - for key, default in ( - ("input_file", "lobsterin"), - ("wavecar", "WAVECAR"), - ("structure_file", "POSCAR"), - ("potcar", "POTCAR"), - ) - ) - return () + raise ValueError(f"input link {target} must name a declared dependency") + name = _basename(specification.get("file", target), "input link file") + _relative_link(directories[dependency] / name, destination / target) def _auto_plugin_inputs( @@ -399,8 +317,7 @@ def _auto_plugin_inputs( dependencies: tuple[WorkflowStep, ...], directories: dict[str, Path], ) -> None: - options = _plugin_options(step) - structure_name = str(options.get("structure_file", "CONTCAR")) + structure_name = str(_plugin_options(step).get("structure_file", "CONTCAR")) for raw_name in _plugin_required_names(step): name = _basename(raw_name, f"step {step.id} plugin input") target = destination / name @@ -408,79 +325,42 @@ def _auto_plugin_inputs( continue if step.kind in {"critic2", "lobster"} and name == structure_name: _replace_file(source.structure, target) - continue - candidate = _dependency_source(name, dependencies, directories) - if candidate is not None: + elif (candidate := _dependency_source(name, dependencies, directories)) is not None: _relative_link(candidate, target) -def _validate_lobster_input(step: WorkflowStep, directory: Path) -> None: - options = _plugin_options(step) - input_name = _basename( - options.get("input_file", "lobsterin"), f"step {step.id} lobster input_file" - ) - path = directory / input_name - if not path.is_file() or path.is_symlink(): - raise FileNotFoundError(f"LOBSTER workflow requires a regular reviewed {input_name}") - text = path.read_text(encoding="utf-8", errors="replace") - if _LOBSTER_BOND_REQUEST.search(text) is None: - raise ValueError( - f"LOBSTER step {step.id!r} must request bonds with cohpGenerator or cohpBetween" - ) - if step.capability == "cobi" and _LOBSTER_SKIP_COBI.search(text) is not None: - raise ValueError(f"LOBSTER COBI step {step.id!r} cannot use skipcobi") - - def _prepare_step_directory( step: WorkflowStep, source: FrameSource, destination: Path, - manifest_base: Path, + template_files: tuple[Path, ...], prior_steps: dict[str, WorkflowStep], directories: dict[str, Path], ) -> None: destination.mkdir(parents=True) - if step.template_dir is not None: - _copy_template(_resolve_from(manifest_base, step.template_dir), destination) - dependencies = tuple(prior_steps[identifier] for identifier in step.depends_on) + _copy_template(template_files, destination) if step.kind == "vasp": _replace_file(source.structure, destination / "POSCAR") _apply_incar_overrides(destination / "INCAR", step.options.get("incar_overrides")) - else: - _explicit_input_links(step, source, destination, directories) - _auto_plugin_inputs(step, source, destination, dependencies, directories) - if step.kind == "lobster": - _validate_lobster_input(step, destination) + return + dependencies = tuple(prior_steps[item] for item in step.depends_on) + _explicit_input_links(step, source, destination, directories) + _auto_plugin_inputs(step, source, destination, dependencies, directories) + if step.kind == "lobster": + options = _plugin_options(step) + name = _basename(options.get("input_file", "lobsterin"), "lobster input_file") + path = destination / name + text = path.read_text(encoding="utf-8", errors="replace") if path.is_file() else "" + if path.is_symlink() or _LOBSTER_BOND_REQUEST.search(text) is None: + raise ValueError(f"LOBSTER requires a reviewed bond-requesting {name}") + if step.capability == "cobi" and _LOBSTER_SKIP_COBI.search(text): + raise ValueError("LOBSTER COBI input cannot use skipcobi") def _environment(step: WorkflowStep) -> dict[str, str]: return _string_mapping(step.options.get("environment"), f"step {step.id}.environment") -def _plugin_options(step: WorkflowStep) -> dict[str, JSONValue]: - """Return planner options, accepting a flat public form or nested extension.""" - - reserved = {"environment", "input_links", "plugin_options"} - result = {key: value for key, value in step.options.items() if key not in reserved} - nested = step.options.get("plugin_options") - if nested is None: - return result - if not isinstance(nested, dict): - raise TypeError(f"step {step.id}.options.plugin_options must be a mapping") - duplicates = sorted(set(result) & set(nested)) - if duplicates: - raise ValueError( - f"step {step.id} repeats plugin options in flat and nested forms: " - + ", ".join(duplicates) - ) - result.update(nested) - return result - - -def _executable_argv(base: Path, specification: ExecutableSpec) -> tuple[str, ...]: - return (*specification.argv_prefix, _resolve_command(base, specification.path)) - - def _build_job( manifest: WorkflowManifest, manifest_base: Path, @@ -488,44 +368,41 @@ def _build_job( step: WorkflowStep, directory: Path, ) -> JobSpec: - executable = manifest.executables[step.executable] - command = _executable_argv(manifest_base, executable) - dependencies = tuple(f"f{frame.index:06d}.{identifier}" for identifier in step.depends_on) + specification: ExecutableSpec = manifest.executables[step.executable] + command = ( + *specification.argv_prefix, + _resolve_command(manifest_base, specification.path), + ) job_id = f"f{frame.index:06d}.{step.id}" + dependencies = tuple(f"f{frame.index:06d}.{item}" for item in step.depends_on) if step.kind == "vasp": - raw_arguments = step.options.get("arguments", []) - if not isinstance(raw_arguments, list) or any( - not isinstance(token, str) or not token for token in raw_arguments + arguments = step.options.get("arguments", []) + if not isinstance(arguments, list) or any( + not isinstance(item, str) or not item for item in arguments ): - raise TypeError(f"step {step.id}.options.arguments must be a list of strings") - expected = tuple(directory / name for name in _step_outputs(step)) - required_names = _string_list( - step.options.get("required_inputs"), f"step {step.id}.options.required_inputs" - ) or ("INCAR", "KPOINTS", "POSCAR", "POTCAR") + raise TypeError(f"step {step.id}.options.arguments must be strings") + required = _vasp_required_inputs(step) return JobSpec( id=job_id, frame_index=frame.index, frame_label=frame.label, task=step.id, cwd=directory, - argv=(*command, *cast(list[str], raw_arguments)), + argv=(*command, *cast(list[str], arguments)), stdin=None, stdout_path=directory / "vasp.stdout.log", - required_inputs=tuple(directory / name for name in required_names), - expected_outputs=expected, + required_inputs=tuple(directory / name for name in required), + expected_outputs=tuple(directory / name for name in _step_outputs(step)), dependencies=dependencies, environment=_environment(step), ) - - plugin_options = _plugin_options(step) planned = plan_plugin( step.kind, directory, capability=step.capability, configuration={"executable": command[-1]}, - **plugin_options, + **_plugin_options(step), ) - stdout = planned.stdout_path or directory / f"{step.kind}.stdout.log" return JobSpec( id=job_id, frame_index=frame.index, @@ -534,7 +411,7 @@ def _build_job( cwd=directory, argv=(*command[:-1], *planned.argv), stdin=planned.stdin, - stdout_path=stdout, + stdout_path=planned.stdout_path or directory / f"{step.kind}.stdout.log", required_inputs=planned.required_inputs, expected_outputs=planned.expected_outputs, dependencies=dependencies, @@ -543,34 +420,16 @@ def _build_job( def _workspace_paths(manifest_path: Path, manifest: WorkflowManifest) -> tuple[Path, Path]: - base = manifest_path.parent - # Canonicalize the read-only source boundary before comparing it with the - # lexical workspace. Otherwise a manifest can name the source through a - # symlink while placing the workspace inside that symlink's real target. - source_root = _resolve_from(base, manifest.project.root).resolve() - workspace = _resolve_from(base, manifest.project.workspace) - if ( - workspace == source_root - or source_root in workspace.parents - or workspace in source_root.parents - ): - raise ValueError( - "project.workspace must be outside project.root and must not contain it so " - "source calculations remain read-only" - ) - if workspace == Path(workspace.anchor) or workspace == Path.home(): - raise ValueError(f"refusing unsafe workflow workspace: {workspace}") - if symlink_component(workspace.parent) is not None: - raise ValueError( - f"workflow workspace parent must not traverse a symlink: {workspace.parent}" - ) - return source_root, workspace + source = _resolve_from(manifest_path.parent, manifest.project.root).resolve() + workspace = _resolve_from(manifest_path.parent, manifest.project.workspace) + if workspace == source or source in workspace.parents or workspace in source.parents: + raise ValueError("project.workspace and project.root must not overlap") + if workspace.is_symlink() or symlink_component(workspace.parent) is not None: + raise ValueError(f"workflow workspace must not traverse a symlink: {workspace}") + return source, workspace -def _selected_sources( - manifest: WorkflowManifest, - source_root: Path, -) -> tuple[FrameSource, ...]: +def _selected_sources(manifest: WorkflowManifest, source_root: Path) -> tuple[FrameSource, ...]: discovered = discover_path( source_root, mode=manifest.project.series_kind, @@ -581,19 +440,16 @@ def _selected_sources( return resolve_selected_frames(discovered.frames, manifest.frames) -def _staged_frames( - selected: tuple[FrameSource, ...], - workspace: Path, -) -> tuple[StagedFrame, ...]: +def _staged_frames(selected: tuple[FrameSource, ...], workspace: Path) -> tuple[StagedFrame, ...]: return tuple( StagedFrame( - index=source.index, - label=source.label, - analysis_label=_safe_label(order, source.label), - source_directory=source.directory, - structure=source.structure, - workspace_directory=workspace / "frames" / _safe_label(source.index, source.label), - energy_ev=source.energy_ev, + source.index, + source.label, + _safe_label(order, source.label), + source.directory, + source.structure, + workspace / "frames" / _safe_label(source.index, source.label), + source.energy_ev, ) for order, source in enumerate(selected) ) @@ -601,17 +457,11 @@ def _staged_frames( def _workspace_is_managed(path: Path) -> bool: marker = path / WORKSPACE_RECORD - if marker.is_symlink() or not marker.is_file(): + if path.is_symlink() or marker.is_symlink() or not marker.is_file(): return False try: - payload = json.loads( - read_limited_text( - marker, - limit=MAX_WORKSPACE_RECORD_BYTES, - context="staged workflow record", - ) - ) - except (OSError, RecursionError, ValueError, json.JSONDecodeError): + payload = json.loads(marker.read_text(encoding="utf-8")) + except (OSError, ValueError, json.JSONDecodeError): return False return ( isinstance(payload, dict) @@ -622,14 +472,8 @@ def _workspace_is_managed(path: Path) -> bool: def _rebase_job(job: JobSpec, source_root: Path, destination_root: Path) -> JobSpec: - """Move workspace-owned paths in a staged job from a temporary to its final root.""" - def rebase(path: Path) -> Path: - try: - relative = path.relative_to(source_root) - except ValueError as exc: # pragma: no cover - internal planner invariant - raise RuntimeError(f"staged job path escapes temporary workspace: {path}") from exc - return destination_root / relative + return destination_root / path.relative_to(source_root) return replace( job, @@ -640,21 +484,12 @@ def rebase(path: Path) -> Path: ) -def _publish_workspace(temporary: Path, workspace: Path, *, overwrite: bool) -> None: - """Publish a complete workspace, rolling back a managed replacement on failure.""" - - if not (workspace.exists() or workspace.is_symlink()): +def _publish_workspace(temporary: Path, workspace: Path, overwrite: bool) -> None: + if not workspace.exists(): temporary.replace(workspace) return - if not overwrite: - raise FileExistsError( - f"workflow workspace already exists: {workspace}; use --force to replace it" - ) - if workspace.is_symlink() or not _workspace_is_managed(workspace): - raise ValueError( - f"refusing to replace unmarked or symlinked workflow workspace: {workspace}" - ) - + if not overwrite or not _workspace_is_managed(workspace): + raise ValueError(f"refusing to replace unmarked workflow workspace: {workspace}") backup = Path(tempfile.mkdtemp(prefix=f".{workspace.name}.previous-", dir=workspace.parent)) backup.rmdir() workspace.replace(backup) @@ -672,202 +507,114 @@ def _manifest_jobs( workspace: Path, frames: tuple[StagedFrame, ...], ) -> tuple[tuple[JobSpec, ...], tuple[str, ...]]: - """Rebuild the executable plan instead of trusting serialized commands.""" - - if not frames: - raise ValueError("staged workflow contains no frames") - if len({frame.index for frame in frames}) != len(frames): - raise ValueError("staged workflow contains duplicate frame indices") - - jobs: list[JobSpec] = [] - descriptor_ids: list[str] = [] - for order, frame in enumerate(frames): - if frame.analysis_label != _safe_label(order, frame.label): - raise ValueError( - f"staged frame {frame.index} has an invalid or out-of-order analysis label" - ) - frame_directory = workspace / "frames" / _safe_label(frame.index, frame.label) - if frame.workspace_directory != frame_directory: - raise ValueError( - f"staged frame {frame.index} workspace directory does not match its label" - ) - for step in manifest.steps: - job = _build_job( - manifest, - manifest_path.parent, - frame, - step, - frame_directory / step.id, - ) - jobs.append(job) - if step.kind != "vasp": - descriptor_ids.append(job.id) - return validate_jobs(jobs), tuple(descriptor_ids) - - -def stage_workflow( - manifest_path: str | Path, - *, - overwrite: bool = False, -) -> StagedWorkflow: - """Stage selected frames and return a validated executable job DAG. + jobs = tuple( + _build_job( + manifest, + manifest_path.parent, + frame, + step, + frame.workspace_directory / step.id, + ) + for frame in frames + for step in manifest.steps + ) + descriptors = tuple( + job.id + for job in jobs + if next(step for step in manifest.steps if step.id == job.task).kind != "vasp" + ) + return validate_jobs(jobs), descriptors - ``overwrite=True`` may replace only a workspace carrying ElecTrace's own - marker. An unrelated nonempty directory is never removed. - """ +def stage_workflow(manifest_path: str | Path, *, overwrite: bool = False) -> StagedWorkflow: source_manifest = Path(manifest_path).expanduser().resolve() - manifest_sha256 = sha256_file(source_manifest) manifest = load_workflow_manifest(source_manifest) source_root, workspace = _workspace_paths(source_manifest, manifest) + manifest_hash = sha256_file(source_manifest) if not source_root.is_dir(): raise FileNotFoundError(f"workflow project root is not a directory: {source_root}") - if workspace.exists() or workspace.is_symlink(): - if not overwrite: - raise FileExistsError( - f"workflow workspace already exists: {workspace}; use --force to replace it" - ) - if workspace.is_symlink() or not _workspace_is_managed(workspace): - raise ValueError( - f"refusing to replace unmarked or symlinked workflow workspace: {workspace}" - ) - + if workspace.exists() and (not overwrite or not _workspace_is_managed(workspace)): + raise ValueError(f"refusing existing or unmarked workflow workspace: {workspace}") selected = _selected_sources(manifest, source_root) + frames = _staged_frames(selected, workspace) + templates = {step.id: _template_files(step, source_manifest.parent) for step in manifest.steps} workspace.parent.mkdir(parents=True, exist_ok=True) temporary = Path(tempfile.mkdtemp(prefix=f".{workspace.name}.stage-", dir=workspace.parent)) - staged_frames = _staged_frames(selected, workspace) jobs: list[JobSpec] = [] - descriptor_job_ids: list[str] = [] + descriptor_ids: list[str] = [] try: - for source, staged in zip(selected, staged_frames, strict=True): - frame_directory = temporary / "frames" / _safe_label(source.index, source.label) - prior_steps: dict[str, WorkflowStep] = {} + for source, frame in zip(selected, frames, strict=True): directories: dict[str, Path] = {} + prior: dict[str, WorkflowStep] = {} for step in manifest.steps: - directories[step.id] = frame_directory / step.id + directory = temporary / "frames" / _safe_label(source.index, source.label) / step.id + directories[step.id] = directory _prepare_step_directory( - step, - source, - directories[step.id], - source_manifest.parent, - prior_steps, - directories, + step, source, directory, templates[step.id], prior, directories ) - temporary_job = _build_job( - manifest, - source_manifest.parent, - staged, - step, - directories[step.id], + job = _rebase_job( + _build_job(manifest, source_manifest.parent, frame, step, directory), + temporary, + workspace, ) - job = _rebase_job(temporary_job, temporary, workspace) jobs.append(job) if step.kind != "vasp": - descriptor_job_ids.append(job.id) - prior_steps[step.id] = step - ordered_jobs = validate_jobs(jobs) + descriptor_ids.append(job.id) + prior[step.id] = step result = StagedWorkflow( - manifest_path=source_manifest, - manifest_sha256=manifest_sha256, - source_root=source_root, - workspace=workspace, - frames=staged_frames, - jobs=ordered_jobs, - descriptor_job_ids=tuple(descriptor_job_ids), + source_manifest, + manifest_hash, + source_root, + workspace, + frames, + validate_jobs(jobs), + tuple(descriptor_ids), + ) + atomic_write_json( + temporary / WORKSPACE_RECORD, + { + "schema_version": WORKSPACE_SCHEMA_VERSION, + "manifest_path": str(source_manifest), + "manifest_sha256": manifest_hash, + "source_root": str(source_root), + "workspace": str(workspace), + }, ) - atomic_write_json(temporary / WORKSPACE_RECORD, result.to_dict()) - if sha256_file(source_manifest) != manifest_sha256: - raise ValueError("workflow manifest changed while staging; run workflow stage again") - _publish_workspace(temporary, workspace, overwrite=overwrite) + if sha256_file(source_manifest) != manifest_hash: + raise ValueError("workflow manifest changed while staging") + _publish_workspace(temporary, workspace, overwrite) except BaseException: - if temporary.exists() and not temporary.is_symlink(): - shutil.rmtree(temporary, ignore_errors=True) + shutil.rmtree(temporary, ignore_errors=True) raise return result def load_staged_workflow(manifest_path: str | Path) -> StagedWorkflow: - """Load a staged plan and reject changes to its source manifest.""" - source_manifest = Path(manifest_path).expanduser().resolve() manifest = load_workflow_manifest(source_manifest) source_root, workspace = _workspace_paths(source_manifest, manifest) - record = workspace / WORKSPACE_RECORD - if record.is_symlink() or not record.is_file(): - raise FileNotFoundError(f"workflow is not staged: {record}") - try: - payload = json.loads( - read_limited_text( - record, - limit=MAX_WORKSPACE_RECORD_BYTES, - context="staged workflow record", - ) - ) - except (RecursionError, json.JSONDecodeError) as exc: - raise ValueError(f"invalid staged workflow record: {record}") from exc - if not isinstance(payload, dict) or payload.get("schema_version") != WORKSPACE_SCHEMA_VERSION: - raise ValueError(f"unsupported staged workflow record: {record}") - data = cast(dict[str, object], payload) - recorded_hash = _record_string(data, "manifest_sha256", "staged workflow") - current_hash = sha256_file(source_manifest) - if recorded_hash != current_hash: - raise ValueError("workflow manifest changed after staging; run workflow stage again") - if Path(_record_string(data, "manifest_path", "staged workflow")) != source_manifest: - raise ValueError("staged manifest path does not match the requested workflow manifest") - if Path(_record_string(data, "source_root", "staged workflow")) != source_root: - raise ValueError("staged source root does not match the workflow manifest") - if Path(_record_string(data, "workspace", "staged workflow")) != workspace: - raise ValueError("staged workspace does not match the workflow manifest") - recorded_frames = tuple( - StagedFrame.from_dict(item) for item in _record_list(data, "frames", "staged workflow") + marker = workspace / WORKSPACE_RECORD + if not _workspace_is_managed(workspace): + raise FileNotFoundError(f"workflow is not staged: {marker}") + payload = json.loads(marker.read_text(encoding="utf-8")) + manifest_hash = sha256_file(source_manifest) + expected = (str(source_manifest), manifest_hash, str(source_root), str(workspace)) + recorded = tuple( + payload.get(key) for key in ("manifest_path", "manifest_sha256", "source_root", "workspace") ) + if recorded != expected: + raise ValueError("staged workflow no longer matches its manifest") frames = _staged_frames(_selected_sources(manifest, source_root), workspace) - if recorded_frames != frames: - raise ValueError( - "staged frame provenance does not match current manifest-derived discovery; " - "run workflow stage again" - ) - jobs, descriptor_ids = _manifest_jobs( - manifest, - source_manifest, - workspace, - frames, - ) - if _record_list(data, "jobs", "staged workflow") != [job.to_dict() for job in jobs]: - raise ValueError( - "staged job record does not match the manifest-derived plan or escapes its " - "managed root; run workflow stage again" - ) - if _record_string_list(data, "descriptor_job_ids", "staged workflow") != descriptor_ids: - raise ValueError("staged descriptor job ids do not match the workflow manifest") + jobs, descriptor_ids = _manifest_jobs(manifest, source_manifest, workspace, frames) return StagedWorkflow( - manifest_path=source_manifest, - manifest_sha256=current_hash, - source_root=source_root, - workspace=workspace, - frames=frames, - jobs=jobs, - descriptor_job_ids=descriptor_ids, + source_manifest, manifest_hash, source_root, workspace, frames, jobs, descriptor_ids ) def descriptor_outputs(staged: StagedWorkflow) -> dict[str, tuple[Path, ...]]: - """Return parser-supported outputs for each staged descriptor job.""" - selected = set(staged.descriptor_job_ids) return { job.id: tuple(path for path in job.expected_outputs if path.name in _DESCRIPTOR_FILENAMES) for job in staged.jobs if job.id in selected } - - -__all__ = [ - "WORKSPACE_RECORD", - "WORKSPACE_SCHEMA_VERSION", - "StagedFrame", - "StagedWorkflow", - "descriptor_outputs", - "load_staged_workflow", - "stage_workflow", -] diff --git a/src/electrace/workflow/validation.py b/src/electrace/workflow/validation.py index 6f29c07..17faf6a 100644 --- a/src/electrace/workflow/validation.py +++ b/src/electrace/workflow/validation.py @@ -1,9 +1,4 @@ -"""Read-only validation and planning for ElecTrace workflows. - -Validation resolves the same frames, commands, and dependency graph as staging, -but represents staged files virtually. It never creates a workspace, copies a -template, or executes an external command. -""" +"""Read-only workflow planning.""" from __future__ import annotations @@ -11,14 +6,12 @@ from pathlib import Path from typing import Any -from electrace.io.discovery import FrameSource from electrace.plugins import resolve_executable from . import staging as _staging -from ._files import symlink_component from .execution import JobSpec, validate_jobs from .manifest import load_workflow_manifest -from .models import SchedulerSpec, WorkflowManifest, WorkflowStep +from .models import SchedulerSpec, WorkflowManifest from .staging import StagedFrame WORKFLOW_VALIDATION_SCHEMA_VERSION = "electrace.workflow-validation.v1" @@ -26,7 +19,7 @@ @dataclass(frozen=True, slots=True) class WorkflowValidation: - """A deterministic, JSON-ready workflow plan produced without filesystem writes.""" + """The resolved job plan for one workflow manifest.""" manifest: Path workspace: Path @@ -34,37 +27,10 @@ class WorkflowValidation: resolved_jobs: tuple[JobSpec, ...] scheduler: SchedulerSpec warnings: tuple[str, ...] = () - schema_version: str = WORKFLOW_VALIDATION_SCHEMA_VERSION - - def __post_init__(self) -> None: - if self.schema_version != WORKFLOW_VALIDATION_SCHEMA_VERSION: - raise ValueError( - f"workflow validation schema_version must be {WORKFLOW_VALIDATION_SCHEMA_VERSION!r}" - ) - for name in ("manifest", "workspace"): - path = getattr(self, name) - if not isinstance(path, Path) or not path.is_absolute(): - raise ValueError(f"workflow validation {name} must be an absolute Path") - if type(self.selected_frames) is not tuple or any( - not isinstance(frame, StagedFrame) for frame in self.selected_frames - ): - raise TypeError("selected_frames must be a tuple of StagedFrame records") - if type(self.resolved_jobs) is not tuple or any( - not isinstance(job, JobSpec) for job in self.resolved_jobs - ): - raise TypeError("resolved_jobs must be a tuple of JobSpec records") - if not isinstance(self.scheduler, SchedulerSpec): - raise TypeError("scheduler must be a SchedulerSpec") - if type(self.warnings) is not tuple or any( - not isinstance(warning, str) or not warning for warning in self.warnings - ): - raise TypeError("warnings must be a tuple of non-empty strings") def to_dict(self) -> dict[str, Any]: - """Return the stable public JSON representation of this validation plan.""" - return { - "schema_version": self.schema_version, + "schema_version": WORKFLOW_VALIDATION_SCHEMA_VERSION, "manifest": str(self.manifest), "workspace": str(self.workspace), "selected_frames": [frame.to_dict() for frame in self.selected_frames], @@ -74,280 +40,6 @@ def to_dict(self) -> dict[str, Any]: } -@dataclass(frozen=True, slots=True) -class _VirtualFile: - """One file that staging can provide before a job is launched. - - ``source`` names a regular file that exists while validation runs. A null - source represents an output promised by an earlier dependency; the staged - link is initially dangling but becomes available after that job succeeds. - """ - - source: Path | None - is_symlink: bool = False - - -def _template_files(manifest_base: Path, step: WorkflowStep) -> dict[str, _VirtualFile]: - if step.template_dir is None: - return {} - template = _staging._resolve_from(manifest_base, step.template_dir) - if not template.is_dir(): - raise FileNotFoundError(f"workflow template directory does not exist: {template}") - if symlink_component(template) is not None: - raise ValueError(f"workflow template directory must not traverse a symlink: {template}") - - files: dict[str, _VirtualFile] = {} - for item in sorted(template.iterdir(), key=lambda path: path.name): - if item.is_symlink(): - raise ValueError(f"workflow templates must not contain symlinks: {item}") - if not item.is_file(): - raise ValueError(f"workflow templates must contain only regular files: {item}") - # Staging copies the complete file. Stream it here as well so the - # read-only preflight cannot approve a template that staging cannot - # actually read, without retaining large inputs such as POTCAR in RAM. - with item.open("rb") as handle: - while handle.read(1024 * 1024): - pass - name = _staging._basename(item.name, "template filename") - files[name] = _VirtualFile(item) - return files - - -def _validate_incar_overrides(step: WorkflowStep, files: dict[str, _VirtualFile]) -> None: - raw = step.options.get("incar_overrides") - if raw is None: - return - if not isinstance(raw, dict): - raise TypeError("vasp options.incar_overrides must be a mapping") - incar = files.get("INCAR") - if incar is None or incar.source is None: - raise FileNotFoundError("vasp options.incar_overrides requires an INCAR template") - # Staging decodes INCAR before applying overrides. Read it here as well so - # validation cannot approve a template that staging would reject. - incar.source.read_text(encoding="utf-8") - for raw_key, value in raw.items(): - if _staging._INCAR_KEY.fullmatch(raw_key) is None: - raise ValueError(f"invalid INCAR tag in workflow: {raw_key!r}") - if isinstance(value, bool): - continue - if not isinstance(value, (str, int, float)) or isinstance(value, bool): - raise TypeError(f"INCAR override {raw_key!r} must be a scalar") - - -def _dependency_file( - name: str, - step: WorkflowStep, - files_by_step: dict[str, dict[str, _VirtualFile]], - outputs_by_step: dict[str, frozenset[str]], -) -> _VirtualFile | None: - """Mirror staging's reverse dependency lookup without touching a workspace.""" - - for dependency in reversed(step.depends_on): - existing = files_by_step[dependency].get(name) - if existing is not None and existing.source is not None: - return existing - if name in outputs_by_step[dependency]: - return _VirtualFile(None) - return None - - -def _apply_explicit_input_links( - step: WorkflowStep, - source_structure: Path, - files: dict[str, _VirtualFile], - files_by_step: dict[str, dict[str, _VirtualFile]], - outputs_by_step: dict[str, frozenset[str]], -) -> None: - raw = step.options.get("input_links") - if raw is None: - return - if not isinstance(raw, dict): - raise TypeError(f"step {step.id}.options.input_links must be a mapping") - for raw_target, specification in raw.items(): - target = _staging._basename(raw_target, f"step {step.id}.options.input_links key") - if specification == "source:structure": - files[target] = _VirtualFile(source_structure) - continue - if not isinstance(specification, dict): - raise TypeError( - f"step {step.id}.options.input_links.{target} must be an object or source:structure" - ) - dependency = specification.get("step") - filename = specification.get("file", target) - if not isinstance(dependency, str) or dependency not in step.depends_on: - raise ValueError( - f"step {step.id}.options.input_links.{target}.step must be a declared dependency" - ) - source_name = _staging._basename( - filename, f"step {step.id}.options.input_links.{target}.file" - ) - if target in files: - raise FileExistsError(f"workflow input target already exists: {target}") - available = files_by_step[dependency].get(source_name) - if available is None and source_name in outputs_by_step[dependency]: - available = _VirtualFile(None) - if available is None: - raise FileNotFoundError( - f"step {step.id!r} input link {target!r} cannot be provided by " - f"dependency {dependency!r} as {source_name!r}" - ) - files[target] = _VirtualFile(available.source, is_symlink=True) - - -def _apply_automatic_plugin_inputs( - step: WorkflowStep, - source_structure: Path, - files: dict[str, _VirtualFile], - files_by_step: dict[str, dict[str, _VirtualFile]], - outputs_by_step: dict[str, frozenset[str]], -) -> None: - options = _staging._plugin_options(step) - structure_name = str(options.get("structure_file", "CONTCAR")) - for raw_name in _staging._plugin_required_names(step): - name = _staging._basename(raw_name, f"step {step.id} plugin input") - if name in files: - continue - if step.kind in {"critic2", "lobster"} and name == structure_name: - files[name] = _VirtualFile(source_structure) - continue - available = _dependency_file(name, step, files_by_step, outputs_by_step) - if available is not None: - files[name] = _VirtualFile(available.source, is_symlink=True) - - -def _validate_lobster_input(step: WorkflowStep, files: dict[str, _VirtualFile]) -> None: - options = _staging._plugin_options(step) - input_name = _staging._basename( - options.get("input_file", "lobsterin"), f"step {step.id} lobster input_file" - ) - entry = files.get(input_name) - if entry is None or entry.source is None or entry.is_symlink or not entry.source.is_file(): - raise FileNotFoundError(f"LOBSTER workflow requires a regular reviewed {input_name}") - text = entry.source.read_text(encoding="utf-8", errors="replace") - if _staging._LOBSTER_BOND_REQUEST.search(text) is None: - raise ValueError( - f"LOBSTER step {step.id!r} must request bonds with cohpGenerator or cohpBetween" - ) - if step.capability == "cobi" and _staging._LOBSTER_SKIP_COBI.search(text) is not None: - raise ValueError(f"LOBSTER COBI step {step.id!r} cannot use skipcobi") - - -def _validate_job_inputs_and_collisions( - step: WorkflowStep, - job: JobSpec, - files: dict[str, _VirtualFile], -) -> None: - missing = [path.name for path in job.required_inputs if path.name not in files] - if missing: - names = ", ".join(sorted(missing)) - raise FileNotFoundError(f"step {step.id!r} cannot provide required staged inputs: {names}") - - required = set(job.required_inputs) - captures = {job.stdout_path, job.stderr_path} - products = {job.stdout_path, job.stderr_path, *job.expected_outputs} - # VASP restart inputs such as CHGCAR and WAVECAR are intentionally updated - # in place. Descriptor templates, by contrast, must never pre-populate a - # result that collection could mistake for newly generated evidence. - protected_products = captures if step.kind == "vasp" else products - overlap = sorted(required & protected_products, key=str) - if overlap: - raise ValueError( - f"step {step.id!r} declares paths as both input and output: " - + ", ".join(str(path) for path in overlap) - ) - - staged_names = set(files) - product_names = {path.name for path in protected_products} - collisions = sorted(staged_names & product_names) - if collisions: - raise ValueError( - f"step {step.id!r} stages files that collide with declared outputs: " - + ", ".join(collisions) - ) - - -def _plan_jobs( - manifest: WorkflowManifest, - manifest_path: Path, - workspace: Path, - selected: tuple[FrameSource, ...], - frames: tuple[StagedFrame, ...], -) -> tuple[JobSpec, ...]: - templates = {step.id: _template_files(manifest_path.parent, step) for step in manifest.steps} - jobs: list[JobSpec] = [] - for source, frame in zip(selected, frames, strict=True): - files_by_step: dict[str, dict[str, _VirtualFile]] = {} - outputs_by_step: dict[str, frozenset[str]] = {} - for step in manifest.steps: - files = dict(templates[step.id]) - if step.kind == "vasp": - files["POSCAR"] = _VirtualFile(source.structure) - _validate_incar_overrides(step, files) - else: - _apply_explicit_input_links( - step, - source.structure, - files, - files_by_step, - outputs_by_step, - ) - _apply_automatic_plugin_inputs( - step, - source.structure, - files, - files_by_step, - outputs_by_step, - ) - if step.kind == "lobster": - _validate_lobster_input(step, files) - - frame_directory = ( - workspace / "frames" / _staging._safe_label(source.index, source.label) - ) - destination = frame_directory / step.id - source_job = _staging._build_job( - manifest, - manifest_path.parent, - frame, - step, - source.directory, - ) - job = _staging._rebase_job(source_job, source.directory, destination) - _validate_job_inputs_and_collisions(step, job, files) - jobs.append(job) - files_by_step[step.id] = files - outputs_by_step[step.id] = frozenset(_staging._step_outputs(step)) - ordered = validate_jobs(jobs) - _validate_descriptor_collisions(ordered, manifest, frames) - return ordered - - -def _validate_descriptor_collisions( - jobs: tuple[JobSpec, ...], - manifest: WorkflowManifest, - frames: tuple[StagedFrame, ...], -) -> None: - """Reject descriptor basenames that collection would flatten to one path.""" - - descriptor_tasks = {step.id for step in manifest.steps if step.kind != "vasp"} - labels = {frame.index: frame.analysis_label for frame in frames} - destinations: dict[tuple[int, str], str] = {} - for job in jobs: - if job.task not in descriptor_tasks: - continue - for output in job.expected_outputs: - if output.name not in _staging._DESCRIPTOR_FILENAMES: - continue - key = (job.frame_index, output.name) - previous = destinations.get(key) - if previous is not None: - destination = Path("descriptors") / labels[job.frame_index] / output.name - raise ValueError( - f"descriptor output collision at {destination}: {previous} and {job.id}" - ) - destinations[key] = job.id - - def _workspace_warnings(workspace: Path) -> tuple[str, ...]: if workspace.is_symlink(): raise ValueError(f"workflow workspace must not be a symlink: {workspace}") @@ -367,56 +59,50 @@ def _executable_warnings( manifest_base: Path, ) -> tuple[str, ...]: warnings: list[str] = [] - for name in sorted(manifest.executables): - specification = manifest.executables[name] + for name, specification in sorted(manifest.executables.items()): command = _staging._resolve_command(manifest_base, specification.path) if resolve_executable(command) is None: warnings.append(f"executable {name!r} is not available on this host: {command}") - if specification.argv_prefix: - wrapper = specification.argv_prefix[0] - if resolve_executable(wrapper) is None: - warnings.append( - f"argv_prefix command for executable {name!r} is not available " - f"on this host: {wrapper}" - ) + if specification.argv_prefix and resolve_executable(specification.argv_prefix[0]) is None: + warnings.append( + f"argv_prefix command for executable {name!r} is not available " + f"on this host: {specification.argv_prefix[0]}" + ) return tuple(warnings) def validate_workflow(manifest_path: str | Path) -> WorkflowValidation: - """Resolve and validate a workflow without creating or modifying any path. - - The function loads the strict manifest, discovers and selects source frames, - validates templates and virtual input links, resolves executable argument - vectors, and validates the complete job DAG and output ownership. Missing - host executables are warnings because Slurm modules or runtime environments - may provide them later; malformed plans and unavailable staged inputs fail. - """ + """Resolve frames and jobs without creating or modifying a workspace.""" source_manifest = Path(manifest_path).expanduser().resolve() manifest = load_workflow_manifest(source_manifest) source_root, workspace = _staging._workspace_paths(source_manifest, manifest) if not source_root.is_dir(): raise FileNotFoundError(f"workflow project root is not a directory: {source_root}") + for step in manifest.steps: + _staging._template_files(step, source_manifest.parent) - workspace_warnings = _workspace_warnings(workspace) selected = _staging._selected_sources(manifest, source_root) frames = _staging._staged_frames(selected, workspace) - jobs = _plan_jobs( - manifest, - source_manifest, - workspace, - selected, - frames, - ) + jobs: list[JobSpec] = [] + for source, frame in zip(selected, frames, strict=True): + frame_directory = workspace / "frames" / _staging._safe_label(source.index, source.label) + for step in manifest.steps: + source_job = _staging._build_job( + manifest, source_manifest.parent, frame, step, source.directory + ) + jobs.append( + _staging._rebase_job(source_job, source.directory, frame_directory / step.id) + ) warnings = ( - *workspace_warnings, + *_workspace_warnings(workspace), *_executable_warnings(manifest, source_manifest.parent), ) return WorkflowValidation( manifest=source_manifest, workspace=workspace, selected_frames=frames, - resolved_jobs=jobs, + resolved_jobs=validate_jobs(jobs), scheduler=manifest.scheduler, warnings=warnings, ) diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..870fb99 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,74 @@ +import pytest +import yaml + +POSCAR = """H\n1\n1 0 0\n0 1 0\n0 0 1\nH\n1\nDirect\n0 0 0\n""" + + +@pytest.fixture +def poscar_text(): + return POSCAR + + +@pytest.fixture +def workflow_manifest(tmp_path): + def write( + *, + base=None, + scheduler="local", + executables=None, + steps=None, + step_options=None, + workspace="workspace", + templates=None, + ): + base = base or tmp_path + for label in ("00", "01"): + frame = base / "images" / label + frame.mkdir(parents=True) + (frame / "POSCAR").write_text(POSCAR, encoding="utf-8") + for directory, files in (templates or {}).items(): + target = base / directory + target.mkdir(parents=True) + for name, content in files.items(): + (target / name).write_text(content, encoding="utf-8") + + programs = executables or {"vasp": "/bin/true"} + options = {"required_inputs": ["POSCAR"], "expected_outputs": ["CHGCAR"]} + options.update(step_options or {}) + payload = { + "schema_version": "electrace.workflow.v1", + "project": { + "root": "images", + "series_kind": "neb", + "workspace": str(workspace), + "structure": "CONTCAR", + "fallback": "POSCAR", + }, + "frames": "all", + "executables": { + name: {"path": value} if isinstance(value, str) else dict(value) + for name, value in programs.items() + }, + "scheduler": { + "kind": scheduler, + "directives": ( + {"partition": "cpu", "exclude": "node[1-8]"} if scheduler == "slurm" else {} + ), + }, + "steps": list( + steps + or [ + { + "id": "density", + "kind": "vasp", + "executable": "vasp", + "options": options, + } + ] + ), + } + path = base / "workflow.yaml" + path.write_text(yaml.safe_dump(payload, sort_keys=False), encoding="utf-8") + return path + + return write diff --git a/tests/test_benchmark.py b/tests/test_benchmark.py deleted file mode 100644 index cb43ef6..0000000 --- a/tests/test_benchmark.py +++ /dev/null @@ -1,126 +0,0 @@ -from __future__ import annotations - -import json -from pathlib import Path - -import pytest -from rich.text import Text -from typer.testing import CliRunner - -from electrace.benchmark import ( - BENCHMARK_SCHEMA_VERSION, - EventLabel, - _event_scores, - benchmark_passed, - run_synthetic_benchmark, - write_synthetic_benchmark, -) -from electrace.cli import app -from electrace.models import Event - - -def _plain(output: str) -> str: - return Text.from_ansi(output).plain - - -def test_synthetic_benchmark_is_deterministic_and_passes() -> None: - first = run_synthetic_benchmark() - second = run_synthetic_benchmark() - - assert json.dumps(first, allow_nan=False, sort_keys=True) == json.dumps( - second, allow_nan=False, sort_keys=True - ) - assert first["schema_version"] == BENCHMARK_SCHEMA_VERSION - assert first["suite"]["synthetic"] is True - assert first["suite"]["scientific_validation"] is False - assert first["summary"] == { - "cases": 15, - "passed": 15, - "failed": 0, - "event_true_positives": 10, - "event_false_positives": 0, - "event_false_negatives": 0, - "event_precision": 1.0, - "event_recall": 1.0, - "event_f1": 1.0, - "mapping_assertions_correct": 30, - "mapping_assertions_total": 30, - "mapping_accuracy": 1.0, - "ambiguity_assertions_correct": 30, - "ambiguity_assertions_total": 30, - "ambiguity_accuracy": 1.0, - } - assert benchmark_passed(first) - - -def test_negative_controls_and_ambiguity_are_explicitly_auditable() -> None: - report = run_synthetic_benchmark() - cases = {case["id"]: case for case in report["cases"]} - - for identifier, reason in ( - ("proximity-without-conservation", "missing_complete_conservation_channel"), - ("failed-conservation", "conservation_tolerance_exceeded"), - ): - case = cases[identifier] - assert case["passed"] is True - assert all(event["type"] != "split" for event in case["observed"]["events"]) - assert case["observed"]["hyperedge_rejections"][0]["reason"] == reason - - for identifier in ("exact-assignment-tie", "near-assignment-tie"): - case = cases[identifier] - assert case["observed"]["feature_ambiguous"] is True - assert case["observed"]["feature_ambiguity_count"] >= 1 - assert case["observed"]["events"] == [] - - resolved = cases["hyperedge-resolves-backbone-tie"] - assert resolved["observed"]["feature_ambiguous"] is False - assert [event["type"] for event in resolved["observed"]["events"]] == ["split"] - - -def test_event_scoring_counts_false_positive_and_false_negative() -> None: - expected = (EventLabel("birth", target_ids=("q",)),) - observed = ( - Event( - "death", - 0, - 1, - source_ids=("p",), - confidence=0.75, - ), - ) - - assert _event_scores(expected, observed) == (0, 1, 1) - - -def test_benchmark_writer_is_atomic_and_requires_explicit_overwrite(tmp_path: Path) -> None: - output = tmp_path / "benchmark.json" - target = tmp_path / "outside.json" - target.write_text("preserve me", encoding="utf-8") - output.symlink_to(target) - - with pytest.raises(FileExistsError, match="use --force"): - write_synthetic_benchmark(output) - - written = write_synthetic_benchmark(output, overwrite=True) - - assert written == output - assert not output.is_symlink() - assert target.read_text(encoding="utf-8") == "preserve me" - assert json.loads(output.read_text(encoding="utf-8"))["summary"]["failed"] == 0 - - -def test_benchmark_cli_writes_report_and_reports_existing_output(tmp_path: Path) -> None: - output = tmp_path / "benchmark.json" - runner = CliRunner() - - first = runner.invoke(app, ["benchmark", "--output", str(output)]) - assert first.exit_code == 0, first.output - assert "Synthetic benchmark passed: 15/15 cases" in _plain(first.stdout) - assert json.loads(output.read_text(encoding="utf-8"))["summary"]["event_recall"] == 1.0 - - refused = runner.invoke(app, ["benchmark", "--output", str(output)]) - assert refused.exit_code == 2 - assert "use --force" in _plain(refused.output) - - replaced = runner.invoke(app, ["benchmark", "--output", str(output), "--force"]) - assert replaced.exit_code == 0, replaced.output diff --git a/tests/test_cli.py b/tests/test_cli.py index 86149a2..1710115 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,7 +1,6 @@ from __future__ import annotations import json -import sys from pathlib import Path import pytest @@ -10,9 +9,7 @@ from electrace import __version__ from electrace.cli import _python_supported, app -from electrace.config import ElecTraceConfig, OutputConfig from electrace.demo import generate_demo -from electrace.export import ExportDependencyError runner = CliRunner() @@ -67,17 +64,14 @@ def test_demo_inspect_doctor_and_plan_workflow(tmp_path: Path) -> None: ) assert diagnosed.exit_code == 0, diagnosed.output diagnosis = json.loads(_plain(diagnosed.stdout)) - assert diagnosis["schema_version"] == "electrace.cli.v1" - assert diagnosis["ok"] is True - assert diagnosis["command"] == "doctor" + assert (diagnosis["schema_version"], diagnosis["ok"], diagnosis["command"]) == ( + "electrace.cli.v1", + True, + "doctor", + ) assert diagnosis["data"]["python_supported"] is True assert [frame["frame"]["label"] for frame in diagnosis["data"]["frames"]] == [ - "00", - "01", - "02", - "03", - "04", - "05", + f"{index:02d}" for index in range(6) ] planned = runner.invoke( @@ -98,35 +92,6 @@ def test_demo_inspect_doctor_and_plan_workflow(tmp_path: Path) -> None: assert planned_payload["data"]["plugin"] == "critic2" assert "ready" not in planned_payload["data"] - batch = runner.invoke( - app, - [ - "plan", - "critic2", - str(demo), - "--all-frames", - "--capability", - "qtaim_critical_points", - "--config", - str(demo / "electrace.toml"), - "--json", - ], - ) - assert batch.exit_code == 0, batch.output - batch_payload = json.loads(_plain(batch.stdout)) - assert [plan["frame"]["label"] for plan in batch_payload["data"]["plans"]] == [ - "00", - "01", - "02", - "03", - "04", - "05", - ] - assert all( - "CPREPORT critic2.features.json GRAPH" in plan["command"]["stdin"] - for plan in batch_payload["data"]["plans"] - ) - def test_init_refuses_overwrite_without_force(tmp_path: Path) -> None: images = tmp_path / "images" @@ -141,81 +106,7 @@ def test_init_refuses_overwrite_without_force(tmp_path: Path) -> None: assert "Refusing to replace" in _plain(second.output) -def test_workflow_init_json_reports_manifest_artifact(tmp_path: Path) -> None: - images = tmp_path / "images" - images.mkdir() - manifest = tmp_path / "workflow.yaml" - - result = runner.invoke( - app, - ["workflow", "init", str(manifest), "--images", str(images), "--json"], - ) - - assert result.exit_code == 0, result.output - payload = json.loads(result.stdout) - assert payload["command"] == "workflow.init" - assert payload["data"]["manifest"] == str(manifest.resolve()) - assert payload["artifacts"] == [{"path": str(manifest.resolve()), "role": "workflow_manifest"}] - - -def test_analyze_reports_missing_hdf5_dependency_without_traceback( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - config_path = tmp_path / "electrace.toml" - config_path.write_text( - "[outputs]\nhdf5 = true\n", - encoding="utf-8", - ) - config = ElecTraceConfig( - outputs=OutputConfig(html_report=False, json=False, csv=False, hdf5=True) - ) - analysis = {"schema_version": "0.1", "events": [], "tracks": []} - monkeypatch.setattr( - "electrace.pipeline.load_and_analyze", - lambda *_args, **_kwargs: (analysis, config), - ) - monkeypatch.setitem(sys.modules, "h5py", None) - - result = runner.invoke( - app, - [ - "analyze", - str(config_path), - "--output", - str(tmp_path / "results"), - ], - ) - - assert result.exit_code == 2 - assert "pip install electrace[hdf5]" in _plain(result.output) - assert "Traceback" not in result.output - - -def test_analyze_strict_rejects_ambiguity_and_permissive_reports_warning( - tmp_path: Path, -) -> None: - config_path = generate_demo(tmp_path / "demo") - descriptor_path = config_path.parent / "01" / "features.json" - descriptor = json.loads(descriptor_path.read_text(encoding="utf-8")) - duplicate = dict(descriptor["features"][0]) - duplicate["id"] = "ambiguous-duplicate" - descriptor["features"].append(duplicate) - descriptor_path.write_text(json.dumps(descriptor), encoding="utf-8") - - strict = runner.invoke(app, ["analyze", str(config_path)]) - - assert strict.exit_code == 2 - assert "Ambiguous correspondence in strict mode" in _plain(strict.output) - assert "--permissive" in _plain(strict.output) - assert not (config_path.parent / "electrace-results").exists() - - permissive = runner.invoke(app, ["analyze", str(config_path), "--permissive"]) - - assert permissive.exit_code == 0, permissive.output - assert "warnings" in _plain(permissive.output) - - -def test_analyze_json_reports_counts_and_written_artifacts(tmp_path: Path) -> None: +def test_analyze_json_reports_counts(tmp_path: Path) -> None: config_path = generate_demo(tmp_path / "demo") result = runner.invoke(app, ["analyze", str(config_path), "--json"]) @@ -225,60 +116,12 @@ def test_analyze_json_reports_counts_and_written_artifacts(tmp_path: Path) -> No assert payload["command"] == "analyze" assert payload["data"]["counts"]["frames"] == 6 assert payload["data"]["counts"]["events"] == 6 - assert any(item["role"] == "analysis_artifact" for item in payload["artifacts"]) - - -def test_demo_reports_export_dependency_without_traceback( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - def fail_export(*_args: object, **_kwargs: object) -> None: - raise ExportDependencyError("optional exporter unavailable") - - monkeypatch.setattr("electrace.cli.run_project", fail_export) - - result = runner.invoke( - app, - ["demo", "--output", str(tmp_path / "demo")], - ) - - assert result.exit_code == 2 - assert "optional exporter unavailable" in _plain(result.output) - assert "Traceback" not in result.output - - -def test_json_error_with_empty_exception_message_remains_valid( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - def fail_without_message(*_args: object, **_kwargs: object) -> None: - raise ValueError - - monkeypatch.setattr("electrace.cli.generate_demo", fail_without_message) - - result = runner.invoke( - app, - ["demo", "--output", str(tmp_path / "demo"), "--json"], - ) - - assert result.exit_code == 2 - assert result.stderr == "" - assert result.stdout.count("\n") == 1 - payload = json.loads(result.stdout) - assert payload["error"]["code"] == "invalid_input" - assert payload["error"]["message"] == "ValueError" def test_inspect_malformed_analysis_returns_json_error(tmp_path: Path) -> None: analysis = tmp_path / "analysis.json" analysis.write_text( - json.dumps( - { - "project": {}, - "frames": None, - "tracks": [], - "events": [], - "warnings": [], - } - ), + json.dumps({"project": {}, "frames": None, "tracks": [], "events": []}), encoding="utf-8", ) @@ -289,45 +132,7 @@ def test_inspect_malformed_analysis_returns_json_error(tmp_path: Path) -> None: payload = json.loads(result.stdout) assert payload["command"] == "inspect" assert payload["error"]["code"] == "invalid_input" - assert payload["error"]["message"] == "analysis frames must be an array" - - -def test_inspect_json_recursion_failure_returns_json_error( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - analysis = tmp_path / "analysis.json" - analysis.write_text("{}", encoding="utf-8") - original_loads = json.loads - - def fail_recursively(_document: str) -> object: - raise RecursionError("simulated JSON nesting limit") - - monkeypatch.setattr("electrace.cli.json.loads", fail_recursively) - - result = runner.invoke(app, ["inspect", str(analysis), "--json"]) - - assert result.exit_code == 2 - assert result.stderr == "" - payload = original_loads(result.stdout) - assert payload["command"] == "inspect" - assert payload["error"]["code"] == "invalid_input" - assert payload["error"]["message"].startswith(f"Cannot read {analysis}:") - - -def test_benchmark_json_reports_summary_and_artifact(tmp_path: Path) -> None: - report = tmp_path / "benchmark.json" - - result = runner.invoke( - app, - ["benchmark", "--output", str(report), "--json"], - ) - - assert result.exit_code == 0, result.output - payload = json.loads(result.stdout) - assert payload["command"] == "benchmark" - assert payload["data"]["passed"] is True - assert payload["data"]["summary"]["passed"] == payload["data"]["summary"]["cases"] - assert payload["artifacts"] == [{"path": str(report.resolve()), "role": "benchmark_report"}] + assert "NoneType" in payload["error"]["message"] def test_invalid_input_filename_config_is_a_clean_cli_error(tmp_path: Path) -> None: @@ -346,38 +151,6 @@ def test_invalid_input_filename_config_is_a_clean_cli_error(tmp_path: Path) -> N assert "Traceback" not in result.output -def test_quoted_boolean_config_is_a_clean_cli_error(tmp_path: Path) -> None: - config = tmp_path / "invalid-boolean.toml" - config.write_text( - """[outputs] -json = "false" -""", - encoding="utf-8", - ) - - result = runner.invoke(app, ["doctor", "--config", str(config)]) - - assert result.exit_code == 2 - assert "outputs.json must be a boolean" in _plain(result.output) - assert "Traceback" not in result.output - - -def test_misspelled_root_config_table_is_a_clean_cli_error(tmp_path: Path) -> None: - config = tmp_path / "typo.toml" - config.write_text( - """[matcing] -position_weight = 2.0 -""", - encoding="utf-8", - ) - - result = runner.invoke(app, ["doctor", "--config", str(config)]) - - assert result.exit_code == 2 - assert "configuration root contains unknown keys: matcing" in _plain(result.output) - assert "Traceback" not in result.output - - def test_configured_structure_name_cannot_inject_path_wide_critic2_input( tmp_path: Path, ) -> None: @@ -406,126 +179,11 @@ def test_configured_structure_name_cannot_inject_path_wide_critic2_input( assert "CPREPORT" not in result.output -def test_mixed_case_plugin_name_still_honors_normalized_configuration(tmp_path: Path) -> None: - frame = tmp_path / "frame" - frame.mkdir() - config = tmp_path / "electrace.toml" - config.write_text( - """[plugins.critic2] -enabled = false -executable = "must-not-be-ignored" -""", - encoding="utf-8", - ) - - result = runner.invoke( - app, - ["plan", "Critic2", str(frame), "--config", str(config)], - ) - - assert result.exit_code == 2 - assert "plugin 'critic2' is disabled" in _plain(result.output) - - -def test_single_frame_plan_honors_configured_structure_name(tmp_path: Path) -> None: - frame = tmp_path / "frame" - frame.mkdir() - structure = frame / "FINAL.vasp" - structure.write_text("planned input only\n", encoding="utf-8") - config = tmp_path / "electrace.toml" - config.write_text( - """[inputs] -structure_name = "FINAL.vasp" -""", - encoding="utf-8", - ) - - result = runner.invoke( - app, - ["plan", "critic2", str(frame), "--config", str(config)], - ) - - assert result.exit_code == 0, result.output - payload = json.loads(_plain(result.stdout)) - assert payload["required_inputs"][0] == str(structure.resolve()) - assert "CRYSTAL FINAL.vasp" in payload["stdin"] - - -@pytest.mark.parametrize( - "arguments", - [ - ["doctor", "--json"], - ["demo", "--help"], - ["benchmark", "--help"], - ["analyze", "--help"], - ["inspect", "--help"], - ["plan", "--help"], - ["volume", "--help"], - ["init", "--help"], - ["workflow", "init", "--help"], - ["workflow", "validate", "--help"], - ["workflow", "stage", "--help"], - ["workflow", "run", "--help"], - ["workflow", "submit", "--help"], - ["workflow", "status", "--help"], - ["workflow", "collect", "--help"], - ["workflow", "analyze", "--help"], - ], -) -def test_every_command_supports_json_or_emits_the_json_contract( - arguments: list[str], -) -> None: - result = runner.invoke(app, arguments) - - assert result.exit_code == 0, result.output - if "--help" in arguments: - assert "--json" in _plain(result.stdout) - else: - assert json.loads(result.stdout)["schema_version"] == "electrace.cli.v1" - - -@pytest.mark.parametrize( - ("arguments", "command", "error_code"), - [ - (["workflow", "validate", "--json"], "workflow.validate", "missing_parameter"), - ( - ["workflow", "validate", "missing.yaml", "--json"], - "workflow.validate", - "not_found", - ), - (["doctor", "--unknown", "--json"], "doctor", "invalid_option"), - (["doctor", "--json=1"], "doctor", "invalid_input"), - ( - ["workflow", "run", "missing.yaml", "--max-parallel", "0", "--json"], - "workflow.run", - "invalid_input", - ), - ], -) -def test_parse_time_json_errors_are_single_unstyled_documents( - arguments: list[str], - command: str, - error_code: str, -) -> None: - result = runner.invoke(app, arguments) +def test_parse_errors_use_normal_cli_output() -> None: + result = runner.invoke(app, ["doctor", "--unknown", "--json"]) assert result.exit_code == 2 - assert result.stderr == "" - assert "\x1b" not in result.stdout - assert result.stdout.count("\n") == 1 - payload = json.loads(result.stdout) - assert payload == { - "artifacts": [], - "command": command, - "error": { - "code": error_code, - "details": None, - "message": payload["error"]["message"], - }, - "ok": False, - "schema_version": "electrace.cli.v1", - "warnings": [], - } + assert "No such option" in _plain(result.output) def test_workflow_validation_failure_is_a_machine_readable_error(tmp_path: Path) -> None: diff --git a/tests/test_cli_output.py b/tests/test_cli_output.py index 342736f..c2a7b49 100644 --- a/tests/test_cli_output.py +++ b/tests/test_cli_output.py @@ -1,167 +1,37 @@ from __future__ import annotations import json -from collections.abc import Callable -from pathlib import Path import pytest -from electrace.cli_output import ( - CLI_SCHEMA_VERSION, - failure, - normalize_error_code, - serialize, - success, -) +from electrace.cli_output import CLI_SCHEMA_VERSION, failure, serialize, success -def test_success_builds_a_versioned_json_safe_envelope() -> None: - envelope = success( - "workflow.validate", - {"frames": (0, 2), "valid": True}, - warnings=("optional tool unavailable",), - artifacts=({"kind": "manifest", "path": "workflow.yaml"},), - ) - - assert envelope == { +def test_success_and_failure_use_one_small_envelope() -> None: + assert success("inspect", {"frames": 3}, warnings=("partial",)) == { "schema_version": CLI_SCHEMA_VERSION, "ok": True, - "command": "workflow.validate", - "data": {"frames": [0, 2], "valid": True}, - "warnings": ["optional tool unavailable"], - "artifacts": [{"kind": "manifest", "path": "workflow.yaml"}], + "command": "inspect", + "data": {"frames": 3}, + "warnings": ["partial"], } - - -def test_failure_builds_a_stable_error_envelope() -> None: - envelope = failure( - "workflow.validate", - FileNotFoundError("missing.yaml"), - "Manifest was not found", - details={"path": "missing.yaml"}, - warnings=("No workflow was inspected",), - ) - - assert envelope == { + assert failure("inspect", "invalid_input", "bad input") == { "schema_version": CLI_SCHEMA_VERSION, "ok": False, - "command": "workflow.validate", + "command": "inspect", "error": { - "code": "not_found", - "message": "Manifest was not found", - "details": {"path": "missing.yaml"}, + "code": "invalid_input", + "message": "bad input", + "details": None, }, - "warnings": ["No workflow was inspected"], - "artifacts": [], + "warnings": [], } -def test_serializer_is_compact_deterministic_and_single_document() -> None: - first = success("inspect", {"z": 1, "a": "ELF Δ"}) - second = success("inspect", {"a": "ELF Δ", "z": 1}) - - serialized = serialize(first) - - assert serialized == serialize(second) - assert serialized.endswith("\n") - assert serialized.count("\n") == 1 - assert ": " not in serialized - assert ", " not in serialized - assert json.loads(serialized) == first - assert "\\u0394" in serialized - - -def test_serializer_revalidates_an_envelope_modified_after_construction() -> None: - envelope = success("inspect", {"energy": 0.0}) - envelope["data"] = {"energy": float("nan")} - - with pytest.raises(ValueError, match="non-finite"): - serialize(envelope) - - -@pytest.mark.parametrize( - ("value", "code"), - [ - ("ValueError", "invalid_input"), - ("OSError", "io_error"), - ("IsADirectoryError", "invalid_path"), - ("FILE-NOT-FOUND", "not_found"), - ("EACCES", "permission_denied"), - ("Already invalid!", "already_invalid"), - ("503 Unavailable", "error_503_unavailable"), - (FileNotFoundError(), "not_found"), - (FileExistsError(), "already_exists"), - (PermissionError(), "permission_denied"), - (NotADirectoryError(), "invalid_path"), - (TimeoutError(), "timeout"), - (ValueError(), "invalid_input"), - (OSError(), "io_error"), - (RuntimeError(), "internal_error"), - (KeyboardInterrupt(), "interrupted"), - ], -) -def test_error_code_mapping(value: str | BaseException, code: str) -> None: - assert normalize_error_code(value) == code - - -@pytest.mark.parametrize("value", [float("nan"), float("inf"), float("-inf")]) -def test_nonfinite_values_are_rejected(value: float) -> None: - with pytest.raises(ValueError, match="non-finite"): - success("analyze", {"energy": value}) - - with pytest.raises(ValueError, match="non-finite"): - failure("analyze", "invalid_result", "Invalid result", details=value) - - with pytest.raises(ValueError, match="non-finite"): - success("analyze", artifacts=({"energy": value},)) - - -@pytest.mark.parametrize("value", [{1, 2}, Path("result.json"), object()]) -def test_unserializable_values_are_rejected(value: object) -> None: - with pytest.raises(TypeError, match="non-JSON value"): - success("inspect", value) - - -def test_non_string_object_keys_are_rejected_instead_of_coerced() -> None: - with pytest.raises(TypeError, match="non-string object key"): - success("inspect", {1: "frame"}) - - -def test_circular_data_is_rejected() -> None: - data: list[object] = [] - data.append(data) - - with pytest.raises(ValueError, match="circular reference"): - success("inspect", data) - +def test_serialize_emits_one_compact_document_and_rejects_nan() -> None: + document = serialize(success("inspect", {"value": 1.0})) -@pytest.mark.parametrize( - ("call", "error", "message"), - [ - (lambda: success(""), ValueError, "command must not be empty"), - (lambda: failure("inspect", "", "bad"), ValueError, "error code must not be empty"), - (lambda: failure("inspect", "bad", " "), ValueError, "error message must not be empty"), - ( - lambda: success("inspect", warnings=("",)), - ValueError, - "warning must not be empty", - ), - ( - lambda: success("inspect", warnings="not-a-sequence"), - TypeError, - "warnings must be a sequence", - ), - ( - lambda: success("inspect", artifacts="result.json"), - TypeError, - "artifacts must be a sequence", - ), - ], -) -def test_invalid_envelope_metadata_is_rejected( - call: Callable[[], object], - error: type[Exception], - message: str, -) -> None: - with pytest.raises(error, match=message): - call() + assert document.count("\n") == 1 + assert json.loads(document)["data"] == {"value": 1.0} + with pytest.raises(ValueError): + serialize(success("inspect", {"value": float("nan")})) diff --git a/tests/test_cli_schema.py b/tests/test_cli_schema.py index 524ad15..22bdd35 100644 --- a/tests/test_cli_schema.py +++ b/tests/test_cli_schema.py @@ -1,96 +1,18 @@ from __future__ import annotations -import importlib -import importlib.util import json from pathlib import Path -from typing import Any import pytest from electrace.cli_output import failure, success -_SCHEMA_PATH = Path(__file__).parents[1] / "schemas" / "electrace.cli.v1.schema.json" +def test_public_cli_schema_accepts_both_envelopes() -> None: + jsonschema = pytest.importorskip("jsonschema") + path = Path(__file__).parents[1] / "schemas" / "electrace.cli.v1.schema.json" + schema = json.loads(path.read_text(encoding="utf-8")) + validator = jsonschema.Draft202012Validator(schema) -def _schema() -> dict[str, Any]: - payload = json.loads(_SCHEMA_PATH.read_text(encoding="utf-8")) - assert isinstance(payload, dict) - return payload - - -def _validator(schema: dict[str, Any]) -> Any | None: - if importlib.util.find_spec("jsonschema") is None: - return None - jsonschema = importlib.import_module("jsonschema") - jsonschema.Draft202012Validator.check_schema(schema) - return jsonschema.Draft202012Validator(schema) - - -def test_cli_success_and_failure_envelopes_match_the_public_schema() -> None: - schema = _schema() - assert schema["$schema"] == "https://json-schema.org/draft/2020-12/schema" - assert schema["$id"].endswith("/schemas/electrace.cli.v1.schema.json") - assert {branch["$ref"] for branch in schema["oneOf"]} == { - "#/$defs/success", - "#/$defs/failure", - } - validator = _validator(schema) - if validator is None: - return - - validator.validate( - success( - "workflow.validate", - {"schema_version": "electrace.workflow-validation.v1", "jobs": []}, - warnings=("executable is not installed on this host",), - artifacts=({"path": "workflow.yaml", "role": "manifest"},), - ) - ) - validator.validate( - failure( - "workflow.validate", - "invalid_input", - "manifest has an invalid frame selector", - details={"path": "workflow.yaml"}, - ) - ) - - -@pytest.mark.parametrize( - "payload", - [ - { - "schema_version": "electrace.cli.v1", - "ok": True, - "command": "inspect", - "warnings": [], - "artifacts": [], - }, - { - "schema_version": "electrace.cli.v1", - "ok": False, - "command": "inspect", - "data": {}, - "warnings": [], - "artifacts": [], - }, - { - "schema_version": "electrace.cli.v0", - "ok": True, - "command": "inspect", - "data": {}, - "warnings": [], - "artifacts": [], - }, - ], -) -def test_cli_schema_rejects_incomplete_or_mismatched_envelopes( - payload: dict[str, Any], -) -> None: - validator = _validator(_schema()) - if validator is None: - return - jsonschema = importlib.import_module("jsonschema") - with pytest.raises(jsonschema.exceptions.ValidationError): - validator.validate(payload) + validator.validate(success("inspect", {"frames": 2})) + validator.validate(failure("inspect", "invalid_input", "bad input")) diff --git a/tests/test_config.py b/tests/test_config.py index c10e5aa..05a7f3e 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,119 +1,35 @@ from __future__ import annotations -from datetime import date from pathlib import Path from typing import Any import pytest -from electrace.config import ( - ElecTraceConfig, - EventSettings, - InputConfig, - load_config, - write_default_config, -) +from electrace.config import InputConfig, load_config, write_default_config from electrace.pipeline import _event_config, _matching_configs def test_default_config_round_trip(tmp_path: Path) -> None: - path = write_default_config(tmp_path / "electrace.toml") - config = load_config(path) + config = load_config(write_default_config(tmp_path / "electrace.toml")) assert config.project_name == "ElecTrace analysis" assert config.series_kind == "auto" assert config.matching.max_atom_displacement == pytest.approx(1.5) - assert config.matching.descriptor_weight == pytest.approx(0.35) - assert config.matching.ambiguity_tolerance == pytest.approx(0.05) - assert config.matching.conservation_tolerance == pytest.approx(0.35) - assert config.events.minimum_event_confidence == pytest.approx(0.10) assert config.events.descriptor_thresholds["icohp"] == pytest.approx(0.5) assert config.plugins["critic2"]["enabled"] is True - assert config.inputs.descriptor_source == "auto" - - _, matching = _matching_configs(config) - events = _event_config(config) - assert matching.descriptor_scales == {} - assert matching.ambiguity_tolerance == pytest.approx(0.05) - assert matching.conservation_tolerance == pytest.approx(0.35) - assert events.hysteresis_frames == 1 - assert events.descriptor_thresholds["icobi"] == pytest.approx(0.1) -def test_default_config_refuses_to_overwrite(tmp_path: Path) -> None: +def test_default_config_refuses_overwrite(tmp_path: Path) -> None: path = write_default_config(tmp_path / "electrace.toml") - with pytest.raises(FileExistsError): write_default_config(path) -@pytest.mark.parametrize("target_exists", [False, True]) -def test_default_config_never_follows_leaf_symlinks(tmp_path: Path, target_exists: bool) -> None: - target = tmp_path / "outside.toml" - if target_exists: - target.write_text("outside\n", encoding="utf-8") - destination = tmp_path / "electrace.toml" - destination.symlink_to(target) - - with pytest.raises(OSError, match="symbolic link"): - write_default_config(destination, overwrite=True) - - assert destination.is_symlink() - assert target.exists() is target_exists - if target_exists: - assert target.read_text(encoding="utf-8") == "outside\n" - - -def test_default_config_write_is_atomic_on_failure( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - destination = tmp_path / "electrace.toml" - destination.write_text("original\n", encoding="utf-8") - write_text = Path.write_text - - def fail_temporary_write(path: Path, data: str, **kwargs: Any) -> int: - if path.name.startswith(f".{destination.name}."): - write_text(path, "partial\n", **kwargs) - raise OSError("simulated write failure") - return write_text(path, data, **kwargs) - - monkeypatch.setattr(Path, "write_text", fail_temporary_write) - - with pytest.raises(OSError, match="simulated write failure"): - write_default_config(destination, overwrite=True) - - assert destination.read_text(encoding="utf-8") == "original\n" - assert list(tmp_path.glob(f".{destination.name}.*.tmp")) == [] - - -@pytest.mark.parametrize( - ("field", "value"), - [ - ("structure_name", "../CONTCAR"), - ("fallback_structure_name", "inputs/POSCAR"), - ("descriptor_name", "/tmp/features.json"), - ], -) -def test_input_filenames_must_be_basenames(field: str, value: str) -> None: - options: dict[str, Any] = {field: value} - with pytest.raises(ValueError, match="must be a basename"): - InputConfig(**options) - - -def test_descriptor_source_is_normalized_and_requires_external_location() -> None: - assert InputConfig(descriptor_source=" AUTO ").descriptor_source == "auto" - with pytest.raises(ValueError, match="descriptor_source must be one of"): - InputConfig(descriptor_source="prefer_external") - with pytest.raises(ValueError, match="descriptor_dir is required"): - InputConfig(descriptor_source="merge") - - -def test_descriptor_scales_are_loaded_and_propagated(tmp_path: Path) -> None: +def test_configured_scales_and_thresholds_reach_runtime(tmp_path: Path) -> None: path = tmp_path / "electrace.toml" path.write_text( """[matching] descriptor_weight = 0.8 -ambiguity_tolerance = 0.02 conservation_tolerance = 0.2 [matching.descriptor_scales] @@ -121,7 +37,6 @@ def test_descriptor_scales_are_loaded_and_propagated(tmp_path: Path) -> None: rho = 0.02 [events] -hysteresis_frames = 2 minimum_event_confidence = 0.25 [events.descriptor_thresholds] @@ -129,169 +44,48 @@ def test_descriptor_scales_are_loaded_and_propagated(tmp_path: Path) -> None: """, encoding="utf-8", ) + config = load_config(path) _, matching = _matching_configs(config) events = _event_config(config) - assert matching.descriptor_weight == pytest.approx(0.8) - assert matching.ambiguity_tolerance == pytest.approx(0.02) - assert matching.conservation_tolerance == pytest.approx(0.2) assert matching.descriptor_scales == {"icohp": 0.25, "rho": 0.02} - assert events.hysteresis_frames == 2 assert events.min_event_confidence == pytest.approx(0.25) assert events.descriptor_thresholds == {"icohp": 0.4} @pytest.mark.parametrize( - ("medium", "high"), - [(-0.1, 0.8), (0.9, 0.8), (0.5, 1.1), (float("nan"), 0.8)], -) -def test_confidence_threshold_order_is_validated(medium: float, high: float) -> None: - with pytest.raises(ValueError, match=r"confidence_threshold|confidence thresholds"): - EventSettings( - medium_confidence_threshold=medium, - high_confidence_threshold=high, - ) - - -def test_equal_confidence_thresholds_are_allowed() -> None: - settings = EventSettings( - medium_confidence_threshold=0.7, - high_confidence_threshold=0.7, - ) - assert settings.medium_confidence_threshold == settings.high_confidence_threshold - - -@pytest.mark.parametrize("value", [-0.1, 1.1, float("nan")]) -def test_minimum_event_confidence_is_validated(value: float) -> None: - with pytest.raises(ValueError, match="minimum_event_confidence"): - EventSettings(minimum_event_confidence=value) - - -def test_toml_dates_are_rejected_in_json_bound_metadata(tmp_path: Path) -> None: - path = tmp_path / "dated.toml" - path.write_text("[metadata]\ncreated = 2026-08-07\n", encoding="utf-8") - - with pytest.raises(ValueError, match=r"configuration\.metadata\.created.*date"): - load_config(path) - - with pytest.raises(ValueError, match=r"configuration\.metadata\.created.*date"): - ElecTraceConfig(metadata={"created": date(2026, 8, 7)}) - - -@pytest.mark.parametrize("literal", ["nan", "inf", "-inf"]) -def test_nonfinite_toml_values_are_rejected_recursively(tmp_path: Path, literal: str) -> None: - path = tmp_path / "nonfinite.toml" - path.write_text(f"[metadata]\nvalue = {literal}\n", encoding="utf-8") - - with pytest.raises(ValueError, match=r"configuration\.metadata\.value must be finite"): - load_config(path) - - -def test_to_dict_revalidates_mutated_metadata() -> None: - config = ElecTraceConfig() - config.metadata["bad"] = float("nan") - - with pytest.raises(ValueError, match=r"configuration\.metadata\.bad must be finite"): - config.to_dict() - - -@pytest.mark.parametrize( - ("document", "message"), - [ - ('[inputs]\nprefer_contcar = "false"\n', "inputs.prefer_contcar must be a boolean"), - ('[outputs]\njson = "false"\n', "outputs.json must be a boolean"), - ( - '[plugins.critic2]\nenabled = "false"\n', - "plugins.critic2.enabled must be a boolean", - ), - ( - "[plugins.critic2]\nexecutable = false\n", - "plugins.critic2.executable must be a string", - ), - ( - "[plugins.critic2]\ncapability = 7\n", - "plugins.critic2.capability must be a string", - ), - ], -) -def test_behavioral_booleans_and_reserved_plugin_options_are_strict( - tmp_path: Path, document: str, message: str -) -> None: - path = tmp_path / "invalid.toml" - path.write_text(document, encoding="utf-8") - - with pytest.raises((TypeError, ValueError), match=message): - load_config(path) - - -@pytest.mark.parametrize("literal", ["2.9", "2.0", "true"]) -def test_integer_event_settings_reject_nonintegers(tmp_path: Path, literal: str) -> None: - path = tmp_path / "invalid-integer.toml" - path.write_text(f"[events]\nminimum_persistence = {literal}\n", encoding="utf-8") - - with pytest.raises( - (TypeError, ValueError), match=r"events\.minimum_persistence must be an integer" - ): - load_config(path) - - -@pytest.mark.parametrize( - "document", + ("field", "value"), [ - '[matching.descriptor_scales]\n"icohp " = 0.5\n', - '[events.descriptor_thresholds]\n" icobi" = 0.1\n', + ("structure_name", "../CONTCAR"), + ("fallback_structure_name", "inputs/POSCAR"), + ("descriptor_name", "/tmp/features.json"), ], ) -def test_descriptor_keys_reject_surrounding_whitespace(tmp_path: Path, document: str) -> None: - path = tmp_path / "whitespace-key.toml" - path.write_text(document, encoding="utf-8") - - with pytest.raises(ValueError, match="must not contain surrounding whitespace"): - load_config(path) +def test_input_filenames_must_be_basenames(field: str, value: str) -> None: + options: dict[str, Any] = {field: value} + with pytest.raises(ValueError, match="must be a basename"): + InputConfig(**options) @pytest.mark.parametrize( ("document", "message"), [ - ( - "[matching]\nsplit_merge_threshold = 2\n", - "matching.split_merge_threshold must be at most 1", - ), - ( - "[matching]\nconservation_tolerance = 2\n", - "matching.conservation_tolerance must be at most 1", - ), - ( - "[matching]\nambiguity_tolerance = -0.1\n", - "matching.ambiguity_tolerance must be at least 0", - ), - ( - "[matching]\nmax_feature_displacement = 0\n", - "matching.max_feature_displacement must be greater than 0", - ), - ( - "[matching.descriptor_scales]\nicohp = 0\n", - "matching.descriptor_scales.icohp must be greater than 0", - ), - ( - "[events]\ncharge_change_threshold = 0\n", - "events.charge_change_threshold must be greater than 0", - ), - ('[outputs]\ndirectory = ""\n', "outputs.directory must be a non-empty string"), + ('[inputs]\nprefer_contcar = "false"\n', "must be a boolean"), + ("[matching]\nmax_feature_displacement = 0\n", "must be greater than 0"), + ("[events]\nminimum_persistence = 2.5\n", "must be an integer"), + ('[plugins.critic2]\nenabled = "false"\n', "must be a boolean"), + ("[matcing]\nposition_weight = 2\n", "unknown keys"), ], ) -def test_scientific_numeric_ranges_and_output_directory_are_validated( - tmp_path: Path, document: str, message: str -) -> None: - path = tmp_path / "invalid-range.toml" +def test_invalid_configuration_fails_cleanly(tmp_path: Path, document: str, message: str) -> None: + path = tmp_path / "invalid.toml" path.write_text(document, encoding="utf-8") - with pytest.raises((TypeError, ValueError), match=message): load_config(path) -def test_arbitrary_json_compatible_plugin_options_are_preserved(tmp_path: Path) -> None: +def test_json_compatible_plugin_options_are_preserved(tmp_path: Path) -> None: path = tmp_path / "plugin.toml" path.write_text( """[plugins.custom] @@ -299,53 +93,13 @@ def test_arbitrary_json_compatible_plugin_options_are_preserved(tmp_path: Path) executable = "custom-tool" capability = "custom_evidence" iterations = 4 -labels = ["a", "b"] [plugins.custom.parameters] alpha = 0.25 """, encoding="utf-8", ) - config = load_config(path) - - assert config.plugins["custom"]["iterations"] == 4 - assert config.plugins["custom"]["labels"] == ["a", "b"] - assert config.plugins["custom"]["parameters"] == {"alpha": 0.25} - - -@pytest.mark.parametrize( - ("document", "message"), - [ - ("[matcing]\nposition_weight = 2\n", "configuration root contains unknown keys: matcing"), - ('[project]\nseries_knd = "neb"\n', r"\[project\] contains unknown keys: series_knd"), - ], -) -def test_unknown_fixed_configuration_keys_are_rejected( - tmp_path: Path, document: str, message: str -) -> None: - path = tmp_path / "typo.toml" - path.write_text(document, encoding="utf-8") - - with pytest.raises(ValueError, match=message): - load_config(path) - - -def test_plugin_table_names_are_lowercase_and_normalization_collisions_fail( - tmp_path: Path, -) -> None: - uppercase = tmp_path / "uppercase.toml" - uppercase.write_text("[plugins.Critic2]\nenabled = true\n", encoding="utf-8") - duplicate = tmp_path / "duplicate.toml" - duplicate.write_text( - """[plugins.critic2] -enabled = true -[plugins." critic2 "] -enabled = false -""", - encoding="utf-8", - ) + options = load_config(path).plugins["custom"] - with pytest.raises(ValueError, match="must be lowercase"): - load_config(uppercase) - with pytest.raises(ValueError, match="duplicate normalized plugin name"): - load_config(duplicate) + assert options["iterations"] == 4 + assert options["parameters"] == {"alpha": 0.25} diff --git a/tests/test_events.py b/tests/test_events.py index 191fc7b..6992cae 100644 --- a/tests/test_events.py +++ b/tests/test_events.py @@ -2,14 +2,9 @@ import pytest -from electrace.events import ( - EventDetectionConfig, - _topology_persistence, - detect_sequence_events, - detect_transition_events, -) +from electrace.events import EventDetectionConfig, detect_sequence_events, detect_transition_events from electrace.matching import match_features -from electrace.models import Event, EventType, Feature, Frame, MatchEdge, MatchResult, Sequence +from electrace.models import EventType, Feature, Frame, Sequence CELL = ((10.0, 0.0, 0.0), (0.0, 10.0, 0.0), (0.0, 0.0, 10.0)) @@ -21,239 +16,29 @@ def frame(index: int, *features: Feature) -> Frame: def test_transition_reports_birth_and_death() -> None: previous = frame(0, Feature("gone", "nna", (0.1, 0.1, 0.1))) current = frame(1, Feature("born", "bcp", (0.8, 0.8, 0.8))) + events = detect_transition_events(previous, current, match_features(previous, current)) + assert {event.event_type for event in events} == {EventType.BIRTH, EventType.DEATH} assert all(event.evidence["unmatched"] for event in events) -def test_transition_reports_split_with_conservation_evidence() -> None: +def test_transition_reports_conserving_split() -> None: previous = frame(0, Feature("p", "elf_basin", (0.5, 0.5, 0.5), 1.0, 10.0)) current = frame( 1, Feature("a", "elf_basin", (0.49, 0.5, 0.5), 0.4, 4.0), Feature("b", "elf_basin", (0.51, 0.5, 0.5), 0.6, 6.0), ) + events = detect_transition_events(previous, current, match_features(previous, current)) + assert len(events) == 1 assert events[0].event_type is EventType.SPLIT assert "conservation" in events[0].evidence["match"] -def test_ambiguous_assignment_suppresses_mapping_birth_and_property_claims() -> None: - previous = frame( - 0, - Feature( - "source", - "bond_critical_point", - (0.5, 0.5, 0.5), - attributes={"icohp": -0.2}, - ), - ) - targets = ( - Feature( - "a", - "bond_critical_point", - (0.5, 0.5, 0.5), - attributes={"icohp": -1.2}, - ), - Feature( - "b", - "bond_critical_point", - (0.5, 0.5, 0.5), - attributes={"icohp": -1.2}, - ), - ) - results = [] - for ordered in (targets, tuple(reversed(targets))): - current = frame(1, *ordered) - matches = match_features(previous, current) - results.append((matches, detect_transition_events(previous, current, matches))) - - assert all(matches.diagnostics["ambiguous"] is True for matches, _ in results) - assert all(events == [] for _, events in results) - - -def test_proximity_only_candidate_never_becomes_a_persistent_split() -> None: - sequence = Sequence( - ( - frame(0, Feature("parent", "elf_basin", (0.5, 0.5, 0.5))), - frame( - 1, - Feature("left-1", "elf_basin", (0.48, 0.5, 0.5)), - Feature("right-1", "elf_basin", (0.52, 0.5, 0.5)), - ), - frame( - 2, - Feature("left-2", "elf_basin", (0.47, 0.5, 0.5)), - Feature("right-2", "elf_basin", (0.53, 0.5, 0.5)), - ), - ) - ) - - events = detect_sequence_events(sequence) - - assert not {event.event_type for event in events} & { - EventType.SPLIT, - EventType.MERGE, - EventType.REARRANGEMENT, - } - - -def test_merge_persistence_ends_when_merged_descendant_splits() -> None: - sequence = Sequence( - ( - frame( - 0, - Feature("a", "elf_basin", (0.48, 0.5, 0.5), 0.4, 4.0), - Feature("b", "elf_basin", (0.52, 0.5, 0.5), 0.6, 6.0), - ), - frame(1, Feature("c", "elf_basin", (0.5, 0.5, 0.5), 1.0, 10.0)), - frame( - 2, - Feature("d", "elf_basin", (0.48, 0.5, 0.5), 0.4, 4.0), - Feature("e", "elf_basin", (0.52, 0.5, 0.5), 0.6, 6.0), - ), - ) - ) - - merge = next( - event for event in detect_sequence_events(sequence) if event.event_type is EventType.MERGE - ) - - assert merge.persistence == 1 - assert merge.metadata["persistence_status"] == "transient" - assert merge.metadata["persistence_end_frame"] == 1 - assert merge.evidence["persistence_termination"] == "merge_participant_expanded" - assert ( - merge.evidence["persistence_cardinality_criterion"] - == "all_target_participants_distinct_one_to_one" - ) - - -def test_observed_split_termination_is_transient_not_right_censored() -> None: - sequence = Sequence( - ( - frame(0, Feature("parent", "elf_basin", (0.5, 0.5, 0.5), 1.0, 10.0)), - frame( - 1, - Feature("left", "elf_basin", (0.48, 0.5, 0.5), 0.4, 4.0), - Feature("right", "elf_basin", (0.52, 0.5, 0.5), 0.6, 6.0), - ), - frame(2, Feature("merged", "elf_basin", (0.5, 0.5, 0.5), 1.0, 10.0)), - ) - ) - - split = next( - event - for event in detect_sequence_events( - sequence, - event_config=EventDetectionConfig( - min_persistence=3, - hysteresis_frames=0, - ), - ) - if event.event_type is EventType.SPLIT - ) - - assert split.persistence == 1 - assert split.evidence["maximum_observable_persistence"] == 2 - assert split.evidence["persistence_termination"] == "split_participants_coalesced" - assert split.metadata["persistence_status"] == "transient" - - -def test_observed_termination_above_hysteresis_is_still_transient() -> None: - sequence = Sequence( - ( - frame(0, Feature("parent", "elf_basin", (0.5, 0.5, 0.5), 1.0, 10.0)), - frame( - 1, - Feature("left-1", "elf_basin", (0.48, 0.5, 0.5), 0.4, 4.0), - Feature("right-1", "elf_basin", (0.52, 0.5, 0.5), 0.6, 6.0), - ), - frame( - 2, - Feature("left-2", "elf_basin", (0.47, 0.5, 0.5), 0.4, 4.0), - Feature("right-2", "elf_basin", (0.53, 0.5, 0.5), 0.6, 6.0), - ), - frame(3, Feature("merged", "elf_basin", (0.5, 0.5, 0.5), 1.0, 10.0)), - ) - ) - - split = next( - event - for event in detect_sequence_events( - sequence, - event_config=EventDetectionConfig( - min_persistence=4, - hysteresis_frames=1, - ), - ) - if event.event_type is EventType.SPLIT - ) - - assert split.persistence == 2 - assert split.persistence > 1 - assert split.evidence["maximum_observable_persistence"] == 3 - assert split.evidence["persistence_termination"] == "split_participants_coalesced" - assert split.metadata["persistence_status"] == "transient" - - -def test_rearrangement_requires_distinct_one_to_one_participant_lineages() -> None: - event = Event( - EventType.REARRANGEMENT, - 0, - 1, - ("a", "b"), - ("c", "d"), - 0.8, - ) - onset = MatchResult( - 0, - 1, - edges=(MatchEdge(("a", "b"), ("c", "d"), 0.1, 0.8),), - ) - participant_expands = MatchResult( - 1, - 2, - edges=( - MatchEdge(("c",), ("e", "f"), 0.1, 0.8), - MatchEdge(("d",), ("g",), 0.1, 0.8), - ), - ) - - persistence, end_position, evidence = _topology_persistence( - event, - 0, - (onset, participant_expands), - ) - - assert persistence == 1 - assert end_position == 1 - assert evidence["persistence_termination"] == "rearrangement_participant_expanded" - assert ( - evidence["persistence_cardinality_criterion"] - == "all_target_participants_distinct_one_to_one" - ) - - -def test_property_change_requires_configured_effect_size() -> None: - previous = frame( - 0, - Feature("a", "bond_critical_point", (0.5, 0.5, 0.5), attributes={"icohp": -0.2}), - ) - current = frame( - 1, - Feature("b", "bond_critical_point", (0.5, 0.5, 0.5), attributes={"icohp": -1.2}), - ) - events = detect_transition_events(previous, current, match_features(previous, current)) - assert len(events) == 1 - event = events[0] - assert event.event_type is EventType.PROPERTY_CHANGE - assert event.evidence["descriptor_changes"]["icohp"]["delta"] == pytest.approx(-1.0) - assert event.evidence["property_signals"]["attribute:icohp"]["baseline"] == pytest.approx(-0.2) - - -def test_property_change_is_confirmed_when_changed_state_persists() -> None: +def test_property_change_is_confirmed_when_lineage_persists() -> None: sequence = Sequence( ( frame( @@ -272,61 +57,19 @@ def test_property_change_is_confirmed_when_changed_state_persists() -> None: ) event = next( - item - for item in detect_sequence_events(sequence) - if item.event_type is EventType.PROPERTY_CHANGE - ) - assert event.persistence == 2 - assert event.metadata["persistence_status"] == "confirmed" - assert event.metadata["persistence_end_frame"] == 2 - - -def test_property_persistence_tolerates_only_configured_gap_frames() -> None: - sequence = Sequence( - ( - frame( - 0, - Feature("f0", "bond_critical_point", (0.5, 0.5, 0.5), attributes={"icohp": -0.2}), - ), - frame( - 1, - Feature("f1", "bond_critical_point", (0.5, 0.5, 0.5), attributes={"icohp": -1.2}), - ), - frame( - 2, - Feature("f2", "bond_critical_point", (0.5, 0.5, 0.5), attributes={"icohp": -0.4}), - ), - frame( - 3, - Feature("f3", "bond_critical_point", (0.5, 0.5, 0.5), attributes={"icohp": -1.1}), - ), - ) - ) - - tolerant = next( - item - for item in detect_sequence_events( - sequence, event_config=EventDetectionConfig(hysteresis_frames=1) - ) - if item.event_type is EventType.PROPERTY_CHANGE and item.frame_after == 1 - ) - strict = next( item for item in detect_sequence_events( - sequence, event_config=EventDetectionConfig(hysteresis_frames=0) + sequence, event_config=EventDetectionConfig(min_persistence=2) ) - if item.event_type is EventType.PROPERTY_CHANGE and item.frame_after == 1 + if item.event_type is EventType.PROPERTY_CHANGE ) - assert tolerant.persistence == 2 - assert tolerant.metadata["persistence_status"] == "confirmed" - assert tolerant.metadata["persistence_end_frame"] == 3 - assert tolerant.evidence["hysteresis_gap_frames"] == [2] - assert strict.persistence == 1 - assert strict.metadata["persistence_status"] == "transient" + assert event.persistence == 2 + assert event.metadata["persistence_status"] == "confirmed" + assert event.evidence["descriptor_changes"]["icohp"]["delta"] == pytest.approx(-1.0) -def test_hysteresis_rejects_one_frame_topology_flicker_even_when_minimum_is_one() -> None: +def test_one_frame_birth_is_transient() -> None: sequence = Sequence( ( frame(0), @@ -334,76 +77,29 @@ def test_hysteresis_rejects_one_frame_topology_flicker_even_when_minimum_is_one( frame(2), ) ) - rejected = next( - item - for item in detect_sequence_events( - sequence, - event_config=EventDetectionConfig(min_persistence=1, hysteresis_frames=1), - ) - if item.event_type is EventType.BIRTH - ) - accepted = next( - item - for item in detect_sequence_events( - sequence, - event_config=EventDetectionConfig(min_persistence=1, hysteresis_frames=0), - ) - if item.event_type is EventType.BIRTH - ) - - assert rejected.metadata["persistence_status"] == "transient" - assert accepted.metadata["persistence_status"] == "confirmed" - -def test_negative_hysteresis_frames_are_rejected() -> None: - with pytest.raises(ValueError, match="hysteresis_frames"): - EventDetectionConfig(hysteresis_frames=-1) - - -def test_sequence_birth_is_confirmed_when_descendant_persists() -> None: - sequence = Sequence( - ( - frame(0), - frame(1, Feature("new-1", "nna", (0.5, 0.5, 0.5))), - frame(2, Feature("new-2", "nna", (0.51, 0.5, 0.5))), - ) + birth = next( + event for event in detect_sequence_events(sequence) if event.event_type is EventType.BIRTH ) - events = detect_sequence_events(sequence) - birth = next(event for event in events if event.event_type is EventType.BIRTH) - assert birth.persistence == 2 - assert birth.metadata["persistence_status"] == "confirmed" - -def test_sequence_one_frame_birth_is_downweighted_as_transient() -> None: - sequence = Sequence( - ( - frame(0), - frame(1, Feature("noise", "nna", (0.5, 0.5, 0.5))), - frame(2), - ) - ) - events = detect_sequence_events(sequence) - birth = next(event for event in events if event.event_type is EventType.BIRTH) assert birth.persistence == 1 assert birth.metadata["persistence_status"] == "transient" - assert birth.confidence == pytest.approx(0.75 / 2) - filtered = detect_sequence_events( - sequence, event_config=EventDetectionConfig(include_transient=False) - ) - assert not any(event.event_type is EventType.BIRTH for event in filtered) +def test_death_persistence_is_invariant_to_later_local_id_reuse() -> None: + def death_status(later_id: str) -> tuple[int, str]: + sequence = Sequence( + ( + frame(0, Feature("local", "nna", (0.1, 0.1, 0.1))), + frame(1), + frame(2, Feature(later_id, "nna", (0.8, 0.8, 0.8))), + ) + ) + death = next( + event + for event in detect_sequence_events(sequence) + if event.event_type is EventType.DEATH and event.frame_before == 0 + ) + return death.persistence, str(death.metadata["persistence_status"]) -def test_event_round_trip_is_serializable() -> None: - event = Event( - EventType.MERGE, - 2, - 3, - ("a", "b"), - ("c",), - 0.8, - 3, - {"charge": {"delta": 0.1}}, - "merge", - ) - assert Event.from_dict(event.to_dict()).to_dict() == event.to_dict() + assert death_status("local") == death_status("different") == (2, "confirmed") diff --git a/tests/test_export.py b/tests/test_export.py index 448e265..d052759 100644 --- a/tests/test_export.py +++ b/tests/test_export.py @@ -2,44 +2,26 @@ import csv import json -import sys from pathlib import Path import pytest -from electrace.export import ( - ExportDependencyError, - write_events_csv, - write_hdf5, - write_json, - write_tracks_csv, -) +from electrace.export import write_events_csv, write_json, write_tracks_csv def test_machine_readable_exports(tmp_path: Path) -> None: - analysis = {"schema_version": "0.1", "events": [{"type": "birth"}]} - json_path = write_json(analysis, tmp_path / "analysis.json") + json_path = write_json({"schema_version": "0.1"}, tmp_path / "analysis.json") events_path = write_events_csv( - [ - { - "event_id": "E0001", - "type": "birth", - "source_ids": [], - "target_ids": ["nna-1"], - "confidence": 0.9, - } - ], + [{"event_id": "E1", "type": "birth", "target_ids": ["nna-1"]}], tmp_path / "events.csv", ) tracks_path = write_tracks_csv( [ { - "track_id": "T0001", + "track_id": "T1", "frame": 0, "feature_id": "nna-1", - "kind": "nna", - "parent_track_ids": ["T0000"], - "attributes": {"accepted": True, "note": None}, + "attributes": {"accepted": True}, } ], tmp_path / "tracks.csv", @@ -49,116 +31,25 @@ def test_machine_readable_exports(tmp_path: Path) -> None: with events_path.open(newline="") as handle: assert next(csv.DictReader(handle))["target_ids"] == '["nna-1"]' with tracks_path.open(newline="") as handle: - track = next(csv.DictReader(handle)) - assert track["track_id"] == "T0001" - assert json.loads(track["parent_track_ids"]) == ["T0000"] - assert json.loads(track["attributes"]) == {"accepted": True, "note": None} + assert json.loads(next(csv.DictReader(handle))["attributes"]) == {"accepted": True} -@pytest.mark.parametrize("bad_value", [float("nan"), object()]) -def test_json_export_rejects_non_finite_and_non_json_values( - tmp_path: Path, bad_value: object -) -> None: +def test_json_export_rejects_non_finite_values(tmp_path: Path) -> None: output = tmp_path / "analysis.json" - - with pytest.raises( - ValueError, - match="analysis JSON contains a non-finite or non-JSON value", - ): - write_json({"nested": {"bad": bad_value}}, output) - + with pytest.raises(ValueError, match="non-finite or non-JSON"): + write_json({"bad": float("nan")}, output) assert not output.exists() -def test_hdf5_export_rejects_non_finite_analysis_before_writing(tmp_path: Path) -> None: - output = tmp_path / "analysis.h5" - - with pytest.raises( - ValueError, - match="HDF5 analysis JSON contains a non-finite or non-JSON value", - ): - write_hdf5({"nested": [float("inf")]}, output) - - assert not output.exists() - - -def test_missing_hdf5_dependency_raises_dedicated_error( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - monkeypatch.setitem(sys.modules, "h5py", None) - - with pytest.raises(ExportDependencyError, match=r"pip install electrace\[hdf5\]"): - write_hdf5({"schema_version": "0.1"}, tmp_path / "analysis.h5") - - -@pytest.mark.parametrize( - "writer,row,context", - [ - ( - write_events_csv, - {"event_id": "E1", "metadata": {"bad": float("nan")}}, - "events CSV", - ), - ( - write_tracks_csv, - {"track_id": "T1", "attributes": {"bad": object()}}, - "feature-tracks CSV", - ), - ], -) -def test_csv_exports_reject_invalid_nested_json_without_partial_files( - tmp_path: Path, - writer: object, - row: dict[str, object], - context: str, -) -> None: - output = tmp_path / "output.csv" - - with pytest.raises( - ValueError, - match=rf"{context} row 1, field .* contains a non-finite or non-JSON value", - ): - writer([row], output) # type: ignore[operator] - - assert not output.exists() - - -@pytest.mark.parametrize( - "dangerous", - [ - '=HYPERLINK("https://example.invalid")', - "+SUM(1,1)", - "-2+3", - "@SUM(1,1)", - " \t=1+1", - ], -) -def test_csv_exports_neutralize_formula_like_strings(tmp_path: Path, dangerous: str) -> None: - events_path = write_events_csv( - [ - { - "event_id": dangerous, - "type": "birth", - "description": dangerous, - "@untrusted_header": dangerous, - } - ], +def test_csv_exports_neutralize_formula_strings(tmp_path: Path) -> None: + dangerous = '=HYPERLINK("https://example.invalid")' + path = write_events_csv( + [{"event_id": dangerous, "type": "birth", "description": dangerous}], tmp_path / "events.csv", ) - tracks_path = write_tracks_csv( - [{"track_id": dangerous, "feature_id": dangerous, "kind": "nna"}], - tmp_path / "tracks.csv", - ) - with events_path.open(newline="") as handle: + with path.open(newline="") as handle: event = next(csv.DictReader(handle)) - with tracks_path.open(newline="") as handle: - track = next(csv.DictReader(handle)) assert event["event_id"] == f"'{dangerous}" assert event["description"] == f"'{dangerous}" - assert event["'@untrusted_header"] == f"'{dangerous}" - assert event["type"] == "birth" - assert track["track_id"] == f"'{dangerous}" - assert track["feature_id"] == f"'{dangerous}" - assert track["kind"] == "nna" diff --git a/tests/test_geometry.py b/tests/test_geometry.py index 7de5117..a8a7779 100644 --- a/tests/test_geometry.py +++ b/tests/test_geometry.py @@ -6,55 +6,17 @@ import pytest from electrace.geometry import ( - cartesian_to_fractional, deformation_aware_distance, - deformation_gradient, - fractional_to_cartesian, - minimum_image_fractional_displacement, - pairwise_periodic_distances, - periodic_distance, - periodic_fractional_centroid, + minimum_image_displacement, periodic_voxel_overlap, - wrap_fractional, ) -def test_fractional_cartesian_round_trip() -> None: - cell = np.array([[3.0, 0.0, 0.0], [0.7, 2.5, 0.0], [0.2, 0.3, 4.0]]) - fractional = np.array([[0.2, 0.4, 0.9], [1.1, -0.2, 0.0]]) - cartesian = fractional_to_cartesian(fractional, cell) - np.testing.assert_allclose(cartesian_to_fractional(cartesian, cell), fractional) - - -def test_wrap_respects_nonperiodic_axes() -> None: - wrapped = wrap_fractional([1.2, -0.1, 2.5], pbc=(True, True, False)) - np.testing.assert_allclose(wrapped, [0.2, 0.9, 2.5]) - - -def test_minimum_image_across_boundary() -> None: - cell = np.diag([10.0, 10.0, 10.0]) - displacement = minimum_image_fractional_displacement([0.98, 0.2, 0.3], [0.02, 0.2, 0.3], cell) - np.testing.assert_allclose(displacement, [0.04, 0.0, 0.0], atol=1e-14) - assert periodic_distance([0.98, 0.2, 0.3], [0.02, 0.2, 0.3], cell) == pytest.approx(0.4) - - -def test_triclinic_nearest_image_is_not_component_rounding() -> None: - # For this skew lattice, subtracting one of the nearly parallel lattice - # vectors is much shorter than leaving both components at +0.49. - cell = np.array([[1.0, 0.0, 0.0], [0.9, 0.1, 0.0], [0.0, 0.0, 1.0]]) - displacement = minimum_image_fractional_displacement([0.0, 0.0, 0.0], [0.49, 0.49, 0.0], cell) - np.testing.assert_allclose(displacement, [0.49, -0.51, 0.0]) - assert periodic_distance([0, 0, 0], [0.49, 0.49, 0], cell) == pytest.approx( - np.hypot(0.031, 0.051) - ) - - -def test_triclinic_nearest_image_matches_bounded_brute_force() -> None: +def test_triclinic_minimum_image_matches_brute_force() -> None: cell = np.array([[3.0, 0.0, 0.0], [1.2, 2.4, 0.0], [0.3, 0.2, 2.8]]) rng = np.random.default_rng(20260808) - for first, second in zip(rng.random((32, 3)), rng.random((32, 3)), strict=True): - displacement = minimum_image_fractional_displacement(first, second, cell) - actual = np.linalg.norm(displacement @ cell) + for first, second in zip(rng.random((12, 3)), rng.random((12, 3)), strict=True): + actual = np.linalg.norm(minimum_image_displacement(first, second, cell)) delta = second - first expected = min( np.linalg.norm((delta - np.asarray(shift)) @ cell) @@ -63,53 +25,19 @@ def test_triclinic_nearest_image_matches_bounded_brute_force() -> None: assert actual == pytest.approx(expected, abs=1.0e-12) -def test_pairwise_distances_factorize_the_cell_once(monkeypatch: pytest.MonkeyPatch) -> None: - cell = np.diag([10.0, 11.0, 12.0]) - first = np.linspace(0.0, 0.9, 24).reshape(8, 3) - second = np.linspace(0.05, 0.95, 18).reshape(6, 3) - real_svd = np.linalg.svd - calls = 0 - - def counted_svd(*args: object, **kwargs: object) -> object: - nonlocal calls - calls += 1 - return real_svd(*args, **kwargs) - - monkeypatch.setattr(np.linalg, "svd", counted_svd) - distances = pairwise_periodic_distances(first, second, cell) - - assert distances.shape == (8, 6) - assert calls == 1 - - -def test_pairwise_distances_validate_empty_inputs() -> None: - with pytest.raises(ValueError, match="finite"): - pairwise_periodic_distances( - np.empty((0, 3)), - np.array([[np.nan, 0.0, 0.0]]), - np.eye(3), - ) - - def test_deformation_aware_distance_uses_symmetric_metric() -> None: previous = np.diag([10.0, 10.0, 10.0]) current = np.diag([12.0, 12.0, 12.0]) distance = deformation_aware_distance([0.2, 0.2, 0.2], [0.3, 0.2, 0.2], previous, current) assert distance == pytest.approx(0.1 * np.sqrt((10.0**2 + 12.0**2) / 2.0)) - np.testing.assert_allclose(deformation_gradient(previous, current), np.eye(3) * 1.2) -def test_periodic_voxel_overlap_and_centroid() -> None: +def test_periodic_voxel_overlap_wraps_grid_indices() -> None: first = [(0, 0, 0), (9, 0, 0)] second = [(1, 0, 0), (0, 0, 0)] assert periodic_voxel_overlap(first, second, (10, 10, 10), shift_b=(-1, 0, 0)) == 1.0 - centroid = periodic_fractional_centroid([[0.98, 0.2, 0.3], [0.02, 0.2, 0.3]]) - assert min(abs(centroid[0]), abs(centroid[0] - 1.0)) < 1e-12 - np.testing.assert_allclose(centroid[1:], [0.2, 0.3]) -def test_invalid_cells_and_coordinate_shapes_are_rejected() -> None: +def test_invalid_cell_is_rejected() -> None: with pytest.raises(ValueError, match="nonsingular"): - periodic_distance([0, 0, 0], [0, 0, 0], np.zeros((3, 3))) - with pytest.raises(ValueError, match="axis of length 3"): - wrap_fractional([0.0, 0.0]) + minimum_image_displacement([0, 0, 0], [0, 0, 0], np.zeros((3, 3))) diff --git a/tests/test_io.py b/tests/test_io.py index 7cf6a72..1e0929d 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -1,19 +1,12 @@ from __future__ import annotations import json -import math -import re from pathlib import Path -from types import SimpleNamespace -from typing import Any import pytest -import electrace.io.descriptors as descriptor_module from electrace.io import ( DescriptorError, - DiscoveryError, - VaspParseError, descriptor_from_dict, discover_neb, discover_static_series, @@ -21,17 +14,12 @@ load_descriptor, load_path, parse_poscar, - pressure_from_label, - pressure_from_outcar, read_poscar, read_vasp_energy, ) -def _poscar(*, x: float = 0.0, cartesian: bool = False, selective: bool = False) -> str: - mode = "Cartesian" if cartesian else "Direct" - selective_line = "Selective dynamics\n" if selective else "" - flags = " T F T" if selective else "" +def _poscar(*, x: float = 0.0, cartesian: bool = False) -> str: return ( "C N test\n" "1.0\n" @@ -40,10 +28,9 @@ def _poscar(*, x: float = 0.0, cartesian: bool = False, selective: bool = False) "0.0 0.0 2.0\n" "C N\n" "1 1\n" - f"{selective_line}" - f"{mode}\n" - f"{x} 0.0 0.0{flags}\n" - f"1.0 1.0 1.0{flags}\n" + f"{'Cartesian' if cartesian else 'Direct'}\n" + f"{x} 0.0 0.0\n" + "1.0 1.0 1.0\n" ) @@ -52,7 +39,6 @@ def _write_frame(directory: Path, *, x: float = 0.0, energy: float | None = None (directory / "POSCAR").write_text(_poscar(x=x), encoding="utf-8") if energy is not None: (directory / "OUTCAR").write_text( - f" free energy TOTEN = {energy + 1.0: .8f} eV\n" f" free energy TOTEN = {energy: .8f} eV\n", encoding="utf-8", ) @@ -69,7 +55,6 @@ def _canonical_feature(*, frame_index: int = 0, frame_label: str = "00") -> dict "kind": "bond_critical_point", "fractional_position": [0.25, 0.0, 0.0], "neighbor_atom_indices": [0, 1], - "voxel_indices": [[1, 2, 3], [2, 3, 4]], "attributes": {"rho": 0.14}, } ], @@ -77,790 +62,188 @@ def _canonical_feature(*, frame_index: int = 0, frame_label: str = "00") -> dict } -def test_parse_poscar_direct_selective_and_core_frame(tmp_path: Path) -> None: - path = tmp_path / "POSCAR" - path.write_text(_poscar(x=0.25, selective=True), encoding="utf-8") - - parsed = parse_poscar(path) - frame = read_poscar(path, index=3, label="image-3", energy=-12.5) - - assert parsed.cell == ((2.0, 0.0, 0.0), (0.0, 2.0, 0.0), (0.0, 0.0, 2.0)) +def test_poscar_parsing_handles_direct_cartesian_and_negative_scale(tmp_path: Path) -> None: + direct = tmp_path / "POSCAR" + direct.write_text(_poscar(x=0.25), encoding="utf-8") + parsed = parse_poscar(direct) assert parsed.fractional_positions[0] == (0.25, 0.0, 0.0) - assert parsed.selective_dynamics[0] == (True, False, True) - assert [atom.element for atom in frame.atoms] == ["C", "N"] - assert frame.atoms[0].attributes["selective_dynamics"] == (True, False, True) - assert frame.energy == -12.5 + assert [atom.element for atom in read_poscar(direct).atoms] == ["C", "N"] - -def test_parse_cartesian_and_negative_volume_scale(tmp_path: Path) -> None: cartesian = tmp_path / "cartesian.vasp" cartesian.write_text(_poscar(x=1.0, cartesian=True), encoding="utf-8") assert parse_poscar(cartesian).fractional_positions[0] == pytest.approx((0.5, 0.0, 0.0)) - volume_scaled = tmp_path / "volume.vasp" - volume_scaled.write_text( + scaled = tmp_path / "scaled.vasp" + scaled.write_text( _poscar().replace("1.0\n2.0 0.0 0.0", "-64.0\n2.0 0.0 0.0", 1), encoding="utf-8", ) - parsed = parse_poscar(volume_scaled) - determinant = parsed.cell[0][0] * parsed.cell[1][1] * parsed.cell[2][2] - assert determinant == pytest.approx(64.0) - - component_scaled = tmp_path / "components.vasp" - component_scaled.write_text( - _poscar(cartesian=True) - .replace("1.0\n2.0 0.0 0.0", "2.0 3.0 4.0\n2.0 0.0 0.0", 1) - .replace("0.0 2.0 0.0", "1.0 2.0 0.0", 1), - encoding="utf-8", - ) - component_data = parse_poscar(component_scaled) - assert component_data.cell[1] == (2.0, 6.0, 0.0) - + cell = parse_poscar(scaled).cell + assert cell[0][0] * cell[1][1] * cell[2][2] == pytest.approx(64.0) -def test_parse_vasp4_placeholder_symbols_and_reject_bad_input(tmp_path: Path) -> None: vasp4 = tmp_path / "POSCAR.v4" vasp4.write_text(_poscar().replace("C N\n1 1", "1 1", 1), encoding="utf-8") - parsed = parse_poscar(vasp4) - assert parsed.elements == ("X1", "X2") - assert parsed.symbols_inferred is True - - bad = tmp_path / "bad" - bad.write_text("not a POSCAR\n", encoding="utf-8") - with pytest.raises(VaspParseError, match="too short"): - parse_poscar(bad) - - -def test_energy_and_pressure_extraction_use_last_records(tmp_path: Path) -> None: - outcar = tmp_path / "OUTCAR" - outcar.write_text( - "free energy TOTEN = -1.0 eV\n" - " external pressure = 100.0 kB Pullay stress = 0.0 kB\n" - "free energy TOTEN = -2.5E+01 eV\n" - " external pressure = 225.0 kB\n", - encoding="utf-8", - ) - oszicar = tmp_path / "OSZICAR" - oszicar.write_text( - " 1 F= -.10 E0= -.20 d E =0\n 2 F= 1.0 E0= 0.0 d E =0\n", - encoding="utf-8", - ) + assert parse_poscar(vasp4).elements == ("X1", "X2") - assert read_vasp_energy(outcar) == -25.0 - assert read_vasp_energy(oszicar) == 0.0 - assert pressure_from_outcar(outcar) == 22.5 - assert pressure_from_label("run_pressure_7.5_gpa") == 7.5 - assert pressure_from_label("100kbar") == 10.0 - assert pressure_from_label("relax") is None + oszicar = tmp_path / "OSZICAR" + oszicar.write_text(" 1 F= -1.0 E0= -1.2 d E =0\n", encoding="utf-8") + assert read_vasp_energy(oszicar) == -1.2 -def test_neb_discovery_orders_numeric_directories_and_validates_gaps(tmp_path: Path) -> None: - _write_frame(tmp_path / "10", energy=-8.0) - _write_frame(tmp_path / "8", energy=-6.0) - _write_frame(tmp_path / "9", energy=-7.0) - discovered = discover_neb(tmp_path) +def test_discovery_orders_neb_and_pressure_series(tmp_path: Path) -> None: + neb = tmp_path / "neb" + _write_frame(neb / "10", energy=-8.0) + _write_frame(neb / "8", energy=-6.0) + _write_frame(neb / "9", energy=-7.0) + discovered = discover_neb(neb) assert [frame.label for frame in discovered.frames] == ["8", "9", "10"] assert [frame.index for frame in discovered.frames] == [0, 1, 2] - assert [frame.metadata["directory_index"] for frame in discovered.frames] == [8, 9, 10] assert [frame.energy_ev for frame in discovered.frames] == [-6.0, -7.0, -8.0] - (tmp_path / "9" / "POSCAR").unlink() - with pytest.raises(DiscoveryError, match="no usable CONTCAR or POSCAR"): - discover_neb(tmp_path) - non_strict = discover_neb(tmp_path, strict=False) - assert [frame.label for frame in non_strict.frames] == ["8", "10"] - assert [frame.index for frame in non_strict.frames] == [0, 1] - - -def test_nonzero_neb_directory_numbers_use_canonical_descriptor_frame_indices( - tmp_path: Path, -) -> None: - for canonical_index, label in enumerate(("08", "09")): - directory = tmp_path / label - _write_frame(directory) - payload = _canonical_feature( - frame_index=canonical_index, - frame_label=label, - ) - payload["features"][0]["id"] = f"feature-{canonical_index}" # type: ignore[index] - (directory / "features.json").write_text(json.dumps(payload), encoding="utf-8") - - frames = load_path(tmp_path) - - assert [frame.index for frame in frames] == [0, 1] - assert [frame.label for frame in frames] == ["08", "09"] - assert [frame.metadata["directory_index"] for frame in frames] == [8, 9] - assert [frame.features[0].id for frame in frames] == ["feature-0", "feature-1"] - - -def test_neb_discovery_rejects_normalized_duplicate_indices(tmp_path: Path) -> None: - _write_frame(tmp_path / "01") - _write_frame(tmp_path / "1") - with pytest.raises(DiscoveryError, match="ambiguous NEB indices"): - discover_neb(tmp_path) - - -def test_static_pressure_series_orders_by_physical_pressure(tmp_path: Path) -> None: - _write_frame(tmp_path / "p20") - _write_frame(tmp_path / "5GPa") - _write_frame(tmp_path / "100kbar") - - discovered = discover_static_series(tmp_path, require_pressure=True) - assert discovered.kind.value == "pressure" - assert [frame.pressure_gpa for frame in discovered.frames] == [5.0, 10.0, 20.0] - assert [frame.label for frame in discovered.frames] == ["5GPa", "100kbar", "p20"] - - -def test_ordered_mode_alias_loads_non_pressure_static_series(tmp_path: Path) -> None: - _write_frame(tmp_path / "state10") - _write_frame(tmp_path / "state2") - frames = load_path(tmp_path, mode="ordered") - assert [frame.label for frame in frames] == ["state2", "state10"] - - -def test_custom_structure_names_and_fallback_order_reach_discovery(tmp_path: Path) -> None: - for label in ("00", "01"): - directory = tmp_path / label - directory.mkdir() - (directory / "START.vasp").write_text(_poscar(x=0.1), encoding="utf-8") - (tmp_path / "00" / "FINAL.vasp").write_text(_poscar(x=0.2), encoding="utf-8") - - preferred = discover_neb( - tmp_path, - structure_name="FINAL.vasp", - fallback_structure_name="START.vasp", - ) - reversed_order = discover_neb( - tmp_path, - prefer_contcar=False, - structure_name="FINAL.vasp", - fallback_structure_name="START.vasp", - ) - - assert [frame.structure.name for frame in preferred.frames] == [ - "FINAL.vasp", - "START.vasp", - ] - assert [frame.structure.name for frame in reversed_order.frames] == [ - "START.vasp", - "START.vasp", - ] - - -def test_load_path_descriptor_source_policies_are_explicit_and_auditable( - tmp_path: Path, -) -> None: - root = tmp_path / "series" - descriptors = tmp_path / "external" - _write_frame(root / "state") - descriptors.mkdir() - - local = _canonical_feature(frame_label="state") - local["features"][0]["id"] = "local" # type: ignore[index] - (root / "state" / "electronic.json").write_text(json.dumps(local), encoding="utf-8") - external = _canonical_feature(frame_label="state") - external["features"][0]["id"] = "external" # type: ignore[index] - external_file = descriptors / "state.json" - external_file.write_text(json.dumps(external), encoding="utf-8") - - automatic = load_path( - root, - descriptor_dir=descriptors, - descriptor_name="electronic.json", - mode="static", - ) - local_only = load_path( - root, - descriptor_dir=descriptors, - descriptor_name="electronic.json", - descriptor_source="local", - mode="static", - ) - merged = load_path( - root, - descriptor_dir=descriptors, - descriptor_name="electronic.json", - descriptor_source="merge", - mode="static", - ) + pressure = tmp_path / "pressure" + for label in ("p20", "5GPa", "100kbar"): + _write_frame(pressure / label) + series = discover_static_series(pressure, require_pressure=True) + assert [frame.pressure_gpa for frame in series.frames] == [5.0, 10.0, 20.0] - assert [feature.id for feature in automatic[0].features] == ["external"] - assert automatic[0].features[0].attributes["descriptor_producer"] == "critic2" - assert [feature.id for feature in local_only[0].features] == ["local"] - assert [feature.id for feature in merged[0].features] == ["local", "external"] - assert automatic[0].metadata["descriptor_source_policy"] == "external" - assert local_only[0].metadata["descriptor_source_policy"] == "local" - assert merged[0].metadata["descriptor_sources"] == [ - str((root / "state" / "electronic.json").resolve()), - str(external_file.resolve()), - ] - external_file.unlink() - empty = load_path( - root, - descriptor_dir=descriptors, - descriptor_source="external", - mode="static", - )[0] - assert empty.features == () - assert empty.metadata["descriptor_source_policy"] == "external" - assert empty.metadata["descriptor_sources"] == [] - - -def test_load_path_rejects_ambiguous_or_unsafe_descriptor_selection(tmp_path: Path) -> None: - _write_frame(tmp_path / "state") - with pytest.raises(DescriptorError, match="descriptor_dir is required"): - load_path(tmp_path, descriptor_source="merge", mode="static") - with pytest.raises(DescriptorError, match="basename"): - load_path(tmp_path, descriptor_name="../features.json", mode="static") - with pytest.raises(DiscoveryError, match="basename"): - load_path(tmp_path, structure_name="../CONTCAR", mode="static") - - -def test_canonical_descriptor_roundtrip_and_strict_validation(tmp_path: Path) -> None: +def test_canonical_descriptor_round_trip_and_validation(tmp_path: Path) -> None: document = descriptor_from_dict(_canonical_feature()) - destination = dump_descriptor(document, tmp_path / "features.json") - loaded = load_descriptor(destination) + loaded = load_descriptor(dump_descriptor(document, tmp_path / "features.json")) assert loaded.producer == "critic2" assert loaded.features[0].kind == "bond_critical_point" assert loaded.features[0].neighbor_atom_indices == (0, 1) - assert loaded.features[0].voxel_indices == ((1, 2, 3), (2, 3, 4)) duplicate = _canonical_feature() duplicate["features"] = [duplicate["features"][0], duplicate["features"][0]] # type: ignore[index] with pytest.raises(DescriptorError, match="unique"): descriptor_from_dict(duplicate) - non_finite = _canonical_feature() - non_finite["features"][0]["charge"] = math.inf # type: ignore[index] - with pytest.raises(DescriptorError, match="finite"): - descriptor_from_dict(non_finite) - duplicate_key = tmp_path / "duplicate-key.json" - duplicate_key.write_text( - '{"schema_version":"electrace.features.v1","producer":{"name":"x","name":"y"},"features":[]}', - encoding="utf-8", - ) - with pytest.raises(DescriptorError, match="duplicate object key"): - load_descriptor(duplicate_key) - - unknown_producer = _canonical_feature() - unknown_producer["producer"]["command"] = "critic2 input.cri" # type: ignore[index] - with pytest.raises(DescriptorError, match="producer contains unknown fields: command"): - descriptor_from_dict(unknown_producer) - - unknown_frame = _canonical_feature() - unknown_frame["frame"]["source"] = "00/CONTCAR" # type: ignore[index] - with pytest.raises(DescriptorError, match="frame contains unknown fields: source"): - descriptor_from_dict(unknown_frame) - - missing_convention = _canonical_feature() - missing_convention["features"][0]["charge"] = 0.2 # type: ignore[index] - with pytest.raises(DescriptorError, match=r"metadata\.charge_convention"): - descriptor_from_dict(missing_convention) - - missing_convention["metadata"]["charge_convention"] = ( # type: ignore[index] - "Signed net charge in elementary-charge units." - ) - assert descriptor_from_dict(missing_convention).features[0].charge == pytest.approx(0.2) - - -def test_descriptor_limit_is_enforced_during_read( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - descriptor = (tmp_path / "features.json").resolve() - limit = 64 - descriptor.write_bytes(b"{" + b" " * limit) - real_stat = Path.stat - - def stale_stat(path: Path, *args: Any, **kwargs: Any) -> Any: - result = real_stat(path, *args, **kwargs) - if path == descriptor: - return SimpleNamespace(st_mode=result.st_mode, st_size=limit) - return result - - monkeypatch.setattr(descriptor_module, "MAX_DESCRIPTOR_BYTES", limit) - monkeypatch.setattr(Path, "stat", stale_stat) - - with pytest.raises(DescriptorError, match="safety limit"): - load_descriptor(descriptor) - - -@pytest.mark.parametrize( - "filename", - ["features.json", "ICOHPLIST.lobster", "ACF.dat", "DDEC6_charges.xyz"], -) -def test_descriptor_text_formats_require_utf8(tmp_path: Path, filename: str) -> None: - structure = tmp_path / "POSCAR" - structure.write_text(_poscar(), encoding="utf-8") - frame = read_poscar(structure) - descriptor = tmp_path / filename - descriptor.write_bytes(b"valid-prefix\n\xff\n") - - with pytest.raises(DescriptorError, match="not valid UTF-8"): - load_descriptor(descriptor, frame=frame) - - -def test_deeply_nested_descriptor_json_has_a_bounded_error(tmp_path: Path) -> None: - descriptor = tmp_path / "features.json" - descriptor.write_text("[" * 10_000 + "0" + "]" * 10_000, encoding="utf-8") - - with pytest.raises(DescriptorError, match="cannot parse descriptor JSON"): - load_descriptor(descriptor) - - -@pytest.mark.parametrize("target_exists", [False, True]) -def test_dump_descriptor_never_follows_leaf_symlinks(tmp_path: Path, target_exists: bool) -> None: - target = tmp_path / "outside.json" - if target_exists: - target.write_text("outside\n", encoding="utf-8") - destination = tmp_path / "features.json" - destination.symlink_to(target) - document = descriptor_from_dict(_canonical_feature()) - - with pytest.raises(OSError, match="symbolic link"): - dump_descriptor(document, destination, overwrite=True) - - assert destination.is_symlink() - assert target.exists() is target_exists - if target_exists: - assert target.read_text(encoding="utf-8") == "outside\n" - - -def test_dump_descriptor_is_atomic_on_failure( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - destination = tmp_path / "features.json" - destination.write_text("original\n", encoding="utf-8") - document = descriptor_from_dict(_canonical_feature()) - write_text = Path.write_text - - def fail_temporary_write(path: Path, data: str, **kwargs: Any) -> int: - if path.name.startswith(f".{destination.name}."): - write_text(path, "partial\n", **kwargs) - raise OSError("simulated write failure") - return write_text(path, data, **kwargs) - - monkeypatch.setattr(Path, "write_text", fail_temporary_write) - - with pytest.raises(OSError, match="simulated write failure"): - dump_descriptor(document, destination, overwrite=True) - - assert destination.read_text(encoding="utf-8") == "original\n" - assert list(tmp_path.glob(f".{destination.name}.*.tmp")) == [] - - -def test_charge_convention_schema_matches_runtime_nonwhitespace_rule() -> None: - schema_path = Path(__file__).parents[1] / "schemas" / "electrace.features.v1.schema.json" - schema = json.loads(schema_path.read_text(encoding="utf-8")) - convention_schema = schema["allOf"][0]["then"]["properties"]["metadata"]["properties"][ - "charge_convention" - ] - pattern = convention_schema["pattern"] - - assert re.search(pattern, " ") is None - assert re.search(pattern, "signed net charge") is not None - - payload = _canonical_feature() - payload["features"][0]["charge"] = 0.2 # type: ignore[index] - payload["metadata"]["charge_convention"] = " " # type: ignore[index] - with pytest.raises(DescriptorError, match=r"metadata\.charge_convention"): - descriptor_from_dict(payload) - - -def test_native_baderkit_critic2_lobster_and_chargemol_adapters(tmp_path: Path) -> None: +def test_bader_baderkit_lobster_and_chargemol_adapters(tmp_path: Path) -> None: structure = tmp_path / "POSCAR" structure.write_text(_poscar(x=0.95), encoding="utf-8") frame = read_poscar(structure, label="00") - bader = tmp_path / "bader.json" - bader.write_text( - json.dumps( - { - "maxima_frac": [[0.95, 0.0, 0.0], [0.5, 0.5, 0.5]], - "basin_charges": [5.8, 0.2], - "basin_volumes": [6.0, 0.4], - "basin_atoms": [0, 2], - "atom_charges": [5.8, 7.1], - "atom_volumes": [6.0, 5.5], - } - ), + acf = tmp_path / "ACF.dat" + acf.write_text( + "1 0 0 0 5.8 0 6.0\n2 0 0 0 7.1 0 5.5\n", encoding="utf-8", ) - bader_document = load_descriptor(bader, frame=frame) - assert [feature.kind for feature in bader_document.features] == [ - "atomic_basin", - "non_nuclear_attractor", - "atomic_charge", - "atomic_charge", + assert [feature.charge for feature in load_descriptor(acf, frame=frame).features] == [ + 5.8, + 7.1, ] - critic = tmp_path / "critic2.json" - critic.write_text( + baderkit = tmp_path / "bader.json" + baderkit.write_text( json.dumps( { - "critical_points": [ - { - "id": "cp7", - "type": "BCP", - "frac": [0.99, 0.0, 0.0], - "atoms": [0, 1], - "rho": 0.2, - } - ] - } - ), - encoding="utf-8", - ) - critic_document = load_descriptor(critic, producer="critic2") - assert critic_document.features[0].kind == "bond_critical_point" - assert critic_document.features[0].attributes["rho"] == 0.2 - - lobster = tmp_path / "ICOHPLIST.lobster" - lobster.write_text( - "No. atom1 atom2 distance translation ICOHP\n1 C1 N2 1.20 0 0 0 -4.75\n", - encoding="utf-8", - ) - lobster_feature = load_descriptor(lobster, frame=frame).features[0] - assert lobster_feature.attributes["icohp"] == -4.75 - assert lobster_feature.attributes["icohp_ev"] == -4.75 - assert lobster_feature.attributes["descriptor_family"] == "icohp" - assert lobster_feature.attributes["lattice_translation"] == [0, 0, 0] - # Explicit zero-translation midpoint between fractional x=0.95 and x=1.0. - assert lobster_feature.fractional_position[0] == pytest.approx(0.975) - - chargemol = tmp_path / "DDEC6_even_tempered_net_atomic_charges.xyz" - chargemol.write_text( - "2\nDDEC6 charges\nC 0 0 0 0.30\nN 1 1 1 -0.30\n", - encoding="utf-8", - ) - ddec = load_descriptor(chargemol, frame=frame) - assert [feature.charge for feature in ddec.features] == [0.3, -0.3] - assert all( - feature.attributes["charge_definition"] == "ddec_net_atomic_charge" - for feature in ddec.features - ) - chargemol_json = tmp_path / "chargemol.json" - chargemol_json.write_text( - json.dumps({"atoms": [{"index": 0, "charge": 0.3}, {"index": 1, "charge": -0.3}]}), - encoding="utf-8", - ) - ddec_json = load_descriptor(chargemol_json, frame=frame) - assert all( - feature.attributes["charge_definition"] == "ddec_net_atomic_charge" - for feature in ddec_json.features - ) - - lobster_json = tmp_path / "lobster.json" - lobster_json.write_text( - json.dumps({"bonds": [{"atom1": 1, "atom2": 2, "icohp": -4.75, "icobi": 0.42}]}), - encoding="utf-8", - ) - multi_family = load_descriptor(lobster_json, frame=frame).features[0] - assert multi_family.attributes["descriptor_family"] == "icohp+icobi" - - -@pytest.mark.parametrize( - ("filename", "descriptor", "value"), - [ - ("ICOHPLIST.lobster", "icohp_ev", -4.2), - ("ICOBILIST.lobster", "icobi", 0.31), - ], -) -def test_lobster_native_translation_selects_atom2_image_for_midpoint( - tmp_path: Path, filename: str, descriptor: str, value: float -) -> None: - structure = tmp_path / "POSCAR" - structure.write_text( - _poscar(x=0.1).replace("1.0 1.0 1.0", "0.2 0.0 0.0", 1), - encoding="utf-8", - ) - frame = read_poscar(structure, label="00") - native = tmp_path / filename - native.write_text( - f"No. atom1 atom2 distance translation value\n1 C1 N2 2.20 1 0 0 {value}\n", - encoding="utf-8", - ) - - feature = load_descriptor(native, frame=frame).features[0] - - # Atom 2 is at fractional x=0.2 in cell +1, so the unwrapped midpoint - # with atom 1 at x=0.1 is 0.65. A minimum-image implementation gives 0.15. - assert feature.fractional_position == pytest.approx((0.65, 0.0, 0.0)) - assert feature.attributes[descriptor] == pytest.approx(value) - assert feature.attributes["lattice_translation"] == [1, 0, 0] - assert feature.attributes["midpoint_convention"] == "explicit_atom2_lattice_translation" - - -def test_lobster_simplified_row_keeps_minimum_image_midpoint(tmp_path: Path) -> None: - structure = tmp_path / "POSCAR" - structure.write_text( - _poscar(x=0.1).replace("1.0 1.0 1.0", "0.2 0.0 0.0", 1), - encoding="utf-8", - ) - frame = read_poscar(structure, label="00") - native = tmp_path / "simplified-ICOHPLIST.lobster" - native.write_text("1 C1 N2 0.20 -1.5\n", encoding="utf-8") - - feature = load_descriptor(native, frame=frame).features[0] - - assert feature.fractional_position == pytest.approx((0.15, 0.0, 0.0)) - assert feature.attributes["midpoint_convention"] == "minimum_image" - assert "lattice_translation" not in feature.attributes - - -@pytest.mark.parametrize( - ("row", "message"), - [ - ("2 C1 N2 2.20 1 0 -4.2", "7 columns"), - ("2 C1 N2 2.20 1.0 0 0 -4.2", "lattice translation"), - ("2 C1 N2 nan 1 0 0 -4.2", "distance must be finite"), - ("2 C1 N2 2.20 1 0 0 nan", "ICOHP must be finite"), - ], -) -def test_lobster_native_malformed_record_is_not_silently_dropped( - tmp_path: Path, row: str, message: str -) -> None: - structure = tmp_path / "POSCAR" - structure.write_text(_poscar(x=0.1), encoding="utf-8") - frame = read_poscar(structure, label="00") - native = tmp_path / "ICOHPLIST.lobster" - native.write_text( - f"No. atom1 atom2 distance translation ICOHP\n1 C1 N2 1.80 0 0 0 -1.0\n{row}\n", - encoding="utf-8", - ) - - with pytest.raises(DescriptorError, match=message): - load_descriptor(native, frame=frame) - - -def test_lobster_json_translation_uses_same_periodic_image_contract(tmp_path: Path) -> None: - structure = tmp_path / "POSCAR" - structure.write_text( - _poscar(x=0.1).replace("1.0 1.0 1.0", "0.2 0.0 0.0", 1), - encoding="utf-8", - ) - frame = read_poscar(structure, label="00") - lobster_json = tmp_path / "lobster.json" - lobster_json.write_text( - json.dumps( - {"bonds": [{"atom1": 1, "atom2": 2, "icohp": -2.0, "cell_translation": [1, 0, 0]}]} - ), - encoding="utf-8", - ) - - feature = load_descriptor(lobster_json, frame=frame).features[0] - - assert feature.fractional_position == pytest.approx((0.65, 0.0, 0.0)) - assert feature.attributes["lattice_translation"] == [1, 0, 0] - - lobster_json.write_text( - json.dumps( - {"bonds": [{"atom1": 1, "atom2": 2, "icohp": -2.0, "translation": [1.0, 0, 0]}]} - ), - encoding="utf-8", - ) - with pytest.raises(DescriptorError, match="translation must contain exactly three integers"): - load_descriptor(lobster_json, frame=frame) - - -def test_official_baderkit_bader_json_nested_results(tmp_path: Path) -> None: - structure = tmp_path / "POSCAR" - structure.write_text(_poscar(x=0.95), encoding="utf-8") - frame = read_poscar(structure, label="00") - - bader = tmp_path / "bader.json" - bader.write_text( - json.dumps( - { - "method_kwargs": {"version": "0.10.0", "method": "neargrid"}, "atom_results": { + "basin_atoms": [0, 2], "atom_charges": [5.8, 7.1], "atom_volumes": [6.0, 5.5], - "oxidation_states": [0.2, -0.1], - "basin_atoms": [0, 2], }, "maxima_results": { "maxima_frac": [[0.95, 0.0, 0.0], [0.5, 0.5, 0.5]], "basin_charges": [5.8, 0.2], "basin_volumes": [6.0, 0.4], }, - "base_summary_props": {"spin_system": "total"}, } ), encoding="utf-8", ) - - document = load_descriptor(bader, frame=frame) - - assert document.producer_version == "0.10.0" - assert document.metadata["native_format"] == "bader.json" - assert [feature.kind for feature in document.features] == [ + bader_features = load_descriptor(baderkit, frame=frame).features + assert [feature.kind for feature in bader_features] == [ "atomic_basin", "non_nuclear_attractor", "atomic_charge", "atomic_charge", ] - assert document.features[0].neighbor_atom_indices == (0,) - assert document.features[1].neighbor_atom_indices == () - assert [feature.charge for feature in document.features[2:]] == [5.8, 7.1] - assert all( - feature.attributes["charge_definition"] == "bader_electron_population" - for feature in document.features[2:] - ) - assert [feature.attributes["oxidation_state"] for feature in document.features[2:]] == [ - 0.2, - -0.1, - ] - oxidation_only = tmp_path / "renamed-bader-results.json" - oxidation_only.write_text( - json.dumps( - { - "atom_results": {"oxidation_states": [1.25, -1.25]}, - "maxima_results": {}, - } - ), + lobster = tmp_path / "ICOHPLIST.lobster" + lobster.write_text( + "No. atom1 atom2 distance translation ICOHP\n1 C1 N2 1.20 0 0 0 -4.75\n", encoding="utf-8", ) - oxidation_document = load_descriptor(oxidation_only, frame=frame) - assert [feature.charge for feature in oxidation_document.features] == [1.25, -1.25] - assert all( - feature.attributes["charge_definition"] == "bader_oxidation_state" - for feature in oxidation_document.features + bond = load_descriptor(lobster, frame=frame).features[0] + assert bond.attributes["icohp"] == -4.75 + assert bond.neighbor_atom_indices == (0, 1) + assert bond.attributes["lattice_translation"] == [0, 0, 0] + + icobi = tmp_path / "ICOBILIST.lobster" + icobi.write_text("1 C1 N2 1.20 0.42\n", encoding="utf-8") + assert load_descriptor(icobi, frame=frame).features[0].attributes["icobi"] == 0.42 + + chargemol = tmp_path / "DDEC6_even_tempered_net_atomic_charges.xyz" + chargemol.write_text( + "2\nDDEC6 charges\nC 0 0 0 0.30\nN 1 1 1 -0.30\n", + encoding="utf-8", ) - assert oxidation_document.metadata["native_format"] == "renamed-bader-results.json" + ddec = load_descriptor(chargemol, frame=frame) + assert [feature.charge for feature in ddec.features] == [0.3, -0.3] -def test_official_baderkit_badelf_json_maps_appended_nna_sites(tmp_path: Path) -> None: +def test_badelf_maps_appended_nna_site(tmp_path: Path) -> None: structure = tmp_path / "POSCAR" structure.write_text(_poscar(x=0.95), encoding="utf-8") frame = read_poscar(structure, label="00") - - badelf = tmp_path / "badelf.json" - badelf.write_text( + path = tmp_path / "badelf.json" + path.write_text( json.dumps( { - "method_kwargs": {"version": "0.10.0", "partition_method": "badelf"}, + "method_kwargs": {"version": "0.10.0"}, "atom_results": { "atom_charges": [5.7, 7.0, 0.65], "atom_volumes": [5.9, 5.4, 0.7], - "oxidation_states": [0.3, 0.0, -0.65], - "species": ["C", "N", "Xmc"], - "maxima_elf_values": [0.81, 0.88, 0.96], - "total_electron_number": 13.35, + "species": ["C", "N", "X"], }, "nna_results": { - "nna_dimensionality": 2, "num_nnas": 1, - "nnas_per_formula": 0.65, "nna_structure": { - "@module": "pymatgen.core.structure", - "@class": "Structure", "sites": [ - {"species": [{"element": "C", "occu": 1}], "abc": [0.95, 0, 0]}, - {"species": [{"element": "N", "occu": 1}], "abc": [0, 0, 0]}, - { - "species": [{"element": "Xmc", "occu": 1}], - "abc": [0.5, 0.5, 0.5], - }, - ], + {"abc": [0.95, 0, 0]}, + {"abc": [0, 0, 0]}, + {"abc": [0.5, 0.5, 0.5]}, + ] }, }, - "base_summary_props": {"spin_system": "total"}, } ), encoding="utf-8", ) - document = load_descriptor(badelf, frame=frame) - - assert document.producer_version == "0.10.0" - assert document.metadata["native_format"] == "badelf.json" - assert [feature.kind for feature in document.features] == [ - "atomic_charge", - "atomic_charge", - "non_nuclear_attractor", - ] + document = load_descriptor(path, frame=frame) nna = document.features[-1] - assert nna.id == "baderkit:nna:0" + assert nna.kind == "non_nuclear_attractor" assert nna.fractional_position == pytest.approx((0.5, 0.5, 0.5)) - assert nna.neighbor_atom_indices == () assert nna.charge == pytest.approx(0.65) - assert nna.volume == pytest.approx(0.7) - assert nna.attributes["charge_definition"] == "bader_electron_population" - assert nna.attributes["species"] == "Xmc" - assert nna.attributes["maxima_elf_value"] == pytest.approx(0.96) - assert nna.attributes["oxidation_state"] == pytest.approx(-0.65) -def test_critic2_native_cpreport_json_is_joined_conservatively(tmp_path: Path) -> None: - report = tmp_path / "critic2.features.json" - report.write_text( +def test_critic2_cpreport_joins_topology_and_position(tmp_path: Path) -> None: + path = tmp_path / "critic2.features.json" + path.write_text( json.dumps( { - "units": "bohr", - "metadata": { - "charge_convention": ( - "critic2 critical-point integrated population in elementary-charge units" - ) - }, "structure": {"number_of_cell_atoms": 2}, - "field": {"id": 1, "name": "rho", "source": "CHGCAR"}, "critical_points": { - "number_of_nonequivalent_cps": 4, "nonequivalent_cps": [ - {"id": 1, "signature": -3, "is_nucleus": True}, - {"id": 2, "signature": -3, "is_nucleus": True}, { "id": 3, - "name": "b1", - "rank": 3, "signature": -1, "is_nucleus": False, "field": 0.2, - "charge": 0.25, - "volume": 1.5, - "hessian": [1.0, 0.0, 0.0, 0.0, 2.0, 0.0, 0.0, 0.0, -4.0], - }, - { - "id": 4, - "name": "nna1", - "rank": 3, - "signature": -3, - "is_nucleus": False, - "field": 0.1, - "volume": 0.5, - }, + "hessian": [1, 0, 0, 0, 2, 0, 0, 0, -4], + } ], - "number_of_cell_cps": 4, "cell_cps": [ - { - "id": 1, - "signature": -3, - "nonequivalent_id": 1, - "fractional_coordinates": [0.0, 0.0, 0.0], - }, - { - "id": 2, - "signature": -3, - "nonequivalent_id": 2, - "fractional_coordinates": [0.5, 0.5, 0.5], - }, { "id": 3, - "signature": -1, "nonequivalent_id": 3, "fractional_coordinates": [0.25, 0.25, 0.25], "attractors": [{"cell_id": 1}, {"cell_id": 2}], - }, - { - "id": 4, - "signature": -3, - "nonequivalent_id": 4, - "fractional_coordinates": [0.75, 0.75, 0.75], - "volume": 0.75, - }, + } ], }, } @@ -868,120 +251,25 @@ def test_critic2_native_cpreport_json_is_joined_conservatively(tmp_path: Path) - encoding="utf-8", ) - document = load_descriptor(report) - - assert [feature.id for feature in document.features] == ["critic2:cp:3", "critic2:cp:4"] - assert [feature.kind for feature in document.features] == [ - "bond_critical_point", - "non_nuclear_attractor", - ] - assert document.features[0].neighbor_atom_indices == (0, 1) - assert document.features[0].charge == pytest.approx(0.25) - assert document.features[0].volume == pytest.approx(1.5) - # The explicit cell value overrides the nonequivalent definition's 0.5. - assert document.features[1].volume == pytest.approx(0.75) - assert document.features[0].attributes["field"] == pytest.approx(0.2) - assert document.features[0].attributes["laplacian"] == pytest.approx(-1.0) - assert document.features[0].attributes["hessian"] == [ - 1.0, - 0.0, - 0.0, - 0.0, - 2.0, - 0.0, - 0.0, - 0.0, - -4.0, - ] - assert document.metadata == { - "native_format": "cpreport.json", - "units": "bohr", - "charge_convention": ( - "critic2 critical-point integrated population in elementary-charge units" - ), - } - - malformed = tmp_path / "critic2-malformed.json" - malformed.write_text( - json.dumps({"critical_points": {"cell_cps": "not-an-array"}}), - encoding="utf-8", - ) - with pytest.raises(DescriptorError, match="CPREPORT JSON must contain"): - load_descriptor(malformed) + feature = load_descriptor(path).features[0] + assert feature.kind == "bond_critical_point" + assert feature.neighbor_atom_indices == (0, 1) + assert feature.attributes["field"] == pytest.approx(0.2) + assert feature.attributes["laplacian"] == pytest.approx(-1.0) -def test_load_path_combines_structure_energy_and_external_descriptors(tmp_path: Path) -> None: +def test_load_path_combines_frames_energy_and_external_descriptors(tmp_path: Path) -> None: root = tmp_path / "neb" descriptors = tmp_path / "descriptors" - _write_frame(root / "00", x=0.0, energy=-10.0) + _write_frame(root / "00", energy=-10.0) _write_frame(root / "01", x=0.1, energy=-9.0) descriptors.mkdir() for index, label in enumerate(("00", "01")): payload = _canonical_feature(frame_index=index, frame_label=label) payload["features"][0]["id"] = f"bcp-{index}" # type: ignore[index] - if index == 0: - payload["metadata"]["warnings"] = ["grid convergence was marginal"] # type: ignore[index] (descriptors / f"{label}.json").write_text(json.dumps(payload), encoding="utf-8") frames = load_path(root, descriptor_dir=descriptors) - assert [frame.index for frame in frames] == [0, 1] assert [frame.energy for frame in frames] == [-10.0, -9.0] assert [frame.features[0].id for frame in frames] == ["bcp-0", "bcp-1"] - assert frames[0].metadata["series_kind"] == "neb" - assert frames[0].metadata["descriptor_producers"] == ["critic2"] - assert frames[0].metadata["descriptor_documents"] == [ - { - "schema_version": "electrace.features.v1", - "producer": {"name": "critic2", "version": "1.2"}, - "metadata": { - "grid": "fine", - "warnings": ["grid convergence was marginal"], - }, - } - ] - assert frames[0].features[0].attributes["descriptor_producer"] == "critic2" - - -def test_load_path_rejects_descriptor_frame_mismatch(tmp_path: Path) -> None: - root = tmp_path / "series" - _write_frame(root / "only") - descriptor = tmp_path / "features.json" - descriptor.write_text( - json.dumps(_canonical_feature(frame_index=2, frame_label="other")), encoding="utf-8" - ) - with pytest.raises(DescriptorError, match="does not match"): - load_path(root, descriptor_dir=descriptor, mode="static") - - -def test_load_path_rejects_conflicting_per_feature_producer_tag(tmp_path: Path) -> None: - root = tmp_path / "series" - _write_frame(root / "only") - payload = _canonical_feature(frame_label="only") - payload["features"][0]["attributes"]["descriptor_producer"] = "lobster" # type: ignore[index] - (root / "only" / "features.json").write_text(json.dumps(payload), encoding="utf-8") - - with pytest.raises(DescriptorError, match="conflicting with document producer"): - load_path(root, mode="static") - - -def test_load_path_propagates_charge_convention_and_rejects_conflict( - tmp_path: Path, -) -> None: - root = tmp_path / "series" - _write_frame(root / "only") - payload = _canonical_feature(frame_label="only") - feature = payload["features"][0] # type: ignore[index] - feature["charge"] = 0.2 # type: ignore[index] - payload["metadata"]["charge_convention"] = "Signed net feature charge." # type: ignore[index] - descriptor = root / "only" / "features.json" - descriptor.write_text(json.dumps(payload), encoding="utf-8") - - loaded = load_path(root, mode="static") - - assert loaded[0].features[0].attributes["charge_convention"] == "Signed net feature charge." - - feature["attributes"]["charge_convention"] = "Integrated electron population." # type: ignore[index] - descriptor.write_text(json.dumps(payload), encoding="utf-8") - with pytest.raises(DescriptorError, match="conflicting with document convention"): - load_path(root, mode="static") diff --git a/tests/test_matching.py b/tests/test_matching.py index 4c590e4..321c191 100644 --- a/tests/test_matching.py +++ b/tests/test_matching.py @@ -3,14 +3,7 @@ import numpy as np import pytest -from electrace.matching import ( - FeatureMatchConfig, - _assignment_ambiguities, - _hungarian_fallback, - match_atoms, - match_features, - solve_assignment, -) +from electrace.matching import FeatureMatchConfig, _hungarian_fallback, match_atoms, match_features from electrace.models import Atom, Feature, Frame, MatchRelation CELL = ((10.0, 0.0, 0.0), (0.0, 10.0, 0.0), (0.0, 0.0, 10.0)) @@ -27,146 +20,12 @@ def frame( def test_hungarian_fallback_finds_global_rectangular_optimum() -> None: - costs = np.array([[4.0, 1.0, 3.0], [2.0, 0.0, 5.0]]) - rows, columns = _hungarian_fallback(costs) + rows, columns = _hungarian_fallback(np.array([[4.0, 1.0, 3.0], [2.0, 0.0, 5.0]])) assert list(rows) == [0, 1] assert list(columns) == [1, 0] - scipy_rows, scipy_columns = solve_assignment(costs) - assert costs[rows, columns].sum() == costs[scipy_rows, scipy_columns].sum() - - -@pytest.mark.parametrize("value", [-0.1, 1.1, float("nan")]) -def test_conservation_tolerance_is_a_probability_like_fraction(value: float) -> None: - with pytest.raises(ValueError, match="conservation_tolerance"): - FeatureMatchConfig(conservation_tolerance=value) - - -def test_local_tie_is_not_reported_when_global_assignment_resolves_it() -> None: - costs = np.array([[0.0, 0.0], [0.0, 10.0]]) - - ambiguities, ambiguous_pairs = _assignment_ambiguities( - costs, - 100.0, - [(0, 1), (1, 0)], - set(), - set(), - 0.0, - ["source-a", "source-b"], - ["target-a", "target-b"], - 0.05, - ) - - assert ambiguities == [] - assert ambiguous_pairs == {} - -def test_exact_feature_tie_is_auditable_and_reorder_safe() -> None: - previous = frame( - 0, - features=(Feature("source", "bond_critical_point", (0.5, 0.5, 0.5)),), - ) - targets = ( - Feature("a", "bond_critical_point", (0.5, 0.5, 0.5)), - Feature("b", "bond_critical_point", (0.5, 0.5, 0.5)), - ) - - results = [ - match_features(previous, frame(1, features=ordered)) - for ordered in (targets, tuple(reversed(targets))) - ] - assert all(result.mapping == {} for result in results) - assert all(result.diagnostics["ambiguous"] is True for result in results) - assert all(result.edges[0].ambiguous for result in results) - assert all( - { - str(item) - for record in result.diagnostics["ambiguities"] - for item in record["selected"].values() - } - == {"source", "a", "b"} - for result in results - ) - - -def test_near_tie_uses_configured_complete_assignment_gap() -> None: - previous = frame( - 0, - features=(Feature("source", "bond_critical_point", (0.5, 0.5, 0.5)),), - ) - current = frame( - 1, - features=( - Feature("best", "bond_critical_point", (0.5, 0.5, 0.5)), - Feature("near", "bond_critical_point", (0.5001, 0.5, 0.5)), - ), - ) - - ambiguous = match_features( - previous, - current, - config=FeatureMatchConfig(ambiguity_tolerance=0.002), - ) - separated = match_features( - previous, - current, - config=FeatureMatchConfig(ambiguity_tolerance=0.001), - ) - - assert ambiguous.diagnostics["ambiguous"] is True - assert ambiguous.diagnostics["ambiguities"][0]["assignment_cost_gap"] == pytest.approx( - 0.0013333333333333333 - ) - assert separated.diagnostics["ambiguous"] is False - assert separated.mapping == {"source": "best"} - - -def test_exact_atom_tie_is_excluded_from_unambiguous_mapping() -> None: - previous = frame( - 0, - atoms=( - Atom(0, "C", (0.5, 0.5, 0.5)), - Atom(1, "C", (0.5, 0.5, 0.5)), - ), - ) - targets = ( - Atom(10, "C", (0.5, 0.5, 0.5)), - Atom(11, "C", (0.5, 0.5, 0.5)), - ) - - results = [ - match_atoms(previous, frame(1, atoms=ordered)) - for ordered in (targets, tuple(reversed(targets))) - ] - - assert all(result.mapping == {} for result in results) - assert all(result.diagnostics["ambiguous"] is True for result in results) - assert all(len(result.diagnostics["ambiguities"]) == 2 for result in results) - assert all(edge.ambiguous for result in results for edge in result.edges) - - -def test_symmetric_backbone_tie_is_resolved_by_unique_conserving_hyperedge() -> None: - previous = frame( - 0, - features=(Feature("parent", "elf_basin", (0.5, 0.5, 0.5), volume=10.0),), - ) - current = frame( - 1, - features=( - Feature("left", "elf_basin", (0.48, 0.5, 0.5), volume=5.0), - Feature("right", "elf_basin", (0.52, 0.5, 0.5), volume=5.0), - ), - ) - - result = match_features(previous, current) - - assert result.diagnostics["ambiguous"] is False - assert result.diagnostics["ambiguities"] == [] - assert result.edges[0].relation is MatchRelation.SPLIT - assert len(result.edges[0].evidence["resolved_assignment_ambiguities"]) == 2 - - -def test_atom_matching_is_periodic_element_aware_and_variable_cell_safe() -> None: +def test_atom_matching_is_periodic_element_aware_across_changing_cells() -> None: previous = frame( 0, atoms=( @@ -182,34 +41,11 @@ def test_atom_matching_is_periodic_element_aware_and_variable_cell_safe() -> Non ), cell=((10.2, 0.0, 0.0), (0.0, 10.2, 0.0), (0.0, 0.0, 10.2)), ) + result = match_atoms(previous, current) + assert result.mapping == {10: 1, 20: 2} assert not result.unmatched_previous - assert all(edge.evidence["element_match"] for edge in result.edges) - - -def test_atom_matching_reuses_one_pairwise_geometry_factorization( - monkeypatch: pytest.MonkeyPatch, -) -> None: - atoms = tuple(Atom(index, "C", (index / 20.0, 0.2, 0.3)) for index in range(8)) - shifted = tuple( - Atom(index + 20, "C", ((index / 20.0 + 0.01) % 1.0, 0.2, 0.3)) for index in range(8) - ) - real_svd = np.linalg.svd - calls = 0 - - def counted_svd(*args: object, **kwargs: object) -> object: - nonlocal calls - calls += 1 - return real_svd(*args, **kwargs) - - monkeypatch.setattr(np.linalg, "svd", counted_svd) - result = match_atoms(frame(0, atoms=atoms), frame(1, atoms=shifted)) - - assert len(result.edges) == len(atoms) - # Two endpoint validations plus one validation of the mean-metric cell; - # the count is independent of the 8x8 candidate-pair count. - assert calls == 3 def test_feature_matching_uses_atom_neighbor_mapping() -> None: @@ -223,322 +59,45 @@ def test_feature_matching_uses_atom_neighbor_mapping() -> None: atoms=(Atom(10, "N", (0.8, 0.2, 0.2)), Atom(11, "C", (0.22, 0.2, 0.2))), features=(Feature("new", "elf_basin", (0.27, 0.2, 0.2), 1.02, 2.05, (11,)),), ) - atoms = match_atoms(previous, current) - result = match_features(previous, current, atoms) - assert result.mapping == {"old": "new"} - assert result.edges[0].evidence["neighbor_similarity"] == 1.0 - - -def test_invalid_grid_shape_iterables_are_consumed_boundedly() -> None: - grid_shape = iter(range(1, 100)) - previous = frame( - 0, - features=( - Feature( - "old", - "elf_basin", - (0.5, 0.5, 0.5), - attributes={"grid_shape": grid_shape}, - voxel_indices=((0, 0, 0),), - ), - ), - ) - current = frame( - 1, - features=( - Feature( - "new", - "elf_basin", - (0.5, 0.5, 0.5), - attributes={"grid_shape": (4, 4, 4)}, - voxel_indices=((0, 0, 0),), - ), - ), - ) - result = match_features(previous, current) + result = match_features(previous, current, match_atoms(previous, current)) assert result.mapping == {"old": "new"} - assert "voxel_similarity" not in result.edges[0].evidence - assert next(grid_shape) == 5 + assert result.edges[0].evidence["neighbor_similarity"] == 1.0 -def test_split_is_promoted_to_a_conservation_backed_hyperedge() -> None: +def test_split_requires_complete_conservation_evidence() -> None: previous = frame( 0, - features=(Feature("parent", "elf_basin", (0.5, 0.5, 0.5), 1.0, 10.0),), + features=(Feature("parent", "elf_basin", (0.5, 0.5, 0.5), volume=10.0),), ) - current = frame( + conserving = frame( 1, features=( - Feature("left", "elf_basin", (0.48, 0.5, 0.5), 0.4, 4.0), - Feature("right", "elf_basin", (0.52, 0.5, 0.5), 0.6, 6.0), + Feature("left", "elf_basin", (0.48, 0.5, 0.5), volume=4.0), + Feature("right", "elf_basin", (0.52, 0.5, 0.5), volume=6.0), ), ) - result = match_features(previous, current) - assert len(result.edges) == 1 - edge = result.edges[0] - assert edge.relation is MatchRelation.SPLIT - assert edge.source_ids == ("parent",) - assert set(edge.target_ids) == {"left", "right"} - assert edge.evidence["conservation"]["volume_relative_error"] == pytest.approx(0) - assert edge.confidence > 0.5 - assert not result.unmatched_current - - -def test_merge_is_promoted_to_a_conservation_backed_hyperedge() -> None: - previous = frame( - 0, - features=( - Feature("left", "elf_basin", (0.48, 0.5, 0.5), 0.4, 4.0), - Feature("right", "elf_basin", (0.52, 0.5, 0.5), 0.6, 6.0), - ), - ) - current = frame( + missing = frame( 1, - features=(Feature("child", "elf_basin", (0.5, 0.5, 0.5), 1.0, 10.0),), - ) - result = match_features(previous, current) - assert result.edges[0].relation is MatchRelation.MERGE - assert set(result.edges[0].source_ids) == {"left", "right"} - assert result.edges[0].target_ids == ("child",) - - -@pytest.mark.parametrize("relation", ["split", "merge"]) -@pytest.mark.parametrize( - ("case", "parent_values", "child_values", "accepted", "available", "failed"), - [ - ("no evidence", {}, ({}, {}), False, (), ()), - ( - "partial volume", - {"volume": 10.0}, - ({"volume": 4.0}, {}), - False, - (), - (), - ), - ( - "charge only", - {"charge": 1.0}, - ({"charge": 0.4}, {"charge": 0.6}), - True, - ("charge",), - (), - ), - ( - "volume only", - {"volume": 10.0}, - ({"volume": 4.0}, {"volume": 6.0}), - True, - ("volume",), - (), - ), - ( - "charge and volume", - {"charge": 1.0, "volume": 10.0}, - ( - {"charge": 0.4, "volume": 4.0}, - {"charge": 0.6, "volume": 6.0}, - ), - True, - ("charge", "volume"), - (), - ), - ( - "tolerance boundary", - {"volume": 10.0}, - ({"volume": 3.0}, {"volume": 3.5}), - True, - ("volume",), - (), - ), - ( - "one failed channel", - {"charge": 1.0, "volume": 10.0}, - ( - {"charge": 0.4, "volume": 2.0}, - {"charge": 0.6, "volume": 2.0}, - ), - False, - ("charge", "volume"), - ("volume",), - ), - ], -) -def test_hyperedges_require_a_complete_conserving_channel( - relation: str, - case: str, - parent_values: dict[str, float], - child_values: tuple[dict[str, float], dict[str, float]], - accepted: bool, - available: tuple[str, ...], - failed: tuple[str, ...], -) -> None: - parent = Feature("parent", "elf_basin", (0.5, 0.5, 0.5), **parent_values) - children = ( - Feature("left", "elf_basin", (0.48, 0.5, 0.5), **child_values[0]), - Feature("right", "elf_basin", (0.53, 0.5, 0.5), **child_values[1]), - ) - previous_features = (parent,) if relation == "split" else children - current_features = children if relation == "split" else (parent,) - - result = match_features( - frame(0, features=previous_features), - frame(1, features=current_features), - config=FeatureMatchConfig(ambiguity_tolerance=0.0), - ) - - hyperedges = [edge for edge in result.edges if edge.relation is not MatchRelation.ONE_TO_ONE] - if accepted: - assert len(hyperedges) == 1, case - conservation = hyperedges[0].evidence["conservation"] - assert tuple(conservation["available_channels"]) == available - assert tuple(conservation["failed_channels"]) == failed - assert not result.diagnostics["hyperedge_rejections"] - else: - assert not hyperedges, case - rejection = result.diagnostics["hyperedge_rejections"][0] - assert rejection["relation"] == relation - conservation = rejection["conservation"] - assert tuple(conservation["available_channels"]) == available - assert tuple(conservation["failed_channels"]) == failed - expected_reason = ( - "missing_complete_conservation_channel" - if not available - else "conservation_tolerance_exceeded" - ) - assert rejection["reason"] == expected_reason - - -@pytest.mark.parametrize("relation", ["split", "merge"]) -@pytest.mark.parametrize("with_volume", [False, True]) -def test_hyperedge_charge_conservation_requires_compatible_semantics( - relation: str, - with_volume: bool, -) -> None: - parent = Feature( - "parent", - "elf_basin", - (0.5, 0.5, 0.5), - charge=1.0, - volume=10.0 if with_volume else None, - ) - children = ( - Feature( - "left", - "elf_basin", - (0.48, 0.5, 0.5), - charge=0.4, - volume=4.0 if with_volume else None, - attributes={"charge_convention": "electron population"}, - ), - Feature( - "right", - "elf_basin", - (0.52, 0.5, 0.5), - charge=0.6, - volume=6.0 if with_volume else None, - attributes={"charge_convention": "signed net charge"}, + features=( + Feature("left", "elf_basin", (0.48, 0.5, 0.5)), + Feature("right", "elf_basin", (0.52, 0.5, 0.5)), ), ) - previous_features = (parent,) if relation == "split" else children - current_features = children if relation == "split" else (parent,) - - result = match_features( - frame(0, features=previous_features), - frame(1, features=current_features), - ) - - hyperedges = [edge for edge in result.edges if edge.relation is not MatchRelation.ONE_TO_ONE] - if with_volume: - assert len(hyperedges) == 1 - conservation = hyperedges[0].evidence["conservation"] - assert conservation["available_channels"] == ["volume"] - assert conservation["incompatible_channels"] == ["charge"] - else: - assert not hyperedges - rejection = result.diagnostics["hyperedge_rejections"][0] - assert rejection["reason"] == "incompatible_conservation_semantics" - assert rejection["conservation"]["incompatible_channels"] == ["charge"] - - -def test_conservation_backed_split_is_periodic_and_input_order_invariant() -> None: - parent = Feature("parent", "elf_basin", (0.99, 0.5, 0.5), volume=10.0) - left = Feature("left", "elf_basin", (0.97, 0.5, 0.5), volume=4.0) - wrapped = Feature("wrapped", "elf_basin", (0.01, 0.5, 0.5), volume=6.0) - - results = [ - match_features( - frame(0, features=(parent,)), - frame(1, features=children), - ) - for children in ((left, wrapped), (wrapped, left)) - ] - assert all(result.edges[0].relation is MatchRelation.SPLIT for result in results) - assert all(set(result.edges[0].target_ids) == {"left", "wrapped"} for result in results) - assert results[0].edges[0].confidence == pytest.approx(results[1].edges[0].confidence) + accepted = match_features(previous, conserving) + rejected = match_features(previous, missing) - -def test_unknown_kinds_are_allowed_but_do_not_cross_match_by_default() -> None: - previous = frame(0, features=(Feature("a", "plugin_custom", (0.2, 0.2, 0.2)),)) - current = frame(1, features=(Feature("b", "other_custom", (0.2, 0.2, 0.2)),)) - result = match_features(previous, current) - assert result.unmatched_previous == ("a",) - assert result.unmatched_current == ("b",) - permissive = match_features( - previous, - current, - config=FeatureMatchConfig(allow_kind_mismatch=True), - ) - # Permissive matching makes the mismatch finite and explicitly maps it. - assert permissive.mapping == {"a": "b"} - - -def test_configured_descriptor_scales_disambiguate_feature_identity() -> None: - previous = frame( - 0, - features=( - Feature( - "source", - "bond_critical_point", - (0.5, 0.5, 0.5), - attributes={"icohp": -1.0}, - ), - ), - ) - current = frame( - 1, - features=( - Feature( - "wrong", - "bond_critical_point", - (0.5, 0.5, 0.5), - attributes={"icohp": 1.0}, - ), - Feature( - "right", - "bond_critical_point", - (0.5, 0.5, 0.5), - attributes={"icohp": -1.0}, - ), - ), + assert accepted.edges[0].relation is MatchRelation.SPLIT + assert accepted.edges[0].evidence["conservation"]["volume_relative_error"] == 0.0 + assert all(edge.relation is MatchRelation.ONE_TO_ONE for edge in rejected.edges) + assert rejected.diagnostics["hyperedge_rejections"][0]["reason"] == ( + "missing_complete_conservation_channel" ) - result = match_features( - previous, - current, - config=FeatureMatchConfig( - descriptor_scales={"icohp": 0.5}, - descriptor_weight=1.0, - detect_splits_merges=False, - ), - ) - - assert result.mapping == {"source": "right"} - assert result.edges[0].evidence["descriptor_normalized_deltas"] == {"icohp": 0.0} -def test_incompatible_lobster_families_cannot_cross_match_when_colocated() -> None: +def test_incompatible_lobster_families_do_not_cross_match() -> None: previous = frame( 0, features=( @@ -546,26 +105,16 @@ def test_incompatible_lobster_families_cannot_cross_match_when_colocated() -> No "p-hp", "bond_descriptor", (0.5, 0.5, 0.5), - attributes={ - "descriptor_family": "ICOHP", - "descriptor_producer": "LOBSTER", - "icohp": -1.0, - }, + attributes={"descriptor_family": "ICOHP", "icohp": -1.0}, ), Feature( "p-bi", "bond_descriptor", (0.5, 0.5, 0.5), - attributes={ - "descriptor_family": "ICOBI", - "descriptor_producer": "LOBSTER", - "icobi": 0.2, - }, + attributes={"descriptor_family": "ICOBI", "icobi": 0.2}, ), ), ) - # Put each incompatible family first. The former missing-key behavior - # assigned both cross-family pairs at zero cost. current = frame( 1, features=( @@ -573,21 +122,13 @@ def test_incompatible_lobster_families_cannot_cross_match_when_colocated() -> No "q-bi", "bond_descriptor", (0.5, 0.5, 0.5), - attributes={ - "descriptor_family": "icobi", - "descriptor_producer": "lobster", - "icobi": 0.3, - }, + attributes={"descriptor_family": "icobi", "icobi": 0.3}, ), Feature( "q-hp", "bond_descriptor", (0.5, 0.5, 0.5), - attributes={ - "descriptor_family": "icohp", - "descriptor_producer": "lobster", - "icohp": -1.5, - }, + attributes={"descriptor_family": "icohp", "icohp": -1.5}, ), ), ) @@ -604,279 +145,31 @@ def test_incompatible_lobster_families_cannot_cross_match_when_colocated() -> No assert result.mapping == {"p-hp": "q-hp", "p-bi": "q-bi"} assert result.diagnostics["semantic_rejections"] == 2 - assert all(edge.evidence["semantic_compatibility"]["compatible"] for edge in result.edges) - - -def test_multifamily_descriptor_declaration_matches_on_shared_family() -> None: - result = match_features( - frame( - 0, - features=( - Feature( - "multi", - "bond_descriptor", - (0.5, 0.5, 0.5), - attributes={ - "descriptor_family": "ICOHP+ICOBI", - "descriptor_producer": "LOBSTER", - "icohp": -1.0, - "icobi": 0.2, - }, - ), - ), - ), - frame( - 1, - features=( - Feature( - "bi", - "bond_descriptor", - (0.5, 0.5, 0.5), - attributes={ - "descriptor_family": "icobi", - "descriptor_producer": "lobster", - "icobi": 0.3, - }, - ), - ), - ), - config=FeatureMatchConfig(descriptor_scales={"icohp": 0.5, "icobi": 0.1}), - ) - - assert result.mapping == {"multi": "bi"} - assert result.diagnostics["semantic_rejections"] == 0 - - -def test_disjoint_configured_key_families_gate_untagged_legacy_features() -> None: - previous = frame( - 0, - features=( - Feature( - "p-hp", - "bond_descriptor", - (0.5, 0.5, 0.5), - attributes={"icohp_ev": -1.0}, - ), - Feature( - "p-bi", - "bond_descriptor", - (0.5, 0.5, 0.5), - attributes={"icobi": 0.2}, - ), - ), - ) - current = frame( - 1, - features=( - Feature( - "q-bi", - "bond_descriptor", - (0.5, 0.5, 0.5), - attributes={"icobi": 0.3}, - ), - Feature( - "q-hp", - "bond_descriptor", - (0.5, 0.5, 0.5), - attributes={"icohp": -1.5}, - ), - ), - ) - - result = match_features( - previous, - current, - config=FeatureMatchConfig( - descriptor_scales={"icohp": 0.5, "icohp_ev": 0.5, "icobi": 0.1}, - descriptor_weight=1.0, - detect_splits_merges=False, - ), - ) - - assert result.mapping == {"p-hp": "q-hp", "p-bi": "q-bi"} -def test_incompatible_charge_definitions_are_forbidden() -> None: - previous = frame( - 0, - features=( - Feature( - "ddec", - "atomic_charge", - (0.5, 0.5, 0.5), - charge=0.2, - attributes={"charge_definition": "DDEC net atomic charge"}, - ), - ), - ) - current = frame( - 1, - features=( - Feature( - "bader", - "atomic_charge", - (0.5, 0.5, 0.5), - charge=0.2, - attributes={"charge_definition": "bader-electron-population"}, - ), - ), - ) - - result = match_features(previous, current) - - assert not result.edges - assert result.unmatched_previous == ("ddec",) - assert result.unmatched_current == ("bader",) - assert result.diagnostics["semantic_rejections"] == 1 - - -def test_incompatible_charge_conventions_are_gated_only_for_charged_features() -> None: - charged = match_features( - frame( - 0, - features=( - Feature( - "population", - "atomic_basin", - (0.5, 0.5, 0.5), - charge=1.2, - attributes={ - "descriptor_producer": "same-tool", - "charge_convention": "integrated electron population", - }, - ), - ), - ), - frame( - 1, - features=( - Feature( - "net-charge", - "atomic_basin", - (0.5, 0.5, 0.5), - charge=1.2, - attributes={ - "descriptor_producer": "same-tool", - "charge_convention": "signed net atomic charge", - }, - ), - ), - ), - ) - uncharged = match_features( - frame( - 0, - features=( - Feature( - "bcp-a", - "bond_critical_point", - (0.5, 0.5, 0.5), - attributes={ - "descriptor_producer": "same-tool", - "charge_convention": "integrated electron population", - }, - ), - ), - ), - frame( - 1, - features=( - Feature( - "bcp-b", - "bond_critical_point", - (0.5, 0.5, 0.5), - attributes={ - "descriptor_producer": "same-tool", - "charge_convention": "signed net atomic charge", - }, - ), - ), - ), - ) - - assert not charged.edges - assert charged.diagnostics["semantic_rejections"] == 1 - assert uncharged.mapping == {"bcp-a": "bcp-b"} - - -def test_incompatible_declared_descriptor_producers_are_forbidden() -> None: +@pytest.mark.parametrize( + ("key", "left", "right"), + [ + ("descriptor_producer", "critic2", "badelf"), + ("charge_definition", "bader", "ddec6"), + ("charge_convention", "electron_count", "net_atomic_charge"), + ], +) +def test_explicit_semantic_conflicts_prevent_matching(key: str, left: str, right: str) -> None: previous = frame( - 0, - features=( - Feature( - "critic", - "bond_critical_point", - (0.5, 0.5, 0.5), - attributes={"descriptor_producer": "Critic2"}, - ), - ), + 0, features=(Feature("p", "basin", (0.5, 0.5, 0.5), 1.0, attributes={key: left}),) ) current = frame( - 1, - features=( - Feature( - "other", - "bond_critical_point", - (0.5, 0.5, 0.5), - attributes={"descriptor_producer": "different-qtaim"}, - ), - ), - ) - - result = match_features(previous, current) - - assert not result.edges - assert result.unmatched_previous == ("critic",) - assert result.unmatched_current == ("other",) - - -def test_missing_optional_semantic_evidence_does_not_forbid_matching() -> None: - descriptor_result = match_features( - frame( - 0, - features=( - Feature( - "old", - "bond_descriptor", - (0.5, 0.5, 0.5), - attributes={"icohp": -1.0, "descriptor_producer": "lobster"}, - ), - ), - ), - frame( - 1, - features=(Feature("new", "bond_descriptor", (0.5, 0.5, 0.5)),), - ), - config=FeatureMatchConfig(descriptor_scales={"icohp": 0.5}), - ) - charge_result = match_features( - frame( - 0, - features=( - Feature( - "old-charge", - "atomic_charge", - (0.5, 0.5, 0.5), - charge=0.1, - attributes={"charge_definition": "ddec_net_atomic_charge"}, - ), - ), - ), - frame( - 1, - features=(Feature("new-charge", "atomic_charge", (0.5, 0.5, 0.5), charge=0.1),), - ), + 1, features=(Feature("q", "basin", (0.5, 0.5, 0.5), 1.0, attributes={key: right}),) ) - assert descriptor_result.mapping == {"old": "new"} - assert charge_result.mapping == {"old-charge": "new-charge"} + assert not match_features(previous, current).edges -def test_models_round_trip_through_json_compatible_dicts() -> None: +def test_frame_models_round_trip() -> None: original = frame( 7, atoms=(Atom(0, "H", (0.1, 0.2, 0.3), {"tag": np.int64(2)}),), features=(Feature("f", "nna", (0.9, 0.2, 0.3), -0.2, 1.1, (0,)),), ) - restored = Frame.from_dict(original.to_dict()) - assert restored.to_dict() == original.to_dict() + assert Frame.from_dict(original.to_dict()).to_dict() == original.to_dict() diff --git a/tests/test_models.py b/tests/test_models.py index c2b50f4..9ddc12f 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -1,20 +1,6 @@ from __future__ import annotations -import pytest - -from electrace.models import Atom, Feature, Frame - - -def test_fixed_shape_validation_consumes_only_one_excess_item() -> None: - coordinates = iter(range(100)) - with pytest.raises(ValueError, match="exactly three"): - Atom(0, "H", coordinates) - assert next(coordinates) == 4 - - rows = iter([(1.0, 0.0, 0.0)] * 100) - with pytest.raises(ValueError, match="exactly three row vectors"): - Frame(0, "0", rows) - assert next(rows) == (1.0, 0.0, 0.0) +from electrace.models import Feature def test_voxel_indices_are_deduplicated_and_sorted() -> None: diff --git a/tests/test_packaging.py b/tests/test_packaging.py index 9d37204..bdaa979 100644 --- a/tests/test_packaging.py +++ b/tests/test_packaging.py @@ -31,8 +31,7 @@ def test_runtime_extras_match_import_boundaries() -> None: assert required == {"numpy", "pyyaml", "rich", "typer"} assert _names(optional["accelerate"]) == {"scipy"} - assert _names(optional["hdf5"]) == {"h5py"} - assert _names(optional["all"]) == {"h5py", "scipy"} + assert _names(optional["all"]) == {"scipy"} optional_names = {name for requirements in optional.values() for name in _names(requirements)} assert optional_names.isdisjoint({"ase", "mkdocstrings", "types-setuptools"}) assert "typer>=0.20" in project["dependencies"] @@ -50,4 +49,4 @@ def test_sdist_excludes_repository_only_configuration() -> None: document: dict[str, Any] = tomllib.loads((ROOT / "pyproject.toml").read_text()) excluded = set(document["tool"]["hatch"]["build"]["targets"]["sdist"]["exclude"]) - assert {"/.github", "/.pre-commit-config.yaml", "/AGENTS.md"} <= excluded + assert {"/.github", "/AGENTS.md"} <= excluded diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index e60d98f..99a80a2 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -3,100 +3,22 @@ import csv import json from pathlib import Path -from typing import Any -import pytest - -from electrace.config import ElecTraceConfig from electrace.demo import generate_demo -from electrace.models import Event, EventType -from electrace.pipeline import ( - _normalize_events, - load_and_analyze, - run_project, - write_analysis_bundle, -) +from electrace.pipeline import run_project from electrace.provenance import sha256_file -def _add_ambiguous_demo_feature(config_path: Path) -> str: - descriptor_path = config_path.parent / "01" / "features.json" - descriptor = json.loads(descriptor_path.read_text(encoding="utf-8")) - duplicate = dict(descriptor["features"][0]) - duplicate["id"] = "ambiguous-duplicate" - descriptor["features"].append(duplicate) - descriptor_path.write_text(json.dumps(descriptor), encoding="utf-8") - return duplicate["id"] - - -def test_demo_pipeline_detects_expected_electronic_events(tmp_path: Path) -> None: +def test_demo_pipeline_analyzes_and_writes_report_bundle(tmp_path: Path) -> None: config_path = generate_demo(tmp_path / "demo") - analysis, _ = load_and_analyze(config_path) + artifacts = run_project(config_path) + analysis = artifacts.analysis event_types = [event["type"] for event in analysis["events"]] assert "merge" in event_types - assert event_types.count("birth") == 2 assert "property_change" in event_types - assert analysis["project"]["series_kind"] == "neb" - assert analysis["warnings"] == [] assert len(analysis["frames"]) == 6 - assert len(analysis["tracks"]) == 15 assert analysis["provenance"]["descriptor_producers"] == ["electrace-demo"] - assert all(source["sha256"] for source in analysis["provenance"]["source_files"]) - - -def test_strict_matching_fails_and_permissive_matching_suppresses_ambiguous_claims( - tmp_path: Path, -) -> None: - config_path = generate_demo(tmp_path / "demo") - duplicate_id = _add_ambiguous_demo_feature(config_path) - - with pytest.raises(ValueError, match="Ambiguous correspondence in strict mode"): - run_project(config_path) - assert not (config_path.parent / "electrace-results").exists() - - analysis, _ = load_and_analyze(config_path, strict=False) - - assert any("Ambiguous feature correspondence" in warning for warning in analysis["warnings"]) - first_transition = analysis["transitions"][0]["feature_matches"] - assert first_transition["diagnostics"]["ambiguous"] is True - assert first_transition["diagnostics"]["ambiguities"] - assert all( - duplicate_id not in {*event["source_ids"], *event["target_ids"]} - for event in analysis["events"] - ) - duplicate_track = next( - item for item in analysis["tracks"] if item["feature_id"] == duplicate_id - ) - assert duplicate_track["parent_track_ids"] == [] - - -def test_descriptor_document_provenance_and_warnings_reach_analysis( - tmp_path: Path, -) -> None: - config_path = generate_demo(tmp_path / "demo") - descriptor_path = config_path.parent / "00" / "features.json" - descriptor = json.loads(descriptor_path.read_text(encoding="utf-8")) - descriptor["metadata"]["warnings"] = ["synthetic parser warning "] - descriptor_path.write_text(json.dumps(descriptor), encoding="utf-8") - - analysis, _ = load_and_analyze(config_path) - - assert ( - "Descriptor warning (00, electrace-demo): synthetic parser warning " - in analysis["warnings"] - ) - first = analysis["provenance"]["descriptor_documents"][0] - assert first["frame"] == {"index": 0, "label": "00"} - assert first["producer"] == {"name": "electrace-demo", "version": "0.1"} - assert first["metadata"]["warnings"] == ["synthetic parser warning "] - assert "charge_convention" in first["metadata"] - - -def test_result_bundle_is_complete_and_integrity_manifest_matches(tmp_path: Path) -> None: - config_path = generate_demo(tmp_path / "demo") - artifacts = run_project(config_path) - assert {path.name for path in artifacts.files} == { "analysis.json", "events.csv", @@ -104,245 +26,10 @@ def test_result_bundle_is_complete_and_integrity_manifest_matches(tmp_path: Path "manifest.json", "report.html", } + manifest = json.loads((artifacts.output_directory / "manifest.json").read_text()) for record in manifest["files"]: - path = artifacts.output_directory / record["path"] - assert record["sha256"] == sha256_file(path) + assert record["sha256"] == sha256_file(artifacts.output_directory / record["path"]) with (artifacts.output_directory / "events.csv").open(newline="") as handle: - rows = list(csv.DictReader(handle)) - assert len(rows) == len(artifacts.analysis["events"]) + assert len(list(csv.DictReader(handle))) == len(analysis["events"]) assert "Electronic event timeline" in (artifacts.output_directory / "report.html").read_text() - - with pytest.raises(FileExistsError): - run_project(config_path) - rerun = run_project(config_path, overwrite=True) - assert rerun.output_directory == artifacts.output_directory - - -def test_overwrite_removes_only_unselected_known_bundle_artifacts( - tmp_path: Path, -) -> None: - analysis: dict[str, Any] = { - "schema_version": "0.1", - "project": {"name": "bundle regression"}, - "events": [{"event_id": "E1", "type": "birth"}], - "tracks": [{"track_id": "T1", "frame": 0}], - "frames": [], - } - config = ElecTraceConfig() - output = tmp_path / "bundle" - first = write_analysis_bundle(analysis, config, output) - assert {path.name for path in first.files} == { - "analysis.json", - "events.csv", - "feature_tracks.csv", - "manifest.json", - "report.html", - } - (output / "analysis.h5").write_bytes(b"stale known artifact") - user_file = output / "research-notes.txt" - user_file.write_text("preserve me", encoding="utf-8") - - config.outputs.json = False - config.outputs.html_report = False - config.outputs.hdf5 = False - rerun = write_analysis_bundle(analysis, config, output, overwrite=True) - - assert {path.name for path in rerun.files} == { - "events.csv", - "feature_tracks.csv", - "manifest.json", - } - assert not (output / "analysis.json").exists() - assert not (output / "analysis.h5").exists() - assert not (output / "report.html").exists() - assert user_file.read_text(encoding="utf-8") == "preserve me" - manifest = json.loads((output / "manifest.json").read_text(encoding="utf-8")) - assert {record["path"] for record in manifest["files"]} == { - "events.csv", - "feature_tracks.csv", - } - - -def test_overwrite_never_follows_reserved_artifact_symlinks(tmp_path: Path) -> None: - analysis: dict[str, Any] = { - "schema_version": "0.1", - "project": {"name": "symlink regression"}, - "events": [], - "tracks": [], - "frames": [], - } - config = ElecTraceConfig() - output = tmp_path / "bundle" - write_analysis_bundle(analysis, config, output) - external = tmp_path / "external.json" - external.write_text("do not replace", encoding="utf-8") - analysis_path = output / "analysis.json" - analysis_path.unlink() - analysis_path.symlink_to(external) - unknown_target = tmp_path / "unknown-target.txt" - unknown_target.write_text("user data", encoding="utf-8") - unknown_link = output / "user-link.txt" - unknown_link.symlink_to(unknown_target) - - write_analysis_bundle(analysis, config, output, overwrite=True) - - assert external.read_text(encoding="utf-8") == "do not replace" - assert not analysis_path.is_symlink() - assert json.loads(analysis_path.read_text(encoding="utf-8"))["schema_version"] == "0.1" - assert unknown_link.is_symlink() - assert unknown_link.read_text(encoding="utf-8") == "user data" - - -def test_late_export_failure_never_publishes_a_partial_bundle( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - analysis: dict[str, Any] = { - "schema_version": "0.1", - "project": {"name": "transaction regression"}, - "events": [{"event_id": "E1", "type": "birth"}], - "tracks": [{"track_id": "T1", "frame": 0}], - "frames": [], - } - config = ElecTraceConfig() - existing = tmp_path / "existing" - write_analysis_bundle(analysis, config, existing) - before = {path.name: path.read_bytes() for path in existing.iterdir() if path.is_file()} - - def fail_report(*_args: object, **_kwargs: object) -> Path: - raise RuntimeError("forced late report failure") - - monkeypatch.setattr("electrace.pipeline.render_html_report", fail_report) - - with pytest.raises(RuntimeError, match="forced late report failure"): - write_analysis_bundle(analysis, config, existing, overwrite=True) - after = {path.name: path.read_bytes() for path in existing.iterdir() if path.is_file()} - assert after == before - - new_output = tmp_path / "new" - with pytest.raises(RuntimeError, match="forced late report failure"): - write_analysis_bundle(analysis, config, new_output) - assert new_output.is_dir() - assert not any(new_output.iterdir()) - - -def test_load_and_analyze_forwards_all_configured_input_selectors( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - root = tmp_path / "images" - descriptors = tmp_path / "descriptors" - root.mkdir() - descriptors.mkdir() - config_path = tmp_path / "electrace.toml" - config_path.write_text( - """[project] -name = "custom inputs" -series_kind = "ordered" - -[inputs] -root = "images" -descriptor_dir = "descriptors" -structure_name = "FINAL.vasp" -fallback_structure_name = "START.vasp" -descriptor_name = "electronic.json" -descriptor_source = "merge" -prefer_contcar = false -""", - encoding="utf-8", - ) - captured: dict[str, Any] = {} - sentinel = object() - - def fake_load_sequence(path: Path, **kwargs: Any) -> object: - captured["path"] = path - captured.update(kwargs) - return sentinel - - def fake_analyze_sequence( - sequence: object, config: object, *, config_path: Path, strict: bool - ) -> dict[str, bool]: - assert sequence is sentinel - captured["config_path"] = config_path - captured["analysis_strict"] = strict - return {"ok": True} - - monkeypatch.setattr("electrace.pipeline.load_sequence", fake_load_sequence) - monkeypatch.setattr("electrace.pipeline.analyze_sequence", fake_analyze_sequence) - - analysis, _ = load_and_analyze(config_path) - - assert analysis == {"ok": True} - assert captured["path"] == root.resolve() - assert captured["descriptor_dir"] == descriptors.resolve() - assert captured["structure_name"] == "FINAL.vasp" - assert captured["fallback_structure_name"] == "START.vasp" - assert captured["descriptor_name"] == "electronic.json" - assert captured["descriptor_source"] == "merge" - assert captured["prefer_contcar"] is False - assert captured["analysis_strict"] is True - - -def test_event_normalization_uses_observed_persistence_end_frame() -> None: - event = Event( - type=EventType.PROPERTY_CHANGE, - frame_before=10, - frame_after=20, - source_ids=("before",), - target_ids=("after",), - confidence=0.8, - persistence=2, - metadata={ - "persistence_status": "confirmed", - "persistence_end_frame": 40, - }, - ) - - normalized = _normalize_events((event,), minimum_persistence=2) - - assert normalized[0]["start_frame"] == 20 - assert normalized[0]["end_frame"] == 40 - assert normalized[0]["confidence_label"] == "high" - - -def test_event_confidence_labels_use_configured_inclusive_thresholds() -> None: - events = tuple( - Event( - type=EventType.BIRTH, - frame_before=0, - frame_after=index, - target_ids=(f"f{index}",), - confidence=confidence, - metadata={"persistence_status": "confirmed"}, - ) - for index, confidence in enumerate((0.75, 0.50, 0.49), start=1) - ) - - normalized = _normalize_events( - events, - minimum_persistence=1, - medium_confidence_threshold=0.50, - high_confidence_threshold=0.75, - ) - - assert [event["confidence_label"] for event in normalized] == [ - "high", - "medium", - "low", - ] - assert all(event["persistent"] is True for event in normalized) - - -def test_transient_status_is_not_reported_as_persistent_when_minimum_is_one() -> None: - event = Event( - type=EventType.BIRTH, - frame_before=0, - frame_after=1, - target_ids=("f",), - confidence=0.9, - persistence=1, - metadata={"persistence_status": "transient"}, - ) - - normalized = _normalize_events((event,), minimum_persistence=1) - - assert normalized[0]["persistent"] is False diff --git a/tests/test_plugins.py b/tests/test_plugins.py index 04be9c0..8ce1083 100644 --- a/tests/test_plugins.py +++ b/tests/test_plugins.py @@ -1,20 +1,12 @@ from __future__ import annotations from pathlib import Path -from typing import Any - -import pytest from electrace.plugins import ( DEFAULT_REGISTRY, - BaderKitPlugin, Capability, - CommandPlan, - PluginRegistry, doctor_plugins, - doctor_plugins_path, plan_plugin, - plan_plugin_path, ) @@ -24,7 +16,7 @@ def _touch(directory: Path, *names: str) -> None: (directory / name).write_text("fixture\n", encoding="utf-8") -def test_builtin_registry_exposes_capabilities() -> None: +def test_builtin_registry_exposes_supported_tools() -> None: assert DEFAULT_REGISTRY.names() == ( "bader", "baderkit", @@ -35,156 +27,63 @@ def test_builtin_registry_exposes_capabilities() -> None: assert [plugin.name for plugin in DEFAULT_REGISTRY.for_capability(Capability.COHP)] == [ "lobster" ] - assert {plugin.name for plugin in DEFAULT_REGISTRY.for_capability("bader_charges")} == { - "bader", - "baderkit", - } - -def test_registry_rejects_duplicate_and_unknown_plugins() -> None: - registry = PluginRegistry([BaderKitPlugin()]) - with pytest.raises(ValueError, match="already registered"): - registry.register(BaderKitPlugin()) - with pytest.raises(KeyError, match="unknown plugin"): - registry.get("missing") +def test_bader_and_badelf_plans_declare_inputs_and_outputs(tmp_path: Path) -> None: + _touch(tmp_path, "CHGCAR", "ELFCAR", "CHGCAR_sum") -def test_baderkit_plans_argv_without_executing( - tmp_path: Path, monkeypatch: pytest.MonkeyPatch -) -> None: - _touch(tmp_path, "CHGCAR", "CHGCAR_sum") - executed = False - - def forbidden(*args: object, **kwargs: object) -> None: - nonlocal executed - executed = True - raise AssertionError("planner attempted to execute an external tool") - - monkeypatch.setattr("subprocess.run", forbidden) - plan = plan_plugin( - "baderkit", + bader = plan_plugin( + "bader", tmp_path, capability="bader_charges", reference_file="CHGCAR_sum", ) - - assert plan.argv == ("baderkit", "bader", "CHGCAR", "-tot", "CHGCAR_sum") - assert plan.cwd == tmp_path.resolve() - assert plan.expected_outputs == (tmp_path / "bader.json",) - assert plan.missing_inputs == () - assert executed is False - assert ">" not in plan.argv - - -def test_baderkit_badelf_uses_documented_grid_order_and_output(tmp_path: Path) -> None: - _touch(tmp_path, "CHGCAR", "ELFCAR", "CHGCAR_sum") - - plan = plan_plugin( + badelf = plan_plugin( "baderkit", tmp_path, capability="elf_basins", - configuration={"executable": "/opt/baderkit"}, reference_file="CHGCAR_sum", ) - assert plan.argv == ( - "/opt/baderkit", + assert bader.argv == ("bader", "CHGCAR", "-ref", "CHGCAR_sum") + assert bader.expected_outputs == (tmp_path / "ACF.dat",) + assert badelf.argv == ( + "baderkit", "badelf", "CHGCAR", "ELFCAR", "-tot", "CHGCAR_sum", ) - assert plan.expected_outputs == (tmp_path / "badelf.json",) - - -def test_bader_plan_supports_optional_reference_density(tmp_path: Path) -> None: - _touch(tmp_path, "CHGCAR", "CHGCAR_sum") - - plan = plan_plugin( - "bader", - tmp_path, - capability="bader_charges", - reference_file="CHGCAR_sum", - ) + assert badelf.expected_outputs == (tmp_path / "badelf.json",) - assert plan.argv == ("bader", "CHGCAR", "-ref", "CHGCAR_sum") - assert plan.expected_outputs == (tmp_path / "ACF.dat",) - assert plan.missing_inputs == () - assert "not formal oxidation states" in plan.notes[0] - -def test_critic2_plan_uses_explicit_stdin_and_stdout_capture(tmp_path: Path) -> None: +def test_critic2_plan_uses_explicit_stdin(tmp_path: Path) -> None: _touch(tmp_path, "CONTCAR", "CHGCAR") + plan = plan_plugin("critic2", tmp_path, capability=Capability.QTAIM_CRITICAL_POINTS) assert plan.argv == ("critic2", "-q") assert plan.stdin == ( "CRYSTAL CONTCAR\nLOAD CHGCAR\nAUTO\nCPREPORT critic2.features.json GRAPH\nEND\n" ) - assert plan.stdout_path == tmp_path / "critic2.cro" assert plan.expected_outputs == ( tmp_path / "critic2.cro", tmp_path / "critic2.features.json", ) - assert "does not parse critic2.cro" in plan.notes[1] - assert "review" in plan.notes[0].lower() -def test_lobster_and_chargemol_plans_declare_inputs_and_outputs(tmp_path: Path) -> None: +def test_lobster_and_chargemol_plans(tmp_path: Path) -> None: _touch(tmp_path, "lobsterin", "WAVECAR", "POSCAR", "POTCAR", "job_control.txt") + lobster = plan_plugin("lobster", tmp_path, capability="cobi") chargemol = plan_plugin("chargemol", tmp_path) - assert lobster.argv == ("lobster",) assert lobster.expected_outputs == (tmp_path / "ICOBILIST.lobster",) - assert lobster.missing_inputs == () assert chargemol.expected_outputs == (tmp_path / "DDEC6_even_tempered_net_atomic_charges.xyz",) -def test_plan_rejects_disabled_plugin_and_unknown_options(tmp_path: Path) -> None: - _touch(tmp_path, "CHGCAR") - with pytest.raises(ValueError, match="disabled"): - plan_plugin("baderkit", tmp_path, configuration={"enabled": False}) - with pytest.raises(TypeError, match="unknown BaderKit"): - plan_plugin("baderkit", tmp_path, surprise=True) - - -@pytest.mark.parametrize( - ("plugin", "options"), - [ - ("baderkit", {"charge_file": "../CHGCAR"}), - ("critic2", {"field_file": "CHGCAR\nEND"}), - ("lobster", {"input_file": "../lobsterin"}), - ("chargemol", {"job_control": "/tmp/job_control.txt"}), - ], -) -def test_planner_filenames_must_be_basenames( - tmp_path: Path, plugin: str, options: dict[str, Any] -) -> None: - with pytest.raises(ValueError, match="basename"): - plan_plugin(plugin, tmp_path, **options) - - -@pytest.mark.parametrize( - ("plugin", "options", "fixed_name"), - [ - ("lobster", {"input_file": "custom-lobsterin"}, "lobsterin"), - ("lobster", {"structure_file": "CONTCAR"}, "POSCAR"), - ("chargemol", {"job_control": "custom-control.txt"}, "job_control.txt"), - ], -) -def test_fixed_name_tools_reject_unstaged_aliases( - tmp_path: Path, - plugin: str, - options: dict[str, Any], - fixed_name: str, -) -> None: - with pytest.raises(ValueError, match=rf"fixed filename {fixed_name!r}"): - plan_plugin(plugin, tmp_path, **options) - - -def test_doctor_is_read_only_and_honors_configuration(tmp_path: Path) -> None: +def test_doctor_checks_files_without_launching_executable(tmp_path: Path) -> None: _touch(tmp_path, "CHGCAR") executable = tmp_path / "fake-baderkit" executable.write_text("#!/bin/sh\nexit 99\n", encoding="utf-8") @@ -202,105 +101,5 @@ def test_doctor_is_read_only_and_honors_configuration(tmp_path: Path) -> None: ) assert len(reports) == 1 - assert reports[0].plugin == "baderkit" assert reports[0].installed is True assert reports[0].ready is True - # The fake executable exits 99 if launched; reaching this assertion proves - # the doctor only inspected the filesystem. - assert reports[0].executable_path == executable.resolve() - - -def test_path_wide_plan_and_doctor_are_ordered_and_use_discovered_structure( - tmp_path: Path, -) -> None: - root = tmp_path / "neb" - for label in ("01", "00"): - _touch(root / label, "FINAL.vasp") - _touch(root / "00", "CHGCAR") - executable = tmp_path / "fake-critic2" - executable.write_text("#!/bin/sh\nexit 99\n", encoding="utf-8") - executable.chmod(executable.stat().st_mode | 0o111) - configuration: dict[str, dict[str, Any]] = { - "bader": {"enabled": False}, - "baderkit": {"enabled": False}, - "critic2": {"enabled": True, "executable": str(executable)}, - "lobster": {"enabled": False}, - "chargemol": {"enabled": False}, - } - - plans = plan_plugin_path( - "critic2", - root, - structure_name="FINAL.vasp", - fallback_structure_name="START.vasp", - configuration=configuration["critic2"], - ) - reports = doctor_plugins_path( - root, - structure_name="FINAL.vasp", - fallback_structure_name="START.vasp", - configuration=configuration, - ) - - assert [plan.label for plan in plans] == ["00", "01"] - assert [plan.directory for plan in plans] == [root / "00", root / "01"] - assert all("CRYSTAL FINAL.vasp" in (plan.command.stdin or "") for plan in plans) - assert [report.label for report in reports] == ["00", "01"] - assert reports[0].ready is True - assert reports[1].ready is False - assert reports[1].plugins[0].missing_inputs == (root / "01" / "CHGCAR",) - assert reports[0].to_dict()["frame"]["label"] == "00" - - -def test_nonzero_neb_labels_keep_plan_and_doctor_indices_canonical(tmp_path: Path) -> None: - root = tmp_path / "neb" - for label in ("09", "08"): - _touch(root / label, "FINAL.vasp") - configuration = { - "bader": {"enabled": False}, - "baderkit": {"enabled": False}, - "critic2": {"enabled": False}, - "lobster": {"enabled": False}, - "chargemol": {"enabled": False}, - } - - plans = plan_plugin_path( - "critic2", - root, - structure_name="FINAL.vasp", - fallback_structure_name="START.vasp", - ) - reports = doctor_plugins_path( - root, - structure_name="FINAL.vasp", - fallback_structure_name="START.vasp", - configuration=configuration, - ) - - assert [(plan.index, plan.label) for plan in plans] == [(0, "08"), (1, "09")] - assert [(report.index, report.label) for report in reports] == [ - (0, "08"), - (1, "09"), - ] - - -def test_command_plan_validation_and_serialization(tmp_path: Path) -> None: - with pytest.raises(ValueError, match="absolute"): - CommandPlan( - plugin="test", - capability=Capability.BADER_CHARGES, - argv=("tool",), - cwd=Path("relative"), - ) - plan = CommandPlan( - plugin="test", - capability=Capability.BADER_CHARGES, - argv=("missing-tool", "input"), - cwd=tmp_path.resolve(), - environment={"OMP_NUM_THREADS": "2"}, - ) - payload = plan.to_dict() - assert payload["argv"] == ["missing-tool", "input"] - assert payload["capability"] == "bader_charges" - assert payload["environment"] == {"OMP_NUM_THREADS": "2"} - assert plan.ready is False diff --git a/tests/test_report.py b/tests/test_report.py index 00ca4dd..622eddf 100644 --- a/tests/test_report.py +++ b/tests/test_report.py @@ -2,23 +2,20 @@ from pathlib import Path -import pytest - from electrace.report import render_html_report -def test_report_is_self_contained_and_escapes_script_content(tmp_path: Path) -> None: +def test_report_is_self_contained_and_escapes_input(tmp_path: Path) -> None: output = render_html_report( { "schema_version": "0.1", "project": {"name": "Demo "}, - "frames": [{"label": "00", "energy": -1.0, "atom_count": 2, "feature_count": 1}], + "frames": [{"label": "00", "energy": -1.0}], "events": [ { "type": "birth", "start_frame": 0, "confidence": 0.9, - "confidence_label": "high", "description": "appeared ", } ], @@ -26,45 +23,8 @@ def test_report_is_self_contained_and_escapes_script_content(tmp_path: Path) -> }, tmp_path / "report.html", ) - document = output.read_text() + document = output.read_text() assert "Demo <system>" in document assert "\\u003c/script>" in document assert "https://" not in document - assert 'e.confidence_label || ""' in document - assert 'e.confidence_label||"unclassified"' in document - assert "Number(e.confidence || 0) >= .8" not in document - - -@pytest.mark.parametrize("bad_value", [float("nan"), object()]) -def test_report_rejects_non_finite_and_non_json_analysis_values( - tmp_path: Path, bad_value: object -) -> None: - output = tmp_path / "report.html" - - with pytest.raises( - ValueError, - match="HTML report analysis JSON contains a non-finite or non-JSON value", - ): - render_html_report({"nested": {"bad": bad_value}}, output) - - assert not output.exists() - - -def test_energy_chart_preserves_sparse_frame_positions(tmp_path: Path) -> None: - output = render_html_report( - { - "frames": [ - {"label": str(index), "energy": energy} - for index, energy in enumerate((-1.0, None, None, None, 0.0)) - ], - "events": [], - "tracks": [], - }, - tmp_path / "report.html", - ) - - document = output.read_text(encoding="utf-8") - assert '"energy":null' in document - assert "maxX=Math.max(frames.length-1,1)" in document - assert "maxX=Math.max(points.length-1,1)" not in document diff --git a/tests/test_volume.py b/tests/test_volume.py index 4ef3360..b05c500 100644 --- a/tests/test_volume.py +++ b/tests/test_volume.py @@ -1,6 +1,5 @@ from __future__ import annotations -import textwrap from pathlib import Path import numpy as np @@ -9,282 +8,81 @@ from electrace.volume import VaspGridError, read_vasp_grid -def _write_grid(path: Path, document: str) -> Path: - path.write_text(textwrap.dedent(document).lstrip(), encoding="utf-8") +def _write_grid( + path: Path, + dimensions: str, + values: str, + *, + cell: tuple[str, str, str] = ("1 0 0", "0 1 0", "0 0 1"), + species: str = "H", + counts: str = "1", + mode: str = "Direct", + positions: tuple[str, ...] = ("0 0 0",), +) -> Path: + document = ["grid", "1", *cell, species, counts, mode, *positions, "", dimensions, values] + path.write_text("\n".join(document), encoding="utf-8") return path -def test_non_cubic_x_fastest_order_consecutive_dataset_and_summary(tmp_path: Path) -> None: +def test_non_cubic_grid_uses_vasp_x_fastest_order(tmp_path: Path) -> None: source = _write_grid( tmp_path / "ELFCAR", - """ - non-cubic two-channel grid - 1.0 - 2.0 0.0 0.0 - 0.5 3.0 0.0 - 0.0 0.25 4.0 - C H - 1 1 - Direct - 0.0 0.0 0.0 - 0.5 0.5 0.5 - - 2 3 2 - 0 1 2 3 4 - 5 6 7 8 9 - 10 11 - - 2 3 2 - 100 101 102 103 - 104 105 106 107 - 108 109 110 111 - """, + "2 3 2", + "0 1 2 3 4 5 6 7 8 9 10 11", + cell=("2 0 0", "0.5 3 0", "0 0.25 4"), + species="C H", + counts="1 1", + positions=("0 0 0", "0.5 0.5 0.5"), ) - first = read_vasp_grid(source) - second = read_vasp_grid(source, dataset=1) + grid = read_vasp_grid(source) - assert first.shape == (2, 3, 2) - assert first.metadata.cell == ( + assert grid.shape == (2, 3, 2) + assert grid.metadata.cell == ( (2.0, 0.0, 0.0), (0.5, 3.0, 0.0), (0.0, 0.25, 4.0), ) - assert first.metadata.elements == ("C", "H") - assert first.metadata.dataset_count == 2 - # VASP writes x fastest, then y, then z. - assert first.data[0, 0, 0] == pytest.approx(0.0) - assert first.data[1, 0, 0] == pytest.approx(1.0) - assert first.data[0, 1, 0] == pytest.approx(2.0) - assert first.data[1, 2, 1] == pytest.approx(11.0) - assert second.data[0, 0, 0] == pytest.approx(100.0) - assert second.data[1, 2, 1] == pytest.approx(111.0) - assert second.data.flags.writeable is False - assert second.summary() == { - "source": str(source.resolve()), - "dataset_index": 1, - "dataset_count": 2, - "shape": [2, 3, 2], - "point_count": 12, - "cell_volume": pytest.approx(24.0), - "minimum": 100.0, - "maximum": 111.0, - "mean": 105.5, - } + assert grid.data[1, 0, 0] == pytest.approx(1.0) + assert grid.data[0, 1, 0] == pytest.approx(2.0) + assert grid.data[1, 2, 1] == pytest.approx(11.0) -def test_periodic_trilinear_sampling_and_vasp4_blank_lines(tmp_path: Path) -> None: +def test_periodic_trilinear_sampling(tmp_path: Path) -> None: source = _write_grid( - tmp_path / "vasp4-grid", - """ - VASP 4 periodic interpolation - - 1 - 1 0 0 - 0 1 0 - 0 0 1 - - 1 - Direct - - 0 0 0 - - 2 2 2 - 0 1 10 11 - 100 101 110 111 - """, + tmp_path / "grid", + "2 2 2", + "0 1 10 11 100 101 110 111", ) grid = read_vasp_grid(source) - assert grid.metadata.symbols_inferred is True - assert grid.metadata.species == ("X1",) - assert grid.sample((0.0, 0.0, 0.0)) == pytest.approx(0.0) - assert grid.sample((0.5, 0.5, 0.5)) == pytest.approx(111.0) assert grid.sample((0.25, 0.25, 0.25)) == pytest.approx(55.5) assert grid.sample((1.0, 0.0, 0.0)) == pytest.approx(0.0) - assert grid.sample((-0.5, 0.0, 0.0)) == pytest.approx(1.0) - with pytest.raises(ValueError, match="three finite numbers"): - grid.sample((float("nan"), 0.0, 0.0)) - - -def test_selective_cartesian_coordinates_and_positive_scale(tmp_path: Path) -> None: - source = _write_grid( - tmp_path / "CHGCAR", - """ - scaled Cartesian structure - 2.0 - 1 0 0 - 0 2 0 - 0 0 3 - Si O - 1 1 - Selective dynamics - Cartesian - 0.5 1.0 1.5 T F T - 0.0 0.0 0.0 F F F - - 1 1 1 - 1.25D+00 - """, - ) - grid = read_vasp_grid(source) - - assert grid.metadata.coordinate_mode == "cartesian" - assert grid.metadata.cell == ((2.0, 0.0, 0.0), (0.0, 4.0, 0.0), (0.0, 0.0, 6.0)) - np.testing.assert_allclose( - grid.metadata.fractional_positions, - ((0.5, 0.5, 0.5), (0.0, 0.0, 0.0)), - ) - assert grid.metadata.selective_dynamics == ((True, False, True), (False, False, False)) - assert grid.data[0, 0, 0] == pytest.approx(1.25) - -def test_cartesian_positions_use_row_vector_cell_convention(tmp_path: Path) -> None: +def test_cartesian_positions_follow_row_vector_convention(tmp_path: Path) -> None: source = _write_grid( tmp_path / "triclinic-grid", - """ - row-vector Cartesian conversion - 1 - 2.0 0.0 0.0 - 0.5 3.0 0.0 - 0.0 0.25 4.0 - C - 1 - Cartesian - 0.55 1.0 1.6 - - 1 1 1 - 0.5 - """, + "1 1 1", + "0.5", + cell=("2 0 0", "0.5 3 0", "0 0.25 4"), + species="C", + mode="Cartesian", + positions=("0.55 1 1.6",), ) grid = read_vasp_grid(source) - # (0.2, 0.3, 0.4) @ cell = (0.55, 1.0, 1.6). np.testing.assert_allclose(grid.metadata.fractional_positions[0], (0.2, 0.3, 0.4)) -@pytest.mark.parametrize( - ("scale", "expected_cell"), - [ - ("-8", ((2.0, 0.0, 0.0), (0.0, 2.0, 0.0), (0.0, 0.0, 2.0))), - ("2 3 4", ((2.0, 0.0, 0.0), (0.0, 3.0, 0.0), (0.0, 0.0, 4.0))), - ], -) -def test_negative_volume_and_component_scaling( - tmp_path: Path, - scale: str, - expected_cell: tuple[tuple[float, float, float], ...], -) -> None: +def test_malformed_grid_is_rejected(tmp_path: Path) -> None: source = _write_grid( - tmp_path / "scaled-grid", - f""" - scale variants - {scale} - 1 0 0 - 0 1 0 - 0 0 1 - H - 1 - Direct - 0.25 0.5 0.75 - - 1 1 1 - 2.0 - """, + tmp_path / "bad-grid", + "2 2 1", + "1 2 3", ) - grid = read_vasp_grid(source) - - assert grid.metadata.cell == expected_cell - assert grid.metadata.fractional_positions == ((0.25, 0.5, 0.75),) - - -def _minimal_document(data_and_tail: str, *, shape: str = "2 2 1") -> str: - return f"""\ -minimal malformed grid -1 -1 0 0 -0 1 0 -0 0 1 -H -1 -Direct -0 0 0 - -{shape} -{data_and_tail} -""" - - -@pytest.mark.parametrize( - ("data_and_tail", "match"), - [ - ("1 2 3", "missing values for dataset 0"), - ("1 2 nan 4", "non-finite number"), - ("1 2 3 4\naugmentation occupancies 1 1", "augmentation blocks are not supported"), - ("1 2 3 4\nunknown tail", "unsupported content follows"), - ("1 2 3 4\n1 1 1\n5", "shape .* differs"), - ("1 2 3 4 5", "extra values"), - ], -) -def test_rejects_truncated_nonfinite_augmentation_and_ambiguous_content( - tmp_path: Path, - data_and_tail: str, - match: str, -) -> None: - source = _write_grid(tmp_path / "bad-grid", _minimal_document(data_and_tail)) - - with pytest.raises(VaspGridError, match=match): + with pytest.raises(VaspGridError, match="missing values"): read_vasp_grid(source) - - -def test_max_points_and_dataset_bounds_are_enforced(tmp_path: Path) -> None: - oversized = _write_grid( - tmp_path / "oversized", - _minimal_document("", shape="11 10 10"), - ) - valid = _write_grid( - tmp_path / "single", - _minimal_document("1 2 3 4"), - ) - - with pytest.raises(VaspGridError, match="1100 points, exceeding max_points=1099"): - read_vasp_grid(oversized, max_points=1099) - with pytest.raises(VaspGridError, match=r"dataset 1 is unavailable.*1 dataset"): - read_vasp_grid(valid, dataset=1) - with pytest.raises(ValueError, match="max_points must be a positive integer"): - read_vasp_grid(valid, max_points=0) - - -def test_data_order_matches_explicit_numpy_reference(tmp_path: Path) -> None: - source = _write_grid( - tmp_path / "order", - _minimal_document("0 1 2 3 4 5", shape="3 1 2"), - ) - - grid = read_vasp_grid(source) - expected = np.array([0, 1, 2, 3, 4, 5], dtype=float).reshape((3, 1, 2), order="F") - - np.testing.assert_array_equal(grid.data, expected) - - -def test_reader_streams_instead_of_materializing_the_text_file( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - source = _write_grid( - tmp_path / "streamed-grid", - _minimal_document("1 2 3 4"), - ) - - def reject_read_text(*_args: object, **_kwargs: object) -> str: - raise AssertionError("read_vasp_grid must not materialize the full text file") - - monkeypatch.setattr(Path, "read_text", reject_read_text) - - grid = read_vasp_grid(source) - - np.testing.assert_array_equal(grid.data.ravel(order="F"), (1.0, 2.0, 3.0, 4.0)) diff --git a/tests/test_workflow_collection.py b/tests/test_workflow_collection.py index 4499ed2..dd3b4a7 100644 --- a/tests/test_workflow_collection.py +++ b/tests/test_workflow_collection.py @@ -1,266 +1,49 @@ -from __future__ import annotations - -import json -from pathlib import Path - import pytest -import yaml # type: ignore[import-untyped] - -from electrace.config import EventSettings, MatchingSettings, load_config -from electrace.io.descriptors import DescriptorError -from electrace.io.vasp import read_vasp_energy -from electrace.workflow.collection import ( - COLLECTION_RECORD, - COLLECTION_SCHEMA_VERSION, - analyze_collected_workflow, - collect_workflow_outputs, -) -from electrace.workflow.staging import ( - StagedWorkflow, - descriptor_outputs, - stage_workflow, -) -POSCAR = """\ -H collection fixture -1.0 -1 0 0 -0 1 0 -0 0 1 -H -1 -Direct -0 0 0 -""" +from electrace.workflow.collection import analyze_collected_workflow, collect_workflow_outputs +from electrace.workflow.staging import descriptor_outputs, stage_workflow -ACF = """\ - # X Y Z CHARGE MIN DIST ATOMIC VOL - 1 0.0 0.0 0.0 1.0 0.0 10.0 +ACF = """# X Y Z CHARGE MIN DIST ATOMIC VOL +1 0.0 0.0 0.0 1.0 0.0 10.0 """ -def _stage_fixture( - tmp_path: Path, - *, - duplicate_bader: bool = False, - write_outputs: bool = True, -) -> tuple[Path, StagedWorkflow]: - images = tmp_path / "images" - for order, directory_index in enumerate((8, 9)): - frame = images / f"{directory_index:02d}" - frame.mkdir(parents=True) - (frame / "POSCAR").write_text(POSCAR, encoding="utf-8") - energy = -10.0 + order * 0.25 - (frame / "OSZICAR").write_text(f" 1 F= {energy} E0= {energy} d E = 0.0\n", encoding="utf-8") - steps = [ - { - "id": "charges", - "kind": "bader", - "executable": "bader", - "depends_on": [], - "options": {}, - "resources": {}, - } - ] - if duplicate_bader: - steps.append( - { - "id": "charges_again", - "kind": "bader", - "executable": "bader", - "depends_on": [], - "options": {}, - "resources": {}, - } - ) - document = { - "schema_version": "electrace.workflow.v1", - "project": { - "root": "images", - "series_kind": "neb", - "workspace": "workspace", - "structure": "CONTCAR", - "fallback": "POSCAR", - }, - "frames": "all", - "executables": {"bader": {"path": "/opt/bader"}}, - "scheduler": {"kind": "local", "directives": {}}, - "steps": steps, - } - manifest = tmp_path / "workflow.yaml" - manifest.write_text(yaml.safe_dump(document, sort_keys=False), encoding="utf-8") +def _staged(factory, base): + manifest = factory( + base=base, + executables={"bader": "/opt/bader"}, + steps=[{"id": "charges", "kind": "bader", "executable": "bader"}], + ) staged = stage_workflow(manifest) - if write_outputs: - for paths in descriptor_outputs(staged).values(): - for path in paths: - path.write_text(ACF, encoding="utf-8") - return manifest, staged - - -def _first_descriptor(staged: StagedWorkflow) -> Path: - return next(iter(descriptor_outputs(staged).values()))[0] - - -def test_collect_copies_selected_evidence_and_emits_strict_config(tmp_path: Path) -> None: - manifest, staged = _stage_fixture(tmp_path) - source_text = (tmp_path / "images/08/POSCAR").read_text(encoding="utf-8") - - artifacts = collect_workflow_outputs(manifest) - - analysis_root = staged.workspace / "analysis" - assert artifacts.config_path == analysis_root / "electrace.toml" - assert artifacts.input_root == analysis_root / "input" - assert artifacts.descriptor_root == analysis_root / "descriptors" - for order, (source_label, energy) in enumerate((("08", -10.0), ("09", -9.75))): - label = f"{order:06d}-{source_label}" - assert (artifacts.input_root / label / "CONTCAR").read_text(encoding="utf-8") == POSCAR - assert read_vasp_energy(artifacts.input_root / label / "OSZICAR") == energy - assert (artifacts.descriptor_root / label / "ACF.dat").read_text(encoding="utf-8") == ACF - assert (tmp_path / "images/08/POSCAR").read_text(encoding="utf-8") == source_text - assert not (tmp_path / "images/analysis").exists() - - config = load_config(artifacts.config_path) - assert config.series_kind == "ordered" - assert config.inputs.root == "input" - assert config.inputs.descriptor_dir == "descriptors" - assert config.inputs.descriptor_source == "external" - assert config.matching == MatchingSettings() - assert config.events == EventSettings() - assert config.outputs.directory == "results" - assert config.metadata["workflow_manifest_sha256"] == staged.manifest_sha256 - assert config.metadata["workflow_frame_indices"] == [0, 1] - - marker = json.loads((analysis_root / COLLECTION_RECORD).read_text(encoding="utf-8")) - assert marker["schema_version"] == COLLECTION_SCHEMA_VERSION - assert marker["workflow_workspace_record"] == str(staged.record_path) - assert {record["role"] for record in marker["files"]} == { - "configuration", - "descriptor", - "structure", - "synthetic_energy", - } - assert set(artifacts.files) == {path for path in analysis_root.rglob("*") if path.is_file()} - - -@pytest.mark.parametrize("failure", ["missing", "empty", "symlink"]) -def test_collect_hard_fails_missing_empty_or_symlinked_descriptor_outputs( - tmp_path: Path, failure: str -) -> None: - manifest, staged = _stage_fixture(tmp_path, write_outputs=failure != "missing") - descriptor = _first_descriptor(staged) - external = tmp_path / "external-acf" - if failure == "empty": - descriptor.write_bytes(b"") - elif failure == "symlink": - descriptor.unlink() - external.write_text(ACF, encoding="utf-8") - descriptor.symlink_to(external) - - with pytest.raises((FileNotFoundError, ValueError), match=failure): - collect_workflow_outputs(manifest) - - assert not (staged.workspace / "analysis").exists() - if failure == "symlink": - assert external.read_text(encoding="utf-8") == ACF - - -def test_collect_parses_every_descriptor_before_publishing(tmp_path: Path) -> None: - manifest, staged = _stage_fixture(tmp_path) - _first_descriptor(staged).write_text("not an ACF file\n", encoding="utf-8") - - with pytest.raises(DescriptorError, match="no atomic charge records"): - collect_workflow_outputs(manifest) - - assert not (staged.workspace / "analysis").exists() - + for outputs in descriptor_outputs(staged).values(): + outputs[0].write_text(ACF, encoding="utf-8") + return manifest, base / "images/00/POSCAR" -def test_collect_rejects_descriptor_destination_collisions(tmp_path: Path) -> None: - manifest, staged = _stage_fixture(tmp_path, duplicate_bader=True) - with pytest.raises(ValueError, match="descriptor output collision"): - collect_workflow_outputs(manifest) +def test_collect_and_analyze_preserve_sources(tmp_path, poscar_text, workflow_manifest): + manifest, source = _staged(workflow_manifest, tmp_path) - assert not (staged.workspace / "analysis").exists() + collected = collect_workflow_outputs(manifest) + artifacts = analyze_collected_workflow(manifest) + assert source.read_text(encoding="utf-8") == poscar_text + assert (collected.input_root / "000000-00/CONTCAR").is_file() + assert (collected.descriptor_root / "000000-00/ACF.dat").is_file() + assert collected.config_path.is_file() + assert (artifacts.output_directory / "analysis.json").is_file() -def test_overwrite_replaces_only_a_marker_owned_analysis(tmp_path: Path) -> None: - manifest, staged = _stage_fixture(tmp_path) - artifacts = collect_workflow_outputs(manifest) - analysis_root = artifacts.config_path.parent - sentinel = analysis_root / "old-managed-file" - sentinel.write_text("old", encoding="utf-8") - with pytest.raises(FileExistsError, match="overwrite=True"): +def test_collect_refuses_missing_output_and_unmarked_replacement(tmp_path, workflow_manifest): + manifest, _ = _staged(workflow_manifest, tmp_path / "missing") + next((tmp_path / "missing/workspace").rglob("ACF.dat")).unlink() + with pytest.raises((FileNotFoundError, ValueError), match=r"output|supported"): collect_workflow_outputs(manifest) - collect_workflow_outputs(manifest, overwrite=True) - assert not sentinel.exists() - marker = analysis_root / COLLECTION_RECORD - marker.unlink() - protected = analysis_root / "user-file" - protected.write_text("preserve", encoding="utf-8") + manifest, _ = _staged(workflow_manifest, tmp_path / "unmarked") + analysis = tmp_path / "unmarked/workspace/analysis" + analysis.mkdir() + sentinel = analysis / "keep" + sentinel.write_text("user data", encoding="utf-8") with pytest.raises(ValueError, match="unmarked"): collect_workflow_outputs(manifest, overwrite=True) - assert protected.read_text(encoding="utf-8") == "preserve" - assert staged.record_path.is_file() - - -def test_failed_overwrite_preflight_preserves_prior_collection(tmp_path: Path) -> None: - manifest, staged = _stage_fixture(tmp_path) - artifacts = collect_workflow_outputs(manifest) - config_hash = artifacts.config_path.read_bytes() - _first_descriptor(staged).unlink() - - with pytest.raises(FileNotFoundError, match="missing"): - collect_workflow_outputs(manifest, overwrite=True) - - assert artifacts.config_path.read_bytes() == config_hash - assert (artifacts.config_path.parent / COLLECTION_RECORD).is_file() - - -def test_collect_rejects_symlinked_selected_structure(tmp_path: Path) -> None: - manifest, staged = _stage_fixture(tmp_path) - structure = staged.frames[0].structure - external = tmp_path / "external-structure" - external.write_text(POSCAR, encoding="utf-8") - structure.unlink() - structure.symlink_to(external) - - with pytest.raises(ValueError, match="symlink"): - collect_workflow_outputs(manifest) - - assert external.read_text(encoding="utf-8") == POSCAR - assert not (staged.workspace / "analysis").exists() - - -def test_collect_rejects_tampered_output_path_outside_job_directory(tmp_path: Path) -> None: - manifest, staged = _stage_fixture(tmp_path) - external = tmp_path / "external" / "ACF.dat" - external.parent.mkdir() - external.write_text(ACF, encoding="utf-8") - record = json.loads(staged.record_path.read_text(encoding="utf-8")) - descriptor_id = staged.descriptor_job_ids[0] - job = next(item for item in record["jobs"] if item["id"] == descriptor_id) - job["expected_outputs"] = [str(external)] - staged.record_path.write_text(json.dumps(record), encoding="utf-8") - - with pytest.raises(ValueError, match="escapes its managed root"): - collect_workflow_outputs(manifest) - - assert external.read_text(encoding="utf-8") == ACF - - -def test_collect_then_analyze_reuses_valid_collection_without_force(tmp_path: Path) -> None: - manifest, _ = _stage_fixture(tmp_path) - collected = collect_workflow_outputs(manifest) - - analysis = analyze_collected_workflow(manifest) - - assert analysis.output_directory == collected.config_path.parent / "results" - assert (analysis.output_directory / "analysis.json").is_file() - assert (analysis.output_directory / "manifest.json").is_file() - with pytest.raises(FileExistsError, match="overwrite=True"): - analyze_collected_workflow(manifest) - - replaced = analyze_collected_workflow(manifest, overwrite=True) - assert (replaced.output_directory / "analysis.json").is_file() + assert sentinel.read_text(encoding="utf-8") == "user data" diff --git a/tests/test_workflow_end_to_end.py b/tests/test_workflow_end_to_end.py index 7fb0d38..b85c1c3 100644 --- a/tests/test_workflow_end_to_end.py +++ b/tests/test_workflow_end_to_end.py @@ -1,233 +1,57 @@ -from __future__ import annotations - -import json from pathlib import Path -import yaml -from rich.text import Text -from typer.testing import CliRunner - -from electrace.cli import app -from electrace.workflow.collection import ( - analyze_collected_workflow, - collect_workflow_outputs, -) +from electrace.workflow.collection import analyze_collected_workflow, collect_workflow_outputs from electrace.workflow.operations import run_staged_workflow from electrace.workflow.staging import stage_workflow -runner = CliRunner() - -POSCAR = """\ -H2 -1.0 -2 0 0 -0 2 0 -0 0 2 -H -2 -Direct -0.0 0.0 0.0 -0.5 0.0 0.0 -""" - -def _write_executable(path: Path, body: str) -> None: +def _executable(path: Path, body: str) -> Path: path.write_text("#!/usr/bin/env python3\n" + body, encoding="utf-8") path.chmod(0o700) + return path -def _end_to_end_manifest(tmp_path: Path) -> Path: - images = tmp_path / "images" - for label in ("00", "01"): - frame = images / label - frame.mkdir(parents=True) - (frame / "POSCAR").write_text(POSCAR, encoding="utf-8") - - vasp_template = tmp_path / "templates" / "vasp" - vasp_template.mkdir(parents=True) - (vasp_template / "INCAR").write_text("ENCUT = 300\n", encoding="utf-8") - (vasp_template / "KPOINTS").write_text("Gamma\n0\nGamma\n1 1 1\n0 0 0\n", encoding="utf-8") - (vasp_template / "POTCAR").write_text("licensed-test-placeholder\n", encoding="utf-8") - lobster_template = tmp_path / "templates" / "lobster" - lobster_template.mkdir(parents=True) - (lobster_template / "lobsterin").write_text("cohpGenerator from 0.1 to 2.0\n", encoding="utf-8") - - _write_executable( - tmp_path / "fake_vasp.py", - "from pathlib import Path\n" - "import shutil\n" - "shutil.copyfile('POSCAR', 'CONTCAR')\n" - "for name in ('CHGCAR', 'ELFCAR', 'WAVECAR'):\n" - " Path(name).write_text(name + ' fixture')\n", - ) - _write_executable( - tmp_path / "fake_baderkit.py", - "from pathlib import Path\n" - "import json\n" - "index = int(Path.cwd().parent.name.split('-', 1)[0])\n" - "oxidation = float(index)\n" - "payload = {'method_kwargs': {'version': 'test'}, " - "'atom_results': {'atom_charges': [1.0, 1.0], " - "'oxidation_states': [oxidation, -oxidation], 'atom_volumes': [4.0, 4.0]}, " - "'maxima_results': {}}\n" - "Path('badelf.json').write_text(json.dumps(payload))\n", +def test_stage_run_collect_analyze_end_to_end(tmp_path, workflow_manifest): + vasp = _executable( + tmp_path / "vasp.py", + "from pathlib import Path\nimport shutil\n" + "Path('CHGCAR').write_text('density')\nshutil.copyfile('POSCAR', 'CONTCAR')\n", ) - _write_executable( - tmp_path / "fake_lobster.py", + bader = _executable( + tmp_path / "bader.py", "from pathlib import Path\n" - "index = int(Path.cwd().parent.name.split('-', 1)[0])\n" - "value = -1.0 - index\n" - "Path('ICOHPLIST.lobster').write_text(" - "f'No. atom1 atom2 distance translation ICOHP\\n'" - "+ f'1 H1 H2 1.00 0 0 0 {value}\\n')\n", + "i=int(Path.cwd().parent.name.split('-', 1)[0])\n" + "Path('ACF.dat').write_text(f'# X Y Z CHARGE MIN DIST ATOMIC VOL\\n'" + "+f'1 0 0 0 {1.0+i} 0 10\\n')\n", ) - - payload = { - "schema_version": "electrace.workflow.v1", - "project": { - "root": "images", - "series_kind": "neb", - "workspace": "workspace", - "structure": "CONTCAR", - "fallback": "POSCAR", - }, - "frames": "all", - "executables": { - "vasp": {"path": "./fake_vasp.py"}, - "baderkit": {"path": "./fake_baderkit.py"}, - "lobster": {"path": "./fake_lobster.py"}, - }, - "scheduler": {"kind": "local", "directives": {}}, - "steps": [ - { - "id": "vasp_elf", - "kind": "vasp", - "executable": "vasp", - "template_dir": "templates/vasp", - "options": { - "expected_outputs": ["CHGCAR", "ELFCAR", "CONTCAR"], - "incar_overrides": {"LELF": True, "LCHARG": True}, - }, - }, - { - "id": "badelf", - "kind": "baderkit", - "executable": "baderkit", - "capability": "elf_basins", - "depends_on": ["vasp_elf"], - }, + manifest = workflow_manifest( + executables={"vasp": str(vasp), "bader": str(bader)}, + steps=[ { - "id": "vasp_lobster", + "id": "density", "kind": "vasp", "executable": "vasp", - "template_dir": "templates/vasp", "options": { - "expected_outputs": ["WAVECAR", "CONTCAR"], - "incar_overrides": {"NSW": 0, "ISYM": -1, "LWAVE": True}, + "required_inputs": ["POSCAR"], + "expected_outputs": ["CHGCAR", "CONTCAR"], }, }, { - "id": "icohp", - "kind": "lobster", - "executable": "lobster", - "capability": "cohp", - "template_dir": "templates/lobster", - "depends_on": ["vasp_lobster"], + "id": "charges", + "kind": "bader", + "executable": "bader", + "depends_on": ["density"], }, ], - } - manifest = tmp_path / "workflow.yaml" - manifest.write_text(yaml.safe_dump(payload, sort_keys=False), encoding="utf-8") - return manifest - + ) -def test_full_fake_external_workflow_collects_and_analyzes_changes(tmp_path: Path) -> None: - manifest = _end_to_end_manifest(tmp_path) staged = stage_workflow(manifest) - results = run_staged_workflow(manifest, confirmed=True, max_parallel=2) - - assert len(results) == 8 - assert all(result.succeeded for result in results) collected = collect_workflow_outputs(manifest) - assert sorted(path.name for path in collected.descriptor_root.rglob("*.*")) == [ - "ICOHPLIST.lobster", - "ICOHPLIST.lobster", - "badelf.json", - "badelf.json", - ] - artifacts = analyze_collected_workflow(manifest) - analysis = artifacts.analysis - assert len(analysis["frames"]) == 2 - assert {frame["label"] for frame in analysis["frames"]} == { - staged.frames[0].analysis_label, - staged.frames[1].analysis_label, - } - serialized_events = json.dumps(analysis["events"], sort_keys=True) - assert "oxidation_state" in serialized_events - assert "icohp" in serialized_events + assert len(staged.frames) == 2 + assert len(results) == 4 and all(result.succeeded for result in results) + assert len(list(collected.descriptor_root.rglob("ACF.dat"))) == 2 + assert len(artifacts.analysis["frames"]) == 2 assert (artifacts.output_directory / "analysis.json").is_file() - - -def test_workflow_cli_runs_collects_and_analyzes_end_to_end(tmp_path: Path) -> None: - manifest = _end_to_end_manifest(tmp_path) - - validated = runner.invoke(app, ["workflow", "validate", str(manifest), "--json"]) - assert validated.exit_code == 0, validated.output - assert not (tmp_path / "workspace").exists() - validation = json.loads(validated.stdout) - assert validation["schema_version"] == "electrace.cli.v1" - assert validation["command"] == "workflow.validate" - assert len(validation["data"]["selected_frames"]) == 2 - assert len(validation["data"]["resolved_jobs"]) == 8 - - staged = runner.invoke(app, ["workflow", "stage", str(manifest), "--json"]) - executed = runner.invoke( - app, - [ - "workflow", - "run", - str(manifest), - "--yes", - "--max-parallel", - "2", - "--json", - ], - ) - status = runner.invoke(app, ["workflow", "status", str(manifest), "--json"]) - collected = runner.invoke(app, ["workflow", "collect", str(manifest), "--json"]) - analyzed = runner.invoke(app, ["workflow", "analyze", str(manifest), "--json"]) - - assert staged.exit_code == 0, staged.output - assert executed.exit_code == 0, executed.output - assert status.exit_code == 0, status.output - assert collected.exit_code == 0, collected.output - assert analyzed.exit_code == 0, analyzed.output - assert json.loads(staged.stdout)["command"] == "workflow.stage" - assert json.loads(executed.stdout)["data"]["counts"]["succeeded"] == 8 - assert json.loads(status.stdout)["data"]["complete"] is True - assert json.loads(collected.stdout)["command"] == "workflow.collect" - analysis_payload = json.loads(Text.from_ansi(analyzed.stdout).plain) - assert analysis_payload["command"] == "workflow.analyze" - assert analysis_payload["data"]["counts"]["frames"] == 2 - - -def test_volume_cli_reads_and_samples_plain_elfcar(tmp_path: Path) -> None: - elfcar = tmp_path / "ELFCAR" - elfcar.write_text( - "H ELF\n1\n1 0 0\n0 1 0\n0 0 1\nH\n1\nDirect\n0 0 0\n\n2 1 1\n0.0 1.0\n", - encoding="utf-8", - ) - - result = runner.invoke( - app, - ["volume", str(elfcar), "--sample", "0.25,0,0", "--json"], - ) - - assert result.exit_code == 0, result.output - payload = json.loads(Text.from_ansi(result.stdout).plain) - assert payload["schema_version"] == "electrace.cli.v1" - assert payload["data"]["shape"] == [2, 1, 1] - assert payload["data"]["sample"]["value"] == 0.5 diff --git a/tests/test_workflow_execution.py b/tests/test_workflow_execution.py index e03dfb9..861d6d4 100644 --- a/tests/test_workflow_execution.py +++ b/tests/test_workflow_execution.py @@ -1,424 +1,81 @@ from __future__ import annotations -import json -import os -import subprocess import sys from dataclasses import replace from pathlib import Path import pytest -from electrace.workflow.execution import ( - JobSpec, - render_slurm_script, - run_jobs, - submit_slurm_script, - validate_jobs, - write_slurm_script, -) +from electrace.workflow.execution import JobSpec, render_slurm_script, run_jobs def _job( root: Path, - job_id: str, + identifier: str, + code: str, *, - code: str = "print('ok')", - arguments: tuple[str, ...] = (), - stdin: str | None = None, - required_inputs: tuple[Path, ...] = (), - expected_outputs: tuple[Path, ...] = (), dependencies: tuple[str, ...] = (), - environment: dict[str, str] | None = None, + required: tuple[Path, ...] = (), ) -> JobSpec: + directory = root / identifier + directory.mkdir() + output = directory / "result" return JobSpec( - id=job_id, - frame_index=int(job_id.removeprefix("job") or 0), - frame_label=job_id, - task=f"test task {job_id}", - cwd=root.resolve(), - argv=(sys.executable, "-c", code, *arguments), - stdin=stdin, - stdout_path=(root / f"{job_id}.stdout").resolve(), - required_inputs=required_inputs, - expected_outputs=expected_outputs, + id=identifier, + frame_index=0, + frame_label="00", + task=identifier, + cwd=directory, + argv=(sys.executable, "-c", code, str(output)), + stdin=None, + stdout_path=directory / "stdout.log", + required_inputs=required, + expected_outputs=(output,), dependencies=dependencies, - environment=environment or {}, - ) - - -def test_job_spec_validates_paths_tokens_and_environment(tmp_path: Path) -> None: - with pytest.raises(ValueError, match="cwd must be an absolute path"): - JobSpec( - id="job0", - frame_index=0, - frame_label="00", - task="test", - cwd=Path("relative"), - argv=(sys.executable,), - stdin=None, - stdout_path=(tmp_path / "stdout").resolve(), - ) - with pytest.raises(ValueError, match=r"argv\[2\].*newline"): - _job(tmp_path, "job0", code="bad\nargument") - with pytest.raises(ValueError, match="invalid environment variable"): - _job(tmp_path, "job0", environment={"BAD=NAME": "value"}) - with pytest.raises(ValueError, match=r"environment value.*newline"): - _job(tmp_path, "job0", environment={"SAFE_NAME": "bad\nvalue"}) - - -def test_validate_jobs_returns_stable_dependency_order_and_rejects_bad_graphs( - tmp_path: Path, -) -> None: - first = _job(tmp_path, "job0") - second = _job(tmp_path, "job1", dependencies=(first.id,)) - - assert [job.id for job in validate_jobs((second, first))] == ["job0", "job1"] - with pytest.raises(ValueError, match="duplicate job id"): - validate_jobs((first, first)) - with pytest.raises(ValueError, match="unknown dependencies"): - validate_jobs((replace(second, dependencies=("missing",)),)) - cyclic_first = replace(first, dependencies=(second.id,)) - with pytest.raises(ValueError, match="contains a cycle"): - validate_jobs((cyclic_first, second)) - with pytest.raises(ValueError, match="at least one job"): - run_jobs((), confirmed=True, max_parallel=10_000) - - -def test_run_jobs_requires_confirmation_without_starting_process(tmp_path: Path) -> None: - marker = tmp_path / "must-not-exist" - job = _job( - tmp_path, - "job0", - code="from pathlib import Path; import sys; Path(sys.argv[1]).write_text('ran')", - arguments=(str(marker),), ) - with pytest.raises(PermissionError, match="confirmed=True"): - run_jobs((job,)) - - assert not marker.exists() - assert not job.stdout_path.exists() - -def test_run_jobs_executes_dependency_dag_and_emits_json_ready_results( - tmp_path: Path, -) -> None: - required = tmp_path / "required.txt" - required.write_text("input", encoding="utf-8") - product = tmp_path / "product.txt" - first = _job( - tmp_path, - "job0", - code=( - "from pathlib import Path; import os, sys; " - "Path(sys.argv[1]).write_text(Path(sys.argv[2]).read_text() + '-product'); " - "print(os.environ['ELECTRACE_TEST_VALUE']); " - "print('first warning', file=sys.stderr)" - ), - arguments=(str(product), str(required)), - stdin="ignored input\n", - required_inputs=(required,), - expected_outputs=(product,), - environment={"ELECTRACE_TEST_VALUE": "environment value"}, - ) - second = _job( +def test_local_dag_requires_confirmation_and_runs_in_parallel(tmp_path: Path) -> None: + write = "from pathlib import Path; import sys; Path(sys.argv[1]).write_text('ok')" + first = _job(tmp_path, "first", write) + independent = _job(tmp_path, "independent", write) + dependent = _job( tmp_path, - "job1", - code="from pathlib import Path; import sys; print(Path(sys.argv[1]).read_text())", - arguments=(str(product),), - required_inputs=(product,), + "dependent", + write, dependencies=(first.id,), + required=(first.expected_outputs[0],), ) - results = run_jobs((second, first), confirmed=True, max_parallel=2) + with pytest.raises(PermissionError, match="confirmed=True"): + run_jobs((dependent, independent, first), max_parallel=2) + results = run_jobs((dependent, independent, first), confirmed=True, max_parallel=2) - assert [result.id for result in results] == ["job0", "job1"] + assert [result.id for result in results] == ["independent", "first", "dependent"] assert all(result.succeeded for result in results) - assert product.read_text(encoding="utf-8") == "input-product" - assert first.stdout_path.read_text(encoding="utf-8") == "environment value\n" - assert first.stderr_path.read_text(encoding="utf-8") == "first warning\n" - assert second.stdout_path.read_text(encoding="utf-8") == "input-product\n" - json.dumps([result.to_dict() for result in results], allow_nan=False) - - -def test_missing_input_fails_job_and_blocks_its_dependency(tmp_path: Path) -> None: - missing = tmp_path / "missing-input" - dependent_marker = tmp_path / "dependent-ran" - first = _job(tmp_path, "job0", required_inputs=(missing,)) - second = _job( - tmp_path, - "job1", - code="from pathlib import Path; import sys; Path(sys.argv[1]).write_text('ran')", - arguments=(str(dependent_marker),), - dependencies=(first.id,), + assert all( + path.is_file() for job in (first, independent, dependent) for path in job.expected_outputs ) - first_result, second_result = run_jobs((first, second), confirmed=True, stop_on_error=False) - - assert first_result.status == "failed" - assert first_result.returncode is None - assert "required inputs are missing" in (first_result.error or "") - assert second_result.status == "blocked" - assert "dependency did not succeed" in (second_result.error or "") - assert not dependent_marker.exists() - -def test_stop_on_error_prevents_later_independent_job(tmp_path: Path) -> None: - marker = tmp_path / "later-ran" - failing = _job(tmp_path, "job0", code="raise SystemExit(7)") - later = _job( +def test_failure_blocks_dependents_and_slurm_commands_are_quoted(tmp_path: Path) -> None: + failing = _job(tmp_path, "fail", "raise SystemExit(2)") + dependent = _job( tmp_path, - "job1", - code="from pathlib import Path; import sys; Path(sys.argv[1]).write_text('ran')", - arguments=(str(marker),), + "after", + "raise SystemExit('must not run')", + dependencies=(failing.id,), ) - failed, blocked = run_jobs((failing, later), confirmed=True, max_parallel=1, stop_on_error=True) + failed, blocked = run_jobs((failing, dependent), confirmed=True, stop_on_error=False) assert failed.status == "failed" - assert failed.returncode == 7 assert blocked.status == "blocked" - assert not marker.exists() - - -def test_keep_going_runs_independent_work_and_transitively_blocks_dependents( - tmp_path: Path, -) -> None: - marker = tmp_path / "independent-ran" - failing = _job(tmp_path, "job0", code="raise SystemExit(9)") - independent = _job( - tmp_path, - "job1", - code="from pathlib import Path; import sys; Path(sys.argv[1]).write_text('ran')", - arguments=(str(marker),), - ) - dependent = _job( - tmp_path, - "job2", - dependencies=(failing.id, independent.id), - ) - grandchild = _job(tmp_path, "job3", dependencies=(dependent.id,)) - - results = run_jobs( - (grandchild, dependent, independent, failing), - confirmed=True, - max_parallel=2, - stop_on_error=False, - ) - - by_id = {result.id: result for result in results} - assert {job_id: result.status for job_id, result in by_id.items()} == { - "job0": "failed", - "job1": "success", - "job2": "blocked", - "job3": "blocked", - } - assert by_id["job2"].error == "dependency did not succeed: job0" - assert by_id["job3"].error == "dependency did not succeed: job2" - assert marker.read_text(encoding="utf-8") == "ran" - - -def test_failed_dependency_blocks_a_long_chain_without_recursion(tmp_path: Path) -> None: - chain_length = 1_500 - jobs = [_job(tmp_path, "job0", required_inputs=(tmp_path / "missing",))] - jobs.extend( - _job(tmp_path, f"job{index}", dependencies=(f"job{index - 1}",)) - for index in range(1, chain_length) - ) - - results = run_jobs(jobs, confirmed=True, stop_on_error=False) - - assert len(results) == chain_length - assert results[0].status == "failed" - assert all(result.status == "blocked" for result in results[1:]) - assert results[-1].error == f"dependency did not succeed: job{chain_length - 2}" - - -@pytest.mark.parametrize("symlink_field", ["stdout", "expected"]) -def test_run_jobs_never_follows_output_symlinks(tmp_path: Path, symlink_field: str) -> None: - external = tmp_path / "external-target" - external.write_text("unchanged", encoding="utf-8") - marker = tmp_path / "process-ran" - expected = tmp_path / "expected-output" - job = _job( - tmp_path, - "job0", - code="from pathlib import Path; import sys; Path(sys.argv[1]).write_text('ran')", - arguments=(str(marker),), - expected_outputs=(expected,) if symlink_field == "expected" else (), - ) - symlink = job.stdout_path if symlink_field == "stdout" else expected - symlink.symlink_to(external) - - (result,) = run_jobs((job,), confirmed=True) - - assert result.status == "failed" - assert "symlink" in (result.error or "") - assert external.read_text(encoding="utf-8") == "unchanged" - assert not marker.exists() - - -def test_run_jobs_rejects_symlinked_output_parent(tmp_path: Path) -> None: - external_directory = tmp_path / "external-directory" - external_directory.mkdir() - linked_directory = tmp_path / "linked-directory" - linked_directory.symlink_to(external_directory, target_is_directory=True) - marker = tmp_path / "process-ran" - job = JobSpec( - id="job0", - frame_index=0, - frame_label="00", - task="parent symlink regression", - cwd=tmp_path.resolve(), - argv=( - sys.executable, - "-c", - "from pathlib import Path; import sys; Path(sys.argv[1]).write_text('ran')", - str(marker), - ), - stdin=None, - stdout_path=linked_directory / "stdout.log", - ) - - (result,) = run_jobs((job,), confirmed=True) - - assert result.status == "failed" - assert "symlink component" in (result.error or "") - assert not marker.exists() - assert not (external_directory / "stdout.log").exists() - - -def test_run_jobs_rejects_expected_symlink_created_by_process(tmp_path: Path) -> None: - external = tmp_path / "external-target" - external.write_text("unchanged", encoding="utf-8") - expected = tmp_path / "expected-output" - job = _job( - tmp_path, - "job0", - code="from pathlib import Path; import sys; Path(sys.argv[1]).symlink_to(sys.argv[2])", - arguments=(str(expected), str(external)), - expected_outputs=(expected,), - ) - - (result,) = run_jobs((job,), confirmed=True) - - assert result.status == "failed" - assert "unsafe symlinks" in (result.error or "") - assert external.read_text(encoding="utf-8") == "unchanged" - - -def test_rendered_slurm_script_orders_dependencies_and_quotes_untrusted_tokens( - tmp_path: Path, -) -> None: - injection_marker = tmp_path / "injection-marker" - expected = tmp_path / "expected.txt" - hostile_argument = f"value; touch {injection_marker}" - hostile_environment = f"$(touch {injection_marker})" - first = _job( - tmp_path, - "job0", - code=( - "from pathlib import Path; import os, sys; " - "Path(sys.argv[1]).write_text('ready'); " - "print(sys.argv[2]); print(os.environ['ELECTRACE_PAYLOAD'])" - ), - arguments=(str(expected), hostile_argument), - expected_outputs=(expected,), - environment={"ELECTRACE_PAYLOAD": hostile_environment}, - ) - second = _job( - tmp_path, - "job1", - code="print('dependent')", - dependencies=(first.id,), - ) - - document = render_slurm_script( - (second, first), - { - "partition": "gpu", - "exclude": "node[1-8]", - "exclusive": True, - "nice": 1.5, - }, - "inpaint_test", - ) - script_path = write_slurm_script(document, (tmp_path / "jobs.slurm").resolve()) - completed = subprocess.run( - ["/bin/sh", str(script_path)], - capture_output=True, - text=True, - check=False, - shell=False, - ) - - assert document.startswith("#!/bin/sh\n#SBATCH --job-name=inpaint_test\n") - assert "#SBATCH --exclude=node[1-8]" in document - assert "#SBATCH --nice=1.5" in document - assert document.index("ElecTrace job job0") < document.index("ElecTrace job job1") - assert completed.returncode == 0, completed.stderr - assert expected.read_text(encoding="utf-8") == "ready" - assert first.stdout_path.read_text(encoding="utf-8") == ( - f"{hostile_argument}\n{hostile_environment}\n" - ) - assert second.stdout_path.read_text(encoding="utf-8") == "dependent\n" - assert not injection_marker.exists() - - -@pytest.mark.parametrize( - ("directives", "job_name"), - [ - ({"partition\n--exclusive": "gpu"}, "safe"), - ({"partition": "gpu\n#SBATCH --exclusive"}, "safe"), - ({"wrap": "python-code"}, "safe"), - ({"partition": "gpu"}, "unsafe\nname"), - ], -) -def test_render_slurm_script_rejects_directive_and_name_injection( - tmp_path: Path, - directives: dict[str, str], - job_name: str, -) -> None: - with pytest.raises(ValueError, match=r"unsafe|job_name|executable content"): - render_slurm_script((_job(tmp_path, "job0"),), directives, job_name) - - -def test_write_slurm_script_atomically_replaces_symlink_not_target(tmp_path: Path) -> None: - external = tmp_path / "external-script" - external.write_text("external", encoding="utf-8") - output = (tmp_path / "job.slurm").resolve() - output.symlink_to(external) - - written = write_slurm_script("#!/bin/sh\nexit 0\n", output, overwrite=True) - - assert written == output - assert not output.is_symlink() - assert output.read_text(encoding="utf-8") == "#!/bin/sh\nexit 0\n" - assert external.read_text(encoding="utf-8") == "external" - assert os.access(output, os.X_OK) - - -def test_submit_slurm_script_requires_confirmation_and_parses_fake_job_id( - tmp_path: Path, -) -> None: - script = write_slurm_script("#!/bin/sh\nexit 0\n", (tmp_path / "job.slurm").resolve()) - fake_submit = tmp_path / "fake_submit.py" - fake_submit.write_text( - "import pathlib, sys\n" - "assert pathlib.Path(sys.argv[1]).is_file()\n" - "print('Submitted batch job 424242')\n", - encoding="utf-8", - ) - - with pytest.raises(PermissionError, match="confirmed=True"): - submit_slurm_script(script, (sys.executable, str(fake_submit))) - - assert ( - submit_slurm_script( - script, - (sys.executable, str(fake_submit)), - confirmed=True, - ) - == "424242" - ) + hostile = replace( + failing, + argv=(sys.executable, "value; touch never"), + expected_outputs=(), + ) + script = render_slurm_script((hostile,), {"partition": "cpu"}, "inpaint_test") + assert "'value; touch never'" in script + assert "#SBATCH --job-name=inpaint_test" in script diff --git a/tests/test_workflow_manifest.py b/tests/test_workflow_manifest.py index c2954a4..dee966c 100644 --- a/tests/test_workflow_manifest.py +++ b/tests/test_workflow_manifest.py @@ -1,428 +1,108 @@ from __future__ import annotations -import copy -import json +from collections.abc import Callable from pathlib import Path -from typing import Any import pytest +import yaml -import electrace.workflow.manifest as manifest_module -from electrace.io.discovery import FrameSource from electrace.workflow import ( - WORKFLOW_SCHEMA_VERSION, - ExecutableSpec, FrameSelectionError, - SchedulerSpec, WorkflowManifest, - WorkflowProject, - WorkflowStep, + WorkflowManifestError, dump_starter_manifest, load_workflow_manifest, parse_frame_selection, - resolve_frame_selection, - resolve_selected_frames, starter_workflow_manifest, ) -def _manifest_data() -> dict[str, Any]: +def _manifest() -> dict[str, object]: return { - "schema_version": WORKFLOW_SCHEMA_VERSION, + "schema_version": "electrace.workflow.v1", "project": { - "root": "./images", + "root": "images", "series_kind": "neb", - "workspace": ".electrace-work", + "workspace": "work", "structure": "CONTCAR", "fallback": "POSCAR", }, "frames": "0,2:5:2", - "executables": { - "vasp": {"path": "vasp_std", "argv_prefix": ["srun", "--cpu-bind=cores"]}, - "bader": {"path": "/opt/bader/bader"}, - }, - "scheduler": { - "kind": "slurm", - "directives": { - "partition": "compute", - "time": "01:00:00", - "nodes": 1, - "exclusive": False, - }, - }, + "executables": {"vasp": {"path": "vasp_std"}, "baderkit": {"path": "baderkit"}}, + "scheduler": {"kind": "local", "directives": {}}, "steps": [ - { - "id": "vasp", - "kind": "vasp", - "executable": "vasp", - "template_dir": "templates/vasp", - "options": {"restart": False, "encut": 600.0}, - "resources": {"ranks": 8}, - }, + {"id": "vasp", "kind": "vasp", "executable": "vasp"}, { "id": "bader", - "kind": "bader", - "executable": "bader", - "capability": "atomic_charge", + "kind": "baderkit", + "executable": "baderkit", + "capability": "bader_charges", "depends_on": ["vasp"], - "options": {"reference": "CHGCAR_sum"}, }, ], } -def _write_data(path: Path, data: object) -> Path: - path.write_text(json.dumps(data, indent=2, allow_nan=False), encoding="utf-8") +def _write(path: Path, data: object) -> Path: + path.write_text(yaml.safe_dump(data, sort_keys=False), encoding="utf-8") return path -def _frame(tmp_path: Path, index: int) -> FrameSource: - directory = tmp_path / f"{index:02d}" - return FrameSource( - index=index, - label=directory.name, - directory=directory, - structure=directory / "CONTCAR", - metadata={"directory_index": index + 8}, - ) - - -def test_valid_manifest_loads_to_typed_json_ready_models(tmp_path: Path) -> None: - source = _write_data(tmp_path / "workflow.yaml", _manifest_data()) - +def test_manifest_round_trip_and_starter_write(tmp_path: Path) -> None: + source = _write(tmp_path / "workflow.yaml", _manifest()) manifest = load_workflow_manifest(source) + assert WorkflowManifest.from_dict(manifest.to_dict()) == manifest + assert [step.id for step in manifest.steps] == ["vasp", "bader"] - assert manifest.schema_version == WORKFLOW_SCHEMA_VERSION - assert manifest.project == WorkflowProject( - root="./images", - series_kind="neb", - workspace=".electrace-work", - structure="CONTCAR", - fallback="POSCAR", - ) - assert manifest.project.root == "./images" - assert manifest.executables["vasp"].argv_prefix == ("srun", "--cpu-bind=cores") - assert manifest.scheduler.directives["nodes"] == 1 - assert manifest.steps[1].kind == "bader" - assert manifest.steps[1].depends_on == ("vasp",) - encoded = json.dumps(manifest.to_dict(), allow_nan=False) - assert '"schema_version": "electrace.workflow.v1"' in encoded - - -def test_programmatic_dataclasses_enforce_and_serialize_the_contract() -> None: - manifest = WorkflowManifest( - schema_version=WORKFLOW_SCHEMA_VERSION, - project=WorkflowProject(".", "static", "work", "CONTCAR", "POSCAR"), - frames="1:5:2", - executables={"critic2": ExecutableSpec("critic2", ("srun",))}, - scheduler=SchedulerSpec("local"), - steps=( - WorkflowStep( - id="critic2", - kind="critic2", - executable="critic2", - capability="qtaim_critical_points", - options={"nested": {"thresholds": [0.1, 0.2]}}, - ), - ), - ) - - assert manifest.to_dict()["executables"] == { - "critic2": {"path": "critic2", "argv_prefix": ["srun"]} - } - assert manifest.to_dict()["steps"][0]["depends_on"] == [] - - -def test_starter_dump_round_trips_and_preserves_existing_file(tmp_path: Path) -> None: - target = tmp_path / "nested" / "workflow.yaml" - written = dump_starter_manifest(target, root="./images") - - assert written == target - assert ( - load_workflow_manifest(target).to_dict() - == starter_workflow_manifest(root="./images").to_dict() - ) - starter = load_workflow_manifest(target) - assert starter.steps[0].options["incar_overrides"] == { - "LELF": True, - "LCHARG": True, - "NSW": 0, - } + starter_path = tmp_path / "starter.yaml" + assert dump_starter_manifest(starter_path, root="/images") == starter_path + starter = load_workflow_manifest(starter_path) + assert starter == starter_workflow_manifest(root="/images") with pytest.raises(FileExistsError): - dump_starter_manifest(target) - - dump_starter_manifest(target, root="./replacement", overwrite=True) - assert load_workflow_manifest(target).project.root == "./replacement" - - -def test_starter_force_refuses_to_follow_manifest_symlink(tmp_path: Path) -> None: - protected = tmp_path / "protected.yaml" - protected.write_text("user content\n", encoding="utf-8") - target = tmp_path / "workflow.yaml" - target.symlink_to(protected) - - with pytest.raises(ValueError, match="symlinked workflow manifest"): - dump_starter_manifest(target, overwrite=True) - - assert target.is_symlink() - assert protected.read_text(encoding="utf-8") == "user content\n" - - -@pytest.mark.parametrize( - ("section", "unknown_key"), - [ - ("root", "matcing"), - ("project", "series_knd"), - ("executable", "arguments"), - ("scheduler", "queue"), - ("step", "after"), - ], -) -def test_unknown_fixed_keys_are_rejected(tmp_path: Path, section: str, unknown_key: str) -> None: - data = _manifest_data() - if section == "root": - data[unknown_key] = {} - elif section == "project": - data["project"][unknown_key] = "neb" - elif section == "executable": - data["executables"]["vasp"][unknown_key] = [] - elif section == "scheduler": - data["scheduler"][unknown_key] = "compute" - else: - data["steps"][0][unknown_key] = "vasp" - - with pytest.raises(ValueError, match="unknown keys"): - load_workflow_manifest(_write_data(tmp_path / f"{section}.yaml", data)) + dump_starter_manifest(starter_path) + dump_starter_manifest(starter_path, overwrite=True) @pytest.mark.parametrize( - ("mutate", "message"), + "mutation", [ - (lambda data: data.update(schema_version="electrace.workflow.v2"), "schema_version"), - (lambda data: data.update(frames=[0, 1]), "frames must be a string"), - ( - lambda data: data["project"].update(series_kind=True), - "project.series_kind must be a string", - ), - ( - lambda data: data["executables"]["vasp"].update(argv_prefix="srun"), - "argv_prefix must be a sequence", - ), - ( - lambda data: data["scheduler"].update(directives=[]), - "scheduler.directives must be a mapping", - ), - ( - lambda data: data["steps"][1].update(depends_on="vasp"), - "depends_on must be a sequence", - ), + lambda data: data.update(schema_version="wrong"), + lambda data: data["steps"].append(dict(data["steps"][0])), + lambda data: data["steps"][1].update(executable="missing"), + lambda data: data["steps"][0].update(depends_on=["bader"]), + lambda data: data.update(extra=True), ], ) -def test_wrong_manifest_field_types_are_rejected(tmp_path: Path, mutate: Any, message: str) -> None: - data = _manifest_data() - mutate(data) - - with pytest.raises((TypeError, ValueError), match=message): - load_workflow_manifest(_write_data(tmp_path / "wrong-type.yaml", data)) - - -def test_all_declared_step_kinds_are_accepted(tmp_path: Path) -> None: - for kind in ("vasp", "bader", "baderkit", "critic2", "lobster"): - data = _manifest_data() - data["executables"] = {kind: {"path": kind}} - data["steps"] = [{"id": kind, "kind": kind, "executable": kind}] - manifest = load_workflow_manifest(_write_data(tmp_path / f"{kind}.yaml", data)) - assert manifest.steps[0].kind == kind - - -def test_dependency_identity_and_executable_references_are_strict(tmp_path: Path) -> None: - cases: list[tuple[dict[str, Any], str]] = [] - - duplicate = _manifest_data() - duplicate["steps"][1]["id"] = "vasp" - cases.append((duplicate, "duplicate ids")) - - executable = _manifest_data() - executable["steps"][1]["executable"] = "missing" - cases.append((executable, "unknown executable")) - - unknown = _manifest_data() - unknown["steps"][1]["depends_on"] = ["density"] - cases.append((unknown, "unknown dependencies")) - - forward = _manifest_data() - forward["steps"][0]["depends_on"] = ["bader"] - forward["steps"][1]["depends_on"] = [] - cases.append((forward, "forward dependencies")) - - cycle = _manifest_data() - cycle["steps"][0]["depends_on"] = ["bader"] - cycle["steps"][1]["depends_on"] = ["vasp"] - cases.append((cycle, "dependency cycle")) - - for index, (data, message) in enumerate(cases): - with pytest.raises(ValueError, match=message): - load_workflow_manifest(_write_data(tmp_path / f"dependency-{index}.yaml", data)) - - -def test_duplicate_yaml_mapping_keys_are_rejected(tmp_path: Path) -> None: - source = tmp_path / "duplicate.yaml" - source.write_text( - "schema_version: electrace.workflow.v1\nschema_version: electrace.workflow.v1\n", - encoding="utf-8", - ) - - with pytest.raises(ValueError, match="duplicate mapping key 'schema_version'"): - load_workflow_manifest(source) - - -def test_manifest_byte_limit_is_enforced_during_the_read( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, +def test_manifest_rejects_invalid_contract( + tmp_path: Path, mutation: Callable[[dict[str, object]], None] ) -> None: - source = _write_data(tmp_path / "large.yaml", _manifest_data()) - monkeypatch.setattr(manifest_module, "MAX_WORKFLOW_MANIFEST_BYTES", 64) - - with pytest.raises(ValueError, match="exceeds 64 bytes"): - load_workflow_manifest(source) - - -def test_yaml_only_and_non_finite_values_are_rejected(tmp_path: Path) -> None: - date_data = _manifest_data() - date_text = json.dumps(date_data, indent=2).replace( - '"options": {\n "restart": false,', - '"options": {\n "submitted": 2026-08-08,\n "restart": false,', - 1, - ) - date_source = tmp_path / "date.yaml" - date_source.write_text(date_text, encoding="utf-8") - with pytest.raises(ValueError, match="non-JSON-compatible value of type date"): - load_workflow_manifest(date_source) - - non_finite = _manifest_data() - non_finite_text = json.dumps(non_finite, indent=2).replace('"encut": 600.0', '"encut": .nan') - non_finite_source = tmp_path / "nan.yaml" - non_finite_source.write_text(non_finite_text, encoding="utf-8") - with pytest.raises(ValueError, match="finite for JSON serialization"): - load_workflow_manifest(non_finite_source) - - set_data = _manifest_data() - set_text = json.dumps(set_data, indent=2).replace( - '"reference": "CHGCAR_sum"', '"reference": !!set {CHGCAR: null}' - ) - set_source = tmp_path / "set.yaml" - set_source.write_text(set_text, encoding="utf-8") - with pytest.raises(ValueError, match="non-JSON-compatible value of type set"): - load_workflow_manifest(set_source) - - alias_source = tmp_path / "alias.yaml" - alias_source.write_text("value: &shared [1, 2]\ncopy: *shared\n", encoding="utf-8") - with pytest.raises(ValueError, match="aliases are not allowed"): - load_workflow_manifest(alias_source) - - -def test_programmatic_recursive_json_values_are_rejected() -> None: - recursive: list[Any] = [] - recursive.append(recursive) - - with pytest.raises(ValueError, match="cyclic sequence"): - WorkflowStep( - id="vasp", - kind="vasp", - executable="vasp", - options={"recursive": recursive}, - ) - + data = _manifest() + mutation(data) + with pytest.raises((TypeError, ValueError)): + load_workflow_manifest(_write(tmp_path / "bad.yaml", data)) -def test_control_characters_and_command_directives_are_rejected(tmp_path: Path) -> None: - newline = _manifest_data() - newline["steps"][0]["options"]["note"] = "safe\n--wrap=payload" - with pytest.raises(ValueError, match="control characters or newlines"): - load_workflow_manifest(_write_data(tmp_path / "newline.yaml", newline)) - wrap = _manifest_data() - wrap["scheduler"]["directives"]["wrap"] = "arbitrary command" - with pytest.raises(ValueError, match="not allowed to embed executable content"): - load_workflow_manifest(_write_data(tmp_path / "wrap.yaml", wrap)) +def test_malformed_yaml_is_rejected(tmp_path: Path) -> None: + alias = tmp_path / "bad.yaml" + alias.write_text("value: [unclosed\n", encoding="utf-8") + with pytest.raises(WorkflowManifestError): + load_workflow_manifest(alias) - directive = _manifest_data() - directive["scheduler"]["directives"]["--partition"] = "compute" - with pytest.raises(ValueError, match="unsafe characters"): - load_workflow_manifest(_write_data(tmp_path / "directive.yaml", directive)) - underscore = _manifest_data() - underscore["scheduler"]["directives"]["cpus_per_task"] = 2 - with pytest.raises(ValueError, match="unsafe characters"): - load_workflow_manifest(_write_data(tmp_path / "underscore.yaml", underscore)) +def test_duplicate_yaml_keys_are_rejected(tmp_path: Path) -> None: + manifest = tmp_path / "duplicate.yaml" + manifest.write_text("schema_version: first\nschema_version: second\n", encoding="utf-8") + with pytest.raises(WorkflowManifestError, match=r"duplicate.*schema_version"): + load_workflow_manifest(manifest) - job_name = _manifest_data() - job_name["scheduler"]["directives"]["job-name"] = "embedded-name" - with pytest.raises(ValueError, match="reserved"): - load_workflow_manifest(_write_data(tmp_path / "job-name.yaml", job_name)) - unsafe_value = _manifest_data() - unsafe_value["scheduler"]["directives"]["comment"] = "contains whitespace" - with pytest.raises(ValueError, match="unsafe string value"): - load_workflow_manifest(_write_data(tmp_path / "directive-value.yaml", unsafe_value)) - - local = _manifest_data() - local["scheduler"]["kind"] = "local" - with pytest.raises(ValueError, match="must be empty"): - load_workflow_manifest(_write_data(tmp_path / "local.yaml", local)) - - -def test_parse_frame_selection_supports_indices_and_python_slices() -> None: +def test_frame_selection_supports_indices_slices_and_reverse_order() -> None: assert parse_frame_selection("all", 5) == (0, 1, 2, 3, 4) - assert parse_frame_selection("0, 2:5:2", 5) == (0, 2, 4) - assert parse_frame_selection("-3:", 5) == (2, 3, 4) - assert parse_frame_selection("::-2", 5) == (4, 2, 0) - assert parse_frame_selection("1:99", 5) == (1, 2, 3, 4) - - -@pytest.mark.parametrize( - "selection", - ["", "ALL", "all,1", "1,,2", "frame-1", "-1", "1:2:0", "1:2:3:4"], -) -def test_malformed_frame_selections_are_rejected(selection: str) -> None: - with pytest.raises(FrameSelectionError): - parse_frame_selection(selection, 4) - - -def test_unresolvable_or_duplicate_frame_selections_are_rejected() -> None: - with pytest.raises(FrameSelectionError, match="outside"): - parse_frame_selection("4", 4) - with pytest.raises(FrameSelectionError, match="more than once"): - parse_frame_selection("0:2,1", 4) - with pytest.raises(FrameSelectionError, match="no frames"): - parse_frame_selection("2:2", 4) - with pytest.raises(TypeError, match="integer"): - parse_frame_selection("all", True) - - -def test_selection_resolves_canonical_frame_sources(tmp_path: Path) -> None: - frames = tuple(_frame(tmp_path, index) for index in range(5)) - - selected = resolve_frame_selection("0,2:5:2", frames) - wrapped = resolve_selected_frames(frames, "::-2") - - assert [frame.index for frame in selected] == [0, 2, 4] - assert [frame.index for frame in wrapped] == [4, 2, 0] - assert [frame.metadata["directory_index"] for frame in selected] == [8, 10, 12] - - -def test_selection_rejects_noncanonical_frame_source_order(tmp_path: Path) -> None: - frames = [_frame(tmp_path, 0), _frame(tmp_path, 2)] - - with pytest.raises(FrameSelectionError, match="canonical order"): - resolve_frame_selection("all", frames) - - -def test_nested_input_mapping_is_copied_before_validation(tmp_path: Path) -> None: - data = _manifest_data() - original = copy.deepcopy(data) + assert parse_frame_selection("0,2:6:2", 6) == (0, 2, 4) + assert parse_frame_selection("4:0:-2", 5) == (4, 2) - manifest = load_workflow_manifest(_write_data(tmp_path / "copy.yaml", data)) - manifest.steps[0].options["encut"] = 700 - assert data == original - assert manifest.to_dict()["steps"][0]["options"]["encut"] == 700 +@pytest.mark.parametrize("selection", ["", "all,0", "-1", "0::0", "0,0", "9"]) +def test_frame_selection_rejects_ambiguous_or_empty_requests(selection: str) -> None: + with pytest.raises((FrameSelectionError, ValueError)): + parse_frame_selection(selection, 5) diff --git a/tests/test_workflow_operations.py b/tests/test_workflow_operations.py index 5bc50d1..7c13214 100644 --- a/tests/test_workflow_operations.py +++ b/tests/test_workflow_operations.py @@ -1,15 +1,8 @@ -from __future__ import annotations - import json import sys -from pathlib import Path import pytest -import yaml -from typer.testing import CliRunner -import electrace.workflow.operations as workflow_operations -from electrace.cli import app from electrace.workflow.operations import ( SlurmSubmissionError, prepare_slurm_scripts, @@ -19,464 +12,68 @@ ) from electrace.workflow.staging import stage_workflow -POSCAR = """\ -H -1 -1 0 0 -0 1 0 -0 0 1 -H -1 -Direct -0 0 0 -""" -runner = CliRunner() - -def _manifest(tmp_path: Path, scheduler: str) -> Path: - images = tmp_path / "images" - for label in ("00", "01"): - frame = images / label - frame.mkdir(parents=True) - (frame / "POSCAR").write_text(POSCAR, encoding="utf-8") - executable = tmp_path / "fake_vasp.py" - executable.write_text( - "#!/usr/bin/env python3\nfrom pathlib import Path\nPath('CHGCAR').write_text('density')\n", - encoding="utf-8", +def _manifest(factory, base, scheduler): + script = base / "fake.py" + base.mkdir(parents=True, exist_ok=True) + script.write_text( + "from pathlib import Path; Path('CHGCAR').write_text('density')", encoding="utf-8" + ) + return factory( + base=base, + scheduler=scheduler, + executables={"vasp": sys.executable}, + step_options={"arguments": [str(script)]}, ) - executable.chmod(0o700) - directives = {"partition": "compute", "exclude": "node[1-8]"} if scheduler == "slurm" else {} - payload = { - "schema_version": "electrace.workflow.v1", - "project": { - "root": "images", - "series_kind": "neb", - "workspace": "workspace", - "structure": "CONTCAR", - "fallback": "POSCAR", - }, - "frames": "all", - "executables": {"vasp": {"path": "./fake_vasp.py"}}, - "scheduler": {"kind": scheduler, "directives": directives}, - "steps": [ - { - "id": "density", - "kind": "vasp", - "executable": "vasp", - "depends_on": [], - "options": { - "required_inputs": ["POSCAR"], - "expected_outputs": ["CHGCAR"], - }, - "resources": {}, - } - ], - } - path = tmp_path / "workflow.yaml" - path.write_text(yaml.safe_dump(payload, sort_keys=False), encoding="utf-8") - return path - -def test_local_run_is_confirmation_gated_and_records_status(tmp_path: Path) -> None: - manifest = _manifest(tmp_path, "local") - staged = stage_workflow(manifest) +def test_local_run_is_confirmed_and_reports_status(tmp_path, workflow_manifest): + manifest = _manifest(workflow_manifest, tmp_path, "local") + stage_workflow(manifest) with pytest.raises(PermissionError, match="confirmed=True"): run_staged_workflow(manifest) results = run_staged_workflow(manifest, confirmed=True, max_parallel=2) - assert len(results) == 2 assert all(result.succeeded for result in results) assert workflow_status(manifest)["complete"] is True - record = json.loads((staged.workspace / "run-results.json").read_text(encoding="utf-8")) - assert [item["status"] for item in record["results"]] == ["success", "success"] - -def test_slurm_prepare_uses_exact_name_and_exclusion_then_submits(tmp_path: Path) -> None: - manifest = _manifest(tmp_path, "slurm") - staged = stage_workflow(manifest) - - scripts = prepare_slurm_scripts(manifest, job_name="inpaint_test") +def test_slurm_happy_path_and_partial_submission_are_recorded(tmp_path, workflow_manifest): + happy = _manifest(workflow_manifest, tmp_path / "happy", "slurm") + stage_workflow(happy) + scripts = prepare_slurm_scripts(happy, job_name="inpaint_test") assert len(scripts) == 2 - first = scripts[0].read_text(encoding="utf-8") - assert "#SBATCH --job-name=inpaint_test" in first - assert "#SBATCH --exclude=node[1-8]" in first - attempt_count = tmp_path / "attempt-count" - fake_submit = tmp_path / "submit.py" - fake_submit.write_text( - "from pathlib import Path\n" - "import sys\n" - "counter = Path(sys.argv[1])\n" - "attempt = int(counter.read_text()) if counter.exists() else 0\n" - "counter.write_text(str(attempt + 1))\n" - "assert Path(sys.argv[2]).is_file()\n" - "print('Submitted batch job 789')\n", - encoding="utf-8", - ) + assert "#SBATCH --exclude=node[1-8]" in scripts[0].read_text(encoding="utf-8") submissions = submit_staged_workflow( - manifest, + happy, job_name="inpaint_test", - submit_command=(sys.executable, str(fake_submit), str(attempt_count)), + submit_command=(sys.executable, "-c", "print('Submitted batch job 789')"), confirmed=True, overwrite_scripts=True, ) - assert set(submissions.values()) == {"789"} - record = json.loads((staged.workspace / "slurm/submissions.json").read_text(encoding="utf-8")) - assert record["status"] == "complete" - assert record["failed_script"] is None - assert record["submissions"] == submissions - record_before_retry = (staged.workspace / "slurm/submissions.json").read_bytes() - scripts = sorted((staged.workspace / "slurm").glob("*.slurm")) - scripts_before_retry = {path: path.read_bytes() for path in scripts} - - with pytest.raises(FileExistsError, match="prior Slurm submission record"): - submit_staged_workflow( - manifest, - job_name="must_not_replace_inpaint_test", - submit_command=(sys.executable, str(fake_submit), str(attempt_count)), - confirmed=True, - overwrite_scripts=True, - ) - - assert attempt_count.read_text(encoding="utf-8") == "2" - assert (staged.workspace / "slurm/submissions.json").read_bytes() == record_before_retry - assert {path: path.read_bytes() for path in scripts} == scripts_before_retry - - -def test_failed_script_preparation_releases_empty_submission_reservation( - tmp_path: Path, -) -> None: - manifest = _manifest(tmp_path, "slurm") - staged = stage_workflow(manifest) - - with pytest.raises(ValueError, match="job_name must be"): - submit_staged_workflow( - manifest, - job_name="invalid job name", - submit_command="must-not-run", - confirmed=True, - overwrite_scripts=True, - ) - - assert not (staged.workspace / "slurm/submissions.json").exists() - - -def test_slurm_submission_failure_records_accepted_jobs_before_retry(tmp_path: Path) -> None: - manifest = _manifest(tmp_path, "slurm") - staged = stage_workflow(manifest) - attempt_count = tmp_path / "attempt-count" - fake_submit = tmp_path / "submit-once.py" - fake_submit.write_text( - "from pathlib import Path\n" - "import sys\n" - "counter = Path(sys.argv[1])\n" - "attempt = int(counter.read_text()) if counter.exists() else 0\n" - "counter.write_text(str(attempt + 1))\n" - "if attempt == 0:\n" - " print('Submitted batch job 789')\n" - "else:\n" - " print('scheduler unavailable', file=sys.stderr)\n" - " raise SystemExit(7)\n", - encoding="utf-8", - ) - - with pytest.raises(SlurmSubmissionError) as raised: - submit_staged_workflow( - manifest, - job_name="inpaint_test", - submit_command=(sys.executable, str(fake_submit), str(attempt_count)), - confirmed=True, - overwrite_scripts=True, - ) - - record_path = staged.workspace / "slurm/submissions.json" - first_script = staged.workspace / "slurm/frame-000000.slurm" - second_script = staged.workspace / "slurm/frame-000001.slurm" - expected_submissions = {str(first_script): "789"} - error = raised.value - assert error.phase == "submission" - assert error.failed_script == second_script - assert dict(error.partial_submissions) == expected_submissions - assert error.record_path == record_path - assert error.submission_complete is False - assert isinstance(error.__cause__, RuntimeError) - assert "inspect" in str(error) - record = json.loads(record_path.read_text(encoding="utf-8")) - assert record == { - "failed_script": str(second_script), - "job_name": "inpaint_test", - "manifest_sha256": staged.manifest_sha256, - "schema_version": "electrace.slurm-submissions.v1", - "status": "failed", - "submissions": expected_submissions, - } - record_before_retry = record_path.read_bytes() - + record = scripts[0].parent / "submissions.json" + assert json.loads(record.read_text(encoding="utf-8"))["status"] == "complete" with pytest.raises(FileExistsError, match="prior Slurm submission record"): - submit_staged_workflow( - manifest, - job_name="inpaint_test", - submit_command=(sys.executable, str(fake_submit), str(attempt_count)), - confirmed=True, - overwrite_scripts=True, - ) - - assert attempt_count.read_text(encoding="utf-8") == "2" - assert record_path.read_bytes() == record_before_retry - - -def test_first_slurm_submission_failure_records_no_job_ids(tmp_path: Path) -> None: - manifest = _manifest(tmp_path, "slurm") - staged = stage_workflow(manifest) - fake_submit = tmp_path / "submit-never.py" - fake_submit.write_text( - "import sys\nprint('rejected', file=sys.stderr)\nraise SystemExit(3)\n", - encoding="utf-8", - ) - - with pytest.raises(SlurmSubmissionError) as raised: - submit_staged_workflow( - manifest, - submit_command=(sys.executable, str(fake_submit)), - confirmed=True, - overwrite_scripts=True, - ) - - first_script = staged.workspace / "slurm/frame-000000.slurm" - record_path = staged.workspace / "slurm/submissions.json" - assert raised.value.failed_script == first_script - assert raised.value.phase == "submission" - assert dict(raised.value.partial_submissions) == {} - record = json.loads(record_path.read_text(encoding="utf-8")) - assert record["status"] == "failed" - assert record["failed_script"] == str(first_script) - assert record["submissions"] == {} - - -def test_workflow_submit_json_reports_partial_submission_without_blind_retry( - tmp_path: Path, -) -> None: - manifest = _manifest(tmp_path, "slurm") - staged = stage_workflow(manifest) - counter = tmp_path / "submit-count" - fake_submit = tmp_path / "submit-partially.py" - fake_submit.write_text( - f"#!{sys.executable}\n" - "from pathlib import Path\n" - "import sys\n" - f"counter = Path({str(counter)!r})\n" - "attempt = int(counter.read_text()) if counter.exists() else 0\n" - "counter.write_text(str(attempt + 1))\n" - "if attempt == 0:\n" - " print('Submitted batch job 789')\n" - "else:\n" - " print('scheduler unavailable', file=sys.stderr)\n" - " raise SystemExit(7)\n", - encoding="utf-8", - ) - fake_submit.chmod(0o700) - - result = runner.invoke( - app, - [ - "workflow", - "submit", - str(manifest), - "--job-name", - "inpaint_test", - "--submit-command", - str(fake_submit), - "--yes", - "--force", - "--json", - ], - ) - - assert result.exit_code == 1 - assert result.stderr == "" - assert result.stdout.count("\n") == 1 - assert "\x1b" not in result.stdout - payload = json.loads(result.stdout) - first_script = staged.workspace / "slurm/frame-000000.slurm" - second_script = staged.workspace / "slurm/frame-000001.slurm" - record_path = staged.workspace / "slurm/submissions.json" - assert payload["command"] == "workflow.submit" - assert payload["error"]["code"] == "slurm_submission_failed" - assert payload["error"]["details"] == { - "failed_script": str(second_script), - "partial_submissions": {str(first_script): "789"}, - "phase": "submission", - "record_path": str(record_path), - "submission_complete": False, - } - assert payload["artifacts"][-1] == { - "path": str(record_path), - "role": "slurm_submission_record", - } - - -def test_record_write_failure_after_accept_exposes_unrecorded_job_id( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - manifest = _manifest(tmp_path, "slurm") - staged = stage_workflow(manifest) - fake_submit = tmp_path / "submit.py" - fake_submit.write_text("print('Submitted batch job 789')\n", encoding="utf-8") - real_write = workflow_operations.atomic_write_json - - def fail_first_accepted_record(path: Path, payload: object) -> Path: - if ( - path.name == "submissions.json" - and isinstance(payload, dict) - and len(payload["submissions"]) == 1 - ): - raise OSError("simulated disk failure") - return real_write(path, payload) - - monkeypatch.setattr( - workflow_operations, - "atomic_write_json", - fail_first_accepted_record, - ) - - with pytest.raises(SlurmSubmissionError) as raised: - submit_staged_workflow( - manifest, - submit_command=(sys.executable, str(fake_submit)), - confirmed=True, - overwrite_scripts=True, - ) - - first_script = staged.workspace / "slurm/frame-000000.slurm" - error = raised.value - assert error.phase == "recording" - assert error.failed_script is None - assert dict(error.partial_submissions) == {str(first_script): "789"} - assert error.submission_complete is False - assert isinstance(error.__cause__, OSError) - record = json.loads(error.record_path.read_text(encoding="utf-8")) - assert record["status"] == "submitting" - assert record["submissions"] == {} - - -def test_final_record_write_failure_reports_all_jobs_were_accepted( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - manifest = _manifest(tmp_path, "slurm") - staged = stage_workflow(manifest) - attempt_count = tmp_path / "attempt-count" - fake_submit = tmp_path / "submit.py" - fake_submit.write_text( - "from pathlib import Path\n" - "import sys\n" - "counter = Path(sys.argv[1])\n" - "attempt = int(counter.read_text()) if counter.exists() else 0\n" - "counter.write_text(str(attempt + 1))\n" - "print(f'Submitted batch job {789 + attempt}')\n", - encoding="utf-8", + submit_staged_workflow(happy, confirmed=True, overwrite_scripts=True) + + partial = _manifest(workflow_manifest, tmp_path / "partial", "slurm") + stage_workflow(partial) + counter = tmp_path / "partial/count" + code = ( + "from pathlib import Path; import sys; p=Path(sys.argv[1]); " + "n=int(p.read_text()) if p.exists() else 0; p.write_text(str(n+1)); " + "print('Submitted batch job 456') if n == 0 else raise_error()" ) - real_write = workflow_operations.atomic_write_json - - def fail_complete_record(path: Path, payload: object) -> Path: - if ( - path.name == "submissions.json" - and isinstance(payload, dict) - and payload["status"] == "complete" - ): - raise OSError("simulated disk failure") - return real_write(path, payload) - - monkeypatch.setattr(workflow_operations, "atomic_write_json", fail_complete_record) - with pytest.raises(SlurmSubmissionError) as raised: submit_staged_workflow( - manifest, - submit_command=(sys.executable, str(fake_submit), str(attempt_count)), + partial, + submit_command=(sys.executable, "-c", code, str(counter)), confirmed=True, - overwrite_scripts=True, ) - - first_script = staged.workspace / "slurm/frame-000000.slurm" - second_script = staged.workspace / "slurm/frame-000001.slurm" - accepted = {str(first_script): "789", str(second_script): "790"} error = raised.value - assert error.phase == "recording" - assert error.failed_script is None - assert dict(error.partial_submissions) == accepted - assert error.submission_complete is True - assert "all 2 Slurm jobs were accepted" in str(error) - record = json.loads(error.record_path.read_text(encoding="utf-8")) - assert record["status"] == "submitting" - assert record["submissions"] == accepted - - -def test_slurm_prepare_preflights_all_scripts_before_writing(tmp_path: Path) -> None: - manifest = _manifest(tmp_path, "slurm") - staged = stage_workflow(manifest) - script_directory = staged.workspace / "slurm" - script_directory.mkdir() - first = script_directory / "frame-000000.slurm" - second = script_directory / "frame-000001.slurm" - second.write_text("user-owned sentinel\n", encoding="utf-8") - - with pytest.raises(FileExistsError, match=r"frame-000001\.slurm"): - prepare_slurm_scripts(manifest) - - assert not first.exists() - assert second.read_text(encoding="utf-8") == "user-owned sentinel\n" - - -def test_scheduler_kind_separates_local_execution_and_submission(tmp_path: Path) -> None: - local = _manifest(tmp_path / "local", "local") - stage_workflow(local) - with pytest.raises(ValueError, match=r"scheduler\.kind: slurm"): - prepare_slurm_scripts(local) - - slurm = _manifest(tmp_path / "slurm", "slurm") - stage_workflow(slurm) - with pytest.raises(ValueError, match=r"scheduler\.kind: local"): - run_staged_workflow(slurm, confirmed=True) - - -def test_workflow_submit_json_reports_prepared_scripts(tmp_path: Path) -> None: - manifest = _manifest(tmp_path, "slurm") - stage_workflow(manifest) - - result = runner.invoke( - app, - [ - "workflow", - "submit", - str(manifest), - "--job-name", - "inpaint_test", - "--json", - ], - ) - - assert result.exit_code == 0, result.output - payload = json.loads(result.stdout) - assert payload["command"] == "workflow.submit" - assert payload["data"]["submitted"] is False - assert payload["data"]["job_name"] == "inpaint_test" - assert len(payload["data"]["scripts"]) == 2 - assert all(item["role"] == "slurm_script" for item in payload["artifacts"]) - - -def test_workflow_run_json_reports_missing_authorization(tmp_path: Path) -> None: - manifest = _manifest(tmp_path, "local") - stage_workflow(manifest) - - result = runner.invoke(app, ["workflow", "run", str(manifest), "--json"]) - - assert result.exit_code == 2 - assert result.stderr == "" - payload = json.loads(result.stdout) - assert payload["command"] == "workflow.run" - assert payload["error"]["code"] == "authorization_required" - assert not any((tmp_path / "workspace").rglob("CHGCAR")) + assert len(error.partial_submissions) == 1 + payload = json.loads(error.record_path.read_text(encoding="utf-8")) + assert payload["status"] == "failed" + assert payload["submissions"] == dict(error.partial_submissions) diff --git a/tests/test_workflow_schema.py b/tests/test_workflow_schema.py index ec5547b..f10af1c 100644 --- a/tests/test_workflow_schema.py +++ b/tests/test_workflow_schema.py @@ -1,202 +1,29 @@ from __future__ import annotations -import copy -import importlib -import importlib.util import json -import re from pathlib import Path -from typing import Any import pytest from electrace.workflow import WorkflowManifest, load_workflow_manifest -_ROOT = Path(__file__).parents[1] -_SCHEMA_PATH = _ROOT / "schemas" / "electrace.workflow.v1.schema.json" -_EXAMPLE_PATH = _ROOT / "examples" / "electrace.workflow.yaml" +ROOT = Path(__file__).parents[1] -def _load_schema() -> dict[str, Any]: - payload = json.loads(_SCHEMA_PATH.read_text(encoding="utf-8")) - assert isinstance(payload, dict) - return payload +def test_public_workflow_example_matches_schema() -> None: + jsonschema = pytest.importorskip("jsonschema") + schema = json.loads((ROOT / "schemas/electrace.workflow.v1.schema.json").read_text()) + payload = load_workflow_manifest(ROOT / "examples/electrace.workflow.yaml").to_dict() + jsonschema.Draft202012Validator.check_schema(schema) + jsonschema.validate(payload, schema) -def _validate_with_jsonschema_when_installed( - schema: dict[str, Any], payload: dict[str, Any] -) -> None: - """Use the docs extra when present without making it a test dependency.""" - - if importlib.util.find_spec("jsonschema") is None: - return - jsonschema = importlib.import_module("jsonschema") - validator = jsonschema.Draft202012Validator - validator.check_schema(schema) - validator(schema).validate(payload) - - -def _reject_with_jsonschema_when_installed(schema: dict[str, Any], payload: dict[str, Any]) -> None: - """Assert invalid examples are rejected when the optional validator exists.""" - - if importlib.util.find_spec("jsonschema") is None: - return - jsonschema = importlib.import_module("jsonschema") - validator = jsonschema.Draft202012Validator - validator.check_schema(schema) - with pytest.raises(jsonschema.exceptions.ValidationError): - validator(schema).validate(payload) - - -def _frame_schema_accepts(schema: dict[str, Any], value: str) -> bool: - frame_schema = schema["properties"]["frames"] - trimmed_pattern = schema["$defs"]["trimmedString"]["pattern"] - if re.search(trimmed_pattern, value) is None: - return False - alternatives = frame_schema["allOf"][1]["oneOf"] - accepted = any( - alternative.get("const") == value - or ("pattern" in alternative and re.search(alternative["pattern"], value) is not None) - for alternative in alternatives - ) - zero_step = frame_schema["allOf"][2]["not"]["pattern"] - return accepted and re.search(zero_step, value) is None - - -def test_public_workflow_example_matches_runtime_and_schema_contract() -> None: - schema = _load_schema() - manifest = load_workflow_manifest(_EXAMPLE_PATH) - payload = manifest.to_dict() - - assert schema["$schema"] == "https://json-schema.org/draft/2020-12/schema" - assert schema["$id"].endswith("/schemas/electrace.workflow.v1.schema.json") - assert set(schema["required"]) == set(schema["properties"]) == set(payload) - assert schema["additionalProperties"] is False - assert set(schema["$defs"]["project"]["required"]) == set(payload["project"]) - assert schema["$defs"]["project"]["additionalProperties"] is False - assert set(schema["$defs"]["project"]["properties"]["series_kind"]["enum"]) == { - "auto", - "neb", - "ordered", - "pressure", - "static", - } - assert set(schema["$defs"]["step"]["properties"]["kind"]["enum"]) == { - "bader", - "baderkit", - "critic2", - "lobster", - "vasp", - } - assert set(schema["$defs"]["scheduler"]["properties"]["kind"]["enum"]) == { - "local", - "slurm", - } - assert ( - schema["$defs"]["scheduler"]["allOf"][0]["then"]["properties"]["directives"][ - "maxProperties" - ] - == 0 - ) - directive_name = schema["$defs"]["schedulerDirectiveName"] - assert directive_name["allOf"][0]["pattern"] == "^[a-z][a-z0-9-]*$" - assert set(directive_name["allOf"][1]["not"]["enum"]) == { - "command", - "job-name", - "script", - "wrap", - } - directive_value = schema["$defs"]["schedulerDirectiveValue"] - string_value = next( - branch for branch in directive_value["oneOf"] if branch.get("type") == "string" - ) - assert string_value["pattern"] == r"^[A-Za-z0-9_./,:@%+=\[\]*?{}~-]+$" - assert "unique step ids" in schema["$comment"] - - assert payload["frames"] == "0,2:7:2" - assert set(payload["executables"]) == {"vasp", "baderkit", "lobster"} - assert payload["scheduler"]["kind"] == "slurm" - assert payload["scheduler"]["directives"]["exclude"] == "node[1-8]" - steps = {step["id"]: step for step in payload["steps"]} - assert steps["badelf"]["kind"] == "baderkit" - assert steps["badelf"]["capability"] == "elf_basins" - assert steps["badelf"]["depends_on"] == ["vasp"] - assert "template_dir" not in steps["badelf"] - assert steps["lobster"]["kind"] == "lobster" - assert steps["lobster"]["capability"] == "cohp" - assert steps["lobster"]["depends_on"] == ["vasp"] - - # The canonical model result is strict JSON and validates against the full - # public schema when the optional docs validator is installed. - json.dumps(payload, allow_nan=False) - _validate_with_jsonschema_when_installed(schema, payload) - - -@pytest.mark.parametrize( - "selection", - ["all", "0", "+0", "-0", "0:7", "::-1", "0,2:7:2", "0 , 2:7:2"], -) -def test_frame_selection_schema_accepts_every_representative_runtime_form( - selection: str, -) -> None: - schema = _load_schema() - payload = load_workflow_manifest(_EXAMPLE_PATH).to_dict() - payload["frames"] = selection - - assert _frame_schema_accepts(schema, selection) - assert WorkflowManifest.from_dict(payload).frames == selection - - -@pytest.mark.parametrize( - "selection", - ["", " 0", "0 ", "all,0", "-1", "0::0", "0,,1", "0:1:2:3"], -) -def test_frame_selection_schema_rejects_representative_runtime_errors( - selection: str, -) -> None: - schema = _load_schema() - payload = load_workflow_manifest(_EXAMPLE_PATH).to_dict() - payload["frames"] = selection - - assert not _frame_schema_accepts(schema, selection) - with pytest.raises((TypeError, ValueError)): +def test_schema_and_runtime_reject_unknown_root_keys() -> None: + schema = json.loads((ROOT / "schemas/electrace.workflow.v1.schema.json").read_text()) + payload = load_workflow_manifest(ROOT / "examples/electrace.workflow.yaml").to_dict() + payload["unknown"] = True + with pytest.raises(ValueError): WorkflowManifest.from_dict(payload) - - -def test_scheduler_schema_matches_local_and_slurm_runtime_rules() -> None: - schema = _load_schema() - payload = load_workflow_manifest(_EXAMPLE_PATH).to_dict() - - local = copy.deepcopy(payload) - local["scheduler"] = {"kind": "local", "directives": {}} - assert WorkflowManifest.from_dict(local).scheduler.kind == "local" - _validate_with_jsonschema_when_installed(schema, local) - - local_with_directive = copy.deepcopy(local) - local_with_directive["scheduler"]["directives"] = {"nodes": 1} - with pytest.raises(ValueError, match="must be empty"): - WorkflowManifest.from_dict(local_with_directive) - - unsafe_slurm = copy.deepcopy(payload) - unsafe_slurm["scheduler"]["directives"] = {"wrap": "python payload.py"} - with pytest.raises(ValueError, match="not allowed to embed executable content"): - WorkflowManifest.from_dict(unsafe_slurm) - _reject_with_jsonschema_when_installed(schema, unsafe_slurm) - - for directives in ( - {"cpus_per_task": 8}, - {"job-name": "embedded-name"}, - {"comment": "contains whitespace"}, - {"comment": ""}, - ): - invalid = copy.deepcopy(payload) - invalid["scheduler"]["directives"] = directives - with pytest.raises(ValueError): - WorkflowManifest.from_dict(invalid) - _reject_with_jsonschema_when_installed(schema, invalid) - - for value in ("gpu:a100", 2, 2.5, True, False, None): - valid = copy.deepcopy(payload) - valid["scheduler"]["directives"] = {"constraint": value} - assert WorkflowManifest.from_dict(valid).scheduler.directives["constraint"] == value - _validate_with_jsonschema_when_installed(schema, valid) + jsonschema = pytest.importorskip("jsonschema") + with pytest.raises(jsonschema.ValidationError): + jsonschema.validate(payload, schema) diff --git a/tests/test_workflow_staging.py b/tests/test_workflow_staging.py index 1f16b3b..9caef13 100644 --- a/tests/test_workflow_staging.py +++ b/tests/test_workflow_staging.py @@ -1,248 +1,27 @@ -from __future__ import annotations - -import json -from pathlib import Path - import pytest -import yaml - -import electrace.workflow.staging as staging_module -from electrace.workflow.staging import ( - WORKSPACE_RECORD, - descriptor_outputs, - load_staged_workflow, - stage_workflow, -) - -POSCAR = """\ -H -1.0 -1 0 0 -0 1 0 -0 0 1 -H -1 -Direct -0 0 0 -""" - - -def _workflow_fixture(tmp_path: Path, *, workspace: str = "workspace") -> Path: - images = tmp_path / "images" - for index in range(3): - frame = images / f"{index:02d}" - frame.mkdir(parents=True) - (frame / "POSCAR").write_text(POSCAR, encoding="utf-8") - template = tmp_path / "templates" / "elf" - template.mkdir(parents=True) - (template / "INCAR").write_text("ENCUT = 400\nLELF = .FALSE.\n", encoding="utf-8") - (template / "KPOINTS").write_text("Gamma\n0\nGamma\n1 1 1\n0 0 0\n", encoding="utf-8") - (template / "POTCAR").write_text("private fixture\n", encoding="utf-8") - document = { - "schema_version": "electrace.workflow.v1", - "project": { - "root": "images", - "series_kind": "neb", - "workspace": workspace, - "structure": "CONTCAR", - "fallback": "POSCAR", - }, - "frames": "0:3:2", - "executables": { - "vasp": {"path": "/opt/vasp_std", "argv_prefix": ["srun"]}, - "baderkit": {"path": "/opt/baderkit"}, - }, - "scheduler": {"kind": "local", "directives": {}}, - "steps": [ - { - "id": "vasp_elf", - "kind": "vasp", - "executable": "vasp", - "template_dir": "templates/elf", - "depends_on": [], - "options": { - "outputs": ["ELFCAR", "CHGCAR", "CONTCAR"], - "incar_overrides": {"LELF": True, "LCHARG": True}, - }, - "resources": {}, - }, - { - "id": "elf_basins", - "kind": "baderkit", - "executable": "baderkit", - "capability": "elf_basins", - "depends_on": ["vasp_elf"], - "options": {}, - "resources": {}, - }, - ], - } - manifest = tmp_path / "electrace-workflow.yaml" - manifest.write_text(yaml.safe_dump(document, sort_keys=False), encoding="utf-8") - return manifest +from electrace.workflow.staging import load_staged_workflow, stage_workflow -def test_stage_selects_frames_without_touching_sources_and_builds_job_dag( - tmp_path: Path, -) -> None: - manifest = _workflow_fixture(tmp_path) +def test_stage_and_reload_preserve_source(tmp_path, poscar_text, workflow_manifest): + manifest = workflow_manifest() staged = stage_workflow(manifest) - assert [frame.index for frame in staged.frames] == [0, 2] - assert [job.id for job in staged.jobs] == [ - "f000000.vasp_elf", - "f000000.elf_basins", - "f000002.vasp_elf", - "f000002.elf_basins", - ] - first_vasp, first_badelf = staged.jobs[:2] - assert first_vasp.argv == ("srun", "/opt/vasp_std") - assert first_badelf.argv == ( - "/opt/baderkit", - "badelf", - "CHGCAR", - "ELFCAR", - ) - assert first_badelf.dependencies == (first_vasp.id,) - assert first_badelf.expected_outputs == (first_badelf.cwd / "badelf.json",) - assert (first_badelf.cwd / "CHGCAR").is_symlink() - assert (first_badelf.cwd / "ELFCAR").is_symlink() - assert (first_badelf.cwd / "CHGCAR").readlink() == Path("../vasp_elf/CHGCAR") - incar = (first_vasp.cwd / "INCAR").read_text(encoding="utf-8") - assert "LELF = .TRUE." in incar - assert "LCHARG = .TRUE." in incar - assert (tmp_path / "images/00/POSCAR").read_text(encoding="utf-8") == POSCAR - assert staged.record_path.name == WORKSPACE_RECORD + assert [job.id for job in staged.jobs] == ["f000000.density", "f000001.density"] + assert staged.jobs[0].required_inputs == (staged.jobs[0].cwd / "POSCAR",) + assert (tmp_path / "images/00/POSCAR").read_text(encoding="utf-8") == poscar_text assert load_staged_workflow(manifest).to_dict() == staged.to_dict() - assert descriptor_outputs(staged) == { - "f000000.elf_basins": (first_badelf.cwd / "badelf.json",), - "f000002.elf_basins": (staged.jobs[3].cwd / "badelf.json",), - } -def test_stage_refuses_unmarked_workspace_even_with_force(tmp_path: Path) -> None: - manifest = _workflow_fixture(tmp_path) - workspace = tmp_path / "workspace" - workspace.mkdir() - protected = workspace / "user-file" - protected.write_text("keep", encoding="utf-8") +def test_stage_refuses_source_overlap_and_unmarked_overwrite(tmp_path, workflow_manifest): + with pytest.raises(ValueError, match="must not overlap"): + stage_workflow(workflow_manifest(base=tmp_path / "overlap", workspace="images/work")) + manifest = workflow_manifest(base=tmp_path / "unmarked") + workspace = tmp_path / "unmarked/workspace" + workspace.mkdir() + sentinel = workspace / "keep" + sentinel.write_text("user data", encoding="utf-8") with pytest.raises(ValueError, match="unmarked"): stage_workflow(manifest, overwrite=True) - - assert protected.read_text(encoding="utf-8") == "keep" - - -def test_failed_force_restaging_preserves_the_previous_workspace(tmp_path: Path) -> None: - manifest = _workflow_fixture(tmp_path) - staged = stage_workflow(manifest) - sentinel = staged.workspace / "previous-run" - sentinel.write_text("preserve", encoding="utf-8") - (tmp_path / "templates/elf/INCAR").unlink() - - with pytest.raises(FileNotFoundError, match="INCAR template"): - stage_workflow(manifest, overwrite=True) - - assert sentinel.read_text(encoding="utf-8") == "preserve" - assert load_staged_workflow(manifest).to_dict() == staged.to_dict() - - -def test_staged_record_is_size_bounded_and_bound_to_the_manifest_plan( - tmp_path: Path, - monkeypatch: pytest.MonkeyPatch, -) -> None: - manifest = _workflow_fixture(tmp_path) - staged = stage_workflow(manifest) - record = json.loads(staged.record_path.read_text(encoding="utf-8")) - record["jobs"][0]["argv"] = ["/tmp/tampered-command"] - staged.record_path.write_text(json.dumps(record), encoding="utf-8") - - with pytest.raises(ValueError, match="manifest-derived plan"): - load_staged_workflow(manifest) - - staged = stage_workflow(manifest, overwrite=True) - monkeypatch.setattr(staging_module, "MAX_WORKSPACE_RECORD_BYTES", 128) - with pytest.raises(ValueError, match="exceeds 128 bytes"): - load_staged_workflow(manifest) - - -@pytest.mark.parametrize( - "tamper", - ("source_directory", "structure", "energy_ev", "label", "index", "order"), -) -def test_staged_record_frame_provenance_is_bound_to_manifest_discovery( - tmp_path: Path, - tamper: str, -) -> None: - manifest = _workflow_fixture(tmp_path) - staged = stage_workflow(manifest) - record = json.loads(staged.record_path.read_text(encoding="utf-8")) - frames = record["frames"] - if tamper in {"source_directory", "structure"}: - frames[0][tamper] = frames[1][tamper] - elif tamper == "energy_ev": - frames[0][tamper] = 123.5 - elif tamper == "label": - frames[0][tamper] = "substituted-frame" - elif tamper == "index": - frames[0][tamper] = 99 - else: - frames.reverse() - staged.record_path.write_text(json.dumps(record), encoding="utf-8") - - with pytest.raises(ValueError, match="frame provenance"): - load_staged_workflow(manifest) - - -def test_manifest_change_requires_restaging(tmp_path: Path) -> None: - manifest = _workflow_fixture(tmp_path) - stage_workflow(manifest) - manifest.write_text(manifest.read_text(encoding="utf-8") + "\n", encoding="utf-8") - - with pytest.raises(ValueError, match="changed after staging"): - load_staged_workflow(manifest) - - -def test_workspace_inside_source_tree_is_rejected(tmp_path: Path) -> None: - manifest = _workflow_fixture(tmp_path, workspace="images/.electrace-work") - - with pytest.raises(ValueError, match=r"outside project\.root"): - stage_workflow(manifest) - - -def test_source_inside_managed_workspace_is_rejected_without_data_loss(tmp_path: Path) -> None: - manifest = _workflow_fixture(tmp_path) - staged = stage_workflow(manifest) - nested_source = staged.workspace / "images" - for index in range(2): - frame = nested_source / f"{index:02d}" - frame.mkdir(parents=True) - (frame / "POSCAR").write_text(POSCAR, encoding="utf-8") - sentinel = nested_source / "00/POSCAR" - - document = yaml.safe_load(manifest.read_text(encoding="utf-8")) - document["project"]["root"] = "workspace/images" - manifest.write_text(yaml.safe_dump(document, sort_keys=False), encoding="utf-8") - - with pytest.raises(ValueError, match=r"must not contain"): - stage_workflow(manifest, overwrite=True) - - assert sentinel.read_text(encoding="utf-8") == POSCAR - assert staged.record_path.is_file() - - -def test_symlinked_source_root_cannot_hide_workspace_inside_real_source( - tmp_path: Path, -) -> None: - manifest = _workflow_fixture(tmp_path, workspace="images/generated-workspace") - source_alias = tmp_path / "source-alias" - source_alias.symlink_to(tmp_path / "images", target_is_directory=True) - document = yaml.safe_load(manifest.read_text(encoding="utf-8")) - document["project"]["root"] = source_alias.name - manifest.write_text(yaml.safe_dump(document, sort_keys=False), encoding="utf-8") - - with pytest.raises(ValueError, match=r"outside project\.root"): - stage_workflow(manifest) - - assert not (tmp_path / "images/generated-workspace").exists() + assert sentinel.read_text(encoding="utf-8") == "user data" diff --git a/tests/test_workflow_validation.py b/tests/test_workflow_validation.py index 7f481fb..3eb4dee 100644 --- a/tests/test_workflow_validation.py +++ b/tests/test_workflow_validation.py @@ -1,449 +1,117 @@ -from __future__ import annotations - -import hashlib -import importlib -import importlib.util import json -import os -import stat from pathlib import Path -from typing import Any import pytest import yaml -from electrace.workflow import ( - WORKFLOW_VALIDATION_SCHEMA_VERSION, - WorkflowValidation, - stage_workflow, - validate_workflow, -) - -POSCAR = """\ -H -1.0 -1 0 0 -0 1 0 -0 0 1 -H -1 -Direct -0 0 0 -""" - - -def _write_executable(path: Path) -> Path: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text("#!/bin/sh\nexit 0\n", encoding="utf-8") - path.chmod(0o700) - return path - +from electrace.workflow import stage_workflow, validate_workflow -def _workflow_fixture(tmp_path: Path) -> Path: - images = tmp_path / "images" - for index in range(3): - frame = images / f"{index:02d}" - frame.mkdir(parents=True) - (frame / "POSCAR").write_text(POSCAR, encoding="utf-8") - template = tmp_path / "templates" / "vasp-elf" - template.mkdir(parents=True) - (template / "INCAR").write_text("ENCUT = 400\nLELF = .FALSE.\n", encoding="utf-8") - (template / "KPOINTS").write_text("Gamma\n0\nGamma\n1 1 1\n0 0 0\n", encoding="utf-8") - (template / "POTCAR").write_text("test-only placeholder\n", encoding="utf-8") - - wrapper = _write_executable(tmp_path / "bin" / "launcher") - vasp = _write_executable(tmp_path / "bin" / "vasp_std") - baderkit = _write_executable(tmp_path / "bin" / "baderkit") - payload = { - "schema_version": "electrace.workflow.v1", - "project": { - "root": "images", - "series_kind": "neb", - "workspace": "generated/workspace", - "structure": "CONTCAR", - "fallback": "POSCAR", - }, - "frames": "0,2", - "executables": { - "vasp": {"path": str(vasp), "argv_prefix": [str(wrapper)]}, - "baderkit": {"path": str(baderkit)}, +@pytest.fixture +def valid_manifest(workflow_manifest): + return workflow_manifest( + templates={ + "template": { + "INCAR": "LELF = .TRUE.\n", + "KPOINTS": "Gamma\n", + "POTCAR": "test fixture\n", + } }, - "scheduler": { - "kind": "slurm", - "directives": {"partition": "compute", "time": "00:30:00", "nodes": 1}, - }, - "steps": [ + steps=[ { - "id": "vasp_elf", + "id": "elf", "kind": "vasp", "executable": "vasp", - "template_dir": "templates/vasp-elf", - "options": { - "arguments": ["--gamma"], - "expected_outputs": ["CHGCAR", "ELFCAR", "CONTCAR"], - "incar_overrides": {"LELF": True, "LCHARG": True}, - "environment": {"OMP_NUM_THREADS": "2"}, - }, - }, - { - "id": "elf_basins", - "kind": "baderkit", - "executable": "baderkit", - "capability": "elf_basins", - "depends_on": ["vasp_elf"], - }, + "template_dir": "template", + "options": {"expected_outputs": ["ELFCAR", "CHGCAR"]}, + } ], - } - manifest = tmp_path / "workflow.yaml" - manifest.write_text(yaml.safe_dump(payload, sort_keys=False), encoding="utf-8") - return manifest - - -def _tree_snapshot(root: Path) -> tuple[tuple[object, ...], ...]: - records: list[tuple[object, ...]] = [] - for path in sorted(root.rglob("*"), key=lambda item: str(item.relative_to(root))): - relative = str(path.relative_to(root)) - status = path.lstat() - if path.is_symlink(): - records.append((relative, "symlink", os.readlink(path), status.st_mtime_ns)) - elif path.is_file(): - digest = hashlib.sha256(path.read_bytes()).hexdigest() - records.append( - (relative, "file", status.st_mode, status.st_size, status.st_mtime_ns, digest) - ) - else: - records.append((relative, "directory", status.st_mode, status.st_mtime_ns)) - return tuple(records) - - -def _normalized(value: Any, root: Path) -> Any: - if isinstance(value, dict): - return {key: _normalized(item, root) for key, item in value.items()} - if isinstance(value, list): - return [_normalized(item, root) for item in value] - if isinstance(value, str): - return value.replace(str(root), "") - return value - + ) -def test_validate_workflow_snapshots_plan_without_creating_workspace(tmp_path: Path) -> None: - manifest = _workflow_fixture(tmp_path) - before = _tree_snapshot(tmp_path) - validation = validate_workflow(manifest) +def test_validate_is_read_only_and_returns_resolved_plan(tmp_path, valid_manifest): + before = {path.relative_to(tmp_path) for path in tmp_path.rglob("*")} - assert isinstance(validation, WorkflowValidation) - assert _tree_snapshot(tmp_path) == before - assert not (tmp_path / "generated").exists() - payload = validation.to_dict() - json.dumps(payload, allow_nan=False) - assert _normalized(payload, tmp_path) == { - "schema_version": WORKFLOW_VALIDATION_SCHEMA_VERSION, - "manifest": "/workflow.yaml", - "workspace": "/generated/workspace", - "selected_frames": [ - { - "index": 0, - "label": "00", - "analysis_label": "000000-00", - "source_directory": "/images/00", - "structure": "/images/00/POSCAR", - "workspace_directory": "/generated/workspace/frames/000000-00", - "energy_ev": None, - }, - { - "index": 2, - "label": "02", - "analysis_label": "000001-02", - "source_directory": "/images/02", - "structure": "/images/02/POSCAR", - "workspace_directory": "/generated/workspace/frames/000002-02", - "energy_ev": None, - }, - ], - "resolved_jobs": [ - { - "id": "f000000.vasp_elf", - "frame_index": 0, - "frame_label": "00", - "task": "vasp_elf", - "cwd": "/generated/workspace/frames/000000-00/vasp_elf", - "argv": ["/bin/launcher", "/bin/vasp_std", "--gamma"], - "stdin": None, - "stdout_path": ( - "/generated/workspace/frames/000000-00/vasp_elf/vasp.stdout.log" - ), - "stderr_path": ( - "/generated/workspace/frames/000000-00/vasp_elf/vasp.stdout.log.stderr.log" - ), - "required_inputs": [ - "/generated/workspace/frames/000000-00/vasp_elf/INCAR", - "/generated/workspace/frames/000000-00/vasp_elf/KPOINTS", - "/generated/workspace/frames/000000-00/vasp_elf/POSCAR", - "/generated/workspace/frames/000000-00/vasp_elf/POTCAR", - ], - "expected_outputs": [ - "/generated/workspace/frames/000000-00/vasp_elf/CHGCAR", - "/generated/workspace/frames/000000-00/vasp_elf/ELFCAR", - "/generated/workspace/frames/000000-00/vasp_elf/CONTCAR", - ], - "dependencies": [], - "environment": {"OMP_NUM_THREADS": "2"}, - }, - { - "id": "f000000.elf_basins", - "frame_index": 0, - "frame_label": "00", - "task": "elf_basins", - "cwd": "/generated/workspace/frames/000000-00/elf_basins", - "argv": ["/bin/baderkit", "badelf", "CHGCAR", "ELFCAR"], - "stdin": None, - "stdout_path": ( - "/generated/workspace/frames/000000-00/elf_basins/baderkit.stdout.log" - ), - "stderr_path": ( - "/generated/workspace/frames/000000-00/elf_basins/" - "baderkit.stdout.log.stderr.log" - ), - "required_inputs": [ - "/generated/workspace/frames/000000-00/elf_basins/ELFCAR", - "/generated/workspace/frames/000000-00/elf_basins/CHGCAR", - ], - "expected_outputs": [ - "/generated/workspace/frames/000000-00/elf_basins/badelf.json" - ], - "dependencies": ["f000000.vasp_elf"], - "environment": {}, - }, - { - "id": "f000002.vasp_elf", - "frame_index": 2, - "frame_label": "02", - "task": "vasp_elf", - "cwd": "/generated/workspace/frames/000002-02/vasp_elf", - "argv": ["/bin/launcher", "/bin/vasp_std", "--gamma"], - "stdin": None, - "stdout_path": ( - "/generated/workspace/frames/000002-02/vasp_elf/vasp.stdout.log" - ), - "stderr_path": ( - "/generated/workspace/frames/000002-02/vasp_elf/vasp.stdout.log.stderr.log" - ), - "required_inputs": [ - "/generated/workspace/frames/000002-02/vasp_elf/INCAR", - "/generated/workspace/frames/000002-02/vasp_elf/KPOINTS", - "/generated/workspace/frames/000002-02/vasp_elf/POSCAR", - "/generated/workspace/frames/000002-02/vasp_elf/POTCAR", - ], - "expected_outputs": [ - "/generated/workspace/frames/000002-02/vasp_elf/CHGCAR", - "/generated/workspace/frames/000002-02/vasp_elf/ELFCAR", - "/generated/workspace/frames/000002-02/vasp_elf/CONTCAR", - ], - "dependencies": [], - "environment": {"OMP_NUM_THREADS": "2"}, - }, - { - "id": "f000002.elf_basins", - "frame_index": 2, - "frame_label": "02", - "task": "elf_basins", - "cwd": "/generated/workspace/frames/000002-02/elf_basins", - "argv": ["/bin/baderkit", "badelf", "CHGCAR", "ELFCAR"], - "stdin": None, - "stdout_path": ( - "/generated/workspace/frames/000002-02/elf_basins/baderkit.stdout.log" - ), - "stderr_path": ( - "/generated/workspace/frames/000002-02/elf_basins/" - "baderkit.stdout.log.stderr.log" - ), - "required_inputs": [ - "/generated/workspace/frames/000002-02/elf_basins/ELFCAR", - "/generated/workspace/frames/000002-02/elf_basins/CHGCAR", - ], - "expected_outputs": [ - "/generated/workspace/frames/000002-02/elf_basins/badelf.json" - ], - "dependencies": ["f000002.vasp_elf"], - "environment": {}, - }, - ], - "scheduler": { - "kind": "slurm", - "directives": {"partition": "compute", "time": "00:30:00", "nodes": 1}, - }, - "warnings": [], - } + plan = validate_workflow(valid_manifest) + assert {path.relative_to(tmp_path) for path in tmp_path.rglob("*")} == before + assert not plan.workspace.exists() + assert [frame.label for frame in plan.selected_frames] == ["00", "01"] + assert [job.id for job in plan.resolved_jobs] == ["f000000.elf", "f000001.elf"] + assert plan.resolved_jobs[0].argv == ("/bin/true",) -def test_validation_jobs_match_subsequent_staging_exactly(tmp_path: Path) -> None: - manifest = _workflow_fixture(tmp_path) - validation = validate_workflow(manifest) - assert not validation.workspace.parent.exists() - staged = stage_workflow(manifest) +def test_validate_and_stage_share_job_plan(valid_manifest): + plan = validate_workflow(valid_manifest) + staged = stage_workflow(valid_manifest) - assert [job.to_dict() for job in validation.resolved_jobs] == [ - job.to_dict() for job in staged.jobs - ] - assert [frame.to_dict() for frame in validation.selected_frames] == [ + assert [frame.to_dict() for frame in plan.selected_frames] == [ frame.to_dict() for frame in staged.frames ] + assert [job.to_dict() for job in plan.resolved_jobs] == [job.to_dict() for job in staged.jobs] -def test_read_only_incar_template_validates_and_stages_with_overrides(tmp_path: Path) -> None: - manifest = _workflow_fixture(tmp_path) - template = tmp_path / "templates/vasp-elf/INCAR" - template.chmod(0o444) - - validation = validate_workflow(manifest) - staged = stage_workflow(manifest) - - assert validation.resolved_jobs[0].to_dict() == staged.jobs[0].to_dict() - staged_incar = staged.jobs[0].cwd / "INCAR" - assert "LELF = .TRUE." in staged_incar.read_text(encoding="utf-8") - assert stat.S_IMODE(template.stat().st_mode) == 0o444 - - -def test_validation_rejects_missing_staged_input_without_writing(tmp_path: Path) -> None: - manifest = _workflow_fixture(tmp_path) - (tmp_path / "templates/vasp-elf/POTCAR").unlink() - before = _tree_snapshot(tmp_path) - - with pytest.raises(FileNotFoundError, match="required staged inputs: POTCAR"): - validate_workflow(manifest) - - assert _tree_snapshot(tmp_path) == before - assert not (tmp_path / "generated").exists() - - -def test_validation_rejects_unreadable_template_without_writing(tmp_path: Path) -> None: - manifest = _workflow_fixture(tmp_path) - template = tmp_path / "templates/vasp-elf/POTCAR" - original = _tree_snapshot(tmp_path) - original_mode = stat.S_IMODE(template.stat().st_mode) - template.chmod(0) - - try: - try: - template.open("rb").close() - except PermissionError: - pass - else: - pytest.skip("filesystem permissions do not make mode-000 files unreadable") - with pytest.raises(PermissionError): - validate_workflow(manifest) - finally: - template.chmod(original_mode) - - assert _tree_snapshot(tmp_path) == original - assert not (tmp_path / "generated").exists() - - -def test_validation_rejects_source_inside_workspace_without_writing(tmp_path: Path) -> None: - manifest = _workflow_fixture(tmp_path) - nested_source = tmp_path / "generated/workspace/images" - for index in range(2): - frame = nested_source / f"{index:02d}" - frame.mkdir(parents=True) - (frame / "POSCAR").write_text(POSCAR, encoding="utf-8") - payload = yaml.safe_load(manifest.read_text(encoding="utf-8")) - payload["project"]["root"] = "generated/workspace/images" - manifest.write_text(yaml.safe_dump(payload, sort_keys=False), encoding="utf-8") - before = _tree_snapshot(tmp_path) - - with pytest.raises(ValueError, match=r"must not contain"): - validate_workflow(manifest) - - assert _tree_snapshot(tmp_path) == before - - -def test_validation_rejects_template_output_collision_without_writing(tmp_path: Path) -> None: - manifest = _workflow_fixture(tmp_path) - template = tmp_path / "templates" / "badelf" - template.mkdir() - (template / "badelf.json").write_text("stale output\n", encoding="utf-8") - payload = yaml.safe_load(manifest.read_text(encoding="utf-8")) - payload["steps"][1]["template_dir"] = "templates/badelf" - manifest.write_text(yaml.safe_dump(payload, sort_keys=False), encoding="utf-8") - before = _tree_snapshot(tmp_path) - - with pytest.raises(ValueError, match=r"collide with declared outputs: badelf.json"): - validate_workflow(manifest) - - assert _tree_snapshot(tmp_path) == before - assert not (tmp_path / "generated").exists() - - -def test_validation_rejects_linked_lobster_input_like_staging(tmp_path: Path) -> None: - manifest = _workflow_fixture(tmp_path) - template = tmp_path / "templates/vasp-elf" - (template / "lobsterin").write_text("cohpGenerator from 1.0 to 2.0\n", encoding="utf-8") - payload = yaml.safe_load(manifest.read_text(encoding="utf-8")) - payload["executables"]["lobster"] = dict(payload["executables"]["baderkit"]) - payload["steps"][0]["options"]["expected_outputs"].append("WAVECAR") - payload["steps"][1] = { - "id": "bonding", - "kind": "lobster", - "executable": "lobster", - "capability": "cohp", - "depends_on": ["vasp_elf"], - "options": {"input_links": {"lobsterin": {"step": "vasp_elf", "file": "lobsterin"}}}, - } - manifest.write_text(yaml.safe_dump(payload, sort_keys=False), encoding="utf-8") +def test_validate_and_stage_reject_missing_vasp_input(tmp_path, valid_manifest): + (tmp_path / "template/KPOINTS").unlink() + for operation in (validate_workflow, stage_workflow): + with pytest.raises(FileNotFoundError, match=r"required inputs.*KPOINTS"): + operation(valid_manifest) + assert not (tmp_path / "workspace").exists() - with pytest.raises(FileNotFoundError, match="regular reviewed lobsterin"): - validate_workflow(manifest) - with pytest.raises(FileNotFoundError, match="regular reviewed lobsterin"): - stage_workflow(manifest) - - assert not (tmp_path / "generated/workspace").exists() +def test_descriptor_template_cannot_prepopulate_output(workflow_manifest): + manifest = workflow_manifest( + executables={"baderkit": "/bin/true"}, + templates={"template": {"bader.json": "stale\n"}}, + steps=[ + { + "id": "charges", + "kind": "baderkit", + "executable": "baderkit", + "capability": "bader_charges", + "template_dir": "template", + } + ], + ) + for operation in (validate_workflow, stage_workflow): + with pytest.raises(ValueError, match=r"collide.*bader.json"): + operation(manifest) + + +def test_vasp_restart_input_may_also_be_an_output(tmp_path, valid_manifest): + payload = yaml.safe_load(valid_manifest.read_text(encoding="utf-8")) + payload["steps"][0]["options"]["required_inputs"] = [ + "INCAR", + "KPOINTS", + "POSCAR", + "POTCAR", + "CHGCAR", + ] + valid_manifest.write_text(yaml.safe_dump(payload, sort_keys=False), encoding="utf-8") + (tmp_path / "template/CHGCAR").write_text("restart\n", encoding="utf-8") -def test_validation_rejects_collection_descriptor_basename_collision(tmp_path: Path) -> None: - manifest = _workflow_fixture(tmp_path) - payload = yaml.safe_load(manifest.read_text(encoding="utf-8")) - payload["steps"].append( - { - "id": "elf_basins_again", - "kind": "baderkit", - "executable": "baderkit", - "capability": "elf_basins", - "depends_on": ["vasp_elf"], - } + validate_workflow(valid_manifest) + staged = stage_workflow(valid_manifest) + assert all( + (job.cwd / "CHGCAR").read_text(encoding="utf-8") == "restart\n" for job in staged.jobs ) - manifest.write_text(yaml.safe_dump(payload, sort_keys=False), encoding="utf-8") - with pytest.raises(ValueError, match=r"descriptor output collision.*badelf.json"): - validate_workflow(manifest) - assert not (tmp_path / "generated").exists() +def test_validation_payload_matches_public_schema(valid_manifest): + jsonschema = pytest.importorskip("jsonschema") + schema_path = Path(__file__).parents[1] / "schemas/electrace.workflow-validation.v1.schema.json" + schema = json.loads(schema_path.read_text(encoding="utf-8")) + payload = validate_workflow(valid_manifest).to_dict() + jsonschema.Draft202012Validator(schema).validate(payload) + del payload["resolved_jobs"][0]["argv"] + with pytest.raises(jsonschema.ValidationError): + jsonschema.Draft202012Validator(schema).validate(payload) -def test_validation_reads_incar_during_override_preflight(tmp_path: Path) -> None: - manifest = _workflow_fixture(tmp_path) - (tmp_path / "templates/vasp-elf/INCAR").write_bytes(b"ENCUT = 400\n\xff") +def test_validate_rejects_malformed_manifest_without_writing(tmp_path): + manifest = tmp_path / "workflow.yaml" + manifest.write_text("schema_version: electrace.workflow.v1\n", encoding="utf-8") - with pytest.raises(UnicodeDecodeError): + with pytest.raises(ValueError): validate_workflow(manifest) - - assert not (tmp_path / "generated").exists() - - -def test_workflow_validation_payload_matches_its_public_schema(tmp_path: Path) -> None: - schema_path = ( - Path(__file__).parents[1] / "schemas" / "electrace.workflow-validation.v1.schema.json" - ) - schema = json.loads(schema_path.read_text(encoding="utf-8")) - assert schema["$schema"] == "https://json-schema.org/draft/2020-12/schema" - assert schema["properties"]["schema_version"]["const"] == (WORKFLOW_VALIDATION_SCHEMA_VERSION) - assert set(schema["required"]) == set(schema["properties"]) - if importlib.util.find_spec("jsonschema") is None: - return - jsonschema = importlib.import_module("jsonschema") - jsonschema.Draft202012Validator.check_schema(schema) - jsonschema.Draft202012Validator(schema).validate( - validate_workflow(_workflow_fixture(tmp_path)).to_dict() - ) + assert list(tmp_path.iterdir()) == [manifest]