diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index ef7229d0..d7015fea 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -47,3 +47,9 @@ Describe the testing strategy for this change and include any relevant details a If the change is covered by the CI test suite, state that explicitly. If manual testing was performed, describe the testing process and results. --> + + +## Checklist + +- [ ] Architecture/docs impact considered. Updated `ARCHITECTURE.md` and/or + developer docs where needed, or explained why not. diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 94c14a50..2c81d437 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -23,6 +23,8 @@ jobs: steps: - name: Checkout Myna uses: actions/checkout@v6 + with: + fetch-depth: 0 - name: Install uv uses: astral-sh/setup-uv@v7 with: diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 70e11bfd..6e4d3fec 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -9,6 +9,8 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + fetch-depth: 0 - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: '3.14' diff --git a/AGENTS.md b/AGENTS.md index 5cb0f51c..569f78b2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -109,8 +109,11 @@ Branch names use: ``` Examples: `docs/agent-harness`, `fix/database-paths`, `feat(cli): add dry-run flag`. -Complete `.github/pull_request_template.md` and state behavior impact, risk, and -testing strategy. +When asked to draft or open a pull request, first inspect and complete +`.github/pull_request_template.md`. Preserve the template headings and checklist, and +state behavior impact, risk, and testing strategy. If the requested PR body is scoped +to a subset of commits while the branch contains other commits, state that scope +explicitly in the body. ## Security and Data Handling @@ -131,3 +134,7 @@ testing strategy. - Public behavior changes, compatibility risks, or known gaps are called out. - Docs and examples are updated when commands, inputs, outputs, or extension points change. +- If code changes touch `src/myna/core/workflow/`, `src/myna/core/components/`, + `src/myna/core/app/`, `src/myna/core/context.py`, `src/myna/database/`, CLI behavior, + app extension patterns, or subsystem boundaries, update `ARCHITECTURE.md` and + relevant developer docs, or state why no docs update was needed. diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index db841857..243c9378 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -2,7 +2,7 @@ ## Status -- Last verified: 2026-06-01 during the documentation-harness update +- Last verified: 2026-06-01 during the workflow-context update - Audience: maintainers, contributors, and coding agents - Scope: describes current architecture; does not replace API docs or user docs @@ -44,6 +44,7 @@ The main user workflows are: | `uv.lock` | Locked `uv` dependency graph | Commit updates when dependency declarations change | | `src/myna/` | Python package source | Runtime code lives here | | `src/myna/core/` | Workflow orchestration, components, file abstractions, metadata, app base helpers, utilities | Prefer generic workflow behavior here | +| `src/myna/core/context.py` | Explicit workflow context shared by orchestration, components, app wrappers, metadata, and sync helpers | Prefer this over workflow-specific environment variables for new shared behavior | | `src/myna/core/workflow/` | `config`, `run`, `sync`, `status`, and input loading stages | CLI entrypoint delegates here | | `src/myna/core/components/` | Workflow component classes and lookup table | Component string keys are user-facing compatibility surface | | `src/myna/core/files/` | Output file classes and validation/sync value extraction | Add new output formats here | @@ -62,7 +63,7 @@ The main user workflows are: | `docs/api-docs/` | Generated API docs from `scripts/group_docs.py` | Ignored by git; regenerate before docs builds | | `scripts/group_docs.py` | Generates and groups LazyDocs API documentation | CI runs this before `mkdocs build --strict` | | `scripts/check_dev_tools.py` | Checks local development tool availability and writable caches | Run first in new agent or container shells | -| `scripts/check_docs_harness.py` | Validates required agent-harness docs and links | Runs in pre-commit and CI | +| `scripts/check_docs_harness.py` | Validates required agent-harness docs, links, and architecture-doc updates for sensitive code changes | Runs in pre-commit and CI | | `.pre-commit-config.yaml` | Local quality hooks | Includes Ruff, codespell, license headers, and docs harness check | | `.github/workflows/CI.yml` | Package build, default tests, pylint, API docs, MkDocs, and external-app example CI | External-app job runs in a container | | `.github/workflows/pre-commit.yml` | Pre-commit CI | Runs hooks on pull requests | @@ -87,11 +88,17 @@ On import, `src/myna/core/__init__.py` sets: - `MYNA_INSTALL_PATH` to the installed `myna` package root; - `MYNA_APP_PATH` to the installed `myna/application` directory. -Workflow commands set additional runtime environment variables, including `MYNA_INPUT`, -`MYNA_CONFIG_INPUT`, `MYNA_RUN_INPUT`, `MYNA_SYNC_INPUT`, `MYNA_STEP_NAME`, -`MYNA_STEP_CLASS`, and `MYNA_STEP_INDEX`. Usage of runtime environment variables -is planned to be deprecated--the `*_INPUT` variables are marked in code as -future deprecation targets. Prefer using `MynaApp` class parameters for new features. +Workflow run and sync state is carried through `myna.core.context.WorkflowContext`. +This context includes the active input file, current step, step class, step index, and +previous-step information. `myna.core.context` resolves that explicit context first and +still falls back to legacy `MYNA_*` environment variables so direct stage-script +invocation remains compatible in Myna 1.x. That env-var fallback is deprecated and +scheduled for removal in Myna 2.0. `MynaApp` consumes that shared context resolution +rather than reading workflow env vars directly. + +`myna config` still sets `MYNA_INPUT` and the deprecated `MYNA_CONFIG_INPUT` while it +extracts metadata. New shared workflow code should prefer `WorkflowContext`, +`MynaApp` attributes, or `get_workflow_input_file()` over direct `os.environ` reads. ## Main Concepts and Domain Model @@ -107,8 +114,8 @@ future deprecation targets. Prefer using `MynaApp` class parameters for new feat `data_requirements`, `input_requirement`, `output_requirement`, and hierarchical `types` such as `build`, `build_region`, `part`, `region`, and `layer`. - **Application wrapper**: Tool-specific code under `src/myna/application///` - that may provide `configure.py`, `execute.py`, and `postprocess.py` stages. Wrappers - commonly use `myna.core.app.MynaApp`. + that may provide `configure.py`, `execute.py`, and `postprocess.py` stage modules. + Wrappers commonly use `myna.core.app.MynaApp`. - **Database adapter**: A subclass of `myna.core.db.Database` under `src/myna/database/` that loads metadata and syncs outputs for a supported data source. @@ -138,7 +145,7 @@ flowchart TD Component --> Cases[case directories and myna_data.yaml] CLI --> Run[myna run] Run --> ComponentRun[Component.run_component] - ComponentRun --> AppStages[application configure/execute/postprocess scripts] + ComponentRun --> AppStages[application configure/execute/postprocess modules] AppStages --> Outputs[component output files] CLI --> Sync[myna sync] Sync --> FileValidation[file validation] @@ -150,9 +157,11 @@ the input file, resolves the database adapter with `return_datatype_class`, extr component metadata requirements, creates case directories, writes per-case `myna_data.yaml`, records expected output paths, and writes the configured input. For `run`, Myna reloads the input before each step, applies step settings to a component, -sets step environment variables, and runs the available app-stage scripts. For `sync`, -Myna validates component outputs and delegates supported sync behavior to the selected -database adapter. +and calls available app-stage modules in-process through `Component.run_component`. +Stage command-line arguments from the input file are still exposed through `sys.argv` +while each stage runs, preserving `argparse`-based app wrappers. For `sync`, Myna +validates component outputs and delegates supported sync behavior to the selected +database adapter while providing the active input file through `WorkflowContext`. ## Dependency Boundaries @@ -222,7 +231,7 @@ Common validation commands: | External examples | `uv run pytest -m "examples and not parallel"` | Requires external tools and example fixtures | | Generate API docs | `uv run scripts/group_docs.py` | Writes ignored `docs/api-docs/` | | Build docs | `uv run mkdocs build --strict` | Run after API docs generation for parity with CI | -| Docs harness | `uv run python scripts/check_docs_harness.py` | Verifies agent-doc structure, links, and compactness | +| Docs harness | `uv run python scripts/check_docs_harness.py` | Verifies required agent docs, links, and architecture-doc updates for sensitive code changes | | Pre-commit | `uv run pre-commit run --all-files` | May require network the first time hooks install | CI behavior: diff --git a/docs/developer_guide.md b/docs/developer_guide.md index 76154c73..a74d6c56 100644 --- a/docs/developer_guide.md +++ b/docs/developer_guide.md @@ -119,33 +119,54 @@ input file, just provide an arbitrary name for the application, e.g., "test". ## Developing new applications Once a new component is implemented, you will have to implement a corresponding -application (app) to use with `myna run`. Applications consist of up to three scripts: - -1. `config.py`: Script that configures the files within each Myna case folder generated -during `myna config`. At the end of this script, each case folder should be a valid -case directory for whatever model is being run. -2. `execute.py`: Script that executes the model for each case. -3. `postprocess.py`: Script that converts the output of the model into the required -myna file format for the component. This may be part of execute.py, as well. - -These scripts are run sequentially and are mainly separated for clarity of the app -functionality and for handling runtime issues. *Technically* all functionality can -be in one script, however, this may get confusing so it is recommended to split -functions into three scripts. If the model is not Python-based, these scripts can -simply wrap other scripts as needed. All steps are optional and if one of these -scripts is not present, it will be ignored. +application (app) to use with `myna run`. Applications consist of up to three stage +modules: + +1. `configure.py`: Module that configures the files within each Myna case folder +generated during `myna config`. At the end of this stage, each case folder should be a +valid case directory for whatever model is being run. +2. `execute.py`: Module that executes the model for each case. +3. `postprocess.py`: Module that converts the output of the model into the required +myna file format for the component. This may be part of `execute.py`, as well. + +These stage modules are imported and called sequentially by `Component.run_component`. +Each module should define a function named for the stage (`configure`, `execute`, or +`postprocess`) or a `main()` function. The stages are mainly separated for clarity of +the app functionality and for handling runtime issues. *Technically* all functionality +can be in one stage, however, this may get confusing so it is recommended to split +functions into three stages. If the model is not Python-based, these stages can simply +wrap other commands as needed. All steps are optional and if one of these modules is +not present, it will be ignored. Many of the already implemented apps use the `argparse` library to parse user-specified inputs. In the input file, `configure`, `execute`, and -`postprocess` allow users to pass options to each of the scripts -for the app. Any parameters that you wish to have accessible to users are -intended to be adjusted through such options, which are passed to the script via -command line in the format `--key value` or `--key` for Boolean flags. For Boolean -flags, the assumed behavior is False if the flag is not passed and True if the flag is -passed. +`postprocess` allow users to pass options to each stage for the app. Any parameters +that you wish to have accessible to users are intended to be adjusted through such +options, which are exposed to the active stage through `sys.argv` in the format +`--key value` or `--key` for Boolean flags. For Boolean flags, the assumed behavior is +False if the flag is not passed and True if the flag is passed. + +Applications that derive from `MynaApp` should read workflow state from app attributes +such as `self.input_file`, `self.step_name`, `self.last_step_name`, and +`self.step_index`. Avoid reading `MYNA_*` environment variables directly in new app +code; those names are only populated temporarily while a stage is running for +compatibility with existing wrappers and direct stage invocation in Myna 1.x. That +env-var fallback is deprecated and will be removed in Myna 2.0. + +Because stage modules are imported and executed in-process, they should restore any +global process state they modify, especially the current working directory. Prefer +`myna.core.utils.working_directory()` over bare `os.chdir()` calls. It is likely that your app will require a `template` directory, or a set of input files for your model that get copied into every case. If you are using a template -directory, then the intended functionality is that during `config.py` the template +directory, then the intended functionality is that during `configure.py` the template folder is copied into each of the case directory *and then updated*. Updating the files inside the original template folder should be avoided. + +## Versioning applications + +There is base `MynaApp` functionality to extract a version number from process output +given a regex pattern to match. This can be used to have branching behavior in a Myna +application based on the version of the executable that is being used. Adding in branching +logic instead of updating the logic to handle only the most recent version of a code +is generally preferable for backwards compatibility. diff --git a/docs/documentation.md b/docs/documentation.md index f378353f..83113920 100644 --- a/docs/documentation.md +++ b/docs/documentation.md @@ -70,7 +70,11 @@ The documentation harness is intentionally small: - `scripts/check_dev_tools.py` verifies that the current shell can run the repository's development toolchain. - `scripts/check_docs_harness.py` verifies required headings, required `AGENTS.md` - links, relative link targets, and the compact size of `AGENTS.md`. + links, relative link targets, compact size of `AGENTS.md`, and architecture-doc + updates when architecture-sensitive code paths change. It also verifies required PR + template guidance and PR template headings. These checks enforce repository design + decisions: the required documentation headers and related harness policies are + not optional formatting preferences. Run the harness check with: diff --git a/examples/cases/microstructure_region/input.yaml b/examples/cases/microstructure_region/input.yaml index 7d99dab8..8ee946e6 100644 --- a/examples/cases/microstructure_region/input.yaml +++ b/examples/cases/microstructure_region/input.yaml @@ -8,7 +8,7 @@ steps: ry: 0.48e-3 rz: 0.16e-3 pad-xy: 0.48e-3 - pad-z: 0.08e-3 + pad-z: 0.48e-3 pad-sub: 0.64e-3 refine-layer: 1 refine-region: 2 @@ -30,6 +30,7 @@ steps: nd: 250 mu: 21 std: 3 + template: ./template execute: np: 2 data: diff --git a/examples/cases/microstructure_region/template/analysis.json b/examples/cases/microstructure_region/template/analysis.json new file mode 100644 index 00000000..4436ccf7 --- /dev/null +++ b/examples/cases/microstructure_region/template/analysis.json @@ -0,0 +1,28 @@ +{ + "Regions": { + "XY": { + "units": "Cells", + "zBounds": [360,360], + "printStats": ["GrainTypeFractions"], + "printPerGrainStats": ["Size","IPFZ-RGB"], + "printPoleFigureData": true, + "printInversePoleFigureData": true + }, + "XZ": { + "units": "Cells", + "yBounds": [200,200], + "printStats": ["GrainTypeFractions"], + "printPerGrainStats": ["Size","IPFZ-RGB"], + "printPoleFigureData": true, + "printInversePoleFigureData": true + }, + "YZ": { + "units": "Cells", + "xBounds": [200,200], + "printStats": ["GrainTypeFractions"], + "printPerGrainStats": ["Size","IPFZ-RGB"], + "printPoleFigureData": true, + "printInversePoleFigureData": true + } + } +} diff --git a/examples/cases/microstructure_region/template/inputs.json b/examples/cases/microstructure_region/template/inputs.json new file mode 100644 index 00000000..1e1630ca --- /dev/null +++ b/examples/cases/microstructure_region/template/inputs.json @@ -0,0 +1,38 @@ +{ + "SimulationType": "FromFile", + "MaterialFileName": "", + "GrainOrientationFile": "", + "RandomSeed": 0.0, + "Domain": { + "CellSize": 2.5, + "TimeStep": 0.0625, + "NumberOfLayers": 10, + "LayerOffset": 20.0 + }, + "Nucleation": { + "Density": 250.0, + "MeanUndercooling": 21.0, + "StDev": 3.0 + }, + "TemperatureData": { + "TemperatureFiles": [ + ], + "LayerwiseTempRead": true + }, + "Substrate": { + "MeanSize": 12.3 + }, + "Printing": { + "PathToOutput": "./", + "OutputFile": "exaca", + "PrintBinary": true, + "Interlayer": { + "Fields": [ + "GrainID", + "LayerID", + "GrainMisorientation", + "MeltTimeStep" + ] + } + } +} diff --git a/examples/cases/microstructure_region/template/runCase.sh b/examples/cases/microstructure_region/template/runCase.sh new file mode 100644 index 00000000..f458bae2 --- /dev/null +++ b/examples/cases/microstructure_region/template/runCase.sh @@ -0,0 +1,10 @@ +#!/bin/bash +cd ${0%/*} || exit 1 # run from this directory + +# source the correct version of ExaCA based +# on {{EXACA_BIN_PATH}} and {{EXACA_EXEC}} specified +# in the app's configure.py script +EXE="{{EXACA_BIN_PATH}}/{{EXACA_EXEC}}" + +# Run the ExaCA case +mpiexec -n {{RANKS}} $EXE inputs.json > exaca_run.log 2>&1 diff --git a/scripts/check_docs_harness.py b/scripts/check_docs_harness.py index 6858d594..7438e3c2 100644 --- a/scripts/check_docs_harness.py +++ b/scripts/check_docs_harness.py @@ -7,6 +7,7 @@ from __future__ import annotations import re +import subprocess import sys from pathlib import Path @@ -49,6 +50,29 @@ "docs/documentation.md", ] +ARCHITECTURE_SENSITIVE_PATHS = [ + "src/myna/core/workflow/", + "src/myna/core/components/", + "src/myna/core/app/", + "src/myna/core/context.py", + "src/myna/database/", + "src/myna/application/readme.md", +] + +ARCHITECTURE_DOC_PATHS = [ + "ARCHITECTURE.md", + "docs/developer_guide.md", +] + +PR_TEMPLATE_REQUIRED_HEADINGS = [ + "## Summary", + "## Related issues", + "## Details", + "## Impact", + "## Testing strategy", + "## Checklist", +] + def fail(message: str) -> None: print(f"docs harness check failed: {message}", file=sys.stderr) @@ -118,10 +142,102 @@ def check_agents_is_compact() -> None: ) +def check_pr_template_guidance() -> None: + template_text = read_required_file(".github/pull_request_template.md") + for heading in PR_TEMPLATE_REQUIRED_HEADINGS: + if not has_heading(template_text, heading): + fail( + f".github/pull_request_template.md is missing required heading: {heading}" + ) + + agents_text = read_required_file("AGENTS.md") + required_phrases = [ + ".github/pull_request_template.md", + "Preserve the template headings and checklist", + "state behavior impact, risk, and testing strategy", + ] + for phrase in required_phrases: + if phrase not in agents_text: + fail(f"AGENTS.md is missing PR template guidance: {phrase}") + + +def run_git_command(args: list[str]) -> list[str]: + try: + result = subprocess.run( + ["git", *args], + cwd=REPO_ROOT, + check=False, + capture_output=True, + text=True, + ) + except FileNotFoundError: + return [] + + if result.returncode != 0: + return [] + return [line.strip() for line in result.stdout.splitlines() if line.strip()] + + +def get_comparison_base() -> str | None: + for ref in ["origin/main", "main"]: + merge_base = run_git_command(["merge-base", "HEAD", ref]) + if merge_base: + return merge_base[0] + return None + + +def get_changed_files() -> set[str]: + merge_base = get_comparison_base() + if merge_base is not None: + changed = set(run_git_command(["diff", "--name-only", f"{merge_base}..HEAD"])) + else: + changed = set(run_git_command(["diff", "--name-only", "HEAD"])) + changed.update(run_git_command(["ls-files", "--others", "--exclude-standard"])) + return changed + + +def path_matches_any(path: str, prefixes_or_paths: list[str]) -> bool: + return any( + path == candidate.rstrip("/") or path.startswith(candidate) + for candidate in prefixes_or_paths + ) + + +def check_architecture_docs_updated_for_sensitive_changes() -> None: + changed_files = get_changed_files() + if not changed_files: + return + + sensitive_changes = sorted( + path + for path in changed_files + if path_matches_any(path, ARCHITECTURE_SENSITIVE_PATHS) + ) + if not sensitive_changes: + return + + docs_changes = sorted( + path for path in changed_files if path_matches_any(path, ARCHITECTURE_DOC_PATHS) + ) + if docs_changes: + return + + changed_list = "\n".join(f" - {path}" for path in sensitive_changes) + required_list = "\n".join(f" - {path}" for path in ARCHITECTURE_DOC_PATHS) + fail( + "architecture-sensitive files changed without architecture documentation " + "updates.\nChanged files:\n" + f"{changed_list}\nUpdate at least one of:\n{required_list}\n" + "If no docs update is needed, state that explicitly in the PR." + ) + + def main() -> None: check_required_headings() check_agents_links() check_agents_is_compact() + check_pr_template_guidance() + check_architecture_docs_updated_for_sensitive_changes() print("docs harness check passed") diff --git a/src/myna/application/additivefoam/solidification_region_reduced/app.py b/src/myna/application/additivefoam/solidification_region_reduced/app.py index b3f3f094..88d9b46f 100644 --- a/src/myna/application/additivefoam/solidification_region_reduced/app.py +++ b/src/myna/application/additivefoam/solidification_region_reduced/app.py @@ -495,10 +495,11 @@ def postprocess_case(self, mynafile): with working_directory(case_dict["case_dir"]): # Determine if parallel execution + step_index = self.step_index + if step_index is None: + step_index = self.step_number np = nested_get( - self.settings["steps"][int(os.environ["MYNA_STEP_INDEX"])][ - self.step_name - ], + self.settings["steps"][step_index][self.step_name], ["execute", "np"], default_value=1, ) @@ -519,20 +520,33 @@ def postprocess_case(self, mynafile): # Check data exists datafiles = sorted(glob.glob(f"{case_dict['case_dir']}/ExaCA/*")) if len(datafiles) > 0: - # Header - process = self.start_subprocess( - ["echo", "x (m),y (m),z (m),tm (s),ts (s),cr (k/s)"], - stdout=mf, - stderr=f, - ) - self.wait_for_process_success(process) - # Data - process = self.start_subprocess( - ["cat", *datafiles], - stdout=mf, - stderr=f, - ) - self.wait_for_process_success(process) + # Write header + header = "x (m),y (m),z (m),tm (s),ts (s),cr (k/s)\n" + mf.write(header) + + # Write data, removing header if found + for datafile in datafiles: + with open(datafile, mode="r", encoding="utf-8") as src: + first_line = src.readline() + + if not first_line: + continue + + # Check if line is header, write if not + first_field = first_line.split(",", 1)[0].strip() + is_header = False + try: + float(first_field) + except ValueError: + is_header = True + except ValueError: + pass + if not is_header: + mf.write(first_line) + + # Write the rest of the file + # (shutil.copyfileobj starts from current position) + shutil.copyfileobj(src, mf) # Clean up parallel case files if parallel: diff --git a/src/myna/application/bnpy/bnpy.py b/src/myna/application/bnpy/bnpy.py index 78d6f2e5..4c715785 100644 --- a/src/myna/application/bnpy/bnpy.py +++ b/src/myna/application/bnpy/bnpy.py @@ -18,8 +18,10 @@ def __init__(self): self.app_type = "bnpy" self.sF = 0.5 self.gamma = 8 - self.settings = load_input(os.environ["MYNA_INPUT"]) - self.input_dir = os.path.dirname(os.environ["MYNA_INPUT"]) + if self.input_file is None: + raise ValueError("Bnpy apps require a Myna workflow input file.") + self.settings = load_input(self.input_file) + self.input_dir = os.path.dirname(self.input_file) self.resource_dir = os.path.join(self.input_dir, "myna_resources") self.resource_template_dir = os.path.join( self.resource_dir, *self.name.split("/") diff --git a/src/myna/application/bnpy/cluster_solidification/execute.py b/src/myna/application/bnpy/cluster_solidification/execute.py index c0a8f98f..c2663dd6 100644 --- a/src/myna/application/bnpy/cluster_solidification/execute.py +++ b/src/myna/application/bnpy/cluster_solidification/execute.py @@ -14,6 +14,7 @@ import glob import matplotlib.pyplot as plt from myna.application.bnpy import get_representative_distribution +from myna.core.utils import working_directory from .app import BnpyClusterSolidification @@ -51,9 +52,6 @@ def train_voxel_model(myna_files, thermal_files, sF, gamma, input_dir): 'Myna bnpy app requires "pip install .[bnpy]" optional dependencies!' ) - # Store original working directory - orig_dir = os.getcwd() - # Create blank dataframe df_training = pd.DataFrame({"logG": [], "logV": []}) @@ -61,43 +59,42 @@ def train_voxel_model(myna_files, thermal_files, sF, gamma, input_dir): for myna_file, thermal_file in zip(myna_files, thermal_files): # Go to case directory case_dir = os.path.dirname(myna_file) - os.chdir(case_dir) - - # Get case myna_data - resource_dir = os.path.join(input_dir, "myna_resources") - - # Set directory for training data - resource_template_dir = os.path.join(resource_dir, "cluster_solidification") - - # Create symbolic links to all available thermal results - thermal_dir = "thermal_data" - os.makedirs(thermal_dir, exist_ok=True) - copy_path = os.path.join("thermal_data", os.path.basename(thermal_file)) - if not os.path.exists(copy_path): - try: - os.symlink(thermal_file, copy_path) - except FileExistsError: - os.remove(copy_path) - os.symlink(thermal_file, copy_path) - thermal_file_local = copy_path - - # Set up folder structure - training_dir = os.path.join(resource_template_dir, "training_voxels") - os.makedirs(training_dir, exist_ok=True) - - # Reduce data to only the columns needed for training - _, df_case_training = reduce_thermal_file_to_df(thermal_file_local) - - # Get a representative sampling of the case dataset - training_data, _ = get_representative_distribution( - df_case_training.to_numpy(), bins=20 - ) - df_case_training = pd.DataFrame( - data=training_data, columns=df_case_training.columns - ) + with working_directory(case_dir): + # Get case myna_data + resource_dir = os.path.join(input_dir, "myna_resources") + + # Set directory for training data + resource_template_dir = os.path.join(resource_dir, "cluster_solidification") + + # Create symbolic links to all available thermal results + thermal_dir = "thermal_data" + os.makedirs(thermal_dir, exist_ok=True) + copy_path = os.path.join("thermal_data", os.path.basename(thermal_file)) + if not os.path.exists(copy_path): + try: + os.symlink(thermal_file, copy_path) + except FileExistsError: + os.remove(copy_path) + os.symlink(thermal_file, copy_path) + thermal_file_local = copy_path + + # Set up folder structure + training_dir = os.path.join(resource_template_dir, "training_voxels") + os.makedirs(training_dir, exist_ok=True) + + # Reduce data to only the columns needed for training + _, df_case_training = reduce_thermal_file_to_df(thermal_file_local) + + # Get a representative sampling of the case dataset + training_data, _ = get_representative_distribution( + df_case_training.to_numpy(), bins=20 + ) + df_case_training = pd.DataFrame( + data=training_data, columns=df_case_training.columns + ) - # Concatenate the case data to the training dataset - df_training = pd.concat([df_training, df_case_training], ignore_index=True) + # Concatenate the case data to the training dataset + df_training = pd.concat([df_training, df_case_training], ignore_index=True) # Get a representative sampling of the training dataset for the current cases training_data, _ = get_representative_distribution(df_training.to_numpy(), bins=20) @@ -192,9 +189,6 @@ def train_voxel_model(myna_files, thermal_files, sF, gamma, input_dir): print(f'{info_dict["task_output_path"]=}') trained_model_path = info_dict["task_output_path"] - # Return to original working directory - os.chdir(orig_dir) - return trained_model_path @@ -214,117 +208,114 @@ def run_clustering( 'Myna bnpy app requires "pip install .[bnpy]" optional dependencies!' ) - # Go to case directory - orig_dir = os.getcwd() - os.chdir(case_dir) - - # Get case myna_data - resource_dir = os.path.join(input_dir, "myna_resources") - - # Set directory for training data - resource_template_dir = os.path.join(resource_dir, "cluster_solidification") - - # Create symbolic links to all available thermal results - thermal_dir = "thermal_data" - os.makedirs(thermal_dir, exist_ok=True) - copy_path = os.path.join("thermal_data", os.path.basename(thermal_file)) - if not os.path.exists(copy_path): - try: - os.symlink(thermal_file, copy_path) - except FileExistsError: - os.remove(copy_path) - os.symlink(thermal_file, copy_path) - thermal_file_local = copy_path - - # Set up folder structure for voxel clustering results - cluster_dir = os.path.join(case_dir, "cluster_voxels") - os.makedirs(cluster_dir, exist_ok=True) - - # Convert the thermal dataset to a bnpy dataset for clustering - df_output, df_cluster = reduce_thermal_file_to_df(thermal_file_local) - dataset_current = bnpy.data.XData.from_dataframe(df_cluster) - - # Get the latest trained model - model_dir = os.path.join( - resource_template_dir, f"voxel_model-sF={sF}-gamma={gamma}" - ) - latest_model = sorted(glob.glob(os.path.join(model_dir, "*")), reverse=True)[0] - latest_model_iteration = sorted( - glob.glob(os.path.join(latest_model, "*")), reverse=True - )[0] - task_output_path = latest_model_iteration - cur_model, lap_val = bnpy.load_model_at_lap(task_output_path, None) - - # Assign cluster IDs, or overwrite them if specified - result_file = os.path.join(cluster_dir, "cluster_ids.csv") - if not os.path.exists(result_file) or overwrite: - K = cur_model.allocModel.K - - # Assign current data to clusters - local_params = cur_model.calc_local_params(dataset_current) - resp = local_params["resp"] - soft_cluster = np.argmax(resp, axis=1) - n_digits = cur_model.allocModel.K - df_output["id"] = soft_cluster - df_output.to_csv(result_file, index=False) - - # If file already exists and should not be overwritten, load cluster IDs - else: - df_output = pd.read_csv(result_file) - n_digits = cur_model.allocModel.K - K = n_digits - - # Generate plots - dpi = 300 - colors, cmap, _ = myna_bnpy.cluster_colormap(n_digits) - suffix = f"sF={sF}-g={gamma}-K={K}" - - # GV plot - gv_plot_file = os.path.join(cluster_dir, f"cluster_GV-{suffix}.png") - if not os.path.exists(gv_plot_file) or overwrite: - myna_bnpy.voxel_GV_plot(df_output, colors, cmap, gv_plot_file, dpi=dpi) - - # Field histograms - fields = ["logG", "logV"] - for field in fields: - field_value_file = os.path.join(cluster_dir, f"cluster_{field}-{suffix}.png") - if not os.path.exists(field_value_file) or overwrite: - myna_bnpy.voxel_id_stacked_histogram( - df_output, - field, - colors, - field_value_file, - dpi=dpi, - ids=[x for x in range(K)], - ) + with working_directory(case_dir): + # Get case myna_data + resource_dir = os.path.join(input_dir, "myna_resources") - # Spatial map of clusters - map_plot_file = os.path.join(cluster_dir, f"cluster_map-{suffix}.png") - if not os.path.exists(map_plot_file) or overwrite: - plt.figure(dpi=dpi) - colormap = "tab10" - if n_digits > 10: - colormap = "tab20" - colors, cmap, colorValues = myna_bnpy.cluster_colormap( - n_digits, colorspace=colormap - ) - fig, ax = plt.subplots() - ax.scatter( - df_output["x (m)"] * 1e3, - df_output["y (m)"] * 1e3, - c=df_output["id"], - s=1, - marker="s", - cmap=cmap, + # Set directory for training data + resource_template_dir = os.path.join(resource_dir, "cluster_solidification") + + # Create symbolic links to all available thermal results + thermal_dir = "thermal_data" + os.makedirs(thermal_dir, exist_ok=True) + copy_path = os.path.join("thermal_data", os.path.basename(thermal_file)) + if not os.path.exists(copy_path): + try: + os.symlink(thermal_file, copy_path) + except FileExistsError: + os.remove(copy_path) + os.symlink(thermal_file, copy_path) + thermal_file_local = copy_path + + # Set up folder structure for voxel clustering results + cluster_dir = os.path.join(case_dir, "cluster_voxels") + os.makedirs(cluster_dir, exist_ok=True) + + # Convert the thermal dataset to a bnpy dataset for clustering + df_output, df_cluster = reduce_thermal_file_to_df(thermal_file_local) + dataset_current = bnpy.data.XData.from_dataframe(df_cluster) + + # Get the latest trained model + model_dir = os.path.join( + resource_template_dir, f"voxel_model-sF={sF}-gamma={gamma}" ) - ax.set_aspect("equal") - ax.set_xlabel("X (mm)") - ax.set_ylabel("Y (mm)") - plt.savefig(map_plot_file, dpi=dpi) - plt.close() - - # Return to original working directory and return result file path - os.chdir(orig_dir) + latest_model = sorted(glob.glob(os.path.join(model_dir, "*")), reverse=True)[0] + latest_model_iteration = sorted( + glob.glob(os.path.join(latest_model, "*")), reverse=True + )[0] + task_output_path = latest_model_iteration + cur_model, lap_val = bnpy.load_model_at_lap(task_output_path, None) + + # Assign cluster IDs, or overwrite them if specified + result_file = os.path.join(cluster_dir, "cluster_ids.csv") + if not os.path.exists(result_file) or overwrite: + K = cur_model.allocModel.K + + # Assign current data to clusters + local_params = cur_model.calc_local_params(dataset_current) + resp = local_params["resp"] + soft_cluster = np.argmax(resp, axis=1) + n_digits = cur_model.allocModel.K + df_output["id"] = soft_cluster + df_output.to_csv(result_file, index=False) + + # If file already exists and should not be overwritten, load cluster IDs + else: + df_output = pd.read_csv(result_file) + n_digits = cur_model.allocModel.K + K = n_digits + + # Generate plots + dpi = 300 + colors, cmap, _ = myna_bnpy.cluster_colormap(n_digits) + suffix = f"sF={sF}-g={gamma}-K={K}" + + # GV plot + gv_plot_file = os.path.join(cluster_dir, f"cluster_GV-{suffix}.png") + if not os.path.exists(gv_plot_file) or overwrite: + myna_bnpy.voxel_GV_plot(df_output, colors, cmap, gv_plot_file, dpi=dpi) + + # Field histograms + fields = ["logG", "logV"] + for field in fields: + field_value_file = os.path.join( + cluster_dir, f"cluster_{field}-{suffix}.png" + ) + if not os.path.exists(field_value_file) or overwrite: + myna_bnpy.voxel_id_stacked_histogram( + df_output, + field, + colors, + field_value_file, + dpi=dpi, + ids=[x for x in range(K)], + ) + + # Spatial map of clusters + map_plot_file = os.path.join(cluster_dir, f"cluster_map-{suffix}.png") + if not os.path.exists(map_plot_file) or overwrite: + plt.figure(dpi=dpi) + colormap = "tab10" + if n_digits > 10: + colormap = "tab20" + colors, cmap, colorValues = myna_bnpy.cluster_colormap( + n_digits, colorspace=colormap + ) + fig, ax = plt.subplots() + ax.scatter( + df_output["x (m)"] * 1e3, + df_output["y (m)"] * 1e3, + c=df_output["id"], + s=1, + marker="s", + cmap=cmap, + ) + ax.set_aspect("equal") + ax.set_xlabel("X (mm)") + ax.set_ylabel("Y (mm)") + plt.savefig(map_plot_file, dpi=dpi) + plt.close() + return result_file @@ -342,16 +333,16 @@ def main(): # Parse command line arguments args = parser.parse_args() - settings = load_input(os.environ["MYNA_INPUT"]) + settings = load_input(app.input_file) train_model = args.train_model overwrite = args.overwrite # Get expected Myna output files - step_name = os.environ["MYNA_STEP_NAME"] + step_name = app.step_name myna_files = settings["data"]["output_paths"][step_name] thermal_step_name = args.thermal if thermal_step_name is None: - thermal_step_name = os.environ["MYNA_LAST_STEP_NAME"] + thermal_step_name = app.last_step_name thermal_files = settings["data"]["output_paths"][thermal_step_name] # Assemble training data and train model diff --git a/src/myna/application/bnpy/cluster_supervoxel/execute.py b/src/myna/application/bnpy/cluster_supervoxel/execute.py index 99fce5b6..9402bc3d 100644 --- a/src/myna/application/bnpy/cluster_supervoxel/execute.py +++ b/src/myna/application/bnpy/cluster_supervoxel/execute.py @@ -13,6 +13,7 @@ import numpy as np import matplotlib.pyplot as plt import myna.application.bnpy as myna_bnpy +from myna.core.utils import working_directory from .app import BnpyClusterSupervoxel from myna.application.bnpy import ( add_cluster_colormap_colorbar, @@ -101,9 +102,6 @@ def train_supervoxel_model( 'Myna bnpy app requires "pip install .[bnpy]" optional dependencies!' ) - # Store original working directory - orig_dir = os.getcwd() - # Create blank dataframe col_names = [f"c_{x}" for x in range(app.n_voxel_clusters)] schema = {} @@ -116,30 +114,29 @@ def train_supervoxel_model( for myna_file, myna_voxel_file in zip(myna_files, myna_voxel_files): # Get case myna_data case_dir = os.path.dirname(myna_file) - os.chdir(case_dir) - - # Create symbolic links to all available clustering results - voxel_dir = "voxel_data" - os.makedirs(voxel_dir, exist_ok=True) - copy_path = os.path.join(voxel_dir, os.path.basename(myna_voxel_file)) - if not os.path.exists(copy_path): - os.symlink(myna_voxel_file, copy_path) - - # Set output file - composition_file = os.path.join(case_dir, comp_file_name) - - # Load voxel information - df_composition = reduce_voxel_file_to_supervoxel_df( - myna_voxel_file, - app, - write_csv=True, - output_file=composition_file, - ) - composition_files.append(composition_file) - - # Remove spatial information for training - df_case_training = df_composition.drop(["x (m)", "y (m)"]) - df_training = df_training.vstack(df_case_training) + with working_directory(case_dir): + # Create symbolic links to all available clustering results + voxel_dir = "voxel_data" + os.makedirs(voxel_dir, exist_ok=True) + copy_path = os.path.join(voxel_dir, os.path.basename(myna_voxel_file)) + if not os.path.exists(copy_path): + os.symlink(myna_voxel_file, copy_path) + + # Set output file + composition_file = os.path.join(case_dir, comp_file_name) + + # Load voxel information + df_composition = reduce_voxel_file_to_supervoxel_df( + myna_voxel_file, + app, + write_csv=True, + output_file=composition_file, + ) + composition_files.append(composition_file) + + # Remove spatial information for training + df_case_training = df_composition.drop(["x (m)", "y (m)"]) + df_training = df_training.vstack(df_case_training) # Check for other preexitsting training data to include suffix_training = "training.csv" @@ -227,8 +224,6 @@ def train_supervoxel_model( print(f'{info_dict["task_output_path"]=}') trained_model_path = info_dict["task_output_path"] - # Return to original working directory and return result file path - os.chdir(orig_dir) return trained_model_path, composition_files @@ -256,9 +251,6 @@ def run( 'Myna bnpy app requires "pip install .[bnpy]" optional dependencies!' ) - # Store original working directory - orig_dir = os.getcwd() - # Create bnpy dataset for clustering df = pl.read_csv(composition_file) df_composition = df.drop("x (m)", "y (m)") @@ -335,7 +327,6 @@ def run( export_name=composition_file.replace(".csv", ".png"), ) - os.chdir(orig_dir) return @@ -389,11 +380,11 @@ def main(): app.n_voxel_clusters = max(voxel_model.allocModel.K, 2) # Get expected Myna output files - step_name = os.environ["MYNA_STEP_NAME"] + step_name = app.step_name myna_files = app.settings["data"]["output_paths"][step_name] cluster_step_name = app.args.cluster if cluster_step_name == "": - cluster_step_name = os.environ["MYNA_LAST_STEP_NAME"] + cluster_step_name = app.last_step_name voxel_cluster_files = app.settings["data"]["output_paths"][cluster_step_name] # Assemble training data and train model diff --git a/src/myna/application/exaca/exaca.py b/src/myna/application/exaca/exaca.py index 69ee64bc..114aba2c 100644 --- a/src/myna/application/exaca/exaca.py +++ b/src/myna/application/exaca/exaca.py @@ -8,9 +8,11 @@ # import json import os +import re import shutil import numpy as np +import polars as pl from myna.core.app.base import MynaApp from myna.core.utils import nested_set @@ -68,6 +70,65 @@ def parse_postprocess_arguments(self): self.parse_shared_arguments() self.parse_known_args() + def get_executable_version(self, timeout=30): + """Return the ExaCA version reported by its executable banner. + + ExaCA prints its version when invoked without an input file, then exits + unsuccessfully because that input is required. The base helper preserves + that output for version extraction. + """ + return super().get_executable_version( + default="ExaCA", + version_args=None, + version_regex=r"^ExaCA version:\s*(?P\S+)", + timeout=timeout, + ) + + @staticmethod + def _uses_exaca_21_input_schema(version): + """Return whether an ExaCA version uses the 2.1 input schema.""" + match = re.match(r"^(\d+)\.(\d+)", version) + if match is None: + raise ValueError( + f"Could not compare unrecognized ExaCA version {version!r}." + ) + return tuple(int(value) for value in match.groups()) >= (2, 1) + + def convert_case_input_for_exaca_version(self, case_dir, input_settings): + """Update deprecated substrate settings for the installed ExaCA version. + + ExaCA 2.1 renamed ``MeanSize`` and ``PowderDensity``. A canonical setting + already present in a custom template takes precedence over its deprecated + counterpart. + """ + version = self.get_executable_version() + if not self._uses_exaca_21_input_schema(version): + return input_settings + + substrate = input_settings.get("Substrate") + if substrate is None: + return input_settings + # Direct replacement + for deprecated, replacement in { + "MeanSize": "MeanBaseplateGrainSize", + }.items(): + if deprecated in substrate: + substrate.setdefault(replacement, substrate[deprecated]) + del substrate[deprecated] + # Derived replacement: density -> size + for deprecated, replacement in { + "PowderDensity": "MeanPowderGrainSize", + }.items(): + if deprecated in substrate: + substrate.setdefault( + replacement, 1 / np.power(substrate[deprecated], 1 / 3) + ) + del substrate[deprecated] + + with open(os.path.join(case_dir, "inputs.json"), "w", encoding="utf-8") as f: + json.dump(input_settings, f, indent=2) + return input_settings + def _get_material_file(self, myna_settings): """Resolve the ExaCA material definition file for the current build.""" material = myna_settings["build"]["build_data"]["material"]["value"] @@ -104,6 +165,46 @@ def _update_input_settings( nested_set(input_settings, ["Substrate", "MeanSize"], self.args.sub_size) return input_settings + def _sanitize_temperature_files(self, case_dir, solid_files): + """Convert Myna reduced-solidification CSVs into ExaCA's bare-header schema.""" + expected_columns = ("x", "y", "z", "tm", "ts", "cr") + sanitized_files = [] + for index, solid_file in enumerate(solid_files): + df = pl.read_csv(solid_file) + if set(expected_columns).issubset(df.columns): + sanitized_files.append(solid_file) + continue + + rename_map = {} + for column in expected_columns: + unit_name = ( + f"{column} (m)" + if column in ("x", "y", "z") + else (f"{column} (s)" if column in ("tm", "ts") else "cr (k/s)") + ) + if unit_name in df.columns: + rename_map[unit_name] = column + if rename_map: + df = df.rename(rename_map) + + cast_exprs = [ + pl.col(column).cast(pl.Float64, strict=False).alias(column) + for column in expected_columns + if column in df.columns + ] + if cast_exprs: + df = df.with_columns(cast_exprs) + if not set(expected_columns).issubset(df.columns): + sanitized_files.append(solid_file) + continue + df = df.drop_nulls(list(expected_columns)) + df = df.select(list(expected_columns)) + + case_file = os.path.join(case_dir, f"exaca_input_{index:03}.csv") + df.write_csv(case_file) + sanitized_files.append(case_file) + return sanitized_files + def _replace_run_script_placeholders(self, run_script, replacements): """Replace template placeholders in a case run script.""" with open(run_script, "r", encoding="utf-8") as f: @@ -141,6 +242,7 @@ def _patch_case_ranks(self, case_dir): def setup_exaca_case(self, case_dir, solid_files, layer_thickness): """Copy a template and populate shared ExaCA inputs for a case directory.""" self.copy_template_to_case(case_dir) + solid_files = self._sanitize_temperature_files(case_dir, solid_files) myna_settings = load_input(os.path.join(case_dir, "myna_data.yaml")) input_file = os.path.join(case_dir, "inputs.json") diff --git a/src/myna/application/exaca/microstructure_region/app.py b/src/myna/application/exaca/microstructure_region/app.py index 7bacce05..4043041d 100644 --- a/src/myna/application/exaca/microstructure_region/app.py +++ b/src/myna/application/exaca/microstructure_region/app.py @@ -22,6 +22,9 @@ def __init__(self): def setup_case(self, case_dir, solid_files, layer_thickness): """Configure a valid ExaCA case directory for the current region.""" input_settings = self.setup_exaca_case(case_dir, solid_files, layer_thickness) + input_settings = self.convert_case_input_for_exaca_version( + case_dir, input_settings + ) self._configure_case_analysis(case_dir, input_settings) def _configure_case_analysis(self, case_dir, input_settings): @@ -35,6 +38,7 @@ def _configure_case_analysis(self, case_dir, input_settings): ) if df.is_empty(): return + df = self._normalize_temperature_columns(df) xmin, xmax = [df["x"].min(), df["x"].max()] ymin, ymax = [df["y"].min(), df["y"].max()] spacing = nested_get(input_settings, ["Domain", "CellSize"]) @@ -61,6 +65,28 @@ def _configure_case_analysis(self, case_dir, input_settings): with open(analysis_file, "w", encoding="utf-8") as f: json.dump(analysis_settings, f, indent=2) + def _normalize_temperature_columns(self, df): + """Support both legacy bare coordinates and unit-bearing Myna CSV headers.""" + rename_map = {} + for axis in ("x", "y", "z"): + if axis in df.columns: + continue + unit_name = f"{axis} (m)" + if unit_name in df.columns: + rename_map[unit_name] = axis + if rename_map: + df = df.rename(rename_map) + cast_columns = [axis for axis in ("x", "y", "z") if axis in df.columns] + if cast_columns: + df = df.with_columns( + [ + pl.col(axis).cast(pl.Float64, strict=False).alias(axis) + for axis in cast_columns + ] + ) + df = df.drop_nulls(cast_columns) + return df + def _get_region_case_setup_data(self): """Return region case directories paired with their solidification inputs.""" myna_files = self.get_step_output_paths() diff --git a/src/myna/application/readme.md b/src/myna/application/readme.md index 51fab8eb..35fa6084 100644 --- a/src/myna/application/readme.md +++ b/src/myna/application/readme.md @@ -8,29 +8,32 @@ functionality). `myna` expects a specific directory and file structure for each application (app): -- Parent directories are named after the class names defined in -[../src/myna/components/component_class_lookup.py]. -- Child directory names then correspond to an available app name -for the parent class -- Each available app can contain three stages of executables -that will be called sequentially with the corresponding arguments from -the myna_run input file: +- Parent directories are named after available application names. +- Child directory names correspond to the component class names defined in + `src/myna/core/components/component_class_lookup.py`. +- Each available app can contain three Python stage modules that are imported and + called sequentially with the corresponding arguments from the `myna run` input file: 1. configure.py 2. execute.py 3. postprocess.py -- The expectation is that each configure, execute, and postprocess -script will use the Python argparse module to parse command line inputs -- If the model has a case template that it will copy from, then the -convention is to name that directory "template" (this is -not strictly necessary) -- Other files can be included in the app directory for documentation or -as resources, such as a readme file +- Each stage module should define a function named for the stage (`configure`, + `execute`, or `postprocess`) or a `main()` function. +- The expectation is that each configure, execute, and postprocess stage will use the + Python argparse module to parse command line inputs. `myna` temporarily populates + `sys.argv` for the active stage to preserve command-line parsing behavior. +- Stage code should get workflow state from `MynaApp` attributes rather than directly + reading `MYNA_*` environment variables. Those env vars are exposed only during the + active stage call for compatibility in Myna 1.x and are deprecated for removal in + Myna 2.0. +- Stage code should restore process-global state it changes. Prefer + `myna.core.utils.working_directory()` over bare `os.chdir()` calls. +- If the model has a case template that it will copy from, then the convention is to + name that directory "template" (this is not strictly necessary). +- Other files can be included in the app directory for documentation or as resources, + such as a readme file. -There is no technical difference between the three stages of executables -on the backend of `myna`. The `os.system()` call is the same for each stage. -The split into stages is more for organization of tasks into case setup, -case execution, and tasks to be completed after a (successful) execution, -which will be useful if implementing a more robust workflow manager in -the future. +There is no technical difference between the three stages on the backend of `myna`. +The split into stages is for organization of tasks into case setup, case execution, +and tasks to be completed after a successful execution. diff --git a/src/myna/application/thesis/melt_pool_geometry_part/app.py b/src/myna/application/thesis/melt_pool_geometry_part/app.py index 58fc4517..df505c21 100644 --- a/src/myna/application/thesis/melt_pool_geometry_part/app.py +++ b/src/myna/application/thesis/melt_pool_geometry_part/app.py @@ -56,21 +56,6 @@ def configure_case(self, case_dir, myna_input="myna_data.yaml"): elapsed_time = 0.0 total_segments = 0 for index, pair in enumerate(index_pairs): - segment_dir = Path(case_dir) / f"path_segment_{index:03}" - os.makedirs(segment_dir, exist_ok=True) - for case_file in configured_case_files: - shutil.copy(case_file, segment_dir / Path(case_file).name) - - segment_scanfile = segment_dir / "Path.txt" - df_segment = df[0 : pair[1] + 1] - df_segment.write_csv(segment_scanfile, separator="\t") - - # Write temporary path file for getting elapsed time of the segment to calculate - # the write times for the melt pool geometry: - # - Ignore wait times at beginning and end of the scan segment - # - Distribute `nout` proportionally by segment row count - # - If last segment, ensure that any missing segments due to int() rounding - # are included with tempfile.NamedTemporaryFile() as fp: df_segment_only = df[pair[0] : pair[1] + 1] df_segment_only.write_csv(fp.name, separator="\t") @@ -93,6 +78,17 @@ def configure_case(self, case_dir, myna_input="myna_data.yaml"): fraction_segments, ) elapsed_time += segment_time + if len(segment_times) == 0: + continue + + segment_dir = Path(case_dir) / f"path_segment_{index:03}" + os.makedirs(segment_dir, exist_ok=True) + for case_file in configured_case_files: + shutil.copy(case_file, segment_dir / Path(case_file).name) + + segment_scanfile = segment_dir / "Path.txt" + df_segment = df[0 : pair[1] + 1] + df_segment.write_csv(segment_scanfile, separator="\t") mode_file = segment_dir / "Mode.txt" adjust_parameter( diff --git a/src/myna/core/app/base.py b/src/myna/core/app/base.py index d95ed0a6..4c9a590a 100644 --- a/src/myna/core/app/base.py +++ b/src/myna/core/app/base.py @@ -10,6 +10,7 @@ import argparse import os +import re import sys import time import shutil @@ -19,6 +20,7 @@ import docker from docker.models.containers import Container from myna.core.app._argument_registrar import _ArgumentRegistrar +from myna.core.context import get_workflow_context from myna.core.workflow.load_input import load_input from myna.core.utils import is_executable, get_quoted_str from myna.core.components import return_step_class @@ -32,29 +34,41 @@ class MynaApp: using the MynaApp functionality where possible for consistent behavior across apps. """ - # Define ENV class variables that have relevant workflow information - ENV_SETTINGS_FILE = "MYNA_INPUT" ENV_APP_PATH = "MYNA_APP_PATH" - ENV_STEP_NAME = "MYNA_STEP_NAME" - ENV_LAST_STEP_NAME = "MYNA_LAST_STEP_NAME" def __init__(self): # Set the print name as well as the Myna app and class names self.class_name: str | None = None self.app_type: str | None = None - - # Get the names for the current and previous workflow step - self.step_name = os.environ.get(self.ENV_STEP_NAME) - self.last_step_name = os.environ.get(self.ENV_LAST_STEP_NAME) + self.step_class: str | None = None + self.last_step_class: str | None = None + + # Get workflow context, setting to None if context is not found + context = get_workflow_context() + if context is not None: + self.step_name = context.step_name + self.last_step_name = context.last_step_name + self.step_class = context.step_class + self.last_step_class = context.last_step_class + self.step_index = context.step_index + self.input_file = context.input_file + else: + self.step_name = None + self.last_step_name = None + self.step_class = None + self.last_step_class = None + self.input_file = None + self.step_index = None # Get the input file contents and parse additional step information - self.input_file = os.environ.get(self.ENV_SETTINGS_FILE) self.settings = {} self.step_number = None if self.input_file is not None: self.settings = load_input(self.input_file) step_names = [list(x.keys())[0] for x in self.settings.get("steps", [])] - if self.step_name in step_names: + if self.step_index is not None: + self.step_number = self.step_index + elif self.step_name in step_names: self.step_number = step_names.index(self.step_name) # Set up argparse @@ -169,7 +183,13 @@ def template(self): """Set the path to the template directory based on the path to the app directory""" if self.args.template is None: return Path(self.path) / "template" - return Path(self.args.template) + template_path = Path(self.args.template) + if template_path.is_absolute(): + return template_path + if self.input_file is not None: + input_dir = Path(self.input_file).resolve(strict=False).parent + return input_dir / template_path + return template_path @property def component(self): @@ -349,6 +369,95 @@ def validate_executable(self, default): + "does not have execute permissions." ) + def get_executable(self, default=None): + """Return the configured executable, falling back to ``default``.""" + executable = self.args.exec + if executable is None: + executable = default + if executable is None: + raise ValueError( + f"MynaApp {self.name} requires an executable, but no executable " + "was provided and no default was set." + ) + return executable + + def get_executable_version( + self, + default=None, + version_args=("--version",), + version_regex=None, + timeout=30, + ): + """Return a version identifier from the configured executable. + + ``version_regex`` may contain a named ``version`` group; otherwise the + first capture group is returned. The executable output is inspected even + when the version command exits unsuccessfully, because some applications + print their banner before reporting a missing required input. + """ + executable = self.get_executable(default) + version_args = [] if version_args is None else list(version_args) + output, returncode = self._run_executable_version_command( + [executable, *version_args], timeout=timeout + ) + try: + return self.extract_executable_version(output, version_regex) + except ValueError as exc: + if returncode != 0: + raise RuntimeError( + f"Could not determine version for {self.name} executable " + f'"{executable}". Version command exited with return code ' + f"{returncode}." + ) from exc + raise + + @staticmethod + def extract_executable_version(output, version_regex=None): + """Extract a version identifier from executable output.""" + output = output.strip() + if not output: + raise ValueError("Executable version output was empty.") + if version_regex is None: + return next(line.strip() for line in output.splitlines() if line.strip()) + + match = re.search(version_regex, output, flags=re.MULTILINE) + if match is None: + raise ValueError( + f"Could not extract executable version using pattern {version_regex!r}." + ) + version = match.groupdict().get("version") + if version is not None: + return version.strip() + if match.groups(): + return next(group.strip() for group in match.groups() if group is not None) + return match.group(0).strip() + + def _run_executable_version_command(self, cmd_args, timeout=30): + """Run a version command and return its combined output and exit code. + + Process launch is delegated to :meth:`start_subprocess` so version checks + honor the application's configured environment and Docker image in the + same way as normal execution. + """ + process = self.start_subprocess( + [str(arg) for arg in cmd_args], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + if isinstance(process, Container): + result = process.wait(timeout=timeout) + output = process.logs(stdout=True, stderr=True) + return output.decode("utf-8", errors="replace"), result["StatusCode"] + + try: + stdout, stderr = process.communicate(timeout=timeout) + except subprocess.TimeoutExpired: + process.kill() + process.communicate() + raise + output = b"\n".join(stream for stream in (stdout, stderr) if stream) + return output.decode("utf-8", errors="replace"), process.returncode + def _set_procs(self): """Set processor information based on the `maxproc` and `np` inputs. If the available CPU count can be determined by `os.cpu_count()` then it will be used diff --git a/src/myna/core/components/component.py b/src/myna/core/components/component.py index d43a274a..755a6657 100644 --- a/src/myna/core/components/component.py +++ b/src/myna/core/components/component.py @@ -8,13 +8,17 @@ # """Base class for workflow components""" +import inspect +import importlib import os -import sys from pathlib import Path -from typing import Literal import subprocess +import sys +from contextlib import contextmanager +from typing import Literal import myna import myna.database +from myna.core.context import get_workflow_input_file, workflow_context, workflow_env from myna.core.workflow import load_input from myna.core.utils import nested_get, get_quoted_str import warnings @@ -43,57 +47,17 @@ def __init__(self): self.data = {} self.types = ["build"] self.workspace = None + self.input_file = None + self.step_index = None + self.last_step_name = None + self.last_step_class = None def run_component(self): - """Runs the configure.py, execute.py, and postprocess.py - for selected component class & application combination""" + """Run configure, execute, and postprocess stages for this component.""" - # Run component configure.py script - configure_path = os.path.join( - os.environ["MYNA_APP_PATH"], - self.component_application, - self.component_class, - "configure.py", - ) - if os.path.exists(configure_path): - cmd = [sys.executable, configure_path] - cmd.extend(self.get_step_args_list("configure")) - cmd = self.cmd_preformat(cmd) - print(f"myna run: {cmd=}") - with subprocess.Popen(cmd) as p: - p.wait() - - # Run component execute.py script - has_executed = False - execute_path = os.path.join( - os.environ["MYNA_APP_PATH"], - self.component_application, - self.component_class, - "execute.py", - ) - if os.path.exists(execute_path): - cmd = [sys.executable, execute_path] - cmd.extend(self.get_step_args_list("execute")) - cmd = self.cmd_preformat(cmd) - print(f"myna run: {cmd=}") - with subprocess.Popen(cmd) as p: - p.wait() - has_executed = True - - # Run component postprocess.py script - postprocess_path = os.path.join( - os.environ["MYNA_APP_PATH"], - self.component_application, - self.component_class, - "postprocess.py", - ) - if os.path.exists(postprocess_path): - cmd = [sys.executable, postprocess_path] - cmd.extend(self.get_step_args_list("postprocess")) - cmd = self.cmd_preformat(cmd) - print(f"myna run: {cmd=}") - with subprocess.Popen(cmd) as p: - p.wait() + self._run_stage("configure") + has_executed = self._run_stage("execute") + self._run_stage("postprocess") # Check output of component output_files, _, valid = self.get_output_files() @@ -113,6 +77,115 @@ def run_component(self): ) [print("\t" + x) for x in output_files] + def _run_stage(self, operation: Literal["configure", "execute", "postprocess"]): + """Import and run an application stage module in the current process.""" + + module_name = ".".join( + [ + "myna", + "application", + self.component_application, + self.component_class, + operation, + ] + ) + script_name = os.path.join( + os.environ["MYNA_APP_PATH"], + self.component_application, + self.component_class, + f"{operation}.py", + ) + if not os.path.exists(script_name): + return False + + stage_args = self.cmd_preformat(self.get_step_args_list(operation)) + cmd = [sys.executable, script_name] + cmd.extend(stage_args) + print(f"myna run: {cmd=}") + + with workflow_context( + input_file=self.input_file, + step_name=self.name, + step_class=self.component_class, + step_index=self.step_index, + last_step_name=self.last_step_name, + last_step_class=self.last_step_class, + ) as context: + with workflow_env(context, operation="run"): + if self._should_run_stage_in_process(script_name): + self._run_stage_in_process( + module_name, operation, script_name, stage_args + ) + else: + self._run_stage_subprocess(cmd) + return True + + def _run_stage_in_process(self, module_name, operation, script_name, stage_args): + module = importlib.import_module(module_name) + try: + module_file = inspect.getsourcefile(module) + except TypeError: + module_file = None + module_file = module_file or getattr(module, "__file__", None) + if module_file is not None and not self._same_stage_path( + module_file, script_name + ): + cmd = [sys.executable, script_name] + cmd.extend(stage_args) + self._run_stage_subprocess(cmd) + return + + stage_func = getattr(module, operation, None) + if stage_func is None: + stage_func = getattr(module, "main", None) + if stage_func is None: + raise AttributeError( + f"Application stage module '{module_name}' does not define " + f"'{operation}()' or 'main()'." + ) + + with self._stage_process_state(script_name, stage_args): + try: + stage_func() + except SystemExit as exc: + if exc.code not in (None, 0): + raise + + def _run_stage_subprocess(self, cmd): + subprocess.run(cmd, check=True) + + def _should_run_stage_in_process(self, script_name): + installed_script = os.path.join( + os.environ["MYNA_INSTALL_PATH"], + "application", + self.component_application, + self.component_class, + f"{Path(script_name).stem}.py", + ) + return self._same_stage_path(script_name, installed_script) + + def _same_stage_path(self, left, right): + return Path(left).resolve(strict=False) == Path(right).resolve(strict=False) + + @contextmanager + def _stage_process_state(self, script_name, args): + previous_argv = sys.argv + previous_cwd = os.getcwd() + sys.argv = [script_name] + sys.argv.extend(args) + try: + yield + finally: + sys.argv = previous_argv + os.chdir(previous_cwd) + + @contextmanager + def _stage_argv(self, script_name, args): + """Deprecated helper kept for compatibility with older tests/imports.""" + + with self._stage_process_state(script_name, args): + yield + def cmd_preformat(self, raw_cmd): """Replace placeholder names in command arguments and adds executable argument if a custom executable path is specified. @@ -202,7 +275,11 @@ def get_files_from_template(self, template, abspath=True): files = [] # Get build name - input_dir = os.path.abspath(os.path.dirname(os.environ["MYNA_INPUT"])) + input_file = self.input_file or get_workflow_input_file() + if input_file is None: + input_dir = os.getcwd() + else: + input_dir = os.path.abspath(os.path.dirname(input_file)) build = nested_get(self.data, ["build", "name"], "myna_output") filled_template = template.replace("{{build}}", build) @@ -460,9 +537,21 @@ def sync_output_files(self): print(f"- step {self.name}: No valid output files found.") else: # Use the database sync functionality - synced_files = datatype.sync( - self.component_application, self.types, self.output_requirement, files - ) + with workflow_context( + input_file=self.input_file, + step_name=self.name, + step_class=self.component_class, + step_index=self.step_index, + last_step_name=self.last_step_name, + last_step_class=self.last_step_class, + ) as context: + with workflow_env(context, operation="sync"): + synced_files = datatype.sync( + self.component_application, + self.types, + self.output_requirement, + files, + ) return synced_files diff --git a/src/myna/core/context.py b/src/myna/core/context.py new file mode 100644 index 00000000..ba3f26be --- /dev/null +++ b/src/myna/core/context.py @@ -0,0 +1,222 @@ +# +# Copyright (c) Oak Ridge National Laboratory. +# +# This file is part of Myna. For details, see the top-level license +# at https://github.com/ORNL-MDF/Myna/LICENSE.md. +# +# License: 3-clause BSD, see https://opensource.org/licenses/BSD-3-Clause. +# +"""Workflow context shared between Myna workflow orchestration and apps.""" + +from __future__ import annotations + +from contextlib import contextmanager +from contextvars import ContextVar +from dataclasses import dataclass +import os +import warnings + + +WORKFLOW_ENV_INPUT_FILE = "MYNA_INPUT" +WORKFLOW_ENV_RUN_INPUT = "MYNA_RUN_INPUT" +WORKFLOW_ENV_SYNC_INPUT = "MYNA_SYNC_INPUT" +WORKFLOW_ENV_STEP_NAME = "MYNA_STEP_NAME" +WORKFLOW_ENV_STEP_CLASS = "MYNA_STEP_CLASS" +WORKFLOW_ENV_STEP_INDEX = "MYNA_STEP_INDEX" +WORKFLOW_ENV_LAST_STEP_NAME = "MYNA_LAST_STEP_NAME" +WORKFLOW_ENV_LAST_STEP_CLASS = "MYNA_LAST_STEP_CLASS" + + +@dataclass(frozen=True) +class WorkflowContext: + """Explicit workflow state for an active Myna operation.""" + + input_file: str | None = None + step_name: str | None = None + step_class: str | None = None + step_index: int | None = None + last_step_name: str | None = None + last_step_class: str | None = None + + +_CURRENT_WORKFLOW_CONTEXT: ContextVar[WorkflowContext | None] = ContextVar( + "myna_current_workflow_context", default=None +) +_LEGACY_ENV_FALLBACK_WARNED = False + + +def current_workflow_context() -> WorkflowContext | None: + """Return the active workflow context, if one has been set.""" + + return _CURRENT_WORKFLOW_CONTEXT.get() + + +def get_workflow_context() -> WorkflowContext | None: + """Return explicit workflow context or derive it from legacy env vars.""" + + context = current_workflow_context() + if context is not None: + return context + + if not _has_legacy_workflow_env(): + return None + + _warn_legacy_env_fallback() + + env_context = WorkflowContext( + input_file=os.environ.get(WORKFLOW_ENV_INPUT_FILE), + step_name=os.environ.get(WORKFLOW_ENV_STEP_NAME), + step_class=os.environ.get(WORKFLOW_ENV_STEP_CLASS), + step_index=_parse_optional_int(os.environ.get(WORKFLOW_ENV_STEP_INDEX)), + last_step_name=os.environ.get(WORKFLOW_ENV_LAST_STEP_NAME), + last_step_class=os.environ.get(WORKFLOW_ENV_LAST_STEP_CLASS), + ) + return env_context + + +@contextmanager +def workflow_context( + *, + input_file: str | None = None, + step_name: str | None = None, + step_class: str | None = None, + step_index: int | None = None, + last_step_name: str | None = None, + last_step_class: str | None = None, +): + """Temporarily set explicit workflow state for in-process app execution.""" + + parent = current_workflow_context() + context = WorkflowContext( + input_file=( + input_file + if input_file is not None + else _parent_value(parent, "input_file") + ), + step_name=( + step_name if step_name is not None else _parent_value(parent, "step_name") + ), + step_class=( + step_class + if step_class is not None + else _parent_value(parent, "step_class") + ), + step_index=( + step_index + if step_index is not None + else _parent_value(parent, "step_index") + ), + last_step_name=last_step_name + if last_step_name is not None + else _parent_value(parent, "last_step_name"), + last_step_class=last_step_class + if last_step_class is not None + else _parent_value(parent, "last_step_class"), + ) + token = _CURRENT_WORKFLOW_CONTEXT.set(context) + try: + yield context + finally: + _CURRENT_WORKFLOW_CONTEXT.reset(token) + + +def get_workflow_input_file(default: str | None = None) -> str | None: + """Return the active workflow input file, falling back to legacy env state.""" + + context = get_workflow_context() + if context is not None and context.input_file is not None: + return context.input_file + return default + + +@contextmanager +def workflow_env( + context: WorkflowContext | None = None, *, operation: str | None = None +): + """Temporarily expose workflow state through legacy environment variables.""" + + if context is None: + context = get_workflow_context() + + updates = { + WORKFLOW_ENV_INPUT_FILE: None if context is None else context.input_file, + WORKFLOW_ENV_STEP_NAME: None if context is None else context.step_name, + WORKFLOW_ENV_STEP_CLASS: None if context is None else context.step_class, + WORKFLOW_ENV_STEP_INDEX: ( + None + if context is None or context.step_index is None + else str(context.step_index) + ), + WORKFLOW_ENV_LAST_STEP_NAME: None + if context is None + else context.last_step_name, + WORKFLOW_ENV_LAST_STEP_CLASS: ( + None if context is None else context.last_step_class + ), + } + if operation == "run": + updates[WORKFLOW_ENV_RUN_INPUT] = ( + None if context is None else context.input_file + ) + elif operation == "sync": + updates[WORKFLOW_ENV_SYNC_INPUT] = ( + None if context is None else context.input_file + ) + + previous_values = {key: os.environ.get(key) for key in updates} + previous_presence = {key: key in os.environ for key in updates} + try: + for key, value in updates.items(): + if value is None: + os.environ.pop(key, None) + else: + os.environ[key] = value + yield context + finally: + for key in updates: + if previous_presence[key]: + os.environ[key] = previous_values[key] + else: + os.environ.pop(key, None) + + +def _parent_value(parent: WorkflowContext | None, field: str): + if parent is None: + return None + return getattr(parent, field) + + +def _has_legacy_workflow_env() -> bool: + keys = [ + WORKFLOW_ENV_INPUT_FILE, + WORKFLOW_ENV_RUN_INPUT, + WORKFLOW_ENV_SYNC_INPUT, + WORKFLOW_ENV_STEP_NAME, + WORKFLOW_ENV_STEP_CLASS, + WORKFLOW_ENV_STEP_INDEX, + WORKFLOW_ENV_LAST_STEP_NAME, + WORKFLOW_ENV_LAST_STEP_CLASS, + ] + return any(key in os.environ for key in keys) + + +def _warn_legacy_env_fallback() -> None: + global _LEGACY_ENV_FALLBACK_WARNED + + if _LEGACY_ENV_FALLBACK_WARNED: + return + + warnings.warn( + "Workflow state derived from legacy MYNA_* environment variables is " + "deprecated and will be removed in Myna 2.0. Use explicit workflow " + "context or MynaApp workflow attributes instead.", + DeprecationWarning, + stacklevel=3, + ) + _LEGACY_ENV_FALLBACK_WARNED = True + + +def _parse_optional_int(value: str | None) -> int | None: + if value is None: + return None + return int(value) diff --git a/src/myna/core/db/database.py b/src/myna/core/db/database.py index d29582b4..34a84874 100644 --- a/src/myna/core/db/database.py +++ b/src/myna/core/db/database.py @@ -11,6 +11,7 @@ import os from datetime import datetime import yaml +from myna.core.context import get_workflow_input_file from myna.core.workflow import load_input from myna.core.utils import strf_datetime @@ -55,7 +56,10 @@ def write_segment_sync_metadata( simulation_file_path (str): Path to the simulation file being synced segment_key (str): Key identifying the segment in the database""" step_name = os.path.basename(os.path.dirname(simulation_file_path)) - step_list = load_input(os.environ["MYNA_INPUT"])["steps"] + input_file = get_workflow_input_file() + if input_file is None: + raise ValueError("Cannot write sync metadata without a Myna input file.") + step_list = load_input(input_file)["steps"] step_dict = step_list[ [list(step.keys())[0] for step in step_list].index(step_name) ] diff --git a/src/myna/core/metadata/file.py b/src/myna/core/metadata/file.py index 37c17e0f..cc4e2cad 100644 --- a/src/myna/core/metadata/file.py +++ b/src/myna/core/metadata/file.py @@ -10,6 +10,7 @@ import shutil import os +from myna.core.context import get_workflow_input_file class BuildFile: @@ -42,9 +43,11 @@ def copy_file(self, destination=None, overwrite=True): def set_local_resource_dir(self): """Get the local resource directory and make if it doesn't exist""" - input_dir = os.path.abspath(os.path.dirname(os.environ["MYNA_INPUT"])) - if input_dir is None: + input_file = get_workflow_input_file() + if input_file is None: input_dir = "." + else: + input_dir = os.path.abspath(os.path.dirname(input_file)) resource_dir = os.path.abspath(os.path.join(input_dir, "myna_resources")) os.makedirs(resource_dir, exist_ok=True) self.resource_dir = resource_dir diff --git a/src/myna/core/workflow/__init__.py b/src/myna/core/workflow/__init__.py index 2dd553d2..c771014c 100644 --- a/src/myna/core/workflow/__init__.py +++ b/src/myna/core/workflow/__init__.py @@ -49,6 +49,13 @@ load_input, write_input, ) +from myna.core.context import ( + WorkflowContext, + current_workflow_context, + get_workflow_context, + get_workflow_input_file, + workflow_context, +) from .status import format_class_string_to_list, write_codebase_status_to_file __all__ = [ @@ -60,6 +67,11 @@ "get_validated_input_filetype", "load_input", "write_input", + "WorkflowContext", + "current_workflow_context", + "get_workflow_context", + "get_workflow_input_file", + "workflow_context", "format_class_string_to_list", "write_codebase_status_to_file", ] diff --git a/src/myna/core/workflow/run.py b/src/myna/core/workflow/run.py index c0a51e52..f0db58ec 100644 --- a/src/myna/core/workflow/run.py +++ b/src/myna/core/workflow/run.py @@ -45,17 +45,14 @@ def run(input_file, step=None): step: name of step to run, runs all if None""" steps_to_run = myna.core.utils.str_to_list(step) - - # Set environmental variable for input file location - os.environ["MYNA_INPUT"] = os.path.abspath(input_file) - os.environ["MYNA_RUN_INPUT"] = os.path.abspath( - input_file - ) # MYNA_RUN_INPUT will be deprecated in future versions + input_file = os.path.abspath(input_file) # Load the initial input file to get the steps initial_settings = load_input(input_file) # Run through each step + last_step_name = "" + last_step_class = "" for index, step in enumerate(initial_settings["steps"]): # Load the input file at each step in case one the previous step has updated the inputs settings = load_input(input_file) @@ -67,17 +64,10 @@ def run(input_file, step=None): step_obj.name = step_name step_obj.component_class = component_class_name step_obj.component_application = step[step_name]["application"] - - # Set environmental variable for the step name - if index != 0: - os.environ["MYNA_LAST_STEP_NAME"] = os.environ["MYNA_STEP_NAME"] - os.environ["MYNA_LAST_STEP_CLASS"] = os.environ["MYNA_STEP_CLASS"] - else: - os.environ["MYNA_LAST_STEP_NAME"] = "" - os.environ["MYNA_LAST_STEP_CLASS"] = "" - os.environ["MYNA_STEP_NAME"] = step_name - os.environ["MYNA_STEP_CLASS"] = component_class_name - os.environ["MYNA_STEP_INDEX"] = str(index) + step_obj.input_file = input_file + step_obj.step_index = index + step_obj.last_step_name = last_step_name + step_obj.last_step_class = last_step_class # Apply the settings and execute the component, as needed run_step = True @@ -93,3 +83,6 @@ def run(input_file, step=None): step[step_name], settings.get("data"), settings.get("myna") ) step_obj.run_component() + + last_step_name = step_name + last_step_class = component_class_name diff --git a/src/myna/core/workflow/sync.py b/src/myna/core/workflow/sync.py index 89993c68..eee856cb 100644 --- a/src/myna/core/workflow/sync.py +++ b/src/myna/core/workflow/sync.py @@ -46,17 +46,14 @@ def sync(input_file, step=None): step: name of step to sync, syncs all if None""" steps_to_sync = myna.core.utils.str_to_list(step) - - # Set environmental variable for input file location - os.environ["MYNA_INPUT"] = os.path.abspath(input_file) - os.environ["MYNA_SYNC_INPUT"] = os.path.abspath( - input_file - ) # MYNA_SYNC_INPUT will be deprecated in future versions + input_file = os.path.abspath(input_file) # Load the initial input file to get the steps initial_settings = load_input(input_file) # Run through each step + last_step_name = "" + last_step_class = "" for index, step in enumerate(initial_settings["steps"]): # Load the input file at each step in case one the previous step has updated the inputs settings = load_input(input_file) @@ -69,17 +66,10 @@ def sync(input_file, step=None): step_obj.name = step_name step_obj.component_class = component_class_name step_obj.component_application = component_app_name - - # Set environmental variable for the step name - if index != 0: - os.environ["MYNA_LAST_STEP_NAME"] = os.environ["MYNA_STEP_NAME"] - os.environ["MYNA_LAST_STEP_CLASS"] = os.environ["MYNA_STEP_CLASS"] - else: - os.environ["MYNA_LAST_STEP_NAME"] = "" - os.environ["MYNA_LAST_STEP_CLASS"] = "" - os.environ["MYNA_STEP_NAME"] = step_name - os.environ["MYNA_STEP_CLASS"] = component_class_name - os.environ["MYNA_STEP_INDEX"] = str(index) + step_obj.input_file = input_file + step_obj.step_index = index + step_obj.last_step_name = last_step_name + step_obj.last_step_class = last_step_class # Apply the settings and execute the component, as needed sync_step = True @@ -93,3 +83,6 @@ def sync(input_file, step=None): if sync_step: step_obj.apply_settings(step[step_name], settings["data"], settings["myna"]) step_obj.sync_output_files() + + last_step_name = step_name + last_step_class = component_class_name diff --git a/src/myna/database/nist_ambench_2022.py b/src/myna/database/nist_ambench_2022.py index eb44acb5..dccc18b4 100644 --- a/src/myna/database/nist_ambench_2022.py +++ b/src/myna/database/nist_ambench_2022.py @@ -11,6 +11,7 @@ from myna.core.db import Database from myna.core import metadata from myna.core.utils import downsample_to_image, nested_get +from myna.core.context import get_workflow_input_file from myna.core.workflow import load_input import matplotlib.pyplot as plt import numpy as np @@ -302,7 +303,10 @@ def sync(self, component_type, step_types, output_class, files): component_name = os.path.basename(os.path.dirname(files[0])) unique_regions = sorted(set(regions)) unique_parts = sorted(set(parts)) - settings = load_input(os.environ["MYNA_INPUT"]) + input_file = get_workflow_input_file() + if input_file is None: + raise ValueError("Region sync requires a Myna input file.") + settings = load_input(input_file) for region in unique_regions: for part in unique_parts: part_dict = nested_get(settings, ["data", "build", "parts"]).get( diff --git a/src/myna/database/pelican.py b/src/myna/database/pelican.py index 8f1cd8d0..886ecc1c 100644 --- a/src/myna/database/pelican.py +++ b/src/myna/database/pelican.py @@ -18,6 +18,7 @@ from polars.datatypes import Float64 from myna.core import metadata from myna.core.db import Database +from myna.core.context import get_workflow_input_file from myna.core.utils import nested_get, nested_set @@ -104,7 +105,9 @@ def get_segment_details(self, part, layer): layer: (str) "layer" corresponding to the time_segment for the part in the Myna input file """ - input_file = os.environ["MYNA_INPUT"] + input_file = get_workflow_input_file() + if input_file is None: + raise ValueError("Pelican segment details require a Myna input file.") with open(input_file, "r", encoding="utf-8") as f: input_dict = yaml.safe_load(f) segment = nested_get( diff --git a/src/myna/database/peregrine.py b/src/myna/database/peregrine.py index c1b992dc..dc6b436d 100644 --- a/src/myna/database/peregrine.py +++ b/src/myna/database/peregrine.py @@ -11,6 +11,7 @@ from myna.core.db import Database from myna.core import metadata from myna.core.utils import downsample_to_image, get_synonymous_key, nested_get +from myna.core.context import get_workflow_input_file from myna.core.workflow import load_input import matplotlib.pyplot as plt import os @@ -261,7 +262,10 @@ def sync(self, component_type, step_types, output_class, files): component_name = os.path.basename(os.path.dirname(files[0])) unique_regions = sorted(set(regions)) unique_parts = sorted(set(parts)) - settings = load_input(os.environ["MYNA_INPUT"]) + input_file = get_workflow_input_file() + if input_file is None: + raise ValueError("Region sync requires a Myna input file.") + settings = load_input(input_file) for region in unique_regions: for part in unique_parts: part_dict = nested_get(settings, ["data", "build", "parts"]).get( diff --git a/tests/test_app_argument_parsing.py b/tests/test_app_argument_parsing.py index 4825120d..0849a9a7 100644 --- a/tests/test_app_argument_parsing.py +++ b/tests/test_app_argument_parsing.py @@ -7,6 +7,7 @@ # License: 3-clause BSD, see https://opensource.org/licenses/BSD-3-Clause. # import sys +import stat import pytest @@ -23,6 +24,11 @@ def _count_option_actions(parser, option_string): return sum(option_string in action.option_strings for action in parser._actions) +def _write_shell_executable(path, body): + path.write_text(f"#!/bin/sh\n{body}", encoding="utf-8") + path.chmod(path.stat().st_mode | stat.S_IXUSR) + + def test_register_argument_skips_duplicate_option_registration(monkeypatch): monkeypatch.setattr(sys, "argv", ["test"]) app = MynaApp() @@ -242,6 +248,21 @@ def test_exaca_stage_parsers_set_default_executable(monkeypatch, stage_call): assert app.args.exec == "ExaCA" +def test_exaca_get_executable_version_reads_banner_before_missing_input_error( + monkeypatch, tmp_path +): + executable = tmp_path / "ExaCA" + _write_shell_executable( + executable, + 'printf "%s\\n" "ExaCA version: 2.1.0-dev"\n' + 'printf "%s\\n" "Error: Must provide path to input file" >&2\n' + "exit 1\n", + ) + monkeypatch.setattr(sys, "argv", ["test", "--exec", str(executable)]) + + assert ExaCA().get_executable_version() == "2.1.0-dev" + + @pytest.mark.parametrize( "stage_calls", [ diff --git a/tests/test_component_output_paths.py b/tests/test_component_output_paths.py index 9eb3f056..663cbb40 100644 --- a/tests/test_component_output_paths.py +++ b/tests/test_component_output_paths.py @@ -33,6 +33,7 @@ def component(tmp_path, monkeypatch): component = Component() component.name = "demo-step" + component.input_file = str(input_file) component.data = {"build": {"name": "build-1"}} component.output_requirement = DummyOutputFile return component diff --git a/tests/test_configure.py b/tests/test_configure.py index 0c97f79a..db97a339 100644 --- a/tests/test_configure.py +++ b/tests/test_configure.py @@ -10,8 +10,10 @@ import sys import os import shutil +import warnings import myna +import myna.core.context as context_module from .example_paths import CASES_DIR @@ -42,6 +44,7 @@ def run_configure(example): def test_configure(): + context_module._LEGACY_ENV_FALLBACK_WARNED = False examples = [ "cluster_solidification", "melt_pool_geometry_part", @@ -62,5 +65,12 @@ def test_configure(): "vtk_to_exodus_region", ] - for example in examples: - run_configure(example) + with warnings.catch_warnings(record=True) as record: + warnings.simplefilter("always") + for example in examples: + run_configure(example) + + unexpected = [ + warning for warning in record if warning.category is not DeprecationWarning + ] + assert unexpected == [] diff --git a/tests/test_database.py b/tests/test_database.py index 6576d159..269e1779 100644 --- a/tests/test_database.py +++ b/tests/test_database.py @@ -8,9 +8,11 @@ # import os import shutil +import warnings import myna import myna.database +import myna.core.context as context_module from .example_paths import CASES_DIR, DATABASES_DIR @@ -57,6 +59,7 @@ def test_database_Peregrine(): def test_database_pelican(tmp_path): """Test if the Pelican database class is working for the expected metadata types""" + context_module._LEGACY_ENV_FALLBACK_WARNED = False database_copy = tmp_path / "Pelican" shutil.copytree(DATABASES_DIR / "Pelican", database_copy) @@ -75,4 +78,10 @@ def test_database_pelican(tmp_path): os.environ["MYNA_INPUT"] = os.fspath( CASES_DIR / "solidification_part_pelican" / "input.yaml" ) - assert db.load(metadata_type, part="P0", layer=0) is not None + with warnings.catch_warnings(record=True) as record: + warnings.simplefilter("always") + assert db.load(metadata_type, part="P0", layer=0) is not None + unexpected = [ + warning for warning in record if warning.category is not DeprecationWarning + ] + assert unexpected == [] diff --git a/tests/test_docs_harness.py b/tests/test_docs_harness.py new file mode 100644 index 00000000..ae8bebcf --- /dev/null +++ b/tests/test_docs_harness.py @@ -0,0 +1,76 @@ +# +# Copyright (c) Oak Ridge National Laboratory. +# +# This file is part of Myna. For details, see the top-level license +# at https://github.com/ORNL-MDF/Myna/LICENSE.md. +# +# License: 3-clause BSD, see https://opensource.org/licenses/BSD-3-Clause. +# +from __future__ import annotations + +import importlib.util +from pathlib import Path + +import pytest + + +REPO_ROOT = Path(__file__).resolve().parents[1] +MODULE_PATH = REPO_ROOT / "scripts" / "check_docs_harness.py" +SPEC = importlib.util.spec_from_file_location("test_docs_harness_module", MODULE_PATH) +DOCS_HARNESS = importlib.util.module_from_spec(SPEC) +assert SPEC.loader is not None +SPEC.loader.exec_module(DOCS_HARNESS) + + +def test_get_changed_files_uses_merge_base_diff_and_untracked(monkeypatch): + def fake_run_git_command(args): + command_map = { + ("merge-base", "HEAD", "origin/main"): ["abc123"], + ("diff", "--name-only", "abc123..HEAD"): ["src/myna/core/context.py"], + ("ls-files", "--others", "--exclude-standard"): ["scratch.txt"], + } + return command_map.get(tuple(args), []) + + monkeypatch.setattr(DOCS_HARNESS, "run_git_command", fake_run_git_command) + + assert DOCS_HARNESS.get_changed_files() == { + "src/myna/core/context.py", + "scratch.txt", + } + + +def test_get_changed_files_falls_back_to_dirty_worktree(monkeypatch): + def fake_run_git_command(args): + command_map = { + ("merge-base", "HEAD", "origin/main"): [], + ("merge-base", "HEAD", "main"): [], + ("diff", "--name-only", "HEAD"): ["scripts/check_docs_harness.py"], + ("ls-files", "--others", "--exclude-standard"): [], + } + return command_map.get(tuple(args), []) + + monkeypatch.setattr(DOCS_HARNESS, "run_git_command", fake_run_git_command) + + assert DOCS_HARNESS.get_changed_files() == {"scripts/check_docs_harness.py"} + + +def test_architecture_docs_required_for_sensitive_branch_changes(monkeypatch, capsys): + monkeypatch.setattr( + DOCS_HARNESS, + "get_changed_files", + lambda: {"src/myna/core/context.py"}, + ) + + with pytest.raises(SystemExit): + DOCS_HARNESS.check_architecture_docs_updated_for_sensitive_changes() + assert "architecture-sensitive files changed" in capsys.readouterr().err + + +def test_architecture_docs_check_passes_when_docs_are_updated(monkeypatch): + monkeypatch.setattr( + DOCS_HARNESS, + "get_changed_files", + lambda: {"src/myna/core/context.py", "ARCHITECTURE.md"}, + ) + + DOCS_HARNESS.check_architecture_docs_updated_for_sensitive_changes() diff --git a/tests/test_exaca_app_behavior.py b/tests/test_exaca_app_behavior.py index 80c462cb..c4f37430 100644 --- a/tests/test_exaca_app_behavior.py +++ b/tests/test_exaca_app_behavior.py @@ -8,15 +8,19 @@ # import json import os +from pathlib import Path from types import SimpleNamespace import pandas as pd +import numpy as np +import pytest from myna.application.exaca.microstructure_region import ExaCAMicrostructureRegion from myna.application.exaca.microstructure_region_slice import ( ExaCAMicrostructureRegionSlice, ) import myna.application.exaca.microstructure_region_slice.app as slice_app_module +import myna.core.context as context_module def _write_settings(tmp_path, step_output_paths, last_step_output_paths): @@ -69,7 +73,10 @@ def _create_exaca_install(tmp_path): install_dir = tmp_path / "install" exe_path = install_dir / "bin" / "ExaCA" exe_path.parent.mkdir(parents=True) - exe_path.write_text("#!/bin/sh\n", encoding="utf-8") + exe_path.write_text( + '#!/bin/sh\nprintf "%s\\n" "ExaCA version: 2.0.0"\nexit 1\n', + encoding="utf-8", + ) exe_path.chmod(0o755) orientation_file = install_dir / "share" / "ExaCA" / "GrainOrientationVectors.csv" @@ -154,12 +161,15 @@ def _build_app_args(template_dir, exe_path): np=4, overwrite=False, batch=False, + docker_image=None, + env=None, ) def test_region_and_slice_configure_match_outputs_to_region_inputs( monkeypatch, tmp_path ): + monkeypatch.setattr(context_module, "_LEGACY_ENV_FALLBACK_WARNED", False) step_output_paths = [ str(tmp_path / "build-1" / "part-a" / "region-1" / "micro.vtk"), str(tmp_path / "build-1" / "part-a" / "region-2" / "micro.vtk"), @@ -177,7 +187,9 @@ def capture_setup(case_dir, solid_files, layer_thickness): captured.append((case_dir, solid_files, layer_thickness)) for app_cls in (ExaCAMicrostructureRegion, ExaCAMicrostructureRegionSlice): - app = app_cls() + monkeypatch.setattr(context_module, "_LEGACY_ENV_FALLBACK_WARNED", False) + with pytest.warns(DeprecationWarning, match="Myna 2.0"): + app = app_cls() captured.clear() monkeypatch.setattr(app, "parse_configure_arguments", lambda: None) monkeypatch.setattr(app, "setup_case", capture_setup) @@ -199,6 +211,7 @@ def capture_setup(case_dir, solid_files, layer_thickness): def test_region_setup_case_populates_inputs_run_script_and_analysis( monkeypatch, tmp_path ): + monkeypatch.setattr(context_module, "_LEGACY_ENV_FALLBACK_WARNED", False) _configure_minimal_app_env(monkeypatch, tmp_path) material_file = _create_material_root(monkeypatch, tmp_path) exe_path, orientation_file = _create_exaca_install(tmp_path) @@ -210,7 +223,8 @@ def test_region_setup_case_populates_inputs_run_script_and_analysis( case_dir = tmp_path / "case" _write_case_metadata(case_dir) - app = ExaCAMicrostructureRegion() + with pytest.warns(DeprecationWarning, match="Myna 2.0"): + app = ExaCAMicrostructureRegion() app.args = _build_app_args(template_dir, exe_path) app.setup_case(str(case_dir), [str(solid_file)], 50.0) @@ -228,9 +242,53 @@ def test_region_setup_case_populates_inputs_run_script_and_analysis( assert analysis_settings["Regions"]["XY"]["zBounds"] == [16, 16] assert analysis_settings["Regions"]["XZ"]["yBounds"] == [4, 4] assert analysis_settings["Regions"]["YZ"]["xBounds"] == [2, 2] + assert input_settings["TemperatureData"]["TemperatureFiles"] == [str(solid_file)] + + +def test_region_setup_case_accepts_unit_bearing_temperature_headers( + monkeypatch, tmp_path +): + monkeypatch.setattr(context_module, "_LEGACY_ENV_FALLBACK_WARNED", False) + _configure_minimal_app_env(monkeypatch, tmp_path) + _create_material_root(monkeypatch, tmp_path) + exe_path, _ = _create_exaca_install(tmp_path) + solid_file = tmp_path / "solid.csv" + solid_file.write_text( + "x (m),y (m),z (m),tm (s),ts (s),cr (k/s)\n" + "0.0,0.0,0.0,0.0,0.0,0.0\n" + "x,y,z,tm,ts,cr\n" + "0.00001,0.00002,0.00003,0.0,0.0,0.0\n", + encoding="utf-8", + ) + + template_dir = tmp_path / "template" + _write_template(template_dir, include_analysis=True) + case_dir = tmp_path / "case" + _write_case_metadata(case_dir) + + with pytest.warns(DeprecationWarning, match="Myna 2.0"): + app = ExaCAMicrostructureRegion() + app.args = _build_app_args(template_dir, exe_path) + app.setup_case(str(case_dir), [str(solid_file)], 50.0) + + input_settings = json.loads((case_dir / "inputs.json").read_text(encoding="utf-8")) + analysis_settings = json.loads( + (case_dir / "analysis.json").read_text(encoding="utf-8") + ) + sanitized_file = input_settings["TemperatureData"]["TemperatureFiles"][0] + sanitized_headers = ( + Path(sanitized_file).read_text(encoding="utf-8").splitlines()[0].split(",") + ) + assert Path(sanitized_file).is_file() + assert Path(sanitized_file).parent == case_dir + assert sanitized_headers == ["x", "y", "z", "tm", "ts", "cr"] + assert analysis_settings["Regions"]["XY"]["zBounds"] == [16, 16] + assert analysis_settings["Regions"]["XZ"]["yBounds"] == [4, 4] + assert analysis_settings["Regions"]["YZ"]["xBounds"] == [2, 2] def test_slice_setup_case_uses_shared_setup_without_analysis(monkeypatch, tmp_path): + monkeypatch.setattr(context_module, "_LEGACY_ENV_FALLBACK_WARNED", False) _configure_minimal_app_env(monkeypatch, tmp_path) material_file = _create_material_root(monkeypatch, tmp_path) exe_path, orientation_file = _create_exaca_install(tmp_path) @@ -242,7 +300,8 @@ def test_slice_setup_case_uses_shared_setup_without_analysis(monkeypatch, tmp_pa case_dir = tmp_path / "case" _write_case_metadata(case_dir) - app = ExaCAMicrostructureRegionSlice() + with pytest.warns(DeprecationWarning, match="Myna 2.0"): + app = ExaCAMicrostructureRegionSlice() app.args = _build_app_args(template_dir, exe_path) app.setup_case(str(case_dir), [str(solid_file)], 50.0) @@ -252,14 +311,65 @@ def test_slice_setup_case_uses_shared_setup_without_analysis(monkeypatch, tmp_pa assert not (case_dir / "analysis.json").exists() +@pytest.mark.parametrize( + ("app_cls", "include_analysis"), + [ + (ExaCAMicrostructureRegion, True), + (ExaCAMicrostructureRegionSlice, False), + ], +) +def test_setup_case_converts_deprecated_substrate_settings_for_exaca_21( + monkeypatch, tmp_path, app_cls, include_analysis +): + _configure_minimal_app_env(monkeypatch, tmp_path) + _create_material_root(monkeypatch, tmp_path) + exe_path, _ = _create_exaca_install(tmp_path) + solid_file = tmp_path / "solid.csv" + solid_file.write_text("x,y\n0.0,0.0\n", encoding="utf-8") + + template_dir = tmp_path / "template" + _write_template(template_dir, include_analysis=include_analysis) + template_input = template_dir / "inputs.json" + template_settings = json.loads(template_input.read_text(encoding="utf-8")) + template_settings["Substrate"]["PowderDensity"] = 7.8 + template_input.write_text(json.dumps(template_settings), encoding="utf-8") + case_dir = tmp_path / "case" + _write_case_metadata(case_dir) + + app = app_cls() + app.args = _build_app_args(template_dir, exe_path) + monkeypatch.setattr(app, "get_executable_version", lambda: "2.1.0-dev") + app.setup_case(str(case_dir), [str(solid_file)], 50.0) + + substrate = json.loads((case_dir / "inputs.json").read_text(encoding="utf-8"))[ + "Substrate" + ] + assert substrate == { + "MeanBaseplateGrainSize": 12.3, + "MeanPowderGrainSize": 1 / np.power(7.8, 1 / 3), + } + + +def test_convert_case_input_preserves_pre_21_substrate_settings(monkeypatch, tmp_path): + app = ExaCAMicrostructureRegion() + monkeypatch.setattr(app, "get_executable_version", lambda: "2.0.9") + input_settings = {"Substrate": {"MeanSize": 12.3, "PowderDensity": 7.8}} + + converted = app.convert_case_input_for_exaca_version(str(tmp_path), input_settings) + + assert converted == input_settings + + def test_region_execute_moves_exaca_vtk_to_workflow_output(monkeypatch, tmp_path): + monkeypatch.setattr(context_module, "_LEGACY_ENV_FALLBACK_WARNED", False) _configure_minimal_app_env(monkeypatch, tmp_path) case_dir = tmp_path / "case" case_dir.mkdir() workflow_output = tmp_path / "result.vtk" raw_output = case_dir / "exaca.vtk" - app = ExaCAMicrostructureRegion() + with pytest.warns(DeprecationWarning, match="Myna 2.0"): + app = ExaCAMicrostructureRegion() app.args = SimpleNamespace(overwrite=False, batch=False) monkeypatch.setattr(app, "parse_execute_arguments", lambda: None) monkeypatch.setattr(app, "get_step_output_paths", lambda: [str(workflow_output)]) @@ -285,6 +395,7 @@ def test_region_execute_moves_exaca_vtk_to_workflow_output(monkeypatch, tmp_path def test_slice_execute_writes_csv_statistics_from_raw_vtk(monkeypatch, tmp_path): + monkeypatch.setattr(context_module, "_LEGACY_ENV_FALLBACK_WARNED", False) _configure_minimal_app_env(monkeypatch, tmp_path) case_dir = tmp_path / "case" case_dir.mkdir() @@ -311,7 +422,8 @@ def GetOutput(self): } ) - app = ExaCAMicrostructureRegionSlice() + with pytest.warns(DeprecationWarning, match="Myna 2.0"): + app = ExaCAMicrostructureRegionSlice() app.args = SimpleNamespace(overwrite=False, batch=False) monkeypatch.setattr(app, "parse_execute_arguments", lambda: None) monkeypatch.setattr(app, "get_step_output_paths", lambda: [str(workflow_output)]) @@ -348,6 +460,7 @@ def GetOutput(self): def test_slice_postprocess_colors_raw_exaca_vtk(monkeypatch, tmp_path): + monkeypatch.setattr(context_module, "_LEGACY_ENV_FALLBACK_WARNED", False) _configure_minimal_app_env(monkeypatch, tmp_path) case_dir = tmp_path / "case" case_dir.mkdir() @@ -364,7 +477,8 @@ def test_slice_postprocess_colors_raw_exaca_vtk(monkeypatch, tmp_path): ) calls = {} - app = ExaCAMicrostructureRegionSlice() + with pytest.warns(DeprecationWarning, match="Myna 2.0"): + app = ExaCAMicrostructureRegionSlice() monkeypatch.setattr(app, "parse_postprocess_arguments", lambda: None) monkeypatch.setattr(app, "get_step_output_paths", lambda: [str(workflow_output)]) monkeypatch.setattr( diff --git a/tests/test_examples.py b/tests/test_examples.py index 1cc93857..418fa85e 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -22,7 +22,6 @@ # # License: 3-clause BSD, see https://opensource.org/licenses/BSD-3-Clause. # -import os import shutil import pytest from myna.core.utils import working_directory @@ -44,8 +43,7 @@ def run_example_test(example_name): # The `tmp` directories need to be in `examples/cases/` to find the database files. example_dir = get_example_dir(example_name) tmp_dir = example_dir.parent / f"{example_dir.name}_tmp" - os.makedirs(tmp_dir) - shutil.copyfile(example_dir / "input.yaml", tmp_dir / "input.yaml") + shutil.copytree(example_dir, tmp_dir) # Run and clean the example try: diff --git a/tests/test_thesis_app_behavior.py b/tests/test_thesis_app_behavior.py index 58d4d6a7..aebf42b4 100644 --- a/tests/test_thesis_app_behavior.py +++ b/tests/test_thesis_app_behavior.py @@ -11,6 +11,7 @@ import pandas as pd import polars as pl +import pytest from myna.application.thesis import read_parameter from myna.application.thesis.melt_pool_geometry_part import ThesisMeltPoolGeometryPart @@ -21,6 +22,7 @@ from myna.application.thesis.temperature_part import ThesisTemperaturePart import myna.application.thesis.melt_pool_geometry_part.app as melt_pool_app_module import myna.application.thesis.thesis as thesis_module +import myna.core.context as context_module def _write_settings(tmp_path, step_name, step_output_paths): @@ -191,6 +193,7 @@ def get_property(self, name, *_args): def test_temperature_and_solidification_part_setup_share_case_configuration( monkeypatch, tmp_path ): + monkeypatch.setattr(context_module, "_LEGACY_ENV_FALLBACK_WARNED", False) _configure_workflow_env(monkeypatch, tmp_path, "temperature_part") monkeypatch.setenv("MYNA_INSTALL_PATH", str(tmp_path / "install")) _patch_material_information(monkeypatch) @@ -206,11 +209,14 @@ def test_temperature_and_solidification_part_setup_share_case_configuration( _write_case_metadata(temperature_case, case_payload) _write_case_metadata(solidification_case, case_payload) - temperature_app = ThesisTemperaturePart() + with pytest.warns(DeprecationWarning, match="Myna 2.0"): + temperature_app = ThesisTemperaturePart() temperature_app.args = _build_args(template_dir, nout=5) temperature_app.configure_case(str(temperature_case)) - solidification_app = ThesisSolidificationPart() + monkeypatch.setattr(context_module, "_LEGACY_ENV_FALLBACK_WARNED", False) + with pytest.warns(DeprecationWarning, match="Myna 2.0"): + solidification_app = ThesisSolidificationPart() solidification_app.args = _build_args(template_dir, nout=5) solidification_app.configure_case(str(solidification_case)) @@ -235,6 +241,7 @@ def test_temperature_and_solidification_part_setup_share_case_configuration( def test_solidification_build_region_configure_creates_ordered_paths_and_beams( monkeypatch, tmp_path ): + monkeypatch.setattr(context_module, "_LEGACY_ENV_FALLBACK_WARNED", False) _configure_workflow_env(monkeypatch, tmp_path, "solidification_build_region") monkeypatch.setenv("MYNA_INSTALL_PATH", str(tmp_path / "install")) _patch_material_information(monkeypatch) @@ -249,7 +256,8 @@ def test_solidification_build_region_configure_creates_ordered_paths_and_beams( case_dir = tmp_path / "case" _write_case_metadata(case_dir, _build_region_case_payload(scanfile_a, scanfile_b)) - app = ThesisSolidificationBuildRegion() + with pytest.warns(DeprecationWarning, match="Myna 2.0"): + app = ThesisSolidificationBuildRegion() app.args = _build_args(template_dir) app.configure_case(str(case_dir)) @@ -268,6 +276,7 @@ def test_solidification_build_region_configure_creates_ordered_paths_and_beams( def test_melt_pool_geometry_configure_creates_segment_cases(monkeypatch, tmp_path): + monkeypatch.setattr(context_module, "_LEGACY_ENV_FALLBACK_WARNED", False) _configure_workflow_env(monkeypatch, tmp_path, "melt_pool_geometry_part") monkeypatch.setenv("MYNA_INSTALL_PATH", str(tmp_path / "install")) _patch_material_information(monkeypatch) @@ -308,7 +317,8 @@ def get_all_scan_stats(self): monkeypatch.setattr(melt_pool_app_module, "Scanpath", FakeScanpath) monkeypatch.setattr(melt_pool_app_module, "ThesisPath", FakeThesisPath) - app = ThesisMeltPoolGeometryPart() + with pytest.warns(DeprecationWarning, match="Myna 2.0"): + app = ThesisMeltPoolGeometryPart() app.args = _build_args(template_dir, nout=5) app.configure_case(str(case_dir)) @@ -344,7 +354,63 @@ def get_all_scan_stats(self): ) +def test_melt_pool_geometry_configure_skips_segments_without_snapshot_times( + monkeypatch, tmp_path +): + monkeypatch.setattr(context_module, "_LEGACY_ENV_FALLBACK_WARNED", False) + _configure_workflow_env(monkeypatch, tmp_path, "melt_pool_geometry_part") + monkeypatch.setenv("MYNA_INSTALL_PATH", str(tmp_path / "install")) + _patch_material_information(monkeypatch) + + scanfile = tmp_path / "scan.txt" + _write_scanfile(scanfile) + template_dir = tmp_path / "template" + _write_template(template_dir) + + case_dir = tmp_path / "case" + _write_case_metadata(case_dir, _build_part_case_payload(scanfile)) + + class FakeScanpath: + def __init__(self, _path, _part, _layer): + self.file_local = str(scanfile) + + def get_constant_z_slice_indices(self): + return ( + [(0, 0), (1, 1), (2, 2), (3, 3)], + pl.DataFrame( + { + "X(mm)": [0.0, 1.0, 2.0, 3.0], + "Y(mm)": [0.0, 0.0, 1.0, 1.0], + "Mode": [1, 0, 0, 0], + "Pmod": [0, 1, 1, 1], + "tParam": [0.0, 0.002, 0.002, 0.002], + } + ), + ) + + class FakeThesisPath: + def loadData(self, _file): + return None + + def get_all_scan_stats(self): + return (10.0, 0.0, 1.0, 1.0) + + monkeypatch.setattr(melt_pool_app_module, "Scanpath", FakeScanpath) + monkeypatch.setattr(melt_pool_app_module, "ThesisPath", FakeThesisPath) + + with pytest.warns(DeprecationWarning, match="Myna 2.0"): + app = ThesisMeltPoolGeometryPart() + app.args = _build_args(template_dir, nout=2) + app.configure_case(str(case_dir)) + + assert not (case_dir / "path_segment_000").exists() + assert not (case_dir / "path_segment_001").exists() + assert not (case_dir / "path_segment_002").exists() + assert (case_dir / "path_segment_003").is_dir() + + def test_temperature_execute_exports_snapshot_schema(monkeypatch, tmp_path): + monkeypatch.setattr(context_module, "_LEGACY_ENV_FALLBACK_WARNED", False) output_path = tmp_path / "temperature.csv" _configure_workflow_env( monkeypatch, @@ -362,7 +428,8 @@ def test_temperature_execute_exports_snapshot_schema(monkeypatch, tmp_path): encoding="utf-8", ) - app = ThesisTemperaturePart() + with pytest.warns(DeprecationWarning, match="Myna 2.0"): + app = ThesisTemperaturePart() app.args = _build_args(tmp_path / "unused", nout=4) monkeypatch.setattr(app, "parse_execute_arguments", lambda: None) monkeypatch.setattr(app, "get_step_output_paths", lambda: [str(output_path)]) @@ -384,6 +451,7 @@ def test_temperature_execute_exports_snapshot_schema(monkeypatch, tmp_path): def test_solidification_part_execute_merges_final_csvs(monkeypatch, tmp_path): + monkeypatch.setattr(context_module, "_LEGACY_ENV_FALLBACK_WARNED", False) case_dir = tmp_path / "case" output_path = case_dir / "solidification.csv" _configure_workflow_env( @@ -407,7 +475,8 @@ def test_solidification_part_execute_merges_final_csvs(monkeypatch, tmp_path): encoding="utf-8", ) - app = ThesisSolidificationPart() + with pytest.warns(DeprecationWarning, match="Myna 2.0"): + app = ThesisSolidificationPart() app.args = _build_args(tmp_path / "unused") monkeypatch.setattr(app, "parse_execute_arguments", lambda: None) monkeypatch.setattr(app, "run_case", lambda proc_list: proc_list) @@ -423,6 +492,7 @@ def test_solidification_part_execute_merges_final_csvs(monkeypatch, tmp_path): def test_solidification_build_region_execute_exports_single_final_csv( monkeypatch, tmp_path ): + monkeypatch.setattr(context_module, "_LEGACY_ENV_FALLBACK_WARNED", False) output_path = tmp_path / "build-region.csv" _configure_workflow_env( monkeypatch, @@ -440,7 +510,8 @@ def test_solidification_build_region_execute_exports_single_final_csv( encoding="utf-8", ) - app = ThesisSolidificationBuildRegion() + with pytest.warns(DeprecationWarning, match="Myna 2.0"): + app = ThesisSolidificationBuildRegion() app.args = _build_args(tmp_path / "unused") monkeypatch.setattr(app, "parse_execute_arguments", lambda: None) monkeypatch.setattr(app, "get_step_output_paths", lambda: [str(output_path)]) @@ -464,6 +535,7 @@ def test_solidification_build_region_execute_exports_single_final_csv( def test_temperature_run_case_reuses_existing_results_unless_overwriting( monkeypatch, tmp_path ): + monkeypatch.setattr(context_module, "_LEGACY_ENV_FALLBACK_WARNED", False) _configure_workflow_env(monkeypatch, tmp_path, "temperature_part") case_dir = tmp_path / "case" @@ -476,7 +548,8 @@ def test_temperature_run_case_reuses_existing_results_unless_overwriting( existing_file = data_dir / "existing.csv" existing_file.write_text("x,y,z,T\n", encoding="utf-8") - app = ThesisTemperaturePart() + with pytest.warns(DeprecationWarning, match="Myna 2.0"): + app = ThesisTemperaturePart() app.args = _build_args(tmp_path / "unused", overwrite=False, nout=4) app.set_case(str(case_dir), str(case_dir)) monkeypatch.setattr( diff --git a/tests/test_workflow_context.py b/tests/test_workflow_context.py new file mode 100644 index 00000000..83a59104 --- /dev/null +++ b/tests/test_workflow_context.py @@ -0,0 +1,656 @@ +# +# Copyright (c) Oak Ridge National Laboratory. +# +# This file is part of Myna. For details, see the top-level license +# at https://github.com/ORNL-MDF/Myna/LICENSE.md. +# +# License: 3-clause BSD, see https://opensource.org/licenses/BSD-3-Clause. +# +import importlib +import json +import os +import sys +import types +import warnings + +import pytest +from myna.core.app.base import MynaApp +from myna.core.components.component import Component +import myna.core.components +import myna.core.context as context_module +from myna.core.context import ( + WorkflowContext, + current_workflow_context, + get_workflow_context, + get_workflow_input_file, + workflow_context, +) +from myna.core.workflow.run import run +from myna.core.workflow.sync import sync + + +WORKFLOW_ENV_KEYS = [ + "MYNA_INPUT", + "MYNA_RUN_INPUT", + "MYNA_SYNC_INPUT", + "MYNA_STEP_NAME", + "MYNA_STEP_CLASS", + "MYNA_STEP_INDEX", + "MYNA_LAST_STEP_NAME", + "MYNA_LAST_STEP_CLASS", +] + + +def _clear_workflow_env(monkeypatch): + for key in WORKFLOW_ENV_KEYS: + monkeypatch.delenv(key, raising=False) + + +def test_myna_app_reads_explicit_workflow_context(monkeypatch, tmp_path): + _clear_workflow_env(monkeypatch) + monkeypatch.setattr(sys, "argv", ["test"]) + input_file = tmp_path / "input.yaml" + input_file.write_text( + """ +steps: +- demo: + class: demo_class + application: demo_app +data: {} +myna: {} +""", + encoding="utf-8", + ) + + with workflow_context( + input_file=os.fspath(input_file), + step_name="demo", + step_class="demo_class", + step_index=0, + last_step_name="previous", + ): + app = MynaApp() + + assert app.input_file == os.fspath(input_file) + assert app.step_name == "demo" + assert app.step_class == "demo_class" + assert app.step_number == 0 + assert app.last_step_name == "previous" + + +def test_get_workflow_context_prefers_explicit_context_over_env(monkeypatch, tmp_path): + _clear_workflow_env(monkeypatch) + input_file = tmp_path / "input.yaml" + input_file.write_text("steps: []\n", encoding="utf-8") + monkeypatch.setenv("MYNA_INPUT", "wrong-input.yaml") + monkeypatch.setenv("MYNA_STEP_NAME", "wrong-step") + + with workflow_context( + input_file=os.fspath(input_file), + step_name="demo", + step_class="demo_class", + step_index=7, + last_step_name="previous", + last_step_class="previous_class", + ): + context = get_workflow_context() + + assert context == WorkflowContext( + input_file=os.fspath(input_file), + step_name="demo", + step_class="demo_class", + step_index=7, + last_step_name="previous", + last_step_class="previous_class", + ) + + +def test_get_workflow_context_falls_back_to_legacy_env(monkeypatch): + _clear_workflow_env(monkeypatch) + monkeypatch.setattr(context_module, "_LEGACY_ENV_FALLBACK_WARNED", False) + monkeypatch.setenv("MYNA_INPUT", "/tmp/input.yaml") + monkeypatch.setenv("MYNA_STEP_NAME", "demo") + monkeypatch.setenv("MYNA_STEP_CLASS", "demo_class") + monkeypatch.setenv("MYNA_STEP_INDEX", "3") + monkeypatch.setenv("MYNA_LAST_STEP_NAME", "previous") + monkeypatch.setenv("MYNA_LAST_STEP_CLASS", "previous_class") + + with pytest.warns(DeprecationWarning, match="Myna 2.0"): + assert get_workflow_context() == WorkflowContext( + input_file="/tmp/input.yaml", + step_name="demo", + step_class="demo_class", + step_index=3, + last_step_name="previous", + last_step_class="previous_class", + ) + + +def test_get_workflow_context_returns_none_without_state(monkeypatch): + _clear_workflow_env(monkeypatch) + + assert get_workflow_context() is None + + +def test_get_workflow_input_file_uses_shared_context_resolution(monkeypatch, tmp_path): + _clear_workflow_env(monkeypatch) + monkeypatch.setattr(context_module, "_LEGACY_ENV_FALLBACK_WARNED", False) + input_file = tmp_path / "input.yaml" + input_file.write_text("steps: []\n", encoding="utf-8") + + assert get_workflow_input_file(default="fallback.yaml") == "fallback.yaml" + + monkeypatch.setenv("MYNA_INPUT", os.fspath(input_file)) + with pytest.warns(DeprecationWarning, match="Myna 2.0"): + assert get_workflow_input_file() == os.fspath(input_file) + + +def test_workflow_context_env_fallback_warns_once_per_process(monkeypatch): + _clear_workflow_env(monkeypatch) + monkeypatch.setattr(context_module, "_LEGACY_ENV_FALLBACK_WARNED", False) + monkeypatch.setenv("MYNA_INPUT", "/tmp/input.yaml") + + with pytest.warns(DeprecationWarning, match="Myna 2.0") as record: + get_workflow_context() + get_workflow_context() + + assert len(record) == 1 + + +def test_explicit_workflow_context_does_not_warn(monkeypatch, tmp_path): + _clear_workflow_env(monkeypatch) + monkeypatch.setattr(context_module, "_LEGACY_ENV_FALLBACK_WARNED", False) + input_file = tmp_path / "input.yaml" + input_file.write_text("steps: []\n", encoding="utf-8") + + with workflow_context(input_file=os.fspath(input_file), step_name="demo"): + with warnings.catch_warnings(record=True) as record: + warnings.simplefilter("always") + context = get_workflow_context() + + assert context == WorkflowContext( + input_file=os.fspath(input_file), step_name="demo" + ) + assert len(record) == 0 + + +def test_get_workflow_context_invalid_step_index_raises_value_error(monkeypatch): + _clear_workflow_env(monkeypatch) + monkeypatch.setattr(context_module, "_LEGACY_ENV_FALLBACK_WARNED", False) + monkeypatch.setenv("MYNA_STEP_INDEX", "not-an-int") + + with pytest.warns(DeprecationWarning, match="Myna 2.0"): + with pytest.raises(ValueError): + get_workflow_context() + + +def test_myna_app_reads_legacy_workflow_env(monkeypatch, tmp_path): + _clear_workflow_env(monkeypatch) + monkeypatch.setattr(context_module, "_LEGACY_ENV_FALLBACK_WARNED", False) + monkeypatch.setattr(sys, "argv", ["test"]) + input_file = tmp_path / "input.yaml" + input_file.write_text( + """ +steps: +- demo: + class: demo_class + application: demo_app +data: {} +myna: {} +""", + encoding="utf-8", + ) + monkeypatch.setenv("MYNA_INPUT", os.fspath(input_file)) + monkeypatch.setenv("MYNA_STEP_NAME", "demo") + monkeypatch.setenv("MYNA_STEP_CLASS", "demo_class") + monkeypatch.setenv("MYNA_STEP_INDEX", "0") + monkeypatch.setenv("MYNA_LAST_STEP_NAME", "previous") + + with pytest.warns(DeprecationWarning, match="Myna 2.0"): + app = MynaApp() + + assert app.input_file == os.fspath(input_file) + assert app.step_name == "demo" + assert app.step_class == "demo_class" + assert app.step_number == 0 + assert app.last_step_name == "previous" + + +def test_myna_app_template_resolves_relative_to_workflow_input(monkeypatch, tmp_path): + _clear_workflow_env(monkeypatch) + monkeypatch.setattr(sys, "argv", ["test"]) + input_dir = tmp_path / "workflow" + input_dir.mkdir() + input_file = input_dir / "input.yaml" + input_file.write_text( + """ +steps: +- demo: + class: demo_class + application: demo_app +data: {} +myna: {} +""", + encoding="utf-8", + ) + + with workflow_context( + input_file=os.fspath(input_file), + step_name="demo", + step_class="demo_class", + step_index=0, + ): + app = MynaApp() + + app.args.template = "./template" + assert app.template == input_dir / "template" + + +def test_component_runs_stage_with_context_without_env(monkeypatch, tmp_path): + _clear_workflow_env(monkeypatch) + input_file = tmp_path / "input.yaml" + input_file.write_text( + "steps: []\ndata:\n build:\n name: build\nmyna: {}\n", + encoding="utf-8", + ) + module_name = "myna.application.fakeapp.fakeclass.configure" + captured = {} + original_import_module = importlib.import_module + monkeypatch.setenv("MYNA_INSTALL_PATH", "/tmp/myna-install") + monkeypatch.setenv("MYNA_APP_PATH", "/tmp/myna-install/application") + + def configure(): + context = current_workflow_context() + captured["context"] = context + captured["argv"] = list(sys.argv) + captured["env"] = { + "input": os.environ.get("MYNA_INPUT"), + "run_input": os.environ.get("MYNA_RUN_INPUT"), + "step": os.environ.get("MYNA_STEP_NAME"), + "step_class": os.environ.get("MYNA_STEP_CLASS"), + "step_index": os.environ.get("MYNA_STEP_INDEX"), + "last_step": os.environ.get("MYNA_LAST_STEP_NAME"), + } + + fake_module = types.SimpleNamespace( + configure=configure, + __file__="/tmp/myna-install/application/fakeapp/fakeclass/configure.py", + ) + + def fake_import_module(name): + if name == module_name: + return fake_module + return original_import_module(name) + + monkeypatch.setattr( + os.path, + "exists", + lambda path: path + == "/tmp/myna-install/application/fakeapp/fakeclass/configure.py", + ) + monkeypatch.setattr(importlib, "import_module", fake_import_module) + + component = Component() + component.name = "demo" + component.component_application = "fakeapp" + component.component_class = "fakeclass" + component.input_file = os.fspath(input_file) + component.step_index = 1 + component.last_step_name = "previous" + component.configure_dict = {"example": "value"} + component.data = {"build": {"name": "build"}} + + previous_argv = list(sys.argv) + component.run_component() + + assert captured["context"].input_file == os.fspath(input_file) + assert captured["context"].step_name == "demo" + assert captured["context"].step_class == "fakeclass" + assert captured["context"].step_index == 1 + assert captured["context"].last_step_name == "previous" + assert captured["argv"] == [ + "/tmp/myna-install/application/fakeapp/fakeclass/configure.py", + "--example", + "value", + ] + assert captured["env"] == { + "input": os.fspath(input_file), + "run_input": os.fspath(input_file), + "step": "demo", + "step_class": "fakeclass", + "step_index": "1", + "last_step": "previous", + } + assert sys.argv == previous_argv + for key in WORKFLOW_ENV_KEYS: + assert key not in os.environ + + +def test_run_passes_workflow_context_without_setting_env(monkeypatch, tmp_path): + _clear_workflow_env(monkeypatch) + input_file = tmp_path / "input.yaml" + input_file.write_text( + """ +steps: +- first: + class: first_class + application: fake +- second: + class: second_class + application: fake +data: + build: + name: build +myna: {} +""", + encoding="utf-8", + ) + calls = [] + + class DummyComponent(Component): + def run_component(self): + calls.append( + { + "name": self.name, + "class": self.component_class, + "input_file": self.input_file, + "step_index": self.step_index, + "last_step_name": self.last_step_name, + "env_step": os.environ.get("MYNA_STEP_NAME"), + } + ) + + monkeypatch.setattr( + myna.core.components, + "return_step_class", + lambda component_class_name: DummyComponent(), + ) + + run(os.fspath(input_file)) + + assert calls == [ + { + "name": "first", + "class": "first_class", + "input_file": os.path.abspath(input_file), + "step_index": 0, + "last_step_name": "", + "env_step": None, + }, + { + "name": "second", + "class": "second_class", + "input_file": os.path.abspath(input_file), + "step_index": 1, + "last_step_name": "first", + "env_step": None, + }, + ] + for key in WORKFLOW_ENV_KEYS: + assert key not in os.environ + + +def test_component_restores_cwd_after_stage_failure(monkeypatch, tmp_path): + _clear_workflow_env(monkeypatch) + monkeypatch.setenv("MYNA_INSTALL_PATH", "/tmp/myna-install") + monkeypatch.setenv("MYNA_APP_PATH", "/tmp/myna-install/application") + input_file = tmp_path / "input.yaml" + input_file.write_text( + "steps: []\ndata:\n build:\n name: build\n", encoding="utf-8" + ) + stage_dir = tmp_path / "stage-dir" + stage_dir.mkdir() + module_name = "myna.application.fakeapp.fakeclass.execute" + original_import_module = importlib.import_module + captured = {} + + def execute(): + os.chdir(stage_dir) + captured["cwd_during_stage"] = os.getcwd() + captured["env_step"] = os.environ.get("MYNA_STEP_NAME") + raise RuntimeError("boom") + + fake_module = types.SimpleNamespace(execute=execute) + + def fake_import_module(name): + if name == module_name: + return fake_module + return original_import_module(name) + + monkeypatch.setattr( + os.path, + "exists", + lambda path: path + == "/tmp/myna-install/application/fakeapp/fakeclass/execute.py", + ) + monkeypatch.setattr(importlib, "import_module", fake_import_module) + + component = Component() + component.name = "demo" + component.component_application = "fakeapp" + component.component_class = "fakeclass" + component.input_file = os.fspath(input_file) + component.step_index = 2 + component.last_step_name = "previous" + component.execute_dict = {"overwrite": True} + component.data = {"build": {"name": "build"}} + + previous_cwd = os.getcwd() + previous_argv = list(sys.argv) + with pytest.raises(RuntimeError, match="boom"): + component._run_stage("execute") + + assert captured["cwd_during_stage"] == os.fspath(stage_dir) + assert captured["env_step"] == "demo" + assert os.getcwd() == previous_cwd + assert sys.argv == previous_argv + for key in WORKFLOW_ENV_KEYS: + assert key not in os.environ + + +def test_component_stage_missing_skips_without_import(monkeypatch): + _clear_workflow_env(monkeypatch) + monkeypatch.setenv("MYNA_APP_PATH", "/tmp/myna-apps") + import_called = False + + def fake_import_module(name): + nonlocal import_called + import_called = True + return importlib.import_module(name) + + monkeypatch.setattr(os.path, "exists", lambda path: False) + monkeypatch.setattr(importlib, "import_module", fake_import_module) + + component = Component() + component.component_application = "fakeapp" + component.component_class = "fakeclass" + + assert component._run_stage("configure") is False + assert import_called is False + + +def test_component_stage_import_error_is_not_treated_as_missing(monkeypatch): + _clear_workflow_env(monkeypatch) + monkeypatch.setenv("MYNA_INSTALL_PATH", "/tmp/myna-install") + monkeypatch.setenv("MYNA_APP_PATH", "/tmp/myna-install/application") + module_name = "myna.application.fakeapp.fakeclass.configure" + + def fake_import_module(name): + if name == module_name: + raise ModuleNotFoundError("No module named 'vtk'") + return importlib.import_module(name) + + monkeypatch.setattr( + os.path, + "exists", + lambda path: path + == "/tmp/myna-install/application/fakeapp/fakeclass/configure.py", + ) + monkeypatch.setattr(importlib, "import_module", fake_import_module) + + component = Component() + component.component_application = "fakeapp" + component.component_class = "fakeclass" + + with pytest.raises(ModuleNotFoundError, match="vtk"): + component._run_stage("configure") + + +def test_component_runs_external_stage_via_subprocess(monkeypatch, tmp_path): + _clear_workflow_env(monkeypatch) + input_file = tmp_path / "input.yaml" + input_file.write_text( + "steps: []\ndata:\n build:\n name: build\nmyna: {}\n", + encoding="utf-8", + ) + app_root = tmp_path / "external-apps" + stage_dir = app_root / "fakeapp" / "fakeclass" + stage_dir.mkdir(parents=True) + capture_path = tmp_path / "capture.json" + script_path = stage_dir / "configure.py" + script_path.write_text( + "\n".join( + [ + "import json", + "import os", + "import sys", + f"capture_path = {os.fspath(capture_path)!r}", + "with open(capture_path, 'w', encoding='utf-8') as handle:", + " json.dump({", + " 'argv': sys.argv,", + " 'env': {", + " 'input': os.environ.get('MYNA_INPUT'),", + " 'run_input': os.environ.get('MYNA_RUN_INPUT'),", + " 'step': os.environ.get('MYNA_STEP_NAME'),", + " 'step_class': os.environ.get('MYNA_STEP_CLASS'),", + " 'step_index': os.environ.get('MYNA_STEP_INDEX'),", + " 'last_step': os.environ.get('MYNA_LAST_STEP_NAME'),", + " },", + " }, handle)", + ] + ), + encoding="utf-8", + ) + monkeypatch.setenv("MYNA_APP_PATH", os.fspath(app_root)) + monkeypatch.setenv("MYNA_INSTALL_PATH", os.fspath(tmp_path / "install-root")) + + def fail_import(name): + raise AssertionError(f"unexpected import: {name}") + + monkeypatch.setattr(importlib, "import_module", fail_import) + + component = Component() + component.name = "demo" + component.component_application = "fakeapp" + component.component_class = "fakeclass" + component.input_file = os.fspath(input_file) + component.step_index = 1 + component.last_step_name = "previous" + component.configure_dict = {"example": "value"} + component.data = {"build": {"name": "build"}} + + previous_argv = list(sys.argv) + assert component._run_stage("configure") is True + + captured = json.loads(capture_path.read_text(encoding="utf-8")) + assert captured["argv"] == [ + os.fspath(script_path), + "--example", + "value", + ] + assert captured["env"] == { + "input": os.fspath(input_file), + "run_input": os.fspath(input_file), + "step": "demo", + "step_class": "fakeclass", + "step_index": "1", + "last_step": "previous", + } + assert sys.argv == previous_argv + for key in WORKFLOW_ENV_KEYS: + assert key not in os.environ + + +def test_sync_passes_legacy_env_compatibility(monkeypatch, tmp_path): + _clear_workflow_env(monkeypatch) + input_file = tmp_path / "input.yaml" + output_file = tmp_path / "build" / "part-a" / "region-1" / "output.csv" + output_file.parent.mkdir(parents=True) + output_file.write_text("x\n1\n", encoding="utf-8") + input_file.write_text( + """ +steps: +- demo: + class: demo_class + application: demo_app +data: + build: + name: build + datatype: fake_db + path: /tmp/fake-db + output_paths: + demo: + - placeholder +myna: {} +""", + encoding="utf-8", + ) + captured = {} + + class DummyOutputRequirement: + filetype = ".csv" + + def __init__(self, path): + self.path = path + + def file_is_valid(self): + return True + + class DummyComponent(Component): + def __init__(self): + super().__init__() + self.output_requirement = DummyOutputRequirement + self.types = ["build", "region"] + + def apply_settings(self, step_settings, data_settings, myna_settings): + super().apply_settings(step_settings, data_settings, myna_settings) + self.data["output_paths"]["demo"] = [os.fspath(output_file)] + + class DummyDatatype: + def set_path(self, path): + captured["path"] = path + + def sync(self, component_application, types, output_requirement, files): + captured["component_application"] = component_application + captured["types"] = types + captured["files"] = files + captured["env"] = {key: os.environ.get(key) for key in WORKFLOW_ENV_KEYS} + return files + + monkeypatch.setattr( + myna.core.components, + "return_step_class", + lambda component_class_name: DummyComponent(), + ) + monkeypatch.setattr( + myna.database, "return_datatype_class", lambda _: DummyDatatype() + ) + + sync(os.fspath(input_file)) + + assert captured["path"] == "/tmp/fake-db" + assert captured["component_application"] == "demo_app" + assert captured["types"] == ["build", "region"] + assert captured["files"] == [os.fspath(output_file)] + assert captured["env"] == { + "MYNA_INPUT": os.path.abspath(input_file), + "MYNA_RUN_INPUT": None, + "MYNA_SYNC_INPUT": os.path.abspath(input_file), + "MYNA_STEP_NAME": "demo", + "MYNA_STEP_CLASS": "demo_class", + "MYNA_STEP_INDEX": "0", + "MYNA_LAST_STEP_NAME": "", + "MYNA_LAST_STEP_CLASS": "", + } + for key in WORKFLOW_ENV_KEYS: + assert key not in os.environ