diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..00ec958 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,42 @@ +name: Lint + +# Lint runs on a cheap GitHub-hosted runner (the unit-test / cluster jobs use a +# self-hosted Windows runner with lab hardware + Confluence secrets, which is +# overkill for a style check and unavailable on forks). black and flake8 read +# their config from pyproject.toml ([tool.black] / [tool.flake8] via +# Flake8-pyproject); versions are pinned to match .pre-commit-config.yaml so CI +# and the local pre-commit hook agree. + +on: + push: + branches: + - master + - develop + pull_request: + branches: + - master + - develop + +jobs: + lint: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + + - name: Install linters + run: | + python -m pip install --upgrade pip + python -m pip install black==26.5.1 flake8==7.1.1 Flake8-pyproject + + - name: black --check + run: black --check . + + - name: flake8 + run: flake8 . diff --git a/.gitignore b/.gitignore index 7f1191e..b3034fb 100644 --- a/.gitignore +++ b/.gitignore @@ -162,6 +162,11 @@ cython_debug/ .DS_Store +# Claude Code: keep local settings and personal notes out of git, but the +# shared standing context in CLAUDE.md IS tracked (do not ignore it). +.claude/ +CLAUDE.local.md + # Generated by setuptools-scm at install time — do not commit picasso_workflow/_version.py @@ -171,5 +176,4 @@ temp/* spinna* nn_redistribution.py -CLAUDE.md -config.yaml \ No newline at end of file +config.yaml diff --git a/.vscode/settings.json b/.vscode/settings.json index c213de7..c84f6ca 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -17,5 +17,5 @@ "picasso_workflow" ], "python.testing.unittestEnabled": false, - "python.testing.pytestEnabled": true + "python.testing.pytestEnabled": true, } diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..03b3c4b --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,79 @@ +# Changelog + +All notable changes to picasso-workflow are documented here. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). +Versions are derived from git tags by setuptools-scm, so entries are collected +under `[Unreleased]` until a tag is cut. + +This file was started after v0.5.6; earlier history is in the git log. + +## [Unreleased] + +### Added + +- Confluence error reports now identify the failing module by index and + name in a heading, list the parameters it was called with, name the + innermost picasso-workflow stack frame, and link the module result + folder and the preceding module's results. Previously only the + exception message and traceback were posted, so diagnosing a failure + meant guessing which parameter value had caused it. +- Failed modules are recorded in `WorkflowRunner.yaml` with their + index, parameters, exception type, message and traceback. +- Per-channel parameters: module parameters can differ between channels + of an aggregation workflow via a `("$$map", "", )` + command backed by a column in `single_dataset_tileparameters`. The + GUI shows the resolved per-channel values beneath the parameter row + and round-trips the dataset table through the generated script. + +### Changed + +- The Zeiss `.czi` reader (`aicsimageio` + `aicspylibczi` + `fsspec`) + moved from the base dependencies to an optional `formats` extra + (`pip install "picasso_workflow[formats]"`). aicspylibczi has no + aarch64 wheels and source-builds via a C++/cmake toolchain, and + aicsimageio dragged an old imagecodecs that failed to build on the + py3.10 arm64 container, so a bare `pip install -e .` now resolves + entirely from wheels. The `convert_zeiss_movie` module raises a clear + ImportError pointing at the extra when the reader is absent; no other + workflow is affected. +- `estimate_density_from_neighbordists` now validates the + `[min_dist, max_dist]` window per neighbour order and fails with a + message naming the parameter and the surviving counts, e.g. + `min_dist=50, max_dist=300 leaves 0 of 530 k=4 nearest-neighbour + distances (observed range 338.3-2.264e+04)`. + + BEHAVIOUR CHANGE: runs in which some neighbour order had no distances + inside the window previously fitted on the remaining orders and now + raise instead. Workflows believed to be healthy may start failing; + widen the window or reduce the number of neighbours fitted. +- Tracebacks are posted inside a Confluence code macro, so their line + structure is preserved. They were previously HTML-escaped without a + wrapper and reflowed into a single unreadable paragraph. +- CI now runs a `black --check` + `flake8` lint job on a GitHub-hosted + runner (`.github/workflows/lint.yml`), so style regressions are caught + in CI and not only by the local pre-commit hook. Linter versions are + pinned to match `.pre-commit-config.yaml`. + +### Fixed + +- An exception other than `AutoPicassoError` escaped + `WorkflowRunner.run()` before `save()`, so the failing module left + no trace in `WorkflowRunner.yaml`. +- A module raising `AutoPicassoError` on the first iteration of + `run()` raised `UnboundLocalError` on `success`, masking the + real error. +- `call_module` re-raised a `copy.copy()` of the exception, which + drops `__traceback__`; the propagated error stopped at the re-raise + rather than pointing at the code that failed. +- `fit_csr` used truthiness to detect optional parameters, so + `min_dist=0`, `max_dist=0` and `bkg_fraction=0` were silently + replaced by defaults. +- `nndistribution_from_csr` raised "zero-size array to reduction + operation maximum" on an empty distance array instead of returning an + empty result. +- The GUI's "Remove Dataset" button did nothing, silently, when a + channel was selected, and its buttons were ordered inconsistently. +- Commands could not be assigned to nested (dict) sub-parameters: the + `cmd` dialog raised `KeyError` on accept, and a nested command + value was discarded when the workflow was reloaded. diff --git a/CHANGELOG.txt b/CHANGELOG.txt deleted file mode 100644 index bbd3b9c..0000000 --- a/CHANGELOG.txt +++ /dev/null @@ -1,73 +0,0 @@ -Changelog -========= - -All notable changes to picasso-workflow are documented here. - -The format follows `Keep a Changelog `_. -Versions are derived from git tags by vcs-versioning, so entries are -collected under "Unreleased" until a tag is cut. - -This file was started after v0.5.6; earlier history is in the git log. - - -Unreleased ----------- - -Added -~~~~~ - -- Confluence error reports now identify the failing module by index and - name in a heading, list the parameters it was called with, name the - innermost picasso-workflow stack frame, and link the module result - folder and the preceding module's results. Previously only the - exception message and traceback were posted, so diagnosing a failure - meant guessing which parameter value had caused it. -- Failed modules are recorded in ``WorkflowRunner.yaml`` with their - index, parameters, exception type, message and traceback. -- Per-channel parameters: module parameters can differ between channels - of an aggregation workflow via a ``("$$map", "", )`` - command backed by a column in ``single_dataset_tileparameters``. The - GUI shows the resolved per-channel values beneath the parameter row - and round-trips the dataset table through the generated script. - -Changed -~~~~~~~ - -- ``estimate_density_from_neighbordists`` now validates the - ``[min_dist, max_dist]`` window per neighbour order and fails with a - message naming the parameter and the surviving counts, e.g. - ``min_dist=50, max_dist=300 leaves 0 of 530 k=4 nearest-neighbour - distances (observed range 338.3-2.264e+04)``. - - BEHAVIOUR CHANGE: runs in which some neighbour order had no distances - inside the window previously fitted on the remaining orders and now - raise instead. Workflows believed to be healthy may start failing; - widen the window or reduce the number of neighbours fitted. - -- Tracebacks are posted inside a Confluence code macro, so their line - structure is preserved. They were previously HTML-escaped without a - wrapper and reflowed into a single unreadable paragraph. - -Fixed -~~~~~ - -- An exception other than ``AutoPicassoError`` escaped - ``WorkflowRunner.run()`` before ``save()``, so the failing module left - no trace in ``WorkflowRunner.yaml``. -- A module raising ``AutoPicassoError`` on the first iteration of - ``run()`` raised ``UnboundLocalError`` on ``success``, masking the - real error. -- ``call_module`` re-raised a ``copy.copy()`` of the exception, which - drops ``__traceback__``; the propagated error stopped at the re-raise - rather than pointing at the code that failed. -- ``fit_csr`` used truthiness to detect optional parameters, so - ``min_dist=0``, ``max_dist=0`` and ``bkg_fraction=0`` were silently - replaced by defaults. -- ``nndistribution_from_csr`` raised "zero-size array to reduction - operation maximum" on an empty distance array instead of returning an - empty result. -- The GUI's "Remove Dataset" button did nothing, silently, when a - channel was selected, and its buttons were ordered inconsistently. -- Commands could not be assigned to nested (dict) sub-parameters: the - ``cmd`` dialog raised ``KeyError`` on accept, and a nested command - value was discarded when the workflow was reloaded. diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..7dc6acf --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,175 @@ +# CLAUDE.md — picasso-workflow + +Standing context for Claude Code (claude.ai/code) working in this repo. Read +this first, then the design doc and the cross-repo pointers below. picasso-workflow +is one repo in the **DNA-PAINT full-automation** stack (siblings: PycroFlow, +monet, picasso, picasso-registry, picasso-agent). + +## What this repo is + +picasso-workflow automates and documents DNA-PAINT **analysis** workflows built +on `picassosr` (the picasso localization/clustering library). It runs a picasso +pipeline as an ordered list of named **modules** — two workflow kinds: + +- **Single-dataset** — one movie is loaded, localized, drift-corrected, + clustered, etc. (`WorkflowRunner`). +- **Aggregation / investigation** — many datasets each run the same + single-dataset modules, then are combined by aggregation modules + (`AggregationWorkflowRunner`). Module parameters can vary per channel via a + `("$$map", "", )` command backed by a column in + `single_dataset_tileparameters` (see the README "Per-channel parameters"). + +Each run documents itself to **Confluence** (`ConfluenceReporter`) and/or a +local **HTML report** (`html_reporter`). A **PyQt6 GUI** (`picasso-workflow-gui`) +builds, edits, and launches workflows, and can generate SLURM scripts to submit +them to a cluster. The importable package is `picasso_workflow/`. + +See `README.md` for the full feature list, installation, per-channel parameters, +the four-tier testing strategy, the SLURM cluster runner, CI, and the release +workflow — this file is the short standing context, not a duplicate of it. + +## Current branch + +`feature-FullAutoS0A` — PRs target `master`. (Upstream also maintains a +`develop` branch; the release workflow merges `develop` → `master` and tags on +`master` — see README "Releasing".) + +## Commands + +```bash +# Install (core; pulls picassosr and the PyQt6 GUI deps). Do this in a +# python>=3.10 env (README recommends a conda env named picasso-workflow). +pip install -e . +pip install -e ".[cluster]" # adds mpi4py for the MPI aggregation path + +# For a local development picasso, install it first, then picasso-workflow: +# cd /path/to/picasso && pip install -r requirements.txt + +# Run the GUI (console script and module form are equivalent) +picasso-workflow-gui +python -m picasso_workflow.gui + +# Test (a bare `pytest` is UNIT-ONLY — the `integration` mark is deselected by +# default in [tool.pytest.ini_options] addopts). See README "Testing". +pytest # tiers 1+2: unit + template validation (fast) +pytest -v # verbose, still unit-only +pytest -k # a single test by keyword +pytest -m integration # tier 3: real picasso pipeline (needs picassosr) +pytest -m "integration and real_data" # tier 4: real acquired data (PW_TEST_DATA_DIR) +pytest -m "" # clear the default filter: run everything + +# Lint / format — pre-commit owns black + flake8 (both run in pre-commit's own +# isolated envs; they are NOT direct package deps, so `pre-commit` is the +# canonical path rather than a bare `black`/`flake8`). +pre-commit install # once, to run hooks on every commit +pre-commit run --all-files # run all hooks (trailing-ws, eof, yaml, black, flake8) now +``` + +## Conventions (aligned across the DNA-PAINT automation repos) + +This repo already matches the aligned target — no S0A-2 migration pending here. + +- **Style:** Black, line length **79** (Black owns line wrapping). Config in + `pyproject.toml [tool.black]`. +- **Lint:** flake8 via **Flake8-pyproject**, config in + `pyproject.toml [tool.flake8]` with `extend-ignore = E203, E501, W503` — E501 + is ignored because **Black owns line length** (it already wraps code; long + strings/comments/HTML it can't split are intentional). `max-line-length = 88` + there is informational only. Some experimental modules and the test-data + fixtures are `extend-exclude`d. +- **Pre-commit:** `pre-commit install` once; hooks run trailing-whitespace, + end-of-file-fixer, check-yaml, check-added-large-files, **black**, and + **flake8** (Flake8-pyproject). isort / bandit / mypy are intentionally **not** + in the pre-commit run (see `.pre-commit-config.yaml`). +- **Versioning:** **setuptools-scm** — **the tag IS the version**; there is no + version string to edit by hand. It writes `picasso_workflow/_version.py` + (gitignored, importable as `picasso_workflow.__version__`); fallback outside a + git checkout is `0.3.3.dev0`. Release = merge to `master`, then + `git tag vX.Y.Z && git push origin vX.Y.Z` (format `vMAJOR.MINOR.PATCH`). +- **Changelog on release:** the changelog is `CHANGELOG.md` at the repo root + (Keep a Changelog, in Markdown, with `### Added` / `### Changed` / `### Fixed` + subsections — matching monet / PycroFlow / picasso-registry). Add an entry + under the top **`## [Unreleased]`** section in every PR; at release, promote + `[Unreleased]` to a dated, tagged section (e.g. `## [1.2.3] - YYYY-MM-DD`). Because the version comes from git tags, the changelog + is the human-facing record of what each tag contains. +- **Packaging:** `pyproject.toml` only (no `setup.py` / `setup.cfg`). Runtime + deps and the `[cluster]` extra live there. +- **Tests:** write/extend tests with every change; keep every tier green. + Picasso is fully mocked in the unit tier so it runs anywhere with no data or + network. Adding a workflow module touches + `util.AbstractModuleCollection`, `analyse.AutoPicasso`, + `confluence.ConfluenceReporter`, and the matching `tests/test_*` files; if a + snapshotted template references it, re-run `python tools/snapshot_templates.py` + (see README "Adding a new workflow module"). + +## Architecture (short) + +`workflow.py` holds the orchestrators (`WorkflowRunner`, +`AggregationWorkflowRunner`): each reads a module list, calls the corresponding +analysis method, and records results/failures to `WorkflowRunner.yaml`. +`analyse.py` (`AutoPicasso`) implements the actual picasso-backed modules; +`util.py` provides `AbstractModuleCollection` (the module contract), +`ParameterTiler` / `ParameterCommandExecutor` (the `$`/`$$map` per-channel +parameter machinery), and typing helpers. `standard_singledataset_workflows.py` +and `standard_aggregation_workflows.py` are the predefined recipes; +`modulespec.py` is the `ModuleSpec` annotation/validation layer; +`picasso_outpost.py` holds picasso-adjacent code not yet upstream. Reporting: +`confluence.py` (`ConfluenceReporter` / `ConfluenceInterface`) and +`html_reporter.py`. `_launcher.py` is the `picasso-workflow-gui` entry point; +`__init__.py` configures loguru logging and deep-merges `config.yaml` +(package → site → per-user). Full module map in `README.md`. + +## Standing pointers + +Paths so later sessions can `@`-reference them. Repo root is +`/workspaces/DNA-PAINT-FullAutomation/repositories/picasso-workflow`; the shared +workspace root is `/workspaces/DNA-PAINT-FullAutomation`. + +**Live (resolve today):** the shared planning docs live in `../../planning/` +(workspace `planning/` folder); start from its document map. +- Document map / reading order: `../../planning/README.md` +- Design doc — recommendation & roadmap (strategy, prioritized initiatives #1–#9, + work packages WP-1–WP-16, Parts I–X): + `../../planning/DNA-PAINT_Automation-Recommendation.md` +- **Playbook** — Claude Code implementation playbook (operating model, Step 0 + foundations, style/repo alignment, gated dependency-ordered work orders): + `../../planning/DNA-PAINT_ClaudeCode-Implementation-Playbook.md` +- **Work-order briefs** — self-contained, paste-ready briefs (S0A-1, S0A-2, + S0B-1/2, WP-1…WP-16); this task is S0A-1: + `../../planning/DNA-PAINT_Work-Order-Briefs.md` +- **Progress tracker** — tick-off worksheet + gates for the work orders: + `../../planning/DNA-PAINT_Implementation-Progress-Tracker.md` +- Module-annotations reference (the `ModuleSpec` layer, data dependencies, + capability registry): + `../../planning/picasso-workflow_Module-Annotations_Reference.md` +- Dev-environment setup (OrbStack dev-container): + `../../planning/DNA-PAINT_ClaudeCode-DevEnvironment.md` +- Sibling repo standing context: + - PycroFlow (experiment orchestration): `../PycroFlow/CLAUDE.md` + - monet (laser-power calibration/control): `../monet/CLAUDE.md` + - picasso-registry (provenance/metrics DB; **owns the schema/API contract**): + `../picasso-registry/CLAUDE.md` + - picasso-agent (agentic layer): `../picasso-agent/CLAUDE.md` + - picasso (upstream localization/clustering library — `picassosr`; no CLAUDE.md + yet): `../picasso` +- Sibling repo roots: `../PycroFlow`, `../monet`, `../picasso`, + `../picasso-registry`, `../picasso-agent` + +**Forthcoming (planned; not yet in-tree — do not treat as resolvable):** +- Cross-repo contracts (after S0B): the picasso-registry OpenAPI spec + generated + client and the shared schemas (metric-vector, workflow-YAML, + `localize_frames` signature, picasso-workflow `ModuleSpec`) — these will be + owned by picasso-registry; see `../picasso-registry/CLAUDE.md` and work orders + S0B-1 / S0B-2 in the briefs above. picasso-workflow's `ModuleSpec` + (`modulespec.py`) is part of that contract set. + +## Notes for editing + +- `.gitignore` **tracks this `CLAUDE.md`** (it is not ignored) but keeps + `.claude/` and `CLAUDE.local.md` ignored (local settings / personal notes) — + keep it that way. +- `picasso_workflow/_version.py` is generated by setuptools-scm and gitignored — + never commit or hand-edit it. +- `spinna_mle.py`, `spinna_mle_2.py`, and `nn_redistribution.py` are isolated + experimental modules (not imported by the package) and are flake8-excluded; + don't expect them to be linted. diff --git a/picasso_workflow/CLAUDE.md b/picasso_workflow/CLAUDE.md new file mode 100644 index 0000000..fdb5cce --- /dev/null +++ b/picasso_workflow/CLAUDE.md @@ -0,0 +1,168 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Project Overview + +This is `picasso-workflow`, a Python package for automated DNA-PAINT analysis workflows. The project provides two main workflow types: +- **Single-dataset workflow**: Process individual datasets through localization, clustering, and analysis +- **Aggregation workflow**: Process multiple datasets and aggregate results + +## Key Architecture + +### Core Modules +- **`workflow.py`**: Main orchestration class `ReportingAnalyzer` that coordinates picasso analysis and confluence reporting. Contains `WorkflowRunner` and `AggregationWorkflowRunner` classes. +- **`analyse.py`**: Contains `AutoPicasso` class - the main picasso interface for localization, clustering, and analysis operations +- **`confluence.py`**: Contains `ConfluenceReporter` and `ConfluenceInterface` classes for generating and uploading analysis reports to Confluence +- **`metaworkflow.py`**: Higher-level analysis functionality for running workflows across multiple conditions and cells with aggregation +- **`util.py`**: Contains `AbstractModuleCollection` and other utility classes for parameter management + +### Specialized Modules +- **`dbscan_molint/`**: DBSCAN clustering with molecular interactions analysis +- **`outpost_modules/`**: Extended analysis modules including binding event analysis, rendering, and Ripley's K analysis +- **`ripleys_analysis/`**: Dedicated Ripley's K spatial analysis functionality +- **`spinna*.py`**: SPINNA (Spatial Point Pattern Analysis) related modules for MLE fitting + +### Standard Workflows +- **`standard_singledataset_workflows.py`**: Pre-built single dataset analysis workflows +- **`standard_aggregation_workflows.py`**: Pre-built aggregation analysis workflows + +## Development Commands + +### Testing +```bash +# Run all tests +pytest -v + +# Run specific test file +pytest -v picasso_workflow/tests/test_workflow.py + +# Run tests with coverage +pytest --cov=picasso_workflow +``` + +### Code Quality +```bash +# Install and run pre-commit hooks +pip install pre-commit +pre-commit install +pre-commit run --all-files + +# Manual linting (based on pyproject.toml config) +black picasso_workflow/ +isort picasso_workflow/ +flake8 picasso_workflow/ +``` + +### Installation +```bash +# Development installation +pip install -e . + +# With cluster support +pip install -e .[cluster] +``` + +## Configuration + +### Environment Variables +- `CONFLUENCE_TOKEN`: API token for operational Confluence integration (legacy + alias `CONFLUENCE_BEARER`). The token is *only* ever an env var — never in + `config.yaml`, generated scripts, or logs. +- `TEST_CONFLUENCE_TOKEN`: API token for the pytest suite's Confluence tests. +- `DRIVEPATHS`: Drive path mappings for multi-machine compatibility (format: "machine:path;machine:path") + +Non-secret Confluence connection settings (URL/Space/DefaultPage/Username) live +in `config.yaml` under `Confluence` (operational) and `ConfluenceTest` (tests); +all credentials resolve through `confluence.resolve_confluence_credentials(profile)`. + +### Code Style +- Line length: 79 characters (Black) / 79 characters (Flake8) +- Uses Black, isort, flake8, mypy, and bandit for code quality +- Docstring convention: NumPy style (numpydoc) — one-line imperative + summary, then `Parameters` / `Returns` / `Raises` / `Notes` sections with + dashed underlines and `name : type` fields. Pair with PEP 604 type hints in + signatures (`from __future__ import annotations`); don't restate a + parameter's type in prose when the annotation already gives it. Matches the + upstream `picasso` package. +- Test coverage requirement: 80% + +## Testing Structure + +Tests are located in `picasso_workflow/tests/` with test data in `TestData/`: +- Unit tests mock dependencies to test individual modules +- Integration tests in `test_z_integration.py` test full workflows +- Test data includes sample DNA-PAINT datasets for realistic testing + +## Key Dependencies + +- **picassosr**: Core analysis engine (>=0.7.3) +- **atlassian-python-api**: Confluence integration +- **mpi4py**: Cluster computing support (optional) +- **scipy**, **numpy**, **pandas**: Scientific computing +- **matplotlib**, **seaborn**: Visualization +- **aicsimageio**: Image I/O for microscopy formats + +## Working inside the Claude Code sandbox + +The default Claude Code sandbox blocks most of the things this package's import chain wants to do. Don't rediscover these every time — work around them up front. + +### `import picasso_workflow` fails with a numba cache error + +The full import pulls `picasso.postprocess`, which calls `@numba.njit(cache=True)` at module load. Numba then tries to write a cache file next to `postprocess.py` inside the installed `picassosr` package; the sandbox denies the write and raises: + +``` +RuntimeError: cannot cache function '_pick_similar': no locator available +for file '/.../site-packages/picasso/postprocess.py' +``` + +This has nothing to do with the code under test — `picasso_workflow/__init__.py` dies at line ~16 (`from picasso_workflow.workflow import ...`) before any of its own logic runs. + +**Workaround for testing `__init__.py` side-effects** (config loading, `.env` discovery, logger setup): stub out the heavy submodules before importing the package, e.g. + +```python +import sys, types +sys.path.insert(0, '/Users/hgrabmayr/GitHub/picasso-workflow') +for name in ('picasso_workflow.workflow', + 'picasso_workflow.standard_singledataset_workflows', + 'picasso_workflow.standard_aggregation_workflows'): + mod = types.ModuleType(name) + if name.endswith('.workflow'): + class _W: pass + mod.WorkflowRunner = _W + mod.AggregationWorkflowRunner = _W + sys.modules[name] = mod +mod = types.ModuleType('picasso_workflow._version') +mod.__version__ = 'test' +sys.modules['picasso_workflow._version'] = mod + +import picasso_workflow # now runs __init__.py without touching numba +``` + +For anything that genuinely needs `picasso.*` (workflow, analyse, …), run the test outside the sandbox or have the user run it. + +### `$HOME` is mostly read-only + +The sandbox `write` allowlist excludes most of `$HOME`. Writes to `~/.picasso_workflow/`, `~/.config/picasso_workflow/`, `~/.matplotlib/`, etc. raise `PermissionError: [Errno 1] Operation not permitted`. Writable paths include `.` (project dir), `$TMPDIR`, and a few specific dotfiles — see the sandbox config in the Bash tool description. + +**To test code that writes under `$HOME`**, redirect `HOME` to a sandbox-writable dir per-invocation: + +```bash +FAKE_HOME="$TMPDIR/fakehome" +rm -rf "$FAKE_HOME" && mkdir -p "$FAKE_HOME" +HOME="$FAKE_HOME" python my_script.py +``` + +`Path.home()` and `~` expansion both honor `$HOME`, so this transparently relocates user-config / log paths. + +### Logger configuration eats your stderr sink + +`picasso_workflow.config_logger()` calls `logger.remove()` and re-adds sinks pointing at the logfile + an ERROR-level stderr sink. If you do `logger.add(sys.stderr, level='DEBUG', ...)` **before** `import picasso_workflow`, the import wipes it. To inspect import-time log output, read the logfile after import (path is printed in the first INFO line: `~/.picasso_workflow/logs/picasso-workflow-job{SLURM_JOB_ID}-rank{SLURM_PROCID}.log`). + +### `python -c` distorts python-dotenv discovery + +`load_dotenv()` / `find_dotenv()` check `_is_interactive()` (true when `__main__` has no `__file__`) and silently switch from frame-inspection to `os.getcwd()` as the search root. In `python -c "..."` invocations, `__main__.__file__` is unset, so dotenv uses cwd and won't find the package-bundled `.env` unless cwd happens to be inside the package. **Always test dotenv behavior with a real `.py` script**, not `python -c`. + +### `cd` in compound Bash commands triggers a permission prompt + +Don't write `cd /some/dir && python ...` — the harness prompts. Use absolute paths, or set cwd-relevant env (`HOME=...`, `PYTHONPATH=...`) inline before the command. diff --git a/picasso_workflow/picasso_outpost.py b/picasso_workflow/picasso_outpost.py index 4713869..c009fcb 100644 --- a/picasso_workflow/picasso_outpost.py +++ b/picasso_workflow/picasso_outpost.py @@ -24,7 +24,15 @@ import yaml import os from datetime import datetime -from aicsimageio import AICSImage + +try: + # The Zeiss .czi reader is an optional dependency (the ``[formats]`` + # extra); it is only needed by ``convert_zeiss_file``. Keep it a module + # attribute (None when absent) so the base install imports cleanly and + # tests can still patch ``picasso_outpost.AICSImage``. + from aicsimageio import AICSImage +except ImportError: # pragma: no cover - exercised only without the extra + AICSImage = None from picasso import ( io, @@ -1905,6 +1913,12 @@ def convert_zeiss_file(filepath_czi, filepath_raw, info=None): are entered. Necessary keys: ``'Byte Order'``, ``'Camera'``, ``'Micro-Manager Metadata'``. """ + if AICSImage is None: + raise ImportError( + "Reading Zeiss .czi files requires the optional 'formats' " + "dependencies. Install them with: pip install " + '"picasso_workflow[formats]"' + ) img = AICSImage(filepath_czi) with open(filepath_raw, "wb") as f: diff --git a/picasso_workflow/start-gui_macos.command b/picasso_workflow/start-gui_macos.command index 082d82d..b72bbf4 100755 --- a/picasso_workflow/start-gui_macos.command +++ b/picasso_workflow/start-gui_macos.command @@ -9,9 +9,9 @@ cd $current_dir echo Starting picasso-workflow GUI conda activate picasso-workflow - + python gui.py echo Shutting down picasso-workflow GUI - -conda deactivate \ No newline at end of file + +conda deactivate diff --git a/picasso_workflow/tests/TestData/integration/3C_30px_1kframes_1/DisplaySettings.json b/picasso_workflow/tests/TestData/integration/3C_30px_1kframes_1/DisplaySettings.json index 993eee7..28dfcd3 100755 --- a/picasso_workflow/tests/TestData/integration/3C_30px_1kframes_1/DisplaySettings.json +++ b/picasso_workflow/tests/TestData/integration/3C_30px_1kframes_1/DisplaySettings.json @@ -685,4 +685,4 @@ "scalar": 2.0 } } -} \ No newline at end of file +} diff --git a/picasso_workflow/tests/TestData/integration/3C_30px_1kframes_1/comments.txt b/picasso_workflow/tests/TestData/integration/3C_30px_1kframes_1/comments.txt index b8aac32..fa9b519 100755 --- a/picasso_workflow/tests/TestData/integration/3C_30px_1kframes_1/comments.txt +++ b/picasso_workflow/tests/TestData/integration/3C_30px_1kframes_1/comments.txt @@ -14,4 +14,4 @@ } } } -} \ No newline at end of file +} diff --git a/picasso_workflow/tests/TestData/integration/3C_30px_1kframes_shifted_1/DisplaySettings.json b/picasso_workflow/tests/TestData/integration/3C_30px_1kframes_shifted_1/DisplaySettings.json index 0e53af4..edb0e2e 100755 --- a/picasso_workflow/tests/TestData/integration/3C_30px_1kframes_shifted_1/DisplaySettings.json +++ b/picasso_workflow/tests/TestData/integration/3C_30px_1kframes_shifted_1/DisplaySettings.json @@ -685,4 +685,4 @@ "scalar": 2.0 } } -} \ No newline at end of file +} diff --git a/picasso_workflow/tests/TestData/integration/3C_30px_1kframes_shifted_1/comments.txt b/picasso_workflow/tests/TestData/integration/3C_30px_1kframes_shifted_1/comments.txt index b8aac32..fa9b519 100755 --- a/picasso_workflow/tests/TestData/integration/3C_30px_1kframes_shifted_1/comments.txt +++ b/picasso_workflow/tests/TestData/integration/3C_30px_1kframes_shifted_1/comments.txt @@ -14,4 +14,4 @@ } } } -} \ No newline at end of file +} diff --git a/pyproject.toml b/pyproject.toml index 8481909..a564385 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,9 +32,6 @@ dependencies = [ # "coverage>=6.5,<6.6", "pytest==8.1.1", "isort==5.13.2", - "aicspylibczi>=3.1.1", - "fsspec>=2022.8.0", - "aicsimageio", "psutil", "atlassian-python-api", "memory-profiler>=0.61.0", @@ -53,6 +50,17 @@ picasso-workflow-gui = "picasso_workflow._launcher:main" cluster = [ "mpi4py>=4.0.2", ] +# Zeiss .czi reader used only by the optional ``convert_zeiss_movie`` module +# (picasso_outpost.convert_zeiss_file). aicspylibczi has no aarch64 wheels and +# source-builds via a C++/cmake toolchain, and aicsimageio drags an old +# imagecodecs that fails to build on the py3.10 arm64 container -- so the CZI +# stack lives here rather than in the base deps, keeping ``pip install -e .`` +# wheel-only. Install with ``pip install -e ".[formats]"`` to enable CZI import. +formats = [ + "aicsimageio", + "aicspylibczi>=3.1.1", + "fsspec>=2022.8.0", +] [tool.setuptools] packages = ["picasso_workflow"] @@ -69,6 +77,7 @@ fallback_version = "0.3.3.dev0" picasso_workflow = ["picasso-workflow.ico"] [tool.black] +target-version = ["py310"] line-length = 79 include = "\\.pyi?$" exclude = ''' @@ -98,7 +107,7 @@ extend-ignore = "E203,E501,W503" # extend-exclude keeps flake8's sensible defaults and additionally skips the # generated/snapshot test fixtures and the isolated experimental modules # (not imported by the package) that still carry WIP F-codes. -extend-exclude = "docs,picasso_workflow/spinna_mle.py,picasso_workflow/spinna_mle_2.py,picasso_workflow/nn_redistribution.py,picasso_workflow/tests/TestData" +extend-exclude = "docs,picasso_workflow/_version.py,picasso_workflow/spinna_mle.py,picasso_workflow/spinna_mle_2.py,picasso_workflow/nn_redistribution.py,picasso_workflow/tests/TestData" [tool.bandit] exclude = "/tests"