diff --git a/CLAUDE.md b/CLAUDE.md index 89496fb..6c0f6dd 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -83,6 +83,9 @@ clean-up-workspaces.py # Cron-style cleanup of stale workspace director docker/entrypoint.sh # Canonical container entrypoint (copied to /app/entrypoint.sh in the image); entrypoint.sh # auto-detects a read-only root (apptainer/HPC) and moves runtime state to /tmp gdpr_consent/ # GDPR consent Streamlit component (prebuilt JS bundle) +mq_dir_upload/ # JS folder-picker shim: filters a picked MaxQuant folder to the PTXQC-relevant + # files and injects ONLY those into a native st.file_uploader (HTTP transport, + # scales to GBs; never uses setComponentValue). Python wrapper: src/common/mq_dir_upload.py docs/ # User/developer + deployment docs k8s/ # Kustomize: base/ + components/memory-tier-{low,high}/ + overlays/prod/ Dockerfile_simple # The single shipped image (linux/amd64): pyOpenMS (pip) + R + PTXQC + pandoc @@ -110,7 +113,7 @@ Pages are registered in `app.py` under named sidebar sections (the dict key is t The whole app is one `WorkflowManager` subclass implementing the four template methods. This is the example to copy when changing app behavior: -- `upload()` — a reactive **Data type** selector (`MaxQuant directory` / `MaxQuant files` / `mzTab file`) drives which `self.ui.upload_widget(...)` is shown (folder upload vs. multi-file vs. single). +- `upload()` — a reactive **Data type** selector (`MaxQuant directory` / `MaxQuant files` / `mzTab file`) drives which uploader is shown. `MaxQuant files` / `mzTab file` use `self.ui.upload_widget(...)` (multi-file / single). `MaxQuant directory` uses `src/common/mq_dir_upload.filtered_directory_upload`: the `mq_dir_upload` JS **shim** lets the user pick a whole folder but injects **only** the PTXQC-relevant files into a native `st.file_uploader` (so multi-GB `allPeptides.txt` etc. never leave the user's machine — the stock directory uploader filters by extension only, which wouldn't). The native uploader transfers over HTTP (scales; a base64-over-websocket component value does **not** — it silently drops at a few hundred MB, the size real evidence/msms files reach). The wrapper mirrors the uploaded files into the same `input-files/txt-files/` dir the widget would, so `execution()`/`_gather_files` are unchanged. Note: `Workflow.show_file_upload_section()` is overridden to drop the template's generic "⬇️ Download files" button. - `configure()` (`@st.fragment`) — either an "upload a YAML config" path, or the manual advanced-settings grid built from the module-level `NUMBER_WIDGETS` list + a contaminants text field + a metrics multiselect populated from the installed PTXQC version. Degrades gracefully (a warning, defaults only) when R/PTXQC is unavailable. - `execution()` — gathers inputs, writes a JSON run-config (via `ptxqc_config.build_run_config`), then runs **R**: `self.executor.run_command(["Rscript", cfg.RUNNER, "run", "--config", ..., "--in", ..., "--type", "maxquant"|"mztab", "--out", ...])`. Appends a row to the shared usage log afterward. - `results()` (`@st.fragment`) — reads the wrapper's `ptxqc_result.json`, embeds the report HTML via `st.components.v1.html(...)` (with an injected "open in new tab" button), and offers PDF/HTML/YAML/log downloads. diff --git a/Dockerfile_simple b/Dockerfile_simple index 0f0ee47..92f8a69 100644 --- a/Dockerfile_simple +++ b/Dockerfile_simple @@ -119,6 +119,7 @@ COPY content/ /app/content COPY docs/ /app/docs COPY example-data/ /app/example-data COPY gdpr_consent/ /app/gdpr_consent +COPY mq_dir_upload/ /app/mq_dir_upload COPY hooks/ /app/hooks COPY src/ /app/src COPY utils/ /app/utils diff --git a/README.md b/README.md index e5358f8..adad2e4 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,57 @@ long-term maintainability. A hosted instance is available at **[ptxqc.webapps.openms.org](https://ptxqc.webapps.openms.org)**. +## 💻 Run locally (without Docker) + +You can run the app straight from a checkout. Two layers are involved: the **Python/Streamlit** +front end and the **R/PTXQC** engine that actually builds the reports. + +### 1. Python front end (minimum to launch the app) + +```bash +git clone https://github.com/BioinformaticsSolutionCenter/PTXQC-web.git +cd PTXQC-web +python -m venv .venv && source .venv/bin/activate # Windows: .venv\Scripts\activate +pip install -r requirements.txt +streamlit run app.py # opens http://localhost:8501 +``` + +This is enough to launch every page. The committed `settings.json` runs in **local mode** +(single user, no captcha, no Redis), so nothing else is required to start. + +> **Note:** `pip install` only gives you **pyOpenMS** — it does **not** install R or PTXQC. Without +> them the app still loads and degrades gracefully: the *Configure* page warns, the live metric +> list is empty, and actually running a report fails. To generate reports, install the R engine +> below. + +### 2. R engine (needed to generate reports) + +Install **R** (4.x) and **pandoc** (used for the HTML report), then install the PTXQC package and +the two helper packages the runner uses: + +```bash +Rscript -e 'install.packages(c("PTXQC", "jsonlite", "yaml"), repos = "https://cloud.r-project.org")' +``` + +- **Windows / macOS** get precompiled CRAN binaries, so this is quick and needs no extra system + libraries. Install R from and pandoc from + (or `winget install RProject.R pandoc` / + `brew install r pandoc`). +- **Linux** compiles PTXQC's dependencies from source unless you use precompiled binaries. The + fast path (same one the Docker build uses) is Posit Public Package Manager plus the system + `-dev` libraries — see the `install.packages` step and the `apt-get install` list in + [`Dockerfile_simple`](Dockerfile_simple) for the exact repository URL and package names. + +Make sure `Rscript` is on your `PATH` (the app shells out to `Rscript src/ptxqc_runner.R …`). +Verify the engine is visible to the app with: + +```bash +Rscript src/ptxqc_runner.R default-config --out /tmp/ptxqc-default.yaml +``` + +If that writes a YAML file, the *Configure* page will show the live metric list and report +generation will work. + ## 🐳 Run with Docker The app ships as a single prebuilt image (linux/amd64) on the GitHub Container Registry, so you diff --git a/app.py b/app.py index 2d4d4a2..3af57e9 100644 --- a/app.py +++ b/app.py @@ -11,10 +11,10 @@ if __name__ == '__main__': pages = { str(st.session_state.settings["app-name"]): [ - st.Page(Path("content", "ptxqc_upload.py"), title="Upload Data", icon="📁"), - st.Page(Path("content", "ptxqc_configure.py"), title="Configure", icon="⚙️"), - st.Page(Path("content", "ptxqc_run.py"), title="Create Report", icon="🚀"), - st.Page(Path("content", "ptxqc_results.py"), title="Report", icon="📊"), + st.Page(Path("content", "ptxqc_upload.py"), title="1. Upload Data", icon="📁"), + st.Page(Path("content", "ptxqc_configure.py"), title="2. Configure", icon="⚙️"), + st.Page(Path("content", "ptxqc_run.py"), title="3. Create Report", icon="🚀"), + st.Page(Path("content", "ptxqc_results.py"), title="4. Report", icon="📊"), ], "Info": [ st.Page(Path("content", "help.py"), title="Help", icon="❓"), diff --git a/content/ptxqc_configure.py b/content/ptxqc_configure.py index b6ef5ba..5893a7a 100644 --- a/content/ptxqc_configure.py +++ b/content/ptxqc_configure.py @@ -6,3 +6,8 @@ wf = Workflow() wf.show_parameter_section() + +# Configuration is optional — always offer the link to step 3. +import streamlit as st +st.divider() +st.page_link("content/ptxqc_run.py", label="Next: **3. Create Report**", icon="🚀") diff --git a/content/ptxqc_run.py b/content/ptxqc_run.py index 0bd44cb..41bc55b 100644 --- a/content/ptxqc_run.py +++ b/content/ptxqc_run.py @@ -6,3 +6,9 @@ wf = Workflow() wf.show_execution_section() + +# Once a report exists, link to step 4 to view it. +if wf.has_report(): + import streamlit as st + st.success("✅ Report ready. Next step:") + st.page_link("content/ptxqc_results.py", label="**4. Report**", icon="📊") diff --git a/content/ptxqc_upload.py b/content/ptxqc_upload.py index a9927f9..330562f 100644 --- a/content/ptxqc_upload.py +++ b/content/ptxqc_upload.py @@ -6,3 +6,9 @@ wf = Workflow() wf.show_file_upload_section() + +# Step 1 done once data is staged → offer the link to step 2. +if wf.has_inputs(): + import streamlit as st + st.success("✅ Data uploaded. Next step:") + st.page_link("content/ptxqc_configure.py", label="**2. Configure**", icon="⚙️") diff --git a/docs/adr/0001-gui-tests-drive-real-navigation.md b/docs/adr/0001-gui-tests-drive-real-navigation.md new file mode 100644 index 0000000..a255939 --- /dev/null +++ b/docs/adr/0001-gui-tests-drive-real-navigation.md @@ -0,0 +1,23 @@ +# GUI tests drive the real `app.py` navigation shell + +`test_gui.py` boots the actual app via `AppTest.from_file("app.py")` (which runs +`st.navigation`) and then `switch_page`s to each page, instead of loading each +page standalone with `AppTest.from_file("content/.py")`. + +We do this because `st.page_link` resolves its target through the page registry +that `st.navigation` builds. A standalone page run never calls `st.navigation`, +so `st.page_link` raises `KeyError('url_pathname')` — a failure that only exists +in the test, not in production. The tempting "fix" is to wrap every `page_link` +in `try/except`, but that both papers over a test that isn't running pages the +way they actually run *and* silently swallows a genuinely broken page link (a +typo'd target passes CI green). Driving `app.py` + `switch_page` runs pages in +their real navigation context, so `page_link` works without guards and a broken +link fails the test. + +## Consequences + +- No defensive `try/except` around `st.page_link` in the `content/` pages. +- Do **not** "simplify" `test_gui.py` back to per-page `AppTest.from_file` — it + reintroduces the `KeyError`, the guards, and the silent bug-hiding. +- The fixture must inject `settings["test"] = True` (captcha bypass) and a + `workspace` secret, since it now goes through the app's real entrypoint. diff --git a/docs/adr/0002-keep-template-layer-app-agnostic.md b/docs/adr/0002-keep-template-layer-app-agnostic.md new file mode 100644 index 0000000..9e215dd --- /dev/null +++ b/docs/adr/0002-keep-template-layer-app-agnostic.md @@ -0,0 +1,20 @@ +# Keep the inherited template layer (`src/workflow/`) app-agnostic + +`src/workflow/` is vendored from the OpenMS streamlit-template and is merged from +the `template/main` remote periodically. App-specific behaviour must **not** be +added to those files; it lives in the app layer (`src/Workflow.py`, `content/`, +`src/*.py`). Otherwise every template sync produces conflicts and the shared +framework accumulates one app's concerns. + +Concretely: report-generation preconditions such as "is `Rscript` present?" are +guarded in `Workflow.execution()` (app layer, with an app-appropriate "install R +/ use Docker" message), **not** in `CommandExecutor.run_command()`. A `try/except` +that caught a missing executable and logged a PTXQC/R-specific message was removed +from `CommandExecutor.py` for this reason — it was both redundant with the +`execution()` guard and a source of divergence from `template/main`. + +## Consequences + +- Precondition checks and user-facing error messages belong in the app layer. +- Changes under `src/workflow/` should be limited to what can be upstreamed to the + template; keep these files diff-clean against `template/main` where possible. diff --git a/mq_dir_upload/README.md b/mq_dir_upload/README.md new file mode 100644 index 0000000..8baea92 --- /dev/null +++ b/mq_dir_upload/README.md @@ -0,0 +1,50 @@ +# mq_dir_upload — MaxQuant folder-picker shim + +A tiny bidirectional Streamlit custom component that lets the user pick a whole +MaxQuant `txt` folder but upload **only** the PTXQC-relevant files +(`evidence.txt`, `msms.txt`, `msmsScans.txt`, `parameters.txt`, +`proteinGroups.txt`, `summary.txt`, `mqpar.xml`). The large irrelevant MaxQuant +outputs (`allPeptides.txt`, `peptides.txt`, …) never leave the user's machine. + +## How it works (and why it's a *shim*) + +Streamlit's built-in `st.file_uploader` directory mode filters by file +**extension** only, so it can't drop `allPeptides.txt` (it's a `.txt`). +Filtering by file *name* must happen in the browser — which needs JavaScript. + +The naive custom-component approach (read the files, base64-encode them, and +return them to Python via `setComponentValue`) **does not scale**: that value +travels over the websocket, which chokes on the hundreds-of-MB sizes real +MaxQuant files reach (the value is silently dropped → nothing uploads). The +relevant files (`evidence`/`msms`/`msmsScans`) are themselves the big ones, so +this isn't avoidable by filtering. + +So this component is a **shim**, not an uploader. On a folder pick it: + +1. filters the browser `FileList` down to the allow-listed names (in JS), then +2. injects *only* those `File` objects into the page's native + `st.file_uploader` (`DataTransfer` → set `input.files` → dispatch a bubbling + `change` event). + +Streamlit's native uploader then transfers them over its **HTTP** endpoint +(handles GBs, shows a file list / sizes / progress / per-file remove). The shim +**never calls `setComponentValue`** — the heavy bytes never touch the websocket. + +It must be a *declared* component (not `components.html`) so it runs +**same-origin** and can reach `window.parent.document` to find that native +uploader input. The Python side (`src/common/mq_dir_upload.py`) renders the +native uploader (drag-drop area hidden via CSS), mirrors its files into the +workspace dir, and passes `hidden=True` to the shim once files are staged +(uploading more into an existing set is disallowed; the user clears first). + +## Files + +- `index.html` — loads `main.js`. +- `main.js` — **hand-written, no build step.** Speaks the Streamlit iframe + `postMessage` protocol directly (`componentReady` / `render` / + `setFrameHeight`) — there is nothing to `npm build`. Edit `main.js` and reload. + +## Args (from Python) + +- `allowed`: list of relevant file names (matched case-insensitively on basename). +- `hidden`: when true, the picker button is hidden (files already staged). diff --git a/mq_dir_upload/index.html b/mq_dir_upload/index.html new file mode 100644 index 0000000..69f30c4 --- /dev/null +++ b/mq_dir_upload/index.html @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/mq_dir_upload/main.js b/mq_dir_upload/main.js new file mode 100644 index 0000000..ae298f0 --- /dev/null +++ b/mq_dir_upload/main.js @@ -0,0 +1,139 @@ +/* + * mq_dir_upload — MaxQuant "txt" folder-picker shim. + * + * The user picks a whole MaxQuant txt folder; this shim filters the browser's + * FileList down to the PTXQC-relevant files (by name) and injects ONLY those + * into the page's native st.file_uploader (via DataTransfer + a change event). + * Streamlit then uploads them over its HTTP endpoint — so large files work and + * the irrelevant files (allPeptides.txt, …) never leave the machine. + * + * Why a shim instead of returning the bytes: a custom component can only send + * data back over the websocket (setComponentValue), which chokes on hundreds of + * MB. The native uploader uses a separate HTTP transport that scales. This shim + * deliberately never calls setComponentValue — it only orchestrates the native + * widget. It must be a *declared* component (not components.html) so it runs + * same-origin and can reach window.parent.document. + */ +(function () { + "use strict"; + + function post(t, d) { + var m = { isStreamlitMessage: true, type: t }; + for (var k in d) m[k] = d[k]; + window.parent.postMessage(m, "*"); + } + function h(px) { post("streamlit:setFrameHeight", { height: px }); } + function basename(n) { return (n || "").split(/[\\/]/).pop(); } + + var built = false, allowed = [], hidden = false, label, input, status; + + // The native st.file_uploader's file input, in the parent document. On the + // upload page there is exactly one file_uploader, so this is unambiguous. + function nativeInput() { + var pdoc = window.parent.document; + return ( + pdoc.querySelector('[data-testid="stFileUploader"] input[type="file"]') || + pdoc.querySelector('section[data-testid="stFileUploaderDropzone"] input[type="file"]') || + pdoc.querySelector('input[type="file"]') + ); + } + + function onPick(ev) { + var all = Array.prototype.slice.call(ev.target.files || []); + if (!all.length) return; + + var allow = {}; + allowed.forEach(function (n) { allow[n.toLowerCase()] = true; }); + + // De-dupe by basename (a MaxQuant export may nest combined/txt). + var picked = {}, order = []; + all.forEach(function (f) { + var lc = basename(f.name).toLowerCase(); + if (allow[lc] && !picked[lc]) { picked[lc] = f; order.push(lc); } + }); + var matched = order.map(function (lc) { return picked[lc]; }); + + if (!matched.length) { + status.style.color = "#b00"; + status.textContent = + "No PTXQC-relevant files found in that folder (scanned " + all.length + ")."; + ev.target.value = ""; + h(90); + return; + } + + var nat = nativeInput(); + if (!nat) { + status.style.color = "#b00"; + status.textContent = "Could not find the upload target — please reload the page."; + h(90); + return; + } + + // Hand the filtered File objects to the native uploader and let Streamlit + // upload them over HTTP. react-dropzone listens for a bubbling change event. + var dt = new DataTransfer(); + matched.forEach(function (f) { dt.items.add(f); }); + nat.files = dt.files; + nat.dispatchEvent(new Event("input", { bubbles: true })); + nat.dispatchEvent(new Event("change", { bubbles: true })); + + status.style.color = "#444"; + status.textContent = "Uploading " + matched.length + " file(s)…"; + ev.target.value = ""; + h(90); + } + + function build() { + label = document.createElement("label"); + label.textContent = "📁 Select MaxQuant txt folder"; + label.style.cssText = + "display:inline-block;padding:.5rem 1rem;background:#ff4b4b;color:#fff;" + + "border-radius:.5rem;cursor:pointer;font-weight:600;line-height:1.4;" + + "font-family:'Source Sans Pro',sans-serif;"; + + input = document.createElement("input"); + input.type = "file"; + input.multiple = true; + input.setAttribute("webkitdirectory", ""); + input.setAttribute("directory", ""); + input.webkitdirectory = true; + input.style.display = "none"; + input.addEventListener("change", onPick); + label.appendChild(input); + + status = document.createElement("div"); + status.style.cssText = + "margin-top:.5rem;font-size:.85rem;color:#444;line-height:1.4;" + + "font-family:'Source Sans Pro',sans-serif;"; + + document.body.style.margin = "0"; + document.body.appendChild(label); + document.body.appendChild(status); + built = true; + } + + function apply() { + if (!built) return; + // Hidden once files are staged: uploading more into an existing set is not + // allowed — the user must clear first. + label.style.display = hidden ? "none" : "inline-block"; + status.textContent = hidden + ? "" + : "Only the PTXQC-relevant files are uploaded; everything else stays on your machine."; + h(hidden ? 1 : 90); + } + + window.addEventListener("message", function (ev) { + var d = ev.data; + if (!d || d.type !== "streamlit:render") return; + if (!built) build(); + var a = d.args || {}; + if (Array.isArray(a.allowed)) allowed = a.allowed; + hidden = !!a.hidden; + apply(); + }); + + post("streamlit:componentReady", { apiVersion: 1 }); + h(90); +})(); diff --git a/src/Workflow.py b/src/Workflow.py index 6b528bb..ad3d16d 100644 --- a/src/Workflow.py +++ b/src/Workflow.py @@ -15,6 +15,7 @@ import streamlit.components.v1 as components from src.workflow.WorkflowManager import WorkflowManager +from src.common.mq_dir_upload import filtered_directory_upload from src import ptxqc_config as cfg # Map the data-type selector to (input-files key, accepted extensions, upload mode). @@ -79,6 +80,33 @@ def _gather_files(self, files_dir: Path) -> list[str]: if line.strip() and os.path.exists(line.strip())] return out + def show_file_upload_section(self) -> None: + """Render the upload section *without* the template's generic + '⬇️ Download files' button (not useful for this app — finding feedback).""" + self.upload() + + def has_inputs(self) -> bool: + """True if input files are staged for the currently selected data type. + Used by the upload page to reveal the 'next step' link once data is in.""" + # Read the LIVE selector value (same source as upload()), not the persisted + # self.params, so the check stays consistent right after switching data type. + selected = st.session_state.get(self._pk("input-type"), "MaxQuant directory") + key, _ftypes, _mode = INPUT_TYPES[selected] + return bool(self._gather_files(Path(self.workflow_dir, "input-files", key))) + + def has_report(self) -> bool: + """True once a *successful* report has been generated. The R wrapper writes + ptxqc_result.json even on failure (with an `error` field), so existence alone + isn't enough — require no error and an actual HTML/PDF artifact.""" + f = Path(self.workflow_dir, "results", "qc-report", "ptxqc_result.json") + if not f.exists(): + return False + try: + res = json.loads(f.read_text()) + except (ValueError, OSError): + return False + return not res.get("error") and bool(res.get("html") or res.get("pdf")) + # ----- sections ------------------------------------------------------ def upload(self) -> None: self.ui.input_widget( @@ -97,12 +125,18 @@ def upload(self) -> None: if mode == "directory": st.info( - "Select your MaxQuant **txt** folder — your browser uploads its files " - "and only the PTXQC-relevant ones (Evidence, msms, summary, parameters, " - "proteinGroups, msmsScans, mqpar.xml) are used." + "Select your MaxQuant **txt** folder. Your browser reads the whole folder " + "but uploads **only** the PTXQC-relevant files (evidence, msms, msmsScans, " + "parameters, proteinGroups, summary, mqpar.xml) — large files such as " + "allPeptides.txt are never read and stay on your machine." ) - self.ui.upload_widget(key=key, name="MaxQuant txt folder", file_types=ftypes, directory=True) + filtered_directory_upload(Path(self.workflow_dir, "input-files", key)) elif mode == "multi": + st.info( + "Upload only the PTXQC-relevant MaxQuant files " + "(**evidence**, **msms**, **msmsScans**, **parameters**, **proteinGroups**, " + "**summary**, **mqpar.xml**). Any other files are ignored." + ) self.ui.upload_widget(key=key, name="MaxQuant txt files", file_types=ftypes) else: self.ui.upload_widget(key=key, name="mzTab file", file_types=ftypes) @@ -126,6 +160,17 @@ def configure(self) -> None: self.ui.upload_widget(key="yaml-config", name="PTXQC YAML config", file_types=["yaml", "yml"]) return + # The "Show advanced parameters" toggle (rendered by parameter_section) + # gates the threshold grid. Off = PTXQC defaults are used as-is. + if not st.session_state.get("advanced", False): + st.caption( + "Using PTXQC's default thresholds and all metrics. Turn on " + "**Show advanced parameters** above to customise the ID-rate bands, " + "protein/peptide count targets, mass-error tolerances, match-between-runs, " + "the contaminants list, and which metrics to compute." + ) + return + st.markdown("##### Advanced settings") cols = st.columns(3) for i, (key, default, label, help_text, lo, hi) in enumerate(NUMBER_WIDGETS): @@ -166,6 +211,20 @@ def execution(self) -> bool: self.logger.log("ERROR: No input files provided.") raise RuntimeError("No input files provided.") + # Generating a report needs R/PTXQC. rscript_path() also looks in the + # standard Windows R install dir (the installer doesn't add R to PATH). + # When still not found, fail with a clear message and no traceback — + # upload and configuration work fine without R. + rscript = cfg.rscript_path() + if shutil.which(rscript) is None and not os.path.exists(rscript): + self.logger.log( + "ERROR: 'Rscript' was not found. Generating a PTXQC report requires R " + "and the PTXQC package. Install R (see the README's 'Run locally " + "(without Docker)' section), put it on PATH or set the PTXQC_RSCRIPT " + "env var, or use the Docker image. Uploading and tuning work without R." + ) + return False + # Stage inputs into the results dir; PTXQC writes its outputs alongside them. rundir = Path(self.workflow_dir, "results", "qc-report") rundir.mkdir(parents=True, exist_ok=True) @@ -197,12 +256,12 @@ def execution(self) -> bool: # version. run_command returns False (never raises) on failure, so a missing # network / read-only library just falls through to the installed version. self.logger.log("Updating PTXQC to the latest release (Posit PPM)...") - if not self.executor.run_command(["Rscript", cfg.RUNNER, "update"]): + if not self.executor.run_command([cfg.rscript_path(), cfg.RUNNER, "update"]): self.logger.log("WARNING: PTXQC update failed; using the currently installed version.") self.logger.log(f"Generating PTXQC report for {len(in_files)} input file(s) ({selected})...") ok = self.executor.run_command( - ["Rscript", cfg.RUNNER, "run", + [cfg.rscript_path(), cfg.RUNNER, "run", "--config", str(cfg_path), "--in", in_arg, "--type", rtype, "--out", str(rundir)] ) @@ -245,12 +304,43 @@ def results(self) -> None: "Please contact the PTXQC authors: https://github.com/cbielow/PTXQC" ) + # Downloads + actions go at the TOP so they're visible without scrolling + # past the tall embedded report. + st.markdown("##### Downloads") + d_cols = st.columns(4) + for col, (label, path) in zip(d_cols, [ + ("PDF report", res.get("pdf")), + ("HTML report", res.get("html")), + ("YAML config", res.get("yaml")), + ("Log file", res.get("log")), + ]): + if path and Path(path).exists(): + with open(path, "rb") as f: + col.download_button(label, f, file_name=Path(path).name, use_container_width=True) + + # Open a fresh session (new browser tab) on the UPLOAD page. This page + # (Report) is served at "/ptxqc_results"; a relative href of "." + # resolves to that path's parent directory — the app root, which is the + # default (Upload) page — and drops the ?workspace query, so a hosted + # deployment hands the new tab a brand-new workspace. (href="?" would keep + # the /ptxqc_results path and just reopen the Report page.) Works even + # under a server baseUrlPath. The old report stays open in this tab. + st.markdown( + '' + '➕ Create new report (opens a fresh upload page in a new tab)', + unsafe_allow_html=True, + ) + + st.markdown("---") + html_path = res.get("html") if html_path and Path(html_path).exists(): report_html = Path(html_path).read_text(encoding="utf-8", errors="replace") st.caption( "Interactive QC report below. Use **⤢ Open in a new tab** (top-right of the " - "report) or the **HTML report** download to view it full-screen." + "report) or the **HTML report** download above to view it full-screen." ) # The PTXQC report is a self-contained HTML document, so embed it directly. # (Streamlit static serving returns text/plain + nosniff for .html, which makes @@ -274,19 +364,14 @@ def results(self) -> None: """ components.html(report_html + popout, height=900, scrolling=True) - - st.markdown("##### Downloads") - d_cols = st.columns(4) - for col, (label, path) in zip(d_cols, [ - ("PDF report", res.get("pdf")), - ("HTML report", res.get("html")), - ("YAML config", res.get("yaml")), - ("Log file", res.get("log")), - ]): - if path and Path(path).exists(): - with open(path, "rb") as f: - col.download_button(label, f, file_name=Path(path).name, use_container_width=True) - - if st.button("Create new report"): - shutil.rmtree(rundir, ignore_errors=True) - st.rerun() + else: + # PTXQC produced no HTML (only the PDF). The interactive HTML report is + # rendered via pandoc; the Docker image ships it, but a local R install + # often lacks it. Explain rather than showing a blank area. + st.info( + "ℹ️ The interactive **HTML** report isn't available for this run — " + "PTXQC produced only the PDF. The HTML report is rendered with " + "**pandoc**, which the Docker image includes but a local install may " + "lack. Download the **PDF report** above, or install pandoc " + "(`winget install JohnMacFarlane.Pandoc`) and re-run for the interactive HTML." + ) diff --git a/src/common/common.py b/src/common/common.py index 63d845a..da5a30e 100644 --- a/src/common/common.py +++ b/src/common/common.py @@ -768,7 +768,9 @@ def change_workspace(): if st.session_state.settings.get("online_deployment", False): monitor_queue() - # Display OpenMS WebApp Template Version from settings.json + # Display the app name + the installed PTXQC version (read dynamically + # via the R wrapper; 'unknown' when R/PTXQC is unavailable, e.g. local + # dev). NOT the static settings.json "version" (a placeholder). with st.container(): st.markdown( """ @@ -786,10 +788,12 @@ def change_workspace(): """, unsafe_allow_html=True, ) - version_info = st.session_state.settings["version"] + from src.ptxqc_config import get_ptxqc_metadata + meta = get_ptxqc_metadata() + ver = meta["version"] if meta.get("available") else "unknown" app_name = st.session_state.settings["app-name"] st.markdown( - f'
{app_name}
Version: {version_info}
', + f'
{app_name} (version {ver})
', unsafe_allow_html=True, ) diff --git a/src/common/mq_dir_upload.py b/src/common/mq_dir_upload.py new file mode 100644 index 0000000..a33e93d --- /dev/null +++ b/src/common/mq_dir_upload.py @@ -0,0 +1,144 @@ +"""Filtered MaxQuant directory upload (folder shim + native uploader). + +The user picks a whole MaxQuant ``txt`` folder via the ``mq_dir_upload`` +JavaScript shim (repo-root ``mq_dir_upload/``). The shim filters the folder down +to the PTXQC-relevant files and injects ONLY those into a native +``st.file_uploader`` rendered here, which uploads them over Streamlit's HTTP +endpoint. The large irrelevant outputs (``allPeptides.txt``, …) never leave the +user's machine, and — unlike sending bytes back through the component websocket +— this scales to the hundreds-of-MB sizes real MaxQuant files reach. + +The uploaded files are mirrored into the same workspace upload directory that +``StreamlitUI.upload_widget(directory=True)`` would populate +(``/input-files/txt-files/``), so the rest of the workflow +(``Workflow.execution`` / ``_gather_files``) is unchanged. +""" + +from pathlib import Path + +import streamlit as st +import streamlit.components.v1 as components + +# PTXQC-relevant MaxQuant files. Matched case-insensitively on the basename in +# the JS shim; the canonical casing here is what users see in the UI. +PTXQC_FILES = [ + "evidence.txt", + "msms.txt", + "msmsScans.txt", + "parameters.txt", + "proteinGroups.txt", + "summary.txt", + "mqpar.xml", +] + +# Resolve from this file so the declaration works regardless of CWD. +_COMPONENT_DIR = Path(__file__).resolve().parents[2] / "mq_dir_upload" + +# Declare the component at MODULE IMPORT (top level), NOT lazily inside a +# function. A lazy declare_component() can raise +# "RuntimeError: module is None. This should never happen." when Streamlit +# reloads this module on a source change: declare_component's _get_module_name() +# walks the *caller* stack frame and inspect.getmodule() then fails to resolve +# the in-function frame. At module level the caller frame is this module's body, +# which resolves reliably (and re-resolves cleanly on reload). +mq_dir_component = components.declare_component("mq_dir_upload", path=str(_COMPONENT_DIR)) + + +def _fmt_size(num: float) -> str: + """Human-readable byte size.""" + if num < 1024: + return f"{int(num)} B" + for unit in ("KB", "MB", "GB"): + num /= 1024 + if num < 1024 or unit == "GB": + return f"{num:.1f} {unit}" + return f"{num:.1f} GB" + + +def _staged_files(files_dir: Path) -> list[Path]: + """Staged input files (excludes the external_files.txt sidecar).""" + if not files_dir.exists(): + return [] + return sorted( + (p for p in files_dir.iterdir() + if p.is_file() and p.name != "external_files.txt"), + key=lambda p: p.name, + ) + + +def filtered_directory_upload(files_dir: Path, key: str = "mq_dir_upload") -> None: + """Render the folder picker + native uploader and mirror files to ``files_dir``. + + The native ``st.file_uploader`` provides the file list, per-file sizes, the + upload progress bar and per-file removal for free. Its drag-and-drop area is + hidden — the folder shim is the only entry point — and the picker is hidden + once files are staged so a second pick cannot merge into an existing set. + """ + files_dir.mkdir(parents=True, exist_ok=True) + + # Hide the entire native uploader: we use it purely as the HTTP transport + # (the shim injects files into its input) and render our own complete file + # list below — Streamlit's built-in list paginates at 3/page, which is silly + # for <=7 files. + # + # NOTE: this selector is page-global — it hides EVERY stFileUploader on the + # page, not just the one below. That is fine because in "MaxQuant directory" + # mode this is the only uploader rendered (the multi-file / mzTab / YAML + # uploaders live in other data-type branches or on other pages). If a second + # uploader is ever added to the upload page, scope this rule instead. + st.markdown( + "", + unsafe_allow_html=True, + ) + + # Shim button sits above the file list; its `hidden` state depends on what + # is staged, so render it into a slot filled after the uploader is read. + shim_slot = st.container() + + # `key` carries a counter so removals can reset the uploader widget. + ctr = st.session_state.setdefault(f"{key}-ctr", 0) + uploaded = st.file_uploader( + "MaxQuant txt files", + accept_multiple_files=True, + type=["txt", "xml"], + key=f"{key}-native-{ctr}", + label_visibility="collapsed", + ) + + # `files_dir` is the durable source of truth. The uploader only ADDS files — + # its widget state resets when the user navigates away and back, so we must + # NOT delete staged files just because it returns empty (that was the + # "upload disappeared on going back" bug). Removal is explicit (below). + for f in (uploaded or []): + dest = files_dir / Path(f.name).name + if not dest.exists() or dest.stat().st_size != getattr(f, "size", -1): + dest.write_bytes(f.getbuffer()) + + staged = _staged_files(files_dir) + + with shim_slot: + mq_dir_component( + allowed=PTXQC_FILES, hidden=bool(staged), key=f"{key}-shim", default=None + ) + + if staged: + total = sum(p.stat().st_size for p in staged) + st.success(f"✅ {len(staged)} file(s) uploaded ({_fmt_size(total)}):") + for p in staged: + row = st.columns([6, 2, 1]) + row[0].markdown(f"📄 {p.name}") + row[1].markdown( + f"
{_fmt_size(p.stat().st_size)}
", + unsafe_allow_html=True, + ) + if row[2].button("✕", key=f"rm-{key}-{p.name}", help=f"Remove {p.name}"): + p.unlink() + st.session_state[f"{key}-ctr"] = ctr + 1 # reset uploader so it won't re-add + st.rerun() + if st.button( + "🗑️ Clear all", key=f"clear-{key}", use_container_width=True + ): + for p in staged: + p.unlink() + st.session_state[f"{key}-ctr"] = ctr + 1 # fresh, empty uploader + st.rerun() diff --git a/src/ptxqc_config.py b/src/ptxqc_config.py index 742d011..a41313e 100644 --- a/src/ptxqc_config.py +++ b/src/ptxqc_config.py @@ -15,7 +15,9 @@ """ import os +import glob import json +import shutil import subprocess import tempfile from datetime import date @@ -27,6 +29,37 @@ # Path to the R wrapper, relative to the app root (cwd of streamlit / the worker). RUNNER = str(Path("src", "ptxqc_runner.R")) + +def rscript_path() -> str: + """Resolve the ``Rscript`` executable to invoke. + + Order: ``PTXQC_RSCRIPT``/``RSCRIPT`` env override → ``Rscript`` on PATH → + common Windows install locations (the R installer does NOT add R to PATH, so + a plain local install is otherwise invisible to a subprocess) → bare + ``"Rscript"`` as a last resort (which then errors clearly if truly absent). + In Docker/Linux R is on PATH, so this returns the same thing as before. + """ + for env in ("PTXQC_RSCRIPT", "RSCRIPT"): + v = os.environ.get(env) + if v and Path(v).exists(): + return v + found = shutil.which("Rscript") + if found: + return found + candidates: list[str] = [] + for base in ( + r"C:\Program Files\R", + r"C:\Program Files\Microsoft\R Open", + os.path.expandvars(r"%LOCALAPPDATA%\Programs\R"), + ): + candidates += glob.glob(os.path.join(base, "R-*", "bin", "x64", "Rscript.exe")) + candidates += glob.glob(os.path.join(base, "R-*", "bin", "Rscript.exe")) + # Newest version directory first (lexicographic on R-x.y.z is good enough). + for c in sorted(candidates, reverse=True): + if os.path.exists(c): + return c + return "Rscript" + # The 13 numeric/selection parameters exposed by the original PTXQC-web advanced # settings. These names match the `param$...` keys consumed by PTXQC's createYaml # (see PTXQC-web app/server.R build.yaml); the R wrapper maps them into the YAML. @@ -84,7 +117,7 @@ def get_ptxqc_metadata() -> dict: with tempfile.NamedTemporaryFile(suffix=".yaml", delete=False) as tf: yaml_path = tf.name proc = subprocess.run( - ["Rscript", RUNNER, "default-config", "--out", yaml_path], + [rscript_path(), RUNNER, "default-config", "--out", yaml_path], capture_output=True, text=True, timeout=120, ) # The metric list / version are read from the sidecar JSON file (not stdout, diff --git a/src/workflow/StreamlitUI.py b/src/workflow/StreamlitUI.py index a303478..885a6f2 100644 --- a/src/workflow/StreamlitUI.py +++ b/src/workflow/StreamlitUI.py @@ -1398,9 +1398,6 @@ def execution_section( get_status_function=None, stop_workflow_function=None ) -> None: - with st.expander("**Summary**"): - st.markdown(self.export_parameters_markdown()) - c1, c2 = st.columns(2) # Select log level, this can be changed at run time or later without re-running the workflow log_level = c1.selectbox( @@ -1456,28 +1453,37 @@ def execution_section( # Display logs and status if is_running: - # Real-time display during execution - spinner_text = "**Workflow running...**" + # Real-time display during execution. Use a fragment that re-runs on a + # timer so ONLY the status+log area refreshes — re-running the whole + # page every second made the "Workflow running..." banner and the log + # flicker (disappear/reappear) on each cycle. + running_msg = "**Workflow running...**" if job_status == "queued": pos = status.get("queue_position", "?") - spinner_text = f"**Waiting in queue (position {pos})...**" - - with st.spinner(spinner_text): - if log_exists: - with open(log_path, "r", encoding="utf-8") as f: - lines = f.readlines() - if log_lines_count == "all": - display_lines = lines - else: - display_lines = lines[-st.session_state.log_lines_count:] - st.code( - "".join(display_lines), - language="neon", - line_numbers=False, + running_msg = f"**Waiting in queue (position {pos})...**" + + @st.fragment(run_every=2) + def _live_status(msg=running_msg): + # Re-check status inside the fragment so it can stop when the run + # finishes (then trigger one full rerun to show the final view). + cur = get_status_function() if get_status_function else {} + still_running = cur.get("running", False) + pid_alive = self.executor.pid_dir.exists() and list(self.executor.pid_dir.iterdir()) + if not still_running and pid_alive: + still_running = True + if not still_running: + st.rerun() # full rerun → render the completed/static view below + return + st.info(msg) # stable (no spinner re-animation/flicker) + if log_path.exists(): + lines = log_path.read_text(encoding="utf-8", errors="replace").splitlines(keepends=True) + display_lines = ( + lines if log_lines_count == "all" + else lines[-st.session_state.log_lines_count:] ) - # Faster polling for real-time updates - time.sleep(1) - st.rerun() + st.code("".join(display_lines), language="neon", line_numbers=False) + + _live_status() elif log_exists: # Static display after completion diff --git a/test_gui.py b/test_gui.py index 7caac3b..f402e5c 100644 --- a/test_gui.py +++ b/test_gui.py @@ -3,33 +3,43 @@ import json +# Pages to smoke-test, by their script path as registered in app.py's navigation. +PAGES = ( + "content/ptxqc_upload.py", + "content/ptxqc_configure.py", + "content/ptxqc_run.py", + "content/ptxqc_results.py", + "content/help.py", + "content/about.py", +) + + @pytest.fixture -def launch(request): - test = AppTest.from_file(request.param) +def app(): + """Boot the real app (app.py runs st.navigation) so pages are exercised inside + the same navigation shell they get in production. - ## Initialize session state ## + Testing pages standalone via AppTest.from_file("content/.py") skips + st.navigation, so st.page_link cannot resolve its target and raises + KeyError('url_pathname') — and wrapping that in try/except would also let a + genuinely broken page link pass silently. Driving app.py + switch_page tests + navigation for real. See docs/adr/0001-gui-tests-drive-real-navigation.md. + """ + at = AppTest.from_file("app.py") with open("settings.json", "r") as f: - test.session_state.settings = json.load(f) - test.session_state.settings["test"] = True - test.secrets["workspace"] = "test" - return test + at.session_state.settings = json.load(f) + at.session_state.settings["test"] = True # bypass captcha + at.secrets["workspace"] = "test" + at.run(timeout=60) + assert not at.exception, "app shell (app.py) failed to boot" + return at -# Test launching of all PTXQC pages. R/PTXQC is not installed in CI, so the -# config helper degrades gracefully — the pages must still render without error. -@pytest.mark.parametrize( - "launch", - ( - "content/ptxqc_upload.py", - "content/ptxqc_configure.py", - "content/ptxqc_run.py", - "content/ptxqc_results.py", - "content/help.py", - "content/about.py", - ), - indirect=True, -) -def test_launch(launch): - """Test if all pages can be launched without errors.""" - launch.run(timeout=30) - assert not launch.exception +# R/PTXQC is not installed in CI, so the config helper degrades gracefully — every +# page must still render without error inside the real navigation shell. +@pytest.mark.parametrize("page", PAGES) +def test_launch(app, page): + """Test that each page can be launched without errors.""" + app.switch_page(page) + app.run(timeout=60) + assert not app.exception