From 0a2f81d59b4fe01ec162d55e0b2e571dd8d58606 Mon Sep 17 00:00:00 2001 From: TinasheMTapera <15770644+TinasheMTapera@users.noreply.github.com> Date: Mon, 6 Jul 2026 02:08:23 -0400 Subject: [PATCH 1/2] developing dhs madagascar environment --- .../prospectorDHSMadagascar/00_misc.qmd | 160 ++++++++++++++- .../01_base_platform.qmd | 9 - .../01_project-contract.qmd | 98 ++++++++++ .../02_platform-recipe.qmd | 183 ++++++++++++++++++ .../prospectorDHSMadagascar/02_spack.qmd | 9 - .../03_datascience_lang.qmd | 9 - .../03_sci-sysdeps.qmd | 112 +++++++++++ .../04_datascience.qmd | 153 +++++++++++++++ .../prospectorDHSMadagascar/04_devtools.qmd | 9 - .../05_project-activation.qmd | 143 ++++++++++++++ .../prospectorDHSMadagascar/05_recipe.qmd | 9 - .../06_build-launch.qmd | 110 +++++++++++ .../prospectorDHSMadagascar/06_launch.qmd | 9 - .../prospectorDHSMadagascar/setup.qmd | 17 -- .../05_project-activation.qmd | 2 +- 15 files changed, 953 insertions(+), 79 deletions(-) delete mode 100644 notebooks/projects/prospectorDHSMadagascar/01_base_platform.qmd create mode 100644 notebooks/projects/prospectorDHSMadagascar/01_project-contract.qmd create mode 100644 notebooks/projects/prospectorDHSMadagascar/02_platform-recipe.qmd delete mode 100644 notebooks/projects/prospectorDHSMadagascar/02_spack.qmd delete mode 100644 notebooks/projects/prospectorDHSMadagascar/03_datascience_lang.qmd create mode 100644 notebooks/projects/prospectorDHSMadagascar/03_sci-sysdeps.qmd create mode 100644 notebooks/projects/prospectorDHSMadagascar/04_datascience.qmd delete mode 100644 notebooks/projects/prospectorDHSMadagascar/04_devtools.qmd create mode 100644 notebooks/projects/prospectorDHSMadagascar/05_project-activation.qmd delete mode 100644 notebooks/projects/prospectorDHSMadagascar/05_recipe.qmd create mode 100644 notebooks/projects/prospectorDHSMadagascar/06_build-launch.qmd delete mode 100644 notebooks/projects/prospectorDHSMadagascar/06_launch.qmd delete mode 100644 notebooks/projects/prospectorDHSMadagascar/setup.qmd diff --git a/notebooks/projects/prospectorDHSMadagascar/00_misc.qmd b/notebooks/projects/prospectorDHSMadagascar/00_misc.qmd index 3742f85..984105e 100644 --- a/notebooks/projects/prospectorDHSMadagascar/00_misc.qmd +++ b/notebooks/projects/prospectorDHSMadagascar/00_misc.qmd @@ -1,17 +1,163 @@ --- -title: "Legacy: Miscellaneous Functions" +title: "Miscellaneous Helper Functions" format: gfm execute: eval: false +filters: + - sorting-hat + - ripper +extensions: + ripper: + output-name: "helpers" + output-dir: "../../src/rse_workbench" + sorting-hat: + keep: python --- -This numbered notebook has been superseded by the reusable project workflow. +This notebook collects miscellaneous functions that are useful for the RSE +Workbench. -Create a current notebook with: +Here are a couple of functions that allow us to quickly +build our singularity image and iterate on it by executing commands inside the +image. These functions are: + +```{python} +#| sorting-hat: keep +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from rich.columns import Columns +from rich.console import Console +from rich.panel import Panel +from rich.syntax import Syntax + +console = Console() + + +def spython_exec( + command: str, + image: str | Path, + options: list[str] | None = None, + writable: bool = False, + fakeroot: bool = False, + bind_project: str | Path | None = None, + return_result: bool = True, +) -> dict[str, Any]: + """Execute a command in a Singularity image through spython.""" + + from spython.main import Client + + exec_options = list(options or ["--no-home"]) + + if writable and "--writable" not in exec_options: + exec_options.append("--writable") + + if fakeroot and "--fakeroot" not in exec_options: + exec_options.append("--fakeroot") + + if bind_project: + exec_options += ["--bind", f"{Path(bind_project)}:/work"] + + return Client.execute( + image=str(image), + command=["bash", "--noprofile", "--norc", "-c", command], + options=exec_options, + sudo=False, + return_result=return_result, + ) + + +def normalize_spython_result(result: Any) -> dict[str, Any]: + """Normalize spython's mixed result shapes into stdout/stderr/code.""" + + if not isinstance(result, dict): + return {"stdout": str(result), "stderr": "", "return_code": None, "raw": result} + + message = result.get("message", "") + return_code = result.get("return_code", None) + + if isinstance(message, list): + stdout = message[0] if len(message) > 0 else "" + stderr = message[1] if len(message) > 1 else "" + else: + stdout = message or "" + stderr = "" + + return { + "stdout": stdout, + "stderr": stderr, + "return_code": return_code, + "raw": result, + } + + +def show_spython(result: Any, *, tail: int | None = None) -> dict[str, Any]: + """Render a spython result in a notebook-friendly panel.""" + + normalized = normalize_spython_result(result) + stdout = normalized["stdout"] + stderr = normalized["stderr"] + + if tail: + stdout = "\n".join(stdout.splitlines()[-tail:]) + stderr = "\n".join(stderr.splitlines()[-tail:]) + + console.print( + Panel( + Columns( + [ + Panel(Syntax(stdout or "", "text"), title="stdout / message", border_style="green"), + Panel( + Syntax(stderr or "", "text"), + title="stderr", + border_style="red" if normalized["return_code"] else "yellow", + ), + ] + ), + title=f"Exit code: {normalized['return_code']}", + border_style="green" if normalized["return_code"] == 0 else "red", + ) + ) + + return normalized -```bash -cp notebooks/templates/rse-workbench-project-template.qmd notebooks/projects/prospectorDHSMadagascar/environment.qmd ``` -Then edit the Project Contract chapter. Helper logic now lives in -`src/rse_workbench/`. + +To see the ongoing result of a job with `submitit`, we can use the following helper function: + +```{python} +#| sorting-hat: keep +"""Slurm display and launch-script helpers.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from rich.console import Console +from rich.panel import Panel +from rich.text import Text + +console = Console() + + +def show_submitit_job(job: Any) -> Any: + """Show stdout/stderr for a submitit job and return its result when done.""" + + console.print(f"Job ID: {job.job_id}") + console.print(f"Done: {job.done()}") + + stdout = job.stdout() or "" + stderr = job.stderr() or "" + + console.print(Panel(Text(stdout[-8000:] or ""), title="stdout", border_style="green")) + console.print(Panel(Text(stderr[-8000:] or ""), title="stderr", border_style="red")) + + if not job.done(): + return None + + return job.result() +``` diff --git a/notebooks/projects/prospectorDHSMadagascar/01_base_platform.qmd b/notebooks/projects/prospectorDHSMadagascar/01_base_platform.qmd deleted file mode 100644 index 65bc249..0000000 --- a/notebooks/projects/prospectorDHSMadagascar/01_base_platform.qmd +++ /dev/null @@ -1,9 +0,0 @@ ---- -title: "Legacy: Base Platform" -format: gfm -execute: - eval: false ---- - -This numbered notebook has been superseded by the reusable project workflow in -`notebooks/templates/rse-workbench-project-template.qmd`. diff --git a/notebooks/projects/prospectorDHSMadagascar/01_project-contract.qmd b/notebooks/projects/prospectorDHSMadagascar/01_project-contract.qmd new file mode 100644 index 0000000..6a4b00a --- /dev/null +++ b/notebooks/projects/prospectorDHSMadagascar/01_project-contract.qmd @@ -0,0 +1,98 @@ +--- +title: "Project Contract" +format: gfm +filters: + - sorting-hat + - ripper +extensions: + ripper: + output-name: "project_contract" + output-dir: "../../src/rse_workbench" + sorting-hat: + keep: python +execute: + eval: false +--- + +The project contract is the initial source of truth for the +project. In Python, it's fairly easy to work with capital +letter "constants", toss them around with pathlib, +and use them to generate files and folders. The contract +section makes use of this to get the project started: + +```{python} +#| sorting-hat: remove +from pathlib import Path +from pyprojroot import here + +PROJECT_NAME = "prospectorDHSMadagascar" # FILL ME IN +assert PROJECT_NAME, "Please fill in the project name." +PROJECT_DIR = Path("/n/holylabs/LABS/cgolden_lab/Lab/frontier/works/prospectors") / PROJECT_NAME +WORKBENCH_DIR = here() +RUNTIME_DIR = WORKBENCH_DIR / "projects" / PROJECT_NAME + +BASE_IMAGE = "docker://spack/rockylinux8:develop" + +R_VERSION = "4.5.1" +PYTHON_VERSION = "3.14" + +ENABLE_DIRENV = False +``` + +We can dedicate a section to the Slurm variables for the typical +runtime while we're building the project: + +```{python} +#| sorting-hat: remove +SLURM_PARTITION = "hsph" +SLURM_CPUS = 12 +SLURM_MEM = "32G" +SLURM_HOURS = 2 # this is the project build time, not the runtime for a single job +SLURM_EMAIL = "aaaapr3lk7gfdswioulcmvi3te@harvard.org.slack.com" # aaaapr3lk7gfdswioulcmvi3te@harvard.org.slack.com +``` + +Here are a few more useful paths. These names are intentionally +explicit because they are the boundary between this +notebook, the project tree, Slurm, and the container. + +```{python} +#| sorting-hat: remove +SINGULARITY_DEF = PROJECT_DIR / ".devcontainer" / "Singularity.def" +SINGULARITY_SIF = PROJECT_DIR / ".devcontainer" / "Singularity.sif" +SPACK_YAML_PATH = PROJECT_DIR / "spack.yaml" +RPROJECT_TOML_PATH = PROJECT_DIR / "rproject.toml" +PYPROJECT_TOML_PATH = PROJECT_DIR / "pyproject.toml" +CONTAINER_HOME = PROJECT_DIR / ".container-home" +CONTAINER_TMP = PROJECT_DIR / ".container-tmp" +SUBMITIT_DIR = PROJECT_DIR / "slurm" / "submitit" +``` + +Lastly, we can create a `ProjectContract` `dataclass` to help us +store all of these values in one place: + +```{python} +#| sorting-hat: remove + +from rse_workbench.project_contract import ProjectContract + +project_contract = ProjectContract( + project_name=PROJECT_NAME, + project_dir=PROJECT_DIR, + workbench_dir=WORKBENCH_DIR, + runtime_dir=RUNTIME_DIR, + base_image=BASE_IMAGE, + r_version=R_VERSION, + python_version=PYTHON_VERSION, + slurm_partition=SLURM_PARTITION, + slurm_cpus=SLURM_CPUS, + slurm_mem=SLURM_MEM, + slurm_hours=SLURM_HOURS, + enable_direnv=ENABLE_DIRENV, +) +project_contract.ensure_directories() +``` + +By the end of this chapter, you should be able to answer the question, +"what is this project, and where does it live?" + +In the next chapter, you'll define the platform recipe. diff --git a/notebooks/projects/prospectorDHSMadagascar/02_platform-recipe.qmd b/notebooks/projects/prospectorDHSMadagascar/02_platform-recipe.qmd new file mode 100644 index 0000000..258040c --- /dev/null +++ b/notebooks/projects/prospectorDHSMadagascar/02_platform-recipe.qmd @@ -0,0 +1,183 @@ +--- +title: "Platform Recipe" +format: gfm +filters: + - sorting-hat + - ripper +extensions: + ripper: + output-name: "platform_contract" + output-dir: "../../src/rse_workbench" + sorting-hat: + keep: python +execute: + eval: false +--- + +```{python} +from rse_workbench.platform_contract import Installable +from rse_workbench.platform_contract import compose_platform_recipe, recipe_to_definition +``` + +THe platform recipe section tells us what foundation to build +our project on. It picks the base Singularity image, initiates +Spack, and defines the basic structure of the linux machine +that the project will be built on. + +The platform image should provide the bare minimum tools +to start the development environment — not a full +project-specific dependency closure. It starts from the +Rocky/Spack base image, installs minimal OS tools, +creates `/work`, and installs `rv`, `uv`, `arf`, and the VSCode CLI. + +We build the platform image contract using a new +`installable` `dataclass` and the `spython` +library, which is a Python interface to Singularity. It +allows us to programmatically define the image layers and +contents. + +This data structure allows us to quickly define +the main tools we will need installed in +the Linux environment, including test lines +and the path to the tool if it is not in the default `$PATH`. + +Below, we define our 4 tools: + +```{python} +#| sorting-hat: remove +arf = Installable( + name="arf", + kind="cargo", + path="/usr/local/bin/arf", + commands=[ + "curl --proto '=https' --tlsv1.2 -LsSf https://github.com/eitsupi/arf/releases/latest/download/arf-console-installer.sh | sh", + """ +ARF_BIN="$(find /root/.cargo/bin /home -path '*/.cargo/bin/arf' -type f 2>/dev/null | head -n 1)" +if [ -z "$ARF_BIN" ]; then +ARF_BIN="$(find / -path '*/.cargo/bin/arf' -type f 2>/dev/null | head -n 1)" +fi +test -n "$ARF_BIN" +install -m 0755 "$ARF_BIN" /usr/local/bin/arf +""", + ], + test=["command -v arf", "arf --version"], +) + +uv = Installable( + name="uv", + kind="binary", + path="/usr/local/bin/uv", + commands=[ + """ +curl -LsSf https://astral.sh/uv/install.sh | sh +UV_BIN="$(find /root/.local/bin /home -path '*/.local/bin/uv' -type f 2>/dev/null | head -n 1)" +if [ -z "$UV_BIN" ]; then +UV_BIN="$(find / -path '*/.local/bin/uv' -type f 2>/dev/null | head -n 1)" +fi +test -n "$UV_BIN" +install -m 0755 "$UV_BIN" /usr/local/bin/uv +""", + ], + test=["command -v uv", "uv --version"], +) + +rv = Installable( + name="rv", + kind="binary", + path="/usr/local/bin/rv", + commands=[ + """ +curl -sSL https://raw.githubusercontent.com/A2-ai/rv/refs/heads/main/scripts/install.sh | bash +RV_BIN="$(find /root/.local/bin /home -path '*/.local/bin/rv' -type f 2>/dev/null | head -n 1)" +if [ -z "$RV_BIN" ]; then +RV_BIN="$(find / -path '*/.local/bin/rv' -type f 2>/dev/null | head -n 1)" +fi +test -n "$RV_BIN" +install -m 0755 "$RV_BIN" /usr/local/bin/rv +""", + ], + test=["command -v rv", "rv --version"], +) + +code = Installable( + name="code", + kind="binary", + path="/usr/local/bin/code", + commands=[ + """ +tmpdir="$(mktemp -d)" +cd "$tmpdir" +curl -L --fail --show-error --output vscode_cli.tar.gz "https://code.visualstudio.com/sha/download?build=stable&os=cli-alpine-x64" +tar -xzf vscode_cli.tar.gz +CODE_BIN="$(find "$tmpdir" -type f -name code | head -n 1)" +test -n "$CODE_BIN" +test -x "$CODE_BIN" || chmod +x "$CODE_BIN" +install -m 0755 "$CODE_BIN" /usr/local/bin/code +/usr/local/bin/code --version +""", + ], + test=["command -v code", "code --version"], +) +``` + +Here's how we define the platform recipe, using a `spython` +Singularity recipe: + +Project-specific Spack view activation variables do not belong in the image +`%environment`. The project provides those through `/work/env/activate-system-deps.sh`. + +```{python} +#| sorting-hat: remove +recipe = compose_platform_recipe( + base_image=BASE_IMAGE, + installables=[arf, code, rv, uv] + ) + +definition = recipe_to_definition(recipe) +print(definition) +``` + +At this point, we can build the first iteration of the image using the `spython` +library. This will create a Singularity image file in the project directory. To +do this, we need the `Singularity.def` file written out to the project directory, +and then we can use `spython` to build the image. + +```{python} +#| sorting-hat: remove +from spython.main import Client + +TMP_SANDBOX = Path(f"/tmp/{os.environ['USER']}-{PROJECT_NAME}-sandbox") + +SINGULARITY_DEF.write_text(definition) + +Client.build( + image=str(TMP_SANDBOX), + recipe=str(SINGULARITY_DEF), + options=["--fakeroot", "--sandbox", "--fix-perms", "--force"], + sudo=False, +) +``` + +To try out installing a new tool into the platform image, we can experiment +with the image using our `spython_exec` function. Once we are sure of the +commands we want, we can define a new `Installable` and then call +`compose_platform_recipe` again with the new tool added +to the list of installables. + + +```{python} +#| sorting-hat: remove +show_spython( + spython_exec( + command="arf --version", + image=TMP_SANDBOX, + options=["--fakeroot", "--writable", "--no-home"], + bind_project=PROJECT_DIR, + ) +) +``` + +By the end of this chapter, you should be able to answer the question, +"what kind of computer does this project run on?" + +In the next chapter, you'll define the scientific dependencies. diff --git a/notebooks/projects/prospectorDHSMadagascar/02_spack.qmd b/notebooks/projects/prospectorDHSMadagascar/02_spack.qmd deleted file mode 100644 index 50eec48..0000000 --- a/notebooks/projects/prospectorDHSMadagascar/02_spack.qmd +++ /dev/null @@ -1,9 +0,0 @@ ---- -title: "Legacy: Spack Layer" -format: gfm -execute: - eval: false ---- - -This numbered notebook has been superseded by the reusable project workflow. -Use `rse_workbench.spack` from the new template notebook. diff --git a/notebooks/projects/prospectorDHSMadagascar/03_datascience_lang.qmd b/notebooks/projects/prospectorDHSMadagascar/03_datascience_lang.qmd deleted file mode 100644 index df9d9d9..0000000 --- a/notebooks/projects/prospectorDHSMadagascar/03_datascience_lang.qmd +++ /dev/null @@ -1,9 +0,0 @@ ---- -title: "Legacy: Language Layer" -format: gfm -execute: - eval: false ---- - -This numbered notebook has been superseded by the reusable project workflow. -Use `rse_workbench.languages` from the new template notebook. diff --git a/notebooks/projects/prospectorDHSMadagascar/03_sci-sysdeps.qmd b/notebooks/projects/prospectorDHSMadagascar/03_sci-sysdeps.qmd new file mode 100644 index 0000000..fd3af5d --- /dev/null +++ b/notebooks/projects/prospectorDHSMadagascar/03_sci-sysdeps.qmd @@ -0,0 +1,112 @@ +--- +title: "Scientific System Dependency Layer with Spack" +format: gfm +filters: + - sorting-hat + - ripper +extensions: + ripper: + output-name: "sci_sysdeps" + output-dir: "../../src/rse_workbench" + sorting-hat: + keep: python +execute: + eval: false +--- + +```{python} +from rse_workbench.sci_sysdeps import default_spack_env, write_spack_yaml +``` + +Spack is a package manager for scientific software on HPC systems. It +is designed to support multiple versions and configurations of software +on a wide variety of platforms and environments. Spack is used to manage the system +dependency layer of the project. + +Spack works by defining packages as Python classes in a declarative way, +and the workflow is to define a `spack.yaml` file that lists the +packages and their versions in one place. Spack then resolves the +dependencies as a DAG and builds the packages in the correct order. +So long as the package recipe is correct in the Spack package catalog, +Spack will build the environment correctly. + +It's important to delineate responsibilities — Spack owns only the +system dependency layer for the Python and R packages. Avoid +CRAN-level Spack recipes such as `r-fs`, `r-httpuv`, `r-dplyr`, +or `r-rmarkdown` unless there is a strong reason. Those packages +belong to `rv`. + +Installing a library is not the same as making it discoverable. For example, +`httpuv` may find `libuv` through `pkg-config` while still failing to find +`zlib.h`; this is why the activation bridge exists. Also, `spack.yaml` cannot +patch missing `depends_on()` metadata in a Spack package recipe. + +There are some default system dependencies that are going to be required +for most of our projects to build and run. We can define these as defaults +for our environment. + +```{python} +#| sorting-hat: remove +DEFAULT_SYSTEM_SPECS = [ + "libiconv", + "ncurses", + "openblas", + "curl", + "openssl", + "pkgconf", + "abseil-cpp", + "zlib", + "zlib-ng+compat", + "cmake", + "gmake", + "git", + "libx11", + "libxml2", + "freetype", + "libjpeg", + "libjpeg-turbo", + "libpng", + "libtiff", + "libwebp +libwebpmux", + "icu4c", + "fontconfig", + "fribidi", + "harfbuzz", + "libgit2", + "cairo", + "readline", + "libuv", + "libgit2" +] + +DEFAULT_SYSTEM_SPECS.extend([ + "proj", + "geos", + "gdal", + "udunits", + "libuv", + "zlib", + "r-udunits2" +]) + +spack_env = default_spack_env( + r_version=R_VERSION, + python_version=PYTHON_VERSION, + default_specs=DEFAULT_SYSTEM_SPECS +) +spack_env +``` + +When you're ready to write it out to file, you can use the `write_spack_yaml()` +function. This will create the parent directories if they don't exist. + +```{python} +#| sorting-hat: remove +write_spack_yaml(SPACK_YAML_PATH, spack_env) +``` + +By the end of this chapter, you should be able to answer the question, +"what broad area of science does this project entail and what +toolkit will that require?" + +In the next chapter, you'll define the project-specific dependencies. diff --git a/notebooks/projects/prospectorDHSMadagascar/04_datascience.qmd b/notebooks/projects/prospectorDHSMadagascar/04_datascience.qmd new file mode 100644 index 0000000..3ffe49c --- /dev/null +++ b/notebooks/projects/prospectorDHSMadagascar/04_datascience.qmd @@ -0,0 +1,153 @@ +--- +title: "Data Science Language Package Layer" +format: gfm +filters: + - sorting-hat + - ripper +extensions: + ripper: + output-name: "datascience" + output-dir: "../../src/rse_workbench" + sorting-hat: + keep: python +execute: + eval: false +--- + +```{python} +#| sorting-hat: keep +from __future__ import annotations + +from pathlib import Path +from typing import Any +``` + +```{python} +#| sorting-hat: remove +from rse_workbench.datascience import ( + default_pyproject, + default_rproject, + write_pyproject_toml, + write_rproject_toml, +) + +``` + +The data science language package layer is a set of canonical package dependency +files for R and Python. The R package manager `rv` and the Python package +manager `uv` are used to manage these dependencies. + +`rv` owns R package dependencies in `rproject.toml`. `uv` owns Python package +dependencies in `pyproject.toml`. These files live in the project tree so they +travel with the scientific code and are consumed inside `/work`. + +First, we define a few default packages: + +```{python} +#| sorting-hat: keep +DEFAULT_R_PACKAGES = [ + "box", # we use this for namespacing and dependency injection + "remotes", + "rprojroot", # always travel with this + "stringi", + "targets", + "tarchetypes", + "usethis", + "httpgd", + "httpuv", + "parsermd", + "rmarkdown", + "skimr", + "DiagrammeR", + "svglite", + "later", + "rstudioapi", + "dplyr", + "languageserver", + "here", + "fledge", + "readr", + "stringr", + "withr", + "readxl", + "tidyverse", + "websocket", + "codetools", + "quarto", + "rlang", + "purrr", + "tibble", + "devtools", + "jsonlite", +] + +DEFAULT_R_GIT_DEPENDENCIES = [ + { + "name": "sess", + "git": "https://github.com/REditorSupport/vscode-R", + "branch": "master", + "directory": "sess", + } +] + +DEFAULT_PYTHON_PACKAGES = [ + "numpy", + "pandas", + "ipykernel", + "jupyterlab", +] +``` + +```{python} +DEFAULT_R_PACKAGES.extend([ + "sf", + "nanoparquet", + "janitor", + "lubridate", + "frictionless" +]) +``` + +We define the full TOML files like so: + +```{python} +#| sorting-hat: remove +rproject = default_rproject( + project_name=PROJECT_NAME, + r_version=R_VERSION, + packages=DEFAULT_R_PACKAGES, + git_dependencies=DEFAULT_R_GIT_DEPENDENCIES +) +pyproject = default_pyproject( + project_name=PROJECT_NAME, + python_version=PYTHON_VERSION, + packages=DEFAULT_PYTHON_PACKAGES +) +``` + + + + +```{python} +#| sorting-hat: remove +write_rproject_toml(RPROJECT_TOML_PATH, rproject) +write_pyproject_toml(PYPROJECT_TOML_PATH, pyproject) +``` + +It's at this stage that you might want to double back on +the platform recipe and add any additional system dependencies +that are required for project packages. You can do this for the +R TOML like so: + + +```r +pkgs <- toml::parse_toml(readLines("/n/holylabs/LABS/cgolden_lab/Lab/frontier/works/prospectors//rproject.toml"))$project$dependencies +pkgs[[length(pkgs)]] <- NULL +pak::pkg_sysreqs(unlist(pkgs), sysreqs_platform = "rockylinux") +``` + +If any system dependencies come up here, you +can add them either to the platform recipe or to the +`DEFAULT_SYSTEM_SPECS` list in the spack notebook. + +In the next chapter, you'll test out the project activation. diff --git a/notebooks/projects/prospectorDHSMadagascar/04_devtools.qmd b/notebooks/projects/prospectorDHSMadagascar/04_devtools.qmd deleted file mode 100644 index 2ea8aa4..0000000 --- a/notebooks/projects/prospectorDHSMadagascar/04_devtools.qmd +++ /dev/null @@ -1,9 +0,0 @@ ---- -title: "Legacy: Developer Tooling" -format: gfm -execute: - eval: false ---- - -This numbered notebook has been superseded by the reusable project workflow. -Developer tooling is now defined in `rse_workbench.recipe`. diff --git a/notebooks/projects/prospectorDHSMadagascar/05_project-activation.qmd b/notebooks/projects/prospectorDHSMadagascar/05_project-activation.qmd new file mode 100644 index 0000000..b9bf078 --- /dev/null +++ b/notebooks/projects/prospectorDHSMadagascar/05_project-activation.qmd @@ -0,0 +1,143 @@ +--- +title: "Bridge and Activation" +format: gfm +execute: + eval: false +--- + +To test that the project environment is discoverable and +functional, we use this chapter to test the environment. + +First, test that `spack` works: + +```{python} +show_spython( + spython_exec( + """ + set -euo pipefail + . /opt/spack/share/spack/setup-env.sh + cd /work + spack env activate -d . + spack find + """, + image=str(TMP_SANDBOX), + bind_project=PROJECT_DIR, + ), + tail=120, +) +``` + +Now let's see what happens when we check the `spack` environment: + +```{python} +res = show_spython( + spython_exec( + """ + set -euo pipefail + . /opt/spack/share/spack/setup-env.sh + cd /work + spack env activate -d . + + echo $PATH + echo $PKG_CONFIG_PATH + echo $CPATH + echo $LIBRARY_PATH + echo $LD_LIBRARY_PATH + echo $TMPDIR + + spack concretize --force + #spack install --deprecated --fail-fast -j8 --no-checksum + """, + image=str(TMP_SANDBOX), + bind_project=PROJECT_DIR, + ), +) +``` + +We can see that the above environment variables are malformed. We can +add them to our recipe: + +```{python} +recipe.environ.extend( + [ + "PATH=$SPACK_VIEW/bin:${PATH:-}", + "PKG_CONFIG_PATH=$SPACK_VIEW/lib/pkgconfig:$SPACK_VIEW/lib64/pkgconfig:${PKG_CONFIG_PATH:-}", + "CPATH=$SPACK_VIEW/include:${CPATH:-}", + "LIBRARY_PATH=$SPACK_VIEW/lib:$SPACK_VIEW/lib64:${LIBRARY_PATH:-}", + "LD_LIBRARY_PATH=$SPACK_VIEW/lib:$SPACK_VIEW/lib64:${LD_LIBRARY_PATH:-}" + ] +) +``` + +And rebuild: + +```{python} +definition = recipe_to_definition(recipe) +SINGULARITY_DEF.write_text(definition) +Client.build( + image=str(TMP_SANDBOX), + recipe=str(SINGULARITY_DEF), + options=["--fakeroot", "--sandbox", "--fix-perms", "--force"], + sudo=False, +) +``` + +Now, we can see that the environment variables are set: + +```{python} +show_spython( + spython_exec( + """ + set -euo pipefail + . /opt/spack/share/spack/setup-env.sh + cd /work + spack env activate -d . + + echo $PATH + echo $PKG_CONFIG_PATH + echo $CPATH + echo $LIBRARY_PATH + echo $LD_LIBRARY_PATH + echo $TMPDIR + + spack concretize --force + #spack install --deprecated --fail-fast -j8 --no-checksum + """, + image=str(TMP_SANDBOX), + bind_project=PROJECT_DIR, + ), +) +``` + +If you're feeling bold, you can install the project environment in +the sandbox — but that will take a while and the sandbox is +ephemeral. + + + \ No newline at end of file diff --git a/notebooks/projects/prospectorDHSMadagascar/05_recipe.qmd b/notebooks/projects/prospectorDHSMadagascar/05_recipe.qmd deleted file mode 100644 index 4b349a7..0000000 --- a/notebooks/projects/prospectorDHSMadagascar/05_recipe.qmd +++ /dev/null @@ -1,9 +0,0 @@ ---- -title: "Legacy: Build Container" -format: gfm -execute: - eval: false ---- - -This numbered notebook has been superseded by the reusable project workflow. -The final install command now fails loudly and lives in `rse_workbench.project`. diff --git a/notebooks/projects/prospectorDHSMadagascar/06_build-launch.qmd b/notebooks/projects/prospectorDHSMadagascar/06_build-launch.qmd new file mode 100644 index 0000000..8c61967 --- /dev/null +++ b/notebooks/projects/prospectorDHSMadagascar/06_build-launch.qmd @@ -0,0 +1,110 @@ +--- +title: "Build Final SIF & Install" +format: gfm +filters: + - sorting-hat + - ripper +extensions: + ripper: + output-name: "build_install" + output-dir: "../../src/rse_workbench" + sorting-hat: + keep: python +execute: + eval: false +--- + +```{python} +#| sorting-hat: remove +from rse_workbench.helpers import show_submitit_job +from rse_workbench.build_install import install_in_container + +``` + + +Now that the project has been defined, the platform +decided on, and the project-specific dependencies have been +defined, it's time to build the final SIF and install the project environment. + +We're going to build the final read-only SIF through Slurm +so Singularity has node-local temporary space and enough memory. +Here's how you build an SBATCH job with `submitit` to build the SIF: + +```{python} +#| sorting-hat: remove +import submitit +from pathlib import Path +import os + +executor = submitit.AutoExecutor(folder=str(SUBMITIT_DIR / "%j")) +executor.update_parameters( + timeout_min=SLURM_HOURS * 60, + slurm_partition=SLURM_PARTITION, + mem_gb=float(SLURM_MEM.rstrip("G")), + cpus_per_task=SLURM_CPUS, + slurm_job_name=f"build-{PROJECT_NAME}-container", + mail_user=SLURM_EMAIL +) +``` + +The build function takes the Singularity definition file and an installation +string and builds the SIF. It returns the job ID for tracking. + +```{python} +#| sorting-hat: remove +job = executor.submit(build_container, SINGULARITY_DEF, SINGULARITY_SIF) +job.job_id +``` + +```{python} +#| sorting-hat: remove +result = show_submitit_job(job) +if result is not None: + print(result) + +``` + +The final install function runs inside the built SIF, binds the project as `/work`, +activates Spack, installs the Spack layer, then runs `rv sync` to +install the project dependencies. It fails loudly if any required +step fails. + +```{python} +#| sorting-hat: keep +INSTALL_RUNTIME_COMMAND = r""" +set -euo pipefail + +df -h /tmp /work + +. /opt/spack/share/spack/setup-env.sh +cd /work + +spack env activate -d . +spack concretize --force +spack install --deprecated --fail-fast -j8 --no-checksum + +rv sync +""" +``` + +```{python} +#| sorting-hat: remove +job = executor.submit( + install_in_container, + project_dir=PROJECT_DIR, + singularity_sif=SINGULARITY_SIF, + runtime_command=INSTALL_RUNTIME_COMMAND +) +job.job_id +``` + +```{python} +#| sorting-hat: remove +result = show_submitit_job(job) +if result is not None: + print(result) +``` + +```{python} +print(result['message'][0]) +``` diff --git a/notebooks/projects/prospectorDHSMadagascar/06_launch.qmd b/notebooks/projects/prospectorDHSMadagascar/06_launch.qmd deleted file mode 100644 index 3b9abc8..0000000 --- a/notebooks/projects/prospectorDHSMadagascar/06_launch.qmd +++ /dev/null @@ -1,9 +0,0 @@ ---- -title: "Legacy: Launch VSCode" -format: gfm -execute: - eval: false ---- - -This numbered notebook has been superseded by the reusable project workflow. -Launch script generation now lives in `rse_workbench.slurm`. diff --git a/notebooks/projects/prospectorDHSMadagascar/setup.qmd b/notebooks/projects/prospectorDHSMadagascar/setup.qmd deleted file mode 100644 index 8449c8f..0000000 --- a/notebooks/projects/prospectorDHSMadagascar/setup.qmd +++ /dev/null @@ -1,17 +0,0 @@ ---- -title: "Legacy: Project Setup" -format: gfm -execute: - eval: false ---- - -This setup notebook has been superseded by the reusable project workflow. - -Create a current notebook with: - -```bash -cp notebooks/templates/rse-workbench-project-template.qmd notebooks/projects/prospectorDHSMadagascar/environment.qmd -``` - -Then edit the Project Contract chapter. Helper logic now lives in -`src/rse_workbench/`. diff --git a/notebooks/projects/prospectorMahery/05_project-activation.qmd b/notebooks/projects/prospectorMahery/05_project-activation.qmd index 22a3377..b89b9cc 100644 --- a/notebooks/projects/prospectorMahery/05_project-activation.qmd +++ b/notebooks/projects/prospectorMahery/05_project-activation.qmd @@ -9,7 +9,7 @@ To test that the project environment is discoverable and functional, we use this chapter to test the environment. ```{python} -from rse_workbench.config import show_spython, spython_exec +from rse_workbench.helpers import show_spython, spython_exec ``` First, test that `spack` works: From 79deec98d38a8b77931981f63bbe3bd0c39d299c Mon Sep 17 00:00:00 2001 From: TinasheMTapera <15770644+TinasheMTapera@users.noreply.github.com> Date: Mon, 6 Jul 2026 02:49:45 -0400 Subject: [PATCH 2/2] feat(notebooks): update environment variables in activation recipe - refined environment variable settings for spack integration - added additional paths for user and misc cache - ensured compatibility with previous builds by addressing malformed variables --- .../05_project-activation.qmd | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/notebooks/projects/prospectorDHSMadagascar/05_project-activation.qmd b/notebooks/projects/prospectorDHSMadagascar/05_project-activation.qmd index b9bf078..3c11a50 100644 --- a/notebooks/projects/prospectorDHSMadagascar/05_project-activation.qmd +++ b/notebooks/projects/prospectorDHSMadagascar/05_project-activation.qmd @@ -55,16 +55,22 @@ res = show_spython( ``` We can see that the above environment variables are malformed. We can -add them to our recipe: +add them to our recipe below. Pay particular attention to +this; in previous builds, this ended up being the fix: ```{python} recipe.environ.extend( - [ - "PATH=$SPACK_VIEW/bin:${PATH:-}", - "PKG_CONFIG_PATH=$SPACK_VIEW/lib/pkgconfig:$SPACK_VIEW/lib64/pkgconfig:${PKG_CONFIG_PATH:-}", - "CPATH=$SPACK_VIEW/include:${CPATH:-}", - "LIBRARY_PATH=$SPACK_VIEW/lib:$SPACK_VIEW/lib64:${LIBRARY_PATH:-}", - "LD_LIBRARY_PATH=$SPACK_VIEW/lib:$SPACK_VIEW/lib64:${LD_LIBRARY_PATH:-}" + [ + "SPACK_VIEW=/work/.spack-env/view", + "PATH=${SPACK_VIEW}/bin:/usr/local/bin:/opt/views/view/bin:/root/.cargo/bin:/usr/cargo/bin:${PATH:-}", + "PKG_CONFIG_PATH=${SPACK_VIEW}/lib/pkgconfig:${SPACK_VIEW}/lib64/pkgconfig:${SPACK_VIEW}/share/pkgconfig:${PKG_CONFIG_PATH:-}", + "CPATH=${SPACK_VIEW}/include:${CPATH:-}", + "LIBRARY_PATH=${SPACK_VIEW}/lib:${SPACK_VIEW}/lib64:${LIBRARY_PATH:-}", + "LD_LIBRARY_PATH=${SPACK_VIEW}/lib:${SPACK_VIEW}/lib64:${LD_LIBRARY_PATH:-}", + "SPACK_DISABLE_LOCAL_CONFIG=true", + "SPACK_USER_CONFIG_PATH=/tmp/spack-user-config", + "SPACK_USER_CACHE_PATH=/tmp/spack-user-cache", + "SPACK_MISC_CACHE_PATH=/tmp/spack-misc-cache", ] ) ```