diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..8d7feae --- /dev/null +++ b/.dockerignore @@ -0,0 +1,26 @@ +# Keep the build context small and deterministic. The image installs from +# pyproject.toml + uv.lock, so only the package source is needed. +.git +.github +.venv +venv +env +outputs +.tmp +site +docs +analysis +examples +tests +demo_issue_69.sh +**/__pycache__ +**/*.pyc +*.egg-info +.env +*.env +.ruff_cache +.pytest_cache +Dockerfile +.dockerignore +docker-compose.yml +docker-run.sh diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..75ec982 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,42 @@ +# Reproducible runtime image for climate-toolkit. +# +# Reproducibility comes from three pins: +# 1. the base image, pinned by digest (OS + CPython 3.10, matching CI); +# 2. uv, pinned to an exact release; +# 3. uv.lock, which fixes every Python dependency to an exact version + hash. +# +# Earth Engine credentials are NOT baked in (they are per-user OAuth). Mount +# them read-only at runtime and pass GCP_PROJECT_ID as an environment variable +# -- see README "Reproducible environment with Docker". + +FROM python:3.10-slim@sha256:e5300dc020a26a34a19337a57602955a2510e22abeb176edd6de6cd2cc927dd4 AS runtime + +# Pinned uv release: deterministic, lockfile-based installs. +COPY --from=ghcr.io/astral-sh/uv:0.11.28 /uv /uvx /usr/local/bin/ + +ENV UV_LINK_MODE=copy \ + UV_COMPILE_BYTECODE=1 \ + UV_PYTHON_DOWNLOADS=never \ + UV_PROJECT_ENVIRONMENT=/app/.venv \ + MPLBACKEND=Agg \ + PATH="/app/.venv/bin:$PATH" + +WORKDIR /app + +# 1) Dependency layer -- cached until pyproject.toml or uv.lock changes. +COPY pyproject.toml uv.lock README.md ./ +RUN uv sync --locked --no-install-project --no-dev + +# 2) Project layer -- source changes do not invalidate the dependency layer. +COPY climate_toolkit ./climate_toolkit +RUN uv sync --locked --no-dev + +# Run as non-root. Earth Engine credentials mount at +# /home/app/.config/earthengine; outputs/cache persist via /app/outputs. +RUN useradd --create-home app \ + && mkdir -p /app/outputs \ + && chown -R app:app /app +USER app + +ENTRYPOINT ["climate-toolkit"] +CMD ["--help"] diff --git a/README.md b/README.md index 2eb42b0..2f4296c 100644 --- a/README.md +++ b/README.md @@ -281,6 +281,73 @@ python -m pip install .tmp/dist-release/climate_toolkit-*.whl If you only want artifact verification without mutating current environment, rely on automated smoke tests and CI build checks. +### Reproducible environment with Docker + +For a byte-for-byte reproducible runtime that does not depend on your local +Python version, use the provided `Dockerfile`. Reproducibility comes from three +pins: the base image (pinned by digest to CPython 3.10, matching CI), `uv` +(pinned to an exact release), and `uv.lock` (every dependency fixed to an exact +version + hash). + +Build the image: + +```bash +docker build -t climate-toolkit . +``` + +**Recommended: use the `docker-run.sh` wrapper.** It deletes stale containers +from previous runs on every start, then launches a fresh one with `--rm` so it +also cleans up on exit — you never accumulate leftover containers. It builds the +image automatically on first use and forwards all arguments to the CLI: + +```bash +./docker-run.sh --help +GCP_PROJECT_ID=your-project ./docker-run.sh fetch --source nasa_power \ + --lat -1.286 --lon 36.817 --from 2020-01-01 --to 2020-12-31 +``` + +The wrapper mounts your Earth Engine credentials and `outputs/` for you. Under +the hood it runs the plain commands below. + +Offline smoke test — proves the environment reproduces, needs neither Earth +Engine nor network: + +```bash +docker run --rm climate-toolkit --help +docker run --rm --entrypoint python climate-toolkit \ + -c "import climate_toolkit as ct; print(ct.__version__)" +``` + +Earth Engine credentials are **not** baked into the image (they are per-user +OAuth). For Earth Engine-backed sources, mount your credentials read-only and +pass `GCP_PROJECT_ID` at runtime. Mount `outputs/` so cached frames persist +across runs: + +```bash +docker run --rm \ + -e GCP_PROJECT_ID=your-project \ + -v "$HOME/.config/earthengine:/home/app/.config/earthengine:ro" \ + -v "$(pwd)/outputs:/app/outputs" \ + climate-toolkit fetch --source nasa_power \ + --lat -1.286 --lon 36.817 --from 2020-01-01 --to 2020-12-31 +``` + +A `docker-compose.yml` wraps those mounts so you can run +`docker compose run --rm toolkit ` instead. NASA POWER needs no +credentials; the offline unit suite and CLI surface work without Earth Engine. + +**Container cleanup.** Every documented command uses `--rm`, so a successful run +never leaves a container behind. Stale containers can only appear if a run +crashes or is started without `--rm`; the `docker-run.sh` wrapper sweeps those +before each start. To clean up manually at any time: + +```bash +# Remove stopped containers built from the image +docker ps -aq --filter ancestor=climate-toolkit --filter status=exited | xargs -r docker rm +# Or, when using compose +docker compose down --remove-orphans +``` + ### Release strategy Current decision: diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..4e74e1b --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,28 @@ +# Convenience wrapper around the reproducible image. +# +# docker compose build +# docker compose run --rm toolkit --help +# GCP_PROJECT_ID=your-project docker compose run --rm toolkit \ +# fetch --source nasa_power --lat -1.286 --lon 36.817 \ +# --from 2020-01-01 --to 2020-12-31 +# +# Always pass --rm so each run removes its own container on exit. To sweep any +# stale containers left by earlier runs, run: +# docker compose down --remove-orphans +# +# Earth Engine-backed sources also need the mounted credentials below plus a +# real GCP_PROJECT_ID. NASA POWER needs neither. + +services: + toolkit: + build: + context: . + image: climate-toolkit + environment: + # Passed through from your shell; empty is fine for NASA POWER / offline use. + - GCP_PROJECT_ID=${GCP_PROJECT_ID:-} + volumes: + # Earth Engine OAuth credentials (created by `earthengine authenticate`). + - ${HOME}/.config/earthengine:/home/app/.config/earthengine:ro + # Persist cache/outputs across runs so repeat fetches reuse cached frames. + - ./outputs:/app/outputs diff --git a/docker-run.sh b/docker-run.sh new file mode 100755 index 0000000..2204b1f --- /dev/null +++ b/docker-run.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# +# Recommended way to run climate-toolkit in Docker. +# +# Every invocation first deletes stale containers left by previous runs +# (exited / created / dead), then starts a fresh container with --rm so it +# cleans up after itself on exit too. Running containers are never touched, so +# concurrent runs are safe. +# +# Usage: +# ./docker-run.sh --help +# GCP_PROJECT_ID=your-project ./docker-run.sh fetch --source nasa_power \ +# --lat -1.286 --lon 36.817 --from 2020-01-01 --to 2020-12-31 +# +# Override the image name with CLIMATE_TOOLKIT_IMAGE if you tagged it yourself. + +set -euo pipefail + +IMAGE="${CLIMATE_TOOLKIT_IMAGE:-climate-toolkit}" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# Build on first use if the image is missing. +if ! docker image inspect "$IMAGE" >/dev/null 2>&1; then + echo "Image '$IMAGE' not found; building..." >&2 + docker build -t "$IMAGE" "$SCRIPT_DIR" +fi + +# Delete stale containers from previous runs before starting. Different filter +# keys are ANDed (this image) while repeated status filters are ORed, so this +# matches only non-running containers built from this image. +stale="$(docker ps -aq \ + --filter "ancestor=$IMAGE" \ + --filter "status=exited" \ + --filter "status=created" \ + --filter "status=dead")" +if [ -n "$stale" ]; then + echo "Removing stale climate-toolkit containers..." >&2 + # shellcheck disable=SC2086 + docker rm -f $stale >/dev/null +fi + +exec docker run --rm \ + -e GCP_PROJECT_ID="${GCP_PROJECT_ID:-}" \ + -v "$HOME/.config/earthengine:/home/app/.config/earthengine:ro" \ + -v "$PWD/outputs:/app/outputs" \ + "$IMAGE" "$@"