From 047d06817b7921a609b1deca99b0e05040dede92 Mon Sep 17 00:00:00 2001 From: Geoff Boeing Date: Sun, 19 Jul 2026 19:50:16 -0700 Subject: [PATCH 1/6] build cache with 1 process then run notebooks with 4 --- .github/workflows/ci.yml | 7 ++----- notebooks/run_notebooks.sh | 26 ++++++++++++++++++++++++++ 2 files changed, 28 insertions(+), 5 deletions(-) create mode 100755 notebooks/run_notebooks.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ca9d392..33c1392 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -60,14 +60,11 @@ jobs: env: SKIP: no-commit-to-branch - # test notebooks in parallel across 3 jobs with GNU parallel + # test notebooks serially until the cache is complete, then in parallel - name: Test notebooks run: | cd ./notebooks - mkdir -p data images - uv run python -c "import osmnx; print(osmnx.__version__)" - uv run jupyter nbconvert --to python *.ipynb - parallel -j3 --line-buffer --tag 'uv run ipython {}' ::: *.py + ./run_notebooks.sh # allow notebooks test runs to fail, but still save partial cache. this # saves after ordinary failures but not if job is canceled or times out. diff --git a/notebooks/run_notebooks.sh b/notebooks/run_notebooks.sh new file mode 100755 index 0000000..18f0407 --- /dev/null +++ b/notebooks/run_notebooks.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +set -euo pipefail + +CACHE_DIR="./cache" +CACHE_READY_FILE="${CACHE_DIR}/.cache_ready" + +WORKERS=1 +if [[ -f "$CACHE_READY_FILE" ]]; then + WORKERS=4 +fi +echo "Running notebooks with ${WORKERS} process(es)." + +mkdir -p data images "$CACHE_DIR" +uv run python -c "import osmnx; print(osmnx.__version__)" +uv run jupyter nbconvert --to python *.ipynb + +if (( WORKERS == 1 )); then + for filename in *.py; do + uv run ipython "$filename" + done +else + printf '%s\n' *.py | xargs -n1 -P4 uv run ipython +fi + +# only a fully successful run can declare the cache complete +touch "$CACHE_READY_FILE" From 79a15a6c6d6fdd2a9ec6ec4ee3d7205ba209833a Mon Sep 17 00:00:00 2001 From: Geoff Boeing Date: Sun, 19 Jul 2026 19:55:06 -0700 Subject: [PATCH 2/6] use Agg mpl backend --- notebooks/run_notebooks.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/notebooks/run_notebooks.sh b/notebooks/run_notebooks.sh index 18f0407..7cdb972 100755 --- a/notebooks/run_notebooks.sh +++ b/notebooks/run_notebooks.sh @@ -1,6 +1,8 @@ #!/usr/bin/env bash set -euo pipefail +export MPLBACKEND="Agg" + CACHE_DIR="./cache" CACHE_READY_FILE="${CACHE_DIR}/.cache_ready" From e9d2ab09e1bc21d35e3b8903297e344a95cdd0d8 Mon Sep 17 00:00:00 2001 From: Geoff Boeing Date: Sun, 19 Jul 2026 20:57:25 -0700 Subject: [PATCH 3/6] code cleanup --- .github/workflows/ci.yml | 5 +---- notebooks/run_notebooks.sh | 11 ++++++----- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 33c1392..b27a58a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -60,11 +60,8 @@ jobs: env: SKIP: no-commit-to-branch - # test notebooks serially until the cache is complete, then in parallel - name: Test notebooks - run: | - cd ./notebooks - ./run_notebooks.sh + run: cd notebooks && bash run_notebooks.sh # allow notebooks test runs to fail, but still save partial cache. this # saves after ordinary failures but not if job is canceled or times out. diff --git a/notebooks/run_notebooks.sh b/notebooks/run_notebooks.sh index 7cdb972..90fcb0e 100755 --- a/notebooks/run_notebooks.sh +++ b/notebooks/run_notebooks.sh @@ -2,20 +2,21 @@ set -euo pipefail export MPLBACKEND="Agg" - CACHE_DIR="./cache" CACHE_READY_FILE="${CACHE_DIR}/.cache_ready" +# test notebooks serially until the cache is ready, then in parallel WORKERS=1 if [[ -f "$CACHE_READY_FILE" ]]; then WORKERS=4 fi -echo "Running notebooks with ${WORKERS} process(es)." -mkdir -p data images "$CACHE_DIR" -uv run python -c "import osmnx; print(osmnx.__version__)" +# convert the notebooks to .py files to execute them uv run jupyter nbconvert --to python *.ipynb +uv run python -c "import osmnx; print(osmnx.__version__)" +mkdir -p data images "$CACHE_DIR" +echo "Running notebooks with ${WORKERS} process(es)." if (( WORKERS == 1 )); then for filename in *.py; do uv run ipython "$filename" @@ -24,5 +25,5 @@ else printf '%s\n' *.py | xargs -n1 -P4 uv run ipython fi -# only a fully successful run can declare the cache complete +# mark the cache ready after each fully successful run touch "$CACHE_READY_FILE" From f0c07275fe4a777c0220f66badde63642db0bab2 Mon Sep 17 00:00:00 2001 From: Geoff Boeing Date: Sun, 19 Jul 2026 20:59:29 -0700 Subject: [PATCH 4/6] simplify shebang --- notebooks/run_notebooks.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebooks/run_notebooks.sh b/notebooks/run_notebooks.sh index 90fcb0e..813800d 100755 --- a/notebooks/run_notebooks.sh +++ b/notebooks/run_notebooks.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/bin/bash set -euo pipefail export MPLBACKEND="Agg" From b374c58b7e2c18a1942bf299bff8b63bedbcd32e Mon Sep 17 00:00:00 2001 From: Geoff Boeing Date: Sun, 19 Jul 2026 21:08:41 -0700 Subject: [PATCH 5/6] update project config --- pyproject.toml | 4 ++-- uv.lock | 42 +++++++++++++++++++++++------------------- 2 files changed, 25 insertions(+), 21 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 193bf12..706687d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,14 +13,14 @@ version = "2.1.0" # match pinned version above all = ["folium", "jupyterlab", "mapclassify", "igraph"] [dependency-groups] -dev = ["nbdime", "prek"] +dev = ["ipython", "nbconvert", "nbdime", "prek"] [tool.ruff] cache-dir = "~/.cache/prek/ruff" line-length = 100 [tool.ruff.lint] -extend-select = ["A", "B", "E", "F", "I", "UP", "W"] +extend-select = ["A", "B", "E", "F", "I", "RUF200", "UP", "W"] [tool.ruff.lint.pycodestyle] max-line-length = 110 # line length + 10% since it isn't a hard upper bound diff --git a/uv.lock b/uv.lock index 3464d75..bf2ed4d 100644 --- a/uv.lock +++ b/uv.lock @@ -376,7 +376,7 @@ name = "click-plugins" version = "1.1.1.2" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "click" }, + { name = "click", marker = "python_full_version < '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/c3/a4/34847b59150da33690a36da3681d6bbc2ec14ee9a846bc30a6746e5984e4/click_plugins-1.1.1.2.tar.gz", hash = "sha256:d7af3984a99d243c131aa1a828331e7630f4a88a9741fd05c927b204bcf92261", size = 8343, upload-time = "2025-06-25T00:47:37.555Z" } wheels = [ @@ -1680,6 +1680,8 @@ all = [ [package.dev-dependencies] dev = [ + { name = "ipython" }, + { name = "nbconvert" }, { name = "nbdime" }, { name = "prek" }, ] @@ -1696,6 +1698,8 @@ provides-extras = ["all"] [package.metadata.requires-dev] dev = [ + { name = "ipython" }, + { name = "nbconvert" }, { name = "nbdime" }, { name = "prek" }, ] @@ -1802,7 +1806,7 @@ name = "pexpect" version = "4.9.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "ptyprocess" }, + { name = "ptyprocess", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/42/92/cc564bf6381ff43ce1f4d06852fc19a2f11d180f23dc32d9588bee2f149d/pexpect-4.9.0.tar.gz", hash = "sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f", size = 166450, upload-time = "2023-11-25T09:07:26.339Z" } wheels = [ @@ -2276,14 +2280,14 @@ resolution-markers = [ "python_full_version < '3.12' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ - { name = "affine" }, - { name = "attrs" }, - { name = "certifi" }, - { name = "click" }, - { name = "click-plugins" }, - { name = "cligj" }, - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" } }, - { name = "pyparsing" }, + { name = "affine", marker = "python_full_version < '3.12'" }, + { name = "attrs", marker = "python_full_version < '3.12'" }, + { name = "certifi", marker = "python_full_version < '3.12'" }, + { name = "click", marker = "python_full_version < '3.12'" }, + { name = "click-plugins", marker = "python_full_version < '3.12'" }, + { name = "cligj", marker = "python_full_version < '3.12'" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, + { name = "pyparsing", marker = "python_full_version < '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ec/fa/fce8dc9f09e5bc6520b6fc1b4ecfa510af9ca06eb42ad7bdff9c9b8989d0/rasterio-1.4.4.tar.gz", hash = "sha256:c95424e2c7f009b8f7df1095d645c52895cd332c0c2e1b4c2e073ea28b930320", size = 445004, upload-time = "2025-12-12T18:01:08.971Z" } wheels = [ @@ -2338,13 +2342,13 @@ resolution-markers = [ "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ - { name = "affine" }, - { name = "attrs" }, - { name = "certifi" }, - { name = "click" }, - { name = "cligj" }, - { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" } }, - { name = "pyparsing" }, + { name = "affine", marker = "python_full_version >= '3.12'" }, + { name = "attrs", marker = "python_full_version >= '3.12'" }, + { name = "certifi", marker = "python_full_version >= '3.12'" }, + { name = "click", marker = "python_full_version >= '3.12'" }, + { name = "cligj", marker = "python_full_version >= '3.12'" }, + { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "pyparsing", marker = "python_full_version >= '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/f6/88/edb4b66b6cb2c13f123af5a3896bf70c0cbe73ab3cd4243cb4eb0212a0f6/rasterio-1.5.0.tar.gz", hash = "sha256:1e0ea56b02eea4989b36edf8e58a5a3ef40e1b7edcb04def2603accd5ab3ee7b", size = 452184, upload-time = "2026-01-05T16:06:47.169Z" } wheels = [ @@ -2635,7 +2639,7 @@ resolution-markers = [ "python_full_version < '3.12' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ - { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" } }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/7a/97/5a3609c4f8d58b039179648e62dd220f89864f56f7357f5d4f45c29eb2cc/scipy-1.17.1.tar.gz", hash = "sha256:95d8e012d8cb8816c226aef832200b1d45109ed4464303e997c5b13122b297c0", size = 30573822, upload-time = "2026-02-23T00:26:24.851Z" } wheels = [ @@ -2714,7 +2718,7 @@ resolution-markers = [ "python_full_version >= '3.12' and python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'", ] dependencies = [ - { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" } }, + { name = "numpy", version = "2.5.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a7/25/c2700dfaf6442b4effaa91af24ebce5dc9d31bb4a69706313aae70d72cd0/scipy-1.18.0.tar.gz", hash = "sha256:67b2ad2ad54c72ca6d04975a9b2df8c3638c34ddd5b28738e94fc2b57929d378", size = 30774447, upload-time = "2026-06-19T15:01:43.456Z" } wheels = [ From fb916acc3b95ffbdf6d46dc6c953ec2392754564 Mon Sep 17 00:00:00 2001 From: Geoff Boeing Date: Sun, 19 Jul 2026 21:35:09 -0700 Subject: [PATCH 6/6] update project config --- pyproject.toml | 2 +- uv.lock | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 706687d..4044b5f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ requires-python = ">=3.11" version = "2.1.0" # match pinned version above [project.optional-dependencies] -all = ["folium", "jupyterlab", "mapclassify", "igraph"] +examples = ["folium", "jupyterlab", "mapclassify", "igraph"] [dependency-groups] dev = ["ipython", "nbconvert", "nbdime", "prek"] diff --git a/uv.lock b/uv.lock index bf2ed4d..cf2c450 100644 --- a/uv.lock +++ b/uv.lock @@ -1671,7 +1671,7 @@ dependencies = [ ] [package.optional-dependencies] -all = [ +examples = [ { name = "folium" }, { name = "igraph" }, { name = "jupyterlab" }, @@ -1688,13 +1688,13 @@ dev = [ [package.metadata] requires-dist = [ - { name = "folium", marker = "extra == 'all'" }, - { name = "igraph", marker = "extra == 'all'" }, - { name = "jupyterlab", marker = "extra == 'all'" }, - { name = "mapclassify", marker = "extra == 'all'" }, + { name = "folium", marker = "extra == 'examples'" }, + { name = "igraph", marker = "extra == 'examples'" }, + { name = "jupyterlab", marker = "extra == 'examples'" }, + { name = "mapclassify", marker = "extra == 'examples'" }, { name = "osmnx", extras = ["all"], specifier = "==2.1.0" }, ] -provides-extras = ["all"] +provides-extras = ["examples"] [package.metadata.requires-dev] dev = [