Notebook restructuring

.flake8

@@ -19,3 +19,5 @@ exclude =
     .venv
     build
     tutorials_and_tools/tutorials_and_tools/*.py
+    tutorials/**/scripts/*.py
+    tools/**/scripts/*.py

.github/workflows/deploy-books.yml

@@ -5,6 +5,11 @@ on:
   push:
     branches:
     - main
+    - development
+  pull_request:
+    branches:
+    - development
+    - main
     # If your git repository has the Jupyter Book within some-subfolder next to
     # unrelated files, you can make this run only if a file within that specific
     # folder has been modified.
@@ -16,9 +21,8 @@ on:
 jobs:
   deploy-book:
     runs-on: ubuntu-latest
-    environment:
-      name: github-pages
     permissions:
+      contents: write
       pages: write
       id-token: write
     steps:
@@ -47,15 +51,14 @@ jobs:
     # Build the book
     - name: Build the book
       run: |
-        jupyter-book build tutorials_and_tools
+        jupyter-book build tutorials
 
-    # Upload the book's HTML as an artifact
-    - name: Upload artifact
-      uses: actions/upload-pages-artifact@v3
-      with:
-        path: "tutorials_and_tools/_build/html"
-
-    # Deploy the book's HTML to GitHub Pages
+    # Deploy to gh-pages using peaceiris action for all branches
     - name: Deploy to GitHub Pages
-      id: deployment
-      uses: actions/deploy-pages@v4
+      uses: peaceiris/actions-gh-pages@v3
+      with:
+        github_token: ${{ secrets.GITHUB_TOKEN }}
+        publish_dir: tutorials/_build/html
+        publish_branch: gh-pages
+        destination_dir: ${{ github.ref == 'refs/heads/main' && '.' || 'dev-preview' }}
+        keep_files: ${{ github.ref != 'refs/heads/main' }}

.github/workflows/flake8.yaml

@@ -28,4 +28,4 @@ jobs:
       run: echo '::add-matcher::.github/workflows/flake8-linter-matcher.json'
     - name: Run flake8 on changed files only
       run: |
-        git diff origin/$GITHUB_BASE_REF HEAD -- "*.py" | flake8 --config=.flake8 --diff
+        git diff origin/$GITHUB_BASE_REF HEAD --name-only -- "*.py" | grep -v -E "(tutorials/.*/scripts/|tools/.*/scripts/)" | xargs -r git diff origin/$GITHUB_BASE_REF HEAD -- | flake8 --config=.flake8 --diff

.github/workflows/nbval.yaml

@@ -15,22 +15,20 @@ jobs:
     env:
       # Notebooks with estimated runtime in seconds (format: "notebook:runtime")
       ALL_NOTEBOOKS: |
-        "1 - Basics, Humidity, Design.ipynb:10"
-        "2 - Degradation.ipynb:5"
-        "3 - Spectral Degradation.ipynb:5"
-        "4 - Weather Database Access.ipynb:15"
-        "ASTM Live Demo.ipynb:10"
-        "B-O LID - Accelerated Test.ipynb:10"
-        "Custom-Functions-Nopython.ipynb:30"
-        "LETID - Accelerated Test.ipynb:120"
-        "LETID - Outdoor LETID.ipynb:30"
-        "LETID - Outdoor Scenario Based on Accelerated Test Result.ipynb:45"
-        "LETID - Passivated Wafer.ipynb:80"
-        "Monte Carlo - Arrhenius.ipynb:180"
-        "Monte Carlo - Standoff.ipynb:30"
-        "Tools - Edge Seal Oxygen Ingress.ipynb:15"
-        "Van't Hoff Degradation Model.ipynb:5"
-        "Geospatial - Local Scenario.ipynb:1"
+        "01_basics/01_basics_humidity_design.ipynb:10"
+        "01_basics/02_degradation.ipynb:5"
+        "01_basics/03_spectral_degradation.ipynb:5"
+        "01_basics/04_weather_database_access.ipynb:15"
+        "02_degradation/01_bo_lid_accelerated_test.ipynb:10"
+        "02_degradation/02_letid_accelerated_test.ipynb:120"
+        "02_degradation/03_letid_outdoor.ipynb:30"
+        "02_degradation/04_letid_outdoor_scenario.ipynb:45"
+        "02_degradation/05_letid_passivated_wafer.ipynb:80"
+        "02_degradation/06_vant_hoff_degradation_model.ipynb:5"
+        "03_monte_carlo/01_arrhenius.ipynb:180"
+        "03_monte_carlo/02_standoff.ipynb:30"
+        "06_advanced/01_custom_functions_nopython.ipynb:30"
+        "10_workshop_demos/01_astm_live_demo.ipynb:10"
     steps:
       - name: Calculate batch matrix
         id: set-matrix
@@ -150,7 +148,7 @@ jobs:
       fail-fast: false
       matrix: ${{ fromJson(needs.calculate-batches.outputs.matrix) }}
     env:
-      NOTEBOOK_DIR: "tutorials_and_tools/tutorials_and_tools"
+      NOTEBOOK_DIR: "tutorials"
       SANITIZE_CFG: "nbval_sanitization_rules.cfg"
       ALL_NOTEBOOKS: ${{ needs.calculate-batches.outputs.notebooks }}
       BATCH_ASSIGNMENTS: ${{ needs.calculate-batches.outputs.batch_assignments }}
@@ -167,14 +165,14 @@ jobs:
         uses: actions/cache@v4
         with:
           path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ hashFiles('tutorials_and_tools/requirements.txt', 'pyproject.toml') }}
+          key: ${{ runner.os }}-pip-${{ hashFiles('tutorials/requirements.txt', 'pyproject.toml') }}
           restore-keys: |
             ${{ runner.os }}-pip-
 
       - name: Install notebook environment
         run: |
           python -m pip install --upgrade pip wheel
-          pip install --timeout=300 -r tutorials_and_tools/requirements.txt
+          pip install --timeout=300 -r tutorials/requirements.txt
           pip install --timeout=300 -e .[test]
           pip install nbval global_land_mask dotenv imageio nrel-pysam
           pip install nbconvert[webpdf]

.github/workflows/publish-to-pypi.yml

@@ -1,6 +1,9 @@
 name: Publish Python 🐍 distribution 📦 to PyPI and TestPyPI
 
-on: push
+on:
+  push:
+    tags:
+      - '*'
 
 jobs:
   build:
@@ -9,6 +12,8 @@ jobs:
 
     steps:
     - uses: actions/checkout@v4
+      with:
+        fetch-depth: 0  # Fetch all history for setuptools_scm
     - name: Set up Python
       uses: actions/setup-python@v5
       with:
@@ -27,10 +32,35 @@ jobs:
         name: python-package-distributions
         path: dist/
 
+  publish-to-testpypi:
+    name: Publish Python 🐍 distribution 📦 to TestPyPI
+    if: startsWith(github.ref, 'refs/tags/')
+    needs:
+    - build
+    runs-on: ubuntu-latest
+    environment:
+      name: testpypi
+      url: https://test.pypi.org/p/pvdeg
+    permissions:
+      id-token: write  # IMPORTANT: mandatory for trusted publishing
+
+    steps:
+    - name: Download all the dists
+      uses: actions/download-artifact@v4
+      with:
+        name: python-package-distributions
+        path: dist/
+    - name: Publish distribution 📦 to TestPyPI
+      uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+        repository-url: https://test.pypi.org/legacy/
+        skip-existing: true
+
   publish-to-pypi:
     name: >-
       Publish Python 🐍 distribution 📦 to PyPI
-    if: startsWith(github.ref, 'refs/tags/')  # only publish to PyPI on tag pushes
+    # Only publish to PyPI on final release tags (not rc, alpha, beta, etc.)
+    if: startsWith(github.ref, 'refs/tags/') && !contains(github.ref, 'rc') && !contains(github.ref, 'alpha') && !contains(github.ref, 'beta') && !contains(github.ref, 'a') && !contains(github.ref, 'b')
     needs:
     - build
     runs-on: ubuntu-latest

.github/workflows/testbook.yaml

@@ -6,17 +6,16 @@ jobs:
   notebook-testbook-check:
     runs-on: ubuntu-latest
     env:
-      NOTEBOOK_DIR: "tutorials_and_tools/tutorials_and_tools"
+      NOTEBOOK_DIR: "tutorials"
       TESTBOOK_NOTEBOOK_LIST: |
-        "API Call Required - PYSam.ipynb"
-        "API Call Required - load_nsrdb_distributed.ipynb"
-        "HPC Connection Required - Scenario - Geographical Features.ipynb"
-        "HPC Connection Required - Scenario - Geospatial.ipynb"
-        "HPC Connection Required - Scenario - Non-uniform Mountain Prefferential Downselect.ipynb"
-        "HPC Connection Required - Scenario - Single Location.ipynb"
-        "HPC Connection Required - Scenario - Temperature.ipynb"
-        "HPC Connection Required - Tools - Module Standoff for IEC TS 63126.ipynb"
-        "HPC Connection Required - Geospatial Templates.ipynb"
+        "06_advanced/03_pysam_api.ipynb"
+        "06_advanced/04_nsrdb_distributed_api.ipynb"
+        "04_scenario/01_scenario_temperature.ipynb"
+        "04_scenario/02_scenario_single_location.ipynb"
+        "04_scenario/03_scenario_mountain_downselect.ipynb"
+        "05_geospatial/01_geospatial_templates.ipynb"
+        "05_geospatial/04_scenario_geographical_features.ipynb"
+        "05_geospatial/05_scenario_geospatial.ipynb"
 
     steps:
       - uses: actions/checkout@v4
@@ -27,7 +26,7 @@ jobs:
       - name: Install notebook environment
         run: |
           python -m pip install --upgrade pip wheel
-          pip install --timeout=300 -r tutorials_and_tools/requirements.txt
+          pip install --timeout=300 -r tutorials/requirements.txt
           pip install --timeout=300 -e .[test]
           pip install testbook
           pip install global_land_mask dotenv imageio nrel-pysam

.gitignore

@@ -69,7 +69,10 @@ instance/
 docs/sphinx/_build/
 
 # tutorial jupyter-book
+_build/
 pvdeg_tutorials/_build/
+tutorials/_build/
+tutorials_and_tools/_build/
 
 # PyBuilder
 target/

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 # See https://pre-commit.com for more information
 # See https://pre-commit.com/hooks.html for more hooks
-exclude: '^(docs/source/|tutorials_and_tools/_build/)'
+exclude: '^(docs/source/|tutorials/_build/)'
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v6.0.0
@@ -14,9 +14,17 @@ repos:
     rev: 25.9.0
     hooks:
     -   id: black
-        exclude: '^tutorials_and_tools/'
+        exclude: '^tutorials/'
 -   repo: https://github.com/PyCQA/flake8
     rev: 7.3.0 # Use the desired flake8 version
     hooks:
     -   id: flake8
-        exclude: '^tutorials_and_tools/'
+        exclude: '^tutorials/'
+-   repo: https://github.com/mwouts/jupytext
+    rev: v1.18.1
+    hooks:
+    -   id: jupytext
+        args: [--sync, --pipe, black]
+        files: '^tutorials/.*\.ipynb$'
+        additional_dependencies:
+          - black==25.9.0

.readthedocs.yaml

@@ -10,23 +10,20 @@ build:
   os: ubuntu-22.04
   tools:
     python: "3.11"
-    # You can also specify other tool versions:
-    # nodejs: "19"
-    # rust: "1.64"
-    # golang: "1.19"
 
 # Build documentation in the docs/ directory with Sphinx
 sphinx:
    configuration: docs/source/conf.py
 
-# If using Sphinx, optionally build your docs in additional formats such as PDF
-# formats:
-#    - pdf
-
 # Optionally declare the Python requirements required to build your docs
 python:
     install:
         - method: pip
           path: .
           extra_requirements:
               - docs
+
+# Note: Branch-to-version mapping is configured in ReadTheDocs dashboard:
+# - 'main' branch -> 'stable' version (default)
+# - 'development' branch -> 'latest' version
+# To configure: Project Settings > Versions > Activate/set as default