diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index d44b57a..cf79f33 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -44,7 +44,7 @@ "onAutoForward": "notify" } }, - "postCreateCommand": "bash -lc 'set -euo pipefail; cd /workspaces/medtech-edge-analytics; rm -rf .venv; /usr/local/bin/python -m venv .venv; .venv/bin/python -m pip install --upgrade pip setuptools wheel; .venv/bin/pip install -r requirements-dev.txt'", + "postCreateCommand": "bash -lc 'set -euo pipefail; cd /workspaces/medtech-edge-analytics; rm -rf .venv; /usr/local/bin/python -m venv .venv; .venv/bin/python -m pip install --upgrade pip setuptools wheel; .venv/bin/pip install -r requirements-dev.txt; mkdir -p /usr/share/medtech/contracts/vitals /usr/share/medtech/models; cp contracts/vitals/vitals.schema.json /usr/share/medtech/contracts/vitals/vitals.schema.json; cp models/imx8-compatible-sepsis.tflite /usr/share/medtech/models/imx8-compatible-sepsis.tflite'", "postStartCommand": "bash -lc 'set -euo pipefail; cd /workspaces/medtech-edge-analytics; if ! .venv/bin/python -c \"import pytest, pandas, sklearn\" >/dev/null 2>&1; then .venv/bin/pip install -r requirements-dev.txt; fi'", "remoteUser": "root", "remoteEnv": { diff --git a/.github/actions/smoke-test/action.yml b/.github/actions/smoke-test/action.yml index 170ec4f..f311ede 100644 --- a/.github/actions/smoke-test/action.yml +++ b/.github/actions/smoke-test/action.yml @@ -60,7 +60,7 @@ runs: --network smoke-net \ -e MQTT_BROKER=medtech-vitals-publisher \ -e MQTT_PORT=1883 \ - -e MODEL_PATH=models/sepsis_model.tflite \ + -e MODEL_PATH=models/imx8-compatible-sepsis.tflite \ -e LOGLEVEL=INFO \ ${{ inputs.image }} @@ -87,6 +87,20 @@ runs: -C 1 echo "Functional verification passed — prediction received on medtech/predictions/sepsis." + - name: Check for runtime inference errors (informational) + shell: bash + run: | + if docker logs medtech-edge 2>&1 | grep -Eq "Scoring failed|Input shape mismatch|Failed to load model"; then + echo "⚠️ WARNING: Analytics container reported inference/runtime errors:" + docker logs medtech-edge 2>&1 | grep -E "Scoring failed|Input shape mismatch|Failed to load model" || true + echo "" + echo "This may indicate a model version mismatch or compatibility issue." + echo "Check if model regeneration is required after recent changes." + echo "See: https://github.com/chaithubk/medtech-edge-analytics/blob/main/docs/pipeline-internals.md" + else + echo "✓ No inference/runtime model errors detected in analytics logs." + fi + - name: Dump container logs on failure if: ${{ failure() }} shell: bash diff --git a/.github/scripts/detect_model_changes.py b/.github/scripts/detect_model_changes.py new file mode 100644 index 0000000..09894f5 --- /dev/null +++ b/.github/scripts/detect_model_changes.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +""" +Detect model-affecting changes in src/train_and_convert.py. + +This script analyzes git diffs and identifies changes that require +model regeneration based on predefined patterns. + +Usage: + python .github/scripts/detect_model_changes.py [base_branch] [head_branch] + +Environment Variables: + BASE_BRANCH: Git branch to compare against (default: origin/main) + HEAD_BRANCH: Git branch to compare (default: HEAD) + +Output: + JSON format with keys: + - model_changed: boolean + - matched_patterns: list of matched patterns + - message: human-readable summary +""" + +import subprocess +import json +import sys +from pathlib import Path + +# Patterns that require model regeneration +MODEL_AFFECTING_PATTERNS = [ + "def compute_stats", # Feature engineering logic + "feature_cols", # Feature selection/order + "def build_model", # Model architecture + "keras.layers", # Layer changes + "learning_rate", # Hyperparameter: learning rate + "BATCH_SIZE", # Hyperparameter: batch size + "EPOCHS", # Hyperparameter: epochs + "optimizer", # Optimizer changes + "loss=", # Loss function changes + "BinaryCrossentropy", # Loss function config + "class_weight", # Training logic + "validation_split", # Training data handling + "tf.lite.Optimize", # Quantization changes + "inference_input_type", # Quantization config + "inference_output_type", # Quantization config +] + + +def get_git_diff(base_branch: str, head_branch: str) -> str: + """ + Get git diff between base and head branches. + + Args: + base_branch: Base branch to compare against + head_branch: Head branch to compare + + Returns: + Diff content as string + """ + try: + result = subprocess.run( + ["git", "diff", f"{base_branch}...{head_branch}", "--", "src/train_and_convert.py"], + capture_output=True, + text=True, + check=False, + ) + return result.stdout + except Exception as e: + print(f"Error getting git diff: {e}", file=sys.stderr) + return "" + + +def check_for_model_changes(diff_content: str) -> tuple[bool, list[str]]: + """ + Check if diff contains model-affecting changes. + + Args: + diff_content: Git diff content + + Returns: + Tuple of (model_changed, matched_patterns) + """ + matched_patterns = [] + + for pattern in MODEL_AFFECTING_PATTERNS: + if pattern in diff_content: + matched_patterns.append(pattern) + + return len(matched_patterns) > 0, matched_patterns + + +def main(): + """Main entry point.""" + # Get branch arguments or use environment variables + base_branch = sys.argv[1] if len(sys.argv) > 1 else "origin/main" + head_branch = sys.argv[2] if len(sys.argv) > 2 else "HEAD" + + # For GitHub Actions, override with environment variables if set + base_branch = subprocess.os.environ.get("BASE_BRANCH", base_branch) + head_branch = subprocess.os.environ.get("HEAD_BRANCH", head_branch) + + print(f"Comparing {base_branch}...{head_branch} in src/train_and_convert.py") + print() + + # Get the diff + diff_content = get_git_diff(base_branch, head_branch) + + if diff_content: + print("=== Changes detected ===") + print(diff_content[:500]) # Print first 500 chars for debugging + if len(diff_content) > 500: + print(f"... ({len(diff_content)} chars total)") + print() + else: + print("No changes found in src/train_and_convert.py") + + # Check for model-affecting changes + model_changed, matched_patterns = check_for_model_changes(diff_content) + + # Prepare output + output = { + "model_changed": model_changed, + "matched_patterns": matched_patterns, + "message": ( + f"Model regeneration required: {len(matched_patterns)} pattern(s) detected" + if model_changed + else "No model regeneration required" + ), + } + + # Print as JSON for GitHub Actions to parse + print(json.dumps(output)) + + # Also print human-readable output + print() + if model_changed: + print(f"✓ Model-affecting changes detected ({len(matched_patterns)} pattern(s)):") + for pattern in matched_patterns: + print(f" - {pattern}") + else: + print("✓ No model-affecting changes detected") + + return 0 if model_changed else 1 # Exit 0 if changes found, 1 otherwise + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/scripts/detect_model_changes.sh b/.github/scripts/detect_model_changes.sh new file mode 100644 index 0000000..870d8d3 --- /dev/null +++ b/.github/scripts/detect_model_changes.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# Parse model detection output and set GitHub Actions outputs +# +# Usage: bash .github/scripts/detect_model_changes.sh [base_branch] [head_branch] +# +# Reads from detect_model_changes.py and exports GitHub Actions outputs +# If GITHUB_OUTPUT is not set (local testing), outputs to stdout. + +set -e + +# Get the directory where this script is located +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# Run the Python detection script +BASE_BRANCH="${1:-origin/main}" +HEAD_BRANCH="${2:-HEAD}" + +DETECTION_OUTPUT=$(python "$SCRIPT_DIR/detect_model_changes.py" "$BASE_BRANCH" "$HEAD_BRANCH" 2>&1 | tail -1) + +echo "Detection output: $DETECTION_OUTPUT" + +# Parse JSON output +MODEL_CHANGED=$(echo "$DETECTION_OUTPUT" | python -c "import sys, json; print(json.load(sys.stdin)['model_changed'])" 2>/dev/null || echo "false") +MATCHED_PATTERNS=$(echo "$DETECTION_OUTPUT" | python -c "import sys, json; print(','.join(json.load(sys.stdin)['matched_patterns']))" 2>/dev/null || echo "") + +# Set GitHub Actions outputs or print for local testing +if [ -n "$GITHUB_OUTPUT" ]; then + echo "model_changed=$MODEL_CHANGED" >> "$GITHUB_OUTPUT" + echo "matched_patterns=$MATCHED_PATTERNS" >> "$GITHUB_OUTPUT" +fi + +# Print for debugging +echo "" +echo "=== Output ===" +echo "model_changed=$MODEL_CHANGED" +echo "matched_patterns=$MATCHED_PATTERNS" + +exit 0 diff --git a/.github/scripts/format_model_regen_comment.py b/.github/scripts/format_model_regen_comment.py new file mode 100644 index 0000000..8eb5cb7 --- /dev/null +++ b/.github/scripts/format_model_regen_comment.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +""" +Generate PR comment for model regeneration requirements. + +This script generates a formatted GitHub comment with instructions +for model regeneration when model-affecting changes are detected. + +Usage: + python .github/scripts/format_model_regen_comment.py + +Environment Variables: + MATCHED_PATTERNS: Comma-separated list of matched patterns + +Output: + Formatted markdown comment for GitHub PR +""" + +import sys +import textwrap + + +def generate_comment(patterns: list[str]) -> str: + """ + Generate markdown comment for model regeneration. + + Args: + patterns: List of matched pattern strings + + Returns: + Formatted markdown comment + """ + pattern_list = "\n".join(f" - `{p.strip()}`" for p in patterns) + + comment = f"""⚠️ **Model Regeneration Required** + +This PR contains changes to model architecture, features, training logic, or hyperparameters: + +{pattern_list} + +**Action Required After Merge:** + +1. After this PR is merged to `main`, regenerate the model: + ```bash + python -m src.train_and_convert + ``` + +2. Verify the new model works with the smoke tests: + ```bash + bash tools/check_ci.sh + ``` + +3. Create a model update PR to commit the new `models/imx8-compatible-sepsis.tflite` artifact. + +4. Tag the model commit: + ```bash + git tag models/$(cat synthea_version.txt)-$(date +%s) + git push origin models/$(cat synthea_version.txt)-$(date +%s) + ``` + +See [pipeline-internals.md](docs/pipeline-internals.md) for full CI/CD model update workflow.""" + + return comment + + +def main(): + """Main entry point.""" + patterns_input = sys.argv[1] if len(sys.argv) > 1 else "" + + # Parse patterns from comma-separated input + patterns = [p.strip() for p in patterns_input.split(",") if p.strip()] + + if not patterns: + print("Error: No patterns provided", file=sys.stderr) + sys.exit(1) + + comment = generate_comment(patterns) + print(comment) + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/detect-model-changes.yml b/.github/workflows/detect-model-changes.yml new file mode 100644 index 0000000..74914dd --- /dev/null +++ b/.github/workflows/detect-model-changes.yml @@ -0,0 +1,87 @@ +name: Detect Model Regeneration Needs + +on: + pull_request: + paths: + - 'src/train_and_convert.py' + - '.github/workflows/detect-model-changes.yml' + +jobs: + check-model-changes: + runs-on: ubuntu-latest + permissions: + pull-requests: write + + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Check for model-affecting changes + id: check + env: + BASE_BRANCH: origin/${{ github.base_ref }} + HEAD_BRANCH: HEAD + run: | + bash .github/scripts/detect_model_changes.sh "$BASE_BRANCH" "$HEAD_BRANCH" + + - name: Add label to PR + if: steps.check.outputs.model_changed == 'true' + uses: actions/github-script@v7 + with: + script: | + github.rest.issues.addLabels({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + labels: ['model-regeneration-required'] + }); + console.log('✓ Added label: model-regeneration-required'); + + - name: Post PR comment + if: steps.check.outputs.model_changed == 'true' + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const { execSync } = require('child_process'); + + // Generate comment using the Python script + const patterns = '${{ steps.check.outputs.matched_patterns }}'; + const comment = execSync(`python .github/scripts/format_model_regen_comment.py "${patterns}"`, { encoding: 'utf-8' }); + + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: comment + }); + console.log('✓ Posted PR comment with model regeneration instructions'); + + - name: Summary + if: always() + run: | + if [ "${{ steps.check.outputs.model_changed }}" = "true" ]; then + { + echo "## ⚠️ Model Regeneration Needed" + echo "" + echo "This PR will require model regeneration after merge." + echo "" + echo "**Detected changes:**" + echo "\`\`\`" + echo "${{ steps.check.outputs.matched_patterns }}" + echo "\`\`\`" + } >> "$GITHUB_STEP_SUMMARY" + else + { + echo "## ✓ No Model Regeneration Needed" + echo "" + echo "Changes to \`src/train_and_convert.py\` do not affect model architecture or training logic." + } >> "$GITHUB_STEP_SUMMARY" + fi diff --git a/docs/YOCTO_INTEGRATION.md b/docs/YOCTO_INTEGRATION.md index 9272e9a..dfafb99 100644 --- a/docs/YOCTO_INTEGRATION.md +++ b/docs/YOCTO_INTEGRATION.md @@ -6,7 +6,11 @@ model versions for deterministic releases. ## Delivery Contract - Canonical model path in repository: `models/imx8-compatible-sepsis.tflite` -- Model updates are auto-committed by CI after successful retraining +- Model updates are produced by CI after successful retraining. +- NOTE: For safety and branch protection compliance CI creates or updates a + reviewable pull request (`model-update-sepsis-model`) containing the updated + model artifact; maintainers should review and merge the PR to incorporate + the model into `main`. - CI also creates model tags in format: `models/-` - No runtime download is required on target devices @@ -52,6 +56,24 @@ Point your inference service to the installed model path, for example: This keeps deployment deterministic and aligned with Yocto image contents. +## Local development parity + +To make your development environment behave like the Yocto image, the +devcontainer is configured to copy the vendored schema and canonical model into +the expected rootfs locations on container setup. This mirrors production +runtime paths and avoids needing to set environment variables in local runs. + +If you prefer not to copy files into `/usr/share/medtech/`, you can instead +set the environment variables locally: + +```bash +export MEDTECH_VITALS_SCHEMA=contracts/vitals/vitals.schema.json +export MODEL_PATH=models/imx8-compatible-sepsis.tflite +``` + +Both approaches are supported; the devcontainer copy provides the closest +parity with the Yocto image and is recommended for day-to-day development. + ## Security and Credentials - CI uses built-in `GITHUB_TOKEN` for model commit/tag actions. diff --git a/docs/pipeline-internals.md b/docs/pipeline-internals.md index dd79476..38c0550 100644 --- a/docs/pipeline-internals.md +++ b/docs/pipeline-internals.md @@ -21,9 +21,16 @@ the model consumed by edge and Yocto builds. ## CI Triggering and Control -- Scheduled retraining: weekly (Monday 02:00 UTC) -- Manual retraining: workflow dispatch -- Push-triggered validation: regular quality checks and tests +- Manual retraining: workflow dispatch (manual trigger only). +- Scheduled retraining was removed from the automated pipeline to avoid + unreviewed model updates; retraining runs are intended to be run manually + when an operator wants to refresh the model. +- When a trained model changes, CI now creates or updates a stable pull request + (`model-update-sepsis-model`) with the updated artifact for human review and + merge. The pipeline will continue to create a model tag in the format + `models/-` for release lineage. + (Note: CI no longer pushes changes directly to `main` — pull requests are + used to respect branch protection rules.) ## Artifacts and Lineage @@ -45,11 +52,12 @@ Lineage fields captured in reports/metadata include: ## Versioning Strategy - Canonical runtime path: `models/imx8-compatible-sepsis.tflite` -- CI auto-commit: model changes are committed to `main` - CI model tag format: `models/-` Operationally, Yocto should pin to a commit SHA that corresponds to a verified -model tag for deterministic builds. +model tag for deterministic builds. Note that model updates are now surfaced +through a reviewable pull request (`model-update-sepsis-model`) rather than an +automatic push to `main`. ## Failure and Quality Gates