ci.yml

name: ML BootCamp CI Pipeline

on:
  push:
    branches:
      - main
  workflow_dispatch:

# Prevent parallel runs on the self-hosted runner to avoid resource exhaustion
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: false

jobs:
  train-model:
    runs-on: self-hosted
    timeout-minutes: 45  # Prevent infinite hangs
    
    defaults:
      run:
        shell: powershell

    env:
      MLFLOW_TRACKING_URI: sqlite:///D:/ML  101/ML_101_BootCamp/mlflow.db
      MLFLOW_EXPERIMENT_NAME: customer_churn_optimization
      MODEL_NAME: customer_churn_model
      GIT_SHA: ${{ github.sha }}
      GIT_REF: ${{ github.ref_name }}
      PYTHONIOENCODING: utf-8
      PYTHONUTF8: 1
      PIP_DISABLE_PIP_VERSION_CHECK: 1
      PIP_NO_INPUT: 1
      PIP_PREFER_BINARY: 1
      CI: true

    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Verify Python installation
        run: |
          Write-Host "=== Python Environment Check ==="
          $pythonCmd = Get-Command python -ErrorAction SilentlyContinue
          if ($pythonCmd) {
            $pythonPath = $pythonCmd.Source
            Write-Host "✅ Python found at: $pythonPath"
            python --version
            python -m pip --version
          } else {
            Write-Host "❌ Python not found in PATH"
            exit 1
          }

      - name: Cache pip packages
        uses: actions/cache@v4
        with:
          path: ~\AppData\Local\pip\cache
          key: pip-${{ runner.os }}-${{ hashFiles('**/requirements.txt') }}-v3
          restore-keys: |
            pip-${{ runner.os }}-${{ hashFiles('**/requirements.txt') }}-
            pip-${{ runner.os }}-

      - name: Cache virtual environment
        id: cache-venv
        uses: actions/cache@v4
        with:
          path: .venv
          key: venv-${{ runner.os }}-py${{ hashFiles('.python-version') }}-${{ hashFiles('**/requirements.txt') }}-v3
          restore-keys: |
            venv-${{ runner.os }}-py${{ hashFiles('.python-version') }}-${{ hashFiles('**/requirements.txt') }}-
            venv-${{ runner.os }}-py${{ hashFiles('.python-version') }}-

      - name: Display cache status
        run: |
          Write-Host "=== Cache Status ===" -ForegroundColor Cyan
          $cacheHit = "${{ steps.cache-venv.outputs.cache-hit }}"
          if ($cacheHit -eq "true") {
            Write-Host "Virtual env cache hit: $cacheHit" -ForegroundColor Green
          } else {
            Write-Host "Virtual env cache hit: $cacheHit" -ForegroundColor Yellow
          }

      - name: Create virtual environment
        run: |
          if (!(Test-Path ".venv")) {
            Write-Host "Creating virtual environment..."
            python -m venv .venv
            Write-Host "✅ Virtual environment created"
          } else {
            Write-Host "✅ Virtual environment already exists (from cache)"
            Write-Host "Cache hit: ${{ steps.cache-venv.outputs.cache-hit }}"
          }

      - name: Install dependencies
        timeout-minutes: 10
        run: |
          $ErrorActionPreference = "Stop"
          & ".venv\Scripts\Activate.ps1"
          
          Write-Host "Checking for uv..."
          $uv = Get-Command uv -ErrorAction SilentlyContinue
          
          if ($uv) {
            Write-Host "✅ uv found, using uv for faster installation"
            uv pip install --upgrade pip wheel
            uv pip install -r requirements.txt --compile
          } else {
            Write-Host "⚠️ uv not found, falling back to pip"
            python -m pip install --upgrade pip wheel --quiet
            python -m pip install -r requirements.txt --only-binary :all: --compile --no-warn-script-location
          }
          
          Write-Host "Verifying installation..."
          python -c "import mlflow, sklearn, pandas, numpy, optuna; print('All packages verified')"
          Write-Host "Dependencies ready"

      - name: Verify git info
        run: |
          git config user.name "CI Bot"
          git config user.email "ci-bot@example.com"
          git log -1 --pretty=format:"Commit: %H%nAuthor: %an%nDate: %ai%nMessage: %s"

      - name: Verify data files exist
        run: |
          $ErrorActionPreference = "Stop"
          if (!(Test-Path "Data/customer_churn_dataset_prepared.csv")) {
            Write-Host "❌ Training data not found!"
            exit 1
          }
          Write-Host "✅ Data files verified:"
          Get-ChildItem Data/*.csv | ForEach-Object { Write-Host "  - $($_.Name)" }

      - name: Monitor system resources before training
        run: |
          Write-Host "=== System Resources Before Training ===" -ForegroundColor Cyan
          $mem = Get-CimInstance Win32_OperatingSystem
          $totalMemGB = [math]::Round($mem.TotalVisibleMemorySize/1MB, 2)
          $freeMemGB = [math]::Round($mem.FreePhysicalMemory/1MB, 2)
          $usedMemGB = [math]::Round($totalMemGB - $freeMemGB, 2)
          Write-Host "Total Memory: $totalMemGB GB"
          Write-Host "Used Memory: $usedMemGB GB"
          Write-Host "Free Memory: $freeMemGB GB"
          
          if ($freeMemGB -lt 2) {
            Write-Host "WARNING: Less than 2GB free memory. Training may fail." -ForegroundColor Yellow
          }

      - name: Run model training
        timeout-minutes: 30
        run: |
          $ErrorActionPreference = "Stop"
          & ".venv\Scripts\Activate.ps1"
          
          Write-Host "Starting model training with memory constraints..."
          Write-Host "Setting environment variables for memory efficiency..."
          
          # Limit memory usage for numpy/sklearn
          $env:OMP_NUM_THREADS = "2"
          $env:MKL_NUM_THREADS = "2"
          $env:OPENBLAS_NUM_THREADS = "2"
          $env:NUMEXPR_NUM_THREADS = "2"
          
          # Run training with resource monitoring
          $trainingStart = Get-Date
          try {
            python Scripts/model_training.py
            $trainingEnd = Get-Date
            $duration = ($trainingEnd - $trainingStart).TotalMinutes
            Write-Host "✅ Training completed in $([math]::Round($duration, 2)) minutes"
          } catch {
            Write-Host "❌ Training failed: $_" -ForegroundColor Red
            
            # Check if runner is still responsive
            $runnerProcess = Get-Process -Name "Runner.Listener" -ErrorAction SilentlyContinue
            if ($runnerProcess) {
              Write-Host "Runner process is still running (PID: $($runnerProcess.Id))"
            } else {
              Write-Host "WARNING: Runner process may have crashed"
            }
            throw
          }
      
      - name: Monitor system resources after training
        if: always()
        run: |
          Write-Host "=== System Resources After Training ===" -ForegroundColor Cyan
          $mem = Get-CimInstance Win32_OperatingSystem
          $totalMemGB = [math]::Round($mem.TotalVisibleMemorySize/1MB, 2)
          $freeMemGB = [math]::Round($mem.FreePhysicalMemory/1MB, 2)
          $usedMemGB = [math]::Round($totalMemGB - $freeMemGB, 2)
          Write-Host "Total Memory: $totalMemGB GB"
          Write-Host "Used Memory: $usedMemGB GB"
          Write-Host "Free Memory: $freeMemGB GB"

      - name: Training summary
        if: always()
        run: |
          $status = "${{ job.status }}"
          $icon = if ($status -eq "success") { "✅" } else { "❌" }
          
          "# ML Bootcamp Training Report" >> $env:GITHUB_STEP_SUMMARY
          "" >> $env:GITHUB_STEP_SUMMARY
          "### Execution Details" >> $env:GITHUB_STEP_SUMMARY
          "- **Status:** $icon $status" >> $env:GITHUB_STEP_SUMMARY
          "- **Commit:** ${{ github.sha }}" >> $env:GITHUB_STEP_SUMMARY
          "- **Branch:** ${{ github.ref_name }}" >> $env:GITHUB_STEP_SUMMARY
          "- **Runner:** self-hosted (local laptop)" >> $env:GITHUB_STEP_SUMMARY
          "- **Experiment:** $env:MLFLOW_EXPERIMENT_NAME" >> $env:GITHUB_STEP_SUMMARY
          "" >> $env:GITHUB_STEP_SUMMARY
          
          if ($status -eq "success") {
            "### Next Steps" >> $env:GITHUB_STEP_SUMMARY
            "1. Check MLflow UI at http://localhost:5000" >> $env:GITHUB_STEP_SUMMARY
            "2. Verify metrics in the Model Registry" >> $env:GITHUB_STEP_SUMMARY
            "3. CD Pipeline will trigger automatically to deploy if on main" >> $env:GITHUB_STEP_SUMMARY
          } else {
            "### Troubleshooting" >> $env:GITHUB_STEP_SUMMARY
            "1. Check logs above for specific errors" >> $env:GITHUB_STEP_SUMMARY
            "2. Verify if another training process is holding the MLflow DB lock" >> $env:GITHUB_STEP_SUMMARY
          }