Deploy Model API #35
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Deploy Model API | |
| on: | |
| push: | |
| branches: | |
| - main | |
| paths: | |
| - 'api/**' | |
| - '.github/workflows/cd.yml' | |
| workflow_run: | |
| workflows: ["ML BootCamp CI Pipeline"] | |
| types: | |
| - completed | |
| branches: | |
| - main | |
| workflow_dispatch: | |
| inputs: | |
| skip_docker_check: | |
| description: 'Skip Docker startup check (use if Docker is already running)' | |
| required: false | |
| default: 'false' | |
| type: choice | |
| options: | |
| - 'false' | |
| - 'true' | |
| deployment_target: | |
| description: 'Choose deployment target' | |
| required: true | |
| default: 'local' | |
| type: choice | |
| options: | |
| - local | |
| - staging | |
| - production | |
| use_docker: | |
| description: 'Use Docker for deployment' | |
| required: false | |
| default: true | |
| type: boolean | |
| # Prevent parallel deployments on the self-hosted runner | |
| concurrency: | |
| group: deployment-${{ github.ref }} | |
| cancel-in-progress: false | |
| env: | |
| MLFLOW_TRACKING_URI: sqlite:///D:/ML 101/ML_101_BootCamp/mlflow.db | |
| MLFLOW_EXPERIMENT_NAME: customer_churn_optimization | |
| MODEL_NAME: customer_churn_model | |
| PYTHONIOENCODING: utf-8 | |
| PYTHONUTF8: 1 | |
| CI: true | |
| jobs: | |
| # Verify prerequisites | |
| check-prerequisites: | |
| runs-on: self-hosted | |
| if: github.event_name != 'workflow_run' || github.event.workflow_run.conclusion == 'success' | |
| outputs: | |
| model-exists: ${{ steps.check.outputs.model }} | |
| preprocessor-exists: ${{ steps.check.outputs.preprocessor }} | |
| defaults: | |
| run: | |
| shell: powershell | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Check required artifacts | |
| id: check | |
| run: | | |
| $modelExists = $false | |
| $preprocessorExists = $false | |
| # Check for best model artifacts | |
| if (Test-Path "mlruns\best_model_artifacts\customer_churn_optimization") { | |
| $modelDirs = Get-ChildItem "mlruns\best_model_artifacts\customer_churn_optimization" -Directory | |
| if ($modelDirs.Count -gt 0) { | |
| Write-Host "Model artifacts found: $($modelDirs[0].Name)" | |
| $modelExists = $true | |
| } | |
| } | |
| # Check for preprocessor | |
| if (Test-Path "Data\preprocessor.pkl") { | |
| Write-Host "Preprocessor found" | |
| $preprocessorExists = $true | |
| } | |
| # Set outputs | |
| "model=$modelExists" >> $env:GITHUB_OUTPUT | |
| "preprocessor=$preprocessorExists" >> $env:GITHUB_OUTPUT | |
| # Fail if required artifacts are missing | |
| if (-not $modelExists) { | |
| Write-Host "No model artifacts found. Run CI pipeline first to train a model." | |
| exit 1 | |
| } | |
| if (-not $preprocessorExists) { | |
| Write-Host "Preprocessor not found. Run data preprocessing first." | |
| exit 1 | |
| } | |
| Write-Host "All required artifacts are present" | |
| # Test API with native Python (faster for development) | |
| test-api: | |
| needs: check-prerequisites | |
| runs-on: self-hosted | |
| defaults: | |
| run: | |
| shell: powershell | |
| env: | |
| BEST_MODEL_METRIC: f1 | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Set up Python environment | |
| run: | | |
| & ".venv\Scripts\Activate.ps1" | |
| Write-Host "Checking for uv..." | |
| $uv = Get-Command uv -ErrorAction SilentlyContinue | |
| if ($uv) { | |
| Write-Host "✅ uv found, installing API dependencies..." | |
| uv pip install -r api/requirements.txt | |
| } else { | |
| Write-Host "⚠️ uv not found, using pip..." | |
| python -m pip install -q -r api/requirements.txt | |
| } | |
| - name: Start API server and run tests | |
| run: | | |
| Write-Host "=== Starting API Server and Running Tests ===" | |
| # Kill any existing server on port 8000 | |
| $existing = Get-NetTCPConnection -LocalPort 8000 -State Listen -ErrorAction SilentlyContinue | |
| if ($existing) { | |
| $processId = $existing.OwningProcess | |
| Write-Host "Stopping existing process on port 8000 (PID: $processId)" | |
| Stop-Process -Id $processId -Force -ErrorAction SilentlyContinue | |
| Start-Sleep -Seconds 2 | |
| } | |
| # Verify dependencies are installed | |
| Write-Host "Verifying API dependencies..." | |
| python -c "import uvicorn, fastapi" 2>$null | |
| if ($LASTEXITCODE -ne 0) { | |
| Write-Host "Dependencies missing, installing..." | |
| python -m pip install -q -r api/requirements.txt | |
| } else { | |
| Write-Host "Dependencies verified" | |
| } | |
| # Start server in background using Start-Process | |
| Write-Host "`nStarting uvicorn server..." | |
| $process = Start-Process python -ArgumentList @( | |
| "-m", "uvicorn", | |
| "api.main:app", | |
| "--host", "127.0.0.1", | |
| "--port", "8000" | |
| ) -WindowStyle Hidden -PassThru -RedirectStandardOutput "api_stdout.log" -RedirectStandardError "api_stderr.log" | |
| Write-Host "Server starting (PID: $($process.Id))..." | |
| Write-Host "Waiting for API to become ready (initial 15s wait)..." | |
| Start-Sleep -Seconds 15 | |
| # Verify server is responding | |
| $maxRetries = 15 | |
| $retryCount = 0 | |
| $serverReady = $false | |
| while ($retryCount -lt $maxRetries -and -not $serverReady) { | |
| # Check if process is still running | |
| $processCheck = Get-Process -Id $process.Id -ErrorAction SilentlyContinue | |
| if (-not $processCheck) { | |
| Write-Host "❌ Server process died unexpectedly!" | |
| if (Test-Path "api_stderr.log") { | |
| Write-Host "--- Error Log (api_stderr.log) ---" | |
| Get-Content "api_stderr.log" | |
| } | |
| exit 1 | |
| } | |
| try { | |
| $response = Invoke-RestMethod -Uri "http://localhost:8000/health" -Method Get -TimeoutSec 5 -ErrorAction Stop | |
| if ($response.status -eq "ok") { | |
| Write-Host "✅ API server is ready and responding!" | |
| $serverReady = $true | |
| } | |
| } catch { | |
| $retryCount++ | |
| Write-Host "Retry $retryCount/$maxRetries - server not ready yet (Status: $($_.Exception.Message))" | |
| if ($retryCount -eq $maxRetries) { | |
| Write-Host "--- Partial Stdout ---" | |
| if (Test-Path "api_stdout.log") { Get-Content "api_stdout.log" -Tail 20 } | |
| Write-Host "--- Partial Stderr ---" | |
| if (Test-Path "api_stderr.log") { Get-Content "api_stderr.log" -Tail 20 } | |
| } | |
| Start-Sleep -Seconds 3 | |
| } | |
| } | |
| if (-not $serverReady) { | |
| Write-Host "❌ API server failed to respond to health checks after $maxRetries retries" | |
| Stop-Process -Id $process.Id -Force -ErrorAction SilentlyContinue | |
| exit 1 | |
| } | |
| # Run API tests | |
| Write-Host "`n=== Testing API Endpoints ===" | |
| try { | |
| # Test health endpoint | |
| Write-Host "Testing /health..." | |
| $health = Invoke-RestMethod -Uri "http://localhost:8000/health" -Method Get | |
| if ($health.status -eq "ok") { | |
| Write-Host " Health check passed" | |
| } else { | |
| throw "Health check failed" | |
| } | |
| # Test schema endpoint | |
| Write-Host "Testing /schema..." | |
| $schema = Invoke-RestMethod -Uri "http://localhost:8000/schema" -Method Get | |
| if ($schema.required_columns) { | |
| Write-Host " Schema endpoint working" | |
| Write-Host " Required columns: $($schema.required_columns.Count)" | |
| } else { | |
| throw "Schema endpoint failed" | |
| } | |
| # Test prediction endpoint | |
| Write-Host "Testing /predict..." | |
| $body = @{ | |
| features = @{ | |
| Age = 30 | |
| Gender = "Female" | |
| Tenure = 39 | |
| "Usage Frequency" = 14 | |
| "Support Calls" = 5 | |
| "Payment Delay" = 18 | |
| "Subscription Type" = "Standard" | |
| "Contract Length" = "Annual" | |
| "Total Spend" = 932 | |
| "Last Interaction" = 17 | |
| } | |
| } | ConvertTo-Json | |
| $prediction = Invoke-RestMethod -Uri "http://localhost:8000/predict" -Method Post -ContentType "application/json" -Body $body | |
| if ($null -ne $prediction.prediction) { | |
| Write-Host " Prediction endpoint working" | |
| Write-Host " Prediction: $($prediction.prediction)" | |
| Write-Host " Probability: $($prediction.probability)" | |
| } else { | |
| throw "Prediction failed" | |
| } | |
| Write-Host "`nAll API tests passed!" | |
| } catch { | |
| Write-Host "Test failed: $_" | |
| Stop-Process -Id $process.Id -Force -ErrorAction SilentlyContinue | |
| exit 1 | |
| } finally { | |
| # Always stop the server | |
| Write-Host "`nStopping API server (PID: $($process.Id))..." | |
| Stop-Process -Id $process.Id -Force -ErrorAction SilentlyContinue | |
| Write-Host "API server stopped" | |
| } | |
| # Build and test Docker image (optional - runs only if Docker is available) | |
| build-docker: | |
| needs: test-api | |
| runs-on: self-hosted | |
| continue-on-error: true | |
| if: success() | |
| defaults: | |
| run: | |
| shell: powershell | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Force cleanup Docker resources (Prevent GUI errors) | |
| run: | | |
| $ErrorActionPreference = "Continue" | |
| Write-Host "=== Force Cleaning Docker Resources ===" | |
| Write-Host "This step prevents Docker Desktop GUI IPC errors from blocking deployment" | |
| Write-Host "" | |
| # Use CLI to forcefully clean up all containers | |
| Write-Host "Stopping all running containers..." | |
| $runningContainers = docker ps -q 2>&1 | |
| if ($runningContainers -and $runningContainers -notmatch "error") { | |
| docker stop $runningContainers 2>&1 | Out-Null | |
| Write-Host "Stopped $($runningContainers.Count) containers" | |
| } else { | |
| Write-Host "No running containers found" | |
| } | |
| # Remove project-specific containers | |
| $projectContainers = @('ml_churn_api', 'mlflow_server', 'ml-api-test', 'ml-api-staging') | |
| foreach ($container in $projectContainers) { | |
| $exists = docker ps -a --filter "name=^/${container}$" --format "{{.Names}}" 2>&1 | |
| if ($exists -eq $container) { | |
| Write-Host "Force removing: $container" | |
| docker rm -f $container 2>&1 | Out-Null | |
| } | |
| } | |
| # Clean docker-compose resources using CLI (avoid GUI) | |
| Write-Host "Cleaning docker-compose resources via CLI..." | |
| docker-compose down --remove-orphans 2>&1 | Out-Null | |
| Write-Host "Cleanup complete - Docker ready for fresh deployment" | |
| Write-Host "" | |
| # Ensure step exits successfully even if some cleanup commands had errors | |
| exit 0 | |
| - name: Verify Docker is available | |
| run: | | |
| $ErrorActionPreference = "Continue" | |
| Write-Host "=== Checking Docker Installation ===" | |
| # Check if Docker Desktop is installed | |
| try { | |
| $dockerVersion = docker --version 2>&1 | |
| Write-Host "Docker found: $dockerVersion" | |
| $dockerComposeVersion = docker-compose --version 2>&1 | |
| Write-Host "Docker Compose found: $dockerComposeVersion" | |
| } catch { | |
| Write-Host "Docker is not installed" | |
| Write-Host "Please install Docker Desktop from https://www.docker.com/products/docker-desktop" | |
| exit 1 | |
| } | |
| # First, do a quick check if Docker daemon is already accessible | |
| Write-Host "`nQuick Docker daemon check..." | |
| $null = docker ps 2>&1 | |
| if ($LASTEXITCODE -eq 0) { | |
| Write-Host "Docker daemon is already ready!" | |
| docker version --format "Server: {{.Server.Version}}" | |
| $containerCount = (docker ps -q 2>&1 | Measure-Object | Select-Object -ExpandProperty Count) | |
| Write-Host "Containers running: $containerCount" | |
| Write-Host "`nDocker is fully ready for use!" | |
| exit 0 | |
| } | |
| Write-Host "Docker daemon not immediately accessible. Checking Docker Desktop process..." | |
| # Check if Docker Desktop is already running | |
| $dockerDesktopProcess = Get-Process "Docker Desktop" -ErrorAction SilentlyContinue | |
| if (-not $dockerDesktopProcess) { | |
| Write-Host "`n========================================" -ForegroundColor Red | |
| Write-Host "ERROR: Docker Desktop is NOT running" -ForegroundColor Red | |
| Write-Host "========================================" -ForegroundColor Red | |
| Write-Host "`nDocker Desktop must be started BEFORE triggering this workflow." | |
| Write-Host "`nTo fix this issue:" | |
| Write-Host "1. Run the Docker startup script:" | |
| Write-Host " .\start_docker_and_wait.ps1" | |
| Write-Host "`n2. Then trigger the workflow again" | |
| Write-Host "`nAlternatively, configure Docker Desktop to start on Windows login." | |
| exit 1 | |
| } else { | |
| Write-Host "`nDocker Desktop process found (PID: $($dockerDesktopProcess.Id))" | |
| Write-Host "Process is running but daemon is not ready yet. Waiting..." | |
| } | |
| # Wait for Docker daemon to be ready (shorter timeout since process is running) | |
| Write-Host "`nWaiting for Docker daemon to be ready..." | |
| $maxRetries = 30 | |
| $retryCount = 0 | |
| $dockerReady = $false | |
| $lastError = "" | |
| while ($retryCount -lt $maxRetries -and -not $dockerReady) { | |
| try { | |
| $dockerPsOutput = docker ps 2>&1 | |
| $exitCode = $LASTEXITCODE | |
| if ($exitCode -eq 0) { | |
| Write-Host "Docker daemon is ready!" | |
| $dockerReady = $true | |
| break | |
| } else { | |
| $lastError = $dockerPsOutput | |
| } | |
| } catch { | |
| $lastError = $_.Exception.Message | |
| } | |
| $retryCount++ | |
| if ($retryCount % 5 -eq 0) { | |
| $elapsed = $retryCount * 3 | |
| Write-Host "Still waiting... (attempt $retryCount/$maxRetries, $elapsed seconds elapsed)" | |
| } | |
| Start-Sleep -Seconds 3 | |
| } | |
| if (-not $dockerReady) { | |
| $totalWait = $maxRetries * 3 | |
| Write-Host "`n========================================" | |
| Write-Host "ERROR: Docker daemon failed to become ready" | |
| Write-Host "========================================" | |
| Write-Host "Docker Desktop process is running but daemon did not respond after $totalWait seconds" | |
| Write-Host "`nLast error: $lastError" | |
| Write-Host "`nRECOMMENDED SOLUTION:" | |
| Write-Host "1. BEFORE pushing code or triggering workflows, run:" | |
| Write-Host " .\start_docker_and_wait.ps1" | |
| Write-Host "`n2. This ensures Docker daemon is fully ready" | |
| Write-Host "`n3. Then trigger the workflow" | |
| Write-Host "`nAlternative: Set Docker Desktop to auto-start on Windows login" | |
| exit 1 | |
| } | |
| # Verify Docker is working properly | |
| Write-Host "`nVerifying Docker functionality..." | |
| docker version --format "Server: {{.Server.Version}}" | |
| $containerCount = (docker ps -q | Measure-Object | Select-Object -ExpandProperty Count) | |
| Write-Host "Containers running: $containerCount" | |
| Write-Host "`nDocker is fully ready for use!" | |
| - name: Clean up old Docker resources | |
| run: | | |
| Write-Host "=== Cleaning up old Docker resources ===" | |
| # Stop and remove existing containers | |
| $existingContainers = docker ps -a --filter "name=ml-api" --format "{{.Names}}" | |
| if ($existingContainers) { | |
| Write-Host "Stopping existing containers: $existingContainers" | |
| docker stop $existingContainers 2>$null | |
| docker rm $existingContainers 2>$null | |
| } | |
| # Remove old images (keep last 3) | |
| $oldImages = docker images ml-churn-api --format "{{.ID}}" | Select-Object -Skip 3 | |
| if ($oldImages) { | |
| Write-Host "Removing old images..." | |
| $oldImages | ForEach-Object { docker rmi $_ -f 2>$null } | |
| } | |
| Write-Host "Cleanup complete" | |
| - name: Build Docker image | |
| run: | | |
| Write-Host "=== Building Docker Image ===" | |
| $startTime = Get-Date | |
| # Build with build args and tags | |
| docker build ` | |
| --build-arg BUILD_DATE=$(Get-Date -Format 'yyyy-MM-ddTHH:mm:ssZ') ` | |
| --build-arg GIT_COMMIT=${{ github.sha }} ` | |
| -t ml-churn-api:latest ` | |
| -t ml-churn-api:${{ github.sha }} ` | |
| . | |
| if ($LASTEXITCODE -ne 0) { | |
| Write-Host "Docker build failed" | |
| exit 1 | |
| } | |
| $duration = (Get-Date) - $startTime | |
| Write-Host "Docker image built successfully in $($duration.TotalSeconds) seconds" | |
| # Show image info | |
| Write-Host "`nImage details:" | |
| docker images ml-churn-api:latest | |
| - name: Test Docker container | |
| run: | | |
| $ErrorActionPreference = "Continue" | |
| Write-Host "=== Testing Docker Container ===" | |
| # Clean up any existing test containers and containers using port 8000 | |
| Write-Host "Cleaning up existing test containers..." | |
| $testContainer = docker ps -a --filter "name=^/ml-api-test$" --format "{{.Names}}" 2>&1 | |
| if ($testContainer -eq "ml-api-test") { | |
| Write-Host "Removing existing ml-api-test container" | |
| docker rm -f ml-api-test 2>&1 | Out-Null | |
| } else { | |
| Write-Host "No existing ml-api-test container found" | |
| } | |
| # Stop any containers using port 8000 | |
| $containersOnPort = docker ps --filter "publish=8000" --format "{{.Names}}" 2>&1 | Where-Object { $_ -notmatch "error" } | |
| if ($containersOnPort) { | |
| Write-Host "Stopping containers on port 8000: $containersOnPort" | |
| docker stop $containersOnPort 2>&1 | Out-Null | |
| docker rm $containersOnPort 2>&1 | Out-Null | |
| } | |
| # Start container | |
| Write-Host "Starting Docker container..." | |
| docker run -d ` | |
| -p 8000:8000 ` | |
| --name ml-api-test ` | |
| --env BEST_MODEL_METRIC=f1 ` | |
| --env MLFLOW_EXPERIMENT_NAME=customer_churn_optimization ` | |
| ml-churn-api:latest | |
| if ($LASTEXITCODE -ne 0) { | |
| Write-Host "Failed to start container" | |
| exit 1 | |
| } | |
| Write-Host "Container started, waiting for API to be ready..." | |
| Start-Sleep -Seconds 15 | |
| # Check container logs | |
| Write-Host "`nContainer logs:" | |
| docker logs ml-api-test | |
| # Verify container is running | |
| $containerStatus = docker ps --filter "name=ml-api-test" --format "{{.Status}}" | |
| Write-Host "`nContainer status: $containerStatus" | |
| # Test health endpoint | |
| Write-Host "`nTesting containerized API..." | |
| $maxRetries = 10 | |
| $retryCount = 0 | |
| $apiReady = $false | |
| while ($retryCount -lt $maxRetries -and -not $apiReady) { | |
| try { | |
| $health = Invoke-RestMethod -Uri "http://localhost:8000/health" -Method Get -TimeoutSec 5 -ErrorAction Stop | |
| if ($health.status -eq "ok") { | |
| Write-Host "Health check passed" | |
| $apiReady = $true | |
| } | |
| } catch { | |
| $retryCount++ | |
| Write-Host "Retry $retryCount/$maxRetries - waiting for API..." | |
| Start-Sleep -Seconds 3 | |
| } | |
| } | |
| if (-not $apiReady) { | |
| Write-Host "API health check failed" | |
| Write-Host "`nFinal container logs:" | |
| docker logs ml-api-test | |
| docker stop ml-api-test | |
| docker rm ml-api-test | |
| exit 1 | |
| } | |
| # Test prediction endpoint | |
| Write-Host "`nTesting prediction endpoint..." | |
| $body = @{ | |
| features = @{ | |
| Age = 35 | |
| Gender = "Male" | |
| Tenure = 24 | |
| "Usage Frequency" = 15 | |
| "Support Calls" = 2 | |
| "Payment Delay" = 5 | |
| "Subscription Type" = "Standard" | |
| "Contract Length" = "Annual" | |
| "Total Spend" = 1200 | |
| "Last Interaction" = 30 | |
| } | |
| } | ConvertTo-Json | |
| try { | |
| $prediction = Invoke-RestMethod ` | |
| -Uri "http://localhost:8000/predict" ` | |
| -Method Post ` | |
| -ContentType "application/json" ` | |
| -Body $body | |
| Write-Host "Prediction successful" | |
| Write-Host " Prediction: $($prediction.prediction)" | |
| Write-Host " Probability: $($prediction.probability)" | |
| } catch { | |
| Write-Host "Prediction test failed: $_" | |
| docker stop ml-api-test | |
| docker rm ml-api-test | |
| exit 1 | |
| } | |
| # Cleanup test container | |
| Write-Host "`nStopping test container..." | |
| docker stop ml-api-test | |
| docker rm ml-api-test | |
| Write-Host "Docker container tests passed!" | |
| # Deploy to selected environment | |
| deploy: | |
| needs: [check-prerequisites, test-api] | |
| runs-on: self-hosted | |
| if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' | |
| defaults: | |
| run: | |
| shell: powershell | |
| env: | |
| API_PORT_LOCAL: 8000 | |
| API_PORT_STAGING: 8001 | |
| API_PORT_PRODUCTION: 8002 | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Force cleanup Docker resources before deployment | |
| run: | | |
| $ErrorActionPreference = "Continue" | |
| Write-Host "=== Pre-Deployment Docker Cleanup ===" | |
| Write-Host "Preventing Docker Desktop GUI IPC communication errors" | |
| Write-Host "" | |
| # Force stop and remove all project containers using CLI | |
| $projectContainers = @('ml_churn_api', 'mlflow_server', 'ml-api-test', 'ml-api-staging') | |
| foreach ($container in $projectContainers) { | |
| $exists = docker ps -a --filter "name=^/${container}$" --format "{{.Names}}" 2>&1 | |
| if ($exists -eq $container) { | |
| Write-Host "Force removing: $container" | |
| docker rm -f $container 2>&1 | Out-Null | |
| } | |
| } | |
| # Force clean docker-compose using CLI (bypasses GUI) | |
| Write-Host "Force cleaning docker-compose resources..." | |
| docker-compose down --remove-orphans --volumes 2>&1 | Out-Null | |
| # Clean up dangling resources | |
| Write-Host "Removing dangling resources..." | |
| docker system prune -f 2>&1 | Out-Null | |
| Write-Host "Cleanup complete - Ready for deployment" | |
| Write-Host "" | |
| - name: Set deployment target | |
| id: set-target | |
| run: | | |
| $target = "${{ inputs.deployment_target }}" | |
| if ([string]::IsNullOrEmpty($target)) { | |
| $target = "local" | |
| } | |
| $useDocker = "${{ inputs.use_docker }}" | |
| if ([string]::IsNullOrEmpty($useDocker)) { | |
| $useDocker = "true" | |
| } | |
| Write-Host "Deployment target: $target" | |
| Write-Host "Use Docker: $useDocker" | |
| "DEPLOYMENT_TARGET=$target" >> $env:GITHUB_ENV | |
| "USE_DOCKER=$useDocker" >> $env:GITHUB_ENV | |
| "target=$target" >> $env:GITHUB_OUTPUT | |
| "use_docker=$useDocker" >> $env:GITHUB_OUTPUT | |
| - name: Debug deployment target | |
| run: | | |
| Write-Host "DEPLOYMENT_TARGET: $env:DEPLOYMENT_TARGET" | |
| Write-Host "Event name: ${{ github.event_name }}" | |
| - name: Deploy with Docker Compose (Local) | |
| if: ${{ steps.set-target.outputs.target == 'local' }} | |
| run: | | |
| $ErrorActionPreference = "Continue" | |
| Write-Host "=== Deploying to Local with Docker Compose ===" | |
| # Verify Docker is accessible before attempting deployment | |
| Write-Host "Verifying Docker daemon is accessible..." | |
| $null = docker ps 2>&1 | |
| if ($LASTEXITCODE -ne 0) { | |
| Write-Host "" | |
| Write-Host "========================================" -ForegroundColor Red | |
| Write-Host "ERROR: Docker daemon is not accessible" -ForegroundColor Red | |
| Write-Host "========================================" -ForegroundColor Red | |
| Write-Host "" | |
| Write-Host "Cannot deploy with Docker Compose because Docker daemon is not running." | |
| Write-Host "" | |
| Write-Host "SOLUTION:" | |
| Write-Host "1. Run the Docker startup script BEFORE pushing code:" | |
| Write-Host " .\start_docker_and_wait.ps1" | |
| Write-Host "" | |
| Write-Host "2. Then push or trigger the workflow again" | |
| Write-Host "" | |
| Write-Host "Docker must be running before CI/CD workflows are triggered." | |
| exit 1 | |
| } | |
| Write-Host "Docker daemon is accessible - proceeding with deployment" | |
| # Force remove any existing containers with same names | |
| Write-Host "\nCleaning up existing containers..." | |
| $existingContainers = @('ml_churn_api', 'mlflow_server') | |
| foreach ($container in $existingContainers) { | |
| $exists = docker ps -a --filter "name=^/${container}$" --format "{{.Names}}" 2>&1 | |
| if ($exists -eq $container) { | |
| Write-Host "Removing existing container: $container" | |
| docker rm -f $container 2>&1 | Out-Null | |
| } | |
| } | |
| # Stop and remove docker-compose services (force stop with timeout 0) | |
| Write-Host "Force stopping docker-compose services..." | |
| $null = docker-compose down --timeout 0 --remove-orphans 2>&1 | |
| # Start services | |
| Write-Host "Starting services with docker-compose..." | |
| docker-compose up -d 2>&1 | Where-Object { $_ -notmatch 'level=warning' } | Write-Host | |
| if ($LASTEXITCODE -ne 0) { | |
| Write-Host "Docker Compose deployment failed" | |
| Write-Host "Checking Docker daemon status..." | |
| docker ps 2>&1 | |
| exit 1 | |
| } | |
| Write-Host "Waiting for services to start..." | |
| Start-Sleep -Seconds 20 | |
| # Verify deployment | |
| Write-Host "`nVerifying deployment..." | |
| $health = Invoke-RestMethod -Uri "http://localhost:${{ env.API_PORT_LOCAL }}/health" -Method Get | |
| if ($health.status -eq "ok") { | |
| Write-Host "Local deployment successful!" | |
| Write-Host "`n=== Deployment Info ===" | |
| Write-Host "API: http://localhost:${{ env.API_PORT_LOCAL }}" | |
| Write-Host "Docs: http://localhost:${{ env.API_PORT_LOCAL }}/docs" | |
| Write-Host "MLflow: http://localhost:5000" | |
| Write-Host "`nRunning containers:" | |
| docker ps --filter "name=ml_churn_api" | |
| } else { | |
| Write-Host "Health check failed" | |
| Write-Host "`nContainer logs:" | |
| docker logs ml_churn_api | |
| exit 1 | |
| } | |
| - name: Deploy with Docker (Staging) | |
| if: ${{ steps.set-target.outputs.target == 'staging' }} | |
| run: | | |
| Write-Host "=== Deploying to Staging with Docker ===" | |
| # Stop existing staging container | |
| Write-Host "Checking for existing staging container..." | |
| $existingContainer = docker ps -a --filter "name=ml-api-staging" --format "{{.Names}}" | |
| if ($existingContainer) { | |
| Write-Host "Stopping and removing existing staging container..." | |
| docker stop ml-api-staging 2>$null | |
| docker rm ml-api-staging 2>$null | |
| Write-Host "Existing container removed" | |
| } | |
| # Start staging container with production-like settings | |
| Write-Host "Starting staging container on port ${{ env.API_PORT_STAGING }}..." | |
| docker run -d ` | |
| -p ${{ env.API_PORT_STAGING }}:8000 ` | |
| --name ml-api-staging ` | |
| --restart unless-stopped ` | |
| --env BEST_MODEL_METRIC=f1 ` | |
| --env MLFLOW_EXPERIMENT_NAME=customer_churn_optimization ` | |
| --env ENVIRONMENT=staging ` | |
| --memory="2g" ` | |
| --cpus="1.0" ` | |
| --health-cmd="python -c 'import requests; requests.get(\"http://localhost:8000/health\")'" ` | |
| --health-interval=30s ` | |
| --health-timeout=10s ` | |
| --health-retries=3 ` | |
| ml-churn-api:latest | |
| if ($LASTEXITCODE -ne 0) { | |
| Write-Host "Failed to start staging container" | |
| exit 1 | |
| } | |
| Write-Host "Container started, waiting for health check..." | |
| Start-Sleep -Seconds 20 | |
| # Verify container is running | |
| $containerStatus = docker inspect ml-api-staging --format "{{.State.Status}}" | |
| Write-Host "Container status: $containerStatus" | |
| if ($containerStatus -ne "running") { | |
| Write-Host "Container is not running. Checking logs..." | |
| docker logs ml-api-staging | |
| exit 1 | |
| } | |
| # Verify health endpoint | |
| Write-Host "`nVerifying staging deployment..." | |
| $maxRetries = 10 | |
| $retryCount = 0 | |
| $healthPassed = $false | |
| while ($retryCount -lt $maxRetries -and -not $healthPassed) { | |
| try { | |
| $health = Invoke-RestMethod -Uri "http://localhost:${{ env.API_PORT_STAGING }}/health" -Method Get -TimeoutSec 5 -ErrorAction Stop | |
| if ($health.status -eq "ok") { | |
| Write-Host "Health check passed" | |
| $healthPassed = $true | |
| } | |
| } catch { | |
| $retryCount++ | |
| Write-Host "Health check retry $retryCount/$maxRetries..." | |
| Start-Sleep -Seconds 3 | |
| } | |
| } | |
| if (-not $healthPassed) { | |
| Write-Host "Staging health check failed after $maxRetries attempts" | |
| Write-Host "`nContainer logs:" | |
| docker logs ml-api-staging | |
| docker stop ml-api-staging | |
| docker rm ml-api-staging | |
| exit 1 | |
| } | |
| Write-Host "`nStaging deployment successful!" | |
| Write-Host "Staging API: http://localhost:${{ env.API_PORT_STAGING }}" | |
| Write-Host "Docs: http://localhost:${{ env.API_PORT_STAGING }}/docs" | |
| Write-Host "`nContainer Info:" | |
| docker ps --filter "name=ml-api-staging" --format "table {{.ID}}\t{{.Image}}\t{{.Status}}\t{{.Ports}}" | |
| - name: Deploy to Production | |
| if: ${{ steps.set-target.outputs.target == 'production' && github.ref == 'refs/heads/main' }} | |
| run: | | |
| Write-Host "=== Production Deployment ===" | |
| Write-Host "Production deployment configured for manual approval only" | |
| # Production deployment steps would include: | |
| # - Push image to container registry (Docker Hub, ACR, ECR, GCR) | |
| # - Deploy to cloud service (Azure Container Apps, AWS ECS, GCP Cloud Run) | |
| # - Configure load balancer and auto-scaling | |
| # - Set up monitoring and alerts | |
| # - Run smoke tests | |
| # - Gradual rollout with canary deployment | |
| Write-Host "`nProduction deployment checklist:" | |
| Write-Host " - [ ] Tag image: ml-churn-api:${{ github.sha }}" | |
| Write-Host " - [ ] Push to registry" | |
| Write-Host " - [ ] Update Kubernetes/Cloud config" | |
| Write-Host " - [ ] Run smoke tests" | |
| Write-Host " - [ ] Monitor metrics" | |
| - name: Extended Deployment Verification | |
| if: ${{ steps.set-target.outputs.target == 'local' || steps.set-target.outputs.target == 'staging' }} | |
| run: | | |
| Write-Host "`n=== Extended Deployment Verification ===" | |
| # Determine port based on deployment target | |
| $port = if ($env:DEPLOYMENT_TARGET -eq "staging") { 8001 } else { 8000 } | |
| $baseUrl = "http://localhost:$port" | |
| Write-Host "Testing $env:DEPLOYMENT_TARGET environment at $baseUrl" | |
| # Extended health checks (multiple attempts) | |
| Write-Host "`nRunning health checks..." | |
| $successCount = 0 | |
| for ($i = 1; $i -le 5; $i++) { | |
| try { | |
| $health = Invoke-RestMethod -Uri "$baseUrl/health" -Method Get -TimeoutSec 5 | |
| if ($health.status -eq "ok") { | |
| Write-Host " Health check $i/5: PASSED" | |
| $successCount++ | |
| } | |
| } catch { | |
| Write-Host " Health check $i/5: FAILED - $_" | |
| } | |
| Start-Sleep -Seconds 2 | |
| } | |
| if ($successCount -ge 4) { | |
| Write-Host "Health checks: PASSED ($successCount/5)" | |
| } else { | |
| Write-Host "Health checks: FAILED (only $successCount/5 passed)" | |
| exit 1 | |
| } | |
| # Test prediction endpoint with sample data | |
| Write-Host "`nTesting prediction endpoint..." | |
| $body = @{ | |
| features = @{ | |
| Age = 35 | |
| Gender = "Male" | |
| Tenure = 24 | |
| "Usage Frequency" = 15 | |
| "Support Calls" = 2 | |
| "Payment Delay" = 5 | |
| "Subscription Type" = "Standard" | |
| "Contract Length" = "Annual" | |
| "Total Spend" = 1200 | |
| "Last Interaction" = 30 | |
| } | |
| } | ConvertTo-Json | |
| try { | |
| $prediction = Invoke-RestMethod -Uri "$baseUrl/predict" -Method Post -ContentType "application/json" -Body $body | |
| Write-Host " Prediction: $($prediction.prediction)" | |
| Write-Host " Probability: $($prediction.probability)" | |
| Write-Host "Prediction test: PASSED" | |
| } catch { | |
| Write-Host "Prediction test: FAILED - $_" | |
| exit 1 | |
| } | |
| Write-Host "`nAll verification tests PASSED!" | |
| - name: Staging Load Testing | |
| if: ${{ steps.set-target.outputs.target == 'staging' }} | |
| run: | | |
| Write-Host "`n=== Staging Load Testing ===" | |
| $baseUrl = "http://localhost:${{ env.API_PORT_STAGING }}" | |
| $totalRequests = 50 | |
| $successCount = 0 | |
| $failureCount = 0 | |
| $responseTimes = @() | |
| Write-Host "Running $totalRequests prediction requests..." | |
| for ($i = 1; $i -le $totalRequests; $i++) { | |
| $body = @{ | |
| features = @{ | |
| Age = Get-Random -Minimum 18 -Maximum 80 | |
| Gender = @("Male", "Female") | Get-Random | |
| Tenure = Get-Random -Minimum 1 -Maximum 60 | |
| "Usage Frequency" = Get-Random -Minimum 1 -Maximum 30 | |
| "Support Calls" = Get-Random -Minimum 0 -Maximum 10 | |
| "Payment Delay" = Get-Random -Minimum 0 -Maximum 30 | |
| "Subscription Type" = @("Basic", "Standard", "Premium") | Get-Random | |
| "Contract Length" = @("Monthly", "Quarterly", "Annual") | Get-Random | |
| "Total Spend" = Get-Random -Minimum 100 -Maximum 5000 | |
| "Last Interaction" = Get-Random -Minimum 1 -Maximum 60 | |
| } | |
| } | ConvertTo-Json | |
| try { | |
| $startTime = Get-Date | |
| $prediction = Invoke-RestMethod -Uri "$baseUrl/predict" -Method Post -ContentType "application/json" -Body $body -TimeoutSec 10 | |
| $endTime = Get-Date | |
| $duration = ($endTime - $startTime).TotalMilliseconds | |
| $responseTimes += $duration | |
| $successCount++ | |
| if ($i % 10 -eq 0) { | |
| Write-Host "Progress: $i/$totalRequests requests completed" | |
| } | |
| } catch { | |
| $failureCount++ | |
| Write-Host "Request $i failed: $_" | |
| } | |
| } | |
| # Calculate statistics | |
| $avgResponseTime = ($responseTimes | Measure-Object -Average).Average | |
| $minResponseTime = ($responseTimes | Measure-Object -Minimum).Minimum | |
| $maxResponseTime = ($responseTimes | Measure-Object -Maximum).Maximum | |
| $successRate = ($successCount / $totalRequests) * 100 | |
| Write-Host "`n=== Load Test Results ===" | |
| Write-Host "Total Requests: $totalRequests" | |
| Write-Host "Successful: $successCount" | |
| Write-Host "Failed: $failureCount" | |
| Write-Host "Success Rate: $([math]::Round($successRate, 2))%" | |
| Write-Host "Avg Response Time: $([math]::Round($avgResponseTime, 2)) ms" | |
| Write-Host "Min Response Time: $([math]::Round($minResponseTime, 2)) ms" | |
| Write-Host "Max Response Time: $([math]::Round($maxResponseTime, 2)) ms" | |
| # Fail if success rate is below 95% | |
| if ($successRate -lt 95) { | |
| Write-Host "`nLoad test failed: Success rate below 95%" | |
| exit 1 | |
| } | |
| # Warn if average response time is above 500ms | |
| if ($avgResponseTime -gt 500) { | |
| Write-Host "`nWarning: Average response time above 500ms" | |
| } | |
| Write-Host "`nLoad test PASSED!" | |
| - name: Staging Container Health Check | |
| if: ${{ steps.set-target.outputs.target == 'staging' }} | |
| run: | | |
| Write-Host "`n=== Staging Container Health Check ===" | |
| # Check container health status | |
| $healthStatus = docker inspect ml-api-staging --format "{{.State.Health.Status}}" | |
| Write-Host "Container health status: $healthStatus" | |
| # Get resource usage | |
| $stats = docker stats ml-api-staging --no-stream --format "json" | ConvertFrom-Json | |
| Write-Host "`nResource Usage:" | |
| Write-Host " CPU: $($stats.CPUPerc)" | |
| Write-Host " Memory: $($stats.MemUsage)" | |
| # Check logs for errors | |
| Write-Host "`nChecking recent logs for errors..." | |
| $logs = docker logs ml-api-staging --tail 50 2>&1 | |
| $errorCount = ($logs | Select-String -Pattern "error|exception|failed" -AllMatches).Matches.Count | |
| if ($errorCount -gt 0) { | |
| Write-Host "Warning: Found $errorCount potential errors in logs" | |
| Write-Host "Recent logs:" | |
| docker logs ml-api-staging --tail 20 | |
| } else { | |
| Write-Host "No errors found in recent logs" | |
| } | |
| Write-Host "`nContainer health check complete" | |
| - name: Deployment Summary | |
| if: always() | |
| run: | | |
| Write-Host "`n================================================" | |
| Write-Host "=== Deployment Complete ===" | |
| Write-Host "================================================" | |
| Write-Host "Target: $env:DEPLOYMENT_TARGET" | |
| Write-Host "Event: ${{ github.event_name }}" | |
| Write-Host "Commit: ${{ github.sha }}" | |
| Write-Host "Deployment Method: Docker" | |
| Write-Host "Image: ml-churn-api:latest" | |
| if ($env:DEPLOYMENT_TARGET -eq "local") { | |
| Write-Host "`n=== Local Environment ===" | |
| Write-Host "API: http://localhost:8000" | |
| Write-Host "Docs: http://localhost:8000/docs" | |
| Write-Host "MLflow: http://localhost:5000" | |
| Write-Host "`nManage with:" | |
| Write-Host " docker-compose logs -f # View logs" | |
| Write-Host " docker-compose down # Stop services" | |
| Write-Host " docker-compose restart # Restart services" | |
| Write-Host " docker-compose ps # Check status" | |
| } elseif ($env:DEPLOYMENT_TARGET -eq "staging") { | |
| Write-Host "`n=== Staging Environment ===" | |
| Write-Host "API: http://localhost:8001" | |
| Write-Host "Docs: http://localhost:8001/docs" | |
| Write-Host "Health: http://localhost:8001/health" | |
| Write-Host "`nContainer Details:" | |
| docker ps --filter "name=ml-api-staging" --format " ID: {{.ID}}" | |
| docker ps --filter "name=ml-api-staging" --format " Status: {{.Status}}" | |
| docker ps --filter "name=ml-api-staging" --format " Uptime: {{.RunningFor}}" | |
| Write-Host "`nManagement Commands:" | |
| Write-Host " docker logs ml-api-staging # View logs" | |
| Write-Host " docker logs -f ml-api-staging # Follow logs" | |
| Write-Host " docker stop ml-api-staging # Stop container" | |
| Write-Host " docker start ml-api-staging # Start container" | |
| Write-Host " docker restart ml-api-staging # Restart container" | |
| Write-Host " docker stats ml-api-staging # Resource usage" | |
| Write-Host " docker inspect ml-api-staging # Full details" | |
| Write-Host "`nValidation Completed:" | |
| Write-Host " - Health checks: 5/5 passed" | |
| Write-Host " - Prediction endpoint: Verified" | |
| Write-Host " - Load testing: 50 requests" | |
| Write-Host " - Container health: Monitored" | |
| } elseif ($env:DEPLOYMENT_TARGET -eq "production") { | |
| Write-Host "`n=== Production Deployment ===" | |
| Write-Host "Status: Ready for cloud deployment" | |
| Write-Host "Image Tag: ml-churn-api:${{ github.sha }}" | |
| Write-Host "`nNext Steps:" | |
| Write-Host " 1. Push image to container registry" | |
| Write-Host " 2. Deploy to cloud platform" | |
| Write-Host " 3. Configure auto-scaling" | |
| Write-Host " 4. Set up monitoring" | |
| Write-Host " 5. Enable alerts" | |
| } | |
| Write-Host "================================================" |