Fix Docker daemon startup timing issue in CI/CD #21

Workflow file for this run

	name: Deploy Model API

	on:
	push:
	branches:
	- main
	- develop
	paths:
	- 'api/**'
	- 'mlruns/best_model_artifacts/**'
	- 'Data/preprocessor.pkl'
	- 'Dockerfile'
	- 'docker-compose.yml'
	- '.github/workflows/cd.yml'

	pull_request:
	branches:
	- main
	paths:
	- 'api/**'
	- 'mlruns/best_model_artifacts/**'
	- 'Data/preprocessor.pkl'
	- 'Dockerfile'
	- 'docker-compose.yml'
	- '.github/workflows/cd.yml'

	workflow_dispatch:
	inputs:
	skip_docker_check:
	description: 'Skip Docker startup check (use if Docker is already running)'
	required: false
	default: 'false'
	type: choice
	options:
	- 'false'
	- 'true'
	deployment_target:
	description: 'Choose deployment target'
	required: true
	default: 'local'
	type: choice
	options:
	- local
	- staging
	- production
	use_docker:
	description: 'Use Docker for deployment'
	required: false
	default: true
	type: boolean

	jobs:
	# Verify prerequisites
	check-prerequisites:
	runs-on: self-hosted
	outputs:
	model-exists: ${{ steps.check.outputs.model }}
	preprocessor-exists: ${{ steps.check.outputs.preprocessor }}

	defaults:
	run:
	shell: powershell

	steps:
	- name: Checkout repository
	uses: actions/checkout@v4

	- name: Check required artifacts
	id: check
	run: \|
	$modelExists = $false
	$preprocessorExists = $false

	# Check for best model artifacts
	if (Test-Path "mlruns\best_model_artifacts\customer_churn_optimization") {
	$modelDirs = Get-ChildItem "mlruns\best_model_artifacts\customer_churn_optimization" -Directory
	if ($modelDirs.Count -gt 0) {
	Write-Host "Model artifacts found: $($modelDirs[0].Name)"
	$modelExists = $true
	}
	}

	# Check for preprocessor
	if (Test-Path "Data\preprocessor.pkl") {
	Write-Host "Preprocessor found"
	$preprocessorExists = $true
	}

	# Set outputs
	"model=$modelExists" >> $env:GITHUB_OUTPUT
	"preprocessor=$preprocessorExists" >> $env:GITHUB_OUTPUT

	# Fail if required artifacts are missing
	if (-not $modelExists) {
	Write-Host "No model artifacts found. Run CI pipeline first to train a model."
	exit 1
	}

	if (-not $preprocessorExists) {
	Write-Host "Preprocessor not found. Run data preprocessing first."
	exit 1
	}

	Write-Host "All required artifacts are present"

	# Test API with native Python (faster for development)
	test-api:
	needs: check-prerequisites
	runs-on: self-hosted

	defaults:
	run:
	shell: powershell

	env:
	BEST_MODEL_METRIC: f1
	MLFLOW_EXPERIMENT_NAME: customer_churn_optimization

	steps:
	- name: Checkout repository
	uses: actions/checkout@v4

	- name: Set up Python environment
	run: \|
	if (Test-Path ".venv\Scripts\Activate.ps1") {
	Write-Host "Virtual environment found"
	& ".venv\Scripts\Activate.ps1"
	} else {
	Write-Host "No virtual environment found, using system Python"
	}

	# Install API dependencies
	Write-Host "Installing API dependencies..."
	python -m pip install -q -r api/requirements.txt

	- name: Start API server and run tests
	run: \|
	Write-Host "=== Starting API Server and Running Tests ==="

	# Kill any existing server on port 8000
	$existing = Get-NetTCPConnection -LocalPort 8000 -State Listen -ErrorAction SilentlyContinue
	if ($existing) {
	$processId = $existing.OwningProcess
	Write-Host "Stopping existing process on port 8000 (PID: $processId)"
	Stop-Process -Id $processId -Force -ErrorAction SilentlyContinue
	Start-Sleep -Seconds 2
	}

	# Verify dependencies are installed
	Write-Host "Verifying API dependencies..."
	python -c "import uvicorn, fastapi" 2>$null
	if ($LASTEXITCODE -ne 0) {
	Write-Host "Dependencies missing, installing..."
	python -m pip install -q -r api/requirements.txt
	} else {
	Write-Host "Dependencies verified"
	}

	# Start server in background using Start-Process
	Write-Host "`nStarting uvicorn server..."
	$process = Start-Process python -ArgumentList @(
	"-m", "uvicorn",
	"api.main:app",
	"--host", "127.0.0.1",
	"--port", "8000"
	) -WindowStyle Hidden -PassThru -RedirectStandardOutput "api_stdout.log" -RedirectStandardError "api_stderr.log"

	Write-Host "Server starting (PID: $($process.Id))..."
	Write-Host "Waiting for API to become ready..."
	Start-Sleep -Seconds 8

	# Verify server is responding
	$maxRetries = 8
	$retryCount = 0
	$serverReady = $false

	while ($retryCount -lt $maxRetries -and -not $serverReady) {
	# Check if process is still running
	$processCheck = Get-Process -Id $process.Id -ErrorAction SilentlyContinue
	if (-not $processCheck) {
	Write-Host "Server process died unexpectedly!"
	if (Test-Path "api_stderr.log") {
	Write-Host "Error log:"
	Get-Content "api_stderr.log"
	}
	Stop-Process -Id $process.Id -Force -ErrorAction SilentlyContinue
	exit 1
	}

	try {
	$response = Invoke-RestMethod -Uri "http://localhost:8000/health" -Method Get -TimeoutSec 3 -ErrorAction Stop
	if ($response.status -eq "ok") {
	Write-Host "API server is ready and responding!"
	$serverReady = $true
	}
	} catch {
	$retryCount++
	Write-Host "Retry $retryCount/$maxRetries - server not ready yet..."
	Start-Sleep -Seconds 2
	}
	}

	if (-not $serverReady) {
	Write-Host "API server failed to respond to health checks"
	Write-Host "`nStdout log:"
	if (Test-Path "api_stdout.log") { Get-Content "api_stdout.log" }
	Write-Host "`nStderr log:"
	if (Test-Path "api_stderr.log") { Get-Content "api_stderr.log" }
	Stop-Process -Id $process.Id -Force -ErrorAction SilentlyContinue
	exit 1
	}

	# Run API tests
	Write-Host "`n=== Testing API Endpoints ==="

	try {
	# Test health endpoint
	Write-Host "Testing /health..."
	$health = Invoke-RestMethod -Uri "http://localhost:8000/health" -Method Get
	if ($health.status -eq "ok") {
	Write-Host " Health check passed"
	} else {
	throw "Health check failed"
	}

	# Test schema endpoint
	Write-Host "Testing /schema..."
	$schema = Invoke-RestMethod -Uri "http://localhost:8000/schema" -Method Get
	if ($schema.required_columns) {
	Write-Host " Schema endpoint working"
	Write-Host " Required columns: $($schema.required_columns.Count)"
	} else {
	throw "Schema endpoint failed"
	}

	# Test prediction endpoint
	Write-Host "Testing /predict..."
	$body = @{
	features = @{
	Age = 30
	Gender = "Female"
	Tenure = 39
	"Usage Frequency" = 14
	"Support Calls" = 5
	"Payment Delay" = 18
	"Subscription Type" = "Standard"
	"Contract Length" = "Annual"
	"Total Spend" = 932
	"Last Interaction" = 17
	}
	} \| ConvertTo-Json

	$prediction = Invoke-RestMethod -Uri "http://localhost:8000/predict" -Method Post -ContentType "application/json" -Body $body

	if ($null -ne $prediction.prediction) {
	Write-Host " Prediction endpoint working"
	Write-Host " Prediction: $($prediction.prediction)"
	Write-Host " Probability: $($prediction.probability)"
	} else {
	throw "Prediction failed"
	}

	Write-Host "`nAll API tests passed!"

	} catch {
	Write-Host "Test failed: $_"
	Stop-Process -Id $process.Id -Force -ErrorAction SilentlyContinue
	exit 1
	} finally {
	# Always stop the server
	Write-Host "`nStopping API server (PID: $($process.Id))..."
	Stop-Process -Id $process.Id -Force -ErrorAction SilentlyContinue
	Write-Host "API server stopped"
	}

	# Build and test Docker image (optional - runs only if Docker is available)
	build-docker:
	needs: test-api
	runs-on: self-hosted
	continue-on-error: true
	if: success()

	defaults:
	run:
	shell: powershell

	steps:
	- name: Checkout repository
	uses: actions/checkout@v4

	- name: Verify Docker is available
	run: \|
	Write-Host "=== Checking Docker Installation ==="

	# Check if Docker Desktop is installed
	try {
	$dockerVersion = docker --version
	Write-Host "Docker found: $dockerVersion"

	$dockerComposeVersion = docker-compose --version
	Write-Host "Docker Compose found: $dockerComposeVersion"
	} catch {
	Write-Host "Docker is not installed"
	Write-Host "Please install Docker Desktop from https://www.docker.com/products/docker-desktop"
	exit 1
	}

	# Check if Docker Desktop is already running
	$dockerDesktopProcess = Get-Process "Docker Desktop" -ErrorAction SilentlyContinue
	if (-not $dockerDesktopProcess) {
	Write-Host "`nDocker Desktop is not running. Starting Docker Desktop..."
	Start-Process "C:\Program Files\Docker\Docker\Docker Desktop.exe" -ErrorAction SilentlyContinue
	Write-Host "Docker Desktop launch initiated. Waiting for it to start..."
	Start-Sleep -Seconds 10
	} else {
	Write-Host "`nDocker Desktop process found (PID: $($dockerDesktopProcess.Id))"
	}

	# Wait for Docker daemon to be ready with extended timeout
	Write-Host "`nWaiting for Docker daemon to be ready..."
	$maxRetries = 60
	$retryCount = 0
	$dockerReady = $false
	$lastError = ""

	while ($retryCount -lt $maxRetries -and -not $dockerReady) {
	try {
	# Try to run docker ps and capture both output and errors
	$dockerPsOutput = docker ps 2>&1
	$exitCode = $LASTEXITCODE

	if ($exitCode -eq 0) {
	Write-Host "Docker daemon is ready!"
	$dockerReady = $true
	break
	} else {
	$lastError = $dockerPsOutput
	}
	} catch {
	$lastError = $_.Exception.Message
	}

	$retryCount++
	if ($retryCount % 10 -eq 0) {
	Write-Host "Still waiting... ($retryCount/$maxRetries attempts, $($retryCount * 3) seconds elapsed)"
	}
	Start-Sleep -Seconds 3
	}

	if (-not $dockerReady) {
	Write-Host "`n========================================"
	Write-Host "ERROR: Docker daemon failed to start"
	Write-Host "========================================"
	Write-Host "Docker Desktop did not become ready after $($maxRetries * 3) seconds"
	Write-Host "`nLast error: $lastError"
	Write-Host "`nTROUBLESHOOTING STEPS:"
	Write-Host "1. Manually start Docker Desktop before running this workflow"
	Write-Host "2. Verify Docker is running: docker ps"
	Write-Host "3. Check Docker Desktop settings - ensure it's set to start on login"
	Write-Host "4. Restart Docker Desktop if it's stuck"
	Write-Host "`nDocker Desktop needs to be fully started BEFORE triggering CI/CD workflows"
	exit 1
	}

	# Verify Docker is working properly
	Write-Host "`nVerifying Docker functionality..."
	$dockerInfo = docker info 2>&1
	if ($LASTEXITCODE -eq 0) {
	Write-Host "Docker version info:"
	docker version --format 'Server: {{.Server.Version}}'
	Write-Host "Containers running: $(docker ps -q \| Measure-Object \| Select-Object -ExpandProperty Count)"
	Write-Host "`nDocker is fully ready for use!"
	} else {
	Write-Host "Warning: Docker info command failed"
	Write-Host $dockerInfo
	}

	- name: Clean up old Docker resources
	run: \|
	Write-Host "=== Cleaning up old Docker resources ==="

	# Stop and remove existing containers
	$existingContainers = docker ps -a --filter "name=ml-api" --format "{{.Names}}"
	if ($existingContainers) {
	Write-Host "Stopping existing containers: $existingContainers"
	docker stop $existingContainers 2>$null
	docker rm $existingContainers 2>$null
	}

	# Remove old images (keep last 3)
	$oldImages = docker images ml-churn-api --format "{{.ID}}" \| Select-Object -Skip 3
	if ($oldImages) {
	Write-Host "Removing old images..."
	$oldImages \| ForEach-Object { docker rmi $_ -f 2>$null }
	}

	Write-Host "Cleanup complete"

	- name: Build Docker image
	run: \|
	Write-Host "=== Building Docker Image ==="
	$startTime = Get-Date

	# Build with build args and tags
	docker build `
	--build-arg BUILD_DATE=$(Get-Date -Format 'yyyy-MM-ddTHH:mm:ssZ') `
	--build-arg GIT_COMMIT=${{ github.sha }} `
	-t ml-churn-api:latest `
	-t ml-churn-api:${{ github.sha }} `
	.

	if ($LASTEXITCODE -ne 0) {
	Write-Host "Docker build failed"
	exit 1
	}

	$duration = (Get-Date) - $startTime
	Write-Host "Docker image built successfully in $($duration.TotalSeconds) seconds"

	# Show image info
	Write-Host "`nImage details:"
	docker images ml-churn-api:latest

	- name: Test Docker container
	run: \|
	Write-Host "=== Testing Docker Container ==="

	# Start container
	Write-Host "Starting Docker container..."
	docker run -d `
	-p 8000:8000 `
	--name ml-api-test `
	--env BEST_MODEL_METRIC=f1 `
	--env MLFLOW_EXPERIMENT_NAME=customer_churn_optimization `
	ml-churn-api:latest

	if ($LASTEXITCODE -ne 0) {
	Write-Host "Failed to start container"
	exit 1
	}

	Write-Host "Container started, waiting for API to be ready..."
	Start-Sleep -Seconds 15

	# Check container logs
	Write-Host "`nContainer logs:"
	docker logs ml-api-test

	# Verify container is running
	$containerStatus = docker ps --filter "name=ml-api-test" --format "{{.Status}}"
	Write-Host "`nContainer status: $containerStatus"

	# Test health endpoint
	Write-Host "`nTesting containerized API..."
	$maxRetries = 10
	$retryCount = 0
	$apiReady = $false

	while ($retryCount -lt $maxRetries -and -not $apiReady) {
	try {
	$health = Invoke-RestMethod -Uri "http://localhost:8000/health" -Method Get -TimeoutSec 5 -ErrorAction Stop
	if ($health.status -eq "ok") {
	Write-Host "Health check passed"
	$apiReady = $true
	}
	} catch {
	$retryCount++
	Write-Host "Retry $retryCount/$maxRetries - waiting for API..."
	Start-Sleep -Seconds 3
	}
	}

	if (-not $apiReady) {
	Write-Host "API health check failed"
	Write-Host "`nFinal container logs:"
	docker logs ml-api-test
	docker stop ml-api-test
	docker rm ml-api-test
	exit 1
	}

	# Test prediction endpoint
	Write-Host "`nTesting prediction endpoint..."
	$body = @{
	features = @{
	Age = 35
	Gender = "Male"
	Tenure = 24
	"Usage Frequency" = 15
	"Support Calls" = 2
	"Payment Delay" = 5
	"Subscription Type" = "Standard"
	"Contract Length" = "Annual"
	"Total Spend" = 1200
	"Last Interaction" = 30
	}
	} \| ConvertTo-Json

	try {
	$prediction = Invoke-RestMethod `
	-Uri "http://localhost:8000/predict" `
	-Method Post `
	-ContentType "application/json" `
	-Body $body

	Write-Host "Prediction successful"
	Write-Host " Prediction: $($prediction.prediction)"
	Write-Host " Probability: $($prediction.probability)"
	} catch {
	Write-Host "Prediction test failed: $_"
	docker stop ml-api-test
	docker rm ml-api-test
	exit 1
	}

	# Cleanup test container
	Write-Host "`nStopping test container..."
	docker stop ml-api-test
	docker rm ml-api-test

	Write-Host "Docker container tests passed!"

	# Deploy to selected environment
	deploy:
	needs: [check-prerequisites, test-api]
	runs-on: self-hosted
	if: github.event_name == 'push' \|\| github.event_name == 'workflow_dispatch'

	defaults:
	run:
	shell: powershell

	env:
	API_PORT_LOCAL: 8000
	API_PORT_STAGING: 8001
	API_PORT_PRODUCTION: 8002

	steps:
	- name: Checkout repository
	uses: actions/checkout@v4

	- name: Set deployment target
	id: set-target
	run: \|
	$target = "${{ inputs.deployment_target }}"
	if ([string]::IsNullOrEmpty($target)) {
	$target = "local"
	}

	$useDocker = "${{ inputs.use_docker }}"
	if ([string]::IsNullOrEmpty($useDocker)) {
	$useDocker = "true"
	}

	Write-Host "Deployment target: $target"
	Write-Host "Use Docker: $useDocker"

	"DEPLOYMENT_TARGET=$target" >> $env:GITHUB_ENV
	"USE_DOCKER=$useDocker" >> $env:GITHUB_ENV
	"target=$target" >> $env:GITHUB_OUTPUT
	"use_docker=$useDocker" >> $env:GITHUB_OUTPUT

	- name: Debug deployment target
	run: \|
	Write-Host "DEPLOYMENT_TARGET: $env:DEPLOYMENT_TARGET"
	Write-Host "Event name: ${{ github.event_name }}"

	- name: Deploy with Docker Compose (Local)
	if: ${{ steps.set-target.outputs.target == 'local' }}
	run: \|
	Write-Host "=== Deploying to Local with Docker Compose ==="

	# Stop existing containers
	Write-Host "Stopping existing containers..."
	docker-compose down 2>$null

	# Start services
	Write-Host "Starting services with docker-compose..."
	docker-compose up -d

	if ($LASTEXITCODE -ne 0) {
	Write-Host "Docker Compose deployment failed"
	exit 1
	}

	Write-Host "Waiting for services to start..."
	Start-Sleep -Seconds 20

	# Verify deployment
	Write-Host "`nVerifying deployment..."
	$health = Invoke-RestMethod -Uri "http://localhost:${{ env.API_PORT_LOCAL }}/health" -Method Get

	if ($health.status -eq "ok") {
	Write-Host "Local deployment successful!"
	Write-Host "`n=== Deployment Info ==="
	Write-Host "API: http://localhost:${{ env.API_PORT_LOCAL }}"
	Write-Host "Docs: http://localhost:${{ env.API_PORT_LOCAL }}/docs"
	Write-Host "MLflow: http://localhost:5000"
	Write-Host "`nRunning containers:"
	docker ps --filter "name=ml_churn_api"
	} else {
	Write-Host "Health check failed"
	Write-Host "`nContainer logs:"
	docker logs ml_churn_api
	exit 1
	}

	- name: Deploy with Docker (Staging)
	if: ${{ steps.set-target.outputs.target == 'staging' }}
	run: \|
	Write-Host "=== Deploying to Staging with Docker ==="

	# Stop existing staging container
	Write-Host "Checking for existing staging container..."
	$existingContainer = docker ps -a --filter "name=ml-api-staging" --format "{{.Names}}"
	if ($existingContainer) {
	Write-Host "Stopping and removing existing staging container..."
	docker stop ml-api-staging 2>$null
	docker rm ml-api-staging 2>$null
	Write-Host "Existing container removed"
	}

	# Start staging container with production-like settings
	Write-Host "Starting staging container on port ${{ env.API_PORT_STAGING }}..."
	docker run -d `
	-p ${{ env.API_PORT_STAGING }}:8000 `
	--name ml-api-staging `
	--restart unless-stopped `
	--env BEST_MODEL_METRIC=f1 `
	--env MLFLOW_EXPERIMENT_NAME=customer_churn_optimization `
	--env ENVIRONMENT=staging `
	--memory="2g" `
	--cpus="1.0" `
	--health-cmd="python -c 'import requests; requests.get(\"http://localhost:8000/health\")'" `
	--health-interval=30s `
	--health-timeout=10s `
	--health-retries=3 `
	ml-churn-api:latest

	if ($LASTEXITCODE -ne 0) {
	Write-Host "Failed to start staging container"
	exit 1
	}

	Write-Host "Container started, waiting for health check..."
	Start-Sleep -Seconds 20

	# Verify container is running
	$containerStatus = docker inspect ml-api-staging --format "{{.State.Status}}"
	Write-Host "Container status: $containerStatus"

	if ($containerStatus -ne "running") {
	Write-Host "Container is not running. Checking logs..."
	docker logs ml-api-staging
	exit 1
	}

	# Verify health endpoint
	Write-Host "`nVerifying staging deployment..."
	$maxRetries = 10
	$retryCount = 0
	$healthPassed = $false

	while ($retryCount -lt $maxRetries -and -not $healthPassed) {
	try {
	$health = Invoke-RestMethod -Uri "http://localhost:${{ env.API_PORT_STAGING }}/health" -Method Get -TimeoutSec 5 -ErrorAction Stop
	if ($health.status -eq "ok") {
	Write-Host "Health check passed"
	$healthPassed = $true
	}
	} catch {
	$retryCount++
	Write-Host "Health check retry $retryCount/$maxRetries..."
	Start-Sleep -Seconds 3
	}
	}

	if (-not $healthPassed) {
	Write-Host "Staging health check failed after $maxRetries attempts"
	Write-Host "`nContainer logs:"
	docker logs ml-api-staging
	docker stop ml-api-staging
	docker rm ml-api-staging
	exit 1
	}

	Write-Host "`nStaging deployment successful!"
	Write-Host "Staging API: http://localhost:${{ env.API_PORT_STAGING }}"
	Write-Host "Docs: http://localhost:${{ env.API_PORT_STAGING }}/docs"
	Write-Host "`nContainer Info:"
	docker ps --filter "name=ml-api-staging" --format "table {{.ID}}\t{{.Image}}\t{{.Status}}\t{{.Ports}}"

	- name: Deploy to Production
	if: ${{ steps.set-target.outputs.target == 'production' && github.ref == 'refs/heads/main' }}
	run: \|
	Write-Host "=== Production Deployment ==="
	Write-Host "Production deployment configured for manual approval only"

	# Production deployment steps would include:
	# - Push image to container registry (Docker Hub, ACR, ECR, GCR)
	# - Deploy to cloud service (Azure Container Apps, AWS ECS, GCP Cloud Run)
	# - Configure load balancer and auto-scaling
	# - Set up monitoring and alerts
	# - Run smoke tests
	# - Gradual rollout with canary deployment

	Write-Host "`nProduction deployment checklist:"
	Write-Host " - [ ] Tag image: ml-churn-api:${{ github.sha }}"
	Write-Host " - [ ] Push to registry"
	Write-Host " - [ ] Update Kubernetes/Cloud config"
	Write-Host " - [ ] Run smoke tests"
	Write-Host " - [ ] Monitor metrics"

	- name: Extended Deployment Verification
	if: ${{ steps.set-target.outputs.target == 'local' \|\| steps.set-target.outputs.target == 'staging' }}
	run: \|
	Write-Host "`n=== Extended Deployment Verification ==="

	# Determine port based on deployment target
	$port = if ($env:DEPLOYMENT_TARGET -eq "staging") { 8001 } else { 8000 }
	$baseUrl = "http://localhost:$port"

	Write-Host "Testing $env:DEPLOYMENT_TARGET environment at $baseUrl"

	# Extended health checks (multiple attempts)
	Write-Host "`nRunning health checks..."
	$successCount = 0
	for ($i = 1; $i -le 5; $i++) {
	try {
	$health = Invoke-RestMethod -Uri "$baseUrl/health" -Method Get -TimeoutSec 5
	if ($health.status -eq "ok") {
	Write-Host " Health check $i/5: PASSED"
	$successCount++
	}
	} catch {
	Write-Host " Health check $i/5: FAILED - $_"
	}
	Start-Sleep -Seconds 2
	}

	if ($successCount -ge 4) {
	Write-Host "Health checks: PASSED ($successCount/5)"
	} else {
	Write-Host "Health checks: FAILED (only $successCount/5 passed)"
	exit 1
	}

	# Test prediction endpoint with sample data
	Write-Host "`nTesting prediction endpoint..."
	$body = @{
	features = @{
	Age = 35
	Gender = "Male"
	Tenure = 24
	"Usage Frequency" = 15
	"Support Calls" = 2
	"Payment Delay" = 5
	"Subscription Type" = "Standard"
	"Contract Length" = "Annual"
	"Total Spend" = 1200
	"Last Interaction" = 30
	}
	} \| ConvertTo-Json

	try {
	$prediction = Invoke-RestMethod -Uri "$baseUrl/predict" -Method Post -ContentType "application/json" -Body $body
	Write-Host " Prediction: $($prediction.prediction)"
	Write-Host " Probability: $($prediction.probability)"
	Write-Host "Prediction test: PASSED"
	} catch {
	Write-Host "Prediction test: FAILED - $_"
	exit 1
	}

	Write-Host "`nAll verification tests PASSED!"

	- name: Staging Load Testing
	if: ${{ steps.set-target.outputs.target == 'staging' }}
	run: \|
	Write-Host "`n=== Staging Load Testing ==="

	$baseUrl = "http://localhost:${{ env.API_PORT_STAGING }}"
	$totalRequests = 50
	$successCount = 0
	$failureCount = 0
	$responseTimes = @()

	Write-Host "Running $totalRequests prediction requests..."

	for ($i = 1; $i -le $totalRequests; $i++) {
	$body = @{
	features = @{
	Age = Get-Random -Minimum 18 -Maximum 80
	Gender = @("Male", "Female") \| Get-Random
	Tenure = Get-Random -Minimum 1 -Maximum 60
	"Usage Frequency" = Get-Random -Minimum 1 -Maximum 30
	"Support Calls" = Get-Random -Minimum 0 -Maximum 10
	"Payment Delay" = Get-Random -Minimum 0 -Maximum 30
	"Subscription Type" = @("Basic", "Standard", "Premium") \| Get-Random
	"Contract Length" = @("Monthly", "Quarterly", "Annual") \| Get-Random
	"Total Spend" = Get-Random -Minimum 100 -Maximum 5000
	"Last Interaction" = Get-Random -Minimum 1 -Maximum 60
	}
	} \| ConvertTo-Json

	try {
	$startTime = Get-Date
	$prediction = Invoke-RestMethod -Uri "$baseUrl/predict" -Method Post -ContentType "application/json" -Body $body -TimeoutSec 10
	$endTime = Get-Date
	$duration = ($endTime - $startTime).TotalMilliseconds
	$responseTimes += $duration
	$successCount++

	if ($i % 10 -eq 0) {
	Write-Host "Progress: $i/$totalRequests requests completed"
	}
	} catch {
	$failureCount++
	Write-Host "Request $i failed: $_"
	}
	}

	# Calculate statistics
	$avgResponseTime = ($responseTimes \| Measure-Object -Average).Average
	$minResponseTime = ($responseTimes \| Measure-Object -Minimum).Minimum
	$maxResponseTime = ($responseTimes \| Measure-Object -Maximum).Maximum
	$successRate = ($successCount / $totalRequests) * 100

	Write-Host "`n=== Load Test Results ==="
	Write-Host "Total Requests: $totalRequests"
	Write-Host "Successful: $successCount"
	Write-Host "Failed: $failureCount"
	Write-Host "Success Rate: $([math]::Round($successRate, 2))%"
	Write-Host "Avg Response Time: $([math]::Round($avgResponseTime, 2)) ms"
	Write-Host "Min Response Time: $([math]::Round($minResponseTime, 2)) ms"
	Write-Host "Max Response Time: $([math]::Round($maxResponseTime, 2)) ms"

	# Fail if success rate is below 95%
	if ($successRate -lt 95) {
	Write-Host "`nLoad test failed: Success rate below 95%"
	exit 1
	}

	# Warn if average response time is above 500ms
	if ($avgResponseTime -gt 500) {
	Write-Host "`nWarning: Average response time above 500ms"
	}

	Write-Host "`nLoad test PASSED!"

	- name: Staging Container Health Check
	if: ${{ steps.set-target.outputs.target == 'staging' }}
	run: \|
	Write-Host "`n=== Staging Container Health Check ==="

	# Check container health status
	$healthStatus = docker inspect ml-api-staging --format "{{.State.Health.Status}}"
	Write-Host "Container health status: $healthStatus"

	# Get resource usage
	$stats = docker stats ml-api-staging --no-stream --format "json" \| ConvertFrom-Json
	Write-Host "`nResource Usage:"
	Write-Host " CPU: $($stats.CPUPerc)"
	Write-Host " Memory: $($stats.MemUsage)"

	# Check logs for errors
	Write-Host "`nChecking recent logs for errors..."
	$logs = docker logs ml-api-staging --tail 50 2>&1
	$errorCount = ($logs \| Select-String -Pattern "error\|exception\|failed" -AllMatches).Matches.Count

	if ($errorCount -gt 0) {
	Write-Host "Warning: Found $errorCount potential errors in logs"
	Write-Host "Recent logs:"
	docker logs ml-api-staging --tail 20
	} else {
	Write-Host "No errors found in recent logs"
	}

	Write-Host "`nContainer health check complete"

	- name: Deployment Summary
	if: always()
	run: \|
	Write-Host "`n================================================"
	Write-Host "=== Deployment Complete ==="
	Write-Host "================================================"
	Write-Host "Target: $env:DEPLOYMENT_TARGET"
	Write-Host "Event: ${{ github.event_name }}"
	Write-Host "Commit: ${{ github.sha }}"
	Write-Host "Deployment Method: Docker"
	Write-Host "Image: ml-churn-api:latest"

	if ($env:DEPLOYMENT_TARGET -eq "local") {
	Write-Host "`n=== Local Environment ==="
	Write-Host "API: http://localhost:8000"
	Write-Host "Docs: http://localhost:8000/docs"
	Write-Host "MLflow: http://localhost:5000"
	Write-Host "`nManage with:"
	Write-Host " docker-compose logs -f # View logs"
	Write-Host " docker-compose down # Stop services"
	Write-Host " docker-compose restart # Restart services"
	Write-Host " docker-compose ps # Check status"
	} elseif ($env:DEPLOYMENT_TARGET -eq "staging") {
	Write-Host "`n=== Staging Environment ==="
	Write-Host "API: http://localhost:8001"
	Write-Host "Docs: http://localhost:8001/docs"
	Write-Host "Health: http://localhost:8001/health"
	Write-Host "`nContainer Details:"
	docker ps --filter "name=ml-api-staging" --format " ID: {{.ID}}"
	docker ps --filter "name=ml-api-staging" --format " Status: {{.Status}}"
	docker ps --filter "name=ml-api-staging" --format " Uptime: {{.RunningFor}}"
	Write-Host "`nManagement Commands:"
	Write-Host " docker logs ml-api-staging # View logs"
	Write-Host " docker logs -f ml-api-staging # Follow logs"
	Write-Host " docker stop ml-api-staging # Stop container"
	Write-Host " docker start ml-api-staging # Start container"
	Write-Host " docker restart ml-api-staging # Restart container"
	Write-Host " docker stats ml-api-staging # Resource usage"
	Write-Host " docker inspect ml-api-staging # Full details"
	Write-Host "`nValidation Completed:"
	Write-Host " - Health checks: 5/5 passed"
	Write-Host " - Prediction endpoint: Verified"
	Write-Host " - Load testing: 50 requests"
	Write-Host " - Container health: Monitored"
	} elseif ($env:DEPLOYMENT_TARGET -eq "production") {
	Write-Host "`n=== Production Deployment ==="
	Write-Host "Status: Ready for cloud deployment"
	Write-Host "Image Tag: ml-churn-api:${{ github.sha }}"
	Write-Host "`nNext Steps:"
	Write-Host " 1. Push image to container registry"
	Write-Host " 2. Deploy to cloud platform"
	Write-Host " 3. Configure auto-scaling"
	Write-Host " 4. Set up monitoring"
	Write-Host " 5. Enable alerts"
	}
	Write-Host "================================================"

Navigation Menu

Fix Docker daemon startup timing issue in CI/CD #21

Workflow file

Fix Docker daemon startup timing issue in CI/CD #21

Uh oh!

Workflow file for this run