From b40001c5f9171e63b3e14097d36c41820d742cd8 Mon Sep 17 00:00:00 2001 From: Peter Munachi Date: Tue, 13 Jan 2026 18:52:53 +0100 Subject: [PATCH 1/7] feat: render deployment --- Dockerfile | 23 +- Procfile | 4 +- README.md | 8 + RENDER_DEPLOYMENT.md | 359 ++++++++++++++++++++++++ entrypoints/docker-entrypoint-web.sh | 76 +++++ entrypoints/docker-entrypoint-worker.sh | 33 +++ render.yaml | 112 ++++++++ 7 files changed, 610 insertions(+), 5 deletions(-) create mode 100644 RENDER_DEPLOYMENT.md create mode 100755 entrypoints/docker-entrypoint-web.sh create mode 100755 entrypoints/docker-entrypoint-worker.sh create mode 100644 render.yaml diff --git a/Dockerfile b/Dockerfile index aa7e80f..e512d5d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,13 +5,30 @@ ENV PYTHONDONTWRITEBYTECODE=1 WORKDIR /app +# Install system dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + gcc \ + postgresql-client \ + && rm -rf /var/lib/apt/lists/* + +# Copy dependency files COPY uv.lock pyproject.toml ./ + +# Install Python dependencies RUN pip install -U pip && pip install uv && uv sync --frozen --no-install-project --no-dev -RUN addgroup --gid 10000 django && adduser --shell /bin/bash --disabled-password --gecos "" --uid 10000 --ingroup django django -RUN chown -R django:django /app -USER django:django +# Create non-root user +RUN addgroup --gid 10000 django && adduser --shell /bin/bash --disabled-password --gecos "" --uid 10000 --ingroup django django +# Copy application code COPY --chown=django:django . . +# Make entrypoint scripts executable +RUN chmod +x entrypoints/docker-entrypoint-web.sh entrypoints/docker-entrypoint-worker.sh + +USER django:django + EXPOSE 8000 + +# Default to web entrypoint (can be overridden) +CMD ["./entrypoints/docker-entrypoint-web.sh"] diff --git a/Procfile b/Procfile index 59f0f0b..0395dfd 100644 --- a/Procfile +++ b/Procfile @@ -1,2 +1,2 @@ -web: gunicorn flight_blender:app -worker: celery worker --app=flight_blender +web: uvicorn flight_blender.asgi:application --host 0.0.0.0 --port $PORT --workers 3 +worker: celery --app=flight_blender worker --loglevel=info diff --git a/README.md b/README.md index b64d2b3..3a9cc3b 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,14 @@ Follow our simple 5-step guide to deploy Flight Blender and explore its core fea πŸ“– [Read the 20-minute quickstart guide](deployment_support/README.md) to get started now! +--- + +## πŸš€ Deploy to Render.com + +Deploy Flight Blender to Render.com with our comprehensive deployment guide. + +πŸ“– [Read the Render.com deployment guide](RENDER_DEPLOYMENT.md) for step-by-step instructions! + --- ## πŸ’« Join the community [Discord](https://discord.gg/dnRxpZdd9a) diff --git a/RENDER_DEPLOYMENT.md b/RENDER_DEPLOYMENT.md new file mode 100644 index 0000000..0a8a92a --- /dev/null +++ b/RENDER_DEPLOYMENT.md @@ -0,0 +1,359 @@ +# Deploying Flight Blender to Render.com (Docker) + +This guide will walk you through deploying Flight Blender to Render.com using Docker, including the web service, background worker (Celery), PostgreSQL database, and Redis instance. + +> **Note**: This guide uses Docker for deployment. The Dockerfile and entrypoint scripts are configured to automatically handle service dependencies, migrations, and static file collection. + +## Prerequisites + +- A GitHub account with your Flight Blender repository +- A Render.com account (free tier available) +- Basic understanding of environment variables and Docker + +## Overview + +Flight Blender requires the following services on Render: +1. **Web Service** - Main Django application +2. **Background Worker** - Celery worker for async tasks +3. **PostgreSQL Database** - For persistent data storage +4. **Redis** - For caching and Celery message broker + +## Step 1: Create PostgreSQL Database + +1. Go to your Render dashboard +2. Click **"New +"** β†’ **"PostgreSQL"** +3. Configure: + - **Name**: `flight-blender-db` (or your preferred name) + - **Database**: `flight_blender` (or your preferred name) + - **User**: Auto-generated (or custom) + - **Region**: Choose closest to your users + - **PostgreSQL Version**: 17 (or latest) + - **Plan**: Free tier available for testing +4. Click **"Create Database"** +5. **Important**: Note down the connection string from the dashboard (you'll need it later) + +## Step 2: Create Redis Instance + +1. Go to your Render dashboard +2. Click **"New +"** β†’ **"Redis"** +3. Configure: + - **Name**: `flight-blender-redis` (or your preferred name) + - **Region**: Same as PostgreSQL + - **Plan**: Free tier available for testing +4. Click **"Create Redis"** +5. **Important**: Note down the connection details (host, port, password) + +## Step 3: Deploy Web Service (Docker) + +1. Go to your Render dashboard +2. Click **"New +"** β†’ **"Web Service"** +3. Connect your GitHub repository: + - Select your Flight Blender repository + - Choose the branch (usually `main` or `master`) +4. Configure the service: + - **Name**: `flight-blender-web` (or your preferred name) + - **Environment**: `Docker` + - **Region**: Same as your database + - **Branch**: `main` (or your default branch) + - **Dockerfile Path**: `Dockerfile` (or leave empty if Dockerfile is in root) + - **Docker Context**: Leave empty (or set if Dockerfile is in subdirectory) + - **Docker Command**: Leave empty (uses CMD from Dockerfile) + - **Plan**: Choose based on your needs (free tier available) + + **Note**: The Dockerfile is configured to use the web entrypoint by default. The entrypoint script will: + - Wait for Redis and PostgreSQL to be ready + - Collect static files + - Run database migrations + - Start the uvicorn server + +### Environment Variables for Web Service + +Add these environment variables in the Render dashboard under "Environment": + +#### Required Variables + +```bash +# Django Settings +SECRET_KEY=your-secret-key-here-generate-a-long-random-string +IS_DEBUG=0 +ALLOWED_HOSTS=your-app-name.onrender.com,localhost +USE_LOCAL_SQLITE_DATABASE=0 + +# Database (use the connection string from Step 1) +DATABASE_URL=postgresql://user:password@hostname:5432/database_name + +# Redis Configuration (use details from Step 2) +REDIS_HOST=your-redis-host.onrender.com +REDIS_PORT=6379 +REDIS_PASSWORD=your-redis-password +REDIS_BROKER_URL=redis://:password@your-redis-host.onrender.com:6379/0 + +# Application Settings +FLIGHTBLENDER_FQDN=https://your-app-name.onrender.com +HEARTBEAT_RATE_SECS=2 + +# Network Mode (set to 0 for standalone, 1 for DSS integration) +USSP_NETWORK_ENABLED=0 +DSS_SELF_AUDIENCE=your-app-name.onrender.com +``` + +#### Optional Variables (for DSS integration) + +```bash +# Only needed if USSP_NETWORK_ENABLED=1 +AUTH_DSS_CLIENT_ID=your-client-id +AUTH_DSS_CLIENT_SECRET=your-client-secret +DSS_BASE_URL=https://your-dss-url.com +``` + +#### Security Note + +**IMPORTANT**: For production, do NOT set `BYPASS_AUTH_TOKEN_VERIFICATION=1`. This should only be used for local development. + +## Step 4: Deploy Background Worker (Celery) - Docker + +1. Go to your Render dashboard +2. Click **"New +"** β†’ **"Background Worker"** +3. Connect the same GitHub repository +4. Configure: + - **Name**: `flight-blender-worker` (or your preferred name) + - **Environment**: `Docker` + - **Region**: Same as web service + - **Branch**: Same as web service + - **Dockerfile Path**: `Dockerfile` (or leave empty if Dockerfile is in root) + - **Docker Context**: Leave empty + - **Docker Command**: `./entrypoints/docker-entrypoint-worker.sh` + - **Plan**: Choose based on your needs + + **Note**: The worker uses the same Dockerfile but with a different entrypoint command. The entrypoint script will: + - Wait for Redis to be ready + - Start the Celery worker + +### Environment Variables for Worker + +Add the same environment variables as the web service (except `ALLOWED_HOSTS` which is web-only): + +- `SECRET_KEY` +- `DATABASE_URL` +- `REDIS_HOST` +- `REDIS_PORT` +- `REDIS_PASSWORD` +- `REDIS_BROKER_URL` +- All other variables from the web service + +## Step 5: Database Migrations + +**Good News**: Database migrations run automatically when the web service starts! The Docker entrypoint script (`docker-entrypoint-web.sh`) includes: +- Automatic migration execution on startup +- Static file collection + +If you need to run migrations manually or create a superuser: + +1. Go to your web service in Render dashboard +2. Click on **"Shell"** tab +3. Run: + ```bash + python manage.py migrate + ``` +4. (Optional) Create a superuser: + ```bash + python manage.py createsuperuser + ``` + +## Step 6: Verify Deployment + +1. Visit your web service URL: `https://your-app-name.onrender.com` +2. You should see the Flight Blender logo and API documentation links +3. Test the ping endpoint: `https://your-app-name.onrender.com/ping` +4. Check logs in Render dashboard to ensure no errors + +## Using render.yaml (Alternative Method) + +For a more automated setup, you can use a `render.yaml` file. The file is already included in the repository root and configured for Docker deployment: + +```yaml +services: + - type: web + name: flight-blender-web + env: docker + dockerfilePath: Dockerfile + dockerContext: . + envVars: + - key: SECRET_KEY + generateValue: true + - key: IS_DEBUG + value: 0 + - key: USE_LOCAL_SQLITE_DATABASE + value: 0 + - key: ALLOWED_HOSTS + fromService: + type: web + name: flight-blender-web + property: host + - key: DATABASE_URL + fromDatabase: + name: flight-blender-db + property: connectionString + - key: REDIS_HOST + fromService: + type: redis + name: flight-blender-redis + property: host + - key: REDIS_PORT + fromService: + type: redis + name: flight-blender-redis + property: port + - key: REDIS_PASSWORD + fromService: + type: redis + name: flight-blender-redis + property: password + - key: REDIS_BROKER_URL + fromService: + type: redis + name: flight-blender-redis + property: connectionString + - key: FLIGHTBLENDER_FQDN + fromService: + type: web + name: flight-blender-web + property: host + - key: HEARTBEAT_RATE_SECS + value: 2 + - key: USSP_NETWORK_ENABLED + value: 0 + + - type: worker + name: flight-blender-worker + env: docker + dockerfilePath: Dockerfile + dockerContext: . + dockerCommand: ./entrypoints/docker-entrypoint-worker.sh + envVars: + - key: SECRET_KEY + fromService: + type: web + name: flight-blender-web + property: envVar + value: SECRET_KEY + - key: DATABASE_URL + fromDatabase: + name: flight-blender-db + property: connectionString + - key: REDIS_HOST + fromService: + type: redis + name: flight-blender-redis + property: host + - key: REDIS_PORT + fromService: + type: redis + name: flight-blender-redis + property: port + - key: REDIS_PASSWORD + fromService: + type: redis + name: flight-blender-redis + property: password + - key: REDIS_BROKER_URL + fromService: + type: redis + name: flight-blender-redis + property: connectionString + +databases: + - name: flight-blender-db + databaseName: flight_blender + user: flight_blender_user + plan: free + +services: + - type: redis + name: flight-blender-redis + plan: free +``` + +Then deploy via: +1. Go to Render dashboard +2. Click **"New +"** β†’ **"Blueprint"** +3. Connect your repository +4. Render will automatically detect and use `render.yaml` + +## Troubleshooting + +### Common Issues + +1. **Database Connection Errors** + - Verify `DATABASE_URL` is correctly set + - Ensure PostgreSQL service is running + - Check that database name, user, and password are correct + +2. **Redis Connection Errors** + - Verify `REDIS_HOST`, `REDIS_PORT`, and `REDIS_PASSWORD` are set + - Ensure Redis service is running + - Check `REDIS_BROKER_URL` format: `redis://:password@host:port/0` + +3. **Static Files Not Loading** + - Static files are automatically collected by the Docker entrypoint script + - Check logs to see if `collectstatic` ran successfully + - Check `STATIC_URL` setting in `settings.py` + - For Docker: Ensure the entrypoint script has proper permissions + +4. **Worker Not Processing Tasks** + - Verify worker service is running + - Check that `REDIS_BROKER_URL` matches in both web and worker services + - Review worker logs for errors + +5. **Application Crashes on Startup** + - Check logs in Render dashboard + - Verify all required environment variables are set + - Ensure migrations have run (they run automatically in Docker) + - For Docker: Check that entrypoint scripts are executable (`chmod +x entrypoints/docker-entrypoint-*.sh`) + - Verify Docker image builds successfully + +6. **Docker-Specific Issues** + - **Build fails**: Check Dockerfile syntax and ensure all dependencies are listed in `pyproject.toml` + - **Entrypoint script not found**: Ensure scripts are in `entrypoints/` directory and are executable + - **Port binding errors**: Render automatically sets `$PORT` environment variable - ensure your app uses it + - **Service wait timeouts**: Entrypoint scripts wait for Redis/PostgreSQL with 5-second timeouts - increase if services are slow to start + +### Checking Logs + +- **Web Service**: Dashboard β†’ Your Web Service β†’ "Logs" tab +- **Worker**: Dashboard β†’ Your Worker β†’ "Logs" tab +- **Database**: Dashboard β†’ Your Database β†’ "Logs" tab +- **Redis**: Dashboard β†’ Your Redis β†’ "Logs" tab + +## Security Best Practices + +1. **Never commit secrets**: Use Render's environment variables +2. **Use strong SECRET_KEY**: Generate a long random string +3. **Set IS_DEBUG=0**: For production deployments +4. **Configure ALLOWED_HOSTS**: Set to your actual domain +5. **Remove BYPASS_AUTH_TOKEN_VERIFICATION**: Never use in production +6. **Use HTTPS**: Render provides this automatically +7. **Regular updates**: Keep dependencies updated + +## Scaling + +Render allows you to scale services: +- **Web Service**: Scale horizontally by increasing instance count +- **Worker**: Scale workers based on task volume +- **Database**: Upgrade plan for better performance +- **Redis**: Upgrade plan for larger cache/message queue + +## Next Steps + +After deployment: +1. Set up custom domain (optional) +2. Configure Flight Passport for OAuth (production) +3. Set up monitoring and alerts +4. Configure backups for PostgreSQL +5. Review and optimize performance + +## Additional Resources + +- [Render Documentation](https://render.com/docs) +- [Flight Blender API Documentation](http://redocly.github.io/redoc/?url=https://raw.githubusercontent.com/openutm/flight-blender/master/api/flight-blender-server-1.0.0-resolved.yaml) +- [Flight Blender Quickstart Guide](deployment_support/README.md) diff --git a/entrypoints/docker-entrypoint-web.sh b/entrypoints/docker-entrypoint-web.sh new file mode 100755 index 0000000..95eb35f --- /dev/null +++ b/entrypoints/docker-entrypoint-web.sh @@ -0,0 +1,76 @@ +#!/bin/bash +set -e + +echo "Waiting for services..." + +# Wait for Redis +if [ -n "$REDIS_HOST" ]; then + echo "Waiting for Redis at ${REDIS_HOST}:${REDIS_PORT:-6379}..." + until python -c " +import redis +import sys +try: + r = redis.Redis( + host='${REDIS_HOST}', + port=${REDIS_PORT:-6379}, + password='${REDIS_PASSWORD:-}' if '${REDIS_PASSWORD:-}' else None, + decode_responses=True, + socket_connect_timeout=5 + ) + r.ping() + print('Redis is ready!') +except Exception as e: + sys.exit(1) +" 2>/dev/null; do + echo "Waiting for Redis..." + sleep 2 + done + echo "Redis is ready!" +fi + +# Wait for PostgreSQL +if [ -n "$DATABASE_URL" ] || [ -n "$POSTGRES_HOST" ]; then + echo "Waiting for PostgreSQL..." + until python -c " +import sys +try: + if '${DATABASE_URL}': + import psycopg2 + from urllib.parse import urlparse + conn = psycopg2.connect('${DATABASE_URL}') + conn.close() + print('PostgreSQL is ready!') + elif '${POSTGRES_HOST}': + import psycopg2 + conn = psycopg2.connect( + host='${POSTGRES_HOST}', + port=${POSTGRES_PORT:-5432}, + user='${POSTGRES_USER}', + password='${POSTGRES_PASSWORD}', + dbname='${POSTGRES_DB}', + connect_timeout=5 + ) + conn.close() + print('PostgreSQL is ready!') +except Exception as e: + sys.exit(1) +" 2>/dev/null; do + echo "Waiting for PostgreSQL..." + sleep 2 + done + echo "PostgreSQL is ready!" +fi + +echo "All services are ready!" + +# Collect static files +echo "Collecting static files..." +python manage.py collectstatic --noinput || echo "Warning: Static files collection failed, continuing..." + +# Apply database migrations +echo "Applying database migrations..." +python manage.py migrate || echo "Warning: Migrations failed, continuing..." + +# Start server +echo "Starting server on port ${PORT:-8000}..." +exec uvicorn flight_blender.asgi:application --host 0.0.0.0 --port ${PORT:-8000} --workers 3 diff --git a/entrypoints/docker-entrypoint-worker.sh b/entrypoints/docker-entrypoint-worker.sh new file mode 100755 index 0000000..2a5d0f2 --- /dev/null +++ b/entrypoints/docker-entrypoint-worker.sh @@ -0,0 +1,33 @@ +#!/bin/bash +set -e + +# Wait for Redis +if [ -n "$REDIS_HOST" ]; then + echo "Waiting for Redis at ${REDIS_HOST}:${REDIS_PORT:-6379}..." + until python -c " +import redis +import sys +try: + r = redis.Redis( + host='${REDIS_HOST}', + port=${REDIS_PORT:-6379}, + password='${REDIS_PASSWORD:-}' if '${REDIS_PASSWORD:-}' else None, + decode_responses=True, + socket_connect_timeout=5 + ) + r.ping() + print('Redis is ready!') +except Exception as e: + sys.exit(1) +" 2>/dev/null; do + echo "Waiting for Redis..." + sleep 2 + done + echo "Redis is ready!" +else + echo "Warning: REDIS_HOST not set, skipping Redis check" +fi + +# Start Celery worker +echo "Starting Celery worker..." +exec celery --app=flight_blender worker --loglevel=info diff --git a/render.yaml b/render.yaml new file mode 100644 index 0000000..e863bf6 --- /dev/null +++ b/render.yaml @@ -0,0 +1,112 @@ +services: + - type: web + name: flight-blender-web + env: docker + dockerfilePath: Dockerfile + dockerContext: . + envVars: + - key: SECRET_KEY + generateValue: true + - key: IS_DEBUG + value: 0 + - key: USE_LOCAL_SQLITE_DATABASE + value: 0 + - key: ALLOWED_HOSTS + fromService: + type: web + name: flight-blender-web + property: host + - key: DATABASE_URL + fromDatabase: + name: flight-blender-db + property: connectionString + - key: REDIS_HOST + fromService: + type: redis + name: flight-blender-redis + property: host + - key: REDIS_PORT + fromService: + type: redis + name: flight-blender-redis + property: port + - key: REDIS_PASSWORD + fromService: + type: redis + name: flight-blender-redis + property: password + - key: REDIS_BROKER_URL + fromService: + type: redis + name: flight-blender-redis + property: connectionString + - key: FLIGHTBLENDER_FQDN + fromService: + type: web + name: flight-blender-web + property: host + - key: HEARTBEAT_RATE_SECS + value: 2 + - key: USSP_NETWORK_ENABLED + value: 0 + - key: DSS_SELF_AUDIENCE + fromService: + type: web + name: flight-blender-web + property: host + + - type: worker + name: flight-blender-worker + env: docker + dockerfilePath: Dockerfile + dockerContext: . + dockerCommand: ./entrypoints/docker-entrypoint-worker.sh + envVars: + - key: SECRET_KEY + fromService: + type: web + name: flight-blender-web + property: envVar + value: SECRET_KEY + - key: IS_DEBUG + value: 0 + - key: USE_LOCAL_SQLITE_DATABASE + value: 0 + - key: DATABASE_URL + fromDatabase: + name: flight-blender-db + property: connectionString + - key: REDIS_HOST + fromService: + type: redis + name: flight-blender-redis + property: host + - key: REDIS_PORT + fromService: + type: redis + name: flight-blender-redis + property: port + - key: REDIS_PASSWORD + fromService: + type: redis + name: flight-blender-redis + property: password + - key: REDIS_BROKER_URL + fromService: + type: redis + name: flight-blender-redis + property: connectionString + - key: HEARTBEAT_RATE_SECS + value: 2 + - key: USSP_NETWORK_ENABLED + value: 0 + + - type: redis + name: flight-blender-redis + plan: free + +databases: + - name: flight-blender-db + databaseName: flight_blender + user: flight_blender_user + plan: free From c1337b2745a1503b6de9b70113b968afd99f2751 Mon Sep 17 00:00:00 2001 From: Peter Munachi Date: Tue, 13 Jan 2026 19:49:55 +0100 Subject: [PATCH 2/7] feat: prod --- Dockerfile | 8 +- PRODUCTION_DOCKER_SETUP.md | 287 ++++++++++++++++++ docker-compose.yml | 8 +- entrypoints/no-database/entrypoint-celery.sh | 8 +- entrypoints/no-database/entrypoint.sh | 12 +- entrypoints/wait_for_service.py | 45 +++ .../with-database/entrypoint-celery.sh | 8 +- 7 files changed, 353 insertions(+), 23 deletions(-) create mode 100644 PRODUCTION_DOCKER_SETUP.md create mode 100644 entrypoints/wait_for_service.py diff --git a/Dockerfile b/Dockerfile index e512d5d..9d8da23 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,5 @@ -FROM --platform=linux/amd64 python:3.12-slim +# FROM --platform=linux/amd64 +FROM python:3.12-slim ENV PYTHONUNBUFFERED=1 ENV PYTHONDONTWRITEBYTECODE=1 @@ -17,6 +18,9 @@ COPY uv.lock pyproject.toml ./ # Install Python dependencies RUN pip install -U pip && pip install uv && uv sync --frozen --no-install-project --no-dev +# Set PYTHONPATH to include the current directory so Django can find the modules +ENV PYTHONPATH=/app + # Create non-root user RUN addgroup --gid 10000 django && adduser --shell /bin/bash --disabled-password --gecos "" --uid 10000 --ingroup django django @@ -24,7 +28,7 @@ RUN addgroup --gid 10000 django && adduser --shell /bin/bash --disabled-password COPY --chown=django:django . . # Make entrypoint scripts executable -RUN chmod +x entrypoints/docker-entrypoint-web.sh entrypoints/docker-entrypoint-worker.sh +RUN chmod +x entrypoints/docker-entrypoint-web.sh entrypoints/docker-entrypoint-worker.sh entrypoints/wait_for_service.py USER django:django diff --git a/PRODUCTION_DOCKER_SETUP.md b/PRODUCTION_DOCKER_SETUP.md new file mode 100644 index 0000000..e6569e3 --- /dev/null +++ b/PRODUCTION_DOCKER_SETUP.md @@ -0,0 +1,287 @@ +# Running Flight Blender in Production Mode Locally with Docker + +This guide will help you run Flight Blender in production mode locally using Docker. The production setup uses `docker-compose.yml` which is configured for production-like environments. + +## Prerequisites + +- **Docker** (version 20.10 or later) +- **Docker Compose** (version 2.0 or later) +- At least **4GB of RAM** available for Docker +- Ports **8000**, **5432**, and **6379** available on your system + +## Step-by-Step Instructions + +### 1. Create the External Docker Network + +The production Docker Compose setup requires an external network. Create it first: + +```bash +docker network create interop_ecosystem_network +``` + +If the network already exists, you'll see a message indicating that. This is fine - you can proceed. + +### 2. Create Environment File (.env) + +Create a `.env` file in the root directory of `flight_blender` with the following minimum required variables: + +```bash +# Django Settings +SECRET_KEY=your-very-long-random-secret-key-here-minimum-50-characters +IS_DEBUG=0 +ALLOWED_HOSTS=localhost,127.0.0.1 + +# Database Configuration +POSTGRES_USER=flightblender +POSTGRES_PASSWORD=your-secure-password-here +POSTGRES_DB=flightblender +POSTGRES_HOST=db-blender +DATABASE_URL=postgresql://flightblender:your-secure-password-here@db-blender:5432/flightblender + +# Redis Configuration +REDIS_HOST=redis-blender +REDIS_PORT=6379 +REDIS_PASSWORD=your-redis-password-here +REDIS_BROKER_URL=redis://:your-redis-password-here@redis-blender:6379/ + +# Optional: Standalone Mode (set to 0 for standalone, 1 for DSS integration) +USSP_NETWORK_ENABLED=0 + +# Optional: Heartbeat Rate +HEARTBEAT_RATE_SECS=2 + +# Optional: Flight Blender FQDN (for production) +FLIGHTBLENDER_FQDN=http://localhost:8000 +``` + +**⚠️ Important Security Notes:** +- **DO NOT** set `BYPASS_AUTH_TOKEN_VERIFICATION=1` in production mode +- Use strong, unique passwords for `POSTGRES_PASSWORD` and `REDIS_PASSWORD` +- Generate a secure `SECRET_KEY` (you can use: `python -c "from django.core.management.utils import get_random_secret_key; print(get_random_secret_key())"`) + +### 3. Build the Docker Image + +Build the production Docker image: + +```bash +cd /Users/petermunachiali/Documents/Github/SkyTrade/UTM/flight_blender +docker build . -t openutm/flight-blender +``` + +This will: +- Install system dependencies (gcc, postgresql-client) +- Install Python dependencies using `uv` +- Create a non-root user (django:django) +- Copy application code +- Set up entrypoint scripts + +**Note:** The build process may take several minutes on first run as it downloads dependencies. + +### 4. Start the Services + +Start all services using Docker Compose: + +```bash +docker compose up -d +``` + +Or to see logs in real-time: + +```bash +docker compose up +``` + +This will start the following services: +- **db-blender**: PostgreSQL 17 database +- **redis-blender**: Redis/Valkey cache and message broker +- **flight-blender**: Main Django application (port 8000) +- **flight-blender-celery**: Celery worker for background tasks + +### 5. Verify Services are Running + +Check that all containers are running: + +```bash +docker compose ps +``` + +You should see all four services with status "Up" or "Up (healthy)". + +### 6. Check Application Logs + +Monitor the application logs to ensure everything started correctly: + +```bash +# View all logs +docker compose logs -f + +# View logs for specific service +docker compose logs -f flight-blender +docker compose logs -f flight-blender-celery +``` + +Look for: +- Database migrations being applied successfully +- Server starting on port 8000 +- No error messages + +### 7. Access the Application + +Once the services are running, access Flight Blender at: + +- **Web Interface**: http://localhost:8000 +- **API Documentation**: http://localhost:8000/api/docs +- **Health Check**: http://localhost:8000/ping + +You should see the Flight Blender logo and links to the API documentation. + +### 8. Stop the Services + +When you're done, stop all services: + +```bash +docker compose down +``` + +To also remove volumes (this will delete database data): + +```bash +docker compose down -v +``` + +## Production vs Development Differences + +The production setup (`docker-compose.yml`) differs from development (`docker-compose-dev.yml`) in several ways: + +| Feature | Production | Development | +|---------|-----------|-------------| +| Network | External network required | Internal network | +| Volumes | No code volume mount | Code volume mounted | +| Image name | `openutm/flight-blender` | `openutm/flight-blender-dev` | +| Entrypoint | `no-database/entrypoint.sh` | `with-database/entrypoint.sh` | +| Celery Beat | Not included | Included | +| Database port | Not exposed | Exposed (5432) | + +## Troubleshooting + +### Issue: Network not found error + +**Error:** `network interop_ecosystem_network not found` + +**Solution:** +```bash +docker network create interop_ecosystem_network +``` + +### Issue: Port already in use + +**Error:** `Bind for 0.0.0.0:8000 failed: port is already allocated` + +**Solution:** +- Check what's using the port: `lsof -i :8000` (macOS/Linux) or `netstat -ano | findstr :8000` (Windows) +- Stop the conflicting service or change the port in `docker-compose.yml` + +### Issue: Database connection errors + +**Error:** `could not connect to server: Connection refused` + +**Solution:** +1. Verify database container is running: `docker compose ps` +2. Check database logs: `docker compose logs db-blender` +3. Ensure `.env` file has correct `POSTGRES_HOST=db-blender` +4. Wait a few seconds for database to fully initialize + +### Issue: Redis connection errors + +**Error:** `Error connecting to Redis` + +**Solution:** +1. Verify Redis container is running: `docker compose ps` +2. Check Redis logs: `docker compose logs redis-blender` +3. Ensure `REDIS_PASSWORD` in `.env` matches the password used in Redis command +4. Verify `REDIS_BROKER_URL` format: `redis://:password@host:port/` + +### Issue: Migration errors + +**Error:** `django.db.utils.OperationalError` + +**Solution:** +1. Ensure database container is fully started (wait 10-15 seconds) +2. Check database logs: `docker compose logs db-blender` +3. Try restarting: `docker compose restart flight-blender` + +### Issue: Permission errors + +**Error:** `Permission denied` when accessing files + +**Solution:** +- The Docker image runs as non-root user (django:django) +- Ensure entrypoint scripts are executable (handled in Dockerfile) +- Check file ownership if using volumes + +### Viewing Container Logs + +To debug issues, you can view logs for specific services: + +```bash +# All services +docker compose logs + +# Specific service +docker compose logs flight-blender +docker compose logs db-blender +docker compose logs redis-blender +docker compose logs flight-blender-celery + +# Follow logs in real-time +docker compose logs -f flight-blender + +# Last 100 lines +docker compose logs --tail=100 flight-blender +``` + +### Accessing Containers + +To access a running container for debugging: + +```bash +# Access flight-blender container +docker exec -it flight-blender bash + +# Access database container +docker exec -it db-blender psql -U flightblender -d flightblender + +# Access Redis container +docker exec -it redis-blender redis-cli -a your-redis-password +``` + +## Next Steps + +Once Flight Blender is running: + +1. **Test the API**: Import the [Postman Collection](api/flight_blender_api.postman_collection.json) +2. **Generate Access Tokens**: Use the [verification repository](https://github.com/openutm/verification) to generate tokens +3. **Explore API Documentation**: Visit http://localhost:8000/api/docs +4. **Submit Flight Data**: Use the API to submit flight declarations and other data + +## Additional Resources + +- [20-minute Quickstart Guide](deployment_support/README.md) +- [Render.com Deployment Guide](RENDER_DEPLOYMENT.md) +- [API Documentation](http://redocly.github.io/redoc/?url=https://raw.githubusercontent.com/openutm/flight-blender/master/api/flight-blender-server-1.0.0-resolved.yaml) +- [Flight Blender Verification](https://github.com/openutm/verification) + +## Clean Up + +To completely remove all containers, volumes, and networks: + +```bash +# Stop and remove containers +docker compose down -v + +# Remove the external network (if not used by other services) +docker network rm interop_ecosystem_network + +# Remove the Docker image +docker rmi openutm/flight-blender +``` diff --git a/docker-compose.yml b/docker-compose.yml index 8a538ae..bf69355 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ version: "3" services: redis-blender: - platform: linux/amd64 + # platform: linux/amd64 command: ["redis-server", "/redis.conf", "--requirepass", "$REDIS_PASSWORD"] image: "valkey/valkey:latest" expose: @@ -14,7 +14,7 @@ services: networks: - interop_ecosystem_network db-blender: - platform: linux/amd64 + # platform: linux/amd64 container_name: "db-blender" image: postgres:17 expose: @@ -28,7 +28,7 @@ services: - interop_ecosystem_network flight-blender: - platform: linux/amd64 + # platform: linux/amd64 container_name: "flight-blender" env_file: - ".env" @@ -49,7 +49,7 @@ services: flight-blender-celery: - platform: linux/amd64 + # platform: linux/amd64 container_name: worker image: openutm/flight-blender build: diff --git a/entrypoints/no-database/entrypoint-celery.sh b/entrypoints/no-database/entrypoint-celery.sh index ee37b92..d5acab5 100755 --- a/entrypoints/no-database/entrypoint-celery.sh +++ b/entrypoints/no-database/entrypoint-celery.sh @@ -1,10 +1,8 @@ #!/bin/bash -source .venv/bin/activate - echo Waiting for DBs... -if ! wait-for-it --parallel --service $REDIS_HOST:$REDIS_PORT; then - exit +if ! uv run python entrypoints/wait_for_service.py --service $REDIS_HOST:$REDIS_PORT; then + exit 1 fi -celery --app=flight_blender worker --loglevel=info +uv run celery --app=flight_blender worker --loglevel=info diff --git a/entrypoints/no-database/entrypoint.sh b/entrypoints/no-database/entrypoint.sh index 4b3c56b..19ce110 100755 --- a/entrypoints/no-database/entrypoint.sh +++ b/entrypoints/no-database/entrypoint.sh @@ -1,20 +1,18 @@ #!/bin/bash -source .venv/bin/activate - echo Waiting for DBs... -if ! wait-for-it --parallel --service $REDIS_HOST:$REDIS_PORT; then - exit +if ! uv run python entrypoints/wait_for_service.py --service $REDIS_HOST:$REDIS_PORT; then + exit 1 fi # Collect static files #echo "Collect static files" -#python manage.py collectstatic --noinput +#uv run python manage.py collectstatic --noinput # Apply database migrations echo "Apply database migrations" -python manage.py migrate +uv run python manage.py migrate # Start server echo "Starting server" -uvicorn flight_blender.asgi:application --host 0.0.0.0 --port 8000 --workers 3 --reload +uv run uvicorn flight_blender.asgi:application --host 0.0.0.0 --port 8000 --workers 3 --reload diff --git a/entrypoints/wait_for_service.py b/entrypoints/wait_for_service.py new file mode 100644 index 0000000..b732cfb --- /dev/null +++ b/entrypoints/wait_for_service.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +"""Simple script to wait for services to be available.""" +import sys +import socket +import time + +def wait_for_service(host, port, timeout=30): + """Wait for a service to be available on host:port.""" + start_time = time.time() + while time.time() - start_time < timeout: + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(1) + result = sock.connect_ex((host, port)) + sock.close() + if result == 0: + print(f"Service {host}:{port} is available") + return True + except Exception as e: + pass + time.sleep(1) + print(f"Timeout waiting for {host}:{port}") + return False + +if __name__ == "__main__": + services = [] + i = 1 + while i < len(sys.argv): + if sys.argv[i] == "--service" and i + 1 < len(sys.argv): + host, port = sys.argv[i + 1].split(":") + services.append((host, int(port))) + i += 2 + else: + i += 1 + + if not services: + print("Usage: wait_for_service.py --service host:port [--service host:port ...]") + sys.exit(1) + + all_available = True + for host, port in services: + if not wait_for_service(host, port): + all_available = False + + sys.exit(0 if all_available else 1) diff --git a/entrypoints/with-database/entrypoint-celery.sh b/entrypoints/with-database/entrypoint-celery.sh index ee37b92..d5acab5 100755 --- a/entrypoints/with-database/entrypoint-celery.sh +++ b/entrypoints/with-database/entrypoint-celery.sh @@ -1,10 +1,8 @@ #!/bin/bash -source .venv/bin/activate - echo Waiting for DBs... -if ! wait-for-it --parallel --service $REDIS_HOST:$REDIS_PORT; then - exit +if ! uv run python entrypoints/wait_for_service.py --service $REDIS_HOST:$REDIS_PORT; then + exit 1 fi -celery --app=flight_blender worker --loglevel=info +uv run celery --app=flight_blender worker --loglevel=info From 79437bf43342eb0fbd48d01898c32436bbf4047c Mon Sep 17 00:00:00 2001 From: Peter Munachi Date: Tue, 13 Jan 2026 20:05:47 +0100 Subject: [PATCH 3/7] feat: prod --- RENDER_NON_DOCKER_SETUP.md | 299 ++++++++++++++++++++++++ entrypoints/docker-entrypoint-web.sh | 71 ++---- entrypoints/docker-entrypoint-worker.sh | 25 +- render-no-docker.yaml | 115 +++++++++ runtime.txt | 1 + 5 files changed, 442 insertions(+), 69 deletions(-) create mode 100644 RENDER_NON_DOCKER_SETUP.md create mode 100644 render-no-docker.yaml create mode 100644 runtime.txt diff --git a/RENDER_NON_DOCKER_SETUP.md b/RENDER_NON_DOCKER_SETUP.md new file mode 100644 index 0000000..b697a67 --- /dev/null +++ b/RENDER_NON_DOCKER_SETUP.md @@ -0,0 +1,299 @@ +# Deploying Flight Blender to Render.com (Without Docker) + +This guide will walk you through deploying Flight Blender to Render.com using Python buildpacks instead of Docker. This is useful if you prefer native Python deployment or want to avoid Docker overhead. + +## Prerequisites + +- A GitHub account with your Flight Blender repository +- A Render.com account (free tier available) +- Basic understanding of environment variables + +## Overview + +Flight Blender requires the following services on Render: +1. **Web Service** - Main Django application (Python) +2. **Background Worker** - Celery worker for async tasks (Python) +3. **PostgreSQL Database** - For persistent data storage +4. **Redis** - For caching and Celery message broker + +## Step 1: Create PostgreSQL Database + +1. Go to your Render dashboard +2. Click **"New +"** β†’ **"PostgreSQL"** +3. Configure: + - **Name**: `flight-blender-db` (or your preferred name) + - **Database**: `flight_blender` (or your preferred name) + - **User**: Auto-generated (or custom) + - **Region**: Choose closest to your users + - **PostgreSQL Version**: 17 (or latest) + - **Plan**: Free tier available for testing +4. Click **"Create Database"** +5. **Important**: Note down the connection string from the dashboard + +## Step 2: Create Redis Instance + +1. Go to your Render dashboard +2. Click **"New +"** β†’ **"Redis"** +3. Configure: + - **Name**: `flight-blender-redis` (or your preferred name) + - **Region**: Same as PostgreSQL + - **Plan**: Free tier available for testing +4. Click **"Create Redis"** +5. **Important**: Note down the connection details (host, port, password) + +## Step 3: Deploy Web Service (Python Buildpack) + +1. Go to your Render dashboard +2. Click **"New +"** β†’ **"Web Service"** +3. Connect your GitHub repository: + - Select your Flight Blender repository + - Choose the branch (usually `main` or `master`) +4. Configure the service: + - **Name**: `flight-blender-web` (or your preferred name) + - **Environment**: `Python 3` (NOT Docker) + - **Region**: Same as your database + - **Branch**: `main` (or your default branch) + - **Root Directory**: Leave empty (or set if app is in subdirectory) + - **Python Version**: 3.12 (will use `runtime.txt`) + - **Build Command**: + ```bash + pip install uv && uv sync --frozen --no-dev && python manage.py collectstatic --noinput && python manage.py migrate + ``` + - **Start Command**: + ```bash + uvicorn flight_blender.asgi:application --host 0.0.0.0 --port $PORT --workers 3 + ``` + - **Plan**: Choose based on your needs (free tier available) + +### Environment Variables for Web Service + +Add these environment variables in the Render dashboard under "Environment": + +#### Required Variables + +```bash +# Django Settings +SECRET_KEY=your-secret-key-here-generate-a-long-random-string +IS_DEBUG=0 +ALLOWED_HOSTS=your-app-name.onrender.com,localhost +USE_LOCAL_SQLITE_DATABASE=0 + +# Database (use the connection string from Step 1) +DATABASE_URL=postgresql://user:password@hostname:5432/database_name + +# Redis Configuration (use details from Step 2) +REDIS_HOST=your-redis-host.onrender.com +REDIS_PORT=6379 +REDIS_PASSWORD=your-redis-password +REDIS_BROKER_URL=redis://:password@your-redis-host.onrender.com:6379/0 + +# Application Settings +FLIGHTBLENDER_FQDN=https://your-app-name.onrender.com +HEARTBEAT_RATE_SECS=2 + +# Network Mode (set to 0 for standalone, 1 for DSS integration) +USSP_NETWORK_ENABLED=0 +DSS_SELF_AUDIENCE=your-app-name.onrender.com + +# Python Path (important for Python buildpack) +PYTHONPATH=/opt/render/project/src +``` + +#### Optional Variables (for DSS integration) + +```bash +# Only needed if USSP_NETWORK_ENABLED=1 +AUTH_DSS_CLIENT_ID=your-client-id +AUTH_DSS_CLIENT_SECRET=your-client-secret +DSS_BASE_URL=https://your-dss-url.com +``` + +#### Security Note + +**IMPORTANT**: For production, do NOT set `BYPASS_AUTH_TOKEN_VERIFICATION=1`. This should only be used for local development. + +## Step 4: Deploy Background Worker (Celery) - Python Buildpack + +1. Go to your Render dashboard +2. Click **"New +"** β†’ **"Background Worker"** +3. Connect the same GitHub repository +4. Configure: + - **Name**: `flight-blender-worker` (or your preferred name) + - **Environment**: `Python 3` (NOT Docker) + - **Region**: Same as web service + - **Branch**: Same as web service + - **Root Directory**: Leave empty + - **Python Version**: 3.12 + - **Build Command**: + ```bash + pip install uv && uv sync --frozen --no-dev + ``` + - **Start Command**: + ```bash + celery --app=flight_blender worker --loglevel=info + ``` + - **Plan**: Choose based on your needs + +### Environment Variables for Worker + +Add the same environment variables as the web service (except `ALLOWED_HOSTS` which is web-only): + +- `SECRET_KEY` +- `DATABASE_URL` +- `REDIS_HOST` +- `REDIS_PORT` +- `REDIS_PASSWORD` +- `REDIS_BROKER_URL` +- `PYTHONPATH` (set to `/opt/render/project/src`) +- All other variables from the web service + +## Step 5: Using render-no-docker.yaml (Alternative Method) + +For automated setup, you can use the `render-no-docker.yaml` file: + +1. Go to Render dashboard +2. Click **"New +"** β†’ **"Blueprint"** +3. Connect your repository +4. Render will automatically detect and use `render-no-docker.yaml` +5. All services will be created automatically with correct configuration + +**Note**: Make sure to use `render-no-docker.yaml` (not `render.yaml`) for non-Docker deployment. + +## Step 6: Verify Deployment + +1. Visit your web service URL: `https://your-app-name.onrender.com` +2. You should see the Flight Blender logo and API documentation links +3. Test the ping endpoint: `https://your-app-name.onrender.com/ping` +4. Check logs in Render dashboard to ensure no errors + +## Key Differences from Docker Deployment + +### Build Process +- **Docker**: Uses Dockerfile to build container image +- **Python Buildpack**: Uses `buildCommand` to install dependencies and prepare app + +### Start Command +- **Docker**: Uses CMD or entrypoint script from Dockerfile +- **Python Buildpack**: Uses explicit `startCommand` in Render config + +### Dependencies +- **Docker**: Installed during Docker build (in Dockerfile) +- **Python Buildpack**: Installed via `buildCommand` on each deploy + +### Migrations +- **Docker**: Run in entrypoint script on container start +- **Python Buildpack**: Run in `buildCommand` during build (recommended) or manually via Shell + +## Troubleshooting + +### Common Issues + +1. **Build Fails - uv not found** + - Ensure build command includes `pip install uv` first + - Check that `uv` is available in the Python environment + +2. **Module Not Found Errors** + - Verify `PYTHONPATH=/opt/render/project/src` is set + - Check that all dependencies are in `pyproject.toml` + - Ensure `uv sync` completes successfully + +3. **Database Connection Errors** + - Verify `DATABASE_URL` is correctly set + - Ensure PostgreSQL service is running + - Check that database name, user, and password are correct + +4. **Redis Connection Errors** + - Verify `REDIS_HOST`, `REDIS_PORT`, and `REDIS_PASSWORD` are set + - Ensure Redis service is running + - Check `REDIS_BROKER_URL` format: `redis://:password@host:port/0` + +5. **Static Files Not Loading** + - Verify `collectstatic` runs in build command + - Check `STATIC_URL` setting in Django settings + - Ensure static files are being served correctly + +6. **Worker Not Processing Tasks** + - Verify worker service is running + - Check that `REDIS_BROKER_URL` matches in both web and worker services + - Review worker logs for errors + +7. **Port Binding Issues** + - Render automatically sets `$PORT` environment variable + - Ensure start command uses `$PORT` (not hardcoded port) + - Check that uvicorn is configured correctly + +### Checking Logs + +- **Web Service**: Dashboard β†’ Your Web Service β†’ "Logs" tab +- **Worker**: Dashboard β†’ Your Worker β†’ "Logs" tab +- **Database**: Dashboard β†’ Your Database β†’ "Logs" tab +- **Redis**: Dashboard β†’ Your Redis β†’ "Logs" tab + +### Manual Commands via Shell + +To run commands manually: + +1. Go to your service in Render dashboard +2. Click on **"Shell"** tab +3. Run commands like: + ```bash + python manage.py migrate + python manage.py createsuperuser + python manage.py collectstatic + ``` + +## Build Command Breakdown + +The build command does the following: + +```bash +pip install uv && \ +uv sync --frozen --no-dev && \ +python manage.py collectstatic --noinput && \ +python manage.py migrate +``` + +1. `pip install uv` - Installs the uv package manager +2. `uv sync --frozen --no-dev` - Installs all production dependencies from `uv.lock` +3. `python manage.py collectstatic --noinput` - Collects static files for serving +4. `python manage.py migrate` - Runs database migrations + +## Performance Considerations + +- **Build Time**: Python buildpack builds are typically faster than Docker builds +- **Cold Starts**: First request may be slower (JIT compilation) +- **Memory**: Monitor memory usage; Python apps can be memory-intensive +- **Scaling**: Can scale horizontally by increasing instance count + +## Security Best Practices + +1. **Never commit secrets**: Use Render's environment variables +2. **Use strong SECRET_KEY**: Generate a long random string +3. **Set IS_DEBUG=0**: For production deployments +4. **Configure ALLOWED_HOSTS**: Set to your actual domain +5. **Remove BYPASS_AUTH_TOKEN_VERIFICATION**: Never use in production +6. **Use HTTPS**: Render provides this automatically +7. **Regular updates**: Keep dependencies updated + +## Scaling + +Render allows you to scale services: +- **Web Service**: Scale horizontally by increasing instance count +- **Worker**: Scale workers based on task volume +- **Database**: Upgrade plan for better performance +- **Redis**: Upgrade plan for larger cache/message queue + +## Next Steps + +After deployment: +1. Set up custom domain (optional) +2. Configure Flight Passport for OAuth (production) +3. Set up monitoring and alerts +4. Configure backups for PostgreSQL +5. Review and optimize performance + +## Additional Resources + +- [Render Python Documentation](https://render.com/docs/deploy-python) +- [Flight Blender API Documentation](http://redocly.github.io/redoc/?url=https://raw.githubusercontent.com/openutm/flight-blender/master/api/flight-blender-server-1.0.0-resolved.yaml) +- [Flight Blender Quickstart Guide](deployment_support/README.md) diff --git a/entrypoints/docker-entrypoint-web.sh b/entrypoints/docker-entrypoint-web.sh index 95eb35f..a99c518 100755 --- a/entrypoints/docker-entrypoint-web.sh +++ b/entrypoints/docker-entrypoint-web.sh @@ -6,58 +6,31 @@ echo "Waiting for services..." # Wait for Redis if [ -n "$REDIS_HOST" ]; then echo "Waiting for Redis at ${REDIS_HOST}:${REDIS_PORT:-6379}..." - until python -c " -import redis -import sys -try: - r = redis.Redis( - host='${REDIS_HOST}', - port=${REDIS_PORT:-6379}, - password='${REDIS_PASSWORD:-}' if '${REDIS_PASSWORD:-}' else None, - decode_responses=True, - socket_connect_timeout=5 - ) - r.ping() - print('Redis is ready!') -except Exception as e: - sys.exit(1) -" 2>/dev/null; do - echo "Waiting for Redis..." - sleep 2 - done + if ! uv run python entrypoints/wait_for_service.py --service $REDIS_HOST:${REDIS_PORT:-6379}; then + echo "Redis connection failed" + exit 1 + fi echo "Redis is ready!" fi # Wait for PostgreSQL if [ -n "$DATABASE_URL" ] || [ -n "$POSTGRES_HOST" ]; then echo "Waiting for PostgreSQL..." - until python -c " -import sys -try: - if '${DATABASE_URL}': - import psycopg2 - from urllib.parse import urlparse - conn = psycopg2.connect('${DATABASE_URL}') - conn.close() - print('PostgreSQL is ready!') - elif '${POSTGRES_HOST}': - import psycopg2 - conn = psycopg2.connect( - host='${POSTGRES_HOST}', - port=${POSTGRES_PORT:-5432}, - user='${POSTGRES_USER}', - password='${POSTGRES_PASSWORD}', - dbname='${POSTGRES_DB}', - connect_timeout=5 - ) - conn.close() - print('PostgreSQL is ready!') -except Exception as e: - sys.exit(1) -" 2>/dev/null; do - echo "Waiting for PostgreSQL..." - sleep 2 - done + POSTGRES_PORT=${POSTGRES_PORT:-5432} + if [ -n "$DATABASE_URL" ]; then + # Extract host and port from DATABASE_URL + DB_HOST=$(echo $DATABASE_URL | sed -E 's|.*@([^:]+):.*|\1|') + DB_PORT=$(echo $DATABASE_URL | sed -E 's|.*:([0-9]+)/.*|\1|') + if ! uv run python entrypoints/wait_for_service.py --service $DB_HOST:$DB_PORT; then + echo "PostgreSQL connection failed" + exit 1 + fi + elif [ -n "$POSTGRES_HOST" ]; then + if ! uv run python entrypoints/wait_for_service.py --service $POSTGRES_HOST:$POSTGRES_PORT; then + echo "PostgreSQL connection failed" + exit 1 + fi + fi echo "PostgreSQL is ready!" fi @@ -65,12 +38,12 @@ echo "All services are ready!" # Collect static files echo "Collecting static files..." -python manage.py collectstatic --noinput || echo "Warning: Static files collection failed, continuing..." +uv run python manage.py collectstatic --noinput || echo "Warning: Static files collection failed, continuing..." # Apply database migrations echo "Applying database migrations..." -python manage.py migrate || echo "Warning: Migrations failed, continuing..." +uv run python manage.py migrate || echo "Warning: Migrations failed, continuing..." # Start server echo "Starting server on port ${PORT:-8000}..." -exec uvicorn flight_blender.asgi:application --host 0.0.0.0 --port ${PORT:-8000} --workers 3 +exec uv run uvicorn flight_blender.asgi:application --host 0.0.0.0 --port ${PORT:-8000} --workers 3 diff --git a/entrypoints/docker-entrypoint-worker.sh b/entrypoints/docker-entrypoint-worker.sh index 2a5d0f2..34e407a 100755 --- a/entrypoints/docker-entrypoint-worker.sh +++ b/entrypoints/docker-entrypoint-worker.sh @@ -4,25 +4,10 @@ set -e # Wait for Redis if [ -n "$REDIS_HOST" ]; then echo "Waiting for Redis at ${REDIS_HOST}:${REDIS_PORT:-6379}..." - until python -c " -import redis -import sys -try: - r = redis.Redis( - host='${REDIS_HOST}', - port=${REDIS_PORT:-6379}, - password='${REDIS_PASSWORD:-}' if '${REDIS_PASSWORD:-}' else None, - decode_responses=True, - socket_connect_timeout=5 - ) - r.ping() - print('Redis is ready!') -except Exception as e: - sys.exit(1) -" 2>/dev/null; do - echo "Waiting for Redis..." - sleep 2 - done + if ! uv run python entrypoints/wait_for_service.py --service $REDIS_HOST:${REDIS_PORT:-6379}; then + echo "Redis connection failed" + exit 1 + fi echo "Redis is ready!" else echo "Warning: REDIS_HOST not set, skipping Redis check" @@ -30,4 +15,4 @@ fi # Start Celery worker echo "Starting Celery worker..." -exec celery --app=flight_blender worker --loglevel=info +exec uv run celery --app=flight_blender worker --loglevel=info diff --git a/render-no-docker.yaml b/render-no-docker.yaml new file mode 100644 index 0000000..d4417f2 --- /dev/null +++ b/render-no-docker.yaml @@ -0,0 +1,115 @@ +services: + - type: web + name: flight-blender-web + env: python + buildCommand: pip install uv && uv sync --frozen --no-dev && python manage.py collectstatic --noinput && python manage.py migrate + startCommand: uvicorn flight_blender.asgi:application --host 0.0.0.0 --port $PORT --workers 3 + envVars: + - key: SECRET_KEY + generateValue: true + - key: IS_DEBUG + value: 0 + - key: USE_LOCAL_SQLITE_DATABASE + value: 0 + - key: ALLOWED_HOSTS + fromService: + type: web + name: flight-blender-web + property: host + - key: DATABASE_URL + fromDatabase: + name: flight-blender-db + property: connectionString + - key: REDIS_HOST + fromService: + type: redis + name: flight-blender-redis + property: host + - key: REDIS_PORT + fromService: + type: redis + name: flight-blender-redis + property: port + - key: REDIS_PASSWORD + fromService: + type: redis + name: flight-blender-redis + property: password + - key: REDIS_BROKER_URL + fromService: + type: redis + name: flight-blender-redis + property: connectionString + - key: FLIGHTBLENDER_FQDN + fromService: + type: web + name: flight-blender-web + property: host + - key: HEARTBEAT_RATE_SECS + value: 2 + - key: USSP_NETWORK_ENABLED + value: 0 + - key: DSS_SELF_AUDIENCE + fromService: + type: web + name: flight-blender-web + property: host + - key: PYTHONPATH + value: /opt/render/project/src + + - type: worker + name: flight-blender-worker + env: python + buildCommand: pip install uv && uv sync --frozen --no-dev + startCommand: celery --app=flight_blender worker --loglevel=info + envVars: + - key: SECRET_KEY + fromService: + type: web + name: flight-blender-web + property: envVar + value: SECRET_KEY + - key: IS_DEBUG + value: 0 + - key: USE_LOCAL_SQLITE_DATABASE + value: 0 + - key: DATABASE_URL + fromDatabase: + name: flight-blender-db + property: connectionString + - key: REDIS_HOST + fromService: + type: redis + name: flight-blender-redis + property: host + - key: REDIS_PORT + fromService: + type: redis + name: flight-blender-redis + property: port + - key: REDIS_PASSWORD + fromService: + type: redis + name: flight-blender-redis + property: password + - key: REDIS_BROKER_URL + fromService: + type: redis + name: flight-blender-redis + property: connectionString + - key: HEARTBEAT_RATE_SECS + value: 2 + - key: USSP_NETWORK_ENABLED + value: 0 + - key: PYTHONPATH + value: /opt/render/project/src + + - type: redis + name: flight-blender-redis + plan: free + +databases: + - name: flight-blender-db + databaseName: flight_blender + user: flight_blender_user + plan: free diff --git a/runtime.txt b/runtime.txt new file mode 100644 index 0000000..e497a7f --- /dev/null +++ b/runtime.txt @@ -0,0 +1 @@ +python-3.12.12 From afba3868a99720fe96db17976485d1f7d78ee81b Mon Sep 17 00:00:00 2001 From: Peter Munachi Date: Tue, 13 Jan 2026 20:17:58 +0100 Subject: [PATCH 4/7] feat: prod --- RENDER_NON_DOCKER_SETUP.md | 4 +++- flight_blender/settings.py | 6 ++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/RENDER_NON_DOCKER_SETUP.md b/RENDER_NON_DOCKER_SETUP.md index b697a67..675efa7 100644 --- a/RENDER_NON_DOCKER_SETUP.md +++ b/RENDER_NON_DOCKER_SETUP.md @@ -206,15 +206,17 @@ For automated setup, you can use the `render-no-docker.yaml` file: - Verify `REDIS_HOST`, `REDIS_PORT`, and `REDIS_PASSWORD` are set - Ensure Redis service is running - Check `REDIS_BROKER_URL` format: `redis://:password@host:port/0` + - **SSL/rediss:// URLs**: Render's Redis uses SSL (`rediss://`). The settings automatically add `ssl_cert_reqs=CERT_NONE` to the URL for Celery compatibility 5. **Static Files Not Loading** - Verify `collectstatic` runs in build command - Check `STATIC_URL` setting in Django settings - Ensure static files are being served correctly -6. **Worker Not Processing Tasks** +6. **Worker Not Processing Tasks / Celery SSL Error** - Verify worker service is running - Check that `REDIS_BROKER_URL` matches in both web and worker services + - If you see `ssl_cert_reqs` error with `rediss://` URLs, ensure settings.py includes the SSL fix (automatically adds `ssl_cert_reqs=CERT_NONE`) - Review worker logs for errors 7. **Port Binding Issues** diff --git a/flight_blender/settings.py b/flight_blender/settings.py index ff577f5..6443486 100644 --- a/flight_blender/settings.py +++ b/flight_blender/settings.py @@ -158,6 +158,12 @@ else: BROKER_URL = os.getenv("REDIS_BROKER_URL", "redis://redis:6379/") +# Fix for Render.com Redis SSL: Celery requires ssl_cert_reqs parameter for rediss:// URLs +if BROKER_URL.startswith("rediss://") and "ssl_cert_reqs" not in BROKER_URL: + # Add ssl_cert_reqs parameter if not present + separator = "&" if "?" in BROKER_URL else "?" + BROKER_URL = f"{BROKER_URL}{separator}ssl_cert_reqs=CERT_NONE" + CHANNEL_LAYERS = { "default": { From f45f9fd9382bce066f3860266a6be9afc0129f4a Mon Sep 17 00:00:00 2001 From: Peter Munachi Date: Tue, 13 Jan 2026 21:05:58 +0100 Subject: [PATCH 5/7] feat: prod --- flight_blender/settings.py | 1 + 1 file changed, 1 insertion(+) diff --git a/flight_blender/settings.py b/flight_blender/settings.py index 6443486..e56886f 100644 --- a/flight_blender/settings.py +++ b/flight_blender/settings.py @@ -151,6 +151,7 @@ # https://docs.djangoproject.com/en/3.1/howto/static-files/ STATIC_URL = "/static/" +STATIC_ROOT = os.getenv("STATIC_ROOT", BASE_DIR / "staticfiles") DEFAULT_AUTO_FIELD = "django.db.models.AutoField" if DEBUG: From c342c7cf0cbfc283a230c0b4f439d8fefe10ce2f Mon Sep 17 00:00:00 2001 From: Peter Munachi Date: Tue, 20 Jan 2026 04:38:21 +0100 Subject: [PATCH 6/7] complete end to end setup docker --- Dockerfile | 16 +- PRODUCTION_DOCKER_SETUP.md | 287 --------------- README.md | 188 +++++++++- RENDER_DEPLOYMENT.md | 359 ------------------- RENDER_NON_DOCKER_SETUP.md | 301 ---------------- docker-compose-dev.yml | 28 +- entrypoints/with-database/entrypoint-beat.sh | 6 + entrypoints/with-database/entrypoint.sh | 6 + env.template | 41 +++ 9 files changed, 264 insertions(+), 968 deletions(-) delete mode 100644 PRODUCTION_DOCKER_SETUP.md delete mode 100644 RENDER_DEPLOYMENT.md delete mode 100644 RENDER_NON_DOCKER_SETUP.md create mode 100644 env.template diff --git a/Dockerfile b/Dockerfile index 9d8da23..ab0c6a8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,21 +12,29 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ postgresql-client \ && rm -rf /var/lib/apt/lists/* +# Create non-root user first +RUN addgroup --gid 10000 django && adduser --shell /bin/bash --disabled-password --gecos "" --uid 10000 --ingroup django django + # Copy dependency files COPY uv.lock pyproject.toml ./ -# Install Python dependencies +# Install Python dependencies (without the project itself) RUN pip install -U pip && pip install uv && uv sync --frozen --no-install-project --no-dev # Set PYTHONPATH to include the current directory so Django can find the modules ENV PYTHONPATH=/app -# Create non-root user -RUN addgroup --gid 10000 django && adduser --shell /bin/bash --disabled-password --gecos "" --uid 10000 --ingroup django django - # Copy application code COPY --chown=django:django . . +# Change ownership of entire /app directory (including .venv) to django user +RUN chown -R django:django /app + +# Switch to django user and install the project +USER django:django +RUN uv sync --frozen --no-dev +USER root + # Make entrypoint scripts executable RUN chmod +x entrypoints/docker-entrypoint-web.sh entrypoints/docker-entrypoint-worker.sh entrypoints/wait_for_service.py diff --git a/PRODUCTION_DOCKER_SETUP.md b/PRODUCTION_DOCKER_SETUP.md deleted file mode 100644 index e6569e3..0000000 --- a/PRODUCTION_DOCKER_SETUP.md +++ /dev/null @@ -1,287 +0,0 @@ -# Running Flight Blender in Production Mode Locally with Docker - -This guide will help you run Flight Blender in production mode locally using Docker. The production setup uses `docker-compose.yml` which is configured for production-like environments. - -## Prerequisites - -- **Docker** (version 20.10 or later) -- **Docker Compose** (version 2.0 or later) -- At least **4GB of RAM** available for Docker -- Ports **8000**, **5432**, and **6379** available on your system - -## Step-by-Step Instructions - -### 1. Create the External Docker Network - -The production Docker Compose setup requires an external network. Create it first: - -```bash -docker network create interop_ecosystem_network -``` - -If the network already exists, you'll see a message indicating that. This is fine - you can proceed. - -### 2. Create Environment File (.env) - -Create a `.env` file in the root directory of `flight_blender` with the following minimum required variables: - -```bash -# Django Settings -SECRET_KEY=your-very-long-random-secret-key-here-minimum-50-characters -IS_DEBUG=0 -ALLOWED_HOSTS=localhost,127.0.0.1 - -# Database Configuration -POSTGRES_USER=flightblender -POSTGRES_PASSWORD=your-secure-password-here -POSTGRES_DB=flightblender -POSTGRES_HOST=db-blender -DATABASE_URL=postgresql://flightblender:your-secure-password-here@db-blender:5432/flightblender - -# Redis Configuration -REDIS_HOST=redis-blender -REDIS_PORT=6379 -REDIS_PASSWORD=your-redis-password-here -REDIS_BROKER_URL=redis://:your-redis-password-here@redis-blender:6379/ - -# Optional: Standalone Mode (set to 0 for standalone, 1 for DSS integration) -USSP_NETWORK_ENABLED=0 - -# Optional: Heartbeat Rate -HEARTBEAT_RATE_SECS=2 - -# Optional: Flight Blender FQDN (for production) -FLIGHTBLENDER_FQDN=http://localhost:8000 -``` - -**⚠️ Important Security Notes:** -- **DO NOT** set `BYPASS_AUTH_TOKEN_VERIFICATION=1` in production mode -- Use strong, unique passwords for `POSTGRES_PASSWORD` and `REDIS_PASSWORD` -- Generate a secure `SECRET_KEY` (you can use: `python -c "from django.core.management.utils import get_random_secret_key; print(get_random_secret_key())"`) - -### 3. Build the Docker Image - -Build the production Docker image: - -```bash -cd /Users/petermunachiali/Documents/Github/SkyTrade/UTM/flight_blender -docker build . -t openutm/flight-blender -``` - -This will: -- Install system dependencies (gcc, postgresql-client) -- Install Python dependencies using `uv` -- Create a non-root user (django:django) -- Copy application code -- Set up entrypoint scripts - -**Note:** The build process may take several minutes on first run as it downloads dependencies. - -### 4. Start the Services - -Start all services using Docker Compose: - -```bash -docker compose up -d -``` - -Or to see logs in real-time: - -```bash -docker compose up -``` - -This will start the following services: -- **db-blender**: PostgreSQL 17 database -- **redis-blender**: Redis/Valkey cache and message broker -- **flight-blender**: Main Django application (port 8000) -- **flight-blender-celery**: Celery worker for background tasks - -### 5. Verify Services are Running - -Check that all containers are running: - -```bash -docker compose ps -``` - -You should see all four services with status "Up" or "Up (healthy)". - -### 6. Check Application Logs - -Monitor the application logs to ensure everything started correctly: - -```bash -# View all logs -docker compose logs -f - -# View logs for specific service -docker compose logs -f flight-blender -docker compose logs -f flight-blender-celery -``` - -Look for: -- Database migrations being applied successfully -- Server starting on port 8000 -- No error messages - -### 7. Access the Application - -Once the services are running, access Flight Blender at: - -- **Web Interface**: http://localhost:8000 -- **API Documentation**: http://localhost:8000/api/docs -- **Health Check**: http://localhost:8000/ping - -You should see the Flight Blender logo and links to the API documentation. - -### 8. Stop the Services - -When you're done, stop all services: - -```bash -docker compose down -``` - -To also remove volumes (this will delete database data): - -```bash -docker compose down -v -``` - -## Production vs Development Differences - -The production setup (`docker-compose.yml`) differs from development (`docker-compose-dev.yml`) in several ways: - -| Feature | Production | Development | -|---------|-----------|-------------| -| Network | External network required | Internal network | -| Volumes | No code volume mount | Code volume mounted | -| Image name | `openutm/flight-blender` | `openutm/flight-blender-dev` | -| Entrypoint | `no-database/entrypoint.sh` | `with-database/entrypoint.sh` | -| Celery Beat | Not included | Included | -| Database port | Not exposed | Exposed (5432) | - -## Troubleshooting - -### Issue: Network not found error - -**Error:** `network interop_ecosystem_network not found` - -**Solution:** -```bash -docker network create interop_ecosystem_network -``` - -### Issue: Port already in use - -**Error:** `Bind for 0.0.0.0:8000 failed: port is already allocated` - -**Solution:** -- Check what's using the port: `lsof -i :8000` (macOS/Linux) or `netstat -ano | findstr :8000` (Windows) -- Stop the conflicting service or change the port in `docker-compose.yml` - -### Issue: Database connection errors - -**Error:** `could not connect to server: Connection refused` - -**Solution:** -1. Verify database container is running: `docker compose ps` -2. Check database logs: `docker compose logs db-blender` -3. Ensure `.env` file has correct `POSTGRES_HOST=db-blender` -4. Wait a few seconds for database to fully initialize - -### Issue: Redis connection errors - -**Error:** `Error connecting to Redis` - -**Solution:** -1. Verify Redis container is running: `docker compose ps` -2. Check Redis logs: `docker compose logs redis-blender` -3. Ensure `REDIS_PASSWORD` in `.env` matches the password used in Redis command -4. Verify `REDIS_BROKER_URL` format: `redis://:password@host:port/` - -### Issue: Migration errors - -**Error:** `django.db.utils.OperationalError` - -**Solution:** -1. Ensure database container is fully started (wait 10-15 seconds) -2. Check database logs: `docker compose logs db-blender` -3. Try restarting: `docker compose restart flight-blender` - -### Issue: Permission errors - -**Error:** `Permission denied` when accessing files - -**Solution:** -- The Docker image runs as non-root user (django:django) -- Ensure entrypoint scripts are executable (handled in Dockerfile) -- Check file ownership if using volumes - -### Viewing Container Logs - -To debug issues, you can view logs for specific services: - -```bash -# All services -docker compose logs - -# Specific service -docker compose logs flight-blender -docker compose logs db-blender -docker compose logs redis-blender -docker compose logs flight-blender-celery - -# Follow logs in real-time -docker compose logs -f flight-blender - -# Last 100 lines -docker compose logs --tail=100 flight-blender -``` - -### Accessing Containers - -To access a running container for debugging: - -```bash -# Access flight-blender container -docker exec -it flight-blender bash - -# Access database container -docker exec -it db-blender psql -U flightblender -d flightblender - -# Access Redis container -docker exec -it redis-blender redis-cli -a your-redis-password -``` - -## Next Steps - -Once Flight Blender is running: - -1. **Test the API**: Import the [Postman Collection](api/flight_blender_api.postman_collection.json) -2. **Generate Access Tokens**: Use the [verification repository](https://github.com/openutm/verification) to generate tokens -3. **Explore API Documentation**: Visit http://localhost:8000/api/docs -4. **Submit Flight Data**: Use the API to submit flight declarations and other data - -## Additional Resources - -- [20-minute Quickstart Guide](deployment_support/README.md) -- [Render.com Deployment Guide](RENDER_DEPLOYMENT.md) -- [API Documentation](http://redocly.github.io/redoc/?url=https://raw.githubusercontent.com/openutm/flight-blender/master/api/flight-blender-server-1.0.0-resolved.yaml) -- [Flight Blender Verification](https://github.com/openutm/verification) - -## Clean Up - -To completely remove all containers, volumes, and networks: - -```bash -# Stop and remove containers -docker compose down -v - -# Remove the external network (if not used by other services) -docker network rm interop_ecosystem_network - -# Remove the Docker image -docker rmi openutm/flight-blender -``` diff --git a/README.md b/README.md index 3a9cc3b..b4fa50c 100644 --- a/README.md +++ b/README.md @@ -58,11 +58,193 @@ Follow our simple 5-step guide to deploy Flight Blender and explore its core fea --- -## πŸš€ Deploy to Render.com +## πŸš€ How to Run Flight Blender -Deploy Flight Blender to Render.com with our comprehensive deployment guide. +### Prerequisites -πŸ“– [Read the Render.com deployment guide](RENDER_DEPLOYMENT.md) for step-by-step instructions! +- **Docker** and **Docker Compose** installed on your system +- **Python 3.12+** (if running locally without Docker) +- **PostgreSQL** (handled by Docker Compose) +- **Redis/Valkey** (handled by Docker Compose) + +### Quick Start with Docker (Recommended) + +#### 1. Create Environment File + +Create a `.env` file in the root directory. You can use the sample from the [deployment guide](deployment_support/README.md) or create one with the following minimum required variables: + +```bash +# Django Settings +SECRET_KEY=your-secret-key-here +IS_DEBUG=1 +BYPASS_AUTH_TOKEN_VERIFICATION=1 +ALLOWED_HOSTS=* + +# Database Configuration +POSTGRES_USER=flightblender +POSTGRES_PASSWORD=your-password-here +POSTGRES_DB=flightblender +POSTGRES_HOST=db-blender +DATABASE_URL=postgresql://flightblender:your-password-here@db-blender:5432/flightblender + +# Redis Configuration +REDIS_HOST=redis-blender +REDIS_PORT=6379 +REDIS_PASSWORD=your-redis-password +REDIS_BROKER_URL=redis://:your-redis-password@redis-blender:6379/ + +# Optional: Standalone Mode (set to 0 for standalone, 1 for DSS integration) +USSP_NETWORK_ENABLED=0 + +# Optional: Heartbeat Rate +HEARTBEAT_RATE_SECS=2 +``` + +**⚠️ Security Note**: The `BYPASS_AUTH_TOKEN_VERIFICATION=1` setting is for local development only. Remove it for production deployments. + +#### 2. Build and Run with Docker Compose + +For **development** (using `docker-compose-dev.yml`): + +```bash +# Build the Docker image +docker build . -t openutm/flight-blender-dev + +# Start all services +docker compose -f docker-compose-dev.yml up +``` + +For **production-like** setup (using `docker-compose.yml`): + +**⚠️ Important Production Checklist:** + +1. **Update your `.env` file for production:** + - Remove or set `BYPASS_AUTH_TOKEN_VERIFICATION=0` (security risk if enabled) + - Set `IS_DEBUG=0` + - Set `ALLOWED_HOSTS` to your domain name (e.g., `ALLOWED_HOSTS=yourdomain.com,www.yourdomain.com`) + - Ensure `USE_LOCAL_SQLITE_DATABASE=0` (use PostgreSQL) + - Set strong passwords for `SECRET_KEY`, `POSTGRES_PASSWORD`, and `REDIS_PASSWORD` + +2. **Create the external network (if it doesn't exist):** +```bash +docker network create interop_ecosystem_network +``` + +3. **Build the production Docker image:** +```bash +docker build . -t openutm/flight-blender +``` + +4. **Start all services:** +```bash +docker compose up -d # -d runs in detached mode +``` + +**Platform Notes:** +- The production `docker-compose.yml` has `platform: linux/amd64` commented out for macOS compatibility +- For production on Linux servers, uncomment the `platform: linux/amd64` lines in `docker-compose.yml` +- For multi-platform builds: `docker buildx build --platform linux/amd64 -t openutm/flight-blender .` + +Alternatively, use the provided startup script: + +```bash +chmod +x start_flight_blender.sh +./start_flight_blender.sh +``` + +#### 3. Access the Application + +Once the containers are running, access Flight Blender at: + +- **Web Interface**: http://localhost:8000 +- **API Documentation**: http://localhost:8000/api/docs + +You should see the Flight Blender logo and links to the API documentation. + +#### 4. Verify Services + +The Docker Compose setup includes: +- **flight-blender**: Main Django application (port 8000) +- **db-blender**: PostgreSQL database (port 5432) +- **redis-blender**: Redis/Valkey cache and message broker (port 6379) +- **worker**: Celery worker for background tasks +- **flight-blender-beat**: Celery beat scheduler (in dev mode) + +### Running Locally (Without Docker) + +If you prefer to run without Docker: + +#### 1. Install Dependencies + +The project uses `uv` for dependency management: + +```bash +# Install uv if not already installed +pip install uv + +# Install project dependencies +uv sync +``` + +#### 2. Set Up Database + +Ensure PostgreSQL and Redis are running locally, then update your `.env` file: + +```bash +DATABASE_URL=postgresql://user:password@localhost:5432/flightblender +REDIS_HOST=localhost +REDIS_PORT=6379 +REDIS_BROKER_URL=redis://localhost:6379/ +``` + +#### 3. Run Database Migrations + +```bash +python manage.py migrate +``` + +#### 4. Start the Development Server + +```bash +python manage.py runserver +``` + +#### 5. Start Celery Worker (in separate terminal) + +```bash +celery -A flight_blender worker -l info +``` + +#### 6. Start Celery Beat (optional, in another terminal) + +```bash +celery -A flight_blender beat -l info +``` + +### Troubleshooting + +**Issue: Port conflicts** +- Ensure ports 8000, 5432, and 6379 are not in use +- Stop local PostgreSQL/Redis if running: `sudo systemctl stop postgresql` + +**Issue: Docker network errors** +- For `docker-compose.yml`, create the network: `docker network create interop_ecosystem_network` +- For `docker-compose-dev.yml`, the network is created automatically + +**Issue: Database connection errors** +- Verify PostgreSQL container is running: `docker ps` +- Check `.env` file has correct database credentials +- Ensure database migrations have run + +**Issue: Redis connection errors** +- Verify Redis container is running +- Check `REDIS_PASSWORD` matches in `.env` and `redis.conf` + +### Next Steps + +- Import the [Postman Collection](api/flight_blender_api.postman_collection.json) to test the API +- Generate access tokens using the [verification repository](https://github.com/openutm/verification) +- Explore the [API documentation](http://redocly.github.io/redoc/?url=https://raw.githubusercontent.com/openutm/flight-blender/master/api/flight-blender-server-1.0.0-resolved.yaml) --- ## πŸ’« Join the community diff --git a/RENDER_DEPLOYMENT.md b/RENDER_DEPLOYMENT.md deleted file mode 100644 index 0a8a92a..0000000 --- a/RENDER_DEPLOYMENT.md +++ /dev/null @@ -1,359 +0,0 @@ -# Deploying Flight Blender to Render.com (Docker) - -This guide will walk you through deploying Flight Blender to Render.com using Docker, including the web service, background worker (Celery), PostgreSQL database, and Redis instance. - -> **Note**: This guide uses Docker for deployment. The Dockerfile and entrypoint scripts are configured to automatically handle service dependencies, migrations, and static file collection. - -## Prerequisites - -- A GitHub account with your Flight Blender repository -- A Render.com account (free tier available) -- Basic understanding of environment variables and Docker - -## Overview - -Flight Blender requires the following services on Render: -1. **Web Service** - Main Django application -2. **Background Worker** - Celery worker for async tasks -3. **PostgreSQL Database** - For persistent data storage -4. **Redis** - For caching and Celery message broker - -## Step 1: Create PostgreSQL Database - -1. Go to your Render dashboard -2. Click **"New +"** β†’ **"PostgreSQL"** -3. Configure: - - **Name**: `flight-blender-db` (or your preferred name) - - **Database**: `flight_blender` (or your preferred name) - - **User**: Auto-generated (or custom) - - **Region**: Choose closest to your users - - **PostgreSQL Version**: 17 (or latest) - - **Plan**: Free tier available for testing -4. Click **"Create Database"** -5. **Important**: Note down the connection string from the dashboard (you'll need it later) - -## Step 2: Create Redis Instance - -1. Go to your Render dashboard -2. Click **"New +"** β†’ **"Redis"** -3. Configure: - - **Name**: `flight-blender-redis` (or your preferred name) - - **Region**: Same as PostgreSQL - - **Plan**: Free tier available for testing -4. Click **"Create Redis"** -5. **Important**: Note down the connection details (host, port, password) - -## Step 3: Deploy Web Service (Docker) - -1. Go to your Render dashboard -2. Click **"New +"** β†’ **"Web Service"** -3. Connect your GitHub repository: - - Select your Flight Blender repository - - Choose the branch (usually `main` or `master`) -4. Configure the service: - - **Name**: `flight-blender-web` (or your preferred name) - - **Environment**: `Docker` - - **Region**: Same as your database - - **Branch**: `main` (or your default branch) - - **Dockerfile Path**: `Dockerfile` (or leave empty if Dockerfile is in root) - - **Docker Context**: Leave empty (or set if Dockerfile is in subdirectory) - - **Docker Command**: Leave empty (uses CMD from Dockerfile) - - **Plan**: Choose based on your needs (free tier available) - - **Note**: The Dockerfile is configured to use the web entrypoint by default. The entrypoint script will: - - Wait for Redis and PostgreSQL to be ready - - Collect static files - - Run database migrations - - Start the uvicorn server - -### Environment Variables for Web Service - -Add these environment variables in the Render dashboard under "Environment": - -#### Required Variables - -```bash -# Django Settings -SECRET_KEY=your-secret-key-here-generate-a-long-random-string -IS_DEBUG=0 -ALLOWED_HOSTS=your-app-name.onrender.com,localhost -USE_LOCAL_SQLITE_DATABASE=0 - -# Database (use the connection string from Step 1) -DATABASE_URL=postgresql://user:password@hostname:5432/database_name - -# Redis Configuration (use details from Step 2) -REDIS_HOST=your-redis-host.onrender.com -REDIS_PORT=6379 -REDIS_PASSWORD=your-redis-password -REDIS_BROKER_URL=redis://:password@your-redis-host.onrender.com:6379/0 - -# Application Settings -FLIGHTBLENDER_FQDN=https://your-app-name.onrender.com -HEARTBEAT_RATE_SECS=2 - -# Network Mode (set to 0 for standalone, 1 for DSS integration) -USSP_NETWORK_ENABLED=0 -DSS_SELF_AUDIENCE=your-app-name.onrender.com -``` - -#### Optional Variables (for DSS integration) - -```bash -# Only needed if USSP_NETWORK_ENABLED=1 -AUTH_DSS_CLIENT_ID=your-client-id -AUTH_DSS_CLIENT_SECRET=your-client-secret -DSS_BASE_URL=https://your-dss-url.com -``` - -#### Security Note - -**IMPORTANT**: For production, do NOT set `BYPASS_AUTH_TOKEN_VERIFICATION=1`. This should only be used for local development. - -## Step 4: Deploy Background Worker (Celery) - Docker - -1. Go to your Render dashboard -2. Click **"New +"** β†’ **"Background Worker"** -3. Connect the same GitHub repository -4. Configure: - - **Name**: `flight-blender-worker` (or your preferred name) - - **Environment**: `Docker` - - **Region**: Same as web service - - **Branch**: Same as web service - - **Dockerfile Path**: `Dockerfile` (or leave empty if Dockerfile is in root) - - **Docker Context**: Leave empty - - **Docker Command**: `./entrypoints/docker-entrypoint-worker.sh` - - **Plan**: Choose based on your needs - - **Note**: The worker uses the same Dockerfile but with a different entrypoint command. The entrypoint script will: - - Wait for Redis to be ready - - Start the Celery worker - -### Environment Variables for Worker - -Add the same environment variables as the web service (except `ALLOWED_HOSTS` which is web-only): - -- `SECRET_KEY` -- `DATABASE_URL` -- `REDIS_HOST` -- `REDIS_PORT` -- `REDIS_PASSWORD` -- `REDIS_BROKER_URL` -- All other variables from the web service - -## Step 5: Database Migrations - -**Good News**: Database migrations run automatically when the web service starts! The Docker entrypoint script (`docker-entrypoint-web.sh`) includes: -- Automatic migration execution on startup -- Static file collection - -If you need to run migrations manually or create a superuser: - -1. Go to your web service in Render dashboard -2. Click on **"Shell"** tab -3. Run: - ```bash - python manage.py migrate - ``` -4. (Optional) Create a superuser: - ```bash - python manage.py createsuperuser - ``` - -## Step 6: Verify Deployment - -1. Visit your web service URL: `https://your-app-name.onrender.com` -2. You should see the Flight Blender logo and API documentation links -3. Test the ping endpoint: `https://your-app-name.onrender.com/ping` -4. Check logs in Render dashboard to ensure no errors - -## Using render.yaml (Alternative Method) - -For a more automated setup, you can use a `render.yaml` file. The file is already included in the repository root and configured for Docker deployment: - -```yaml -services: - - type: web - name: flight-blender-web - env: docker - dockerfilePath: Dockerfile - dockerContext: . - envVars: - - key: SECRET_KEY - generateValue: true - - key: IS_DEBUG - value: 0 - - key: USE_LOCAL_SQLITE_DATABASE - value: 0 - - key: ALLOWED_HOSTS - fromService: - type: web - name: flight-blender-web - property: host - - key: DATABASE_URL - fromDatabase: - name: flight-blender-db - property: connectionString - - key: REDIS_HOST - fromService: - type: redis - name: flight-blender-redis - property: host - - key: REDIS_PORT - fromService: - type: redis - name: flight-blender-redis - property: port - - key: REDIS_PASSWORD - fromService: - type: redis - name: flight-blender-redis - property: password - - key: REDIS_BROKER_URL - fromService: - type: redis - name: flight-blender-redis - property: connectionString - - key: FLIGHTBLENDER_FQDN - fromService: - type: web - name: flight-blender-web - property: host - - key: HEARTBEAT_RATE_SECS - value: 2 - - key: USSP_NETWORK_ENABLED - value: 0 - - - type: worker - name: flight-blender-worker - env: docker - dockerfilePath: Dockerfile - dockerContext: . - dockerCommand: ./entrypoints/docker-entrypoint-worker.sh - envVars: - - key: SECRET_KEY - fromService: - type: web - name: flight-blender-web - property: envVar - value: SECRET_KEY - - key: DATABASE_URL - fromDatabase: - name: flight-blender-db - property: connectionString - - key: REDIS_HOST - fromService: - type: redis - name: flight-blender-redis - property: host - - key: REDIS_PORT - fromService: - type: redis - name: flight-blender-redis - property: port - - key: REDIS_PASSWORD - fromService: - type: redis - name: flight-blender-redis - property: password - - key: REDIS_BROKER_URL - fromService: - type: redis - name: flight-blender-redis - property: connectionString - -databases: - - name: flight-blender-db - databaseName: flight_blender - user: flight_blender_user - plan: free - -services: - - type: redis - name: flight-blender-redis - plan: free -``` - -Then deploy via: -1. Go to Render dashboard -2. Click **"New +"** β†’ **"Blueprint"** -3. Connect your repository -4. Render will automatically detect and use `render.yaml` - -## Troubleshooting - -### Common Issues - -1. **Database Connection Errors** - - Verify `DATABASE_URL` is correctly set - - Ensure PostgreSQL service is running - - Check that database name, user, and password are correct - -2. **Redis Connection Errors** - - Verify `REDIS_HOST`, `REDIS_PORT`, and `REDIS_PASSWORD` are set - - Ensure Redis service is running - - Check `REDIS_BROKER_URL` format: `redis://:password@host:port/0` - -3. **Static Files Not Loading** - - Static files are automatically collected by the Docker entrypoint script - - Check logs to see if `collectstatic` ran successfully - - Check `STATIC_URL` setting in `settings.py` - - For Docker: Ensure the entrypoint script has proper permissions - -4. **Worker Not Processing Tasks** - - Verify worker service is running - - Check that `REDIS_BROKER_URL` matches in both web and worker services - - Review worker logs for errors - -5. **Application Crashes on Startup** - - Check logs in Render dashboard - - Verify all required environment variables are set - - Ensure migrations have run (they run automatically in Docker) - - For Docker: Check that entrypoint scripts are executable (`chmod +x entrypoints/docker-entrypoint-*.sh`) - - Verify Docker image builds successfully - -6. **Docker-Specific Issues** - - **Build fails**: Check Dockerfile syntax and ensure all dependencies are listed in `pyproject.toml` - - **Entrypoint script not found**: Ensure scripts are in `entrypoints/` directory and are executable - - **Port binding errors**: Render automatically sets `$PORT` environment variable - ensure your app uses it - - **Service wait timeouts**: Entrypoint scripts wait for Redis/PostgreSQL with 5-second timeouts - increase if services are slow to start - -### Checking Logs - -- **Web Service**: Dashboard β†’ Your Web Service β†’ "Logs" tab -- **Worker**: Dashboard β†’ Your Worker β†’ "Logs" tab -- **Database**: Dashboard β†’ Your Database β†’ "Logs" tab -- **Redis**: Dashboard β†’ Your Redis β†’ "Logs" tab - -## Security Best Practices - -1. **Never commit secrets**: Use Render's environment variables -2. **Use strong SECRET_KEY**: Generate a long random string -3. **Set IS_DEBUG=0**: For production deployments -4. **Configure ALLOWED_HOSTS**: Set to your actual domain -5. **Remove BYPASS_AUTH_TOKEN_VERIFICATION**: Never use in production -6. **Use HTTPS**: Render provides this automatically -7. **Regular updates**: Keep dependencies updated - -## Scaling - -Render allows you to scale services: -- **Web Service**: Scale horizontally by increasing instance count -- **Worker**: Scale workers based on task volume -- **Database**: Upgrade plan for better performance -- **Redis**: Upgrade plan for larger cache/message queue - -## Next Steps - -After deployment: -1. Set up custom domain (optional) -2. Configure Flight Passport for OAuth (production) -3. Set up monitoring and alerts -4. Configure backups for PostgreSQL -5. Review and optimize performance - -## Additional Resources - -- [Render Documentation](https://render.com/docs) -- [Flight Blender API Documentation](http://redocly.github.io/redoc/?url=https://raw.githubusercontent.com/openutm/flight-blender/master/api/flight-blender-server-1.0.0-resolved.yaml) -- [Flight Blender Quickstart Guide](deployment_support/README.md) diff --git a/RENDER_NON_DOCKER_SETUP.md b/RENDER_NON_DOCKER_SETUP.md deleted file mode 100644 index 675efa7..0000000 --- a/RENDER_NON_DOCKER_SETUP.md +++ /dev/null @@ -1,301 +0,0 @@ -# Deploying Flight Blender to Render.com (Without Docker) - -This guide will walk you through deploying Flight Blender to Render.com using Python buildpacks instead of Docker. This is useful if you prefer native Python deployment or want to avoid Docker overhead. - -## Prerequisites - -- A GitHub account with your Flight Blender repository -- A Render.com account (free tier available) -- Basic understanding of environment variables - -## Overview - -Flight Blender requires the following services on Render: -1. **Web Service** - Main Django application (Python) -2. **Background Worker** - Celery worker for async tasks (Python) -3. **PostgreSQL Database** - For persistent data storage -4. **Redis** - For caching and Celery message broker - -## Step 1: Create PostgreSQL Database - -1. Go to your Render dashboard -2. Click **"New +"** β†’ **"PostgreSQL"** -3. Configure: - - **Name**: `flight-blender-db` (or your preferred name) - - **Database**: `flight_blender` (or your preferred name) - - **User**: Auto-generated (or custom) - - **Region**: Choose closest to your users - - **PostgreSQL Version**: 17 (or latest) - - **Plan**: Free tier available for testing -4. Click **"Create Database"** -5. **Important**: Note down the connection string from the dashboard - -## Step 2: Create Redis Instance - -1. Go to your Render dashboard -2. Click **"New +"** β†’ **"Redis"** -3. Configure: - - **Name**: `flight-blender-redis` (or your preferred name) - - **Region**: Same as PostgreSQL - - **Plan**: Free tier available for testing -4. Click **"Create Redis"** -5. **Important**: Note down the connection details (host, port, password) - -## Step 3: Deploy Web Service (Python Buildpack) - -1. Go to your Render dashboard -2. Click **"New +"** β†’ **"Web Service"** -3. Connect your GitHub repository: - - Select your Flight Blender repository - - Choose the branch (usually `main` or `master`) -4. Configure the service: - - **Name**: `flight-blender-web` (or your preferred name) - - **Environment**: `Python 3` (NOT Docker) - - **Region**: Same as your database - - **Branch**: `main` (or your default branch) - - **Root Directory**: Leave empty (or set if app is in subdirectory) - - **Python Version**: 3.12 (will use `runtime.txt`) - - **Build Command**: - ```bash - pip install uv && uv sync --frozen --no-dev && python manage.py collectstatic --noinput && python manage.py migrate - ``` - - **Start Command**: - ```bash - uvicorn flight_blender.asgi:application --host 0.0.0.0 --port $PORT --workers 3 - ``` - - **Plan**: Choose based on your needs (free tier available) - -### Environment Variables for Web Service - -Add these environment variables in the Render dashboard under "Environment": - -#### Required Variables - -```bash -# Django Settings -SECRET_KEY=your-secret-key-here-generate-a-long-random-string -IS_DEBUG=0 -ALLOWED_HOSTS=your-app-name.onrender.com,localhost -USE_LOCAL_SQLITE_DATABASE=0 - -# Database (use the connection string from Step 1) -DATABASE_URL=postgresql://user:password@hostname:5432/database_name - -# Redis Configuration (use details from Step 2) -REDIS_HOST=your-redis-host.onrender.com -REDIS_PORT=6379 -REDIS_PASSWORD=your-redis-password -REDIS_BROKER_URL=redis://:password@your-redis-host.onrender.com:6379/0 - -# Application Settings -FLIGHTBLENDER_FQDN=https://your-app-name.onrender.com -HEARTBEAT_RATE_SECS=2 - -# Network Mode (set to 0 for standalone, 1 for DSS integration) -USSP_NETWORK_ENABLED=0 -DSS_SELF_AUDIENCE=your-app-name.onrender.com - -# Python Path (important for Python buildpack) -PYTHONPATH=/opt/render/project/src -``` - -#### Optional Variables (for DSS integration) - -```bash -# Only needed if USSP_NETWORK_ENABLED=1 -AUTH_DSS_CLIENT_ID=your-client-id -AUTH_DSS_CLIENT_SECRET=your-client-secret -DSS_BASE_URL=https://your-dss-url.com -``` - -#### Security Note - -**IMPORTANT**: For production, do NOT set `BYPASS_AUTH_TOKEN_VERIFICATION=1`. This should only be used for local development. - -## Step 4: Deploy Background Worker (Celery) - Python Buildpack - -1. Go to your Render dashboard -2. Click **"New +"** β†’ **"Background Worker"** -3. Connect the same GitHub repository -4. Configure: - - **Name**: `flight-blender-worker` (or your preferred name) - - **Environment**: `Python 3` (NOT Docker) - - **Region**: Same as web service - - **Branch**: Same as web service - - **Root Directory**: Leave empty - - **Python Version**: 3.12 - - **Build Command**: - ```bash - pip install uv && uv sync --frozen --no-dev - ``` - - **Start Command**: - ```bash - celery --app=flight_blender worker --loglevel=info - ``` - - **Plan**: Choose based on your needs - -### Environment Variables for Worker - -Add the same environment variables as the web service (except `ALLOWED_HOSTS` which is web-only): - -- `SECRET_KEY` -- `DATABASE_URL` -- `REDIS_HOST` -- `REDIS_PORT` -- `REDIS_PASSWORD` -- `REDIS_BROKER_URL` -- `PYTHONPATH` (set to `/opt/render/project/src`) -- All other variables from the web service - -## Step 5: Using render-no-docker.yaml (Alternative Method) - -For automated setup, you can use the `render-no-docker.yaml` file: - -1. Go to Render dashboard -2. Click **"New +"** β†’ **"Blueprint"** -3. Connect your repository -4. Render will automatically detect and use `render-no-docker.yaml` -5. All services will be created automatically with correct configuration - -**Note**: Make sure to use `render-no-docker.yaml` (not `render.yaml`) for non-Docker deployment. - -## Step 6: Verify Deployment - -1. Visit your web service URL: `https://your-app-name.onrender.com` -2. You should see the Flight Blender logo and API documentation links -3. Test the ping endpoint: `https://your-app-name.onrender.com/ping` -4. Check logs in Render dashboard to ensure no errors - -## Key Differences from Docker Deployment - -### Build Process -- **Docker**: Uses Dockerfile to build container image -- **Python Buildpack**: Uses `buildCommand` to install dependencies and prepare app - -### Start Command -- **Docker**: Uses CMD or entrypoint script from Dockerfile -- **Python Buildpack**: Uses explicit `startCommand` in Render config - -### Dependencies -- **Docker**: Installed during Docker build (in Dockerfile) -- **Python Buildpack**: Installed via `buildCommand` on each deploy - -### Migrations -- **Docker**: Run in entrypoint script on container start -- **Python Buildpack**: Run in `buildCommand` during build (recommended) or manually via Shell - -## Troubleshooting - -### Common Issues - -1. **Build Fails - uv not found** - - Ensure build command includes `pip install uv` first - - Check that `uv` is available in the Python environment - -2. **Module Not Found Errors** - - Verify `PYTHONPATH=/opt/render/project/src` is set - - Check that all dependencies are in `pyproject.toml` - - Ensure `uv sync` completes successfully - -3. **Database Connection Errors** - - Verify `DATABASE_URL` is correctly set - - Ensure PostgreSQL service is running - - Check that database name, user, and password are correct - -4. **Redis Connection Errors** - - Verify `REDIS_HOST`, `REDIS_PORT`, and `REDIS_PASSWORD` are set - - Ensure Redis service is running - - Check `REDIS_BROKER_URL` format: `redis://:password@host:port/0` - - **SSL/rediss:// URLs**: Render's Redis uses SSL (`rediss://`). The settings automatically add `ssl_cert_reqs=CERT_NONE` to the URL for Celery compatibility - -5. **Static Files Not Loading** - - Verify `collectstatic` runs in build command - - Check `STATIC_URL` setting in Django settings - - Ensure static files are being served correctly - -6. **Worker Not Processing Tasks / Celery SSL Error** - - Verify worker service is running - - Check that `REDIS_BROKER_URL` matches in both web and worker services - - If you see `ssl_cert_reqs` error with `rediss://` URLs, ensure settings.py includes the SSL fix (automatically adds `ssl_cert_reqs=CERT_NONE`) - - Review worker logs for errors - -7. **Port Binding Issues** - - Render automatically sets `$PORT` environment variable - - Ensure start command uses `$PORT` (not hardcoded port) - - Check that uvicorn is configured correctly - -### Checking Logs - -- **Web Service**: Dashboard β†’ Your Web Service β†’ "Logs" tab -- **Worker**: Dashboard β†’ Your Worker β†’ "Logs" tab -- **Database**: Dashboard β†’ Your Database β†’ "Logs" tab -- **Redis**: Dashboard β†’ Your Redis β†’ "Logs" tab - -### Manual Commands via Shell - -To run commands manually: - -1. Go to your service in Render dashboard -2. Click on **"Shell"** tab -3. Run commands like: - ```bash - python manage.py migrate - python manage.py createsuperuser - python manage.py collectstatic - ``` - -## Build Command Breakdown - -The build command does the following: - -```bash -pip install uv && \ -uv sync --frozen --no-dev && \ -python manage.py collectstatic --noinput && \ -python manage.py migrate -``` - -1. `pip install uv` - Installs the uv package manager -2. `uv sync --frozen --no-dev` - Installs all production dependencies from `uv.lock` -3. `python manage.py collectstatic --noinput` - Collects static files for serving -4. `python manage.py migrate` - Runs database migrations - -## Performance Considerations - -- **Build Time**: Python buildpack builds are typically faster than Docker builds -- **Cold Starts**: First request may be slower (JIT compilation) -- **Memory**: Monitor memory usage; Python apps can be memory-intensive -- **Scaling**: Can scale horizontally by increasing instance count - -## Security Best Practices - -1. **Never commit secrets**: Use Render's environment variables -2. **Use strong SECRET_KEY**: Generate a long random string -3. **Set IS_DEBUG=0**: For production deployments -4. **Configure ALLOWED_HOSTS**: Set to your actual domain -5. **Remove BYPASS_AUTH_TOKEN_VERIFICATION**: Never use in production -6. **Use HTTPS**: Render provides this automatically -7. **Regular updates**: Keep dependencies updated - -## Scaling - -Render allows you to scale services: -- **Web Service**: Scale horizontally by increasing instance count -- **Worker**: Scale workers based on task volume -- **Database**: Upgrade plan for better performance -- **Redis**: Upgrade plan for larger cache/message queue - -## Next Steps - -After deployment: -1. Set up custom domain (optional) -2. Configure Flight Passport for OAuth (production) -3. Set up monitoring and alerts -4. Configure backups for PostgreSQL -5. Review and optimize performance - -## Additional Resources - -- [Render Python Documentation](https://render.com/docs/deploy-python) -- [Flight Blender API Documentation](http://redocly.github.io/redoc/?url=https://raw.githubusercontent.com/openutm/flight-blender/master/api/flight-blender-server-1.0.0-resolved.yaml) -- [Flight Blender Quickstart Guide](deployment_support/README.md) diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 21a3453..8f92586 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -11,14 +11,14 @@ services: container_name: "redis-blender" env_file: - ".env" - # networks: - # - interop_ecosystem_network + networks: + - interop_ecosystem_network db-blender: platform: linux/amd64 container_name: "db-blender" image: postgres:17 ports: - - "5432:5432" + - "5433:5432" # Host port 5433, container port 5432 expose: - "5432" restart: unless-stopped @@ -26,8 +26,8 @@ services: - db_data:/var/lib/postgresql/data/ env_file: - ".env" - # networks: - # - interop_ecosystem_network + networks: + - interop_ecosystem_network flight-blender: platform: linux/amd64 @@ -48,8 +48,8 @@ services: - db-blender # volumes: # - .:/app - # networks: - # - interop_ecosystem_network + networks: + - interop_ecosystem_network flight-blender-celery: @@ -66,8 +66,8 @@ services: # - .:/app depends_on: - redis-blender - # networks: - # - interop_ecosystem_network + networks: + - interop_ecosystem_network flight-blender-celery-beat: platform: linux/amd64 @@ -90,13 +90,13 @@ services: condition: service_started flight-blender-celery: condition: service_started - # networks: - # - interop_ecosystem_network + networks: + - interop_ecosystem_network volumes: app: db_data: -# networks: -# interop_ecosystem_network: -# external: true +networks: + interop_ecosystem_network: + external: true diff --git a/entrypoints/with-database/entrypoint-beat.sh b/entrypoints/with-database/entrypoint-beat.sh index 154cb1f..730899d 100755 --- a/entrypoints/with-database/entrypoint-beat.sh +++ b/entrypoints/with-database/entrypoint-beat.sh @@ -3,6 +3,12 @@ source .venv/bin/activate echo Waiting for DBs... +# Use port 5432 for internal Docker network connections (db-blender), 5433 for host connections +if [ "$POSTGRES_HOST" = "db-blender" ]; then + POSTGRES_PORT=5432 +else + POSTGRES_PORT=${POSTGRES_PORT:-5433} +fi if ! wait-for-it --parallel --service $REDIS_HOST:$REDIS_PORT --service $POSTGRES_HOST:$POSTGRES_PORT; then exit fi diff --git a/entrypoints/with-database/entrypoint.sh b/entrypoints/with-database/entrypoint.sh index ef1b4d7..750836f 100755 --- a/entrypoints/with-database/entrypoint.sh +++ b/entrypoints/with-database/entrypoint.sh @@ -3,6 +3,12 @@ source .venv/bin/activate echo Waiting for DBs... +# Use port 5432 for internal Docker network connections (db-blender), 5433 for host connections +if [ "$POSTGRES_HOST" = "db-blender" ]; then + POSTGRES_PORT=5432 +else + POSTGRES_PORT=${POSTGRES_PORT:-5433} +fi if ! wait-for-it --parallel --service $REDIS_HOST:$REDIS_PORT --service $POSTGRES_HOST:$POSTGRES_PORT; then exit fi diff --git a/env.template b/env.template new file mode 100644 index 0000000..6c12a02 --- /dev/null +++ b/env.template @@ -0,0 +1,41 @@ +SECRET_KEY=XhOCvsdPpQPL2Q74hvDNZw3L4QatShA5CBPdEKfzgc1G3tuwKNVsgSN9MKHrYKL5 + +PASSPORT_AUDIENCE=testflight.flightblender.com +PASSPORT_URL=http://flight-passport:9000 +PASSPORT_JWKS_URL=http://flight-passport:9000/.well-known/jwks.json +DSS_AUTH_JWKS_ENDPOINT=http://flight-passport:9000/.well-known/jwks.json + +# PASSPORT_JWKS_URL=http://localhost:9000/.well-known/jwks.json +# PASSPORT_URL=http://localhost:9000 + +IS_DEBUG=0 + +BYPASS_AUTH_TOKEN_VERIFICATION=0 + +ALLOWED_HOSTS=localhost,127.0.0.1,host.docker.internal,flight-blender + +DISABLE_JSON_LOGGING=1 +ENABLE_CONFORMANCE_MONITORING=1 +USSP_NETWORK_ENABLED=0 + +REDIS_HOST=redis-blender +REDIS_PORT=6379 +REDIS_PASSWORD=blender_redis +REDIS_BROKER_URL=redis://:blender_redis@redis-blender:6379 +HEARTBEAT_RATE_SECS=2 + +FLIGHT_SPOTLIGHT_URL=http://flight-spotlight:5000 + +FLIGHTBLENDER_FQDN=http://host.docker.internal:8000 + + +USE_LOCAL_SQLITE_DATABASE=0 +DATABASE_URL=postgresql://mydatabaseuser:mypassword@db-blender:5433/mydatabase + +# Postgres Docker +POSTGRES_USER=mydatabaseuser +POSTGRES_PASSWORD=mypassword +POSTGRES_DB=mydatabase +POSTGRES_HOST=db-blender +PGDATA=/var/lib/postgresql/data/pgdata +POSTGRES_PORT=5433 \ No newline at end of file From e8819d2cd2800e38b437dc1c95d23f9ab461f7cc Mon Sep 17 00:00:00 2001 From: Peter Munachi Date: Wed, 4 Mar 2026 14:39:00 +0100 Subject: [PATCH 7/7] feat: add comprehensive features documentation and enhance deployment instructions --- FEATURES.md | 44 +++++++++++++++++++ README.md | 23 ++++++++++ api/flight-blender-server-1.0.0-resolved.yaml | 2 + auth_helper/dss_auth_helper.py | 3 +- common/data_definitions.py | 1 + .../conformance_checks_handler.py | 2 + constraint_operations/constraints_helper.py | 2 +- .../dss_constraints_helper.py | 2 +- docker-compose-dev.yml | 6 ++- docker-compose.yml | 4 +- entrypoints/no-database/entrypoint.sh | 8 +++- entrypoints/with-database/entrypoint-beat.sh | 8 +--- entrypoints/with-database/entrypoint.sh | 12 ++--- env.template | 16 ++++++- flight_blender/settings.py | 9 +++- flight_declaration_operations/serializers.py | 2 +- flight_declaration_operations/views.py | 12 +++-- pyproject.toml | 1 + rid_operations/dss_rid_helper.py | 2 +- rid_operations/views.py | 6 ++- scd_operations/dss_scd_helper.py | 4 +- scd_operations/opint_helper.py | 3 ++ uv.lock | 11 +++++ 23 files changed, 151 insertions(+), 32 deletions(-) create mode 100644 FEATURES.md diff --git a/FEATURES.md b/FEATURES.md new file mode 100644 index 0000000..b845ac6 --- /dev/null +++ b/FEATURES.md @@ -0,0 +1,44 @@ +Flight Blender is an open-source backend and data-processing engine designed to support standards-compliant UTM (Unmanned Traffic Management) services. It adheres to the latest regulations for UTM/U-Space in the EU and other jurisdictions. With Flight Blender, you can: + +Implement a Remote ID β€œservice provider” compatible with the ASTM-F3411 Remote ID standard, along with Flight Spotlight, an open-source Remote ID Display Application. +Use an open-source implementation of the ASTM F3548 USS-to-USS standard, compatible with EU U-Space regulations for flight authorization. +Interact with interoperability software like interuss/dss to exchange data with other UTM systems. +Process geo-fences using the ED-269 standard. +Monitor conformance and send operator notifications. +Aggregate flight traffic feeds from various sources, including geo-fences, flight declarations, and air-traffic data. +Configure Blender to act as a Surveillance SDSP per the ASTM F3623-23 standard. +Implement alerts / near misses per the ASTM F3442 standard + + +Key Features + +DSS Connectivity +Connect and retrieve data such as Remote ID information or perform strategic de-confliction and flight authorization. + +Flight Tracking +Ingest flight tracking feeds from sources like ADS-B, live telemetry, and Broadcast Remote ID. Outputs a unified JSON feed for real-time display. + +Geofence Management +Submit geofence to Flight Blender, which can then be transmitted to Spotlight for visualization. + +Flight Declaration +Submit future flight plans (up to 24 hours in advance) using the ASTM USS-to-USS API or as a standalone component. Supported DSS APIs are listed below. + +Network Remote ID +Compliant with ASTM standards, this module can act as a β€œdisplay provider” or β€œservice provider” for Network Remote ID. + +Operator Notifications +Send notifications to operators using an AMQP queue, enabling real-time alerts for flight updates, conformance issues, or other critical events. + +Conformance Monitoring +Monitor flight paths against declared 4D volumes for conformance and report outputs. + +Surveillance SDSP +Blender conforms to the requirements for Surveillance supplemental data service providers (SDSPs) and associated equipment and services. + +Detect, Alert and Avoid +Flight Blender implements the Detect Alert and Avoid standard F3442 + + +Can you explore these repos I have attached and create a detailed spec for each of this features on how they can be implemented in the spotlight, the API endpoints in the Blender, that is responsible for these features. I am new to openutm, so make this as much detailed with diagram as you can + diff --git a/README.md b/README.md index b4fa50c..8db685a 100644 --- a/README.md +++ b/README.md @@ -221,6 +221,29 @@ celery -A flight_blender worker -l info celery -A flight_blender beat -l info ``` +### Deploy on Render (Standalone) + +Flight Blender includes Render blueprints for standalone deployments. + +#### Option A: Docker (recommended for parity) +1. In Render, create a **New Blueprint** and point it to this repo. +2. Select `render.yaml`. +3. Render will provision: + - Web service (`flight-blender-web`) + - Worker (`flight-blender-worker`) + - Redis + - Postgres +4. Set/override required env vars (see `env.template`), especially: + - `PASSPORT_URL`, `PASSPORT_AUDIENCE`, `BYPASS_AUTH_TOKEN_VERIFICATION=0` + - `ALLOWED_HOSTS` to your Render host +5. Deploy. The web service will be reachable at your Render URL. + +#### Option B: Non-Docker Python +1. In Render, create a **New Blueprint** and point it to this repo. +2. Select `render-no-docker.yaml`. +3. Render will provision the same services using Python build/start commands. +4. Set/override required env vars as above. + ### Troubleshooting **Issue: Port conflicts** diff --git a/api/flight-blender-server-1.0.0-resolved.yaml b/api/flight-blender-server-1.0.0-resolved.yaml index 03d334b..ed2d78b 100644 --- a/api/flight-blender-server-1.0.0-resolved.yaml +++ b/api/flight-blender-server-1.0.0-resolved.yaml @@ -3564,11 +3564,13 @@ components: FlightDeclarationStateOperatorUpdateEnum: type: integer enum: + - 1 - 2 - 3 - 5 description: > The state of the operation + * `1` - Accepted * `2` - Activated * `3` - Nonconforming * `5` - Ended diff --git a/auth_helper/dss_auth_helper.py b/auth_helper/dss_auth_helper.py index f8ad374..9d35d6e 100644 --- a/auth_helper/dss_auth_helper.py +++ b/auth_helper/dss_auth_helper.py @@ -94,8 +94,9 @@ def _request_credentials(self, audience: str, scopes: list[str]): scopes_str = " ".join(scopes) auth_server_url = env.get("DSS_AUTH_URL", "http://host.docker.internal:8085") + env.get("DSS_AUTH_TOKEN_ENDPOINT", "/auth/token") + use_dummy_oauth = env.get("DSS_USE_DUMMY_OAUTH", "0").lower() in ("1", "true", "yes") - if auth_server_url.startswith("http://local_"): + if use_dummy_oauth or auth_server_url.startswith("http://local_"): payload = { "grant_type": "client_credentials", "intended_audience": env.get("DSS_SELF_AUDIENCE"), diff --git a/common/data_definitions.py b/common/data_definitions.py index 57e85a4..5329543 100644 --- a/common/data_definitions.py +++ b/common/data_definitions.py @@ -56,6 +56,7 @@ # When an operator changes a state, he / she puts a new state (via the API), this object specifies the event when a operator takes action OPERATOR_EVENT_LOOKUP = { + 1: "dss_accepts", 5: "operator_confirms_ended", 2: "operator_activates", 4: "operator_initiates_contingent", diff --git a/conformance_monitoring_operations/conformance_checks_handler.py b/conformance_monitoring_operations/conformance_checks_handler.py index 8cf0658..11aa652 100644 --- a/conformance_monitoring_operations/conformance_checks_handler.py +++ b/conformance_monitoring_operations/conformance_checks_handler.py @@ -81,9 +81,11 @@ def verify_operation_state_transition(self, original_state: int, new_state: int, my_operation_state_machine = FlightOperationStateMachine(state=original_state) logger.info("Current Operation State %s" % my_operation_state_machine.state) + logger.info("Attempting transition: state %s -> %s via event '%s'" % (original_state, new_state, event)) my_operation_state_machine.on_event(event) changed_state = get_status(my_operation_state_machine.state) + logger.info("State after event: %s (int: %s), expected: %s" % (my_operation_state_machine.state, changed_state, new_state)) if changed_state == new_state: return True else: diff --git a/constraint_operations/constraints_helper.py b/constraint_operations/constraints_helper.py index 3c4f909..cc95bd1 100644 --- a/constraint_operations/constraints_helper.py +++ b/constraint_operations/constraints_helper.py @@ -27,7 +27,7 @@ class USSConstraintsOperations: def __init__(self): - self.dss_base_url = env.get("DSS_BASE_URL", "0") + self.dss_base_url = env.get("DSS_BASE_URL", "0").rstrip("/") + "/" self.r = get_redis() self.database_reader = FlightBlenderDatabaseReader() self.database_writer = FlightBlenderDatabaseWriter() diff --git a/constraint_operations/dss_constraints_helper.py b/constraint_operations/dss_constraints_helper.py index 68ef91d..ef646a5 100644 --- a/constraint_operations/dss_constraints_helper.py +++ b/constraint_operations/dss_constraints_helper.py @@ -35,7 +35,7 @@ class ConstraintOperations: def __init__(self): - self.dss_base_url = env.get("DSS_BASE_URL", "0") + self.dss_base_url = env.get("DSS_BASE_URL", "0").rstrip("/") + "/" self.database_reader = FlightBlenderDatabaseReader() self.database_writer = FlightBlenderDatabaseWriter() diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 8f92586..99ffcad 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -17,10 +17,12 @@ services: platform: linux/amd64 container_name: "db-blender" image: postgres:17 + environment: + - PGPORT=5433 ports: - - "5433:5432" # Host port 5433, container port 5432 + - "5433:5433" expose: - - "5432" + - "5433" restart: unless-stopped volumes: - db_data:/var/lib/postgresql/data/ diff --git a/docker-compose.yml b/docker-compose.yml index bf69355..698404d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,8 +17,10 @@ services: # platform: linux/amd64 container_name: "db-blender" image: postgres:17 + environment: + - PGPORT=5433 expose: - - "5432" + - "5433" restart: unless-stopped volumes: - db_data:/var/lib/postgresql/data diff --git a/entrypoints/no-database/entrypoint.sh b/entrypoints/no-database/entrypoint.sh index 19ce110..bf72401 100755 --- a/entrypoints/no-database/entrypoint.sh +++ b/entrypoints/no-database/entrypoint.sh @@ -5,9 +5,13 @@ if ! uv run python entrypoints/wait_for_service.py --service $REDIS_HOST:$REDIS_ exit 1 fi +# Sync dependencies (ensures newly added packages are installed) +echo "Syncing dependencies..." +uv sync --frozen --no-dev + # Collect static files -#echo "Collect static files" -#uv run python manage.py collectstatic --noinput +echo "Collect static files" +uv run python manage.py collectstatic --noinput # Apply database migrations echo "Apply database migrations" diff --git a/entrypoints/with-database/entrypoint-beat.sh b/entrypoints/with-database/entrypoint-beat.sh index 730899d..7224c79 100755 --- a/entrypoints/with-database/entrypoint-beat.sh +++ b/entrypoints/with-database/entrypoint-beat.sh @@ -3,12 +3,8 @@ source .venv/bin/activate echo Waiting for DBs... -# Use port 5432 for internal Docker network connections (db-blender), 5433 for host connections -if [ "$POSTGRES_HOST" = "db-blender" ]; then - POSTGRES_PORT=5432 -else - POSTGRES_PORT=${POSTGRES_PORT:-5433} -fi +# Postgres listens on 5433 both inside Docker and on the host +POSTGRES_PORT=${POSTGRES_PORT:-5433} if ! wait-for-it --parallel --service $REDIS_HOST:$REDIS_PORT --service $POSTGRES_HOST:$POSTGRES_PORT; then exit fi diff --git a/entrypoints/with-database/entrypoint.sh b/entrypoints/with-database/entrypoint.sh index 750836f..3463349 100755 --- a/entrypoints/with-database/entrypoint.sh +++ b/entrypoints/with-database/entrypoint.sh @@ -3,19 +3,15 @@ source .venv/bin/activate echo Waiting for DBs... -# Use port 5432 for internal Docker network connections (db-blender), 5433 for host connections -if [ "$POSTGRES_HOST" = "db-blender" ]; then - POSTGRES_PORT=5432 -else - POSTGRES_PORT=${POSTGRES_PORT:-5433} -fi +# Postgres listens on 5433 both inside Docker and on the host +POSTGRES_PORT=${POSTGRES_PORT:-5433} if ! wait-for-it --parallel --service $REDIS_HOST:$REDIS_PORT --service $POSTGRES_HOST:$POSTGRES_PORT; then exit fi # Collect static files -#echo "Collect static files" -#python manage.py collectstatic --noinput +echo "Collect static files" +python manage.py collectstatic --noinput # Apply database migrations echo "Apply database migrations" diff --git a/env.template b/env.template index 6c12a02..7f19bf9 100644 --- a/env.template +++ b/env.template @@ -38,4 +38,18 @@ POSTGRES_PASSWORD=mypassword POSTGRES_DB=mydatabase POSTGRES_HOST=db-blender PGDATA=/var/lib/postgresql/data/pgdata -POSTGRES_PORT=5433 \ No newline at end of file +POSTGRES_PORT=5433 + + +USSP_NETWORK_ENABLED=1 +DSS_SELF_AUDIENCE=localhost +AUTH_DSS_CLIENT_ID=eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJsb2NhbGhvc3QiLCJleHAiOjE3NzEyNjAxNzgsImlzcyI6ImxvY2FsaG9zdCIsInNjb3BlIjoiZHNzLnJlYWQuaWRlbnRpZmljYXRpb25fc2VydmljZV9hcmVhcyIsInN1YiI6ImZha2VfdXNzIn0.SItbEdY1_Q0nGCZVD-cFzri3i9H1-51jVUSuXHSIyVCy0aCrHJh_gI2su5Upbx_nFs5NmZCLvYO_zxyUsmjHG-FvTkYRW8ggCWXapeaKMdNcuCtjaKyr8iYJnrBgLWAxsB9BFGb5alsggpQ2xQ8N-HMRwt8bU2gjWLJq9m--0AYtERPhPvDzV4NAIf_wXaraTMblnQCBnGqvliWVMk3WHgxPJjJ5-KPwg0-i7yFSlpKbtEn0YQIo2wEWjtNGu1psz4nW5-vb1hzSwQQOqMQ3ZMijQqIFcYt59CxGKQUi2I5yzURdYXFabWBPb1gVjJwmfYwziuWC6XHQyw_-_e55Ug +AUTH_DSS_CLIENT_SECRET=eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJsb2NhbGhvc3QiLCJleHAiOjE3NzEyNjAxNzgsImlzcyI6ImxvY2FsaG9zdCIsInNjb3BlIjoiZHNzLnJlYWQuaWRlbnRpZmljYXRpb25fc2VydmljZV9hcmVhcyIsInN1YiI6ImZha2VfdXNzIn0.SItbEdY1_Q0nGCZVD-cFzri3i9H1-51jVUSuXHSIyVCy0aCrHJh_gI2su5Upbx_nFs5NmZCLvYO_zxyUsmjHG-FvTkYRW8ggCWXapeaKMdNcuCtjaKyr8iYJnrBgLWAxsB9BFGb5alsggpQ2xQ8N-HMRwt8bU2gjWLJq9m--0AYtERPhPvDzV4NAIf_wXaraTMblnQCBnGqvliWVMk3WHgxPJjJ5-KPwg0-i7yFSlpKbtEn0YQIo2wEWjtNGu1psz4nW5-vb1hzSwQQOqMQ3ZMijQqIFcYt59CxGKQUi2I5yzURdYXFabWBPb1gVjJwmfYwziuWC6XHQyw_-_e55Ug +DSS_BASE_URL=http://flight-dss:8082 +DSS_AUTH_URL=http://flight-dss-auth:8085 +DSS_AUTH_TOKEN_ENDPOINT=/token +DSS_USE_DUMMY_OAUTH=1 + + +OPENSKY_NETWORK_USERNAME=opensky +OPENSKY_NETWORK_PASSWORD=opensky diff --git a/flight_blender/settings.py b/flight_blender/settings.py index e56886f..34071f6 100644 --- a/flight_blender/settings.py +++ b/flight_blender/settings.py @@ -65,6 +65,7 @@ MIDDLEWARE = [ "django.middleware.security.SecurityMiddleware", + "whitenoise.middleware.WhiteNoiseMiddleware", "django.contrib.sessions.middleware.SessionMiddleware", "django.middleware.common.CommonMiddleware", "django.middleware.csrf.CsrfViewMiddleware", @@ -98,7 +99,7 @@ DATABASES = {} -USE_LOCAL_SQLITE_DATABASE = os.getenv("USE_LOCAL_SQLITE_DATABASE", 0) +USE_LOCAL_SQLITE_DATABASE = int(os.getenv("USE_LOCAL_SQLITE_DATABASE", 0)) if USE_LOCAL_SQLITE_DATABASE: DATABASES = { "default": { @@ -153,6 +154,12 @@ STATIC_URL = "/static/" STATIC_ROOT = os.getenv("STATIC_ROOT", BASE_DIR / "staticfiles") +STORAGES = { + "staticfiles": { + "BACKEND": "whitenoise.storage.CompressedManifestStaticFilesStorage", + }, +} + DEFAULT_AUTO_FIELD = "django.db.models.AutoField" if DEBUG: BROKER_URL = os.getenv("REDIS_BROKER_URL", "redis://localhost:6379/") diff --git a/flight_declaration_operations/serializers.py b/flight_declaration_operations/serializers.py index 01091a7..da6a713 100644 --- a/flight_declaration_operations/serializers.py +++ b/flight_declaration_operations/serializers.py @@ -122,7 +122,7 @@ def validate_state(self, value: int) -> int: int: The validated state value. """ if self.instance and value not in list(OPERATOR_EVENT_LOOKUP.keys()): - raise serializers.ValidationError("An operator can only set the state to Activated (2), Contingent (4) or Ended (5) using this endpoint") + raise serializers.ValidationError("An operator can only set the state to Accepted (1), Activated (2), Contingent (4) or Ended (5) using this endpoint") current_state = self.instance.state event = OPERATOR_EVENT_LOOKUP[value] diff --git a/flight_declaration_operations/views.py b/flight_declaration_operations/views.py index 6b8b6b3..3751ec4 100644 --- a/flight_declaration_operations/views.py +++ b/flight_declaration_operations/views.py @@ -290,7 +290,9 @@ def set_operational_intent(request): ) else: if declaration_state == 0 and USSP_NETWORK_ENABLED: - submit_flight_declaration_to_dss_async.delay(flight_declaration_id=flight_declaration_id) + from django.db import transaction + + transaction.on_commit(lambda: submit_flight_declaration_to_dss_async.delay(flight_declaration_id=flight_declaration_id)) creation_response = FlightDeclarationCreateResponse( id=flight_declaration_id, @@ -398,7 +400,9 @@ def set_flight_declaration(request): ) else: if declaration_state == 0 and USSP_NETWORK_ENABLED: - submit_flight_declaration_to_dss_async.delay(flight_declaration_id=flight_declaration_id) + from django.db import transaction + + transaction.on_commit(lambda: submit_flight_declaration_to_dss_async.delay(flight_declaration_id=flight_declaration_id)) creation_response = FlightDeclarationCreateResponse( id=flight_declaration_id, @@ -688,7 +692,9 @@ def post(self, request, *args, **kwargs): ) else: if declaration_state == 0 and USSP_NETWORK_ENABLED: - submit_flight_declaration_to_dss_async.delay(flight_declaration_id=flight_declaration_id) + from django.db import transaction + + transaction.on_commit(lambda: submit_flight_declaration_to_dss_async.delay(flight_declaration_id=flight_declaration_id)) creation_response = FlightDeclarationCreateResponse( id=flight_declaration_id, diff --git a/pyproject.toml b/pyproject.toml index 87eb7e0..081d806 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,6 +55,7 @@ dependencies = [ "channels==4.3.1", "channels-redis==4.3.0", "loguru==0.7.3", + "whitenoise==6.9.0", ] [dependency-groups] diff --git a/rid_operations/dss_rid_helper.py b/rid_operations/dss_rid_helper.py index f08bffd..5b088d0 100644 --- a/rid_operations/dss_rid_helper.py +++ b/rid_operations/dss_rid_helper.py @@ -70,7 +70,7 @@ class RemoteIDOperations: def __init__(self): - self.dss_base_url = env.get("DSS_BASE_URL", "000") + self.dss_base_url = env.get("DSS_BASE_URL", "000").rstrip("/") + "/" self.r = get_redis() def compute_polygon_area(self, polygon: Polygon): diff --git a/rid_operations/views.py b/rid_operations/views.py index a892694..f8e52b8 100644 --- a/rid_operations/views.py +++ b/rid_operations/views.py @@ -2,6 +2,7 @@ import json import time import uuid +import dataclasses from dataclasses import asdict from datetime import timedelta from typing import Any @@ -71,6 +72,8 @@ class RIDOutputHelper: def make_json_compatible(self, struct: Any) -> Any: if isinstance(struct, tuple) and hasattr(struct, "_asdict"): return {k: self.make_json_compatible(v) for k, v in struct._asdict().items()} + elif dataclasses.is_dataclass(struct) and not isinstance(struct, type): + return {k: self.make_json_compatible(v) for k, v in asdict(struct).items()} elif isinstance(struct, dict): return {k: self.make_json_compatible(v) for k, v in struct.items()} elif isinstance(struct, str): @@ -116,8 +119,7 @@ def create_new_rid_subscription( subscription_duration_seconds=subscription_duration_seconds, is_simulated=is_simulated, ) - subscription_response = self.my_rid_output_helper.make_json_compatible(subscription_r) - return subscription_response + return subscription_r def start_ussp_polling(self): """ diff --git a/scd_operations/dss_scd_helper.py b/scd_operations/dss_scd_helper.py index f74d01d..dd52f99 100644 --- a/scd_operations/dss_scd_helper.py +++ b/scd_operations/dss_scd_helper.py @@ -705,7 +705,7 @@ def parse_operational_intent_reference_from_dss(self, operational_intent_referen class SCDOperations: def __init__(self): - self.dss_base_url = env.get("DSS_BASE_URL", "0") + self.dss_base_url = env.get("DSS_BASE_URL", "0").rstrip("/") + "/" self.r = get_redis() self.database_reader = FlightBlenderDatabaseReader() self.database_writer = FlightBlenderDatabaseWriter() @@ -838,6 +838,8 @@ def get_nearby_operational_intents(self, volumes: list[Volume4D]) -> list[Operat uss_op_int_id=current_uss_operational_intent_detail.id ) ) + op_int_details_retrieved = False + continue op_int_details_retrieved = True else: # This operational intent details is from a peer uss, need to query peer USS diff --git a/scd_operations/opint_helper.py b/scd_operations/opint_helper.py index 85e70b1..ca8556c 100644 --- a/scd_operations/opint_helper.py +++ b/scd_operations/opint_helper.py @@ -38,6 +38,9 @@ def __init__(self, flight_declaration_id: str): def validate_flight_declaration_start_end_time(self) -> bool: flight_declaration = self.my_database_reader.get_flight_declaration_by_id(flight_declaration_id=self.flight_declaration_id) + if not flight_declaration: + logger.error(f"Flight Declaration with ID {self.flight_declaration_id} not found in database, cannot validate start/end time") + return False # check that flight declaration start and end time is in the next two hours now = arrow.now() two_hours_from_now = now.shift(hours=2) diff --git a/uv.lock b/uv.lock index 097e556..e8e5598 100644 --- a/uv.lock +++ b/uv.lock @@ -577,6 +577,7 @@ dependencies = [ { name = "uas-standards" }, { name = "uvicorn", extra = ["standard"] }, { name = "wait-for-it" }, + { name = "whitenoise" }, ] [package.dev-dependencies] @@ -630,6 +631,7 @@ requires-dist = [ { name = "uas-standards", specifier = "==3.4.0" }, { name = "uvicorn", extras = ["standard"], specifier = "==0.37.0" }, { name = "wait-for-it", specifier = "==2.3.0" }, + { name = "whitenoise", specifier = "==6.9.0" }, ] [package.metadata.requires-dev] @@ -2157,6 +2159,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fa/a8/5b41e0da817d64113292ab1f8247140aac61cbf6cfd085d6a0fa77f4984f/websockets-15.0.1-py3-none-any.whl", hash = "sha256:f7a866fbc1e97b5c617ee4116daaa09b722101d4a3c170c787450ba409f9736f", size = 169743, upload-time = "2025-03-05T20:03:39.41Z" }, ] +[[package]] +name = "whitenoise" +version = "6.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b9/cf/c15c2f21aee6b22a9f6fc9be3f7e477e2442ec22848273db7f4eb73d6162/whitenoise-6.9.0.tar.gz", hash = "sha256:8c4a7c9d384694990c26f3047e118c691557481d624f069b7f7752a2f735d609", size = 25920, upload-time = "2025-02-06T22:16:34.957Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/b2/2ce9263149fbde9701d352bda24ea1362c154e196d2fda2201f18fc585d7/whitenoise-6.9.0-py3-none-any.whl", hash = "sha256:c8a489049b7ee9889617bb4c274a153f3d979e8f51d2efd0f5b403caf41c57df", size = 20161, upload-time = "2025-02-06T22:16:32.589Z" }, +] + [[package]] name = "win32-setctime" version = "1.2.0"