diff --git a/Dockerfile b/Dockerfile index aa7e80f..ab0c6a8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,17 +1,46 @@ -FROM --platform=linux/amd64 python:3.12-slim +# FROM --platform=linux/amd64 +FROM python:3.12-slim ENV PYTHONUNBUFFERED=1 ENV PYTHONDONTWRITEBYTECODE=1 WORKDIR /app +# Install system dependencies +RUN apt-get update && apt-get install -y --no-install-recommends \ + gcc \ + postgresql-client \ + && rm -rf /var/lib/apt/lists/* + +# Create non-root user first +RUN addgroup --gid 10000 django && adduser --shell /bin/bash --disabled-password --gecos "" --uid 10000 --ingroup django django + +# Copy dependency files COPY uv.lock pyproject.toml ./ + +# Install Python dependencies (without the project itself) RUN pip install -U pip && pip install uv && uv sync --frozen --no-install-project --no-dev -RUN addgroup --gid 10000 django && adduser --shell /bin/bash --disabled-password --gecos "" --uid 10000 --ingroup django django +# Set PYTHONPATH to include the current directory so Django can find the modules +ENV PYTHONPATH=/app + +# Copy application code +COPY --chown=django:django . . + +# Change ownership of entire /app directory (including .venv) to django user RUN chown -R django:django /app + +# Switch to django user and install the project USER django:django +RUN uv sync --frozen --no-dev +USER root -COPY --chown=django:django . . +# Make entrypoint scripts executable +RUN chmod +x entrypoints/docker-entrypoint-web.sh entrypoints/docker-entrypoint-worker.sh entrypoints/wait_for_service.py + +USER django:django EXPOSE 8000 + +# Default to web entrypoint (can be overridden) +CMD ["./entrypoints/docker-entrypoint-web.sh"] diff --git a/FEATURES.md b/FEATURES.md new file mode 100644 index 0000000..b845ac6 --- /dev/null +++ b/FEATURES.md @@ -0,0 +1,44 @@ +Flight Blender is an open-source backend and data-processing engine designed to support standards-compliant UTM (Unmanned Traffic Management) services. It adheres to the latest regulations for UTM/U-Space in the EU and other jurisdictions. With Flight Blender, you can: + +Implement a Remote ID “service provider” compatible with the ASTM-F3411 Remote ID standard, along with Flight Spotlight, an open-source Remote ID Display Application. +Use an open-source implementation of the ASTM F3548 USS-to-USS standard, compatible with EU U-Space regulations for flight authorization. +Interact with interoperability software like interuss/dss to exchange data with other UTM systems. +Process geo-fences using the ED-269 standard. +Monitor conformance and send operator notifications. +Aggregate flight traffic feeds from various sources, including geo-fences, flight declarations, and air-traffic data. +Configure Blender to act as a Surveillance SDSP per the ASTM F3623-23 standard. +Implement alerts / near misses per the ASTM F3442 standard + + +Key Features + +DSS Connectivity +Connect and retrieve data such as Remote ID information or perform strategic de-confliction and flight authorization. + +Flight Tracking +Ingest flight tracking feeds from sources like ADS-B, live telemetry, and Broadcast Remote ID. Outputs a unified JSON feed for real-time display. + +Geofence Management +Submit geofence to Flight Blender, which can then be transmitted to Spotlight for visualization. + +Flight Declaration +Submit future flight plans (up to 24 hours in advance) using the ASTM USS-to-USS API or as a standalone component. Supported DSS APIs are listed below. + +Network Remote ID +Compliant with ASTM standards, this module can act as a “display provider” or “service provider” for Network Remote ID. + +Operator Notifications +Send notifications to operators using an AMQP queue, enabling real-time alerts for flight updates, conformance issues, or other critical events. + +Conformance Monitoring +Monitor flight paths against declared 4D volumes for conformance and report outputs. + +Surveillance SDSP +Blender conforms to the requirements for Surveillance supplemental data service providers (SDSPs) and associated equipment and services. + +Detect, Alert and Avoid +Flight Blender implements the Detect Alert and Avoid standard F3442 + + +Can you explore these repos I have attached and create a detailed spec for each of this features on how they can be implemented in the spotlight, the API endpoints in the Blender, that is responsible for these features. I am new to openutm, so make this as much detailed with diagram as you can + diff --git a/Procfile b/Procfile index 59f0f0b..0395dfd 100644 --- a/Procfile +++ b/Procfile @@ -1,2 +1,2 @@ -web: gunicorn flight_blender:app -worker: celery worker --app=flight_blender +web: uvicorn flight_blender.asgi:application --host 0.0.0.0 --port $PORT --workers 3 +worker: celery --app=flight_blender worker --loglevel=info diff --git a/README.md b/README.md index b64d2b3..8db685a 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,219 @@ Follow our simple 5-step guide to deploy Flight Blender and explore its core fea 📖 [Read the 20-minute quickstart guide](deployment_support/README.md) to get started now! +--- + +## 🚀 How to Run Flight Blender + +### Prerequisites + +- **Docker** and **Docker Compose** installed on your system +- **Python 3.12+** (if running locally without Docker) +- **PostgreSQL** (handled by Docker Compose) +- **Redis/Valkey** (handled by Docker Compose) + +### Quick Start with Docker (Recommended) + +#### 1. Create Environment File + +Create a `.env` file in the root directory. You can use the sample from the [deployment guide](deployment_support/README.md) or create one with the following minimum required variables: + +```bash +# Django Settings +SECRET_KEY=your-secret-key-here +IS_DEBUG=1 +BYPASS_AUTH_TOKEN_VERIFICATION=1 +ALLOWED_HOSTS=* + +# Database Configuration +POSTGRES_USER=flightblender +POSTGRES_PASSWORD=your-password-here +POSTGRES_DB=flightblender +POSTGRES_HOST=db-blender +DATABASE_URL=postgresql://flightblender:your-password-here@db-blender:5432/flightblender + +# Redis Configuration +REDIS_HOST=redis-blender +REDIS_PORT=6379 +REDIS_PASSWORD=your-redis-password +REDIS_BROKER_URL=redis://:your-redis-password@redis-blender:6379/ + +# Optional: Standalone Mode (set to 0 for standalone, 1 for DSS integration) +USSP_NETWORK_ENABLED=0 + +# Optional: Heartbeat Rate +HEARTBEAT_RATE_SECS=2 +``` + +**⚠️ Security Note**: The `BYPASS_AUTH_TOKEN_VERIFICATION=1` setting is for local development only. Remove it for production deployments. + +#### 2. Build and Run with Docker Compose + +For **development** (using `docker-compose-dev.yml`): + +```bash +# Build the Docker image +docker build . -t openutm/flight-blender-dev + +# Start all services +docker compose -f docker-compose-dev.yml up +``` + +For **production-like** setup (using `docker-compose.yml`): + +**⚠️ Important Production Checklist:** + +1. **Update your `.env` file for production:** + - Remove or set `BYPASS_AUTH_TOKEN_VERIFICATION=0` (security risk if enabled) + - Set `IS_DEBUG=0` + - Set `ALLOWED_HOSTS` to your domain name (e.g., `ALLOWED_HOSTS=yourdomain.com,www.yourdomain.com`) + - Ensure `USE_LOCAL_SQLITE_DATABASE=0` (use PostgreSQL) + - Set strong passwords for `SECRET_KEY`, `POSTGRES_PASSWORD`, and `REDIS_PASSWORD` + +2. **Create the external network (if it doesn't exist):** +```bash +docker network create interop_ecosystem_network +``` + +3. **Build the production Docker image:** +```bash +docker build . -t openutm/flight-blender +``` + +4. **Start all services:** +```bash +docker compose up -d # -d runs in detached mode +``` + +**Platform Notes:** +- The production `docker-compose.yml` has `platform: linux/amd64` commented out for macOS compatibility +- For production on Linux servers, uncomment the `platform: linux/amd64` lines in `docker-compose.yml` +- For multi-platform builds: `docker buildx build --platform linux/amd64 -t openutm/flight-blender .` + +Alternatively, use the provided startup script: + +```bash +chmod +x start_flight_blender.sh +./start_flight_blender.sh +``` + +#### 3. Access the Application + +Once the containers are running, access Flight Blender at: + +- **Web Interface**: http://localhost:8000 +- **API Documentation**: http://localhost:8000/api/docs + +You should see the Flight Blender logo and links to the API documentation. + +#### 4. Verify Services + +The Docker Compose setup includes: +- **flight-blender**: Main Django application (port 8000) +- **db-blender**: PostgreSQL database (port 5432) +- **redis-blender**: Redis/Valkey cache and message broker (port 6379) +- **worker**: Celery worker for background tasks +- **flight-blender-beat**: Celery beat scheduler (in dev mode) + +### Running Locally (Without Docker) + +If you prefer to run without Docker: + +#### 1. Install Dependencies + +The project uses `uv` for dependency management: + +```bash +# Install uv if not already installed +pip install uv + +# Install project dependencies +uv sync +``` + +#### 2. Set Up Database + +Ensure PostgreSQL and Redis are running locally, then update your `.env` file: + +```bash +DATABASE_URL=postgresql://user:password@localhost:5432/flightblender +REDIS_HOST=localhost +REDIS_PORT=6379 +REDIS_BROKER_URL=redis://localhost:6379/ +``` + +#### 3. Run Database Migrations + +```bash +python manage.py migrate +``` + +#### 4. Start the Development Server + +```bash +python manage.py runserver +``` + +#### 5. Start Celery Worker (in separate terminal) + +```bash +celery -A flight_blender worker -l info +``` + +#### 6. Start Celery Beat (optional, in another terminal) + +```bash +celery -A flight_blender beat -l info +``` + +### Deploy on Render (Standalone) + +Flight Blender includes Render blueprints for standalone deployments. + +#### Option A: Docker (recommended for parity) +1. In Render, create a **New Blueprint** and point it to this repo. +2. Select `render.yaml`. +3. Render will provision: + - Web service (`flight-blender-web`) + - Worker (`flight-blender-worker`) + - Redis + - Postgres +4. Set/override required env vars (see `env.template`), especially: + - `PASSPORT_URL`, `PASSPORT_AUDIENCE`, `BYPASS_AUTH_TOKEN_VERIFICATION=0` + - `ALLOWED_HOSTS` to your Render host +5. Deploy. The web service will be reachable at your Render URL. + +#### Option B: Non-Docker Python +1. In Render, create a **New Blueprint** and point it to this repo. +2. Select `render-no-docker.yaml`. +3. Render will provision the same services using Python build/start commands. +4. Set/override required env vars as above. + +### Troubleshooting + +**Issue: Port conflicts** +- Ensure ports 8000, 5432, and 6379 are not in use +- Stop local PostgreSQL/Redis if running: `sudo systemctl stop postgresql` + +**Issue: Docker network errors** +- For `docker-compose.yml`, create the network: `docker network create interop_ecosystem_network` +- For `docker-compose-dev.yml`, the network is created automatically + +**Issue: Database connection errors** +- Verify PostgreSQL container is running: `docker ps` +- Check `.env` file has correct database credentials +- Ensure database migrations have run + +**Issue: Redis connection errors** +- Verify Redis container is running +- Check `REDIS_PASSWORD` matches in `.env` and `redis.conf` + +### Next Steps + +- Import the [Postman Collection](api/flight_blender_api.postman_collection.json) to test the API +- Generate access tokens using the [verification repository](https://github.com/openutm/verification) +- Explore the [API documentation](http://redocly.github.io/redoc/?url=https://raw.githubusercontent.com/openutm/flight-blender/master/api/flight-blender-server-1.0.0-resolved.yaml) + --- ## 💫 Join the community [Discord](https://discord.gg/dnRxpZdd9a) diff --git a/api/flight-blender-server-1.0.0-resolved.yaml b/api/flight-blender-server-1.0.0-resolved.yaml index 03d334b..ed2d78b 100644 --- a/api/flight-blender-server-1.0.0-resolved.yaml +++ b/api/flight-blender-server-1.0.0-resolved.yaml @@ -3564,11 +3564,13 @@ components: FlightDeclarationStateOperatorUpdateEnum: type: integer enum: + - 1 - 2 - 3 - 5 description: > The state of the operation + * `1` - Accepted * `2` - Activated * `3` - Nonconforming * `5` - Ended diff --git a/auth_helper/dss_auth_helper.py b/auth_helper/dss_auth_helper.py index f8ad374..9d35d6e 100644 --- a/auth_helper/dss_auth_helper.py +++ b/auth_helper/dss_auth_helper.py @@ -94,8 +94,9 @@ def _request_credentials(self, audience: str, scopes: list[str]): scopes_str = " ".join(scopes) auth_server_url = env.get("DSS_AUTH_URL", "http://host.docker.internal:8085") + env.get("DSS_AUTH_TOKEN_ENDPOINT", "/auth/token") + use_dummy_oauth = env.get("DSS_USE_DUMMY_OAUTH", "0").lower() in ("1", "true", "yes") - if auth_server_url.startswith("http://local_"): + if use_dummy_oauth or auth_server_url.startswith("http://local_"): payload = { "grant_type": "client_credentials", "intended_audience": env.get("DSS_SELF_AUDIENCE"), diff --git a/common/data_definitions.py b/common/data_definitions.py index 57e85a4..5329543 100644 --- a/common/data_definitions.py +++ b/common/data_definitions.py @@ -56,6 +56,7 @@ # When an operator changes a state, he / she puts a new state (via the API), this object specifies the event when a operator takes action OPERATOR_EVENT_LOOKUP = { + 1: "dss_accepts", 5: "operator_confirms_ended", 2: "operator_activates", 4: "operator_initiates_contingent", diff --git a/conformance_monitoring_operations/conformance_checks_handler.py b/conformance_monitoring_operations/conformance_checks_handler.py index 8cf0658..11aa652 100644 --- a/conformance_monitoring_operations/conformance_checks_handler.py +++ b/conformance_monitoring_operations/conformance_checks_handler.py @@ -81,9 +81,11 @@ def verify_operation_state_transition(self, original_state: int, new_state: int, my_operation_state_machine = FlightOperationStateMachine(state=original_state) logger.info("Current Operation State %s" % my_operation_state_machine.state) + logger.info("Attempting transition: state %s -> %s via event '%s'" % (original_state, new_state, event)) my_operation_state_machine.on_event(event) changed_state = get_status(my_operation_state_machine.state) + logger.info("State after event: %s (int: %s), expected: %s" % (my_operation_state_machine.state, changed_state, new_state)) if changed_state == new_state: return True else: diff --git a/constraint_operations/constraints_helper.py b/constraint_operations/constraints_helper.py index 3c4f909..cc95bd1 100644 --- a/constraint_operations/constraints_helper.py +++ b/constraint_operations/constraints_helper.py @@ -27,7 +27,7 @@ class USSConstraintsOperations: def __init__(self): - self.dss_base_url = env.get("DSS_BASE_URL", "0") + self.dss_base_url = env.get("DSS_BASE_URL", "0").rstrip("/") + "/" self.r = get_redis() self.database_reader = FlightBlenderDatabaseReader() self.database_writer = FlightBlenderDatabaseWriter() diff --git a/constraint_operations/dss_constraints_helper.py b/constraint_operations/dss_constraints_helper.py index 68ef91d..ef646a5 100644 --- a/constraint_operations/dss_constraints_helper.py +++ b/constraint_operations/dss_constraints_helper.py @@ -35,7 +35,7 @@ class ConstraintOperations: def __init__(self): - self.dss_base_url = env.get("DSS_BASE_URL", "0") + self.dss_base_url = env.get("DSS_BASE_URL", "0").rstrip("/") + "/" self.database_reader = FlightBlenderDatabaseReader() self.database_writer = FlightBlenderDatabaseWriter() diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 21a3453..99ffcad 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -11,23 +11,25 @@ services: container_name: "redis-blender" env_file: - ".env" - # networks: - # - interop_ecosystem_network + networks: + - interop_ecosystem_network db-blender: platform: linux/amd64 container_name: "db-blender" image: postgres:17 + environment: + - PGPORT=5433 ports: - - "5432:5432" + - "5433:5433" expose: - - "5432" + - "5433" restart: unless-stopped volumes: - db_data:/var/lib/postgresql/data/ env_file: - ".env" - # networks: - # - interop_ecosystem_network + networks: + - interop_ecosystem_network flight-blender: platform: linux/amd64 @@ -48,8 +50,8 @@ services: - db-blender # volumes: # - .:/app - # networks: - # - interop_ecosystem_network + networks: + - interop_ecosystem_network flight-blender-celery: @@ -66,8 +68,8 @@ services: # - .:/app depends_on: - redis-blender - # networks: - # - interop_ecosystem_network + networks: + - interop_ecosystem_network flight-blender-celery-beat: platform: linux/amd64 @@ -90,13 +92,13 @@ services: condition: service_started flight-blender-celery: condition: service_started - # networks: - # - interop_ecosystem_network + networks: + - interop_ecosystem_network volumes: app: db_data: -# networks: -# interop_ecosystem_network: -# external: true +networks: + interop_ecosystem_network: + external: true diff --git a/docker-compose.yml b/docker-compose.yml index 8a538ae..698404d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ version: "3" services: redis-blender: - platform: linux/amd64 + # platform: linux/amd64 command: ["redis-server", "/redis.conf", "--requirepass", "$REDIS_PASSWORD"] image: "valkey/valkey:latest" expose: @@ -14,11 +14,13 @@ services: networks: - interop_ecosystem_network db-blender: - platform: linux/amd64 + # platform: linux/amd64 container_name: "db-blender" image: postgres:17 + environment: + - PGPORT=5433 expose: - - "5432" + - "5433" restart: unless-stopped volumes: - db_data:/var/lib/postgresql/data @@ -28,7 +30,7 @@ services: - interop_ecosystem_network flight-blender: - platform: linux/amd64 + # platform: linux/amd64 container_name: "flight-blender" env_file: - ".env" @@ -49,7 +51,7 @@ services: flight-blender-celery: - platform: linux/amd64 + # platform: linux/amd64 container_name: worker image: openutm/flight-blender build: diff --git a/entrypoints/docker-entrypoint-web.sh b/entrypoints/docker-entrypoint-web.sh new file mode 100755 index 0000000..a99c518 --- /dev/null +++ b/entrypoints/docker-entrypoint-web.sh @@ -0,0 +1,49 @@ +#!/bin/bash +set -e + +echo "Waiting for services..." + +# Wait for Redis +if [ -n "$REDIS_HOST" ]; then + echo "Waiting for Redis at ${REDIS_HOST}:${REDIS_PORT:-6379}..." + if ! uv run python entrypoints/wait_for_service.py --service $REDIS_HOST:${REDIS_PORT:-6379}; then + echo "Redis connection failed" + exit 1 + fi + echo "Redis is ready!" +fi + +# Wait for PostgreSQL +if [ -n "$DATABASE_URL" ] || [ -n "$POSTGRES_HOST" ]; then + echo "Waiting for PostgreSQL..." + POSTGRES_PORT=${POSTGRES_PORT:-5432} + if [ -n "$DATABASE_URL" ]; then + # Extract host and port from DATABASE_URL + DB_HOST=$(echo $DATABASE_URL | sed -E 's|.*@([^:]+):.*|\1|') + DB_PORT=$(echo $DATABASE_URL | sed -E 's|.*:([0-9]+)/.*|\1|') + if ! uv run python entrypoints/wait_for_service.py --service $DB_HOST:$DB_PORT; then + echo "PostgreSQL connection failed" + exit 1 + fi + elif [ -n "$POSTGRES_HOST" ]; then + if ! uv run python entrypoints/wait_for_service.py --service $POSTGRES_HOST:$POSTGRES_PORT; then + echo "PostgreSQL connection failed" + exit 1 + fi + fi + echo "PostgreSQL is ready!" +fi + +echo "All services are ready!" + +# Collect static files +echo "Collecting static files..." +uv run python manage.py collectstatic --noinput || echo "Warning: Static files collection failed, continuing..." + +# Apply database migrations +echo "Applying database migrations..." +uv run python manage.py migrate || echo "Warning: Migrations failed, continuing..." + +# Start server +echo "Starting server on port ${PORT:-8000}..." +exec uv run uvicorn flight_blender.asgi:application --host 0.0.0.0 --port ${PORT:-8000} --workers 3 diff --git a/entrypoints/docker-entrypoint-worker.sh b/entrypoints/docker-entrypoint-worker.sh new file mode 100755 index 0000000..34e407a --- /dev/null +++ b/entrypoints/docker-entrypoint-worker.sh @@ -0,0 +1,18 @@ +#!/bin/bash +set -e + +# Wait for Redis +if [ -n "$REDIS_HOST" ]; then + echo "Waiting for Redis at ${REDIS_HOST}:${REDIS_PORT:-6379}..." + if ! uv run python entrypoints/wait_for_service.py --service $REDIS_HOST:${REDIS_PORT:-6379}; then + echo "Redis connection failed" + exit 1 + fi + echo "Redis is ready!" +else + echo "Warning: REDIS_HOST not set, skipping Redis check" +fi + +# Start Celery worker +echo "Starting Celery worker..." +exec uv run celery --app=flight_blender worker --loglevel=info diff --git a/entrypoints/no-database/entrypoint-celery.sh b/entrypoints/no-database/entrypoint-celery.sh index ee37b92..d5acab5 100755 --- a/entrypoints/no-database/entrypoint-celery.sh +++ b/entrypoints/no-database/entrypoint-celery.sh @@ -1,10 +1,8 @@ #!/bin/bash -source .venv/bin/activate - echo Waiting for DBs... -if ! wait-for-it --parallel --service $REDIS_HOST:$REDIS_PORT; then - exit +if ! uv run python entrypoints/wait_for_service.py --service $REDIS_HOST:$REDIS_PORT; then + exit 1 fi -celery --app=flight_blender worker --loglevel=info +uv run celery --app=flight_blender worker --loglevel=info diff --git a/entrypoints/no-database/entrypoint.sh b/entrypoints/no-database/entrypoint.sh index 4b3c56b..bf72401 100755 --- a/entrypoints/no-database/entrypoint.sh +++ b/entrypoints/no-database/entrypoint.sh @@ -1,20 +1,22 @@ #!/bin/bash -source .venv/bin/activate - echo Waiting for DBs... -if ! wait-for-it --parallel --service $REDIS_HOST:$REDIS_PORT; then - exit +if ! uv run python entrypoints/wait_for_service.py --service $REDIS_HOST:$REDIS_PORT; then + exit 1 fi +# Sync dependencies (ensures newly added packages are installed) +echo "Syncing dependencies..." +uv sync --frozen --no-dev + # Collect static files -#echo "Collect static files" -#python manage.py collectstatic --noinput +echo "Collect static files" +uv run python manage.py collectstatic --noinput # Apply database migrations echo "Apply database migrations" -python manage.py migrate +uv run python manage.py migrate # Start server echo "Starting server" -uvicorn flight_blender.asgi:application --host 0.0.0.0 --port 8000 --workers 3 --reload +uv run uvicorn flight_blender.asgi:application --host 0.0.0.0 --port 8000 --workers 3 --reload diff --git a/entrypoints/wait_for_service.py b/entrypoints/wait_for_service.py new file mode 100644 index 0000000..b732cfb --- /dev/null +++ b/entrypoints/wait_for_service.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +"""Simple script to wait for services to be available.""" +import sys +import socket +import time + +def wait_for_service(host, port, timeout=30): + """Wait for a service to be available on host:port.""" + start_time = time.time() + while time.time() - start_time < timeout: + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.settimeout(1) + result = sock.connect_ex((host, port)) + sock.close() + if result == 0: + print(f"Service {host}:{port} is available") + return True + except Exception as e: + pass + time.sleep(1) + print(f"Timeout waiting for {host}:{port}") + return False + +if __name__ == "__main__": + services = [] + i = 1 + while i < len(sys.argv): + if sys.argv[i] == "--service" and i + 1 < len(sys.argv): + host, port = sys.argv[i + 1].split(":") + services.append((host, int(port))) + i += 2 + else: + i += 1 + + if not services: + print("Usage: wait_for_service.py --service host:port [--service host:port ...]") + sys.exit(1) + + all_available = True + for host, port in services: + if not wait_for_service(host, port): + all_available = False + + sys.exit(0 if all_available else 1) diff --git a/entrypoints/with-database/entrypoint-beat.sh b/entrypoints/with-database/entrypoint-beat.sh index 154cb1f..7224c79 100755 --- a/entrypoints/with-database/entrypoint-beat.sh +++ b/entrypoints/with-database/entrypoint-beat.sh @@ -3,6 +3,8 @@ source .venv/bin/activate echo Waiting for DBs... +# Postgres listens on 5433 both inside Docker and on the host +POSTGRES_PORT=${POSTGRES_PORT:-5433} if ! wait-for-it --parallel --service $REDIS_HOST:$REDIS_PORT --service $POSTGRES_HOST:$POSTGRES_PORT; then exit fi diff --git a/entrypoints/with-database/entrypoint-celery.sh b/entrypoints/with-database/entrypoint-celery.sh index ee37b92..d5acab5 100755 --- a/entrypoints/with-database/entrypoint-celery.sh +++ b/entrypoints/with-database/entrypoint-celery.sh @@ -1,10 +1,8 @@ #!/bin/bash -source .venv/bin/activate - echo Waiting for DBs... -if ! wait-for-it --parallel --service $REDIS_HOST:$REDIS_PORT; then - exit +if ! uv run python entrypoints/wait_for_service.py --service $REDIS_HOST:$REDIS_PORT; then + exit 1 fi -celery --app=flight_blender worker --loglevel=info +uv run celery --app=flight_blender worker --loglevel=info diff --git a/entrypoints/with-database/entrypoint.sh b/entrypoints/with-database/entrypoint.sh index ef1b4d7..3463349 100755 --- a/entrypoints/with-database/entrypoint.sh +++ b/entrypoints/with-database/entrypoint.sh @@ -3,13 +3,15 @@ source .venv/bin/activate echo Waiting for DBs... +# Postgres listens on 5433 both inside Docker and on the host +POSTGRES_PORT=${POSTGRES_PORT:-5433} if ! wait-for-it --parallel --service $REDIS_HOST:$REDIS_PORT --service $POSTGRES_HOST:$POSTGRES_PORT; then exit fi # Collect static files -#echo "Collect static files" -#python manage.py collectstatic --noinput +echo "Collect static files" +python manage.py collectstatic --noinput # Apply database migrations echo "Apply database migrations" diff --git a/env.template b/env.template new file mode 100644 index 0000000..7f19bf9 --- /dev/null +++ b/env.template @@ -0,0 +1,55 @@ +SECRET_KEY=XhOCvsdPpQPL2Q74hvDNZw3L4QatShA5CBPdEKfzgc1G3tuwKNVsgSN9MKHrYKL5 + +PASSPORT_AUDIENCE=testflight.flightblender.com +PASSPORT_URL=http://flight-passport:9000 +PASSPORT_JWKS_URL=http://flight-passport:9000/.well-known/jwks.json +DSS_AUTH_JWKS_ENDPOINT=http://flight-passport:9000/.well-known/jwks.json + +# PASSPORT_JWKS_URL=http://localhost:9000/.well-known/jwks.json +# PASSPORT_URL=http://localhost:9000 + +IS_DEBUG=0 + +BYPASS_AUTH_TOKEN_VERIFICATION=0 + +ALLOWED_HOSTS=localhost,127.0.0.1,host.docker.internal,flight-blender + +DISABLE_JSON_LOGGING=1 +ENABLE_CONFORMANCE_MONITORING=1 +USSP_NETWORK_ENABLED=0 + +REDIS_HOST=redis-blender +REDIS_PORT=6379 +REDIS_PASSWORD=blender_redis +REDIS_BROKER_URL=redis://:blender_redis@redis-blender:6379 +HEARTBEAT_RATE_SECS=2 + +FLIGHT_SPOTLIGHT_URL=http://flight-spotlight:5000 + +FLIGHTBLENDER_FQDN=http://host.docker.internal:8000 + + +USE_LOCAL_SQLITE_DATABASE=0 +DATABASE_URL=postgresql://mydatabaseuser:mypassword@db-blender:5433/mydatabase + +# Postgres Docker +POSTGRES_USER=mydatabaseuser +POSTGRES_PASSWORD=mypassword +POSTGRES_DB=mydatabase +POSTGRES_HOST=db-blender +PGDATA=/var/lib/postgresql/data/pgdata +POSTGRES_PORT=5433 + + +USSP_NETWORK_ENABLED=1 +DSS_SELF_AUDIENCE=localhost +AUTH_DSS_CLIENT_ID=eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJsb2NhbGhvc3QiLCJleHAiOjE3NzEyNjAxNzgsImlzcyI6ImxvY2FsaG9zdCIsInNjb3BlIjoiZHNzLnJlYWQuaWRlbnRpZmljYXRpb25fc2VydmljZV9hcmVhcyIsInN1YiI6ImZha2VfdXNzIn0.SItbEdY1_Q0nGCZVD-cFzri3i9H1-51jVUSuXHSIyVCy0aCrHJh_gI2su5Upbx_nFs5NmZCLvYO_zxyUsmjHG-FvTkYRW8ggCWXapeaKMdNcuCtjaKyr8iYJnrBgLWAxsB9BFGb5alsggpQ2xQ8N-HMRwt8bU2gjWLJq9m--0AYtERPhPvDzV4NAIf_wXaraTMblnQCBnGqvliWVMk3WHgxPJjJ5-KPwg0-i7yFSlpKbtEn0YQIo2wEWjtNGu1psz4nW5-vb1hzSwQQOqMQ3ZMijQqIFcYt59CxGKQUi2I5yzURdYXFabWBPb1gVjJwmfYwziuWC6XHQyw_-_e55Ug +AUTH_DSS_CLIENT_SECRET=eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJhdWQiOiJsb2NhbGhvc3QiLCJleHAiOjE3NzEyNjAxNzgsImlzcyI6ImxvY2FsaG9zdCIsInNjb3BlIjoiZHNzLnJlYWQuaWRlbnRpZmljYXRpb25fc2VydmljZV9hcmVhcyIsInN1YiI6ImZha2VfdXNzIn0.SItbEdY1_Q0nGCZVD-cFzri3i9H1-51jVUSuXHSIyVCy0aCrHJh_gI2su5Upbx_nFs5NmZCLvYO_zxyUsmjHG-FvTkYRW8ggCWXapeaKMdNcuCtjaKyr8iYJnrBgLWAxsB9BFGb5alsggpQ2xQ8N-HMRwt8bU2gjWLJq9m--0AYtERPhPvDzV4NAIf_wXaraTMblnQCBnGqvliWVMk3WHgxPJjJ5-KPwg0-i7yFSlpKbtEn0YQIo2wEWjtNGu1psz4nW5-vb1hzSwQQOqMQ3ZMijQqIFcYt59CxGKQUi2I5yzURdYXFabWBPb1gVjJwmfYwziuWC6XHQyw_-_e55Ug +DSS_BASE_URL=http://flight-dss:8082 +DSS_AUTH_URL=http://flight-dss-auth:8085 +DSS_AUTH_TOKEN_ENDPOINT=/token +DSS_USE_DUMMY_OAUTH=1 + + +OPENSKY_NETWORK_USERNAME=opensky +OPENSKY_NETWORK_PASSWORD=opensky diff --git a/flight_blender/settings.py b/flight_blender/settings.py index ff577f5..34071f6 100644 --- a/flight_blender/settings.py +++ b/flight_blender/settings.py @@ -65,6 +65,7 @@ MIDDLEWARE = [ "django.middleware.security.SecurityMiddleware", + "whitenoise.middleware.WhiteNoiseMiddleware", "django.contrib.sessions.middleware.SessionMiddleware", "django.middleware.common.CommonMiddleware", "django.middleware.csrf.CsrfViewMiddleware", @@ -98,7 +99,7 @@ DATABASES = {} -USE_LOCAL_SQLITE_DATABASE = os.getenv("USE_LOCAL_SQLITE_DATABASE", 0) +USE_LOCAL_SQLITE_DATABASE = int(os.getenv("USE_LOCAL_SQLITE_DATABASE", 0)) if USE_LOCAL_SQLITE_DATABASE: DATABASES = { "default": { @@ -151,6 +152,13 @@ # https://docs.djangoproject.com/en/3.1/howto/static-files/ STATIC_URL = "/static/" +STATIC_ROOT = os.getenv("STATIC_ROOT", BASE_DIR / "staticfiles") + +STORAGES = { + "staticfiles": { + "BACKEND": "whitenoise.storage.CompressedManifestStaticFilesStorage", + }, +} DEFAULT_AUTO_FIELD = "django.db.models.AutoField" if DEBUG: @@ -158,6 +166,12 @@ else: BROKER_URL = os.getenv("REDIS_BROKER_URL", "redis://redis:6379/") +# Fix for Render.com Redis SSL: Celery requires ssl_cert_reqs parameter for rediss:// URLs +if BROKER_URL.startswith("rediss://") and "ssl_cert_reqs" not in BROKER_URL: + # Add ssl_cert_reqs parameter if not present + separator = "&" if "?" in BROKER_URL else "?" + BROKER_URL = f"{BROKER_URL}{separator}ssl_cert_reqs=CERT_NONE" + CHANNEL_LAYERS = { "default": { diff --git a/flight_declaration_operations/serializers.py b/flight_declaration_operations/serializers.py index 01091a7..da6a713 100644 --- a/flight_declaration_operations/serializers.py +++ b/flight_declaration_operations/serializers.py @@ -122,7 +122,7 @@ def validate_state(self, value: int) -> int: int: The validated state value. """ if self.instance and value not in list(OPERATOR_EVENT_LOOKUP.keys()): - raise serializers.ValidationError("An operator can only set the state to Activated (2), Contingent (4) or Ended (5) using this endpoint") + raise serializers.ValidationError("An operator can only set the state to Accepted (1), Activated (2), Contingent (4) or Ended (5) using this endpoint") current_state = self.instance.state event = OPERATOR_EVENT_LOOKUP[value] diff --git a/flight_declaration_operations/views.py b/flight_declaration_operations/views.py index 6b8b6b3..3751ec4 100644 --- a/flight_declaration_operations/views.py +++ b/flight_declaration_operations/views.py @@ -290,7 +290,9 @@ def set_operational_intent(request): ) else: if declaration_state == 0 and USSP_NETWORK_ENABLED: - submit_flight_declaration_to_dss_async.delay(flight_declaration_id=flight_declaration_id) + from django.db import transaction + + transaction.on_commit(lambda: submit_flight_declaration_to_dss_async.delay(flight_declaration_id=flight_declaration_id)) creation_response = FlightDeclarationCreateResponse( id=flight_declaration_id, @@ -398,7 +400,9 @@ def set_flight_declaration(request): ) else: if declaration_state == 0 and USSP_NETWORK_ENABLED: - submit_flight_declaration_to_dss_async.delay(flight_declaration_id=flight_declaration_id) + from django.db import transaction + + transaction.on_commit(lambda: submit_flight_declaration_to_dss_async.delay(flight_declaration_id=flight_declaration_id)) creation_response = FlightDeclarationCreateResponse( id=flight_declaration_id, @@ -688,7 +692,9 @@ def post(self, request, *args, **kwargs): ) else: if declaration_state == 0 and USSP_NETWORK_ENABLED: - submit_flight_declaration_to_dss_async.delay(flight_declaration_id=flight_declaration_id) + from django.db import transaction + + transaction.on_commit(lambda: submit_flight_declaration_to_dss_async.delay(flight_declaration_id=flight_declaration_id)) creation_response = FlightDeclarationCreateResponse( id=flight_declaration_id, diff --git a/pyproject.toml b/pyproject.toml index 87eb7e0..081d806 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,6 +55,7 @@ dependencies = [ "channels==4.3.1", "channels-redis==4.3.0", "loguru==0.7.3", + "whitenoise==6.9.0", ] [dependency-groups] diff --git a/render-no-docker.yaml b/render-no-docker.yaml new file mode 100644 index 0000000..d4417f2 --- /dev/null +++ b/render-no-docker.yaml @@ -0,0 +1,115 @@ +services: + - type: web + name: flight-blender-web + env: python + buildCommand: pip install uv && uv sync --frozen --no-dev && python manage.py collectstatic --noinput && python manage.py migrate + startCommand: uvicorn flight_blender.asgi:application --host 0.0.0.0 --port $PORT --workers 3 + envVars: + - key: SECRET_KEY + generateValue: true + - key: IS_DEBUG + value: 0 + - key: USE_LOCAL_SQLITE_DATABASE + value: 0 + - key: ALLOWED_HOSTS + fromService: + type: web + name: flight-blender-web + property: host + - key: DATABASE_URL + fromDatabase: + name: flight-blender-db + property: connectionString + - key: REDIS_HOST + fromService: + type: redis + name: flight-blender-redis + property: host + - key: REDIS_PORT + fromService: + type: redis + name: flight-blender-redis + property: port + - key: REDIS_PASSWORD + fromService: + type: redis + name: flight-blender-redis + property: password + - key: REDIS_BROKER_URL + fromService: + type: redis + name: flight-blender-redis + property: connectionString + - key: FLIGHTBLENDER_FQDN + fromService: + type: web + name: flight-blender-web + property: host + - key: HEARTBEAT_RATE_SECS + value: 2 + - key: USSP_NETWORK_ENABLED + value: 0 + - key: DSS_SELF_AUDIENCE + fromService: + type: web + name: flight-blender-web + property: host + - key: PYTHONPATH + value: /opt/render/project/src + + - type: worker + name: flight-blender-worker + env: python + buildCommand: pip install uv && uv sync --frozen --no-dev + startCommand: celery --app=flight_blender worker --loglevel=info + envVars: + - key: SECRET_KEY + fromService: + type: web + name: flight-blender-web + property: envVar + value: SECRET_KEY + - key: IS_DEBUG + value: 0 + - key: USE_LOCAL_SQLITE_DATABASE + value: 0 + - key: DATABASE_URL + fromDatabase: + name: flight-blender-db + property: connectionString + - key: REDIS_HOST + fromService: + type: redis + name: flight-blender-redis + property: host + - key: REDIS_PORT + fromService: + type: redis + name: flight-blender-redis + property: port + - key: REDIS_PASSWORD + fromService: + type: redis + name: flight-blender-redis + property: password + - key: REDIS_BROKER_URL + fromService: + type: redis + name: flight-blender-redis + property: connectionString + - key: HEARTBEAT_RATE_SECS + value: 2 + - key: USSP_NETWORK_ENABLED + value: 0 + - key: PYTHONPATH + value: /opt/render/project/src + + - type: redis + name: flight-blender-redis + plan: free + +databases: + - name: flight-blender-db + databaseName: flight_blender + user: flight_blender_user + plan: free diff --git a/render.yaml b/render.yaml new file mode 100644 index 0000000..e863bf6 --- /dev/null +++ b/render.yaml @@ -0,0 +1,112 @@ +services: + - type: web + name: flight-blender-web + env: docker + dockerfilePath: Dockerfile + dockerContext: . + envVars: + - key: SECRET_KEY + generateValue: true + - key: IS_DEBUG + value: 0 + - key: USE_LOCAL_SQLITE_DATABASE + value: 0 + - key: ALLOWED_HOSTS + fromService: + type: web + name: flight-blender-web + property: host + - key: DATABASE_URL + fromDatabase: + name: flight-blender-db + property: connectionString + - key: REDIS_HOST + fromService: + type: redis + name: flight-blender-redis + property: host + - key: REDIS_PORT + fromService: + type: redis + name: flight-blender-redis + property: port + - key: REDIS_PASSWORD + fromService: + type: redis + name: flight-blender-redis + property: password + - key: REDIS_BROKER_URL + fromService: + type: redis + name: flight-blender-redis + property: connectionString + - key: FLIGHTBLENDER_FQDN + fromService: + type: web + name: flight-blender-web + property: host + - key: HEARTBEAT_RATE_SECS + value: 2 + - key: USSP_NETWORK_ENABLED + value: 0 + - key: DSS_SELF_AUDIENCE + fromService: + type: web + name: flight-blender-web + property: host + + - type: worker + name: flight-blender-worker + env: docker + dockerfilePath: Dockerfile + dockerContext: . + dockerCommand: ./entrypoints/docker-entrypoint-worker.sh + envVars: + - key: SECRET_KEY + fromService: + type: web + name: flight-blender-web + property: envVar + value: SECRET_KEY + - key: IS_DEBUG + value: 0 + - key: USE_LOCAL_SQLITE_DATABASE + value: 0 + - key: DATABASE_URL + fromDatabase: + name: flight-blender-db + property: connectionString + - key: REDIS_HOST + fromService: + type: redis + name: flight-blender-redis + property: host + - key: REDIS_PORT + fromService: + type: redis + name: flight-blender-redis + property: port + - key: REDIS_PASSWORD + fromService: + type: redis + name: flight-blender-redis + property: password + - key: REDIS_BROKER_URL + fromService: + type: redis + name: flight-blender-redis + property: connectionString + - key: HEARTBEAT_RATE_SECS + value: 2 + - key: USSP_NETWORK_ENABLED + value: 0 + + - type: redis + name: flight-blender-redis + plan: free + +databases: + - name: flight-blender-db + databaseName: flight_blender + user: flight_blender_user + plan: free diff --git a/rid_operations/dss_rid_helper.py b/rid_operations/dss_rid_helper.py index f08bffd..5b088d0 100644 --- a/rid_operations/dss_rid_helper.py +++ b/rid_operations/dss_rid_helper.py @@ -70,7 +70,7 @@ class RemoteIDOperations: def __init__(self): - self.dss_base_url = env.get("DSS_BASE_URL", "000") + self.dss_base_url = env.get("DSS_BASE_URL", "000").rstrip("/") + "/" self.r = get_redis() def compute_polygon_area(self, polygon: Polygon): diff --git a/rid_operations/views.py b/rid_operations/views.py index a892694..f8e52b8 100644 --- a/rid_operations/views.py +++ b/rid_operations/views.py @@ -2,6 +2,7 @@ import json import time import uuid +import dataclasses from dataclasses import asdict from datetime import timedelta from typing import Any @@ -71,6 +72,8 @@ class RIDOutputHelper: def make_json_compatible(self, struct: Any) -> Any: if isinstance(struct, tuple) and hasattr(struct, "_asdict"): return {k: self.make_json_compatible(v) for k, v in struct._asdict().items()} + elif dataclasses.is_dataclass(struct) and not isinstance(struct, type): + return {k: self.make_json_compatible(v) for k, v in asdict(struct).items()} elif isinstance(struct, dict): return {k: self.make_json_compatible(v) for k, v in struct.items()} elif isinstance(struct, str): @@ -116,8 +119,7 @@ def create_new_rid_subscription( subscription_duration_seconds=subscription_duration_seconds, is_simulated=is_simulated, ) - subscription_response = self.my_rid_output_helper.make_json_compatible(subscription_r) - return subscription_response + return subscription_r def start_ussp_polling(self): """ diff --git a/runtime.txt b/runtime.txt new file mode 100644 index 0000000..e497a7f --- /dev/null +++ b/runtime.txt @@ -0,0 +1 @@ +python-3.12.12 diff --git a/scd_operations/dss_scd_helper.py b/scd_operations/dss_scd_helper.py index f74d01d..dd52f99 100644 --- a/scd_operations/dss_scd_helper.py +++ b/scd_operations/dss_scd_helper.py @@ -705,7 +705,7 @@ def parse_operational_intent_reference_from_dss(self, operational_intent_referen class SCDOperations: def __init__(self): - self.dss_base_url = env.get("DSS_BASE_URL", "0") + self.dss_base_url = env.get("DSS_BASE_URL", "0").rstrip("/") + "/" self.r = get_redis() self.database_reader = FlightBlenderDatabaseReader() self.database_writer = FlightBlenderDatabaseWriter() @@ -838,6 +838,8 @@ def get_nearby_operational_intents(self, volumes: list[Volume4D]) -> list[Operat uss_op_int_id=current_uss_operational_intent_detail.id ) ) + op_int_details_retrieved = False + continue op_int_details_retrieved = True else: # This operational intent details is from a peer uss, need to query peer USS diff --git a/scd_operations/opint_helper.py b/scd_operations/opint_helper.py index 85e70b1..ca8556c 100644 --- a/scd_operations/opint_helper.py +++ b/scd_operations/opint_helper.py @@ -38,6 +38,9 @@ def __init__(self, flight_declaration_id: str): def validate_flight_declaration_start_end_time(self) -> bool: flight_declaration = self.my_database_reader.get_flight_declaration_by_id(flight_declaration_id=self.flight_declaration_id) + if not flight_declaration: + logger.error(f"Flight Declaration with ID {self.flight_declaration_id} not found in database, cannot validate start/end time") + return False # check that flight declaration start and end time is in the next two hours now = arrow.now() two_hours_from_now = now.shift(hours=2) diff --git a/uv.lock b/uv.lock index 097e556..e8e5598 100644 --- a/uv.lock +++ b/uv.lock @@ -577,6 +577,7 @@ dependencies = [ { name = "uas-standards" }, { name = "uvicorn", extra = ["standard"] }, { name = "wait-for-it" }, + { name = "whitenoise" }, ] [package.dev-dependencies] @@ -630,6 +631,7 @@ requires-dist = [ { name = "uas-standards", specifier = "==3.4.0" }, { name = "uvicorn", extras = ["standard"], specifier = "==0.37.0" }, { name = "wait-for-it", specifier = "==2.3.0" }, + { name = "whitenoise", specifier = "==6.9.0" }, ] [package.metadata.requires-dev] @@ -2157,6 +2159,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fa/a8/5b41e0da817d64113292ab1f8247140aac61cbf6cfd085d6a0fa77f4984f/websockets-15.0.1-py3-none-any.whl", hash = "sha256:f7a866fbc1e97b5c617ee4116daaa09b722101d4a3c170c787450ba409f9736f", size = 169743, upload-time = "2025-03-05T20:03:39.41Z" }, ] +[[package]] +name = "whitenoise" +version = "6.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b9/cf/c15c2f21aee6b22a9f6fc9be3f7e477e2442ec22848273db7f4eb73d6162/whitenoise-6.9.0.tar.gz", hash = "sha256:8c4a7c9d384694990c26f3047e118c691557481d624f069b7f7752a2f735d609", size = 25920, upload-time = "2025-02-06T22:16:34.957Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/b2/2ce9263149fbde9701d352bda24ea1362c154e196d2fda2201f18fc585d7/whitenoise-6.9.0-py3-none-any.whl", hash = "sha256:c8a489049b7ee9889617bb4c274a153f3d979e8f51d2efd0f5b403caf41c57df", size = 20161, upload-time = "2025-02-06T22:16:32.589Z" }, +] + [[package]] name = "win32-setctime" version = "1.2.0"