diff --git a/.github/workflows/build-and-publish-image.yml b/.github/workflows/build-and-publish-image.yml new file mode 100644 index 0000000..38a01e3 --- /dev/null +++ b/.github/workflows/build-and-publish-image.yml @@ -0,0 +1,49 @@ +name: Build and publish mida-portal image + +on: + push: + branches: [main, dockerdecouple] + workflow_dispatch: + +env: + BASE_TAG: latest + IMAGE_NAME: ghcr.io/ecotrust/mida-portal + +jobs: + build-and-push: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract short SHA + id: meta + run: echo "sha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: . + file: docker/Dockerfile + push: true + build-args: | + BASE_TAG=${{ env.BASE_TAG }} + tags: | + ${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.sha }} + ${{ env.IMAGE_NAME }}:latest + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.gitignore b/.gitignore index 586f65b..314beba 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,6 @@ *.pyc wcoa.egg-info/PKG-INFO deploy/sync_prod_content.sh +docker/.env +docker/static/ +docker/media/ diff --git a/README.md b/README.md index f4fbade..2445ead 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,15 @@ Mid-Atlantic Ocean Data Portal Django web application and Wagtail CMS for the Mid-Atlantic Ocean Data Portal. +## Docker quickstart + +Use the split base/overlay workflow. + +```bash +cp docker/.env.example docker/.env +task base build init +``` + ## Local Vagrant Development Env requirements: diff --git a/Taskfile.yml b/Taskfile.yml new file mode 100644 index 0000000..d617eac --- /dev/null +++ b/Taskfile.yml @@ -0,0 +1,49 @@ +version: '3' + +vars: + CORE: ../../madrona-portal + COMPOSE: docker compose -f docker/compose.yml -f {{.CORE}}/docker/compose.base.yml --env-file docker/.env + +tasks: + base: + desc: build the platform base image locally + cmds: + - sh -c 'if [ -f docker/.env ]; then set -a; . docker/.env; set +a; fi; docker build -t ghcr.io/ecotrust/madrona-portal:${BASE_TAG:-latest} -f {{.CORE}}/docker/Dockerfile {{.CORE}}/../' + + build: + desc: build the mida portal image + cmds: + - '{{.COMPOSE}} build' + + up: + desc: start mida services + cmds: + - '{{.COMPOSE}} up' + + init: + desc: first run with migrations, fixtures, and superuser bootstrap + cmds: + - DB_INIT=1 {{.COMPOSE}} up + + down: + desc: stop mida services + cmds: + - '{{.COMPOSE}} down' + + logs: + desc: tail app logs + cmds: + - '{{.COMPOSE}} logs -f app' + + manage: + desc: run a Django manage.py command (pass with -- cmd="...") + requires: + vars: + - cmd + cmds: + - '{{.COMPOSE}} exec app python marco/manage.py {{.cmd}}' + + shell: + desc: open Django shell + cmds: + - '{{.COMPOSE}} exec app python marco/manage.py shell' \ No newline at end of file diff --git a/docker/.env.example b/docker/.env.example new file mode 100644 index 0000000..8d1db74 --- /dev/null +++ b/docker/.env.example @@ -0,0 +1,26 @@ +COMPOSE_PROJECT_NAME=mida +BASE_TAG=latest + +APP_PORT=8001 +DB_PORT=5433 + +SECRET_KEY= +DEBUG=True +ALLOWED_HOSTS=localhost,127.0.0.1,::1 +DB_ENGINE=django.contrib.gis.db.backends.postgis +DB_NAME=mida_docker_db +DB_USER=postgres +DB_PASSWORD= +REDIS_PASSWORD= + +DB_INIT=0 +DJANGO_SUPERUSER_USERNAME=admin +DJANGO_SUPERUSER_EMAIL=admin@example.com +DJANGO_SUPERUSER_PASSWORD= +DJANGO_ENV=development +GUNICORN_WORKERS=3 + +RECAPTCHA_PUBLIC_KEY= +RECAPTCHA_PRIVATE_KEY= +GA_ACCOUNT= +ARCGIS_API_KEY= \ No newline at end of file diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..d53ce2c --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,13 @@ +ARG BASE_IMAGE=ghcr.io/ecotrust/madrona-portal +ARG BASE_TAG=latest +FROM ${BASE_IMAGE}:${BASE_TAG} + +# Build context is mida-portal repo root +COPY --chown=madrona_user:madrona_user . ./apps/mida-portal + +RUN pip install --no-deps -e ./apps/mida-portal && \ + if [ -s ./apps/mida-portal/docker/requirements.txt ]; then \ + pip install -r ./apps/mida-portal/docker/requirements.txt; \ + fi + +ENV MP_PROJECT_CONFIG=/usr/local/apps/madrona-portal/apps/mida-portal/docker/config.mida.docker.ini \ No newline at end of file diff --git a/docker/compose.yml b/docker/compose.yml new file mode 100644 index 0000000..31fda2c --- /dev/null +++ b/docker/compose.yml @@ -0,0 +1,100 @@ +services: + app: + image: mida-portal:dev + build: + context: .. + dockerfile: docker/Dockerfile + args: + BASE_TAG: ${BASE_TAG:-latest} + env_file: + - .env + environment: + - MP_PROJECT_CONFIG=/usr/local/apps/madrona-portal/apps/mida-portal/docker/config.mida.docker.ini + volumes: + - ./static:/vol/web/static + - ./media:/usr/local/apps/madrona-portal/media + - type: bind + source: ../../../madrona-portal/marco + target: /usr/local/apps/madrona-portal/marco + bind: + create_host_path: false + - type: bind + source: .. + target: /usr/local/apps/madrona-portal/apps/mida-portal + bind: + create_host_path: false + - type: bind + source: ../../mp-data-manager + target: /usr/local/apps/madrona-portal/apps/mp-data-manager + bind: + create_host_path: false + - type: bind + source: ../../mp-layers + target: /usr/local/apps/madrona-portal/apps/mp-layers + bind: + create_host_path: false + - type: bind + source: ../../mp-accounts + target: /usr/local/apps/madrona-portal/apps/mp-accounts + bind: + create_host_path: false + - type: bind + source: ../../mp-drawing + target: /usr/local/apps/madrona-portal/apps/mp-drawing + bind: + create_host_path: false + - type: bind + source: ../../mp-visualize + target: /usr/local/apps/madrona-portal/apps/mp-visualize + bind: + create_host_path: false + - type: bind + source: ../../madrona-features + target: /usr/local/apps/madrona-portal/apps/madrona-features + bind: + create_host_path: false + - type: bind + source: ../../madrona-manipulators + target: /usr/local/apps/madrona-portal/apps/madrona-manipulators + bind: + create_host_path: false + - type: bind + source: ../../madrona-scenarios + target: /usr/local/apps/madrona-portal/apps/madrona-scenarios + bind: + create_host_path: false + - type: bind + source: ../../mp-map-groups + target: /usr/local/apps/madrona-portal/apps/mp-map-groups + bind: + create_host_path: false + - type: bind + source: ../../mp-explore + target: /usr/local/apps/madrona-portal/apps/mp-explore + bind: + create_host_path: false + - type: bind + source: ../../mp-proxy + target: /usr/local/apps/madrona-portal/apps/mp-proxy + bind: + create_host_path: false + - type: bind + source: ../../p97-nursery + target: /usr/local/apps/madrona-portal/apps/p97-nursery + bind: + create_host_path: false + - type: bind + source: ../../django_url_shortener + target: /usr/local/apps/madrona-portal/apps/django_url_shortener + bind: + create_host_path: false + - type: bind + source: ../../madrona-analysistools + target: /usr/local/apps/madrona-portal/apps/madrona-analysistools + bind: + create_host_path: false + - type: bind + source: ../../mp-survey + target: /usr/local/apps/madrona-portal/apps/mp-survey + bind: + create_host_path: false \ No newline at end of file diff --git a/docker/config.mida.docker.ini b/docker/config.mida.docker.ini new file mode 100644 index 0000000..e91f02f --- /dev/null +++ b/docker/config.mida.docker.ini @@ -0,0 +1,88 @@ +# Docker-focused configuration for mida local development. +# Use with MP_PROJECT_CONFIG=config.mida.docker.ini + +[APP] +APP_NAME = Mid-Atlantic Ocean Data Portal +APP_URL = +APP_TEAM_NAME = Marine Planner Team +PROJECT_APP = mida +PROJECT_SETTINGS_FILE = True +DEBUG = True +TEMPLATE_DEBUG = True +ALLOWED_HOSTS = ["localhost", "127.0.0.1", "::1"] +# SECRET_KEY is loaded from environment variable: SECRET_KEY +MEDIA_ROOT = /usr/local/apps/madrona-portal/media +MEDIA_URL = /media/ +TIME_ZONE = UTC +GA_ACCOUNT = +# ReCAPTCHA keys are loaded from env vars: RECAPTCHA_PUBLIC_KEY, RECAPTCHA_PRIVATE_KEY +STATIC_ROOT = /vol/web/static +EMAIL_SUBJECT_PREFIX = [MidA] +MAP_LIBRARY = ol8 +COMPRESS_ENABLED = True +STATIC_CORE = /vol/web/static/ +ADDITIONAL_APPS = [] +ADDITIONAL_MIDDLEWARE = [] + +[REGION] +# TODO(human): verify against prod config.ini/local_settings.py +NAME = Mid-Atlantic +# TODO(human): verify against prod config.ini/local_settings.py +INIT_ZOOM = 7 +# TODO(human): verify against prod config.ini/local_settings.py +INIT_LAT = 38.5 +# TODO(human): verify against prod config.ini/local_settings.py +INIT_LON = -73.5 +# TODO(human): verify against prod config.ini/local_settings.py +MAP = ocean + +[CACHES] +BACKEND = django_redis.cache.RedisCache +LOCATION = redis://tasks:6379/1 +CLIENT_CLASS = django_redis.client.DefaultClient + +[CELERY] +CELERY_RESULT_BACKEND = redis://tasks:6379/1 +CELERY_BROKER_URL = redis://tasks:6379/0 +CELERY_ALWAYS_EAGER = False +CELERY_DISABLE_RATE_LIMITS = True + +[DATABASE] +ENGINE = django.contrib.gis.db.backends.postgis +NAME = mida_docker_db +HOST = db +PORT = 5432 +USER = postgres +# DB password is loaded from environment variable: DB_PASSWORD + +[EMAIL] +HOST = localhost +PORT = 25 +# SMTP credentials are loaded from env vars: EMAIL_HOST_USER, EMAIL_HOST_PASSWORD +# TODO(human): verify from-addresses against prod config.ini/local_settings.py +DEFAULT_FROM_EMAIL = Mid-Atlantic Portal +# TODO(human): verify from-addresses against prod config.ini/local_settings.py +SERVER_EMAIL = MidA Site Errors + +[AWS] +# AWS credentials are loaded from env vars: AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY +AWS_SES_REGION_NAME = us-east-1 +AWS_SES_REGION_ENDPOINT = email.us-east-1.amazonaws.com + +[SOCIAL_AUTH] +# TODO(human): verify social auth credentials/keys against prod config.ini/local_settings.py +# FACEBOOK_KEY= +# FACEBOOK_SECRET= +# TWITTER_KEY= +# TWITTER_SECRET= +# GOOGLE_KEY= +# GOOGLE_SECRET= + +[CATALOG] +# TODO(human): verify catalog settings against prod config.ini/local_settings.py +DATA_CATALOG_ENABLED = True +# TODO(human): verify catalog technology/source/query endpoint against prod +CATALOG_TECHNOLOGY = GeoPortal2 +CATALOG_PROXY = +CATALOG_SOURCE = +CATALOG_QUERY_ENDPOINT = \ No newline at end of file diff --git a/docker/requirements.txt b/docker/requirements.txt new file mode 100644 index 0000000..045cd20 --- /dev/null +++ b/docker/requirements.txt @@ -0,0 +1,2 @@ +# Project-only Python dependencies for mida Docker image. +# Leave empty unless mida requires packages not already in the base image. \ No newline at end of file diff --git a/mida/requirements.txt b/mida/requirements.txt index 5f125c0..ecc3c55 100644 --- a/mida/requirements.txt +++ b/mida/requirements.txt @@ -1,3 +1 @@ --e git+https://github.com/Ecotrust/mida-portal.git@master#egg=mida - # pip3 install selenium diff --git a/scripts/db-restore.sh b/scripts/db-restore.sh new file mode 100755 index 0000000..db5034e --- /dev/null +++ b/scripts/db-restore.sh @@ -0,0 +1,183 @@ +#!/usr/bin/env bash +# ----------------------------------------------------------------------------- +# db-restore.sh — Restore a PostgreSQL dump into the decoupled mida-portal Docker DB. +# +# Usage: +# ./scripts/db-restore.sh +# ./scripts/db-restore.sh --drop # drop & recreate DB first +# ./scripts/db-restore.sh --env-file +# ./scripts/db-restore.sh --core-compose +# ./scripts/db-restore.sh --project-compose +# +# Run from anywhere — this script always operates relative to madrona-apps/mida-portal/. +# +# Prerequisites: +# 1. Docker Compose stack is running with both decoupled compose files: +# docker compose \ +# -f ../../madrona-portal/docker/compose.base.yml \ +# -f docker/compose.yml \ +# --env-file docker/.env up -d +# 2. mida-portal/docker/.env exists and contains DB_NAME, DB_USER, DB_PASSWORD. +# +# Options: +# --drop Terminate all active connections, drop, and recreate the +# target database before restoring. Required for a clean +# import from prod. Without this flag the dump is applied +# on top of existing data. +# --env-file Path to the .env file (default: ./docker/.env). +# --core-compose +# Path to compose.base.yml +# (default: ../../madrona-portal/docker/compose.base.yml). +# --project-compose +# Path to mida-portal compose overlay +# (default: ./docker/compose.yml). +# +# Notes: +# - The dump is streamed directly into the db container (no temp files). +# - psql warnings (e.g. "already exists") are normal when importing a dump +# produced on a different Postgres version (12 → 16) and are not fatal. +# - After a --drop restore, run migrations to pick up any schema drift: +# docker compose -f -f --env-file \ +# exec app python marco/manage.py migrate +# ----------------------------------------------------------------------------- +set -euo pipefail + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- +die() { echo "[db-restore] ERROR: $*" >&2; exit 1; } +info() { echo "[db-restore] $*"; } + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" + +DEFAULT_ENV_FILE="$PROJECT_ROOT/docker/.env" +DEFAULT_CORE_COMPOSE="$PROJECT_ROOT/../../madrona-portal/docker/compose.base.yml" +DEFAULT_PROJECT_COMPOSE="$PROJECT_ROOT/docker/compose.yml" + +resolve_to_abs() { + local path="$1" + [[ -n "$path" ]] || return 1 + if [[ "$path" == ~* ]]; then + path="${path/#\~/$HOME}" + fi + echo "$(cd "$(dirname "$path")" && pwd)/$(basename "$path")" +} + +# --------------------------------------------------------------------------- +# Parse arguments +# --------------------------------------------------------------------------- +DROP_FIRST=false +DUMP_FILE="" +ENV_FILE="$DEFAULT_ENV_FILE" +CORE_COMPOSE_FILE="$DEFAULT_CORE_COMPOSE" +PROJECT_COMPOSE_FILE="$DEFAULT_PROJECT_COMPOSE" + +while [[ $# -gt 0 ]]; do + case "$1" in + --drop) DROP_FIRST=true; shift ;; + --env-file) [[ -n "${2:-}" ]] || die "--env-file requires a path argument" + ENV_FILE="$2"; shift 2 ;; + --core-compose) + [[ -n "${2:-}" ]] || die "--core-compose requires a path argument" + CORE_COMPOSE_FILE="$2"; shift 2 ;; + --project-compose) + [[ -n "${2:-}" ]] || die "--project-compose requires a path argument" + PROJECT_COMPOSE_FILE="$2"; shift 2 ;; + -*) die "Unknown option: '$1'. Usage: $0 [--drop] [--env-file ] [--core-compose ] [--project-compose ] " ;; + *) [[ -z "$DUMP_FILE" ]] || die "Unexpected argument: '$1'" + DUMP_FILE="$1"; shift ;; + esac +done + +[[ -n "$DUMP_FILE" ]] || die "Usage: $0 [--drop] [--env-file ] [--core-compose ] [--project-compose ] " + +# Resolve dump path before we cd away. +DUMP_ABS="$(resolve_to_abs "$DUMP_FILE")" +[[ -f "$DUMP_ABS" ]] || die "Dump file not found: $DUMP_FILE" + +ENV_FILE_ABS="$(resolve_to_abs "$ENV_FILE")" +[[ -f "$ENV_FILE_ABS" ]] || die "Env file not found: $ENV_FILE" + +CORE_COMPOSE_ABS="$(resolve_to_abs "$CORE_COMPOSE_FILE")" +[[ -f "$CORE_COMPOSE_ABS" ]] || die "Core compose file not found: $CORE_COMPOSE_FILE" + +PROJECT_COMPOSE_ABS="$(resolve_to_abs "$PROJECT_COMPOSE_FILE")" +[[ -f "$PROJECT_COMPOSE_ABS" ]] || die "Project compose file not found: $PROJECT_COMPOSE_FILE" + +# --------------------------------------------------------------------------- +# Always operate from the mida-portal repo regardless of where the script is called +# --------------------------------------------------------------------------- +cd "$PROJECT_ROOT" + +# --------------------------------------------------------------------------- +# Load .env for DB credentials and DJANGO_ENV +# --------------------------------------------------------------------------- +set -a +# shellcheck source=/dev/null +source "$ENV_FILE_ABS" +set +a + +DB_NAME="${DB_NAME:-mida_docker_db}" +DB_USER="${DB_USER:-postgres}" +DB_PASSWORD="${DB_PASSWORD:?DB_PASSWORD must be set in .env}" + +# --------------------------------------------------------------------------- +# Compose command helpers +# --------------------------------------------------------------------------- +compose() { + docker compose -f "$CORE_COMPOSE_ABS" -f "$PROJECT_COMPOSE_ABS" --env-file "$ENV_FILE_ABS" "$@" +} + +psql_exec() { + compose exec -T -e "PGPASSWORD=$DB_PASSWORD" db psql -U "$DB_USER" "$@" +} + +info "Using core compose: $CORE_COMPOSE_ABS" +info "Using project compose: $PROJECT_COMPOSE_ABS" +info "Using env file: $ENV_FILE_ABS" + +# --------------------------------------------------------------------------- +# Verify the db container is healthy before doing anything +# --------------------------------------------------------------------------- +info "Checking db service health..." +compose ps db | grep -q "healthy" \ + || die "db container is not healthy. Is the stack running? Try: docker compose -f $CORE_COMPOSE_ABS -f $PROJECT_COMPOSE_ABS --env-file $ENV_FILE_ABS up -d" + +# --------------------------------------------------------------------------- +# Optional: terminate connections, drop, and recreate the database +# --------------------------------------------------------------------------- +if [[ "$DROP_FIRST" == true ]]; then + info "Terminating active connections to '$DB_NAME'..." + psql_exec -d postgres -c \ + "SELECT pg_terminate_backend(pid) + FROM pg_stat_activity + WHERE datname = '$DB_NAME' AND pid <> pg_backend_pid();" \ + > /dev/null + + info "Dropping database '$DB_NAME'..." + psql_exec -d postgres -c "DROP DATABASE IF EXISTS \"$DB_NAME\";" + + info "Creating database '$DB_NAME'..." + psql_exec -d postgres -c "CREATE DATABASE \"$DB_NAME\";" + + info "Enabling PostGIS extension..." + psql_exec -d "$DB_NAME" -c "CREATE EXTENSION IF NOT EXISTS postgis;" +fi + +# --------------------------------------------------------------------------- +# Stream the dump into the database +# --------------------------------------------------------------------------- +DUMP_SIZE="$(du -sh "$DUMP_ABS" | cut -f1)" +info "Restoring '$DUMP_FILE' (${DUMP_SIZE}) → '$DB_NAME'..." +info "psql warnings about existing objects are expected and non-fatal." + +psql_exec -d "$DB_NAME" \ + --set ON_ERROR_STOP=off \ + < "$DUMP_ABS" + +info "Restore complete." +info "" +info "Next steps:" +info " Apply any pending migrations:" +info " docker compose -f $CORE_COMPOSE_ABS -f $PROJECT_COMPOSE_ABS --env-file $ENV_FILE_ABS exec app python marco/manage.py migrate"