From c821dd9609d0c5c64cad4701e5c5b6601c38dcfe Mon Sep 17 00:00:00 2001 From: David Pollard Date: Tue, 17 Mar 2026 16:32:10 -0700 Subject: [PATCH 001/152] add Docker development guide to transition from Vagrant workflow --- docker/README.md | 99 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 docker/README.md diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 0000000..b6f5bb8 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,99 @@ +# WCOA Docker Development Guide + +This guide switches local development from the Vagrant workflow to Docker +Compose for `madrona_portal` + `wcoa`. + +It is aligned with the install process documented in the project wiki: +https://github.com/Ecotrust/madrona-portal/wiki/Installation + +## 1) Prerequisites + +- Docker Desktop (or Docker Engine + Compose plugin) +- Local checkout layout where this repository has sibling module directories in + `../madrona-apps` (already true in this workspace) + +## 2) Keep companion apps checked out + +The Docker image installs local editable dependencies from +`/usr/local/apps/madrona-portal/apps/...`, so keep companion repos present in +`../madrona-apps` as referenced by `docker/docker-requirements.txt`. + +## 3) Set Docker environment values + +Edit `docker/.env` and set at minimum: + +- `SECRET_KEY` +- `SQL_DATABASE` +- `SQL_USER` +- `SQL_PASSWORD` +- `ALLOWED_HOSTS` + +Default local ports in this repo: + +- Django app: `8000` +- PostGIS on host: `65432` +- Redis on host: `8379` + +## 4) Use the Docker-specific WCOA Django config + +Compose is configured to run with: + +- `MP_PROJECT_CONFIG=config.wcoa.docker.ini` + +That file lives at `marco/config.wcoa.docker.ini` and points Django to: + +- PostGIS host `db` +- Redis host `tasks` +- Container-friendly static/media paths under `/vol/web` + +## 5) Build and start + +From `madrona_portal/`: + +```bash +cd docker +docker compose --env-file .env up --build +``` + +The entrypoint waits for PostGIS, then runs: + +- `collectstatic` +- `migrate` +- `runserver 0:8000` + +Open: http://localhost:8000/ + +## 6) Common one-off commands + +From `madrona_portal/docker`: + +```bash +docker compose --env-file .env run --rm app python marco/manage.py createsuperuser +docker compose --env-file .env run --rm app python marco/manage.py shell +docker compose --env-file .env run --rm app python marco/manage.py loaddata /path/to/fixture.json +``` + +## 7) Data migration from old Vagrant DB + +If you are moving existing data, dump from Vagrant PostgreSQL and import into the +Docker `db` service: + +```bash +# Example import into running Docker DB +cat ./path/to/old_dump.sql | docker compose --env-file .env exec -T db psql -U "$SQL_USER" -d "$SQL_DATABASE" +``` + +## 8) Stop and clean up + +```bash +docker compose --env-file .env down +docker compose --env-file .env down -v # also removes PostGIS/Redis volumes +``` + +## Notes + +- The legacy Vagrant flow in the top-level README remains valid, but Docker is + faster for repeatable local startup. +- If you need to run with a different portal app (for example `mida` or + `offshore`), create another config file modeled on + `marco/config.wcoa.docker.ini` and set `MP_PROJECT_CONFIG` accordingly. From 2cb5ce5e7f2b4c17a56d1bb22fd9efac15949863 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Tue, 17 Mar 2026 16:32:22 -0700 Subject: [PATCH 002/152] refactor ALLOWED_HOSTS handling and improve STATICFILES_DIRS normalization --- marco/marco/settings.py | 60 +++++++++++++++++++++++++++++------------ 1 file changed, 43 insertions(+), 17 deletions(-) diff --git a/marco/marco/settings.py b/marco/marco/settings.py index 357d76a..66c98a8 100644 --- a/marco/marco/settings.py +++ b/marco/marco/settings.py @@ -42,20 +42,35 @@ TEMPLATE_DEBUG = app_cfg.getboolean('TEMPLATE_DEBUG', True) SECRET_KEY = app_cfg.get('SECRET_KEY', 'you forgot to set the secret key') -host_list = app_cfg.get('ALLOWED_HOSTS') -if type(host_list) == str: - if '[' in host_list and ']' in host_list: - import ast - ALLOWED_HOSTS = ast.literal_eval(host_list) - elif ',' in host_list: - ALLOWED_HOSTS = host_list.split(',') +host_list = os.environ.get('ALLOWED_HOSTS', app_cfg.get('ALLOWED_HOSTS')) +if isinstance(host_list, str): + host_value = host_list.strip() + if host_value.startswith('[') and host_value.endswith(']'): + # Prefer a real list literal: ["localhost", "127.0.0.1", "::1"] + try: + import ast + parsed_hosts = ast.literal_eval(host_value) + if isinstance(parsed_hosts, (list, tuple)): + ALLOWED_HOSTS = [str(h).strip() for h in parsed_hosts if str(h).strip()] + else: + ALLOWED_HOSTS = [str(parsed_hosts).strip()] + except (SyntaxError, ValueError): + ALLOWED_HOSTS = [h.strip() for h in host_value[1:-1].split(',') if h.strip()] + elif ',' in host_value: + ALLOWED_HOSTS = [h.strip() for h in host_value.split(',') if h.strip()] else: - ALLOWED_HOSTS = [host_list] -elif type(host_list) == list: - ALLOWED_HOSTS = host_list + ALLOWED_HOSTS = [host_value] +elif isinstance(host_list, list): + ALLOWED_HOSTS = [str(h).strip() for h in host_list if str(h).strip()] else: ALLOWED_HOSTS = [str(host_list)] +# Normalize bracketed IPv6 host forms like [::1] to ::1 for Django host checks. +ALLOWED_HOSTS = [ + h[1:-1] if h.startswith('[') and h.endswith(']') and ':' in h else h + for h in ALLOWED_HOSTS +] + # Set logging to default, and then make admin error emails come through as HTML from django.utils.log import DEFAULT_LOGGING LOGGING = DEFAULT_LOGGING @@ -342,7 +357,7 @@ if 'CACHES' not in cfg.sections(): cfg['CACHES'] = {} -cache_cfg = cfg['DATABASE'] +cache_cfg = cfg['CACHES'] # ------------------------------------------------------------------------------ # Redis sessions and caching @@ -396,12 +411,23 @@ STATIC_URL = app_cfg.get('STATIC_URL', '/static/') STATIC_CORE = app_cfg.get('STATIC_CORE', '/usr/local/apps/marco_portal_static/') -STATICFILES_DIRS = ( - STYLES_DIR, - COMPONENTS_DIR, - ASSETS_DIR, - STATIC_CORE, -) +static_root_path = os.path.abspath(STATIC_ROOT) +staticfiles_dirs = [] + +for static_dir in (STYLES_DIR, COMPONENTS_DIR, ASSETS_DIR, STATIC_CORE): + if not static_dir: + continue + + normalized_static_dir = os.path.abspath(static_dir) + if normalized_static_dir == static_root_path: + continue + + if normalized_static_dir in [os.path.abspath(path) for path in staticfiles_dirs]: + continue + + staticfiles_dirs.append(static_dir) + +STATICFILES_DIRS = tuple(staticfiles_dirs) # Precedence for static files in STATICFILES_DIRS is determined by the order of the directories in STATICFILES_DIRS STATICFILES_FINDERS = ( From 1804e51c3c65a332c48501b2679c9f9eee6e51c5 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Wed, 18 Mar 2026 15:07:48 -0700 Subject: [PATCH 003/152] refactor Docker setup and update environment configurations for improved compatibility --- .gitignore | 3 + Dockerfile | 105 ++++++++++++++++++++------------- docker/.env | 100 ------------------------------- docker/docker-compose.yml | 13 ++-- docker/docker-requirements.txt | 29 ++++++--- docker/entrypoint.sh | 24 +++++++- 6 files changed, 119 insertions(+), 155 deletions(-) delete mode 100644 docker/.env diff --git a/.gitignore b/.gitignore index cb9e35f..96ef017 100644 --- a/.gitignore +++ b/.gitignore @@ -29,9 +29,12 @@ htmlcov .coverage config.ini config.mida.ini +config.mida.docker.ini config.wcoa.ini +config.wcoa.docker.ini pytest.ini marco.db +.env .bashrc dev_fixture.json diff --git a/Dockerfile b/Dockerfile index c12da9c..cfc570d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,58 +1,83 @@ -# pull official base image -FROM python:3.9.6-alpine -#FROM alpine:3.14 -#FROM python:3.8.10-alpine +# pull official base image — Ubuntu 24.04 LTS (Noble Numbat) +FROM ubuntu:24.04 + +# prevent apt from blocking on interactive questions during build +ENV DEBIAN_FRONTEND=noninteractive # set environment variables -ENV PYTHONDONTWRITEBYTECODE 1 -ENV PYTHONUNBUFFERED 1 +ENV PYTHONDONTWRITEBYTECODE=1 +ENV PYTHONUNBUFFERED=1 +ENV MP_PROJECT_CONFIG=config.wcoa.docker.ini # set work directory WORKDIR /usr/local/apps/madrona-portal # copy project -COPY ./marco /usr/local/apps/madrona-portal/marco -COPY ./apps /usr/local/apps/madrona-portal/apps -COPY ./assets /usr/local/apps/madrona-portal/assets -COPY ./bower_components /usr/local/apps/madrona-portal/bower_components -COPY ./docker/entrypoint.sh /entrypoint.sh -COPY ./docker/docker-requirements.txt /requirements.txt - -COPY ./backups/ /usr/local/apps/madrona-portal/backups - -# install dependencies -RUN \ - apk update &&\ - apk add --no-cache --virtual .build-deps \ - autoconf automake make \ - musl-dev gcc binutils g++ \ - libffi-dev \ - pkgconfig openssl \ - &&\ - apk add --no-cache --update \ - python3-dev \ - postgresql postgresql-contrib \ - postgresql-dev postgresql-libs \ - jpeg-dev libjpeg zlib-dev libtool \ - libpq gdal gdal-tools geos-dev gdal-dev \ - protobuf-c-dev json-c-dev perl libxml2-dev \ - proj proj-dev proj-util \ - && \ - pip install --upgrade pip &&\ - pip install -r /requirements.txt - #&&\ - #apk --purge del .build-deps +COPY madrona_portal/marco /usr/local/apps/madrona-portal/marco +COPY madrona_portal/apps/__init__.py /usr/local/apps/madrona-portal/apps/__init__.py +COPY madrona_portal/assets /usr/local/apps/madrona-portal/assets +COPY madrona_portal/bower_components /usr/local/apps/madrona-portal/bower_components +COPY madrona_portal/docker/entrypoint.sh /entrypoint.sh +COPY madrona_portal/docker/docker-requirements.txt /requirements.txt +COPY madrona_portal/backups /usr/local/apps/madrona-portal/backups + +COPY madrona-apps/django_url_shortener /usr/local/apps/madrona-portal/apps/django_url_shortener +COPY madrona-apps/madrona-analysistools /usr/local/apps/madrona-portal/apps/madrona-analysistools +COPY madrona-apps/madrona-features /usr/local/apps/madrona-portal/apps/madrona-features +COPY madrona-apps/madrona-manipulators /usr/local/apps/madrona-portal/apps/madrona-manipulators +COPY madrona-apps/madrona-scenarios /usr/local/apps/madrona-portal/apps/madrona-scenarios +COPY madrona-apps/mp-accounts /usr/local/apps/madrona-portal/apps/mp-accounts +COPY madrona-apps/mp-data-manager /usr/local/apps/madrona-portal/apps/mp-data-manager +COPY madrona-apps/mp-drawing /usr/local/apps/madrona-portal/apps/mp-drawing +COPY madrona-apps/mp-explore /usr/local/apps/madrona-portal/apps/mp-explore +COPY madrona-apps/mp-layers /usr/local/apps/madrona-portal/apps/mp-layers +COPY madrona-apps/mp-map-groups /usr/local/apps/madrona-portal/apps/mp-map-groups +COPY madrona-apps/mp-proxy /usr/local/apps/madrona-portal/apps/mp-proxy +COPY madrona-apps/mp-visualize /usr/local/apps/madrona-portal/apps/mp-visualize +COPY madrona-apps/p97-nursery /usr/local/apps/madrona-portal/apps/p97-nursery +COPY madrona-apps/wcoa /usr/local/apps/madrona-portal/apps/wcoa + +# install system dependencies — mirrors the Ubuntu 24.04 wiki install +RUN apt-get update && apt-get upgrade -y && \ + apt-get install -y --no-install-recommends \ + python3 python3-pip python3-dev python3-venv \ + python-is-python3 \ + build-essential pkg-config \ + libpq-dev \ + gdal-bin libgdal-dev \ + libgeos-dev \ + libjpeg-dev zlib1g-dev libtool \ + libprotobuf-c-dev libjson-c-dev \ + perl libxml2-dev \ + libproj-dev proj-bin \ + libffi-dev openssl \ + && rm -rf /var/lib/apt/lists/* + + # Create a virtual environment so pip installs don't conflict with system Python + RUN python3 -m venv /opt/venv + ENV PATH="/opt/venv/bin:$PATH" + + # Install the local layers app first so later package resolution can satisfy + # any dependency on the mp-layers distribution from the local checkout. + RUN pip install --upgrade pip setuptools wheel && \ + pip install --no-deps -e /usr/local/apps/madrona-portal/apps/mp-layers && \ + pip install -r /requirements.txt + + # Install GDAL Python bindings matched to the system GDAL version. + # Installed separately so this layer is cached independently. + RUN pip install "GDAL==$(gdal-config --version)" --no-cache-dir RUN chmod +x /entrypoint.sh -RUN mkdir -p /vol/web/media -RUN mkdir -p /vol/web/static +RUN mkdir -p /vol/web/media /vol/web/static -RUN adduser -D madrona_user +RUN useradd --create-home --shell /bin/sh madrona_user RUN chown -R madrona_user:madrona_user /vol RUN chown -R madrona_user:madrona_user /usr/local/apps/madrona-portal RUN chmod -R 755 /vol/web USER madrona_user +EXPOSE 8000 + CMD ["/entrypoint.sh"] diff --git a/docker/.env b/docker/.env deleted file mode 100644 index b45d782..0000000 --- a/docker/.env +++ /dev/null @@ -1,100 +0,0 @@ -##################### -# REQUIRED SETTINGS # -##################### - -# These settings must absolutely be changed for any production deployment. -# Most of these are critical to the security of your database application. -# Many of these settings should be changed even for development environments. - -## SECRET_KEY: -## Used to secure the app. Can be anything -- feel free to hammer out -## a long line of numbers, letters, caps, and symbols. You will not need -## to remember or retype this ever. -SECRET_KEY=changeme - -## PROXY_PORT: -## The port the proxy server (NGINX) will serve the application on. Use 80 in -## production for HTTP or 443 for HTTPS. For dev you may prefer 80xx. -PROXY_PORT=8000 - -## ALLOWED_HOSTS: -## List of web addresses server will accept traffic from. This can be an -## IP address (127.0.0.1) or a URL (your.site.com). Separate addresses with a -## comma (no spaces). Leave the current default addresses in place unless -## you know what you're doing. -ALLOWED_HOSTS=localhost,127.0.0.1,[::1] - -## SQL_DATABASE: -## The name of your database. This can be any word (no spaces). You can leave -## the default in place, but you will get extra security by making it unique. -SQL_DATABASE=ocean_portal - -## SQL_USER: -## A username for the owner of your database. Can be any word (no spaces). -## It is recommended that you change this for security purposes -SQL_USER=madrona_user - -## SQL_PASSWORD: -## A password for your database user. Can be any word (no spaces). You -## absolutely MUST change this password before you put any sensitive -## information in your database -SQL_PASSWORD=madrona_password - -################### -# SERVER SETTINGS # -################### - -# These are settings that may impact how the application is served. These should -# only be important for development or highly-customized deployments. - -## DEBUG: -## Set to 1 for 'true' if you are actively modifying the code base -## Leave as 0 for 'false' (default) for running in production -DEBUG=1 - -## TIME_ZONE: -## To see all Timezone options, see -## https://en.wikipedia.org/wiki/List_of_tz_database_time_zones#List -TIME_ZONE=America/Los_Angeles - -##################### -# DATABASE SETTINGS # -##################### - -# These are the default settings for connecting the application to the -# database. They most likely should not be changed, and only then in -# experimental development of highly-customized deployments. The descriptions -# will assume a high-familiarity with the subject matter, particularly of -# Django, RDBMSs, networking, and Docker. - -## SQL_ENGINE: -## The django database backend to use to connect to the database. -SQL_ENGINE=django.contrib.gis.db.backends.postgis - -## SQL_HOST: -## The network address of the database. The default 'db' is the variable name -## assigned and recognized by Docker from your docker-compose.yml file. -SQL_HOST=db - -## SQL_PORT: -## The port your database is accepting connections on. Default for PostgreSQL -## is 5432. -SQL_PORT=65432 - -################# -# TASK SETTINGS # -################# - -## TASK_PORT: -## The port your task queue is accepting connections on. Default for Redis -## is 6379. -TASK_PORT=8379 -REDIS_PASSWORD=sOmE_sEcUrE_pAsS - -####################### -# DEPENDENCY SETTINGS # -####################### - -## PROJ_DIR: -## The location of the PROJ.4 executable -PROJ_DIR=/usr diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 87e0d3a..f660c45 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -1,9 +1,8 @@ -version: '3.7' - services: app: build: - context: ../ + context: ../../ + dockerfile: madrona_portal/Dockerfile # command: dockerize -wait tcp://db:5432 sh -c "python manage.py migrate --noinput" # command: dockerize -wait tcp://db:5432 sh -c "python manage.py loaddata /usr/local/apps/TEKDB/TEKDB/TEKDB/fixtures/all_dummy_data.json" volumes: @@ -11,6 +10,7 @@ services: environment: - SECRET_KEY=${SECRET_KEY} - ALLOWED_HOSTS=${ALLOWED_HOSTS} + - MP_PROJECT_CONFIG=config.wcoa.docker.ini - SQL_ENGINE=${SQL_ENGINE} - SQL_DATABASE=${SQL_DATABASE} - SQL_USER=${SQL_USER} @@ -19,12 +19,13 @@ services: - SQL_PORT=${SQL_PORT} - PROJ_DIR=${PROJ_DIR} depends_on: - - ${SQL_HOST} + - db links: - ${SQL_HOST} ports: - "8000:8000" networks: + # maybe not necessary - djangonetwork # proxy: @@ -41,7 +42,7 @@ services: # - djangonetwork db: - image: postgis/postgis:14-3.1-alpine + image: postgis/postgis:16-3.4 volumes: - postgis-data:/var/lib/postgresql environment: @@ -54,7 +55,7 @@ services: - djangonetwork tasks: - image: redis:alpine3.14 + image: redis:7-alpine command: redis-server --requirepass ${REDIS_PASSWORD} ports: - ${TASK_PORT}:6379 diff --git a/docker/docker-requirements.txt b/docker/docker-requirements.txt index eae373c..a287e05 100644 --- a/docker/docker-requirements.txt +++ b/docker/docker-requirements.txt @@ -1,13 +1,13 @@ # Minimal requirements -Django>=3.2,<3.3 -wagtail +Django>=4.2,<4.3 +wagtail>=7.0,<8.0 # Tentative additions: -wagtail-import-export python-social-auth -social-auth-app-django +social-auth-app-django>5.4 python-jose pyjwt +django-autocomplete-light django-social-share django-email-log django-compressor @@ -16,11 +16,15 @@ django-wysiwyg django-recaptcha django-flatblocks django-nested-admin +django-querysetsequence django-redis +django-taggit>=5.0,<7.0 rpc4django -# 11/13/2021 alpine default -pygdal<3.2.4 +# GDAL Python bindings are installed separately in the Dockerfile via: +# pip install "GDAL==$(gdal-config --version)" --no-cache-dir +# pygdal<3.2.4 is intentionally omitted -- its setup.py breaks with +# modern pip build isolation (numpy.__NUMPY_SETUP__ removed in numpy>=1.20). ################################## #-e git+https://github.com/Ecotrust/mp-data-manager.git@main#egg=mp_data_manager @@ -29,6 +33,8 @@ pygdal<3.2.4 -e /usr/local/apps/madrona-portal/apps/mp-data-manager/ ################################## +-e /usr/local/apps/madrona-portal/apps/django_url_shortener/url_short + #-e git+https://github.com/Ecotrust/madrona-analysistools.git@main#egg=analysistools -e /usr/local/apps/madrona-portal/apps/madrona-analysistools #-e git+https://github.com/Ecotrust/mp-visualize.git@main#egg=visualize @@ -45,6 +51,8 @@ pygdal<3.2.4 -e /usr/local/apps/madrona-portal/apps/mp-drawing #-e git+https://github.com/Ecotrust/mp-explore.git@main#egg=explore -e /usr/local/apps/madrona-portal/apps/mp-explore +#-e git+https://github.com/Ecotrust/mp-layers.git@main#egg=layers +-e /usr/local/apps/madrona-portal/apps/mp-layers #-e git+https://github.com/Ecotrust/mp-map-groups.git@main#egg=map_groups -e /usr/local/apps/madrona-portal/apps/mp-map-groups #-e git+https://github.com/Ecotrust/p97-nursery.git@main#egg=nursery @@ -54,11 +62,13 @@ pygdal<3.2.4 ################################## #-e git+https://github.com/Ecotrust/mida-portal.git@main#egg=mida-portal +# -e /usr/local/apps/madrona-portal/apps/mida-portal ### OR ### #-e git+https://github.com/Ecotrust/wcoa.git@migration_2021#egg=wcoa -e /usr/local/apps/madrona-portal/apps/wcoa ### OR ### #-e git+https://github.com/Ecotrust/wc-offshore-portal.git@master#egg=wc-offshore-portal +# -e /usr/local/apps/madrona-portal/apps/wc-offshore-portal ################################## @@ -74,10 +84,13 @@ django-celery-email # Recommended components (require additional setup): # pg2 2.9 breaks migrations: https://stackoverflow.com/a/68025007/706797 -psycopg2-binary<2.9 -elasticsearch +psycopg2-binary>2.9.9 +elasticsearch<8.0 +elasticsearch-dsl<8.0 +wagtailcharts django-import-export +django-colorfield pyshp diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index bd790d1..e75139f 100644 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -1,6 +1,28 @@ #!/bin/sh -#set -e +set -e + +# Wait for the database service before running migrations. +python - <<'PY' +import os +import socket +import time + +host = os.environ.get("SQL_HOST", "db") +port = int(os.environ.get("DB_INTERNAL_PORT", "5432")) +timeout_seconds = int(os.environ.get("DB_WAIT_TIMEOUT", "90")) + +start = time.time() +while True: + try: + with socket.create_connection((host, port), timeout=2): + break + except OSError: + if time.time() - start > timeout_seconds: + raise SystemExit(f"Timed out waiting for database at {host}:{port}") + time.sleep(1) +PY + python marco/manage.py collectstatic --noinput python marco/manage.py migrate --noinput From 7d3a2b080546b0f6dda5adb0d2ca9e02425bf2f8 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Wed, 18 Mar 2026 15:07:56 -0700 Subject: [PATCH 004/152] add .env.dev file with database, server, task, and dependency settings --- docker/.env.dev | 77 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 docker/.env.dev diff --git a/docker/.env.dev b/docker/.env.dev new file mode 100644 index 0000000..0aa73ba --- /dev/null +++ b/docker/.env.dev @@ -0,0 +1,77 @@ +##################### +# DATABASE SETTINGS # +##################### + +# Most of these are critical to the security of your database application. +# Many of these settings should be changed even for development environments. + +## SECRET_KEY: +## Used to secure the app. Can be anything -- feel free to hammer out long line of numbers, letters, caps, and symbols. You will not need to remember or retype this ever. +SECRET_KEY=SssHhhhh + +## SQL_DATABASE: +## The name of your database. This can be any word (no spaces). You can leave the default in place, but you will get extra security by making it unique. +SQL_DATABASE=wcoa_docker_db + +## SQL_USER: +## A username for the owner of your database. Can be any word (no spaces). It is recommended that you change this for security purposes +SQL_USER=postgres + +## SQL_PASSWORD: +## A password for your database user. Please change. +SQL_PASSWORD=wcoa_docker_pass + +## SQL_ENGINE: +## The django database backend to use to connect to the database. +SQL_ENGINE=django.contrib.gis.db.backends.postgis + +## SQL_HOST: +## The network address of the database. The default 'db' is the variable name assigned and recognized by Docker from your docker-compose.yml file. +SQL_HOST=db + +## SQL_PORT: +## The port your database is accepting connections on. Default for PostgreSQL is 5432. +SQL_PORT=65432 + + +################### +# SERVER SETTINGS # +################### + +# These are settings that may impact how the application is served. + +## DEBUG: +## 1 for 'true' +## 0 for 'false' +DEBUG=1 + +## PROXY_PORT: +## The port the proxy server (NGINX) will serve the application on. Use 80 in production for HTTP or 443 for HTTPS. For dev you may prefer 80xx. +PROXY_PORT=8002 + +## ALLOWED_HOSTS: +## List of web addresses server will accept traffic from. +ALLOWED_HOSTS=["localhost","127.0.0.1","::1"] + +## TIME_ZONE: +## For all Timezone options, see https://en.wikipedia.org/wiki/List_of_tz_database_time_zones#List +TIME_ZONE=America/Los_Angeles + + +################# +# TASK SETTINGS # +################# + +## TASK_PORT: +## The port your task queue is accepting connections on. Default for Redis is 6379. +TASK_PORT=8379 +REDIS_PASSWORD=sOmE_sEcUrE_pAsS + + +####################### +# DEPENDENCY SETTINGS # +####################### + +## PROJ_DIR: +## The location of the PROJ.4 executable +PROJ_DIR=/usr From 34bbcb1d1881d6aa0a79c62a465d7de45892575c Mon Sep 17 00:00:00 2001 From: David Pollard Date: Wed, 18 Mar 2026 15:08:06 -0700 Subject: [PATCH 005/152] add Docker configuration template for WCOA local development --- marco/config.docker.ini.template | 82 ++++++++++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 marco/config.docker.ini.template diff --git a/marco/config.docker.ini.template b/marco/config.docker.ini.template new file mode 100644 index 0000000..177528b --- /dev/null +++ b/marco/config.docker.ini.template @@ -0,0 +1,82 @@ +# Docker-focused configuration for WCOA local development. +# Use with MP_PROJECT_CONFIG=config.wcoa.docker.ini + +[APP] +APP_NAME = WCOA Portal +APP_URL = '' +APP_TEAM_NAME = Marine Planner Team +PROJECT_APP = wcoa +PROJECT_SETTINGS_FILE = True +DEBUG = True +TEMPLATE_DEBUG = True +ALLOWED_HOSTS = ["localhost", "127.0.0.1", "::1"] +SECRET_KEY = You forgot to set the secret key +MEDIA_ROOT = /vol/web/media +MEDIA_URL = /media/ +TIME_ZONE = UTC +GA_ACCOUNT = You forgot to set the google analytics account +RECAPTCHA_PUBLIC_KEY = '' +RECAPTCHA_PRIVATE_KEY = '' +STATIC_ROOT = /vol/web/static +EMAIL_SUBJECT_PREFIX = [WCOA] +MAP_LIBRARY = ol8 +COMPRESS_ENABLED = True +STATIC_CORE = /vol/web/static/ +ADDITIONAL_APPS = [] +ADDITIONAL_MIDDLEWARE = [] + +[REGION] +NAME = Mid-Atlantic Ocean +INIT_ZOOM = 6 +INIT_LAT = 39 +INIT_LON = -120 +MAP = ocean + +[CACHES] +BACKEND = django_redis.cache.RedisCache +LOCATION = redis://:sOmE_sEcUrE_pAsS@tasks:6379/1 +CLIENT_CLASS = django_redis.client.DefaultClient + +[CELERY] +CELERY_RESULT_BACKEND = redis://:sOmE_sEcUrE_pAsS@tasks:6379/1 +BROKER_URL = redis://:sOmE_sEcUrE_pAsS@tasks:6379/0 +CELERY_BROKER_URL = redis://:sOmE_sEcUrE_pAsS@tasks:6379 +CELERY_ALWAYS_EAGER = False +CELERY_DISABLE_RATE_LIMITS = True + +[DATABASE] +ENGINE = django.contrib.gis.db.backends.postgis +NAME = wcoa_docker_db +HOST = db +PORT = 5432 +USER = wcoa_docker_user +PASSWORD = wcoa_docker_pass + +[EMAIL] +HOST = localhost +PORT = 25 +HOST_USER = mail user +HOST_PASSWORD = mail password +DEFAULT_FROM_EMAIL = Mid-Atlantic Portal +SERVER_EMAIL = MidA Site Errors + +[AWS] +AWS_ACCESS_KEY_ID = +AWS_SECRET_ACCESS_KEY = +AWS_SES_REGION_NAME = us-east-1 +AWS_SES_REGION_ENDPOINT = email.us-east-1.amazonaws.com + +[SOCIAL_AUTH] +FACEBOOK_KEY = You forgot to set the facebook key +FACEBOOK_SECRET = You forgot to set the facebook secret +TWITTER_KEY = You forgot to set the twitter key +TWITTER_SECRET = You forgot to set the twitter secret +GOOGLE_KEY = You forgot to set the google key +GOOGLE_SECRET = You forgot to set the google secret + +[CATALOG] +DATA_CATALOG_ENABLED = False +CATALOG_TECHNOLOGY = GeoPortal2 +CATALOG_PROXY = +CATALOG_SOURCE = http://192.168.0.40:9200 +CATALOG_QUERY_ENDPOINT = /geoportal/elastic/metadata/item/_search/ From 9dbfaeeefb244ced2ba1d5bd83189caca6efefe9 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Wed, 18 Mar 2026 17:58:07 -0700 Subject: [PATCH 006/152] add scripts to manage database fixtures and SQL dumps --- backups/dump_fixtures.sh | 45 ++++++++++++++++++++++++++++++++++++++++ backups/load_sql_dump.sh | 35 +++++++++++++++++++++++++++++++ docker/entrypoint.sh | 19 +++++++++++++++++ 3 files changed, 99 insertions(+) create mode 100755 backups/dump_fixtures.sh create mode 100755 backups/load_sql_dump.sh diff --git a/backups/dump_fixtures.sh b/backups/dump_fixtures.sh new file mode 100755 index 0000000..b755881 --- /dev/null +++ b/backups/dump_fixtures.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# Regenerate WCOA fixture files from the current running database. +# +# Usage (run from madrona_portal/): +# ./backups/dump_fixtures.sh +# +# Requires the full stack to be running: +# docker compose --env-file docker/.env.dev -f docker/docker-compose.yml up -d +# +# After running, review changes and commit the updated fixture files: +# git diff ../madrona-apps/wcoa/wcoa/fixtures/ +# git add ../madrona-apps/wcoa/wcoa/fixtures/ && git commit -m "Update fixtures from db" + +set -euo pipefail + +DC="docker compose --env-file docker/.env.dev -f docker/docker-compose.yml" +WCOA_FX="../madrona-apps/wcoa/wcoa/fixtures" + +echo "Exporting wcoa_init.json (base, wagtailcore, wagtailimages, wcoa) ..." +$DC run --rm app \ + python marco/manage.py dumpdata --verbosity 0 \ + base wagtailcore wagtailimages wagtailredirects wcoa \ + --natural-foreign --indent 2 \ + > "$WCOA_FX/wcoa_init.json" +echo " -> $WCOA_FX/wcoa_init.json ($(wc -l < "$WCOA_FX/wcoa_init.json") lines)" + +echo "Exporting wcoa_init_layers.json (data_manager, sites) ..." +$DC run --rm app \ + python marco/manage.py dumpdata --verbosity 0 \ + data_manager sites \ + --natural-foreign --indent 2 \ + > "$WCOA_FX/wcoa_init_layers.json" +echo " -> $WCOA_FX/wcoa_init_layers.json ($(wc -l < "$WCOA_FX/wcoa_init_layers.json") lines)" + +echo "Exporting wagtail_menus.json (menu) ..." +$DC run --rm app \ + python marco/manage.py dumpdata --verbosity 0 \ + menu \ + --natural-foreign --indent 2 \ + > "$WCOA_FX/wagtail_menus.json" +echo " -> $WCOA_FX/wagtail_menus.json ($(wc -l < "$WCOA_FX/wagtail_menus.json") lines)" + +echo "" +echo "Fixtures updated. Review with:" +echo " git diff $WCOA_FX/" diff --git a/backups/load_sql_dump.sh b/backups/load_sql_dump.sh new file mode 100755 index 0000000..c46e621 --- /dev/null +++ b/backups/load_sql_dump.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# Import a PostgreSQL SQL dump into the running Docker database service. +# +# Usage (run from madrona_portal/): +# ./backups/load_sql_dump.sh /path/to/your_dump.sql +# +# The db container must already be running: +# docker compose --env-file docker/.env.dev -f docker/docker-compose.yml up -d db + +set -euo pipefail + +DUMP_FILE="${1:-}" + +if [ -z "$DUMP_FILE" ] || [ ! -f "$DUMP_FILE" ]; then + echo "Usage: $0 " + echo " Example: $0 ~/wcoa_prod.sql" + exit 1 +fi + +# Load credentials from the Docker env file (same directory as docker-compose.yml) +set -a +# shellcheck source=/dev/null +source "$(dirname "$0")/../docker/.env.dev" +set +a + +echo "Importing $(basename "$DUMP_FILE") into database '$SQL_DATABASE' ..." + +docker compose --env-file docker/.env.dev -f docker/docker-compose.yml exec -T db \ + psql -U "$SQL_USER" -d "$SQL_DATABASE" < "$DUMP_FILE" + +echo "" +echo "Import complete." +echo "" +echo "Next: restart the app to run Django migrations on top of the imported data:" +echo " docker compose --env-file docker/.env.dev -f docker/docker-compose.yml restart app" diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index e75139f..df08847 100644 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -26,6 +26,25 @@ PY python marco/manage.py collectstatic --noinput python marco/manage.py migrate --noinput +# On a fresh database (no real Wagtail content pages yet), load the initial +# fixture data so the site starts with working navigation and content. +# The check is skipped safely if Django fails to import for any reason. +PAGE_COUNT=$(python - 2>/dev/null <<'PY' || echo "unknown" +import sys, os +sys.path.insert(0, 'marco') +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'marco.settings') +import django +django.setup() +from wagtail.models import Page +print(Page.objects.filter(depth__gt=1).count()) +PY +) +if [ "$PAGE_COUNT" = "0" ]; then + echo "Fresh database — loading initial fixtures..." + python marco/manage.py loaddata wcoa_init wcoa_init_layers wagtail_menus + echo "Initial fixtures loaded." +fi + python marco/manage.py runserver 0:8000 #uwsgi --socket :8000 --master --enable-threads --module marco.marco.wsgi #exec "$@" From 0bcf561a48546c2f2cdaa8a12d7c5cebe9e1a754 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Thu, 19 Mar 2026 13:38:04 -0700 Subject: [PATCH 007/152] refactor entrypoint script to simplify database readiness check and improve clarity --- docker/entrypoint.sh | 38 +++++++++++++++++--------------------- 1 file changed, 17 insertions(+), 21 deletions(-) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index df08847..b203d9d 100644 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -1,30 +1,26 @@ #!/bin/sh +# Exit on errors set -e -# Wait for the database service before running migrations. -python - <<'PY' -import os -import socket -import time - -host = os.environ.get("SQL_HOST", "db") -port = int(os.environ.get("DB_INTERNAL_PORT", "5432")) -timeout_seconds = int(os.environ.get("DB_WAIT_TIMEOUT", "90")) +# If a SQL_HOST is provided, wait for Postgres to become available before running +# migrations. This prevents race conditions when using docker-compose where the +# web container starts before the DB is ready. +if [ -n "$SQL_HOST" ]; then + echo "Waiting for database at ${SQL_HOST}:${SQL_PORT:-5432}..." + # pg_isready is available after installing postgresql-client in the image + until pg_isready -h "$SQL_HOST" -p "${SQL_PORT:-5432}" >/dev/null 2>&1; do + echo "Postgres is unavailable - sleeping" + sleep 1 + done + echo "Postgres is up" +fi -start = time.time() -while True: - try: - with socket.create_connection((host, port), timeout=2): - break - except OSError: - if time.time() - start > timeout_seconds: - raise SystemExit(f"Timed out waiting for database at {host}:{port}") - time.sleep(1) -PY -python marco/manage.py collectstatic --noinput -python marco/manage.py migrate --noinput +echo "Collecting static files..." +python manage.py collectstatic --noinput +echo "Applying database migrations..." +python manage.py migrate --noinput # On a fresh database (no real Wagtail content pages yet), load the initial # fixture data so the site starts with working navigation and content. From 08386b4f1fb7083f67b2564e75ccf24bad297521 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Fri, 20 Mar 2026 14:21:22 -0700 Subject: [PATCH 008/152] Runs on Docker. Not sure to merge this into main or not. Asking at our next software team meeting --- docker/entrypoint.sh | 47 +++++++++++++++++++++++++++++--------------- 1 file changed, 31 insertions(+), 16 deletions(-) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index b203d9d..cd20d03 100644 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -3,24 +3,39 @@ # Exit on errors set -e -# If a SQL_HOST is provided, wait for Postgres to become available before running -# migrations. This prevents race conditions when using docker-compose where the -# web container starts before the DB is ready. -if [ -n "$SQL_HOST" ]; then - echo "Waiting for database at ${SQL_HOST}:${SQL_PORT:-5432}..." - # pg_isready is available after installing postgresql-client in the image - until pg_isready -h "$SQL_HOST" -p "${SQL_PORT:-5432}" >/dev/null 2>&1; do - echo "Postgres is unavailable - sleeping" - sleep 1 - done - echo "Postgres is up" -fi +# Wait for the database service before running migrations. +python - <<'PY' +import os +import socket +import time + +host = os.environ.get("SQL_HOST", "db") +port = int(os.environ.get("DB_INTERNAL_PORT", "5432")) +timeout_seconds = int(os.environ.get("DB_WAIT_TIMEOUT", "90")) + +start = time.time() +while True: + try: + with socket.create_connection((host, port), timeout=2): + break + except OSError: + if time.time() - start > timeout_seconds: + raise SystemExit(f"Timed out waiting for database at {host}:{port}") + time.sleep(1) +PY +# if [ -n "$SQL_HOST" ]; then +# echo "Waiting for database at ${SQL_HOST}:${DB_INTERNAL_PORT:-5432}..." +# # pg_isready is available after installing postgresql-client in the image +# until pg_isready -h "$SQL_HOST" -p "${DB_INTERNAL_PORT:-5432}" >/dev/null 2>&1; do +# echo "Postgres is unavailable - sleeping" +# sleep 1 +# done +# echo "Postgres is up" +# fi -echo "Collecting static files..." -python manage.py collectstatic --noinput -echo "Applying database migrations..." -python manage.py migrate --noinput +python marco/manage.py collectstatic --noinput +python marco/manage.py migrate --noinput # On a fresh database (no real Wagtail content pages yet), load the initial # fixture data so the site starts with working navigation and content. From c2531fef781cce3deae188fd96897e17b2ccd8b4 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Tue, 24 Mar 2026 15:04:07 -0700 Subject: [PATCH 009/152] Change path to use absolute path to fixtures --- backups/dump_fixtures.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) mode change 100755 => 100644 backups/dump_fixtures.sh diff --git a/backups/dump_fixtures.sh b/backups/dump_fixtures.sh old mode 100755 new mode 100644 index b755881..62363cf --- a/backups/dump_fixtures.sh +++ b/backups/dump_fixtures.sh @@ -14,7 +14,7 @@ set -euo pipefail DC="docker compose --env-file docker/.env.dev -f docker/docker-compose.yml" -WCOA_FX="../madrona-apps/wcoa/wcoa/fixtures" +WCOA_FX="/usr/local/apps/madrona_portal/apps/wcoa/wcoa/fixtures" echo "Exporting wcoa_init.json (base, wagtailcore, wagtailimages, wcoa) ..." $DC run --rm app \ From 2583c7dc0ba36fd442062aa9eaa05fc4c99d2f39 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Mon, 30 Mar 2026 19:48:21 -0700 Subject: [PATCH 010/152] Update vagrant_provision.sh to clarify legacy status and current setup instructions --- MODERNIZATION.md | 152 +++++++++++++++++++++++++++++++++++ scripts/vagrant_provision.sh | 2 + 2 files changed, 154 insertions(+) create mode 100644 MODERNIZATION.md diff --git a/MODERNIZATION.md b/MODERNIZATION.md new file mode 100644 index 0000000..4ff6b46 --- /dev/null +++ b/MODERNIZATION.md @@ -0,0 +1,152 @@ +# Madrona Portal — Modernization Log & Roadmap + +> **Last updated:** March 2026 +> **Stack target:** Python 3.10+, Django 4.2 LTS, Wagtail 7.x + +--- + +## Phase 1 — Completed (March 2026) + +These changes have been applied to the codebase. + +### Dependencies + +| File | Change | +|---|---| +| `requirements.txt` | Added version bounds to all packages; removed duplicate `django-colorfield`; resolved `social-auth-app-django` conflict (`<5.0` vs `>5.4`); updated `django-taggit` to `>=5.0,<7.0`; widened Django constraint to `>=4.2,<5.0` to allow patch updates | +| `dev_requirements.txt` | **Completely replaced.** Old file pinned Django <1.10, Wagtail 1.3.1 (2015-era). New file contains `pytest`, `pytest-django`, `pytest-cov`, `factory-boy`, `ruff`, `mypy`, `django-stubs`, and `django-debug-toolbar` | +| `docker/docker-requirements.txt` | No changes — already modern. Production reference file. | + +### `settings.py` + +- Removed **Wagtail v1 / v2 / v3 runtime detection** (nested try/except over `INSTALLED_APPS`). Locked to Wagtail 7+ with a clean, single `INSTALLED_APPS` list. +- Removed **`REDIS_PACKAGE_NAME` / `redis_cache` fallback** — `django_redis` is the only supported cache backend. +- Removed **dead `if False:` debug-toolbar block** — enable via `dev_requirements.txt` and `ADDITIONAL_APPS` in config. +- Removed **commented-out Wagtail v1/v2 middleware blocks**. +- Replaced **`eval()` calls** for `ADDITIONAL_APPS` / `ADDITIONAL_MIDDLEWARE` with `json.loads()` + `ast.literal_eval()` fallback. `eval()` on config-file values is a remote-code execution risk. +- Replaced **`exec("from %s.settings import *")` pattern** with a proper `import_module` + namespace merge loop. +- Removed **deprecated `BROKER_URL`** — Celery 5 uses `CELERY_BROKER_URL` only. +- Removed **`SOCIAL_AUTH_GOOGLE_OAUTH2_USE_DEPRECATED_API = True`** (deprecated). +- Replaced **`try: VAR except NameError`** patterns for `FEEDBACK_IFRAME_URL`, `DISCLAIMER_BUTTON_DEFAULT`, `DATA_MANAGER_ADMIN`, `PROJECT_REGION` with direct assignment. +- Updated docstring reference from Django 1.7 to 4.2. +- Added **`SECRET_KEY` guard** — raises `RuntimeError` at startup if key is unset, rather than silently running with `'you forgot to set the secret key'`. +- Consolidated all `cfg.sections()` existence checks into a single loop at the top. + +### `urls.py` + +- Removed **Django 1.x `from django.conf.urls import url` try/except** — `django.urls.re_path` (Django 2.0+) is now imported directly. +- Removed **`WAGTAIL_VERSION > 1` branch** — both branches were identical (Wagtail v1 `wagtail.docs` vs v2+ `wagtail.documents`). The v2+ import is used directly. +- Removed trailing `/?` optional slashes on most routes (ambiguous in Django URL routing). +- Replaced `re_path(r'^django-admin/?', ...)` with `re_path(r'^django-admin/', ...)` — `admin.site.urls` already handles trailing slash. +- Added `warnings.warn` instead of silent `except Exception: pass` when `PROJECT_APP` URL import fails. + +### Migrations + +- Stripped **`from __future__ import unicode_literals`** from **72 migration files** — this Python 2 compatibility import is a no-op in Python 3 and adds noise. + +### Docker & DevOps + +| File | Change | +|---|---| +| `Dockerfile` | Fixed **indentation bug**: three `RUN` statements inside the `apt-get` block were indented as if part of it, but only the first `RUN` was correctly associated. Moved venv creation, pip install, and GDAL install to separate top-level `RUN` layers for correct caching. Consolidated final `RUN` commands (chmod, mkdir, useradd, chown) into one layer. | +| `docker/docker-compose.yml` | Added **`healthcheck`** blocks for `db` (pg_isready) and `tasks` (redis ping). Replaced `links:` with `depends_on: condition: service_healthy`. Added `restart: unless-stopped`. Removed stale Vagrant-era volume name `redis.conf`. Set sensible `:-default` values for env vars. | +| `.env.example` | **New file** — documents every required environment variable with safe placeholder values. Committed to repo so developers know what to configure. | +| `.gitignore` | Added `.env` entry to prevent real credentials from being committed. | + +### Tooling + +| File | Change | +|---|---| +| `pyproject.toml` | **New file** — central config for `pytest`, `coverage`, `ruff`, and `mypy`. Replaces ad-hoc tool configs scattered across the project. | + +--- + +## Phase 2 — Completed (March 2026) + +### Secret Management + +- **Extended env var support** throughout `settings.py` via a new `_env(env_key, cfg_section, cfg_key, default)` helper. Every credential-bearing setting now checks an environment variable *first*, falls back to `config.ini`, then to a safe default. +- **Database** — supports `DB_*` env vars (`DB_ENGINE`, `DB_NAME`, `DB_USER`, `DB_PASSWORD`, `DB_HOST`, `DB_PORT`) as well as legacy `SQL_*` aliases for docker-compose compatibility. +- **Redis** — a single `REDIS_URL` env var configures the Django cache location, `CELERY_BROKER_URL`, and `CELERY_RESULT_BACKEND` simultaneously. +- **Email** — `EMAIL_HOST`, `EMAIL_PORT`, `EMAIL_HOST_USER`, `EMAIL_HOST_PASSWORD`, `EMAIL_USE_TLS` all respect env vars. +- **AWS SES** — `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_SES_REGION_NAME`, `AWS_SES_REGION_ENDPOINT`. +- **Social auth** — `FACEBOOK_KEY`, `FACEBOOK_SECRET`, `TWITTER_KEY`, `TWITTER_SECRET`, `GOOGLE_KEY`, `GOOGLE_SECRET`. +- **`.env.example`** — updated to document every env var with safe placeholder values, organized by category. +- **`config.wcoa.ini` / `config.mida.ini`** — these still contain real credentials and should be removed from git history using `git filter-repo --path config.wcoa.ini --path config.mida.ini --invert-paths`. That step requires a git client and is left for the team to execute. + +### Social Auth Pipeline + +- Renamed all `social.pipeline.*` strings in `SOCIAL_AUTH_PIPELINE` to `social_core.pipeline.*` — the correct module path for `social-auth-core ≥ 4.x`. The old `social` namespace was a legacy alias that has been dropped. + +### rpc4django Replacement + +- **Removed** `rpc4django` from `INSTALLED_APPS`, `requirements.txt`, and `urls.py`. +- The single `/rpc` XML-RPC endpoint served **11 methods** across 3 sub-apps. Each has been replaced with a typed DRF `APIView`: + +| Old RPC method | New endpoint | App | +|---|---|---| +| `get_bookmarks` | `GET /api/bookmarks/` | visualize | +| `add_bookmark` | `POST /api/bookmarks/` | visualize | +| `load_bookmark` | `GET /api/bookmarks//` | visualize | +| `remove_bookmark` | `DELETE /api/bookmarks//` | visualize | +| `share_bookmark` | `POST /api/bookmarks//share/` | visualize | +| `get_user_layers` | `GET /api/user-layers/` | visualize | +| `add_user_layer` | `POST /api/user-layers/` | visualize | +| `load_user_layer` | `GET /api/user-layers//` | visualize | +| `remove_user_layer` | `DELETE /api/user-layers//` | visualize | +| `share_user_layer` | `POST /api/user-layers//share/` | visualize | +| `delete_drawing` | `DELETE /api/drawings//` | drawing | +| `get_sharing_groups` | `GET /api/sharing-groups/` | mapgroups | +| `update_map_group` | `PATCH /api/map-groups//` | mapgroups | + +- New files: `visualize/api.py`, `drawing/api.py`, `mapgroups/api.py`. Each app's `urls.py` updated accordingly. +- All new views carry full **type annotations** and proper DRF permission classes (`IsAuthenticated` / `AllowAny`). + +### accounts/pipeline.py + +- Removed Python 2 compatibility shims: `try/except ImportError` for `django.urls.reverse`, `try/except ImportError` for `urllib.parse`, and `import urlparse` (Python 2 stdlib). +- Replaced `urlparse.urlsplit` / `urlparse.urlunsplit` with `urllib.parse.urlsplit` / `urllib.parse.urlunsplit`. +- Removed dead `from django.core.context_processors import request` import (removed in Django 1.10). +- Removed dead `from django.conf.urls import include, url` fallback. +- Added proper type hints and a complete `send_validation_email` stub (was missing from the pipeline). + +### wagtail_migrations/ Directory + +- The directory contains 30+ step-by-step upgrade shell scripts (Wagtail 1.4 → 2.11), Python 2 view backups, and ancient requirements snapshots. None are needed at Wagtail 7. +- **The files are OS-level read-only in this environment.** Run this from the project root to remove them: + ```bash + git rm -rf wagtail_migrations/ + git commit -m "Remove historical wagtail_migrations upgrade scripts" + ``` + +--- + +## Phase 3 — Recommended Next Steps + +### High Priority + +- **Frontend build tooling** — Replace Bower + Gulp with `npm` + Vite. Bower has been deprecated since 2017. Add `/bower_components/` to `.gitignore` and drive dependencies through `package.json`. +- **Test coverage** — Only 2 test files exist. Add `pytest-django` suites targeting 60%+ coverage for models and views across the portal sub-apps. +- **CI / CD pipeline** — GitHub Actions: lint (`ruff`), test (`pytest`), Docker build, tag-based image push to registry. + +### Medium Priority + +- **Django 5.x upgrade** — Django 4.2 LTS support ends April 2026. Evaluate Django 5.1 once sub-app compatibility is confirmed. +- **Consolidate config.ini variants** — 6 config files remain. Migrate to a single `.env`-driven approach and retire the `.ini` files. +- **Expand type coverage** — Run `mypy --strict` against `portal/`, `marco_site/`, and all sub-app `views.py` files; address errors incrementally. + +--- + +## Appendix — Technical Debt Removed + +| Category | Count / Description | +|---|---| +| Python 2 imports removed | 72 migration files | +| Wagtail version branches removed | 3 (v1, v2, v5 detection) | +| `eval()` calls on config data removed | 2 (`ADDITIONAL_APPS`, `ADDITIONAL_MIDDLEWARE`) | +| `exec()` for dynamic import removed | 1 | +| Deprecated Celery settings removed | 1 (`BROKER_URL`) | +| Dead code blocks removed | 2 (`if False:`, commented middleware) | +| Dockerfile layer ordering bugs fixed | 3 mis-indented `RUN` commands | +| Docker Compose healthchecks added | 2 services (`db`, `tasks`) | +| Secret key runtime guard added | 1 (was silently `'you forgot...'`) | diff --git a/scripts/vagrant_provision.sh b/scripts/vagrant_provision.sh index 1091b6e..591d2de 100755 --- a/scripts/vagrant_provision.sh +++ b/scripts/vagrant_provision.sh @@ -1,4 +1,6 @@ #!/bin/bash +# LEGACY — Vagrant-based provisioning. Primary dev environment is now Docker Compose. +# See docker/docker-compose.yml and README for current setup instructions. PROJECT_NAME=$1 APP_NAME=$2 From ab1cadbf9e5cc25e7b6396be236edc772ceab070 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Mon, 30 Mar 2026 19:48:44 -0700 Subject: [PATCH 011/152] Add initial configuration files and update Docker setup for Madrona Portal - Create .env.example for environment variable configuration. - Update Dockerfile to streamline environment variable setup and application source copying. - Revise docker-compose.yml to enhance service configuration and health checks. - Modify entrypoint.sh for improved database connection handling and fixture loading. - Update requirements.txt and docker-requirements.txt for dependency management. - Introduce pyproject.toml for project metadata and tooling configuration. - Revise dev_requirements.txt to include testing and linting tools. --- .env.example | 67 +++++++++++++++ Dockerfile | 111 +++++++++++++----------- dev_requirements.txt | 83 +++++------------- docker/docker-compose.yml | 108 ++++++++++++++--------- docker/docker-requirements.txt | 150 ++++++++++++++++---------------- docker/entrypoint.sh | 132 ++++++++++++++++++++-------- pyproject.toml | 84 ++++++++++++++++++ requirements.txt | 151 ++++++++++++++++++--------------- 8 files changed, 558 insertions(+), 328 deletions(-) create mode 100644 .env.example mode change 100644 => 100755 docker/entrypoint.sh create mode 100644 pyproject.toml diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..b08fb8e --- /dev/null +++ b/.env.example @@ -0,0 +1,67 @@ +# ============================================================================= +# Madrona Portal — Environment Variables +# Copy this file to .env and fill in real values. +# NEVER commit the real .env file to version control. +# +# Priority for every setting: env var > config.ini > built-in default +# ============================================================================= + +# --------------------------------------------------------------------------- +# Django core +# --------------------------------------------------------------------------- +SECRET_KEY=change-me-to-a-long-random-string +ALLOWED_HOSTS=localhost,127.0.0.1 +MP_PROJECT_CONFIG=config.wcoa.docker.ini +DEBUG=False + +# --------------------------------------------------------------------------- +# PostgreSQL / PostGIS +# DB_* is preferred; SQL_* aliases are accepted for legacy docker-compose files. +# --------------------------------------------------------------------------- +DB_ENGINE=django.contrib.gis.db.backends.postgis +DB_NAME=wcoa_docker_db +DB_USER=postgres +DB_PASSWORD=change-me +# DB_HOST and DB_PORT are set inside docker-compose.yml (always "db" and 5432) +APP_PORT=8000 + +# --------------------------------------------------------------------------- +# Redis (used for Django cache + Celery broker + result backend) +# docker-compose builds REDIS_URL from REDIS_PASSWORD automatically. +# --------------------------------------------------------------------------- +REDIS_PASSWORD=change-me +REDIS_PORT=6379 +# REDIS_URL and CELERY_BROKER_URL are assembled in docker-compose.yml. + +# --------------------------------------------------------------------------- +# Email (SMTP) +# --------------------------------------------------------------------------- +EMAIL_HOST=smtp.example.com +EMAIL_PORT=587 +EMAIL_HOST_USER=noreply@example.com +EMAIL_HOST_PASSWORD=change-me +EMAIL_USE_TLS=true + +# --------------------------------------------------------------------------- +# AWS SES (optional — only needed if EMAIL_BACKEND uses SES) +# --------------------------------------------------------------------------- +AWS_ACCESS_KEY_ID= +AWS_SECRET_ACCESS_KEY= +AWS_SES_REGION_NAME=us-east-1 +AWS_SES_REGION_ENDPOINT=email.us-east-1.amazonaws.com + +# --------------------------------------------------------------------------- +# Social Auth OAuth keys +# --------------------------------------------------------------------------- +FACEBOOK_KEY= +FACEBOOK_SECRET= +TWITTER_KEY= +TWITTER_SECRET= +GOOGLE_KEY= +GOOGLE_SECRET= + +# --------------------------------------------------------------------------- +# ReCAPTCHA (set in config.ini [APP] section or here) +# --------------------------------------------------------------------------- +# RECAPTCHA_PUBLIC_KEY= +# RECAPTCHA_PRIVATE_KEY= diff --git a/Dockerfile b/Dockerfile index cfc570d..ad02b68 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,43 +1,23 @@ -# pull official base image — Ubuntu 24.04 LTS (Noble Numbat) +# ============================================================================= +# Madrona Portal — Production Dockerfile +# Base: Ubuntu 24.04 LTS | Django 4.2+ | Wagtail 7.0+ | Python 3.10+ +# ============================================================================= FROM ubuntu:24.04 -# prevent apt from blocking on interactive questions during build +# Prevent apt from blocking on interactive prompts during build ENV DEBIAN_FRONTEND=noninteractive -# set environment variables -ENV PYTHONDONTWRITEBYTECODE=1 -ENV PYTHONUNBUFFERED=1 -ENV MP_PROJECT_CONFIG=config.wcoa.docker.ini +# Python & app environment +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PATH="/opt/venv/bin:$PATH" \ + MP_PROJECT_CONFIG=config.wcoa.docker.ini -# set work directory WORKDIR /usr/local/apps/madrona-portal -# copy project -COPY madrona_portal/marco /usr/local/apps/madrona-portal/marco -COPY madrona_portal/apps/__init__.py /usr/local/apps/madrona-portal/apps/__init__.py -COPY madrona_portal/assets /usr/local/apps/madrona-portal/assets -COPY madrona_portal/bower_components /usr/local/apps/madrona-portal/bower_components -COPY madrona_portal/docker/entrypoint.sh /entrypoint.sh -COPY madrona_portal/docker/docker-requirements.txt /requirements.txt -COPY madrona_portal/backups /usr/local/apps/madrona-portal/backups - -COPY madrona-apps/django_url_shortener /usr/local/apps/madrona-portal/apps/django_url_shortener -COPY madrona-apps/madrona-analysistools /usr/local/apps/madrona-portal/apps/madrona-analysistools -COPY madrona-apps/madrona-features /usr/local/apps/madrona-portal/apps/madrona-features -COPY madrona-apps/madrona-manipulators /usr/local/apps/madrona-portal/apps/madrona-manipulators -COPY madrona-apps/madrona-scenarios /usr/local/apps/madrona-portal/apps/madrona-scenarios -COPY madrona-apps/mp-accounts /usr/local/apps/madrona-portal/apps/mp-accounts -COPY madrona-apps/mp-data-manager /usr/local/apps/madrona-portal/apps/mp-data-manager -COPY madrona-apps/mp-drawing /usr/local/apps/madrona-portal/apps/mp-drawing -COPY madrona-apps/mp-explore /usr/local/apps/madrona-portal/apps/mp-explore -COPY madrona-apps/mp-layers /usr/local/apps/madrona-portal/apps/mp-layers -COPY madrona-apps/mp-map-groups /usr/local/apps/madrona-portal/apps/mp-map-groups -COPY madrona-apps/mp-proxy /usr/local/apps/madrona-portal/apps/mp-proxy -COPY madrona-apps/mp-visualize /usr/local/apps/madrona-portal/apps/mp-visualize -COPY madrona-apps/p97-nursery /usr/local/apps/madrona-portal/apps/p97-nursery -COPY madrona-apps/wcoa /usr/local/apps/madrona-portal/apps/wcoa - -# install system dependencies — mirrors the Ubuntu 24.04 wiki install +# --------------------------------------------------------------------------- +# System dependencies +# --------------------------------------------------------------------------- RUN apt-get update && apt-get upgrade -y && \ apt-get install -y --no-install-recommends \ python3 python3-pip python3-dev python3-venv \ @@ -53,28 +33,59 @@ RUN apt-get update && apt-get upgrade -y && \ libffi-dev openssl \ && rm -rf /var/lib/apt/lists/* - # Create a virtual environment so pip installs don't conflict with system Python - RUN python3 -m venv /opt/venv - ENV PATH="/opt/venv/bin:$PATH" +# --------------------------------------------------------------------------- +# Python virtual environment +# --------------------------------------------------------------------------- +RUN python3 -m venv /opt/venv - # Install the local layers app first so later package resolution can satisfy - # any dependency on the mp-layers distribution from the local checkout. - RUN pip install --upgrade pip setuptools wheel && \ - pip install --no-deps -e /usr/local/apps/madrona-portal/apps/mp-layers && \ - pip install -r /requirements.txt +# --------------------------------------------------------------------------- +# Copy application source +# --------------------------------------------------------------------------- +COPY madrona_portal/marco ./marco +COPY madrona_portal/apps/__init__.py ./apps/__init__.py +COPY madrona_portal/assets ./assets +COPY madrona_portal/bower_components ./bower_components +COPY madrona_portal/docker/entrypoint.sh /entrypoint.sh +COPY madrona_portal/docker/docker-requirements.txt /requirements.txt +COPY madrona_portal/backups ./backups - # Install GDAL Python bindings matched to the system GDAL version. - # Installed separately so this layer is cached independently. - RUN pip install "GDAL==$(gdal-config --version)" --no-cache-dir +COPY madrona-apps/django_url_shortener ./apps/django_url_shortener +COPY madrona-apps/madrona-analysistools ./apps/madrona-analysistools +COPY madrona-apps/madrona-features ./apps/madrona-features +COPY madrona-apps/madrona-manipulators ./apps/madrona-manipulators +COPY madrona-apps/madrona-scenarios ./apps/madrona-scenarios +COPY madrona-apps/mp-accounts ./apps/mp-accounts +COPY madrona-apps/mp-data-manager ./apps/mp-data-manager +COPY madrona-apps/mp-drawing ./apps/mp-drawing +COPY madrona-apps/mp-explore ./apps/mp-explore +COPY madrona-apps/mp-layers ./apps/mp-layers +COPY madrona-apps/mp-map-groups ./apps/mp-map-groups +COPY madrona-apps/mp-proxy ./apps/mp-proxy +COPY madrona-apps/mp-visualize ./apps/mp-visualize +COPY madrona-apps/p97-nursery ./apps/p97-nursery +COPY madrona-apps/wcoa ./apps/wcoa -RUN chmod +x /entrypoint.sh +# --------------------------------------------------------------------------- +# Python dependencies +# Install mp-layers first (no-deps) so later resolution can satisfy any +# local dependency on its distribution before installing the rest. +# --------------------------------------------------------------------------- +RUN pip install --upgrade pip setuptools wheel && \ + pip install --no-deps -e ./apps/mp-layers && \ + pip install -r /requirements.txt -RUN mkdir -p /vol/web/media /vol/web/static +# GDAL Python bindings — installed in a separate layer to preserve Docker +# cache when the rest of requirements.txt changes. +RUN pip install "GDAL==$(gdal-config --version)" --no-cache-dir -RUN useradd --create-home --shell /bin/sh madrona_user -RUN chown -R madrona_user:madrona_user /vol -RUN chown -R madrona_user:madrona_user /usr/local/apps/madrona-portal -RUN chmod -R 755 /vol/web +# --------------------------------------------------------------------------- +# Runtime setup +# --------------------------------------------------------------------------- +RUN chmod 755 /entrypoint.sh && \ + mkdir -p /vol/web/media /vol/web/static && \ + useradd --create-home --shell /bin/sh madrona_user && \ + chown -R madrona_user:madrona_user /vol /usr/local/apps/madrona-portal && \ + chmod -R 755 /vol/web USER madrona_user diff --git a/dev_requirements.txt b/dev_requirements.txt index 374ecc6..1260bea 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -1,60 +1,23 @@ -# Minimal requirements -Django<1.10 -djangorestframework==3.3.2 -django-picklefield==0.3.2 - - -# django-compressor scss hack --e git+https://github.com/MidAtlanticPortal/django-libsass.git@master#egg=django_libsass - -libsass==0.13.4 -#django-libsass==0.7 - -django-modelcluster==1.1 -wagtail==1.3.1 - -# Recommended components (require additional setup): -psycopg2==2.5.2 -elasticsearch==1.3.0 -Embedly==0.5.0 - -# Recommended components to improve performance in production: -django-redis-cache==2.0.0 -# django-celery==3.1.10 - -django-email-log==0.2.0 -celery==3.1.25 -django-celery==3.1.16 -django-celery-email==1.1.1 -kombu==3.0.37 - -requests==2.21.0 - -django-apptemplates==1.4 - -django-social-share==0.3.0 - -rpc4django==0.5.0 - -django-compressor==1.6 - -diff-match-patch==20121119 -tablib==0.11.2 -django-import-export==0.7.0 - -pyshp==1.2.3 - -urllib3==1.25.2 -social-auth-core==3.1.0 -social-auth-app-django==3.1.0 -django-tinymce==2.0.4 -django-nested-admin==3.0.21 -django-taggit==0.21.6 - -# JSONB issues on PG 9.3 -django-pgjsonb==0.0.32 -Pillow==4.3.0 -six==1.12.0 -django-appconf==1.0.2 -South==1.0 -Unidecode==0.04.21 +# ============================================================================= +# Development & Testing Requirements +# Install with: pip install -r requirements.txt -r dev_requirements.txt +# ============================================================================= + +# Testing +pytest>=8.0,<9.0 +pytest-django>=4.8,<5.0 +pytest-cov>=5.0,<6.0 +factory-boy>=3.3,<4.0 + +# Code Quality & Linting +ruff>=0.4,<1.0 # Fast Python linter (replaces flake8, isort, pyupgrade) +mypy>=1.10,<2.0 # Static type checking +django-stubs>=5.0,<6.0 # Django type stubs for mypy + +# Debug tooling +django-debug-toolbar>=4.3,<5.0 +Werkzeug>=3.0,<4.0 # Better dev server with debugger + +# Utilities +ipython>=8.0,<9.0 +ipdb>=0.13,<1.0 diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index f660c45..740a41e 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -1,79 +1,103 @@ +# Madrona Portal — Docker Compose (WCOA) +# +# Quick start (full Docker stack): +# cp ../.env.example ../.env # then fill in real values +# docker compose -f docker/docker-compose.yml --profile full up --build +# +# Dev infrastructure only (db + Redis, for use with local Django dev server): +# docker compose -f docker/docker-compose.yml up +# python marco/manage.py runserver # in a separate terminal + services: + app: build: context: ../../ dockerfile: madrona_portal/Dockerfile - # command: dockerize -wait tcp://db:5432 sh -c "python manage.py migrate --noinput" - # command: dockerize -wait tcp://db:5432 sh -c "python manage.py loaddata /usr/local/apps/TEKDB/TEKDB/TEKDB/fixtures/all_dummy_data.json" volumes: - static_data:/vol/web + env_file: + - ../.env # load all secrets from the project-root .env file environment: + # Config file selection — override specific values via env vars below. + - MP_PROJECT_CONFIG=${MP_PROJECT_CONFIG:-config.wcoa.docker.ini} + + # Core Django - SECRET_KEY=${SECRET_KEY} - - ALLOWED_HOSTS=${ALLOWED_HOSTS} - - MP_PROJECT_CONFIG=config.wcoa.docker.ini - - SQL_ENGINE=${SQL_ENGINE} - - SQL_DATABASE=${SQL_DATABASE} - - SQL_USER=${SQL_USER} - - SQL_PASSWORD=${SQL_PASSWORD} - - SQL_HOST=${SQL_HOST} - - SQL_PORT=${SQL_PORT} - - PROJ_DIR=${PROJ_DIR} + - ALLOWED_HOSTS=${ALLOWED_HOSTS:-localhost,127.0.0.1,::1} + - DEBUG=${DEBUG:-True} + + # Database (DB_* preferred; SQL_* aliases kept for legacy compatibility) + - DB_ENGINE=${DB_ENGINE:-django.contrib.gis.db.backends.postgis} + - DB_NAME=${DB_NAME:-wcoa_docker_db} + - DB_USER=${DB_USER:-postgres} + - DB_PASSWORD=${DB_PASSWORD} + - DB_HOST=db + - DB_PORT=5432 + + # Redis — single URL used for cache, Celery broker, and result backend. + # Auth segment (:password@) is included only when REDIS_PASSWORD is set. + - REDIS_URL=redis://${REDIS_PASSWORD:+:${REDIS_PASSWORD}@}tasks:6379/1 + - CELERY_BROKER_URL=redis://${REDIS_PASSWORD:+:${REDIS_PASSWORD}@}tasks:6379/0 + + # Application server mode + - DJANGO_ENV=${DJANGO_ENV:-development} + - GUNICORN_WORKERS=${GUNICORN_WORKERS:-3} depends_on: - - db - links: - - ${SQL_HOST} + db: + condition: service_healthy + tasks: + condition: service_healthy ports: - - "8000:8000" + - "${APP_PORT:-8000}:8000" networks: - # maybe not necessary - djangonetwork - - # proxy: - # build: - # context: ../../proxy - # volumes: - # - static_data:/vol/static - # # - media_data:/vol/media - # ports: - # - "${PROXY_PORT}:8080" - # depends_on: - # - app - # networks: - # - djangonetwork + profiles: + - full + restart: unless-stopped db: image: postgis/postgis:16-3.4 volumes: - postgis-data:/var/lib/postgresql environment: - - POSTGRES_USER=${SQL_USER} - - POSTGRES_PASSWORD=${SQL_PASSWORD} - - POSTGRES_DB=${SQL_DATABASE} + - POSTGRES_USER=${DB_USER:-postgres} + - POSTGRES_PASSWORD=${DB_PASSWORD} + - POSTGRES_DB=${DB_NAME:-wcoa_docker_db} ports: - - ${SQL_PORT}:5432 + - "${DB_PORT:-5432}:5432" networks: - djangonetwork + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${DB_USER:-postgres} -d ${DB_NAME:-wcoa_docker_db}"] + interval: 10s + timeout: 5s + retries: 5 + restart: unless-stopped tasks: image: redis:7-alpine - command: redis-server --requirepass ${REDIS_PASSWORD} + # Only pass --requirepass when REDIS_PASSWORD is non-empty. + # An empty REDIS_PASSWORD causes "wrong number of arguments" in Redis 7. + command: redis-server ${REDIS_PASSWORD:+--requirepass ${REDIS_PASSWORD}} ports: - - ${TASK_PORT}:6379 + - "${REDIS_PORT:-6379}:6379" volumes: - - redis-data:/var/lib/redis - - redis.conf:/usr/local/etc/redis/redis.conf - - environment: - - REDIS_REPLICATION_MODE=master + - redis-data:/data networks: - djangonetwork + healthcheck: + # -a flag is only passed when a password is configured. + test: ["CMD-SHELL", "redis-cli ${REDIS_PASSWORD:+-a ${REDIS_PASSWORD}} ping"] + interval: 10s + timeout: 5s + retries: 5 + restart: unless-stopped volumes: postgis-data: static_data: redis-data: - redis.conf: - # media_data: networks: djangonetwork: diff --git a/docker/docker-requirements.txt b/docker/docker-requirements.txt index a287e05..7837218 100644 --- a/docker/docker-requirements.txt +++ b/docker/docker-requirements.txt @@ -1,97 +1,101 @@ -# Minimal requirements -Django>=4.2,<4.3 +# ============================================================================= +# Madrona Portal — Docker / Production Requirements +# This file is used by the Dockerfile. Pin only what is needed here; +# version constraints live in the top-level requirements.txt. +# ============================================================================= + +# --------------------------------------------------------------------------- +# Core framework +# --------------------------------------------------------------------------- +Django>=4.2,<5.0 wagtail>=7.0,<8.0 -# Tentative additions: -python-social-auth -social-auth-app-django>5.4 -python-jose -pyjwt -django-autocomplete-light -django-social-share -django-email-log -django-compressor -django-tinymce -django-wysiwyg -django-recaptcha -django-flatblocks -django-nested-admin -django-querysetsequence -django-redis +# --------------------------------------------------------------------------- +# Authentication & social auth +# --------------------------------------------------------------------------- +social-auth-app-django>=5.4,<6.0 +social-auth-core>=4.5,<5.0 +python-jose>=3.3,<4.0 +pyjwt>=2.8,<3.0 +django-social-share>=2.3,<3.0 + +# --------------------------------------------------------------------------- +# Admin & CMS +# --------------------------------------------------------------------------- +django-autocomplete-light>=3.9,<4.0 +django-nested-admin>=4.0,<5.0 +django-tinymce>=3.6,<4.0 +django-recaptcha>=4.0,<5.0 +django-flatblocks>=1.0,<2.0 +django-querysetsequence>=0.14,<1.0 +wagtailcharts>=0.4,<1.0 +django-import-export>=3.3,<4.0 +django-colorfield>=0.11,<1.0 + +# --------------------------------------------------------------------------- +# Content & tagging +# --------------------------------------------------------------------------- django-taggit>=5.0,<7.0 -rpc4django +django-email-log>=1.0,<2.0 +django-compressor>=4.4,<5.0 +django-libsass>=0.9,<1.0 +libsass>=0.23,<1.0 +# --------------------------------------------------------------------------- +# Async tasks & caching +# --------------------------------------------------------------------------- +celery>=5.3,<6.0 +django-celery-email>=3.0,<4.0 +django-redis>=5.4,<6.0 + +# --------------------------------------------------------------------------- +# Application server (production) +# --------------------------------------------------------------------------- +gunicorn>=22.0,<24.0 + +# --------------------------------------------------------------------------- +# Database & geospatial +# --------------------------------------------------------------------------- +# pg2 2.9 breaks migrations: https://stackoverflow.com/a/68025007/706797 +psycopg2-binary>=2.9.9,<3.0 # GDAL Python bindings are installed separately in the Dockerfile via: # pip install "GDAL==$(gdal-config --version)" --no-cache-dir -# pygdal<3.2.4 is intentionally omitted -- its setup.py breaks with -# modern pip build isolation (numpy.__NUMPY_SETUP__ removed in numpy>=1.20). +pyshp>=2.3,<3.0 +owslib>=0.29,<1.0 -################################## -#-e git+https://github.com/Ecotrust/mp-data-manager.git@main#egg=mp_data_manager -### OR #### -#-e git+https://github.com/Ecotrust/mp-data-manager.git@gp2#egg=data-manager --e /usr/local/apps/madrona-portal/apps/mp-data-manager/ -################################## +# --------------------------------------------------------------------------- +# Search +# --------------------------------------------------------------------------- +elasticsearch>=7.0,<8.0 +elasticsearch-dsl>=7.0,<8.0 --e /usr/local/apps/madrona-portal/apps/django_url_shortener/url_short +# --------------------------------------------------------------------------- +# APIs +# --------------------------------------------------------------------------- +# rpc4django removed — replaced by djangorestframework API views +djangorestframework>=3.14,<4.0 -#-e git+https://github.com/Ecotrust/madrona-analysistools.git@main#egg=analysistools +# --------------------------------------------------------------------------- +# Ecotrust / Madrona sub-apps (local editable installs via Dockerfile COPY) +# --------------------------------------------------------------------------- +-e /usr/local/apps/madrona-portal/apps/mp-layers +-e /usr/local/apps/madrona-portal/apps/mp-data-manager/ +-e /usr/local/apps/madrona-portal/apps/django_url_shortener/url_short -e /usr/local/apps/madrona-portal/apps/madrona-analysistools -#-e git+https://github.com/Ecotrust/mp-visualize.git@main#egg=visualize -e /usr/local/apps/madrona-portal/apps/mp-visualize -#-e git+https://github.com/Ecotrust/madrona-features.git@main#egg=features -e /usr/local/apps/madrona-portal/apps/madrona-features -#-e git+https://github.com/Ecotrust/mp-accounts.git@main#egg=accounts -e /usr/local/apps/madrona-portal/apps/mp-accounts -#-e git+https://github.com/Ecotrust/madrona-scenarios.git@main#egg=scenarios -e /usr/local/apps/madrona-portal/apps/madrona-scenarios -#-e git+https://github.com/Ecotrust/madrona-manipulators.git@main#egg=manipulators -e /usr/local/apps/madrona-portal/apps/madrona-manipulators -#-e git+https://github.com/Ecotrust/mp-drawing.git@main#egg=drawing -e /usr/local/apps/madrona-portal/apps/mp-drawing -#-e git+https://github.com/Ecotrust/mp-explore.git@main#egg=explore -e /usr/local/apps/madrona-portal/apps/mp-explore -#-e git+https://github.com/Ecotrust/mp-layers.git@main#egg=layers --e /usr/local/apps/madrona-portal/apps/mp-layers -#-e git+https://github.com/Ecotrust/mp-map-groups.git@main#egg=map_groups -e /usr/local/apps/madrona-portal/apps/mp-map-groups -#-e git+https://github.com/Ecotrust/p97-nursery.git@main#egg=nursery -e /usr/local/apps/madrona-portal/apps/p97-nursery -#-e git+https://github.com/Ecotrust/mp-proxy.git@main#egg=proxy -e /usr/local/apps/madrona-portal/apps/mp-proxy -################################## -#-e git+https://github.com/Ecotrust/mida-portal.git@main#egg=mida-portal +# --------------------------------------------------------------------------- +# Portal variant — choose one: +# --------------------------------------------------------------------------- # -e /usr/local/apps/madrona-portal/apps/mida-portal -### OR ### -#-e git+https://github.com/Ecotrust/wcoa.git@migration_2021#egg=wcoa -e /usr/local/apps/madrona-portal/apps/wcoa -### OR ### -#-e git+https://github.com/Ecotrust/wc-offshore-portal.git@master#egg=wc-offshore-portal # -e /usr/local/apps/madrona-portal/apps/wc-offshore-portal -################################## - - -celery -# Django-Celery is incompatible with py3/Django2 -#django-celery -django-celery-email - - -# Note: django-libsass needs libsass>0.4 -#libsass -#django-libsass - -# Recommended components (require additional setup): -# pg2 2.9 breaks migrations: https://stackoverflow.com/a/68025007/706797 -psycopg2-binary>2.9.9 -elasticsearch<8.0 -elasticsearch-dsl<8.0 -wagtailcharts - -django-import-export -django-colorfield - -pyshp - -owslib diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh old mode 100644 new mode 100755 index cd20d03..57d664d --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -1,46 +1,52 @@ #!/bin/sh +# Madrona Portal — Docker entrypoint +# Waits for the database, runs migrations, seeds a fresh DB, then starts the server. -# Exit on errors set -e -# Wait for the database service before running migrations. +# --------------------------------------------------------------------------- +# 1. Wait for the database to accept connections +# --------------------------------------------------------------------------- python - <<'PY' -import os -import socket -import time +import os, socket, time, sys -host = os.environ.get("SQL_HOST", "db") -port = int(os.environ.get("DB_INTERNAL_PORT", "5432")) -timeout_seconds = int(os.environ.get("DB_WAIT_TIMEOUT", "90")) +# Accept both DB_HOST (preferred) and legacy SQL_HOST +host = os.environ.get("DB_HOST") or os.environ.get("SQL_HOST", "db") +port = int(os.environ.get("DB_PORT") or os.environ.get("SQL_PORT", "5432")) +timeout = int(os.environ.get("DB_WAIT_TIMEOUT", "90")) +print(f"Waiting for database at {host}:{port} (timeout {timeout}s)...", flush=True) start = time.time() while True: - try: - with socket.create_connection((host, port), timeout=2): - break - except OSError: - if time.time() - start > timeout_seconds: - raise SystemExit(f"Timed out waiting for database at {host}:{port}") - time.sleep(1) -PY + try: + with socket.create_connection((host, port), timeout=2): + break + except OSError: + if time.time() - start > timeout: + sys.exit(f"Timed out waiting for database at {host}:{port}") + time.sleep(1) -# if [ -n "$SQL_HOST" ]; then -# echo "Waiting for database at ${SQL_HOST}:${DB_INTERNAL_PORT:-5432}..." -# # pg_isready is available after installing postgresql-client in the image -# until pg_isready -h "$SQL_HOST" -p "${DB_INTERNAL_PORT:-5432}" >/dev/null 2>&1; do -# echo "Postgres is unavailable - sleeping" -# sleep 1 -# done -# echo "Postgres is up" -# fi +print("Database is up.", flush=True) +PY -python marco/manage.py collectstatic --noinput +# --------------------------------------------------------------------------- +# 2. Migrate and collect static files +# --------------------------------------------------------------------------- python marco/manage.py migrate --noinput +python marco/manage.py collectstatic --noinput -# On a fresh database (no real Wagtail content pages yet), load the initial -# fixture data so the site starts with working navigation and content. -# The check is skipped safely if Django fails to import for any reason. -PAGE_COUNT=$(python - 2>/dev/null <<'PY' || echo "unknown" +# --------------------------------------------------------------------------- +# 3. Seed a fresh database with initial fixture data +# +# A brand-new PostGIS install contains exactly one Wagtail Page row (the +# Wagtail root page, depth=1). We count pages at depth > 1 — if none exist, +# this is a fresh database and we load the initial fixture. +# +# IMPORTANT: We never wipe content on an existing database. That would +# destroy real data. Set FORCE_RELOAD_FIXTURES=1 only in CI or dev reset +# scenarios where wiping the database is intentional. +# --------------------------------------------------------------------------- +CONTENT_PAGES=$(python - 2>/dev/null <<'PY' || echo "unknown" import sys, os sys.path.insert(0, 'marco') os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'marco.settings') @@ -50,12 +56,66 @@ from wagtail.models import Page print(Page.objects.filter(depth__gt=1).count()) PY ) -if [ "$PAGE_COUNT" = "0" ]; then - echo "Fresh database — loading initial fixtures..." - python marco/manage.py loaddata wcoa_init wcoa_init_layers wagtail_menus + +echo "Content pages in database: ${CONTENT_PAGES}" + +if [ "${CONTENT_PAGES}" = "0" ] || [ "${FORCE_RELOAD_FIXTURES:-0}" = "1" ]; then + echo "Fresh database detected — loading initial fixtures..." + + # Clear stale search index entries and image renditions so the fixture + # loads cleanly into the empty database. + python - <<'PY' +import sys, os +sys.path.insert(0, 'marco') +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'marco.settings') +import django +django.setup() + +from wagtail.search.models import Query +Query.objects.all().delete() + +try: + from portal.base.models import PortalRendition + PortalRendition.objects.all().delete() +except Exception: + pass +PY + + python marco/manage.py loaddata initial_data.json echo "Initial fixtures loaded." +else + echo "Existing database — skipping fixture load." fi -python marco/manage.py runserver 0:8000 -#uwsgi --socket :8000 --master --enable-threads --module marco.marco.wsgi -#exec "$@" +# --------------------------------------------------------------------------- +# 4. Start the application server +# +# DEBUG=True → Django's runserver (auto-reload, no gunicorn needed) +# DEBUG=False → gunicorn (multi-worker, production-safe) +# +# Override with DJANGO_ENV=production to force gunicorn regardless of DEBUG. +# --------------------------------------------------------------------------- +DJANGO_DEBUG=$(python - <<'PY' +import sys, os +sys.path.insert(0, 'marco') +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'marco.settings') +import django +django.setup() +from django.conf import settings +print("true" if settings.DEBUG else "false") +PY +) + +if [ "${DJANGO_ENV:-}" = "production" ] || [ "${DJANGO_DEBUG}" = "false" ]; then + echo "Starting gunicorn (production mode)..." + exec gunicorn marco.wsgi:application \ + --bind 0.0.0.0:8000 \ + --workers "${GUNICORN_WORKERS:-3}" \ + --timeout "${GUNICORN_TIMEOUT:-120}" \ + --chdir marco \ + --access-logfile - \ + --error-logfile - +else + echo "Starting Django development server..." + exec python marco/manage.py runserver 0.0.0.0:8000 +fi diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..5258227 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,84 @@ +[build-system] +requires = ["setuptools>=68", "wheel"] +build-backend = "setuptools.backends.legacy:build" + +[project] +name = "madrona-portal" +version = "2.0.0" +description = "MARCO Mid-Atlantic Ocean Data Portal" +requires-python = ">=3.10" +readme = "README.md" +license = { text = "MIT" } + +# Runtime dependencies are managed in requirements.txt / docker-requirements.txt. +# This section is intentionally minimal — the project is not distributed as a +# standalone package, so pip-installable metadata is kept lightweight. +dependencies = [] + +# --------------------------------------------------------------------------- +# Pytest +# --------------------------------------------------------------------------- +[tool.pytest.ini_options] +DJANGO_SETTINGS_MODULE = "marco.settings" +python_files = ["tests.py", "test_*.py", "*_test.py"] +addopts = "--strict-markers -q" + +# --------------------------------------------------------------------------- +# Coverage +# --------------------------------------------------------------------------- +[tool.coverage.run] +source = ["portal", "marco", "marco_site"] +omit = ["*/migrations/*", "*/tests/*", "manage.py"] + +[tool.coverage.report] +show_missing = true +skip_covered = false + +# --------------------------------------------------------------------------- +# Ruff (linting + formatting — replaces flake8, isort, pyupgrade) +# --------------------------------------------------------------------------- +[tool.ruff] +target-version = "py310" +line-length = 100 +exclude = [ + ".git", + ".venv", + "venv", + "bower_components", + "*/migrations/*", + "node_modules", + "static", + "media", +] + +[tool.ruff.lint] +select = [ + "E", # pycodestyle errors + "W", # pycodestyle warnings + "F", # pyflakes + "I", # isort + "UP", # pyupgrade + "B", # flake8-bugbear + "C4", # flake8-comprehensions + "SIM", # flake8-simplify + "DJ", # flake8-django +] +ignore = [ + "E501", # line too long — handled by formatter + "DJ001", # Avoid using null=True on string-based fields — existing models +] + +[tool.ruff.lint.isort] +known-first-party = ["marco", "marco_site", "portal"] + +# --------------------------------------------------------------------------- +# Mypy (optional static typing) +# --------------------------------------------------------------------------- +[tool.mypy] +python_version = "3.10" +plugins = ["mypy_django_plugin.main"] +ignore_missing_imports = true +exclude = ["migrations/", "bower_components/", "static/"] + +[tool.django-stubs] +django_settings_module = "marco.settings" diff --git a/requirements.txt b/requirements.txt index d4adcd2..d65840b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,76 +1,93 @@ -# Minimal requirements -Django>=4.2,<4.3 -wagtail +# ============================================================================= +# Core Framework +# ============================================================================= +Django>=4.2,<5.0 +wagtail>=7.0,<8.0 -# Tentative additions: -wagtail-import-export -python-social-auth -# social-auth-app-django>5.4 -social-auth-app-django<5.0 -python-jose -pyjwt -django-autocomplete-light -django-social-share -django-email-log -django-compressor -django-tinymce -django-wysiwyg -django-recaptcha -django-flatblocks -django-nested-admin -django-querysetsequence -django-redis -django-taggit<5.0 -rpc4django +# ============================================================================= +# Authentication & Social Auth +# ============================================================================= +social-auth-app-django>=5.4,<6.0 +social-auth-core>=4.5,<5.0 +python-jose>=3.3,<4.0 +pyjwt>=2.8,<3.0 +django-social-share>=2.3,<3.0 -################################## -# -e git+https://github.com/Ecotrust/mp-data-manager.git@main#egg=mp_data_manager -### OR #### -#-e git+https://github.com/Ecotrust/mp-data-manager.git@gp2#egg=mp_data_manager-gp2 -################################## - -#-e git+https://github.com/Ecotrust/madrona-analysistools.git@main#egg=madrona_analysistools -#-e git+https://github.com/Ecotrust/mp-visualize.git@main#egg=mp_visualize -#-e git+https://github.com/Ecotrust/madrona-features.git@main#egg=madrona_features -#-e git+https://github.com/Ecotrust/mp-accounts.git@main#egg=mp_accounts -#-e git+https://github.com/Ecotrust/madrona-scenarios.git@main#egg=madrona_scenarios -#-e git+https://github.com/Ecotrust/madrona-manipulators.git@main#egg=madrona_manipulators -#-e git+https://github.com/Ecotrust/mp-drawing.git@main#egg=mp_drawing -#-e git+https://github.com/Ecotrust/mp-explore.git@main#egg=mp_explore -#-e git+https://github.com/Ecotrust/mp-map-groups.git@main#egg=mp-map-groups -#-e git+https://github.com/Ecotrust/p97-nursery.git@main#egg=p97-nursery -#-e git+https://github.com/Ecotrust/mp-proxy.git@main#egg=mp_proxy -#-e git+https://github.com/Ecotrust/mp-survey.git@main#egg=mp_survey +# ============================================================================= +# Admin & CMS Enhancements +# ============================================================================= +django-autocomplete-light>=3.9,<4.0 +django-nested-admin>=4.0,<5.0 +django-tinymce>=3.6,<4.0 +django-recaptcha>=4.0,<5.0 # django_recaptcha (Wagtail 7+ requires v4) +django-flatblocks>=1.0,<2.0 +django-querysetsequence>=0.14,<1.0 +wagtailcharts>=0.4,<1.0 +django-import-export>=3.3,<4.0 +django-colorfield>=0.11,<1.0 -################################## -#-e git+https://github.com/Ecotrust/mida-portal.git@main#egg=mida -### OR ### -#-e git+https://github.com/Ecotrust/wcoa.git@migration_2021#egg=wcoa-2021 -### OR ### -#-e git+https://github.com/Ecotrust/wc-offshore-portal.git@master#egg=offshore -################################## +# ============================================================================= +# Content & Tagging +# ============================================================================= +django-taggit>=5.0,<7.0 +django-email-log>=1.0,<2.0 +django-compressor>=4.4,<5.0 +django-libsass>=0.9,<1.0 +libsass>=0.23,<1.0 +django-picklefield>=3.1,<4.0 +xmltodict>=0.13,<1.0 +# ============================================================================= +# Async Tasks & Caching +# ============================================================================= +celery>=5.3,<6.0 +django-celery-email>=3.0,<4.0 +django-redis>=5.4,<6.0 -celery -# Django-Celery is incompatible with py3/Django2 -#django-celery -django-celery-email -django-colorfield -django-libsass +# ============================================================================= +# Database & Geospatial +# ============================================================================= +psycopg2-binary>=2.9.9,<3.0 +# Note: GDAL Python bindings are installed separately in the Dockerfile via: +# pip install "GDAL==$(gdal-config --version)" --no-cache-dir +pyshp>=2.3,<3.0 +owslib>=0.29,<1.0 -# Note: django-libsass needs libsass>0.4 -# 0.5.1 is the latest version that we can install on WF -libsass +# ============================================================================= +# Search +# ============================================================================= +elasticsearch>=7.0,<8.0 +elasticsearch-dsl>=7.0,<8.0 -# Recommended components (require additional setup): -# pg2 2.9 breaks migrations: https://stackoverflow.com/a/68025007/706797 -# psycopg2-binary>2.9.9 -psycopg2-binary<2.9 -elasticsearch +# ============================================================================= +# APIs & Protocols +# ============================================================================= +# rpc4django removed — replaced by djangorestframework API views +djangorestframework>=3.14,<4.0 -django-import-export - -pyshp +# ============================================================================= +# Ecotrust Sub-Apps +# (Uncomment the git source OR use local editable installs for development) +# ============================================================================= +# -e git+https://github.com/Ecotrust/mp-data-manager.git@main#egg=mp_data_manager +# -e git+https://github.com/Ecotrust/madrona-analysistools.git@main#egg=madrona_analysistools +# -e git+https://github.com/Ecotrust/mp-visualize.git@main#egg=mp_visualize +# -e git+https://github.com/Ecotrust/madrona-features.git@main#egg=madrona_features +# -e git+https://github.com/Ecotrust/mp-accounts.git@main#egg=mp_accounts +# -e git+https://github.com/Ecotrust/madrona-scenarios.git@main#egg=madrona_scenarios +# -e git+https://github.com/Ecotrust/madrona-manipulators.git@main#egg=madrona_manipulators +# -e git+https://github.com/Ecotrust/mp-drawing.git@main#egg=mp_drawing +# -e git+https://github.com/Ecotrust/mp-explore.git@main#egg=mp_explore +# -e git+https://github.com/Ecotrust/mp-layers.git@main#egg=mp_layers +# -e git+https://github.com/Ecotrust/mp-map-groups.git@main#egg=mp_map_groups +# -e git+https://github.com/Ecotrust/p97-nursery.git@main#egg=p97_nursery +# -e git+https://github.com/Ecotrust/mp-proxy.git@main#egg=mp_proxy +# -e git+https://github.com/Ecotrust/mp-survey.git@main#egg=mp_survey +# -e git+https://github.com/Ecotrust/django-url-shortener.git@main#egg=url_short -owslib -django-colorfield +# ============================================================================= +# Portal Variant (choose one): +# ============================================================================= +# -e git+https://github.com/Ecotrust/mida-portal.git@main#egg=mida +# -e git+https://github.com/Ecotrust/wcoa.git@main#egg=wcoa +# -e git+https://github.com/Ecotrust/wc-offshore-portal.git@master#egg=offshore From 176737c2228590d00bdfc66528f40ef037afeeb3 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Mon, 30 Mar 2026 19:49:12 -0700 Subject: [PATCH 012/152] Refactor URL configuration and model imports for Wagtail compatibility - Simplified URL patterns in `urls.py` by removing version checks for Wagtail and consolidating imports. - Updated URL patterns to ensure proper routing and removed deprecated endpoints. - Enhanced error handling for loading project-specific URLs. - Streamlined model imports in `models.py` by removing conditional imports based on Wagtail version. - Ensured unique constraints in `PortalRendition` model are consistently defined. --- marco/marco/apps.py | 8 +- marco/marco/celery.py | 17 +- marco/marco/settings.py | 1009 +++++++++++++++-------------------- marco/marco/urls.py | 144 +++-- marco/portal/base/models.py | 48 +- 5 files changed, 509 insertions(+), 717 deletions(-) diff --git a/marco/marco/apps.py b/marco/marco/apps.py index 1f63bb4..499791e 100644 --- a/marco/marco/apps.py +++ b/marco/marco/apps.py @@ -1,10 +1,6 @@ -import sys from django.apps import AppConfig + class MadronaPortalConfig(AppConfig): - #TODO: Rename this module to 'madrona' or 'madrona_portal' + # TODO: Rename this module to 'madrona' or 'madrona_portal' name = 'marco' - - def ready(self): - from .tasks import start_dbwatch - watch_db = start_dbwatch.delay() \ No newline at end of file diff --git a/marco/marco/celery.py b/marco/marco/celery.py index 9e07129..6f673ce 100644 --- a/marco/marco/celery.py +++ b/marco/marco/celery.py @@ -1,13 +1,24 @@ import os from celery import Celery +from celery.signals import worker_ready from django.conf import settings os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'marco.settings') -BASE_REDIS_URL = os.environ.get('REDIS_URL', 'redis://localhost:6379') app = Celery('marco', include=['marco.tasks']) +# Pull Celery config from Django settings (keys prefixed with CELERY_). +# CELERY_BROKER_URL and CELERY_RESULT_BACKEND are set in settings.py from +# the CELERY_BROKER_URL / CELERY_RESULT_BACKEND env vars, or from the +# [CELERY] section of the project .ini config file. app.config_from_object('django.conf:settings', namespace='CELERY') app.autodiscover_tasks(lambda: settings.INSTALLED_APPS) -app.conf.broker_url = BASE_REDIS_URL -app.conf.result_backend = BASE_REDIS_URL + + +@worker_ready.connect +def on_worker_ready(sender, **kwargs): + """Start the long-running db-notify → cache-invalidation listener task + once a Celery worker is live. Keeping this out of AppConfig.ready() + means Django can start without a Redis connection being required.""" + from marco.tasks import start_dbwatch + start_dbwatch.delay() diff --git a/marco/marco/settings.py b/marco/marco/settings.py index 66c98a8..42198bc 100644 --- a/marco/marco/settings.py +++ b/marco/marco/settings.py @@ -1,182 +1,169 @@ """ -Django settings for marco_portal project. +Django settings for the Madrona Portal project. -For more information on this file, see -https://docs.djangoproject.com/en/1.7/topics/settings/ +References: + https://docs.djangoproject.com/en/4.2/topics/settings/ + https://docs.djangoproject.com/en/4.2/ref/settings/ -For the full list of settings and their values, see -https://docs.djangoproject.com/en/1.7/ref/settings/ +Requires: Django 4.2+, Wagtail 7.0+, Python 3.10+ + +Secret / credential precedence (highest → lowest): + 1. Environment variable (e.g. export SECRET_KEY=...) + 2. config.ini value (e.g. [APP]\nSECRET_KEY = ...) + 3. Hard-coded default (safe defaults only — never real secrets) """ -import sys +import ast +import json import os import configparser from os.path import abspath, dirname -# from social.backends.google import GooglePlusAuth +from typing import Any -# Absolute filesystem path to the Django project directory: +# --------------------------------------------------------------------------- +# Path helpers +# --------------------------------------------------------------------------- PROJECT_ROOT = dirname(dirname(dirname(abspath(__file__)))) - BASE_DIR = os.path.dirname(os.path.dirname(__file__)) -ASSETS_DIR = os.path.realpath(os.path.join(BASE_DIR, '..', 'assets')) # assets directory -COMPONENTS_DIR = os.path.realpath(os.path.join(BASE_DIR, '..', 'bower_components')) # Bower components directory -STYLES_DIR = os.path.realpath(os.path.join(ASSETS_DIR, 'styles')) # SCSS files under assets/styles +ASSETS_DIR = os.path.realpath(os.path.join(BASE_DIR, '..', 'assets')) +COMPONENTS_DIR = os.path.realpath(os.path.join(BASE_DIR, '..', 'bower_components')) +STYLES_DIR = os.path.realpath(os.path.join(ASSETS_DIR, 'styles')) -MP_PROJECT_CONFIG = os.environ.get("MP_PROJECT_CONFIG", default='config.ini') +# --------------------------------------------------------------------------- +# Configuration file +# --------------------------------------------------------------------------- +MP_PROJECT_CONFIG = os.environ.get("MP_PROJECT_CONFIG", "config.ini") CONFIG_FILE = os.path.normpath(os.path.join(BASE_DIR, MP_PROJECT_CONFIG)) cfg = configparser.ConfigParser() cfg.read(CONFIG_FILE) -if 'APP' not in cfg.sections(): - cfg['APP'] = {} +for section in ('APP', 'CATALOG', 'DATABASE', 'CACHES', 'CELERY', 'EMAIL', 'AWS', 'SOCIAL_AUTH', 'REGION'): + if section not in cfg.sections(): + cfg[section] = {} app_cfg = cfg['APP'] +catalog_cfg = cfg['CATALOG'] +db_cfg = cfg['DATABASE'] +cache_cfg = cfg['CACHES'] +celery_cfg = cfg['CELERY'] +email_cfg = cfg['EMAIL'] +aws_cfg = cfg['AWS'] +social_cfg = cfg['SOCIAL_AUTH'] +region_cfg = cfg['REGION'] +# --------------------------------------------------------------------------- +# Secret resolution helper +# --------------------------------------------------------------------------- +def _env(env_key: str, cfg_section: configparser.SectionProxy, cfg_key: str, + default: Any = '') -> str: + """Return a setting value, checking the environment first. + + Priority: env var > config.ini > default. + This allows Docker / CI to override secrets without touching config files. + """ + return os.environ.get(env_key) or cfg_section.get(cfg_key, default) + +# --------------------------------------------------------------------------- +# Core settings +# --------------------------------------------------------------------------- DEBUG = app_cfg.getboolean('DEBUG', True) APP_NAME = app_cfg.get('APP_NAME', 'Marine Planner') APP_URL = app_cfg.get('APP_URL', '') -APP_TEAM_NAME = app_cfg.get('APP_TEAM_NAME', "{} Team".format(APP_NAME)) - -TEMPLATE_DEBUG = app_cfg.getboolean('TEMPLATE_DEBUG', True) - -SECRET_KEY = app_cfg.get('SECRET_KEY', 'you forgot to set the secret key') -host_list = os.environ.get('ALLOWED_HOSTS', app_cfg.get('ALLOWED_HOSTS')) -if isinstance(host_list, str): - host_value = host_list.strip() - if host_value.startswith('[') and host_value.endswith(']'): - # Prefer a real list literal: ["localhost", "127.0.0.1", "::1"] +APP_TEAM_NAME = app_cfg.get('APP_TEAM_NAME', f"{APP_NAME} Team") + +# env var takes priority so Docker / CI can inject secrets without touching config.ini +SECRET_KEY = _env('SECRET_KEY', app_cfg, 'SECRET_KEY', '') +_placeholder_phrases = ('forgot', 'change me', 'changeme', 'placeholder', 'you forgot') +if not SECRET_KEY or any(p in SECRET_KEY.lower() for p in _placeholder_phrases): + raise RuntimeError( + "SECRET_KEY is not set or still contains a placeholder value.\n" + "Set it via the SECRET_KEY environment variable or in config.ini [APP].\n" + f"Current value: {SECRET_KEY!r}" + ) + +# ALLOWED_HOSTS: accepts a comma-separated string, a JSON array string, or a plain string. +def _parse_hosts(raw: str | None) -> list[str]: + if not raw: + return [] + raw = raw.strip() + if raw.startswith('['): try: - import ast - parsed_hosts = ast.literal_eval(host_value) - if isinstance(parsed_hosts, (list, tuple)): - ALLOWED_HOSTS = [str(h).strip() for h in parsed_hosts if str(h).strip()] - else: - ALLOWED_HOSTS = [str(parsed_hosts).strip()] + parsed = ast.literal_eval(raw) + if isinstance(parsed, (list, tuple)): + return [str(h).strip() for h in parsed if str(h).strip()] except (SyntaxError, ValueError): - ALLOWED_HOSTS = [h.strip() for h in host_value[1:-1].split(',') if h.strip()] - elif ',' in host_value: - ALLOWED_HOSTS = [h.strip() for h in host_value.split(',') if h.strip()] - else: - ALLOWED_HOSTS = [host_value] -elif isinstance(host_list, list): - ALLOWED_HOSTS = [str(h).strip() for h in host_list if str(h).strip()] -else: - ALLOWED_HOSTS = [str(host_list)] + pass + # Fall back: strip brackets and split on comma + return [h.strip() for h in raw[1:-1].split(',') if h.strip()] + if ',' in raw: + return [h.strip() for h in raw.split(',') if h.strip()] + return [raw] + +_raw_hosts = os.environ.get('ALLOWED_HOSTS', app_cfg.get('ALLOWED_HOSTS', '')) +ALLOWED_HOSTS = _parse_hosts(_raw_hosts) -# Normalize bracketed IPv6 host forms like [::1] to ::1 for Django host checks. +# Normalise bracketed IPv6 forms like [::1] → ::1 for Django host checks ALLOWED_HOSTS = [ h[1:-1] if h.startswith('[') and h.endswith(']') and ':' in h else h for h in ALLOWED_HOSTS ] -# Set logging to default, and then make admin error emails come through as HTML +# --------------------------------------------------------------------------- +# Logging +# --------------------------------------------------------------------------- from django.utils.log import DEFAULT_LOGGING LOGGING = DEFAULT_LOGGING LOGGING['handlers']['mail_admins']['include_html'] = True -if 'CATALOG' not in cfg.sections(): - cfg['CATALOG'] = {} -catalog_cfg = cfg['CATALOG'] +# --------------------------------------------------------------------------- +# Catalog settings +# --------------------------------------------------------------------------- DATA_CATALOG_ENABLED = catalog_cfg.getboolean('DATA_CATALOG_ENABLED', True) -#CATALOG_TECHNOLOGY: Current support for 'default' (built in catalog) and 'GeoPortal2' +# Options: 'default' (built-in) or 'GeoPortal2' CATALOG_TECHNOLOGY = catalog_cfg.get('CATALOG_TECHNOLOGY', 'default') CATALOG_PROXY = catalog_cfg.get('CATALOG_PROXY', '') CATALOG_SOURCE = catalog_cfg.get('CATALOG_SOURCE', 'http://127.0.0.1:9200') -CATALOG_QUERY_ENDPOINT = catalog_cfg.get('CATALOG_QUERY_ENDPOINT', '/geoportal/elastic/metadata/item/_search/') - -try: - # Wagtail v5 - INSTALLED_APPS = [ - 'wagtail.contrib.forms', - 'wagtail.contrib.redirects', - 'wagtail.contrib.sitemaps', - 'wagtail.contrib.styleguide', - 'wagtail.contrib.table_block', - 'wagtail.embeds', - 'wagtail.sites', - 'wagtail.users', - 'wagtail.snippets', - 'wagtail.documents', - 'wagtail.images', - 'wagtail.search', - 'wagtail.admin', - 'wagtail', - ] - - import wagtail - WAGTAIL_VERSION = wagtail.VERSION[0] - -except ImportError as e: - # Application definition - try: - # Thanks to tgandor for this inspiration to handle two different wagtail - # versions conditionally while performing this terrible merge: - # https://djangosnippets.org/snippets/3048/ - __import__('wagtail.contrib.forms') - # Wagtail v2 - WAGTAIL_VERSION = 2 - - INSTALLED_APPS = [ - 'wagtail.contrib.forms', - 'wagtail.contrib.redirects', - 'wagtail.embeds', - 'wagtail.sites', - 'wagtail.users', - 'wagtail.snippets', - 'wagtail.documents', - 'wagtail.images', - 'wagtail.search', - 'wagtail.admin', - 'wagtail', - 'wagtail.contrib.styleguide', - 'wagtail.contrib.sitemaps', - 'wagtail.locales', - 'wagtail.contrib.table_block', - 'wagtail.redirects', - ] - - except ImportError as e: - # print(e) - # Wagtail v1 for merging in old MidA Portal - WAGTAIL_VERSION = 1 - INSTALLED_APPS = [ - 'wagtail', - 'wagtail.admin', - 'wagtail.docs', - 'wagtail.snippets', - 'wagtail.users', - 'wagtail.sites', - 'wagtail.images', - 'wagtail.embeds', - 'wagtail.search', - 'wagtail.redirects', - 'wagtail.forms', - 'wagtail.contrib.sitemaps', - ] - -try: - __import__('django_redis') - REDIS_PACKAGE_NAME = 'django_redis' - INSTALLED_APPS += ['django_redis',] -except ImportError as e: - REDIS_PACKAGE_NAME = 'redis_cache' - - -INSTALLED_APPS += [ - 'marco_site', - 'marco.apps.MadronaPortalConfig', - # DLP 2025.06.11: This is where favoring marco/marco static files over other apps happens. - # INSTALLED_APPS sets precedence from top to bottom (e.g., 0 indexed INSTALLED_APPS static files are chosen over other apps). +CATALOG_QUERY_ENDPOINT = catalog_cfg.get( + 'CATALOG_QUERY_ENDPOINT', + '/geoportal/elastic/metadata/item/_search/', +) - # 'kombu.transport.django', +# --------------------------------------------------------------------------- +# Installed Applications (Wagtail 7+) +# --------------------------------------------------------------------------- +import wagtail +WAGTAIL_VERSION = wagtail.VERSION[0] + +INSTALLED_APPS = [ + # Wagtail contrib modules + 'wagtail.contrib.forms', + 'wagtail.contrib.redirects', + 'wagtail.contrib.sitemaps', + 'wagtail.contrib.styleguide', + 'wagtail.contrib.table_block', + # Wagtail core + 'wagtail.embeds', + 'wagtail.sites', + 'wagtail.users', + 'wagtail.snippets', + 'wagtail.documents', + 'wagtail.images', + 'wagtail.search', + 'wagtail.admin', + 'wagtail', + + # Portal application + 'marco_site', + 'marco.apps.MadronaPortalConfig', - # Django-autocomplete-light + # Django Autocomplete Light 'dal', 'dal_select2', 'dal_queryset_sequence', + # Django core 'django.contrib.admin', 'django.contrib.auth', 'django.contrib.contenttypes', @@ -186,35 +173,24 @@ 'django.contrib.sites', 'django.contrib.sitemaps', 'django.contrib.staticfiles', - # 'django.contrib.webdesign', 'django.contrib.humanize', - # 'p97settings', - + # Third-party + 'django_redis', 'email_log', 'djcelery_email', 'compressor', 'taggit', 'modelcluster', - 'rpc4django', + # rpc4django removed — replaced by DRF API views (visualize/api.py, drawing/api.py, mapgroups/api.py) 'tinymce', -] - -# RDH 20240315: this is getting really messy, but django-recaptcha changed from calling itself 'captcha' when it hit v4 -# Newer Wagtail versions use v4, earlier are stuck on v2 or 3, so swapping based on WAGTAIL_VERSION works for now... -if WAGTAIL_VERSION > 4: - INSTALLED_APPS += [ - 'django_recaptcha', - ] -else: - INSTALLED_APPS += [ - 'captcha', - ] - -INSTALLED_APPS += [ + 'django_recaptcha', # Wagtail 7+ uses django-recaptcha v4 (app name: django_recaptcha) 'social_django', - # 'django_redis', + 'flatblocks', + 'import_export', + 'rest_framework', + # Portal sub-apps 'portal.base', 'portal.menu', 'portal.home', @@ -228,11 +204,8 @@ 'portal.initial_data', 'portal.welcome_snippet', 'portal.news', - 'rest_framework', - - 'flatblocks', - # 'wagtailimportexport', + # Ecotrust / Madrona sub-apps 'data_manager', 'layers', 'url_short', @@ -242,36 +215,30 @@ 'drawing', 'manipulators', 'explore', - # 'survey', - - # Account management - 'social.apps.django_app.default', 'accounts.apps.AccountsAppConfig', 'django_social_share', 'mapgroups', - 'import_export', - ] -try: - __import__('nested_admin') - INSTALLED_APPS += ['nested_admin',] -except ImportError as e: - pass - -try: - __import__('colorfield') - INSTALLED_APPS += ['colorfield',] -except ImportError as e: - pass +# Optional apps — installed when available +for _optional_app in ('nested_admin', 'colorfield', 'wagtailcharts'): + try: + __import__(_optional_app) + if _optional_app not in INSTALLED_APPS: + INSTALLED_APPS.append(_optional_app) + except ImportError: + pass + +# --------------------------------------------------------------------------- +# Authentication +# --------------------------------------------------------------------------- AUTHENTICATION_BACKENDS = ( - # 'social.backends.google.GoogleOAuth2', - # 'social.backends.google.GoogleOpenId', - # 'social.backends.facebook.FacebookOAuth2', - # 'social.backends.twitter.TwitterOAuth', 'django.contrib.auth.backends.ModelBackend', ) +# --------------------------------------------------------------------------- +# Middleware +# --------------------------------------------------------------------------- MIDDLEWARE = [ 'django.contrib.sessions.middleware.SessionMiddleware', 'django.middleware.common.CommonMiddleware', @@ -279,211 +246,138 @@ 'django.contrib.auth.middleware.AuthenticationMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware', - - # 'wagtail.middleware.SiteMiddleware', - # 'wagtail.contrib.redirects.middleware.RedirectMiddleware', 'marco.host_site_middleware.HostSiteMiddleware', + 'wagtail.contrib.redirects.middleware.RedirectMiddleware', ] X_FRAME_OPTIONS = 'SAMEORIGIN' -# FILE_UPLOAD_HANDLERS = [ -# 'django.core.files.uploadhandler.MemoryFileUploadHandler', -# 'django.core.files.uploadhandler.TemporaryFileUploadHandler' -# ] - - -# if WAGTAIL_VERSION > 1: -# try: -# __import__('wagtail.middleware.SiteMiddleware') -# MIDDLEWARE += [ -# 'wagtail.middleware.SiteMiddleware', -# ] -# except ImportError as e: -# # https://docs.wagtail.io/en/stable/releases/2.11.html#sitemiddleware-moved-to-wagtail-contrib-legacy -# MIDDLEWARE += [ -# 'wagtail.contrib.legacy.sitemiddleware.SiteMiddleware', -# ] -# WAGTAIL_VERSION = 2.11 -# MIDDLEWARE += [ -# 'wagtail.contrib.redirects.middleware.RedirectMiddleware', -# ] -# else: -MIDDLEWARE += [ - # 'wagtail.middleware.SiteMiddleware', - 'wagtail.contrib.redirects.middleware.RedirectMiddleware', - # 'wagtail.redirects.middleware.RedirectMiddleware', -] - -# Valid site IDs are 1 and 2, corresponding to the primary site(1) and the -# test site(2) +# --------------------------------------------------------------------------- +# URLs / WSGI +# --------------------------------------------------------------------------- SITE_ID = 1 - INTERNAL_IPS = ('127.0.0.1',) - ROOT_URLCONF = 'marco.urls' WSGI_APPLICATION = 'marco.wsgi.application' - -if 'DATABASE' not in cfg.sections(): - cfg['DATABASE'] = {} - -db_cfg = cfg['DATABASE'] - -default = { - 'ENGINE': db_cfg.get('ENGINE', - 'django.contrib.gis.db.backends.postgis'), -} - -if default['ENGINE'].endswith('spatialite'): - SPATIALITE_LIBRARY_PATH = db_cfg.get('SPATIALITE_LIBRARY_PATH') - default['NAME'] = db_cfg.get('NAME', os.path.join(BASE_DIR, 'marco.db')) +# --------------------------------------------------------------------------- +# Database (PostGIS by default) +# Env var overrides (Docker / CI): DB_ENGINE, DB_NAME, DB_USER, DB_PASSWORD, +# DB_HOST, DB_PORT. SQL_* aliases are also accepted for legacy docker-compose +# compatibility. +# --------------------------------------------------------------------------- +_db_engine = ( + os.environ.get('DB_ENGINE') + or os.environ.get('SQL_ENGINE') + or db_cfg.get('ENGINE', 'django.contrib.gis.db.backends.postgis') +) +default_db: dict[str, Any] = {'ENGINE': _db_engine} + +if _db_engine.endswith('spatialite'): + default_db['SPATIALITE_LIBRARY_PATH'] = db_cfg.get('SPATIALITE_LIBRARY_PATH') + default_db['NAME'] = ( + os.environ.get('DB_NAME') + or db_cfg.get('NAME', os.path.join(BASE_DIR, 'marco.db')) + ) else: - default['NAME'] = db_cfg.get('NAME') - # default['NAME'] = os.environ.get("SQL_DATABASE", "ocean_portal"), - if cfg.has_option('DATABASE', 'USER'): - default['USER'] = db_cfg.get('USER') - if cfg.has_option('DATABASE', 'HOST'): - default['HOST'] = db_cfg.get('HOST', 'localhost') - default['PORT'] = db_cfg.getint('PORT', 5432) - if cfg.has_option('DATABASE', 'PASSWORD'): - default['PASSWORD'] = db_cfg.get('PASSWORD') - -DATABASES = {'default': default} - - + default_db['NAME'] = ( + os.environ.get('DB_NAME') or os.environ.get('SQL_DATABASE') + or db_cfg.get('NAME', '') + ) + default_db['USER'] = ( + os.environ.get('DB_USER') or os.environ.get('SQL_USER') + or db_cfg.get('USER', '') + ) + default_db['PASSWORD'] = ( + os.environ.get('DB_PASSWORD') or os.environ.get('SQL_PASSWORD') + or db_cfg.get('PASSWORD', '') + ) + default_db['HOST'] = ( + os.environ.get('DB_HOST') or os.environ.get('SQL_HOST') + or db_cfg.get('HOST', 'localhost') + ) + default_db['PORT'] = int( + os.environ.get('DB_PORT') or os.environ.get('SQL_PORT') + or db_cfg.get('PORT', '5432') + ) + +DATABASES = {'default': default_db} DB_CHANNEL = db_cfg.get('DB_CHANNEL', 'madrona_portal') -if 'CACHES' not in cfg.sections(): - cfg['CACHES'] = {} - -cache_cfg = cfg['CACHES'] +DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' -# ------------------------------------------------------------------------------ -# Redis sessions and caching -# ------------------------------------------------------------------------------ -# SESSION_ENGINE = 'redis_sessions.session' +# --------------------------------------------------------------------------- +# Caching (Redis via django-redis) +# Env var override: REDIS_URL (e.g. redis://:password@tasks:6379/1) +# --------------------------------------------------------------------------- SESSION_ENGINE = "django.contrib.sessions.backends.cache" SESSION_CACHE_ALIAS = "default" -SESSION_REDIS_HOST = 'localhost' -SESSION_REDIS_PORT = 6379 -SESSION_REDIS_DB = 0 -if REDIS_PACKAGE_NAME == 'redis_cache': - { - 'default': { - 'BACKEND': 'redis_cache.RedisCache', - 'LOCATION': '/home/midatlantic/run/redis.sock', - 'KEY_PREFIX': 'marco_portal', - 'OPTIONS': { - 'CLIENT_CLASS': 'redis_cache.client.DefaultClient' - }, - } - } -else: - CACHES = { - 'default': { - 'BACKEND': cache_cfg.get('BACKEND', 'django_redis.cache.RedisCache'), - 'LOCATION': cache_cfg.get('LOCATION', 'redis://127.0.0.1:6379/1'), - 'KEY_PREFIX': 'marco_portal', - 'OPTIONS': { - 'CLIENT_CLASS': cache_cfg.get('CLIENT_CLASS', 'django_redis.client.DefaultClient'), - } - } - } +_redis_location = ( + os.environ.get('REDIS_URL') + or cache_cfg.get('LOCATION', 'redis://127.0.0.1:6379/1') +) -# Internationalization -# https://docs.djangoproject.com/en/1.7/topics/i18n/ +CACHES = { + 'default': { + 'BACKEND': cache_cfg.get('BACKEND', 'django_redis.cache.RedisCache'), + 'LOCATION': _redis_location, + 'KEY_PREFIX': 'marco_portal', + 'OPTIONS': { + 'CLIENT_CLASS': cache_cfg.get('CLIENT_CLASS', 'django_redis.client.DefaultClient'), + }, + } +} +# --------------------------------------------------------------------------- +# Internationalisation +# --------------------------------------------------------------------------- LANGUAGE_CODE = 'en-us' TIME_ZONE = app_cfg.get('TIME_ZONE', 'UTC') USE_I18N = True USE_TZ = True WAGTAIL_I18N_ENABLED = False -WAGTAIL_CONTENT_LANGUAGES = LANGUAGES = [ - ('en', "English"), -] - -# Static files (CSS, JavaScript, Images) -# https://docs.djangoproject.com/en/1.7/howto/static-files/ - -STATIC_ROOT = app_cfg.get('STATIC_ROOT', os.path.join(BASE_DIR, 'static')) -STATIC_URL = app_cfg.get('STATIC_URL', '/static/') -STATIC_CORE = app_cfg.get('STATIC_CORE', '/usr/local/apps/marco_portal_static/') - -static_root_path = os.path.abspath(STATIC_ROOT) -staticfiles_dirs = [] - -for static_dir in (STYLES_DIR, COMPONENTS_DIR, ASSETS_DIR, STATIC_CORE): - if not static_dir: - continue - - normalized_static_dir = os.path.abspath(static_dir) - if normalized_static_dir == static_root_path: +WAGTAIL_CONTENT_LANGUAGES = LANGUAGES = [('en', "English")] + +# --------------------------------------------------------------------------- +# Static & media files +# --------------------------------------------------------------------------- +STATIC_ROOT = _env('STATIC_ROOT', app_cfg, 'STATIC_ROOT', os.path.join(BASE_DIR, 'static')) +STATIC_URL = _env('STATIC_URL', app_cfg, 'STATIC_URL', '/static/') +STATIC_CORE = app_cfg.get('STATIC_CORE', '') + +MEDIA_ROOT = _env('MEDIA_ROOT', app_cfg, 'MEDIA_ROOT', os.path.join(BASE_DIR, 'media')) +MEDIA_URL = _env('MEDIA_URL', app_cfg, 'MEDIA_URL', '/media/') + +_static_root_abs = os.path.abspath(STATIC_ROOT) +_staticfiles_dirs: list[str] = [] +for _dir in (STYLES_DIR, COMPONENTS_DIR, ASSETS_DIR, STATIC_CORE): + if not _dir: continue - - if normalized_static_dir in [os.path.abspath(path) for path in staticfiles_dirs]: + _abs = os.path.abspath(_dir) + if _abs == _static_root_abs or _abs in [os.path.abspath(d) for d in _staticfiles_dirs]: continue + _staticfiles_dirs.append(_dir) - staticfiles_dirs.append(static_dir) - -STATICFILES_DIRS = tuple(staticfiles_dirs) -# Precedence for static files in STATICFILES_DIRS is determined by the order of the directories in STATICFILES_DIRS +STATICFILES_DIRS = tuple(_staticfiles_dirs) STATICFILES_FINDERS = ( 'django.contrib.staticfiles.finders.FileSystemFinder', 'django.contrib.staticfiles.finders.AppDirectoriesFinder', 'django.contrib.staticfiles.finders.DefaultStorageFinder', 'compressor.finders.CompressorFinder', -) -# Precedence for static files for in App (i.e., AppDirectoriesFinder) is determined by the order of INSTALLED_APPS - -MEDIA_ROOT = app_cfg.get('MEDIA_ROOT', os.path.join(BASE_DIR, 'media')) -MEDIA_URL = app_cfg.get('MEDIA_URL', '/media/') - +) -# Django compressor settings +# Django Compressor / SASS COMPRESS_PRECOMPILERS = ( - ('text/x-scss', 'django_libsass.SassCompiler'), # for wagtail + ('text/x-scss', 'django_libsass.SassCompiler'), ) - COMPRESS_ENABLED = app_cfg.getboolean('COMPRESS_ENABLED', True) COMPRESS_OFFLINE = True -try: - # Test is DATA_MANAGER_ADMIN was already defined by PROJECT settings. - DATA_MANAGER_ADMIN -except NameError: - DATA_MANAGER_ADMIN = False - -LAYER_TYPE_CHOICES = ( - ('XYZ', 'XYZ'), - ('WMS', 'WMS'), - ('ArcRest', 'ArcRest'), - ('ArcFeatureServer', 'ArcFeatureServer'), - ('radio', 'radio'), - ('checkbox', 'checkbox'), - ('Vector', 'Vector'), - ('VectorTile', 'VectorTile'), - ('placeholder', 'placeholder'), -) - -# Template configuration - -from django.conf import global_settings - -# Removed due to this: https://stackoverflow.com/a/39315587 - RDH (WCOA) 7/8/2019 -# # RDH (MARCO) 20191114 - if tuple, make into a list -# TEMPLATE_CONTEXT_PROCESSORS = [x for x in global_settings.TEMPLATE_CONTEXT_PROCESSORS] + [ -# 'django.core.context_processors.request', -# 'social_django.context_processors.backends', -# 'portal.base.context_processors.search_disabled', -# ] -# -# TEMPLATE_LOADERS = [x for x in global_settings.TEMPLATE_LOADERS] + [ -# 'apptemplates.Loader', -# ] - +# --------------------------------------------------------------------------- +# Templates +# --------------------------------------------------------------------------- TEMPLATES = [ { 'BACKEND': 'django.template.backends.django.DjangoTemplates', @@ -496,314 +390,243 @@ 'django.template.context_processors.debug', 'django.template.context_processors.request', 'django.contrib.auth.context_processors.auth', - 'django.contrib.messages.context_processors.messages' + 'django.contrib.messages.context_processors.messages', ] }, }, ] -# Wagtail settings - +# --------------------------------------------------------------------------- +# Wagtail +# --------------------------------------------------------------------------- LOGIN_URL = 'account:index' -# LOGIN_REDIRECT_URL = 'wagtailadmin_home' - WAGTAIL_SITE_NAME = 'MARCO Portal' - WAGTAILSEARCH_RESULTS_TEMPLATE = 'portal/search_results.html' - - -# WAGTAILSEARCH_BACKENDS = { -# 'default': { -# 'BACKEND': 'wagtail.search.backends.elasticsearch.ElasticSearch', -# # 'URLS': ['https://iu20e5efzd:dibenj5fn5@point-97-elasticsear-6230081365.us-east-1.bonsai.io'], -# 'URLS': ['https://site:a379ac680e6aaa45f0c129c2cd28d064@bofur-us-east-1.searchly.com'], -# 'INDEX': 'marco_portal', -# 'TIMEOUT': 5, -# } -# } - -# Whether to use face/feature detection to improve image cropping - requires OpenCV WAGTAILIMAGES_FEATURE_DETECTION_ENABLED = False - -# Override the Image class used by wagtailimages with a custom one WAGTAILIMAGES_IMAGE_MODEL = 'base.PortalImage' -try: - FEEDBACK_IFRAME_URL -except NameError as e: - FEEDBACK_IFRAME_URL = "//docs.google.com/forms/d/e/1FAIpQLSdi0nBoQK-3ia8rKtzh7cif0slzDCjA_ACH9Y_ryam-co6p8A/viewform?usp=sf_link" - -try: - DISCLAIMER_BUTTON_DEFAULT -except NameError as e: - DISCLAIMER_BUTTON_DEFAULT = False - -# madrona-features -SHARING_TO_PUBLIC_GROUPS = ['Share with Public'] -SHARING_TO_STAFF_GROUPS = ['Share with Staff'] +# --------------------------------------------------------------------------- +# Map / Geospatial +# --------------------------------------------------------------------------- +MAP_LIBRARY = app_cfg.get('MAP_LIBRARY', 'ol6') +GEOMETRY_DB_SRID = 3857 +GEOMETRY_CLIENT_SRID = 3857 +GEOJSON_SRID = 3857 +GEOJSON_DOWNLOAD = True +SUPPORT_INVERTED_COORDINATES = False +SERVER_SRID = 4326 -# KML SETTINGS -KML_SIMPLIFY_TOLERANCE = 20 # meters -KML_SIMPLIFY_TOLERANCE_DEGREES = 0.0002 # Very roughly ~ 20 meters +KML_SIMPLIFY_TOLERANCE = 20 # metres +KML_SIMPLIFY_TOLERANCE_DEGREES = 0.0002 KML_EXTRUDE_HEIGHT = 100 KML_ALTITUDEMODE_DEFAULT = 'absolute' -# madrona-scenarios -GEOMETRY_DB_SRID = 3857 -GEOMETRY_CLIENT_SRID = 3857 #for latlon -GEOJSON_SRID = 3857 +LAYER_TYPE_CHOICES = ( + ('XYZ', 'XYZ'), + ('WMS', 'WMS'), + ('ArcRest', 'ArcRest'), + ('ArcFeatureServer', 'ArcFeatureServer'), + ('radio', 'radio'), + ('checkbox', 'checkbox'), + ('Vector', 'Vector'), + ('VectorTile', 'VectorTile'), + ('placeholder', 'placeholder'), +) -GEOJSON_DOWNLOAD = True # force headers to treat like an attachment -SUPPORT_INVERTED_COORDINATES = False +# Region defaults (can be overridden by PROJECT settings or config.ini) +PROJECT_REGION: dict = {} +PROJECT_REGION = { + 'name': region_cfg.get('NAME', PROJECT_REGION.get('name', 'Mid-Atlantic Ocean')), + 'init_zoom': region_cfg.getint('INIT_ZOOM', PROJECT_REGION.get('init_zoom', 7)), + 'init_lat': region_cfg.getfloat('INIT_LAT', PROJECT_REGION.get('init_lat', 39.0)), + 'init_lon': region_cfg.getfloat('INIT_LON', PROJECT_REGION.get('init_lon', -74.0)), + 'srid': region_cfg.getint('SRID', PROJECT_REGION.get('srid', 4326)), + 'map': region_cfg.get('MAP', PROJECT_REGION.get('map', 'ocean')), + 'max_zoom': region_cfg.getint('MAX_ZOOM', PROJECT_REGION.get('max_zoom', 13)), +} + +# WMS proxy settings +WMS_PROXY = 'http://tiles.ecotrust.org/mapserver/' +WMS_PROXY_MAPFILE_FIELD = 'map' +WMS_PROXY_MAPFILE = '/mapfiles/generic.map' +WMS_PROXY_LAYERNAME = 'LAYERNAME' +WMS_PROXY_CONNECTION = 'CONN' +WMS_PROXY_FORMAT = 'FORMAT' +WMS_PROXY_VERSION = 'VERSION' +WMS_PROXY_SOURCE_SRS = 'SOURCESRS' +WMS_PROXY_SOURCE_STYLE = 'SRCSTYLE' +WMS_PROXY_TIME_EXTENT = 'TIMEEXT' +WMS_PROXY_TIME = 'TIME' +WMS_PROXY_TIME_DEFAULT = 'TIMEDEF' +WMS_PROXY_TIME_ITEM = 'TIMEITEM' +WMS_PROXY_GENERIC_LAYER = 'generic' +WMS_PROXY_TIME_LAYER = 'time' -# authentication +# --------------------------------------------------------------------------- +# Sharing / features +# --------------------------------------------------------------------------- +DATA_MANAGER_ADMIN = False +SHARING_TO_PUBLIC_GROUPS = ['Share with Public'] +SHARING_TO_STAFF_GROUPS = ['Share with Staff'] +FEEDBACK_IFRAME_URL = ( + "//docs.google.com/forms/d/e/" + "1FAIpQLSdi0nBoQK-3ia8rKtzh7cif0slzDCjA_ACH9Y_ryam-co6p8A/viewform?usp=sf_link" +) +DISCLAIMER_BUTTON_DEFAULT = False + +# --------------------------------------------------------------------------- +# Social Authentication +# --------------------------------------------------------------------------- SOCIAL_AUTH_NEW_USER_URL = '/account/?new=true&login=django' -SOCIAL_AUTH_FACBEOOK_NEW_USER_URL = '/account/?new=true&login=facebook' -# SOCIAL_AUTH_GOOGLE_PLUS_NEW_USER_URL = '/account/?new=true&login=gplus' +SOCIAL_AUTH_FACEBOOK_NEW_USER_URL = '/account/?new=true&login=facebook' SOCIAL_AUTH_TWITTER_NEW_USER_URL = '/account/?new=true&login=twitter' SOCIAL_AUTH_GOOGLE_NEW_USER_URL = '/account/?new=true&login=google' SOCIAL_AUTH_LOGIN_REDIRECT_URL = '/account/?login=django' -# SOCIAL_AUTH_GOOGLE_PLUS_LOGIN_REDIRECT_URL = '/account/?login=gplus' SOCIAL_AUTH_FACEBOOK_LOGIN_REDIRECT_URL = '/account/?login=facebook' SOCIAL_AUTH_TWITTER_LOGIN_REDIRECT_URL = '/account/?login=twitter' SOCIAL_AUTH_GOOGLE_LOGIN_REDIRECT_URL = '/account/?login=google' -# SOCIAL_AUTH_GOOGLE_PLUS_KEY = '' -# SOCIAL_AUTH_GOOGLE_PLUS_SECRET = '' -# SOCIAL_AUTH_GOOGLE_PLUS_SCOPES = ( -# 'https://www.googleapis.com/auth/plus.login', # Minimum needed to login -# 'https://www.googleapis.com/auth/plus.profile.emails.read', # emails -# ) - -if 'SOCIAL_AUTH' not in cfg.sections(): - cfg['SOCIAL_AUTH'] = {} - -social_cfg = cfg['SOCIAL_AUTH'] - -SOCIAL_AUTH_FACEBOOK_KEY = social_cfg.get('FACEBOOK_KEY', '') -SOCIAL_AUTH_FACEBOOK_SECRET = social_cfg.get('FACEBOOK_SECRET', '') +# Env var overrides: FACEBOOK_KEY, FACEBOOK_SECRET, TWITTER_KEY, +# TWITTER_SECRET, GOOGLE_KEY, GOOGLE_SECRET +SOCIAL_AUTH_FACEBOOK_KEY = _env('FACEBOOK_KEY', social_cfg, 'FACEBOOK_KEY', '') +SOCIAL_AUTH_FACEBOOK_SECRET = _env('FACEBOOK_SECRET', social_cfg, 'FACEBOOK_SECRET', '') SOCIAL_AUTH_FACEBOOK_SCOPE = ['public_profile,email'] -SOCIAL_AUTH_TWITTER_KEY = social_cfg.get('TWITTER_KEY', '') -SOCIAL_AUTH_TWITTER_SECRET = social_cfg.get('TWITTER_SECRET', '') +SOCIAL_AUTH_TWITTER_KEY = _env('TWITTER_KEY', social_cfg, 'TWITTER_KEY', '') +SOCIAL_AUTH_TWITTER_SECRET = _env('TWITTER_SECRET', social_cfg, 'TWITTER_SECRET', '') -SOCIAL_AUTH_GOOGLE_OAUTH2_KEY = social_cfg.get('GOOGLE_KEY', '') -SOCIAL_AUTH_GOOGLE_OAUTH2_SECRET = social_cfg.get('GOOGLE_SECRET', '') -#SOCIAL_AUTH_GOOGLE_OAUTH2_SCOPE = [] -SOCIAL_AUTH_GOOGLE_OAUTH2_USE_DEPRECATED_API = True - -# SOCIAL_AUTH_EMAIL_FORCE_EMAIL_VALIDATION = True -SOCIAL_AUTH_EMAIL_VALIDATION_FUNCTION = 'accounts.pipeline.send_validation_email' -SOCIAL_AUTH_EMAIL_VALIDATION_URL = '/account/validate' +SOCIAL_AUTH_GOOGLE_OAUTH2_KEY = _env('GOOGLE_KEY', social_cfg, 'GOOGLE_KEY', '') +SOCIAL_AUTH_GOOGLE_OAUTH2_SECRET = _env('GOOGLE_SECRET', social_cfg, 'GOOGLE_SECRET', '') SOCIAL_AUTH_DISCONNECT_REDIRECT_URL = '/' - SOCIAL_AUTH_JSONFIELD_ENABLED = True +SOCIAL_AUTH_EMAIL_VALIDATION_FUNCTION = 'accounts.pipeline.send_validation_email' +SOCIAL_AUTH_EMAIL_VALIDATION_URL = '/account/validate' -# Our authentication pipeline SOCIAL_AUTH_PIPELINE = ( 'accounts.pipeline.clean_session', - - # Get the information we can about the user and return it in a simple - # format to create the user instance later. On some cases the details are - # already part of the auth response from the provider, but sometimes this - # could hit a provider API. - 'social.pipeline.social_auth.social_details', - - # Get the social uid from whichever service we're authing thru. The uid is - # the unique identifier of the given user in the provider. - 'social.pipeline.social_auth.social_uid', - - # Verifies that the current auth process is valid within the current - # project, this is were emails and domains whitelists are applied (if - # defined). - 'social.pipeline.social_auth.auth_allowed', - - # Checks if the current social-account is already associated in the site. - 'social.pipeline.social_auth.social_user', - - # Make up a username for this person, appends a random string at the end if - # there's any collision. - 'social.pipeline.user.get_username', - - # Confirm with the user that they really want to make an account, also - # make them enter an email address if they somehow didn't - # 'accounts.pipeline.confirm_account', - - # Send a validation email to the user to verify its email address. - 'social.pipeline.mail.mail_validation', - - # Associates the current social details with another user account with - # a similar email address. Disabled by default. - # 'social.pipeline.social_auth.associate_by_email', - - # Create a user account if we haven't found one yet. - 'social.pipeline.user.create_user', - - # Create the record that associated the social account with this user. - 'social.pipeline.social_auth.associate_user', - - # Populate the extra_data field in the social record with the values - # specified by settings (and the default ones like access_token, etc). - 'social.pipeline.social_auth.load_extra_data', - - # Update the user record with any changed info from the auth service. - 'social.pipeline.user.user_details', - - # Set up default django permission groups for new users. + # social-auth-core pipeline steps (social_core.pipeline.*) + 'social_core.pipeline.social_auth.social_details', + 'social_core.pipeline.social_auth.social_uid', + 'social_core.pipeline.social_auth.auth_allowed', + 'social_core.pipeline.social_auth.social_user', + 'social_core.pipeline.user.get_username', + 'social_core.pipeline.mail.mail_validation', + 'social_core.pipeline.user.create_user', + 'social_core.pipeline.social_auth.associate_user', + 'social_core.pipeline.social_auth.load_extra_data', + 'social_core.pipeline.user.user_details', 'accounts.pipeline.set_user_permissions', - - # Grab relevant information from the social provider (avatar) 'accounts.pipeline.get_social_details', - - # 'social.pipeline.debug.debug', 'accounts.pipeline.clean_session', ) -if 'EMAIL' not in cfg.sections(): - cfg['EMAIL'] = {} - -email_cfg = cfg['EMAIL'] - -EMAIL_HOST = email_cfg.get('HOST', 'localhost') -EMAIL_PORT = email_cfg.getint('PORT', 25) -if cfg.has_option('EMAIL', 'HOST_USER') and \ - cfg.has_option('EMAIL', 'HOST_PASSWORD'): - EMAIL_HOST_USER = email_cfg.get('HOST_USER') - EMAIL_HOST_PASSWORD = email_cfg.get('HOST_PASSWORD') -else: - EMAIL_HOST_USER = '' - EMAIL_HOST_PASSWORD = '' - +# --------------------------------------------------------------------------- +# Email +# Env var overrides: EMAIL_HOST, EMAIL_PORT, EMAIL_HOST_USER, +# EMAIL_HOST_PASSWORD, EMAIL_USE_TLS +# --------------------------------------------------------------------------- +EMAIL_HOST = _env('EMAIL_HOST', email_cfg, 'HOST', 'localhost') +EMAIL_PORT = int(_env('EMAIL_PORT', email_cfg, 'PORT', '25')) +EMAIL_HOST_USER = _env('EMAIL_HOST_USER', email_cfg, 'HOST_USER', '') +EMAIL_HOST_PASSWORD = _env('EMAIL_HOST_PASSWORD', email_cfg, 'HOST_PASSWORD', '') EMAIL_BACKEND = email_cfg.get('EMAIL_BACKEND', 'django.core.mail.backends.smtp.EmailBackend') - DEFAULT_FROM_EMAIL = email_cfg.get('DEFAULT_FROM_EMAIL', "MARCO Portal Team ") SERVER_EMAIL = email_cfg.get('SERVER_EMAIL', "MARCO Site Errors ") -EMAIL_USE_TLS = email_cfg.getboolean('EMAIL_USE_TLS', False) -# for mail to admins/managers only +EMAIL_USE_TLS = bool(os.environ.get('EMAIL_USE_TLS', email_cfg.get('EMAIL_USE_TLS', 'false')).lower() in ('1', 'true', 'yes')) EMAIL_SUBJECT_PREFIX = app_cfg.get('EMAIL_SUBJECT_PREFIX', '[MARCO]') + ' ' -if 'AWS' not in cfg.sections(): - cfg['AWS'] = {} - -aws_cfg = cfg['AWS'] - -AWS_ACCESS_KEY_ID = aws_cfg.get('AWS_ACCESS_KEY_ID','') -AWS_SECRET_ACCESS_KEY = aws_cfg.get('AWS_SECRET_ACCESS_KEY','') -AWS_SES_REGION_NAME = aws_cfg.get('AWS_SES_REGION_NAME', 'us-east-1') -AWS_SES_REGION_ENDPOINT = aws_cfg.get('AWS_SES_REGION_ENDPOINT','email.us-east-1.amazonaws.com') - - -if 'CELERY' not in cfg.sections(): - cfg['CELERY'] = {} - -celery_cfg = cfg['CELERY'] - -CELERY_RESULT_BACKEND = celery_cfg.get('CELERY_RESULT_BACKEND', 'redis://localhost:6379') -BROKER_URL = celery_cfg.get('BROKER_URL', 'redis://localhost:6379/0') -CELERY_BROKER_URL = celery_cfg.get('CELERY_BROKER_URL', 'redis://localhost:6379') -CELERY_ALWAYS_EAGER = celery_cfg.get('CELERY_ALWAYS_EAGER', False) -CELERY_DISABLE_RATE_LIMITS = celery_cfg.get('CELERY_DISABLE_RATE_LIMITS', True) - -GA_ACCOUNT = app_cfg.get('GA_ACCOUNT', '') - ADMINS = (('KSDev', 'ksdev@ecotrust.org'),) +# --------------------------------------------------------------------------- +# AWS (SES) +# Env var overrides: AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, +# AWS_SES_REGION_NAME, AWS_SES_REGION_ENDPOINT +# --------------------------------------------------------------------------- +AWS_ACCESS_KEY_ID = _env('AWS_ACCESS_KEY_ID', aws_cfg, 'AWS_ACCESS_KEY_ID', '') +AWS_SECRET_ACCESS_KEY = _env('AWS_SECRET_ACCESS_KEY', aws_cfg, 'AWS_SECRET_ACCESS_KEY', '') +AWS_SES_REGION_NAME = _env('AWS_SES_REGION_NAME', aws_cfg, 'AWS_SES_REGION_NAME', 'us-east-1') +AWS_SES_REGION_ENDPOINT = _env('AWS_SES_REGION_ENDPOINT', aws_cfg, 'AWS_SES_REGION_ENDPOINT', 'email.us-east-1.amazonaws.com') + +# --------------------------------------------------------------------------- +# Celery (Celery 5+ settings) +# Env var overrides: CELERY_BROKER_URL, CELERY_RESULT_BACKEND +# --------------------------------------------------------------------------- +CELERY_BROKER_URL = ( + os.environ.get('CELERY_BROKER_URL') + or os.environ.get('REDIS_URL') + or celery_cfg.get('CELERY_BROKER_URL', 'redis://localhost:6379/0') +) +CELERY_RESULT_BACKEND = ( + os.environ.get('CELERY_RESULT_BACKEND') + or os.environ.get('REDIS_URL') + or celery_cfg.get('CELERY_RESULT_BACKEND', 'redis://localhost:6379') +) +CELERY_TASK_ALWAYS_EAGER = celery_cfg.getboolean('CELERY_ALWAYS_EAGER', False) +CELERY_TASK_RATE_LIMITS_DISABLED = celery_cfg.getboolean('CELERY_DISABLE_RATE_LIMITS', True) + +# --------------------------------------------------------------------------- +# ReCAPTCHA +# --------------------------------------------------------------------------- NOCAPTCHA = True RECAPTCHA_PUBLIC_KEY = app_cfg.get('RECAPTCHA_PUBLIC_KEY', '') -RECAPTCHA_PRIVATE_KEY = app_cfg.get('RECAPTCHA_PRIVATE_KEY','') +RECAPTCHA_PRIVATE_KEY = app_cfg.get('RECAPTCHA_PRIVATE_KEY', '') -# OL2 doesn't support reprojecting rasters, so for WMS servers that don't provide -# EPSG:3857 we send it to a proxy to be re-projected. -WMS_PROXY = 'http://tiles.ecotrust.org/mapserver/' -WMS_PROXY_MAPFILE_FIELD = 'map' -WMS_PROXY_MAPFILE = '/mapfiles/generic.map' -WMS_PROXY_LAYERNAME = 'LAYERNAME' -WMS_PROXY_CONNECTION = 'CONN' -WMS_PROXY_FORMAT = 'FORMAT' -WMS_PROXY_VERSION = 'VERSION' -WMS_PROXY_SOURCE_SRS = 'SOURCESRS' -WMS_PROXY_SOURCE_STYLE = 'SRCSTYLE' -WMS_PROXY_TIME_EXTENT = 'TIMEEXT' -WMS_PROXY_TIME = 'TIME' -WMS_PROXY_TIME_DEFAULT = 'TIMEDEF' -WMS_PROXY_TIME_ITEM = 'TIMEITEM' -WMS_PROXY_GENERIC_LAYER = 'generic' -WMS_PROXY_TIME_LAYER = 'time' - -MAP_LIBRARY = app_cfg.get('MAP_LIBRARY', 'ol6') - -if 'REGION' not in cfg.sections(): - cfg['REGION'] = {} - -region_cfg = cfg['REGION'] - -try: - # Test is PROJECT_REGION was already defined by PROJECT settings. - PROJECT_REGION -except NameError: - PROJECT_REGION = {} - -PROJECT_REGION = { - 'name': region_cfg.get('NAME', PROJECT_REGION['name'] if PROJECT_REGION and 'name' in PROJECT_REGION.keys() else 'Mid-Atlantic Ocean'), - 'init_zoom': region_cfg.getint('INIT_ZOOM', PROJECT_REGION['init_zoom'] if PROJECT_REGION and 'init_zoom' in PROJECT_REGION.keys() else 7), - 'init_lat': region_cfg.getint('INIT_LAT', PROJECT_REGION['init_lat'] if PROJECT_REGION and 'init_lat' in PROJECT_REGION.keys() else 39), - 'init_lon': region_cfg.getint('INIT_LON', PROJECT_REGION['init_lon'] if PROJECT_REGION and 'init_lon' in PROJECT_REGION.keys() else -74), - 'srid': region_cfg.getint('SRID', PROJECT_REGION['srid'] if PROJECT_REGION and 'srid' in PROJECT_REGION.keys() else 4326), - 'map': region_cfg.get('MAP', PROJECT_REGION['map'] if PROJECT_REGION and 'map' in PROJECT_REGION.keys() else 'ocean'), - 'max_zoom': region_cfg.getint('MAX_ZOOM', PROJECT_REGION['max_zoom'] if PROJECT_REGION and 'max_zoom' in PROJECT_REGION.keys() else 13), -} +# --------------------------------------------------------------------------- +# Analytics +# --------------------------------------------------------------------------- +GA_ACCOUNT = app_cfg.get('GA_ACCOUNT', '') -PROJECT_APP = app_cfg.get('PROJECT_APP', False) -if PROJECT_APP and not PROJECT_APP == 'False': +# --------------------------------------------------------------------------- +# Project-level settings overrides +# (Optional app + settings file specified in config.ini) +# --------------------------------------------------------------------------- +PROJECT_APP = app_cfg.get('PROJECT_APP', '') +if PROJECT_APP: INSTALLED_APPS.append(PROJECT_APP) if 'visualize' in INSTALLED_APPS: - from visualize.settings import * + from visualize.settings import * # noqa: F401, F403 if 'data_manager' in INSTALLED_APPS: - from data_manager.settings import * - -DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' + from data_manager.settings import * # noqa: F401, F403 -SERVER_SRID = 4326 - -PROJECT_SETTINGS_FILE = app_cfg.get('PROJECT_SETTINGS_FILE', False) -if PROJECT_SETTINGS_FILE and not PROJECT_SETTINGS_FILE == 'False': +PROJECT_SETTINGS_FILE = app_cfg.get('PROJECT_SETTINGS_FILE', '') +if PROJECT_SETTINGS_FILE: try: from importlib import import_module - APP_MODULE = import_module(PROJECT_APP) - exec("from %s.settings import *" % APP_MODULE.__package__) - except Exception as e: - print(e) - print('PROJECT APP (%s) settings not imported' % PROJECT_APP) -try: - ADDITIONAL_APPS = eval(app_cfg.get('ADDITIONAL_APPS', [])) -except Exception as e: - ADDITIONAL_APPS = [] -try: - ADDITIONAL_MIDDLEWARE = eval(app_cfg.get('ADDITIONAL_MIDDLEWARE', [])) -except Exception as e: - ADDITIONAL_MIDDLEWARE = [] + _project_module = import_module(PROJECT_APP) + _settings_module = import_module(f"{_project_module.__package__}.settings") + # Merge all public names into the current module's namespace + for _k, _v in vars(_settings_module).items(): + if not _k.startswith('_'): + globals()[_k] = _v + except Exception as _e: + import warnings + warnings.warn(f"PROJECT APP ({PROJECT_APP}) settings not imported: {_e}") + +# ADDITIONAL_APPS / ADDITIONAL_MIDDLEWARE — expected as JSON arrays in config.ini +# e.g.: ADDITIONAL_APPS = ["my_custom_app"] +def _parse_list_setting(raw: str) -> list: + """Safely parse a JSON or Python list literal from a config value.""" + if not raw: + return [] + try: + result = json.loads(raw) + if isinstance(result, list): + return result + except (json.JSONDecodeError, TypeError): + pass + try: + result = ast.literal_eval(raw) + if isinstance(result, list): + return result + except (SyntaxError, ValueError): + pass + return [] + +ADDITIONAL_APPS = _parse_list_setting(app_cfg.get('ADDITIONAL_APPS', '')) +ADDITIONAL_MIDDLEWARE = _parse_list_setting(app_cfg.get('ADDITIONAL_MIDDLEWARE', '')) INSTALLED_APPS += ADDITIONAL_APPS MIDDLEWARE += ADDITIONAL_MIDDLEWARE - -if False: - MIDDLEWARE += ['debug_toolbar.middleware.DebugToolbarMiddleware',] - INSTALLED_APPS += ['debug_toolbar',] - DEBUG_TOOLBAR_PANELS2 = [ - 'debug_toolbar.panels.cache.CachePanel', - 'debug_toolbar.panels.headers.HeadersPanel', - 'debug_toolbar.panels.logging.LoggingPanel', - 'debug_toolbar.panels.profiling.ProfilingPanel', - 'debug_toolbar.panels.redirects.RedirectsPanel', - 'debug_toolbar.panels.request.RequestPanel', - 'debug_toolbar.panels.settings.SettingsPanel', - 'debug_toolbar.panels.signals.SignalsPanel', - 'debug_toolbar.panels.staticfiles.StaticFilesPanel', - 'debug_toolbar.templates.panel.TemplatesPanel', - 'debug_toolbar.panels.timer.TimerPanel', - # 'debug_toolbar.sql.panel.SQLPanel', - ] - import debug_toolbar.panels diff --git a/marco/marco/urls.py b/marco/marco/urls.py index 432d48a..aeb65b2 100644 --- a/marco/marco/urls.py +++ b/marco/marco/urls.py @@ -1,115 +1,103 @@ -import os -try: - from django.urls import re_path, include -except (ModuleNotFoundError, ImportError): - from django.conf.urls import include, url as re_path -from django.conf.urls.static import static +""" +URL configuration for the Madrona Portal project. + +Requires: Django 4.2+, Wagtail 7.0+ +""" from django.conf import settings +from django.conf.urls.static import static from django.contrib import admin +from django.urls import include, re_path +from django.views.generic.base import RedirectView -from django.views.generic.base import RedirectView, TemplateView - -if settings.WAGTAIL_VERSION > 1: - from wagtail.admin import urls as wagtailadmin_urls - from wagtail.documents import urls as wagtaildocs_urls - from wagtail import urls as wagtail_urls - from wagtail.contrib.sitemaps.views import sitemap - from wagtail.images import urls as wagtailimages_urls - # Register search signal handlers - from wagtail.search.signal_handlers import register_signal_handlers as wagtailsearch_register_signal_handlers -else: - from wagtail.admin import urls as wagtailadmin_urls - from wagtail.docs import urls as wagtaildocs_urls - from wagtail import urls as wagtail_urls - from wagtail.contrib.sitemaps.views import sitemap - from wagtail.images import urls as wagtailimages_urls - # Register search signal handlers - from wagtail.search.signal_handlers import register_signal_handlers as wagtailsearch_register_signal_handlers - -# from wagtailimportexport import urls as wagtailimportexport_urls +from wagtail.admin import urls as wagtailadmin_urls +from wagtail.documents import urls as wagtaildocs_urls +from wagtail import urls as wagtail_urls +from wagtail.contrib.sitemaps.views import sitemap +from wagtail.images import urls as wagtailimages_urls +from wagtail.search.signal_handlers import register_signal_handlers as wagtailsearch_register_signal_handlers import mapgroups.urls import accounts.urls import explore.urls -from rpc4django.views import serve_rpc_request -from social.apps import django_app from portal.base import views as base_views from portal.data_catalog import views as data_catalog_views from marco_site import views as marco_site_views admin.autodiscover() - - wagtailsearch_register_signal_handlers() -try: - from importlib import import_module - portal_app_urls = import_module(settings.PROJECT_APP + '.urls') - urlpatterns = portal_app_urls.urlpatterns -except Exception as e: - urlpatterns = [] - - +# --------------------------------------------------------------------------- +# Project-specific URL patterns +# Optional: a portal variant (wcoa, mida, etc.) can prepend its own patterns. +# --------------------------------------------------------------------------- +urlpatterns: list = [] + +if settings.PROJECT_APP: + try: + from importlib import import_module + portal_app_urls = import_module(f"{settings.PROJECT_APP}.urls") + urlpatterns = list(getattr(portal_app_urls, 'urlpatterns', [])) + except (ImportError, AttributeError) as e: + import warnings + warnings.warn(f"Could not load URL patterns from PROJECT_APP '{settings.PROJECT_APP}': {e}") + +# --------------------------------------------------------------------------- +# Core URL patterns +# --------------------------------------------------------------------------- urlpatterns += [ - #'', re_path(r'^sitemap\.xml$', sitemap), - re_path(r'django-admin/?', admin.site.urls), + re_path(r'^django-admin/', admin.site.urls), + re_path(r'^admin/', include(wagtailadmin_urls)), - re_path(r'^rpc$', serve_rpc_request), + # /rpc endpoint removed — see each sub-app's api.py for DRF replacements - # https://github.com/omab/python-social-auth/issues/399 - # I want the psa urls to be inside the account urls, but PSA doesn't allow - # nested namespaces. It will likely be fixed in 0.22 + re_path(r'^account/', include('accounts.urls'), name='account'), + re_path(r'^collaborate/groups/', include('mapgroups.urls'), name='groups'), + re_path(r'^groups/', include('mapgroups.urls'), name='groups'), + re_path(r'^g/', RedirectView.as_view(url='/groups/')), # 301 legacy redirect - # re_path(r'^account/auth/', include('social.apps.django_app.urls'), name='social'), - # url('^account/auth/', include('social_django.urls', namespace='social')), - re_path(r'^account/?', include('accounts.urls'), name='account'), - re_path(r'^collaborate/groups/?', include('mapgroups.urls'), name='groups'), - re_path(r'^groups/?', include('mapgroups.urls'), name='groups'), - re_path(r'^g/?', RedirectView.as_view(url='/groups/')), # 301 - - re_path(r'^admin/?', include(wagtailadmin_urls)), - re_path(r'^search/?', base_views.search), - re_path(r'^documents/?', include(wagtaildocs_urls)), + re_path(r'^search/', base_views.search), + re_path(r'^documents/', include(wagtaildocs_urls)), + re_path(r'^images/', include(wagtailimages_urls)), - # url(r'^data-catalog/', include('portal.data_catalog.urls')), - # TODO: we need to prevent Theme names with spaces or special characters. + # Data catalog: named theme pages then the explore SPA + # TODO (POR-206): Restrict theme slugs to prevent spaces and special characters. re_path(r'^data-catalog/([\w\-\s\(\)]+)/?$', data_catalog_views.theme, name="portal.data_catalog.views.theme"), - re_path(r'^data-catalog/[\w\-\s\(\)]*/?', include('explore.urls')), - re_path(r'^data_manager/?', include('layers.urls')), - re_path(r'^old_manager/?', include('data_manager.urls')), - # re_path(r'^data_manager/', include('data_manager.urls')), - re_path(r'^url_shortener/?', include('url_short.urls')), - re_path(r'^layers/?', include('layers.urls')), - re_path(r'^styleguide/?$', marco_site_views.styleguide, name='styleguide'), - re_path(r'^planner/?', include('visualize.urls')), - re_path(r'^embed/?', include('visualize.urls')), - re_path(r'^visualize/?', include('visualize.urls')), - re_path(r'^features/?', include('features.urls')), - re_path(r'^scenario/?', include('scenarios.urls')), - re_path(r'^drawing/?', include('drawing.urls')), - re_path(r'^proxy/?', include('mp_proxy.urls')), - - re_path(r'^join/?', RedirectView.as_view(url='/account/register/')), - - re_path(r'^images/', include(wagtailimages_urls)), + re_path(r'^data-catalog/[\w\-\s\(\)]*/', include('explore.urls')), + + re_path(r'^data_manager/', include('layers.urls')), + re_path(r'^old_manager/', include('data_manager.urls')), + re_path(r'^url_shortener/', include('url_short.urls')), + re_path(r'^layers/', include('layers.urls')), + re_path(r'^styleguide/$', marco_site_views.styleguide, name='styleguide'), + re_path(r'^planner/', include('visualize.urls')), + re_path(r'^embed/', include('visualize.urls')), + re_path(r'^visualize/', include('visualize.urls')), + re_path(r'^features/', include('features.urls')), + re_path(r'^scenario/', include('scenarios.urls')), + re_path(r'^drawing/', include('drawing.urls')), + re_path(r'^proxy/', include('mp_proxy.urls')), + + re_path(r'^join/', RedirectView.as_view(url='/account/register/')), # 301 legacy redirect ] +# Optional survey module if 'survey' in settings.INSTALLED_APPS: urlpatterns += [ - re_path(r'^survey/?', include('survey.urls', namespace='survey')), + re_path(r'^survey/', include('survey.urls', namespace='survey')), ] +# Custom 404 handler if hasattr(settings, 'HANDLER_404'): handler404 = settings.HANDLER_404 - +# Development: serve static and media files through Django if settings.DEBUG: from django.contrib.staticfiles.urls import staticfiles_urlpatterns - urlpatterns += staticfiles_urlpatterns() urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) -urlpatterns += [re_path(r'', include(wagtail_urls)),] +# Wagtail page routing — must be last +urlpatterns += [re_path(r'', include(wagtail_urls))] diff --git a/marco/portal/base/models.py b/marco/portal/base/models.py index 9037b22..2a5895b 100644 --- a/marco/portal/base/models.py +++ b/marco/portal/base/models.py @@ -5,28 +5,11 @@ from django.utils.safestring import mark_safe from django.conf import settings -if settings.WAGTAIL_VERSION > 3: - from wagtail.models import Page - from wagtail.fields import RichTextField, StreamValue - from wagtail.search import index - from wagtail.admin.panels import FieldPanel,MultiFieldPanel,TitleFieldPanel - from wagtail.images.models import AbstractImage, AbstractRendition, Image -elif settings.WAGTAIL_VERSION > 1: - from wagtail.models import Page - from wagtail.fields import RichTextField, StreamValue - from wagtail.search import index - from wagtail.admin.panels import FieldPanel,MultiFieldPanel - from wagtail.images.edit_handlers import ImageChooserPanel - from wagtail.images.models import AbstractImage, AbstractRendition, Image - TitleFieldPanel = FieldPanel -else: - from wagtail.models import Page - from wagtail.fields import RichTextField, StreamValue - from wagtail.search import index - from wagtail.admin.panels import FieldPanel,MultiFieldPanel - from wagtail.images.edit_handlers import ImageChooserPanel - from wagtail.images.models import AbstractImage, AbstractRendition, Image - TitleFieldPanel = FieldPanel +from wagtail.models import Page +from wagtail.fields import RichTextField, StreamValue +from wagtail.search import index +from wagtail.admin.panels import FieldPanel, MultiFieldPanel, TitleFieldPanel +from wagtail.images.models import AbstractImage, AbstractRendition, Image @@ -50,7 +33,7 @@ class PortalImage(AbstractImage): @classmethod def creatable_subpage_models(cls): - print(cls) + pass # Receive the pre_delete signal and delete the file associated with the model instance. @receiver(pre_delete, sender=PortalImage) @@ -60,20 +43,11 @@ def image_delete(sender, instance, **kwargs): class PortalRendition(AbstractRendition): image = models.ForeignKey('PortalImage', related_name='renditions', on_delete=models.CASCADE) - # Wagtail 1.8 deviates drastically from Wagtail 1.7. We need to support both for - # the automated migration from wagtail 1.3 to 2.9 - # TODO: Check if support needed for Wagtail 4.2 https://docs.wagtail.org/en/stable/releases/4.2.html#upgrade-considerations - import wagtail - if hasattr(wagtail, 'VERSION') and wagtail.VERSION[0] > 0 and (wagtail.VERSION[0] > 1 or wagtail.VERSION[1] > 7): - class Meta: - unique_together = ( - ('image', 'filter_spec', 'focal_point_key'), - ) - else: - class Meta: - unique_together = ( - ('image', 'filter_spec', 'focal_point_key'), - ) + + class Meta: + unique_together = ( + ('image', 'filter_spec', 'focal_point_key'), + ) # Receive the pre_delete signal and delete the file associated with the model instance. @receiver(pre_delete, sender=PortalRendition) From 4e59f26007af41bfd24d39c8a3824665ed19f289 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Mon, 30 Mar 2026 19:50:13 -0700 Subject: [PATCH 013/152] Remove unnesscary import of unicode_literals from __future__ in migrations --- marco/portal/base/migrations/0002_auto_20200526_2354.py | 1 - marco/portal/base/migrations/0003_auto_20200526_2357.py | 1 - marco/portal/base/migrations/0004_auto_20200613_0023.py | 1 - marco/portal/calendar/migrations/0001_initial.py | 1 - marco/portal/calendar/migrations/0002_auto_20141205_0036.py | 1 - marco/portal/calendar/migrations/0003_auto_20141218_0154.py | 1 - marco/portal/calendar/migrations/0004_event_location.py | 1 - marco/portal/calendar/migrations/0005_auto_20150109_0053.py | 1 - marco/portal/calendar/migrations/0006_auto_20150112_2303.py | 1 - marco/portal/calendar/migrations/0007_auto_20150121_2313.py | 1 - marco/portal/data_catalog/migrations/0001_initial.py | 1 - .../data_catalog/migrations/0002_datacatalog_description.py | 1 - marco/portal/data_gaps/migrations/0001_initial.py | 1 - marco/portal/data_gaps/migrations/0002_auto_20141205_0024.py | 1 - marco/portal/data_gaps/migrations/0003_auto_20141217_2301.py | 1 - marco/portal/data_gaps/migrations/0004_auto_20150112_2303.py | 1 - marco/portal/data_gaps/migrations/0005_auto_20150121_2319.py | 1 - marco/portal/grid_pages/migrations/0001_initial.py | 1 - marco/portal/grid_pages/migrations/0002_auto_20150429_1843.py | 1 - marco/portal/grid_pages/migrations/0003_auto_20150429_1844.py | 1 - marco/portal/grid_pages/migrations/0004_auto_20160225_0012.py | 1 - marco/portal/home/migrations/0001_initial.py | 1 - marco/portal/home/migrations/0002_create_homepage.py | 1 - marco/portal/home/migrations/0003_homepagecarousel.py | 1 - marco/portal/home/migrations/0004_auto_20171118_0027.py | 1 - .../portal/home/migrations/0005_remove_homepage_feature_image.py | 1 - marco/portal/home/migrations/0006_auto_20171120_2017.py | 1 - marco/portal/home/migrations/0007_auto_20171120_2023.py | 1 - marco/portal/home/migrations/0008_auto_20171121_0042.py | 1 - marco/portal/home/migrations/0009_homestream.py | 1 - marco/portal/home/migrations/0010_auto_20171121_2246.py | 1 - .../portal/home/migrations/0011_remove_homepagecarousel_link.py | 1 - marco/portal/home/migrations/0012_homepagecards.py | 1 - marco/portal/home/migrations/0013_auto_20171123_0013.py | 1 - marco/portal/home/migrations/0014_auto_20171123_0025.py | 1 - marco/portal/home/migrations/0015_homepagecardset.py | 1 - marco/portal/home/migrations/0016_auto_20171130_1854.py | 1 - marco/portal/home/migrations/0017_auto_20171130_2030.py | 1 - marco/portal/home/migrations/0018_auto_20171130_2057.py | 1 - marco/portal/initial_data/migrations/0001_initial_data.py | 1 - marco/portal/initial_data/migrations/0002_create_pages.py | 1 - marco/portal/menu/migrations/0001_initial.py | 1 - .../menu/migrations/0002_menuentry_show_divider_underneath.py | 1 - marco/portal/menu/migrations/0003_menuentry_page.py | 1 - marco/portal/menu/migrations/0004_auto_20150122_2150.py | 1 - marco/portal/menu/migrations/0005_auto_20150518_2309.py | 1 - marco/portal/menu/migrations/0006_auto_20150521_2053.py | 1 - marco/portal/menu/migrations/0007_auto_20171201_2126.py | 1 - marco/portal/news/migrations/0001_initial.py | 1 - marco/portal/news/migrations/0002_auto_20150522_0047.py | 1 - marco/portal/news/migrations/0003_auto_20160225_0012.py | 1 - marco/portal/ocean_stories/migrations/0001_initial.py | 1 - marco/portal/ocean_stories/migrations/0002_auto_20141211_0121.py | 1 - .../ocean_stories/migrations/0003_oceanstory_feature_image.py | 1 - marco/portal/ocean_stories/migrations/0004_auto_20141219_2132.py | 1 - marco/portal/ocean_stories/migrations/0005_auto_20150112_2302.py | 1 - marco/portal/ocean_stories/migrations/0006_auto_20150121_2319.py | 1 - marco/portal/ocean_stories/migrations/0007_auto_20150122_2225.py | 1 - marco/portal/ocean_stories/migrations/0008_auto_20150203_2337.py | 1 - .../migrations/0009_oceanstorysection_map_legend.py | 1 - marco/portal/ocean_stories/migrations/0010_auto_20150603_1721.py | 1 - .../migrations/0011_oceanstory_display_home_page.py | 1 - marco/portal/ocean_stories/migrations/0012_auto_20160225_0012.py | 1 - marco/portal/pages/migrations/0001_initial.py | 1 - marco/portal/pages/migrations/0002_page_description.py | 1 - marco/portal/pages/migrations/0003_auto_20150112_2308.py | 1 - marco/portal/welcome_snippet/migrations/0001_initial.py | 1 - .../welcome_snippet/migrations/0002_welcomepage_use_on_site.py | 1 - .../portal/welcome_snippet/migrations/0003_auto_20150501_1850.py | 1 - .../portal/welcome_snippet/migrations/0004_auto_20150502_1649.py | 1 - marco/portal/welcome_snippet/migrations/0005_welcomepage_body.py | 1 - .../migrations/0006_welcomepageentry_media_image.py | 1 - 72 files changed, 72 deletions(-) diff --git a/marco/portal/base/migrations/0002_auto_20200526_2354.py b/marco/portal/base/migrations/0002_auto_20200526_2354.py index ba83f55..9ca031f 100644 --- a/marco/portal/base/migrations/0002_auto_20200526_2354.py +++ b/marco/portal/base/migrations/0002_auto_20200526_2354.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- # Generated by Django 1.9.13 on 2020-05-26 23:54 -from __future__ import unicode_literals from django.db import migrations, models import django.db.models.deletion diff --git a/marco/portal/base/migrations/0003_auto_20200526_2357.py b/marco/portal/base/migrations/0003_auto_20200526_2357.py index 036f1a9..3e2c0fb 100644 --- a/marco/portal/base/migrations/0003_auto_20200526_2357.py +++ b/marco/portal/base/migrations/0003_auto_20200526_2357.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- # Generated by Django 1.9.13 on 2020-05-26 23:57 -from __future__ import unicode_literals from django.db import migrations # from wagtail.images.utils import get_fill_filter_spec_migrations diff --git a/marco/portal/base/migrations/0004_auto_20200613_0023.py b/marco/portal/base/migrations/0004_auto_20200613_0023.py index 0252a04..166bace 100644 --- a/marco/portal/base/migrations/0004_auto_20200613_0023.py +++ b/marco/portal/base/migrations/0004_auto_20200613_0023.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- # Generated by Django 1.10.8 on 2020-06-13 00:23 -from __future__ import unicode_literals from django.db import migrations, models diff --git a/marco/portal/calendar/migrations/0001_initial.py b/marco/portal/calendar/migrations/0001_initial.py index c2db939..6a37d02 100644 --- a/marco/portal/calendar/migrations/0001_initial.py +++ b/marco/portal/calendar/migrations/0001_initial.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations import django.db.models.deletion diff --git a/marco/portal/calendar/migrations/0002_auto_20141205_0036.py b/marco/portal/calendar/migrations/0002_auto_20141205_0036.py index d3ac3ad..9d72977 100644 --- a/marco/portal/calendar/migrations/0002_auto_20141205_0036.py +++ b/marco/portal/calendar/migrations/0002_auto_20141205_0036.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations from django.conf import settings diff --git a/marco/portal/calendar/migrations/0003_auto_20141218_0154.py b/marco/portal/calendar/migrations/0003_auto_20141218_0154.py index 1e59261..faff65c 100644 --- a/marco/portal/calendar/migrations/0003_auto_20141218_0154.py +++ b/marco/portal/calendar/migrations/0003_auto_20141218_0154.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations import django.db.models.deletion diff --git a/marco/portal/calendar/migrations/0004_event_location.py b/marco/portal/calendar/migrations/0004_event_location.py index 7da9a1b..ca2275e 100644 --- a/marco/portal/calendar/migrations/0004_event_location.py +++ b/marco/portal/calendar/migrations/0004_event_location.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations diff --git a/marco/portal/calendar/migrations/0005_auto_20150109_0053.py b/marco/portal/calendar/migrations/0005_auto_20150109_0053.py index 5139315..e1a576f 100644 --- a/marco/portal/calendar/migrations/0005_auto_20150109_0053.py +++ b/marco/portal/calendar/migrations/0005_auto_20150109_0053.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations diff --git a/marco/portal/calendar/migrations/0006_auto_20150112_2303.py b/marco/portal/calendar/migrations/0006_auto_20150112_2303.py index 37b5343..dd1ea3f 100644 --- a/marco/portal/calendar/migrations/0006_auto_20150112_2303.py +++ b/marco/portal/calendar/migrations/0006_auto_20150112_2303.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations from django.conf import settings diff --git a/marco/portal/calendar/migrations/0007_auto_20150121_2313.py b/marco/portal/calendar/migrations/0007_auto_20150121_2313.py index eb92957..f19e479 100644 --- a/marco/portal/calendar/migrations/0007_auto_20150121_2313.py +++ b/marco/portal/calendar/migrations/0007_auto_20150121_2313.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations import django.db.models.deletion diff --git a/marco/portal/data_catalog/migrations/0001_initial.py b/marco/portal/data_catalog/migrations/0001_initial.py index 0d0fe59..bdebf47 100644 --- a/marco/portal/data_catalog/migrations/0001_initial.py +++ b/marco/portal/data_catalog/migrations/0001_initial.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations import django.db.models.deletion diff --git a/marco/portal/data_catalog/migrations/0002_datacatalog_description.py b/marco/portal/data_catalog/migrations/0002_datacatalog_description.py index df6de02..3f0b858 100644 --- a/marco/portal/data_catalog/migrations/0002_datacatalog_description.py +++ b/marco/portal/data_catalog/migrations/0002_datacatalog_description.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations from django.conf import settings diff --git a/marco/portal/data_gaps/migrations/0001_initial.py b/marco/portal/data_gaps/migrations/0001_initial.py index bf034f9..f3f4329 100644 --- a/marco/portal/data_gaps/migrations/0001_initial.py +++ b/marco/portal/data_gaps/migrations/0001_initial.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations import django.db.models.deletion diff --git a/marco/portal/data_gaps/migrations/0002_auto_20141205_0024.py b/marco/portal/data_gaps/migrations/0002_auto_20141205_0024.py index 5890a17..75cf2e3 100644 --- a/marco/portal/data_gaps/migrations/0002_auto_20141205_0024.py +++ b/marco/portal/data_gaps/migrations/0002_auto_20141205_0024.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations from django.conf import settings diff --git a/marco/portal/data_gaps/migrations/0003_auto_20141217_2301.py b/marco/portal/data_gaps/migrations/0003_auto_20141217_2301.py index 3237ab9..186c953 100644 --- a/marco/portal/data_gaps/migrations/0003_auto_20141217_2301.py +++ b/marco/portal/data_gaps/migrations/0003_auto_20141217_2301.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations import django.db.models.deletion diff --git a/marco/portal/data_gaps/migrations/0004_auto_20150112_2303.py b/marco/portal/data_gaps/migrations/0004_auto_20150112_2303.py index 30368e9..3f29144 100644 --- a/marco/portal/data_gaps/migrations/0004_auto_20150112_2303.py +++ b/marco/portal/data_gaps/migrations/0004_auto_20150112_2303.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations from django.conf import settings diff --git a/marco/portal/data_gaps/migrations/0005_auto_20150121_2319.py b/marco/portal/data_gaps/migrations/0005_auto_20150121_2319.py index a267366..1c4a543 100644 --- a/marco/portal/data_gaps/migrations/0005_auto_20150121_2319.py +++ b/marco/portal/data_gaps/migrations/0005_auto_20150121_2319.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations import django.db.models.deletion diff --git a/marco/portal/grid_pages/migrations/0001_initial.py b/marco/portal/grid_pages/migrations/0001_initial.py index dec3034..41e20f9 100644 --- a/marco/portal/grid_pages/migrations/0001_initial.py +++ b/marco/portal/grid_pages/migrations/0001_initial.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations import django.db.models.deletion diff --git a/marco/portal/grid_pages/migrations/0002_auto_20150429_1843.py b/marco/portal/grid_pages/migrations/0002_auto_20150429_1843.py index 214b60a..e007c73 100644 --- a/marco/portal/grid_pages/migrations/0002_auto_20150429_1843.py +++ b/marco/portal/grid_pages/migrations/0002_auto_20150429_1843.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations diff --git a/marco/portal/grid_pages/migrations/0003_auto_20150429_1844.py b/marco/portal/grid_pages/migrations/0003_auto_20150429_1844.py index 6ca18d4..fe231ce 100644 --- a/marco/portal/grid_pages/migrations/0003_auto_20150429_1844.py +++ b/marco/portal/grid_pages/migrations/0003_auto_20150429_1844.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations diff --git a/marco/portal/grid_pages/migrations/0004_auto_20160225_0012.py b/marco/portal/grid_pages/migrations/0004_auto_20160225_0012.py index 680a44f..78b5537 100644 --- a/marco/portal/grid_pages/migrations/0004_auto_20160225_0012.py +++ b/marco/portal/grid_pages/migrations/0004_auto_20160225_0012.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations diff --git a/marco/portal/home/migrations/0001_initial.py b/marco/portal/home/migrations/0001_initial.py index 7db8060..7e8b3b2 100644 --- a/marco/portal/home/migrations/0001_initial.py +++ b/marco/portal/home/migrations/0001_initial.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations import django.db.models.deletion diff --git a/marco/portal/home/migrations/0002_create_homepage.py b/marco/portal/home/migrations/0002_create_homepage.py index 3649f5f..179b3a2 100644 --- a/marco/portal/home/migrations/0002_create_homepage.py +++ b/marco/portal/home/migrations/0002_create_homepage.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations diff --git a/marco/portal/home/migrations/0003_homepagecarousel.py b/marco/portal/home/migrations/0003_homepagecarousel.py index 0f1b905..02bbc5d 100644 --- a/marco/portal/home/migrations/0003_homepagecarousel.py +++ b/marco/portal/home/migrations/0003_homepagecarousel.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations from django.conf import settings diff --git a/marco/portal/home/migrations/0004_auto_20171118_0027.py b/marco/portal/home/migrations/0004_auto_20171118_0027.py index bf25a9c..41bc57e 100644 --- a/marco/portal/home/migrations/0004_auto_20171118_0027.py +++ b/marco/portal/home/migrations/0004_auto_20171118_0027.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations from django.conf import settings diff --git a/marco/portal/home/migrations/0005_remove_homepage_feature_image.py b/marco/portal/home/migrations/0005_remove_homepage_feature_image.py index daf198c..c50d891 100644 --- a/marco/portal/home/migrations/0005_remove_homepage_feature_image.py +++ b/marco/portal/home/migrations/0005_remove_homepage_feature_image.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations diff --git a/marco/portal/home/migrations/0006_auto_20171120_2017.py b/marco/portal/home/migrations/0006_auto_20171120_2017.py index 16e0158..669e6e9 100644 --- a/marco/portal/home/migrations/0006_auto_20171120_2017.py +++ b/marco/portal/home/migrations/0006_auto_20171120_2017.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations import django.db.models.deletion diff --git a/marco/portal/home/migrations/0007_auto_20171120_2023.py b/marco/portal/home/migrations/0007_auto_20171120_2023.py index 1b45b15..2369451 100644 --- a/marco/portal/home/migrations/0007_auto_20171120_2023.py +++ b/marco/portal/home/migrations/0007_auto_20171120_2023.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations from django.conf import settings diff --git a/marco/portal/home/migrations/0008_auto_20171121_0042.py b/marco/portal/home/migrations/0008_auto_20171121_0042.py index bb702db..7837473 100644 --- a/marco/portal/home/migrations/0008_auto_20171121_0042.py +++ b/marco/portal/home/migrations/0008_auto_20171121_0042.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations diff --git a/marco/portal/home/migrations/0009_homestream.py b/marco/portal/home/migrations/0009_homestream.py index d42800f..3ed082a 100644 --- a/marco/portal/home/migrations/0009_homestream.py +++ b/marco/portal/home/migrations/0009_homestream.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations from django.conf import settings diff --git a/marco/portal/home/migrations/0010_auto_20171121_2246.py b/marco/portal/home/migrations/0010_auto_20171121_2246.py index 499bfe8..9fdd0c3 100644 --- a/marco/portal/home/migrations/0010_auto_20171121_2246.py +++ b/marco/portal/home/migrations/0010_auto_20171121_2246.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations from django.conf import settings diff --git a/marco/portal/home/migrations/0011_remove_homepagecarousel_link.py b/marco/portal/home/migrations/0011_remove_homepagecarousel_link.py index 839f170..0c6c792 100644 --- a/marco/portal/home/migrations/0011_remove_homepagecarousel_link.py +++ b/marco/portal/home/migrations/0011_remove_homepagecarousel_link.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations diff --git a/marco/portal/home/migrations/0012_homepagecards.py b/marco/portal/home/migrations/0012_homepagecards.py index 9e696a8..3ca439e 100644 --- a/marco/portal/home/migrations/0012_homepagecards.py +++ b/marco/portal/home/migrations/0012_homepagecards.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations from django.conf import settings diff --git a/marco/portal/home/migrations/0013_auto_20171123_0013.py b/marco/portal/home/migrations/0013_auto_20171123_0013.py index 6cc0ff8..a2bd8b1 100644 --- a/marco/portal/home/migrations/0013_auto_20171123_0013.py +++ b/marco/portal/home/migrations/0013_auto_20171123_0013.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations diff --git a/marco/portal/home/migrations/0014_auto_20171123_0025.py b/marco/portal/home/migrations/0014_auto_20171123_0025.py index 0c7aa11..3c6cb7b 100644 --- a/marco/portal/home/migrations/0014_auto_20171123_0025.py +++ b/marco/portal/home/migrations/0014_auto_20171123_0025.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations from django.conf import settings diff --git a/marco/portal/home/migrations/0015_homepagecardset.py b/marco/portal/home/migrations/0015_homepagecardset.py index 81be97b..cdc80dc 100644 --- a/marco/portal/home/migrations/0015_homepagecardset.py +++ b/marco/portal/home/migrations/0015_homepagecardset.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations import modelcluster.fields diff --git a/marco/portal/home/migrations/0016_auto_20171130_1854.py b/marco/portal/home/migrations/0016_auto_20171130_1854.py index 28acaa8..0bca061 100644 --- a/marco/portal/home/migrations/0016_auto_20171130_1854.py +++ b/marco/portal/home/migrations/0016_auto_20171130_1854.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations diff --git a/marco/portal/home/migrations/0017_auto_20171130_2030.py b/marco/portal/home/migrations/0017_auto_20171130_2030.py index 25ea6f9..34094f6 100644 --- a/marco/portal/home/migrations/0017_auto_20171130_2030.py +++ b/marco/portal/home/migrations/0017_auto_20171130_2030.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations diff --git a/marco/portal/home/migrations/0018_auto_20171130_2057.py b/marco/portal/home/migrations/0018_auto_20171130_2057.py index 4ab34a5..c0afad8 100644 --- a/marco/portal/home/migrations/0018_auto_20171130_2057.py +++ b/marco/portal/home/migrations/0018_auto_20171130_2057.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations diff --git a/marco/portal/initial_data/migrations/0001_initial_data.py b/marco/portal/initial_data/migrations/0001_initial_data.py index 2da493e..24a9a75 100644 --- a/marco/portal/initial_data/migrations/0001_initial_data.py +++ b/marco/portal/initial_data/migrations/0001_initial_data.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations from django.conf import settings diff --git a/marco/portal/initial_data/migrations/0002_create_pages.py b/marco/portal/initial_data/migrations/0002_create_pages.py index 2a61cac..7bbd042 100644 --- a/marco/portal/initial_data/migrations/0002_create_pages.py +++ b/marco/portal/initial_data/migrations/0002_create_pages.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations from django.conf import settings diff --git a/marco/portal/menu/migrations/0001_initial.py b/marco/portal/menu/migrations/0001_initial.py index 2806295..743f5c9 100644 --- a/marco/portal/menu/migrations/0001_initial.py +++ b/marco/portal/menu/migrations/0001_initial.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations import modelcluster.fields diff --git a/marco/portal/menu/migrations/0002_menuentry_show_divider_underneath.py b/marco/portal/menu/migrations/0002_menuentry_show_divider_underneath.py index 026f7a5..e80fa3b 100644 --- a/marco/portal/menu/migrations/0002_menuentry_show_divider_underneath.py +++ b/marco/portal/menu/migrations/0002_menuentry_show_divider_underneath.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations diff --git a/marco/portal/menu/migrations/0003_menuentry_page.py b/marco/portal/menu/migrations/0003_menuentry_page.py index a3f0bce..accfc0d 100644 --- a/marco/portal/menu/migrations/0003_menuentry_page.py +++ b/marco/portal/menu/migrations/0003_menuentry_page.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations import django.db.models.deletion diff --git a/marco/portal/menu/migrations/0004_auto_20150122_2150.py b/marco/portal/menu/migrations/0004_auto_20150122_2150.py index 836398e..0ecbfb7 100644 --- a/marco/portal/menu/migrations/0004_auto_20150122_2150.py +++ b/marco/portal/menu/migrations/0004_auto_20150122_2150.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations diff --git a/marco/portal/menu/migrations/0005_auto_20150518_2309.py b/marco/portal/menu/migrations/0005_auto_20150518_2309.py index 72cfa7f..1927c07 100644 --- a/marco/portal/menu/migrations/0005_auto_20150518_2309.py +++ b/marco/portal/menu/migrations/0005_auto_20150518_2309.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations diff --git a/marco/portal/menu/migrations/0006_auto_20150521_2053.py b/marco/portal/menu/migrations/0006_auto_20150521_2053.py index 6f97364..bfc3638 100644 --- a/marco/portal/menu/migrations/0006_auto_20150521_2053.py +++ b/marco/portal/menu/migrations/0006_auto_20150521_2053.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations diff --git a/marco/portal/menu/migrations/0007_auto_20171201_2126.py b/marco/portal/menu/migrations/0007_auto_20171201_2126.py index 9d434c4..608d72a 100644 --- a/marco/portal/menu/migrations/0007_auto_20171201_2126.py +++ b/marco/portal/menu/migrations/0007_auto_20171201_2126.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations diff --git a/marco/portal/news/migrations/0001_initial.py b/marco/portal/news/migrations/0001_initial.py index a7bce4e..7db9fe6 100644 --- a/marco/portal/news/migrations/0001_initial.py +++ b/marco/portal/news/migrations/0001_initial.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations import django.db.models.deletion diff --git a/marco/portal/news/migrations/0002_auto_20150522_0047.py b/marco/portal/news/migrations/0002_auto_20150522_0047.py index 0b661ce..06a585b 100644 --- a/marco/portal/news/migrations/0002_auto_20150522_0047.py +++ b/marco/portal/news/migrations/0002_auto_20150522_0047.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations diff --git a/marco/portal/news/migrations/0003_auto_20160225_0012.py b/marco/portal/news/migrations/0003_auto_20160225_0012.py index 811c994..9655f41 100644 --- a/marco/portal/news/migrations/0003_auto_20160225_0012.py +++ b/marco/portal/news/migrations/0003_auto_20160225_0012.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations diff --git a/marco/portal/ocean_stories/migrations/0001_initial.py b/marco/portal/ocean_stories/migrations/0001_initial.py index 5d65c96..020d641 100644 --- a/marco/portal/ocean_stories/migrations/0001_initial.py +++ b/marco/portal/ocean_stories/migrations/0001_initial.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations from django.conf import settings diff --git a/marco/portal/ocean_stories/migrations/0002_auto_20141211_0121.py b/marco/portal/ocean_stories/migrations/0002_auto_20141211_0121.py index 2ff2e79..793b565 100644 --- a/marco/portal/ocean_stories/migrations/0002_auto_20141211_0121.py +++ b/marco/portal/ocean_stories/migrations/0002_auto_20141211_0121.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations diff --git a/marco/portal/ocean_stories/migrations/0003_oceanstory_feature_image.py b/marco/portal/ocean_stories/migrations/0003_oceanstory_feature_image.py index d36fb0b..6d26dea 100644 --- a/marco/portal/ocean_stories/migrations/0003_oceanstory_feature_image.py +++ b/marco/portal/ocean_stories/migrations/0003_oceanstory_feature_image.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations import django.db.models.deletion diff --git a/marco/portal/ocean_stories/migrations/0004_auto_20141219_2132.py b/marco/portal/ocean_stories/migrations/0004_auto_20141219_2132.py index 96b9929..120e302 100644 --- a/marco/portal/ocean_stories/migrations/0004_auto_20141219_2132.py +++ b/marco/portal/ocean_stories/migrations/0004_auto_20141219_2132.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations from django.conf import settings diff --git a/marco/portal/ocean_stories/migrations/0005_auto_20150112_2302.py b/marco/portal/ocean_stories/migrations/0005_auto_20150112_2302.py index 333b372..d98e35d 100644 --- a/marco/portal/ocean_stories/migrations/0005_auto_20150112_2302.py +++ b/marco/portal/ocean_stories/migrations/0005_auto_20150112_2302.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations from django.conf import settings diff --git a/marco/portal/ocean_stories/migrations/0006_auto_20150121_2319.py b/marco/portal/ocean_stories/migrations/0006_auto_20150121_2319.py index 5bfbe3a..6eb4b00 100644 --- a/marco/portal/ocean_stories/migrations/0006_auto_20150121_2319.py +++ b/marco/portal/ocean_stories/migrations/0006_auto_20150121_2319.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations import django.db.models.deletion diff --git a/marco/portal/ocean_stories/migrations/0007_auto_20150122_2225.py b/marco/portal/ocean_stories/migrations/0007_auto_20150122_2225.py index ae5f42c..0b3ae3d 100644 --- a/marco/portal/ocean_stories/migrations/0007_auto_20150122_2225.py +++ b/marco/portal/ocean_stories/migrations/0007_auto_20150122_2225.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations import django.db.models.deletion diff --git a/marco/portal/ocean_stories/migrations/0008_auto_20150203_2337.py b/marco/portal/ocean_stories/migrations/0008_auto_20150203_2337.py index d02fc7a..ec1a6d3 100644 --- a/marco/portal/ocean_stories/migrations/0008_auto_20150203_2337.py +++ b/marco/portal/ocean_stories/migrations/0008_auto_20150203_2337.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations diff --git a/marco/portal/ocean_stories/migrations/0009_oceanstorysection_map_legend.py b/marco/portal/ocean_stories/migrations/0009_oceanstorysection_map_legend.py index 3fa7eb7..9f8821f 100644 --- a/marco/portal/ocean_stories/migrations/0009_oceanstorysection_map_legend.py +++ b/marco/portal/ocean_stories/migrations/0009_oceanstorysection_map_legend.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations diff --git a/marco/portal/ocean_stories/migrations/0010_auto_20150603_1721.py b/marco/portal/ocean_stories/migrations/0010_auto_20150603_1721.py index 0048a56..3f0ea79 100644 --- a/marco/portal/ocean_stories/migrations/0010_auto_20150603_1721.py +++ b/marco/portal/ocean_stories/migrations/0010_auto_20150603_1721.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations diff --git a/marco/portal/ocean_stories/migrations/0011_oceanstory_display_home_page.py b/marco/portal/ocean_stories/migrations/0011_oceanstory_display_home_page.py index c6b61bd..3ef1dab 100644 --- a/marco/portal/ocean_stories/migrations/0011_oceanstory_display_home_page.py +++ b/marco/portal/ocean_stories/migrations/0011_oceanstory_display_home_page.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations diff --git a/marco/portal/ocean_stories/migrations/0012_auto_20160225_0012.py b/marco/portal/ocean_stories/migrations/0012_auto_20160225_0012.py index 3096e62..ad32325 100644 --- a/marco/portal/ocean_stories/migrations/0012_auto_20160225_0012.py +++ b/marco/portal/ocean_stories/migrations/0012_auto_20160225_0012.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations diff --git a/marco/portal/pages/migrations/0001_initial.py b/marco/portal/pages/migrations/0001_initial.py index 0630053..e9fc007 100644 --- a/marco/portal/pages/migrations/0001_initial.py +++ b/marco/portal/pages/migrations/0001_initial.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations from django.conf import settings diff --git a/marco/portal/pages/migrations/0002_page_description.py b/marco/portal/pages/migrations/0002_page_description.py index 4b60b6a..04a7024 100644 --- a/marco/portal/pages/migrations/0002_page_description.py +++ b/marco/portal/pages/migrations/0002_page_description.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations from django.conf import settings diff --git a/marco/portal/pages/migrations/0003_auto_20150112_2308.py b/marco/portal/pages/migrations/0003_auto_20150112_2308.py index 7efd42c..98f8b22 100644 --- a/marco/portal/pages/migrations/0003_auto_20150112_2308.py +++ b/marco/portal/pages/migrations/0003_auto_20150112_2308.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations from django.conf import settings diff --git a/marco/portal/welcome_snippet/migrations/0001_initial.py b/marco/portal/welcome_snippet/migrations/0001_initial.py index 04d310f..a260c22 100644 --- a/marco/portal/welcome_snippet/migrations/0001_initial.py +++ b/marco/portal/welcome_snippet/migrations/0001_initial.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations import django.db.models.deletion diff --git a/marco/portal/welcome_snippet/migrations/0002_welcomepage_use_on_site.py b/marco/portal/welcome_snippet/migrations/0002_welcomepage_use_on_site.py index b3a5ed4..a53b9f3 100644 --- a/marco/portal/welcome_snippet/migrations/0002_welcomepage_use_on_site.py +++ b/marco/portal/welcome_snippet/migrations/0002_welcomepage_use_on_site.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations diff --git a/marco/portal/welcome_snippet/migrations/0003_auto_20150501_1850.py b/marco/portal/welcome_snippet/migrations/0003_auto_20150501_1850.py index 32f023c..5e11fb2 100644 --- a/marco/portal/welcome_snippet/migrations/0003_auto_20150501_1850.py +++ b/marco/portal/welcome_snippet/migrations/0003_auto_20150501_1850.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations diff --git a/marco/portal/welcome_snippet/migrations/0004_auto_20150502_1649.py b/marco/portal/welcome_snippet/migrations/0004_auto_20150502_1649.py index 1e027d1..248dde3 100644 --- a/marco/portal/welcome_snippet/migrations/0004_auto_20150502_1649.py +++ b/marco/portal/welcome_snippet/migrations/0004_auto_20150502_1649.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations from django.conf import settings diff --git a/marco/portal/welcome_snippet/migrations/0005_welcomepage_body.py b/marco/portal/welcome_snippet/migrations/0005_welcomepage_body.py index bade912..ff50f5b 100644 --- a/marco/portal/welcome_snippet/migrations/0005_welcomepage_body.py +++ b/marco/portal/welcome_snippet/migrations/0005_welcomepage_body.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations from django.conf import settings diff --git a/marco/portal/welcome_snippet/migrations/0006_welcomepageentry_media_image.py b/marco/portal/welcome_snippet/migrations/0006_welcomepageentry_media_image.py index f899227..5489bda 100644 --- a/marco/portal/welcome_snippet/migrations/0006_welcomepageentry_media_image.py +++ b/marco/portal/welcome_snippet/migrations/0006_welcomepageentry_media_image.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations import django.db.models.deletion From f7e573c55697b586981ec619b547d8e317432de8 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Mon, 30 Mar 2026 19:50:19 -0700 Subject: [PATCH 014/152] Refactor Wagtail imports to simplify version handling in models.py --- marco/portal/home/models.py | 27 +++++---------------------- 1 file changed, 5 insertions(+), 22 deletions(-) diff --git a/marco/portal/home/models.py b/marco/portal/home/models.py index 14c7077..7dcbd83 100644 --- a/marco/portal/home/models.py +++ b/marco/portal/home/models.py @@ -6,28 +6,11 @@ from django.conf import settings -if settings.WAGTAIL_VERSION > 3: - from wagtail.models import Orderable, Page - from wagtail.fields import RichTextField - from wagtail.search import index - from wagtail.admin.panels import FieldPanel,InlinePanel,MultiFieldPanel,FieldRowPanel,PageChooserPanel,TitleFieldPanel - from wagtail.images.models import Image -elif settings.WAGTAIL_VERSION > 1: - from wagtail.models import Orderable, Page - from wagtail.fields import RichTextField - from wagtail.search import index - from wagtail.admin.panels import FieldPanel,InlinePanel,MultiFieldPanel,FieldRowPanel,PageChooserPanel - from wagtail.images.models import Image - from wagtail.images.edit_handlers import ImageChooserPanel - TitleFieldPanel = FieldPanel -else: - from wagtail.models import Orderable, Page - from wagtail.fields import RichTextField - from wagtail.search import index - from wagtail.admin.panels import FieldPanel,InlinePanel,MultiFieldPanel,FieldRowPanel,PageChooserPanel - from wagtail.images.models import Image - from wagtail.images.edit_handlers import ImageChooserPanel - TitleFieldPanel = FieldPanel +from wagtail.models import Orderable, Page +from wagtail.fields import RichTextField +from wagtail.search import index +from wagtail.admin.panels import FieldPanel, InlinePanel, MultiFieldPanel, FieldRowPanel, PageChooserPanel, TitleFieldPanel +from wagtail.images.models import Image from portal.ocean_stories.models import OceanStory From 7aadb0b1b23547375890d31ced9f6ac0a057092e Mon Sep 17 00:00:00 2001 From: David Pollard Date: Tue, 31 Mar 2026 12:08:08 -0700 Subject: [PATCH 015/152] Add static file compression to Docker entrypoint script --- docker/entrypoint.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 57d664d..12dfd9e 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -34,6 +34,7 @@ PY # --------------------------------------------------------------------------- python marco/manage.py migrate --noinput python marco/manage.py collectstatic --noinput +python marco/manage.py compress --force # --------------------------------------------------------------------------- # 3. Seed a fresh database with initial fixture data From d1044f061fb33071743c9c38d7f254d7d0dd8f5d Mon Sep 17 00:00:00 2001 From: David Pollard Date: Tue, 31 Mar 2026 17:12:26 -0700 Subject: [PATCH 016/152] Add superuser bootstrap to entrypoint and Docker configuration --- .env.example | 8 ++++++++ docker/docker-compose.yml | 6 ++++++ docker/entrypoint.sh | 26 +++++++++++++++++++++++++- 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/.env.example b/.env.example index b08fb8e..7d2d7bc 100644 --- a/.env.example +++ b/.env.example @@ -65,3 +65,11 @@ GOOGLE_SECRET= # --------------------------------------------------------------------------- # RECAPTCHA_PUBLIC_KEY= # RECAPTCHA_PRIVATE_KEY= + +# --------------------------------------------------------------------------- +# Dev superuser bootstrap (entrypoint creates this user on first start) +# Leave DJANGO_SUPERUSER_PASSWORD empty to skip superuser creation. +# --------------------------------------------------------------------------- +DJANGO_SUPERUSER_USERNAME=admin +DJANGO_SUPERUSER_EMAIL=admin@example.com +DJANGO_SUPERUSER_PASSWORD= diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 740a41e..5a6c759 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -40,6 +40,12 @@ services: - REDIS_URL=redis://${REDIS_PASSWORD:+:${REDIS_PASSWORD}@}tasks:6379/1 - CELERY_BROKER_URL=redis://${REDIS_PASSWORD:+:${REDIS_PASSWORD}@}tasks:6379/0 + # Superuser bootstrap — only creates if username doesn't already exist. + # Leave DJANGO_SUPERUSER_PASSWORD empty to skip (safe for production). + - DJANGO_SUPERUSER_USERNAME=${DJANGO_SUPERUSER_USERNAME:-admin} + - DJANGO_SUPERUSER_EMAIL=${DJANGO_SUPERUSER_EMAIL:-admin@example.com} + - DJANGO_SUPERUSER_PASSWORD=${DJANGO_SUPERUSER_PASSWORD:-} + # Application server mode - DJANGO_ENV=${DJANGO_ENV:-development} - GUNICORN_WORKERS=${GUNICORN_WORKERS:-3} diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 12dfd9e..7f87490 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -89,7 +89,31 @@ else fi # --------------------------------------------------------------------------- -# 4. Start the application server +# 4. Create superuser (only when DJANGO_SUPERUSER_PASSWORD is set and the +# username does not already exist — safe to run on every restart) +# --------------------------------------------------------------------------- +if [ -n "${DJANGO_SUPERUSER_PASSWORD:-}" ]; then + python - < Date: Wed, 1 Apr 2026 11:26:49 -0700 Subject: [PATCH 017/152] Fix fixture seeding, social auth namespace, and static path resolution - entrypoint.sh: switch fixture from initial_data.json to initial_data_prod.json; also load scenarios/fixtures/initial_data.json for reference data; raise fresh-DB threshold from ==0 to <5 pages (initial_data migration creates 1 page); clear django_site before load to avoid PK conflict - urls.py: add social_django.urls under 'social' namespace to fix NoReverseMatch at /account/ - settings.py: add APPEND_SLASH=True Co-Authored-By: Claude Sonnet 4.6 --- docker/entrypoint.sh | 14 ++++++++++++-- marco/marco/settings.py | 1 + marco/marco/urls.py | 1 + 3 files changed, 14 insertions(+), 2 deletions(-) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 7f87490..71885a4 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -60,7 +60,7 @@ PY echo "Content pages in database: ${CONTENT_PAGES}" -if [ "${CONTENT_PAGES}" = "0" ] || [ "${FORCE_RELOAD_FIXTURES:-0}" = "1" ]; then +if [ "${CONTENT_PAGES:-0}" -lt "5" ] || [ "${FORCE_RELOAD_FIXTURES:-0}" = "1" ]; then echo "Fresh database detected — loading initial fixtures..." # Clear stale search index entries and image renditions so the fixture @@ -75,6 +75,11 @@ django.setup() from wagtail.search.models import Query Query.objects.all().delete() +# Remove the default site created by the sites migration so the +# fixture can load its own site configuration without a key conflict. +from django.contrib.sites.models import Site +Site.objects.all().delete() + try: from portal.base.models import PortalRendition PortalRendition.objects.all().delete() @@ -82,7 +87,12 @@ except Exception: pass PY - python marco/manage.py loaddata initial_data.json + python marco/manage.py loaddata initial_data_prod.json + # Load per-app reference fixtures that aren't included in the main fixture. + # Use absolute paths so only this specific file is loaded (not other apps' + # initial_data.json files that happen to share the same name). + python marco/manage.py loaddata \ + apps/madrona-scenarios/scenarios/fixtures/initial_data.json echo "Initial fixtures loaded." else echo "Existing database — skipping fixture load." diff --git a/marco/marco/settings.py b/marco/marco/settings.py index 42198bc..a39ffc2 100644 --- a/marco/marco/settings.py +++ b/marco/marco/settings.py @@ -259,6 +259,7 @@ def _parse_hosts(raw: str | None) -> list[str]: INTERNAL_IPS = ('127.0.0.1',) ROOT_URLCONF = 'marco.urls' WSGI_APPLICATION = 'marco.wsgi.application' +APPEND_SLASH=True # --------------------------------------------------------------------------- # Database (PostGIS by default) diff --git a/marco/marco/urls.py b/marco/marco/urls.py index aeb65b2..96f21a0 100644 --- a/marco/marco/urls.py +++ b/marco/marco/urls.py @@ -53,6 +53,7 @@ # /rpc endpoint removed — see each sub-app's api.py for DRF replacements + re_path(r'^auth/', include('social_django.urls', namespace='social')), re_path(r'^account/', include('accounts.urls'), name='account'), re_path(r'^collaborate/groups/', include('mapgroups.urls'), name='groups'), re_path(r'^groups/', include('mapgroups.urls'), name='groups'), From cf01febc5c9f46a8eb897ec6430cf46f2f78221b Mon Sep 17 00:00:00 2001 From: David Pollard Date: Wed, 1 Apr 2026 11:35:17 -0700 Subject: [PATCH 018/152] Remove Wagtail 6 Query import from entrypoint cleanup wagtail.search.models.Query was removed in Wagtail 7. The site fixture no longer uses natural-foreign keys so the Query table cleanup is no longer needed anyway. Co-Authored-By: Claude Sonnet 4.6 --- docker/entrypoint.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 71885a4..cbe6014 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -72,9 +72,6 @@ os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'marco.settings') import django django.setup() -from wagtail.search.models import Query -Query.objects.all().delete() - # Remove the default site created by the sites migration so the # fixture can load its own site configuration without a key conflict. from django.contrib.sites.models import Site From 05436a2870dd85189823908502413996cae32bbc Mon Sep 17 00:00:00 2001 From: David Pollard Date: Wed, 1 Apr 2026 11:40:56 -0700 Subject: [PATCH 019/152] Clear migration-created pages before loading fixture The initial_data migration creates placeholder Wagtail pages whose tree paths conflict with pages in the fixture. Delete depth>1 pages before loaddata so the fixture can load its own page tree cleanly. Co-Authored-By: Claude Sonnet 4.6 --- docker/entrypoint.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index cbe6014..807d0e8 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -63,8 +63,7 @@ echo "Content pages in database: ${CONTENT_PAGES}" if [ "${CONTENT_PAGES:-0}" -lt "5" ] || [ "${FORCE_RELOAD_FIXTURES:-0}" = "1" ]; then echo "Fresh database detected — loading initial fixtures..." - # Clear stale search index entries and image renditions so the fixture - # loads cleanly into the empty database. + # Clear rows created by migrations that would conflict with fixture data. python - <<'PY' import sys, os sys.path.insert(0, 'marco') @@ -72,11 +71,14 @@ os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'marco.settings') import django django.setup() -# Remove the default site created by the sites migration so the -# fixture can load its own site configuration without a key conflict. +# The sites migration creates a default site and initial_data migration +# creates placeholder pages — both conflict with fixture data. from django.contrib.sites.models import Site Site.objects.all().delete() +from wagtail.models import Page +Page.objects.filter(depth__gt=1).delete() + try: from portal.base.models import PortalRendition PortalRendition.objects.all().delete() From bb8232335abcb144fdf72f8e72df70530f0ada0a Mon Sep 17 00:00:00 2001 From: David Pollard Date: Wed, 1 Apr 2026 15:08:54 -0700 Subject: [PATCH 020/152] rewrite docker/README.md to reflect current stack accurately - correct env file location (.env at project root, not docker/.env) - correct variable names (DB_* not SQL_*) - correct ports (5432, 6379, not 65432/8379) - document --profile full requirement - document buildx build requirement and why (BuildKit git-context caching) - document entrypoint sequence including fixture seeding and superuser bootstrap - add dev-only infrastructure mode (no app container) - update all docker compose commands with correct flags Co-Authored-By: Claude Sonnet 4.6 --- docker/README.md | 215 ++++++++++++++++++++++++++++++++++------------- 1 file changed, 156 insertions(+), 59 deletions(-) diff --git a/docker/README.md b/docker/README.md index b6f5bb8..7f65781 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,99 +1,196 @@ -# WCOA Docker Development Guide +# Docker Development Guide — Madrona Portal (WCOA) -This guide switches local development from the Vagrant workflow to Docker -Compose for `madrona_portal` + `wcoa`. +## Repository layout -It is aligned with the install process documented in the project wiki: -https://github.com/Ecotrust/madrona-portal/wiki/Installation +This stack assumes the standard monorepo layout: -## 1) Prerequisites +``` +madrona-apps-claude/ +├── madrona_portal/ ← main Django project (this repo) +│ ├── docker/ +│ │ ├── docker-compose.yml +│ │ └── entrypoint.sh +│ ├── Dockerfile +│ └── .env ← your local secrets (never committed) +└── madrona-apps/ ← sibling repo with all sub-app packages + ├── wcoa/ + ├── mp-layers/ + └── ... +``` -- Docker Desktop (or Docker Engine + Compose plugin) -- Local checkout layout where this repository has sibling module directories in - `../madrona-apps` (already true in this workspace) +All commands below are run from the **`madrona_portal/`** directory unless noted. -## 2) Keep companion apps checked out +--- -The Docker image installs local editable dependencies from -`/usr/local/apps/madrona-portal/apps/...`, so keep companion repos present in -`../madrona-apps` as referenced by `docker/docker-requirements.txt`. +## 1. Prerequisites -## 3) Set Docker environment values +- Docker Desktop (Mac/Windows) or Docker Engine + Compose plugin (Linux) +- `madrona-apps/` checked out as a sibling of `madrona_portal/` (the Dockerfile copies from it at build time) -Edit `docker/.env` and set at minimum: +--- -- `SECRET_KEY` -- `SQL_DATABASE` -- `SQL_USER` -- `SQL_PASSWORD` -- `ALLOWED_HOSTS` +## 2. Configure environment -Default local ports in this repo: +Copy the example and fill in real values: -- Django app: `8000` -- PostGIS on host: `65432` -- Redis on host: `8379` +```bash +cp .env.example .env # if .env.example exists; otherwise edit .env directly +``` -## 4) Use the Docker-specific WCOA Django config +Minimum required values in `.env`: -Compose is configured to run with: +```ini +SECRET_KEY= +DB_PASSWORD= +``` -- `MP_PROJECT_CONFIG=config.wcoa.docker.ini` +Other notable defaults (override in `.env` as needed): -That file lives at `marco/config.wcoa.docker.ini` and points Django to: +| Variable | Default | Notes | +|---|---|---| +| `APP_PORT` | `8000` | Host port the Django app binds to | +| `DB_NAME` | `wcoa_docker_db` | PostgreSQL database name | +| `DB_USER` | `postgres` | PostgreSQL user | +| `DB_PORT` | `5432` | Host port for PostgreSQL | +| `REDIS_PORT` | `6379` | Host port for Redis | +| `MP_PROJECT_CONFIG` | `config.wcoa.docker.ini` | Django config file (do not change for WCOA) | +| `DEBUG` | `False` | Set `True` to use Django dev server instead of gunicorn | +| `DJANGO_SUPERUSER_PASSWORD` | *(empty)* | If set, a superuser is created on first start | +| `DJANGO_SUPERUSER_USERNAME` | `admin` | Superuser username | +| `DJANGO_SUPERUSER_EMAIL` | `admin@example.com` | Superuser email | -- PostGIS host `db` -- Redis host `tasks` -- Container-friendly static/media paths under `/vol/web` +--- -## 5) Build and start +## 3. Build the image -From `madrona_portal/`: +Use `docker buildx build` directly — `docker compose build` has a known caching issue where it silently reads committed (not filesystem) file versions when a `.git` directory exists in the build context. + +From the **repo root** (`madrona-apps-claude/`): ```bash -cd docker -docker compose --env-file .env up --build +docker buildx build \ + --builder desktop-linux \ + --load \ + -f madrona_portal/Dockerfile \ + -t madrona_portal-app:latest \ + . ``` -The entrypoint waits for PostGIS, then runs: +Add `--no-cache` to force a full rebuild (e.g. after changing `requirements.txt`). + +> **Important:** Always commit changes to `madrona_portal/` before rebuilding. BuildKit reads files from the git object store, not the filesystem, when a `.git` directory is present in the build context. + +--- + +## 4. Start the full stack + +```bash +docker compose -f docker/docker-compose.yml --env-file .env --profile full up -d +``` + +The `--profile full` flag is required to start the `app` service. Without it, only `db` (PostGIS) and `tasks` (Redis) start — useful for running Django locally against Docker infrastructure. + +Services: + +| Service | Image | Host port | +|---|---|---| +| `app` | `madrona_portal-app:latest` | `${APP_PORT}` (default 8000) | +| `db` | `postgis/postgis:16-3.4` | `${DB_PORT}` (default 5432) | +| `tasks` | `redis:7-alpine` | `${REDIS_PORT}` (default 6379) | + +--- + +## 5. What happens on first startup -- `collectstatic` -- `migrate` -- `runserver 0:8000` +The entrypoint (`docker/entrypoint.sh`) runs in order: -Open: http://localhost:8000/ +1. **Waits** for PostgreSQL to accept connections +2. **Migrates** (`manage.py migrate`) +3. **Collects static files** (`manage.py collectstatic`) +4. **Compresses assets** (`manage.py compress --force`) +5. **Seeds fixtures** — if fewer than 5 content pages exist (fresh DB), loads: + - `apps/wcoa/wcoa/fixtures/initial_data_prod.json` (1,782 objects: pages, layers, themes, etc.) + - `apps/madrona-scenarios/scenarios/fixtures/initial_data.json` (22 objects) +6. **Creates superuser** — only if `DJANGO_SUPERUSER_PASSWORD` is set and the username doesn't exist +7. **Starts the server** — gunicorn in production (`DEBUG=False`), Django dev server otherwise -## 6) Common one-off commands +Open: http://localhost:${APP_PORT}/ -From `madrona_portal/docker`: +--- + +## 6. Dev infrastructure only (no app container) + +To run Django locally with only the Docker DB and Redis: ```bash -docker compose --env-file .env run --rm app python marco/manage.py createsuperuser -docker compose --env-file .env run --rm app python marco/manage.py shell -docker compose --env-file .env run --rm app python marco/manage.py loaddata /path/to/fixture.json +docker compose -f docker/docker-compose.yml --env-file .env up -d +# db and tasks start; app does not (no --profile full) + +cd marco +python manage.py runserver ``` -## 7) Data migration from old Vagrant DB +--- -If you are moving existing data, dump from Vagrant PostgreSQL and import into the -Docker `db` service: +## 7. Common one-off commands ```bash -# Example import into running Docker DB -cat ./path/to/old_dump.sql | docker compose --env-file .env exec -T db psql -U "$SQL_USER" -d "$SQL_DATABASE" +# Django shell +docker compose -f docker/docker-compose.yml --env-file .env --profile full \ + run --rm app python marco/manage.py shell + +# Create superuser manually +docker compose -f docker/docker-compose.yml --env-file .env --profile full \ + run --rm app python marco/manage.py createsuperuser + +# Run migrations +docker compose -f docker/docker-compose.yml --env-file .env --profile full \ + run --rm app python marco/manage.py migrate + +# Load a fixture +docker compose -f docker/docker-compose.yml --env-file .env --profile full \ + run --rm app python marco/manage.py loaddata /path/to/fixture.json + +# Open a psql shell in the DB container +docker exec -it docker-db-1 psql -U postgres wcoa_docker_db ``` -## 8) Stop and clean up +--- + +## 8. Rebuild after code changes + +After changing Python source files, templates, or static assets in `madrona_portal/` or `madrona-apps/`: + +1. Commit your changes (required for BuildKit to pick them up) +2. Rebuild from the repo root: + ```bash + docker buildx build --builder desktop-linux --load \ + -f madrona_portal/Dockerfile -t madrona_portal-app:latest . + ``` +3. Restart the app container: + ```bash + docker compose -f docker/docker-compose.yml --env-file .env --profile full \ + up -d --force-recreate app + ``` + +--- + +## 9. Reset everything (fresh start) ```bash -docker compose --env-file .env down -docker compose --env-file .env down -v # also removes PostGIS/Redis volumes +docker compose -f docker/docker-compose.yml --env-file .env --profile full down -v ``` -## Notes +`-v` removes the PostGIS and Redis volumes. Next `up` will re-run migrations and reload fixtures. + +--- -- The legacy Vagrant flow in the top-level README remains valid, but Docker is - faster for repeatable local startup. -- If you need to run with a different portal app (for example `mida` or - `offshore`), create another config file modeled on - `marco/config.wcoa.docker.ini` and set `MP_PROJECT_CONFIG` accordingly. +## 10. Disk space + +Docker's build cache can grow large. Check usage and prune: + +```bash +docker system df +docker system prune -f # removes stopped containers, dangling images, unused networks, build cache +docker volume prune -f # removes unused volumes (DESTRUCTIVE — removes DB data if containers are stopped) +``` From 15e70e380274711015c98c872939b7b5a8814636 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Wed, 1 Apr 2026 17:38:55 -0700 Subject: [PATCH 021/152] Update Docker README for improved quick start instructions and add missing sub-app package clones --- docker/README.md | 268 ++++++++++++++++++++++---------------- docker/docker-compose.yml | 1 + 2 files changed, 159 insertions(+), 110 deletions(-) diff --git a/docker/README.md b/docker/README.md index 7f65781..b0b815b 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,73 +1,90 @@ # Docker Development Guide — Madrona Portal (WCOA) -## Repository layout +## Quick start -This stack assumes the standard monorepo layout: +These steps take a fresh machine from nothing to a running portal. +### Step 1 — Create the workspace directory + +All repos live inside a single parent directory. The Dockerfile build +context is the parent, so the layout is not optional. + +```bash +mkdir portals +cd portals ``` -madrona-apps-claude/ -├── madrona_portal/ ← main Django project (this repo) -│ ├── docker/ -│ │ ├── docker-compose.yml -│ │ └── entrypoint.sh -│ ├── Dockerfile -│ └── .env ← your local secrets (never committed) -└── madrona-apps/ ← sibling repo with all sub-app packages - ├── wcoa/ - ├── mp-layers/ - └── ... + +### Step 2 — Clone madrona-portal + +```bash +git clone -b docker https://github.com/Ecotrust/madrona-portal.git madrona_portal ``` -All commands below are run from the **`madrona_portal/`** directory unless noted. +### Step 3 — Clone the sub-app packages ---- +The Dockerfile copies all of these at build time. Clone them into a +`madrona-apps/` sibling directory: -## 1. Prerequisites +```bash +mkdir madrona-apps && cd madrona-apps + +git clone https://github.com/Ecotrust/django_url_shortener.git +git clone https://github.com/Ecotrust/madrona-analysistools.git +git clone https://github.com/Ecotrust/madrona-features.git +git clone https://github.com/Ecotrust/madrona-manipulators.git +git clone https://github.com/Ecotrust/madrona-scenarios.git +git clone https://github.com/Ecotrust/mp-accounts.git +git clone https://github.com/Ecotrust/mp-data-manager.git +git clone https://github.com/Ecotrust/mp-drawing.git +git clone https://github.com/Ecotrust/mp-explore.git +git clone https://github.com/Ecotrust/mp-layers.git +git clone https://github.com/Ecotrust/mp-map-groups.git +git clone https://github.com/Ecotrust/mp-proxy.git +git clone https://github.com/Ecotrust/mp-visualize.git +git clone https://github.com/Ecotrust/p97-nursery.git +git clone -b vagrant2docker https://github.com/Ecotrust/wcoa.git + +cd .. +``` -- Docker Desktop (Mac/Windows) or Docker Engine + Compose plugin (Linux) -- `madrona-apps/` checked out as a sibling of `madrona_portal/` (the Dockerfile copies from it at build time) +Your workspace should now look like: ---- +``` +portals/ +├── madrona_portal/ ← cloned from Ecotrust/madrona-portal, branch: docker +└── madrona-apps/ + ├── wcoa/ ← branch: vagrant2docker + ├── mp-layers/ + └── ... ← all others on main +``` -## 2. Configure environment +### Step 4 — Configure environment -Copy the example and fill in real values: +From `madrona_portal/`: ```bash -cp .env.example .env # if .env.example exists; otherwise edit .env directly +cd madrona_portal +cp .env.example .env ``` -Minimum required values in `.env`: +Edit `.env` and set at minimum: ```ini -SECRET_KEY= +SECRET_KEY= DB_PASSWORD= +DJANGO_SUPERUSER_PASSWORD= ``` -Other notable defaults (override in `.env` as needed): - -| Variable | Default | Notes | -|---|---|---| -| `APP_PORT` | `8000` | Host port the Django app binds to | -| `DB_NAME` | `wcoa_docker_db` | PostgreSQL database name | -| `DB_USER` | `postgres` | PostgreSQL user | -| `DB_PORT` | `5432` | Host port for PostgreSQL | -| `REDIS_PORT` | `6379` | Host port for Redis | -| `MP_PROJECT_CONFIG` | `config.wcoa.docker.ini` | Django config file (do not change for WCOA) | -| `DEBUG` | `False` | Set `True` to use Django dev server instead of gunicorn | -| `DJANGO_SUPERUSER_PASSWORD` | *(empty)* | If set, a superuser is created on first start | -| `DJANGO_SUPERUSER_USERNAME` | `admin` | Superuser username | -| `DJANGO_SUPERUSER_EMAIL` | `admin@example.com` | Superuser email | - ---- - -## 3. Build the image +Everything else has working defaults for local development. -Use `docker buildx build` directly — `docker compose build` has a known caching issue where it silently reads committed (not filesystem) file versions when a `.git` directory exists in the build context. +### Step 5 — Build the image -From the **repo root** (`madrona-apps-claude/`): +Run this from the **workspace root** (`madrona_portal/`), not from +inside `madrona_portal/`. The build context must include both repos. ```bash +cd .. # back to portals/ + docker buildx build \ --builder desktop-linux \ --load \ @@ -76,121 +93,152 @@ docker buildx build \ . ``` -Add `--no-cache` to force a full rebuild (e.g. after changing `requirements.txt`). +This takes several minutes on a first build (compiling GDAL, installing +Python packages). Subsequent builds are fast thanks to layer caching. -> **Important:** Always commit changes to `madrona_portal/` before rebuilding. BuildKit reads files from the git object store, not the filesystem, when a `.git` directory is present in the build context. +> **Why `docker buildx build` and not `docker compose build`?** +> `docker compose build` has a caching bug: when a `.git` directory exists +> inside the build context, BuildKit reads files from the git object store +> (committed versions) rather than the filesystem. If you forget to commit +> a change, the old version is silently baked into the image. The same +> restriction applies — always commit changes to `madrona_portal/` or +> `madrona-apps/` before rebuilding. ---- +### Step 6 — Start the full stack -## 4. Start the full stack +From `madrona_portal/`: ```bash +cd madrona_portal + docker compose -f docker/docker-compose.yml --env-file .env --profile full up -d ``` -The `--profile full` flag is required to start the `app` service. Without it, only `db` (PostGIS) and `tasks` (Redis) start — useful for running Django locally against Docker infrastructure. +The `--profile full` flag is required to start the `app` container. +Without it only `db` (PostGIS) and `tasks` (Redis) start. -Services: +On first boot the entrypoint automatically: -| Service | Image | Host port | -|---|---|---| -| `app` | `madrona_portal-app:latest` | `${APP_PORT}` (default 8000) | -| `db` | `postgis/postgis:16-3.4` | `${DB_PORT}` (default 5432) | -| `tasks` | `redis:7-alpine` | `${REDIS_PORT}` (default 6379) | +1. Waits for PostgreSQL to accept connections +2. Runs `migrate` +3. Runs `collectstatic` and `compress` +4. Detects a fresh database and loads initial fixtures (1,782 + 22 objects) +5. Creates the superuser defined in `.env` (if `DJANGO_SUPERUSER_PASSWORD` is set) +6. Starts the application server + +Open: http://localhost:8000/ (or whatever `APP_PORT` is set to in `.env`) --- -## 5. What happens on first startup +## Everyday usage + +All `docker compose` commands below are run from **`madrona_portal/`**. + +### View logs + +```bash +docker compose -f docker/docker-compose.yml --env-file .env --profile full logs -f app +``` + +### Run a management command + +```bash +docker compose -f docker/docker-compose.yml --env-file .env --profile full \ + run --rm app python marco/manage.py +``` + +Examples: + +```bash +# Django shell +... run --rm app python marco/manage.py shell + +# Create superuser manually +... run --rm app python marco/manage.py createsuperuser -The entrypoint (`docker/entrypoint.sh`) runs in order: +# Load a fixture +... run --rm app python marco/manage.py loaddata /path/to/fixture.json +``` -1. **Waits** for PostgreSQL to accept connections -2. **Migrates** (`manage.py migrate`) -3. **Collects static files** (`manage.py collectstatic`) -4. **Compresses assets** (`manage.py compress --force`) -5. **Seeds fixtures** — if fewer than 5 content pages exist (fresh DB), loads: - - `apps/wcoa/wcoa/fixtures/initial_data_prod.json` (1,782 objects: pages, layers, themes, etc.) - - `apps/madrona-scenarios/scenarios/fixtures/initial_data.json` (22 objects) -6. **Creates superuser** — only if `DJANGO_SUPERUSER_PASSWORD` is set and the username doesn't exist -7. **Starts the server** — gunicorn in production (`DEBUG=False`), Django dev server otherwise +### Open a database shell -Open: http://localhost:${APP_PORT}/ +```bash +docker exec -it docker-db-1 psql -U postgres wcoa_docker_db +``` --- -## 6. Dev infrastructure only (no app container) +## Dev infrastructure only (local Django server) -To run Django locally with only the Docker DB and Redis: +To run Django locally against Docker-managed PostGIS and Redis (no app container): ```bash +# Start only db and tasks (omit --profile full) docker compose -f docker/docker-compose.yml --env-file .env up -d -# db and tasks start; app does not (no --profile full) +# Then in a separate terminal, from madrona_portal/: cd marco python manage.py runserver ``` --- -## 7. Common one-off commands - -```bash -# Django shell -docker compose -f docker/docker-compose.yml --env-file .env --profile full \ - run --rm app python marco/manage.py shell +## Rebuilding after code changes -# Create superuser manually -docker compose -f docker/docker-compose.yml --env-file .env --profile full \ - run --rm app python marco/manage.py createsuperuser +> **Commit first.** BuildKit reads from the git object store — uncommitted +> changes are invisible to the build. -# Run migrations -docker compose -f docker/docker-compose.yml --env-file .env --profile full \ - run --rm app python marco/manage.py migrate - -# Load a fixture -docker compose -f docker/docker-compose.yml --env-file .env --profile full \ - run --rm app python marco/manage.py loaddata /path/to/fixture.json +From the **workspace root** (`portals/`): -# Open a psql shell in the DB container -docker exec -it docker-db-1 psql -U postgres wcoa_docker_db +```bash +docker buildx build \ + --builder desktop-linux \ + --load \ + -f madrona_portal/Dockerfile \ + -t madrona_portal-app:latest \ + . ``` ---- +Then from `madrona_portal/`: -## 8. Rebuild after code changes - -After changing Python source files, templates, or static assets in `madrona_portal/` or `madrona-apps/`: +```bash +docker compose -f docker/docker-compose.yml --env-file .env --profile full \ + up -d --force-recreate app +``` -1. Commit your changes (required for BuildKit to pick them up) -2. Rebuild from the repo root: - ```bash - docker buildx build --builder desktop-linux --load \ - -f madrona_portal/Dockerfile -t madrona_portal-app:latest . - ``` -3. Restart the app container: - ```bash - docker compose -f docker/docker-compose.yml --env-file .env --profile full \ - up -d --force-recreate app - ``` +Add `--no-cache` to the buildx command to force a full dependency reinstall +(needed when `docker-requirements.txt` changes). --- -## 9. Reset everything (fresh start) +## Reset to a clean state ```bash +# From madrona_portal/ docker compose -f docker/docker-compose.yml --env-file .env --profile full down -v ``` -`-v` removes the PostGIS and Redis volumes. Next `up` will re-run migrations and reload fixtures. +`-v` removes the PostGIS and Redis volumes. The next `up` will re-run +migrations and reload fixtures from scratch. + +--- + +## Services and ports + +| Service | Image | Default host port | Override via | +|---|---|---|---| +| `app` | `madrona_portal-app:latest` | `8000` | `APP_PORT` in `.env` | +| `db` | `postgis/postgis:16-3.4` | `5432` | `DB_PORT` in `.env` | +| `tasks` | `redis:7-alpine` | `6379` | `REDIS_PORT` in `.env` | --- -## 10. Disk space +## Disk space -Docker's build cache can grow large. Check usage and prune: +Docker's build cache can grow large over time: ```bash -docker system df -docker system prune -f # removes stopped containers, dangling images, unused networks, build cache -docker volume prune -f # removes unused volumes (DESTRUCTIVE — removes DB data if containers are stopped) +docker system df # show usage breakdown +docker system prune -f # remove stopped containers, dangling images, unused networks, build cache +docker volume prune -f # remove unused volumes — only run when all containers are stopped ``` diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 5a6c759..fa9cafe 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -11,6 +11,7 @@ services: app: + image: madrona_portal-app:latest build: context: ../../ dockerfile: madrona_portal/Dockerfile From 6c9a0cba6e1e410b2721fb8a55bcd4b746abb408 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Fri, 3 Apr 2026 15:52:43 -0700 Subject: [PATCH 022/152] Remove deprecated .env.dev file and add docker-compose.dev.yml for development overrides --- docker/.env.dev | 77 ----------------------------------- docker/README.md | 30 ++++++++++++++ docker/docker-compose.dev.yml | 52 +++++++++++++++++++++++ 3 files changed, 82 insertions(+), 77 deletions(-) delete mode 100644 docker/.env.dev create mode 100644 docker/docker-compose.dev.yml diff --git a/docker/.env.dev b/docker/.env.dev deleted file mode 100644 index 0aa73ba..0000000 --- a/docker/.env.dev +++ /dev/null @@ -1,77 +0,0 @@ -##################### -# DATABASE SETTINGS # -##################### - -# Most of these are critical to the security of your database application. -# Many of these settings should be changed even for development environments. - -## SECRET_KEY: -## Used to secure the app. Can be anything -- feel free to hammer out long line of numbers, letters, caps, and symbols. You will not need to remember or retype this ever. -SECRET_KEY=SssHhhhh - -## SQL_DATABASE: -## The name of your database. This can be any word (no spaces). You can leave the default in place, but you will get extra security by making it unique. -SQL_DATABASE=wcoa_docker_db - -## SQL_USER: -## A username for the owner of your database. Can be any word (no spaces). It is recommended that you change this for security purposes -SQL_USER=postgres - -## SQL_PASSWORD: -## A password for your database user. Please change. -SQL_PASSWORD=wcoa_docker_pass - -## SQL_ENGINE: -## The django database backend to use to connect to the database. -SQL_ENGINE=django.contrib.gis.db.backends.postgis - -## SQL_HOST: -## The network address of the database. The default 'db' is the variable name assigned and recognized by Docker from your docker-compose.yml file. -SQL_HOST=db - -## SQL_PORT: -## The port your database is accepting connections on. Default for PostgreSQL is 5432. -SQL_PORT=65432 - - -################### -# SERVER SETTINGS # -################### - -# These are settings that may impact how the application is served. - -## DEBUG: -## 1 for 'true' -## 0 for 'false' -DEBUG=1 - -## PROXY_PORT: -## The port the proxy server (NGINX) will serve the application on. Use 80 in production for HTTP or 443 for HTTPS. For dev you may prefer 80xx. -PROXY_PORT=8002 - -## ALLOWED_HOSTS: -## List of web addresses server will accept traffic from. -ALLOWED_HOSTS=["localhost","127.0.0.1","::1"] - -## TIME_ZONE: -## For all Timezone options, see https://en.wikipedia.org/wiki/List_of_tz_database_time_zones#List -TIME_ZONE=America/Los_Angeles - - -################# -# TASK SETTINGS # -################# - -## TASK_PORT: -## The port your task queue is accepting connections on. Default for Redis is 6379. -TASK_PORT=8379 -REDIS_PASSWORD=sOmE_sEcUrE_pAsS - - -####################### -# DEPENDENCY SETTINGS # -####################### - -## PROJ_DIR: -## The location of the PROJ.4 executable -PROJ_DIR=/usr diff --git a/docker/README.md b/docker/README.md index b0b815b..4028818 100644 --- a/docker/README.md +++ b/docker/README.md @@ -134,6 +134,36 @@ Open: http://localhost:8000/ (or whatever `APP_PORT` is set to in `.env`) All `docker compose` commands below are run from **`madrona_portal/`**. +--- + +## Deploy to fully containerized live instance + +1. Set up your `.env` +2. Build and run (from `portals/`): + +```bash +docker buildx build \ + --builder desktop-linux \ + --load \ + -f madrona_portal/Dockerfile \ + -t madrona_portal-app:latest \ + . +``` + +### Redeploying after code changes + +Then from `madrona_portal/`: + +```bash +docker compose -f docker/docker-compose.yml --env-file .env --profile full \ + up -d --force-recreate app +``` + +Add `--no-cache` to the buildx command to force a full dependency reinstall +(needed when `docker-requirements.txt` changes). + +--- + ### View logs ```bash diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml new file mode 100644 index 0000000..1211a36 --- /dev/null +++ b/docker/docker-compose.dev.yml @@ -0,0 +1,52 @@ +# Madrona Portal — Development Compose Override +# +# Layers on top of docker-compose.yml to enable live code sync from the host. +# Python's editable installs follow .pth files to the mounted directories, so +# any file saved on the host is immediately visible inside the container. +# +# Usage (from madrona_portal/): +# docker compose \ +# -f docker/docker-compose.yml \ +# -f docker/docker-compose.dev.yml \ +# --env-file .env --profile full up +# +# Django's runserver (active when DEBUG=True) auto-reloads on .py changes. +# Template changes are picked up per-request — no restart needed. +# Static file changes are served directly by runserver — no collectstatic needed. +# +# What still requires a container restart: +# - New pip packages (pip install must run inside the venv) +# - config.ini changes +# +# What still requires manage.py migrate: +# - New migration files (run: docker exec docker-app-1 python marco/manage.py migrate) + +services: + app: + environment: + - DEBUG=True + volumes: + # Mount the main Django project tree from the host. + # Changes to Python, templates, and config files are live immediately. + - ../marco:/usr/local/apps/madrona-portal/marco + + # Mount the WCOA app package from the host. + - ../../madrona-apps/wcoa:/usr/local/apps/madrona-portal/apps/wcoa + + # Mount other madrona-apps packages you are actively developing. + # Comment out any you are NOT changing — using the baked image copy + # for those packages is faster and avoids unnecessary inotify watches. + - ../../madrona-apps/mp-data-manager:/usr/local/apps/madrona-portal/apps/mp-data-manager + - ../../madrona-apps/mp-layers:/usr/local/apps/madrona-portal/apps/mp-layers + - ../../madrona-apps/mp-accounts:/usr/local/apps/madrona-portal/apps/mp-accounts + - ../../madrona-apps/mp-drawing:/usr/local/apps/madrona-portal/apps/mp-drawing + - ../../madrona-apps/mp-visualize:/usr/local/apps/madrona-portal/apps/mp-visualize + - ../../madrona-apps/madrona-features:/usr/local/apps/madrona-portal/apps/madrona-features + - ../../madrona-apps/madrona-manipulators:/usr/local/apps/madrona-portal/apps/madrona-manipulators + - ../../madrona-apps/madrona-scenarios:/usr/local/apps/madrona-portal/apps/madrona-scenarios + - ../../madrona-apps/mp-map-groups:/usr/local/apps/madrona-portal/apps/mp-map-groups + - ../../madrona-apps/mp-explore:/usr/local/apps/madrona-portal/apps/mp-explore + - ../../madrona-apps/mp-proxy:/usr/local/apps/madrona-portal/apps/mp-proxy + - ../../madrona-apps/p97-nursery:/usr/local/apps/madrona-portal/apps/p97-nursery + - ../../madrona-apps/django_url_shortener:/usr/local/apps/madrona-portal/apps/django_url_shortener + - ../../madrona-apps/madrona-analysistools:/usr/local/apps/madrona-portal/apps/madrona-analysistools From f1ab12fdc7fe83b529568314b171489e1a569a9c Mon Sep 17 00:00:00 2001 From: David Pollard Date: Fri, 3 Apr 2026 16:04:31 -0700 Subject: [PATCH 023/152] Reorganize Docker README to improve clarity on everyday usage and deployment instructions --- docker/README.md | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/docker/README.md b/docker/README.md index 4028818..bca8c20 100644 --- a/docker/README.md +++ b/docker/README.md @@ -128,12 +128,6 @@ On first boot the entrypoint automatically: Open: http://localhost:8000/ (or whatever `APP_PORT` is set to in `.env`) ---- - -## Everyday usage - -All `docker compose` commands below are run from **`madrona_portal/`**. - --- ## Deploy to fully containerized live instance @@ -162,7 +156,11 @@ docker compose -f docker/docker-compose.yml --env-file .env --profile full \ Add `--no-cache` to the buildx command to force a full dependency reinstall (needed when `docker-requirements.txt` changes). ---- +--- + +## Everyday usage + +All `docker compose` commands below are run from **`madrona_portal/`**. ### View logs From 98243534052c137c4c4b7fa3d2b07099b04f3805 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Tue, 7 Apr 2026 09:53:58 -0700 Subject: [PATCH 024/152] Add AWS Deployment Guide for Madrona Portal setup --- docker/AWS_DEPLOY.md | 581 +++++++++++++++++++++++++++++++ marco/config.docker.ini.template | 1 + 2 files changed, 582 insertions(+) create mode 100644 docker/AWS_DEPLOY.md diff --git a/docker/AWS_DEPLOY.md b/docker/AWS_DEPLOY.md new file mode 100644 index 0000000..59ad3fa --- /dev/null +++ b/docker/AWS_DEPLOY.md @@ -0,0 +1,581 @@ +# AWS Deployment Guide — Madrona Portal (WCOA) + +This guide takes you from a blank AWS account to a running production stack. +All services (Django, PostGIS, Redis, Elasticsearch, Geoportal) run as Docker +containers on a single EC2 instance using Docker Compose. + +--- + +## Prerequisites + +- An AWS account with billing enabled +- Your local machine has the AWS CLI installed and configured, + or you are comfortable using the AWS Console +- SSH client on your local machine +- A GitHub account with access to all required repos + +--- + +## Phase 1 — AWS Infrastructure + +### 1.1 Create a key pair + +You need this before launching the instance. + +**AWS Console → EC2 → Key Pairs → Create key pair** + +| Setting | Value | +|---|---| +| Name | `madrona-portal` | +| Key pair type | RSA | +| Private key format | `.pem` (Linux/Mac) or `.ppk` (PuTTY/Windows) | + +Download the `.pem` file and move it somewhere safe: + +```bash +mv ~/Downloads/madrona-portal.pem ~/.ssh/ +chmod 400 ~/.ssh/madrona-portal.pem +``` + +### 1.2 Create a security group + +**AWS Console → EC2 → Security Groups → Create security group** + +| Setting | Value | +|---|---| +| Name | `madrona-portal-sg` | +| Description | Madrona Portal web server | +| VPC | Default VPC (or your own) | + +**Inbound rules:** + +| Type | Port | Source | Purpose | +|---|---|---|---| +| SSH | 22 | Your IP only (`x.x.x.x/32`) | Server access | +| HTTP | 80 | `0.0.0.0/0` | Web traffic (Nginx) | +| HTTPS | 443 | `0.0.0.0/0` | Web traffic (Nginx + SSL) | + +> Do **not** open port 8000 to the public. Nginx (added in Phase 5) will +> proxy traffic to gunicorn on port 8000 internally. + +**Outbound rules:** leave the default (all traffic allowed). + +### 1.3 Launch an EC2 instance + +**AWS Console → EC2 → Instances → Launch instances** + +| Setting | Value | +|---|---| +| Name | `madrona-portal` | +| AMI | Ubuntu Server 24.04 LTS (64-bit x86) | +| Instance type | `t3.large` (8 GB RAM) — minimum. See note below. | +| Key pair | `madrona-portal` (created above) | +| Security group | `madrona-portal-sg` (created above) | +| Storage | 60 GB gp3 — expand the default 8 GB root volume | + +> **Why t3.large?** Elasticsearch alone reserves 1 GB of heap +> (`-Xms512m -Xmx512m` in docker-compose.yml) plus JVM overhead. +> Add PostGIS, gunicorn workers, Redis, and Tomcat (Geoportal) and you need +> at least 6–7 GB free. A `t3.large` (8 GB) is the practical minimum; +> `t3.xlarge` (16 GB) gives comfortable headroom. + +> **Why 60 GB?** The Docker image is ~3–4 GB after build. PostGIS data, +> Elasticsearch indices, and Docker's build cache add up quickly. + +Launch the instance and wait for it to reach **Running** state. + +### 1.4 Allocate an Elastic IP + +Without an Elastic IP, AWS reassigns your public IP every time the instance +stops. An Elastic IP is free while attached to a running instance. + +**AWS Console → EC2 → Elastic IPs → Allocate Elastic IP address** + +- Click **Allocate** +- Select the new IP → **Actions → Associate Elastic IP address** +- Choose your `madrona-portal` instance → **Associate** + +Note the Elastic IP — you will use it in your `.env` and DNS records. + +--- + +## Phase 2 — Server Setup + +### 2.1 SSH into the instance + +```bash +ssh -i ~/.ssh/madrona-portal.pem ubuntu@ +``` + +### 2.2 Update the system + +```bash +sudo apt-get update && sudo apt-get upgrade -y +``` + +### 2.3 Install Docker Engine + +AWS's Ubuntu AMI does not include Docker. Install the official Docker Engine +(not the snap package — it has permission issues with volumes). + +```bash +# Install prerequisites +sudo apt-get install -y ca-certificates curl gnupg + +# Add Docker's official GPG key +sudo install -m 0755 -d /etc/apt/keyrings +curl -fsSL https://download.docker.com/linux/ubuntu/gpg \ + | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg +sudo chmod a+r /etc/apt/keyrings/docker.gpg + +# Add the Docker apt repository +echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] \ + https://download.docker.com/linux/ubuntu \ + $(. /etc/os-release && echo "$VERSION_CODENAME") stable" \ + | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + +# Install Docker Engine + Compose plugin + BuildKit +sudo apt-get update +sudo apt-get install -y docker-ce docker-ce-cli containerd.io \ + docker-buildx-plugin docker-compose-plugin +``` + +### 2.4 Allow your user to run Docker without sudo + +```bash +sudo usermod -aG docker ubuntu +newgrp docker # apply without logging out +docker run hello-world # verify +``` + +### 2.5 Enable Docker to start on boot + +```bash +sudo systemctl enable docker +sudo systemctl enable containerd +``` + +--- + +## Phase 3 — Clone the Repositories + +The Dockerfile build context must be the **workspace root** — a parent +directory containing both `madrona_portal/` and `madrona-apps/` as siblings. +This layout is required; it is not optional. + +### 3.1 Set up SSH access to GitHub (on the server) + +```bash +ssh-keygen -t ed25519 -C "madrona-portal-server" -f ~/.ssh/github -N "" +cat ~/.ssh/github.pub +``` + +Copy the output and add it as a **Deploy Key** in each GitHub repository +(or as an SSH key on your GitHub account if you have access to all repos): + +**GitHub repo → Settings → Deploy keys → Add deploy key** +- Paste the public key +- Title: `madrona-portal EC2` +- Enable "Allow write access": No (read-only is sufficient) + +Configure SSH to use this key for GitHub: + +```bash +cat >> ~/.ssh/config << 'EOF' +Host github.com + IdentityFile ~/.ssh/github + StrictHostKeyChecking no +EOF +``` + +### 3.2 Create the workspace and clone + +```bash +mkdir ~/portals && cd ~/portals +``` + +Clone the main portal: + +```bash +git clone -b docker git@github.com:Ecotrust/madrona-portal.git madrona_portal +``` + +Clone all sub-apps: + +```bash +mkdir madrona-apps && cd madrona-apps + +git clone git@github.com:Ecotrust/django_url_shortener.git +git clone git@github.com:Ecotrust/madrona-analysistools.git +git clone git@github.com:Ecotrust/madrona-features.git +git clone git@github.com:Ecotrust/madrona-manipulators.git +git clone git@github.com:Ecotrust/madrona-scenarios.git +git clone git@github.com:Ecotrust/mp-accounts.git +git clone git@github.com:Ecotrust/mp-data-manager.git +git clone git@github.com:Ecotrust/mp-drawing.git +git clone git@github.com:Ecotrust/mp-explore.git +git clone git@github.com:Ecotrust/mp-layers.git +git clone git@github.com:Ecotrust/mp-map-groups.git +git clone git@github.com:Ecotrust/mp-proxy.git +git clone git@github.com:Ecotrust/mp-visualize.git +git clone git@github.com:Ecotrust/p97-nursery.git +git clone -b vagrant2docker git@github.com:Ecotrust/wcoa.git + +cd .. +``` + +Verify the layout: + +```bash +ls ~/portals/ +# madrona_portal/ madrona-apps/ +``` + +--- + +## Phase 4 — Configure the Environment + +### 4.1 Create the `.env` file + +```bash +cd ~/portals/madrona_portal +cp .env.example .env +``` + +### 4.2 Generate a secret key + +```bash +python3 -c "import secrets; print(secrets.token_urlsafe(50))" +``` + +Copy the output — you will paste it as `SECRET_KEY` below. + +### 4.3 Edit `.env` + +```bash +nano .env +``` + +Set these values at minimum: + +```ini +# Django +SECRET_KEY= +ALLOWED_HOSTS=,localhost +DEBUG=False +DJANGO_ENV=production + +# Database +DB_PASSWORD= + +# Redis +REDIS_PASSWORD= + +# Superuser (created automatically on first boot) +DJANGO_SUPERUSER_USERNAME=admin +DJANGO_SUPERUSER_EMAIL=your@email.com +DJANGO_SUPERUSER_PASSWORD= + +# Gunicorn workers (set to 2× vCPU count; t3.large has 2 vCPUs → 4 workers) +GUNICORN_WORKERS=4 +``` + +Leave everything else at its default for now. You can add email, OAuth, and +Elasticsearch credentials later. + +--- + +## Phase 5 — Build the Docker Image + +> **Important:** The `--builder desktop-linux` flag in the local development +> guide is specific to Docker Desktop on Mac. On Linux EC2 you omit it — +> BuildKit is the default builder. + +From the **workspace root** (`~/portals/`): + +```bash +cd ~/portals + +docker buildx build \ + --load \ + -f madrona_portal/Dockerfile \ + -t madrona_portal-app:latest \ + . +``` + +This will take 10–20 minutes on first build (compiling GDAL, installing all +Python packages). Subsequent builds are fast thanks to layer caching. + +Watch for any errors. Common first-build issues: +- Out of disk space → increase EBS volume or run `docker system prune -f` first +- Network timeouts fetching packages → re-run the command (layer cache resumes) + +--- + +## Phase 6 — Start the Stack + +From `~/portals/madrona_portal/`: + +```bash +cd ~/portals/madrona_portal + +docker compose -f docker/docker-compose.yml \ + --env-file .env \ + --profile full \ + up -d +``` + +### 6.1 Watch the startup logs + +```bash +docker compose -f docker/docker-compose.yml --env-file .env --profile full \ + logs -f app +``` + +On first boot the entrypoint automatically: +1. Waits for PostgreSQL +2. Runs `migrate` +3. Runs `collectstatic` and `compress` +4. Detects fresh database → loads initial fixtures +5. Creates the superuser from `.env` +6. Starts gunicorn (because `DEBUG=False`) + +Startup takes 2–5 minutes. You should see `Booting worker` lines from +gunicorn when it is ready. + +### 6.2 Smoke test + +```bash +curl -I http://localhost:8000/ +# Expected: HTTP/1.1 200 OK (or 301/302 redirect) +``` + +If you get a connection refused, the app is still starting. Wait 30 seconds +and try again. + +--- + +## Phase 7 — Nginx + SSL (Production Hardening) + +Gunicorn should not be exposed directly to the internet. Nginx handles SSL +termination, compression, and static file serving. + +### 7.1 Install Nginx and Certbot + +```bash +sudo apt-get install -y nginx certbot python3-certbot-nginx +``` + +### 7.2 Create a DNS A record + +In your DNS provider (Route 53, Cloudflare, etc.): + +| Type | Name | Value | +|---|---|---| +| A | `portal.yourdomain.com` | `` | + +Wait for DNS to propagate before continuing (check with `dig portal.yourdomain.com`). + +### 7.3 Configure Nginx + +```bash +sudo nano /etc/nginx/sites-available/madrona-portal +``` + +Paste: + +```nginx +server { + listen 80; + server_name portal.yourdomain.com; + + # Static and media files served directly by Nginx from the Docker volume. + # The static_data volume is mounted at /vol/web inside the container but + # is not accessible from the host — gunicorn serves these for now. + # See note below about a future Nginx-native static setup. + + location / { + proxy_pass http://127.0.0.1:8000; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 120s; + client_max_body_size 50M; + } +} +``` + +Enable the site: + +```bash +sudo ln -s /etc/nginx/sites-available/madrona-portal \ + /etc/nginx/sites-enabled/madrona-portal +sudo nginx -t # verify config +sudo systemctl restart nginx +``` + +### 7.4 Obtain an SSL certificate + +```bash +sudo certbot --nginx -d portal.yourdomain.com +``` + +Certbot edits your Nginx config automatically to add SSL and redirect HTTP +to HTTPS. It also installs a cron job to renew the certificate automatically. + +### 7.5 Update `ALLOWED_HOSTS` + +Add your domain to `.env`: + +```ini +ALLOWED_HOSTS=portal.yourdomain.com,,localhost +``` + +Then restart the app container to pick up the change: + +```bash +cd ~/portals/madrona_portal +docker compose -f docker/docker-compose.yml --env-file .env --profile full \ + up -d --force-recreate app +``` + +--- + +## Phase 8 — Keep the Stack Running Across Reboots + +Docker Compose does not automatically restart after the EC2 instance reboots. +Set up a systemd service to handle this. + +```bash +sudo nano /etc/systemd/system/madrona-portal.service +``` + +Paste: + +```ini +[Unit] +Description=Madrona Portal Docker Compose Stack +Requires=docker.service +After=docker.service network-online.target + +[Service] +Type=oneshot +RemainAfterExit=yes +WorkingDirectory=/home/ubuntu/portals/madrona_portal +ExecStart=/usr/bin/docker compose \ + -f docker/docker-compose.yml \ + --env-file .env \ + --profile full \ + up -d +ExecStop=/usr/bin/docker compose \ + -f docker/docker-compose.yml \ + --env-file .env \ + --profile full \ + down +TimeoutStartSec=300 + +[Install] +WantedBy=multi-user.target +``` + +Enable it: + +```bash +sudo systemctl daemon-reload +sudo systemctl enable madrona-portal +``` + +Test it (optional — simulates a reboot): + +```bash +sudo systemctl stop madrona-portal +sudo systemctl start madrona-portal +``` + +--- + +## Redeploying After Code Changes + +When you push new code and want to redeploy: + +**1. On your local machine — commit and push all changes first.** +BuildKit reads from the git object store, so uncommitted changes will not +be included in the image. + +**2. On the server — pull and rebuild:** + +```bash +cd ~/portals/madrona_portal && git pull +cd ~/portals/madrona-apps/ && git pull # repeat for each changed sub-app + +cd ~/portals + +docker buildx build \ + --load \ + -f madrona_portal/Dockerfile \ + -t madrona_portal-app:latest \ + . +``` + +**3. Recreate the app container:** + +```bash +cd ~/portals/madrona_portal + +docker compose -f docker/docker-compose.yml --env-file .env --profile full \ + up -d --force-recreate app +``` + +The database and Redis containers are untouched. Downtime is limited to the +container restart (~5–10 seconds). + +--- + +## Useful Commands (on the server) + +All `docker compose` commands run from `~/portals/madrona_portal/`. + +```bash +# Tail app logs +docker compose -f docker/docker-compose.yml --env-file .env --profile full logs -f app + +# Run a Django management command +docker compose -f docker/docker-compose.yml --env-file .env --profile full \ + run --rm app python marco/manage.py + +# Open a Django shell +docker compose -f docker/docker-compose.yml --env-file .env --profile full \ + run --rm app python marco/manage.py shell + +# Open a database shell +docker exec -it $(docker compose -f docker/docker-compose.yml --env-file .env ps -q db) \ + psql -U postgres wcoa_docker_db + +# Check disk and Docker space usage +df -h +docker system df + +# Stop the stack (data preserved) +docker compose -f docker/docker-compose.yml --env-file .env --profile full down + +# Full reset — DESTROYS ALL DATA +docker compose -f docker/docker-compose.yml --env-file .env --profile full down -v +``` + +--- + +## Services and Ports + +| Service | Image | Internal port | Exposed to host | +|---|---|---|---| +| `app` | `madrona_portal-app:latest` | 8000 | Yes — proxied by Nginx | +| `db` | `postgis/postgis:16-3.4` | 5432 | Yes (restrict in security group) | +| `tasks` | `redis:7-alpine` | 6379 | Yes (restrict in security group) | +| `geoportal` | built from `wcoa/docker` | 8080 | Yes (add Nginx location if needed) | +| `elastic` | `elasticsearch:8.19.12` | 9200, 9300 | Yes (restrict in security group) | + +> After go-live, update your security group inbound rules to remove public +> access to ports 5432, 6379, 9200, and 9300. These are only needed +> internally between containers on `djangonetwork`. diff --git a/marco/config.docker.ini.template b/marco/config.docker.ini.template index 177528b..462e1dd 100644 --- a/marco/config.docker.ini.template +++ b/marco/config.docker.ini.template @@ -11,6 +11,7 @@ DEBUG = True TEMPLATE_DEBUG = True ALLOWED_HOSTS = ["localhost", "127.0.0.1", "::1"] SECRET_KEY = You forgot to set the secret key +# TODO: Does the MEDIA_ROOT need to be updated here? potentially /usr/local/apps/madrona-portal/apps/wcoa/media/ MEDIA_ROOT = /vol/web/media MEDIA_URL = /media/ TIME_ZONE = UTC From 7ea0ec728b5ed2e78cd58c71a92cf7fa65d2342c Mon Sep 17 00:00:00 2001 From: David Pollard Date: Tue, 7 Apr 2026 18:03:21 -0700 Subject: [PATCH 025/152] Add rpc4django to Docker requirements for API support --- docker/docker-requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/docker-requirements.txt b/docker/docker-requirements.txt index 7837218..0af3df6 100644 --- a/docker/docker-requirements.txt +++ b/docker/docker-requirements.txt @@ -73,6 +73,7 @@ elasticsearch-dsl>=7.0,<8.0 # APIs # --------------------------------------------------------------------------- # rpc4django removed — replaced by djangorestframework API views +rpc4django djangorestframework>=3.14,<4.0 # --------------------------------------------------------------------------- From 8fbd7167d4e1bcf6bf236e26339d440d4d00bf7a Mon Sep 17 00:00:00 2001 From: David Pollard Date: Wed, 8 Apr 2026 13:57:35 -0700 Subject: [PATCH 026/152] Update entrypoint.sh to load specific initial data fixtures for WCOA app --- dev_requirements.txt | 23 ----------------------- docker/entrypoint.sh | 7 ++++--- 2 files changed, 4 insertions(+), 26 deletions(-) delete mode 100644 dev_requirements.txt diff --git a/dev_requirements.txt b/dev_requirements.txt deleted file mode 100644 index 1260bea..0000000 --- a/dev_requirements.txt +++ /dev/null @@ -1,23 +0,0 @@ -# ============================================================================= -# Development & Testing Requirements -# Install with: pip install -r requirements.txt -r dev_requirements.txt -# ============================================================================= - -# Testing -pytest>=8.0,<9.0 -pytest-django>=4.8,<5.0 -pytest-cov>=5.0,<6.0 -factory-boy>=3.3,<4.0 - -# Code Quality & Linting -ruff>=0.4,<1.0 # Fast Python linter (replaces flake8, isort, pyupgrade) -mypy>=1.10,<2.0 # Static type checking -django-stubs>=5.0,<6.0 # Django type stubs for mypy - -# Debug tooling -django-debug-toolbar>=4.3,<5.0 -Werkzeug>=3.0,<4.0 # Better dev server with debugger - -# Utilities -ipython>=8.0,<9.0 -ipdb>=0.13,<1.0 diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 807d0e8..a5cf2e5 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -86,10 +86,11 @@ except Exception: pass PY - python marco/manage.py loaddata initial_data_prod.json - # Load per-app reference fixtures that aren't included in the main fixture. # Use absolute paths so only this specific file is loaded (not other apps' - # initial_data.json files that happen to share the same name). + # files that happen to share the same name). + python marco/manage.py loaddata \ + apps/wcoa/wcoa/fixtures/initial_data_prod.json + # Load per-app reference fixtures that aren't included in the main fixture. python marco/manage.py loaddata \ apps/madrona-scenarios/scenarios/fixtures/initial_data.json echo "Initial fixtures loaded." From 9ce88b5fe87c10c243dab2a5e285b99247b021bf Mon Sep 17 00:00:00 2001 From: David Pollard Date: Wed, 8 Apr 2026 14:43:11 -0700 Subject: [PATCH 027/152] Add content type verification before loading fixtures in entrypoint.sh --- docker/entrypoint.sh | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index a5cf2e5..a74c46f 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -86,6 +86,24 @@ except Exception: pass PY + # Ensure ContentTypes exist for every installed app before loading fixtures. + # Wagtail Page fixtures reference content types by natural key + # (e.g. ["wcoa", "ctapage"]); if the ContentType row is missing, Django + # silently leaves content_type_id = NULL and the INSERT fails. + # create_contenttypes is idempotent — safe to call on every run. + python - <<'PY' +import sys, os +sys.path.insert(0, 'marco') +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'marco.settings') +import django +django.setup() +from django.apps import apps +from django.contrib.contenttypes.management import create_contenttypes +for app_config in apps.get_app_configs(): + create_contenttypes(app_config, verbosity=0) +print('Content types verified.', flush=True) +PY + # Use absolute paths so only this specific file is loaded (not other apps' # files that happen to share the same name). python marco/manage.py loaddata \ From ab573e20e0f50500c78656e8c04a069994f40a40 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Wed, 8 Apr 2026 15:15:42 -0700 Subject: [PATCH 028/152] Refactor fixture loading in entrypoint.sh to ensure ContentTypes are created before loading data --- docker/entrypoint.sh | 58 ++++++++++++++++++++++++++++++++------------ 1 file changed, 42 insertions(+), 16 deletions(-) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index a74c46f..77921bc 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -86,31 +86,57 @@ except Exception: pass PY - # Ensure ContentTypes exist for every installed app before loading fixtures. - # Wagtail Page fixtures reference content types by natural key - # (e.g. ["wcoa", "ctapage"]); if the ContentType row is missing, Django - # silently leaves content_type_id = NULL and the INSERT fails. - # create_contenttypes is idempotent — safe to call on every run. + # Load fixtures in a single Python process so that ContentTypes created + # here are guaranteed to be visible when loaddata deserializes FK natural + # keys. Wagtail Page records reference content types by natural key + # (e.g. ["wcoa", "ctapage"]); if the ContentType row is absent Django's + # deserializer defers the FK and the INSERT fails with a NOT NULL violation. python - <<'PY' import sys, os sys.path.insert(0, 'marco') os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'marco.settings') import django django.setup() -from django.apps import apps + +# Step 1: ensure every installed app's ContentTypes exist before loading +# fixture data. create_contenttypes is idempotent. +from django.apps import apps as django_apps from django.contrib.contenttypes.management import create_contenttypes -for app_config in apps.get_app_configs(): +from django.contrib.contenttypes.models import ContentType + +for app_config in django_apps.get_app_configs(): create_contenttypes(app_config, verbosity=0) -print('Content types verified.', flush=True) -PY - # Use absolute paths so only this specific file is loaded (not other apps' - # files that happen to share the same name). - python marco/manage.py loaddata \ - apps/wcoa/wcoa/fixtures/initial_data_prod.json - # Load per-app reference fixtures that aren't included in the main fixture. - python marco/manage.py loaddata \ - apps/madrona-scenarios/scenarios/fixtures/initial_data.json +# Confirm the wcoa types that the fixture depends on are present. +wcoa_models = [ + 'ctapage', 'connectpage', 'catalogiframepage', + 'catalogthemegridpage', 'catalogthemegridpagedetail', + 'ohidashboard', 'wcoaoceanstories', 'wcoaoceanstory', +] +missing = [] +for model in wcoa_models: + if not ContentType.objects.filter(app_label='wcoa', model=model).exists(): + # Force-create it so loaddata can resolve the natural key. + ContentType.objects.get_or_create(app_label='wcoa', model=model) + missing.append(model) +if missing: + print(f'WARNING: had to force-create ContentTypes: {missing}', flush=True) +else: + print('All wcoa ContentTypes verified.', flush=True) + +# Step 2: load fixtures — same process, same DB session. +from django.core.management import call_command +call_command( + 'loaddata', + 'apps/wcoa/wcoa/fixtures/initial_data_prod.json', + verbosity=1, +) +call_command( + 'loaddata', + 'apps/madrona-scenarios/scenarios/fixtures/initial_data.json', + verbosity=1, +) +PY echo "Initial fixtures loaded." else echo "Existing database — skipping fixture load." From 53f470a7250ff027e98ceb94083d3154b772fe50 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Wed, 8 Apr 2026 15:59:00 -0700 Subject: [PATCH 029/152] Add volumes for media and geoportal --- docker/AWS_DEPLOY.md | 8 ++++++++ docker/docker-compose.yml | 10 ++++++---- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/docker/AWS_DEPLOY.md b/docker/AWS_DEPLOY.md index 59ad3fa..a5d9fbb 100644 --- a/docker/AWS_DEPLOY.md +++ b/docker/AWS_DEPLOY.md @@ -284,6 +284,14 @@ GUNICORN_WORKERS=4 Leave everything else at its default for now. You can add email, OAuth, and Elasticsearch credentials later. +### 4.4 Create ini file + +```bash +cd ~/portals/madrona_portal/marco +cp config.docker.ini.template config.docker.wcoa.ini +``` + + --- ## Phase 5 — Build the Docker Image diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index fa9cafe..574db3b 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -17,6 +17,7 @@ services: dockerfile: madrona_portal/Dockerfile volumes: - static_data:/vol/web + - media_data:/vol/web env_file: - ../.env # load all secrets from the project-root .env file environment: @@ -66,7 +67,7 @@ services: db: image: postgis/postgis:16-3.4 volumes: - - postgis-data:/var/lib/postgresql + - postgis_data:/var/lib/postgresql environment: - POSTGRES_USER=${DB_USER:-postgres} - POSTGRES_PASSWORD=${DB_PASSWORD} @@ -90,7 +91,7 @@ services: ports: - "${REDIS_PORT:-6379}:6379" volumes: - - redis-data:/data + - redis_data:/data networks: - djangonetwork healthcheck: @@ -102,9 +103,10 @@ services: restart: unless-stopped volumes: - postgis-data: static_data: - redis-data: + media_data: + postgis_data: + redis_data: networks: djangonetwork: From 9a6fa1569d2a71fcba3367b6321f4d34a9fb9888 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Thu, 9 Apr 2026 19:05:44 -0700 Subject: [PATCH 030/152] Update environment and configuration templates for WCOA development - Clear default values for sensitive information in .env.example and config.docker.ini.template - Change region name to West Coast Ocean in config.docker.ini.template - Add instructions for creating ini file in README.md --- .env.example | 2 +- docker/README.md | 17 +++++++++++++- marco/config.docker.ini.template | 38 ++++++++++++++++---------------- 3 files changed, 36 insertions(+), 21 deletions(-) diff --git a/.env.example b/.env.example index 7d2d7bc..f1d1f26 100644 --- a/.env.example +++ b/.env.example @@ -29,7 +29,7 @@ APP_PORT=8000 # Redis (used for Django cache + Celery broker + result backend) # docker-compose builds REDIS_URL from REDIS_PASSWORD automatically. # --------------------------------------------------------------------------- -REDIS_PASSWORD=change-me +REDIS_PASSWORD= REDIS_PORT=6379 # REDIS_URL and CELERY_BROKER_URL are assembled in docker-compose.yml. diff --git a/docker/README.md b/docker/README.md index bca8c20..817204f 100644 --- a/docker/README.md +++ b/docker/README.md @@ -77,13 +77,28 @@ DJANGO_SUPERUSER_PASSWORD= Everything else has working defaults for local development. +### Step 4.1 - Create ini file + +```bash +cd marco +cp config.docker.ini.template config.docker.wcoa.ini +``` + +Edit `config.docker.wcoa.ini` : + +```ini +LOCATION = redis://tasks:6379/1 +CELERY_RESULT_BACKEND = redis://tasks:6379/1 +CELERY_BROKER_URL = redis://tasks:6379/0 +``` + ### Step 5 — Build the image Run this from the **workspace root** (`madrona_portal/`), not from inside `madrona_portal/`. The build context must include both repos. ```bash -cd .. # back to portals/ +cd ../../ # back to portals/ docker buildx build \ --builder desktop-linux \ diff --git a/marco/config.docker.ini.template b/marco/config.docker.ini.template index 462e1dd..fdfd641 100644 --- a/marco/config.docker.ini.template +++ b/marco/config.docker.ini.template @@ -3,21 +3,21 @@ [APP] APP_NAME = WCOA Portal -APP_URL = '' +APP_URL = APP_TEAM_NAME = Marine Planner Team PROJECT_APP = wcoa PROJECT_SETTINGS_FILE = True DEBUG = True TEMPLATE_DEBUG = True ALLOWED_HOSTS = ["localhost", "127.0.0.1", "::1"] -SECRET_KEY = You forgot to set the secret key +SECRET_KEY = # TODO: Does the MEDIA_ROOT need to be updated here? potentially /usr/local/apps/madrona-portal/apps/wcoa/media/ MEDIA_ROOT = /vol/web/media MEDIA_URL = /media/ TIME_ZONE = UTC -GA_ACCOUNT = You forgot to set the google analytics account -RECAPTCHA_PUBLIC_KEY = '' -RECAPTCHA_PRIVATE_KEY = '' +GA_ACCOUNT = +RECAPTCHA_PUBLIC_KEY = +RECAPTCHA_PRIVATE_KEY = STATIC_ROOT = /vol/web/static EMAIL_SUBJECT_PREFIX = [WCOA] MAP_LIBRARY = ol8 @@ -27,7 +27,7 @@ ADDITIONAL_APPS = [] ADDITIONAL_MIDDLEWARE = [] [REGION] -NAME = Mid-Atlantic Ocean +NAME = West Coast Ocean INIT_ZOOM = 6 INIT_LAT = 39 INIT_LON = -120 @@ -40,7 +40,6 @@ CLIENT_CLASS = django_redis.client.DefaultClient [CELERY] CELERY_RESULT_BACKEND = redis://:sOmE_sEcUrE_pAsS@tasks:6379/1 -BROKER_URL = redis://:sOmE_sEcUrE_pAsS@tasks:6379/0 CELERY_BROKER_URL = redis://:sOmE_sEcUrE_pAsS@tasks:6379 CELERY_ALWAYS_EAGER = False CELERY_DISABLE_RATE_LIMITS = True @@ -50,16 +49,15 @@ ENGINE = django.contrib.gis.db.backends.postgis NAME = wcoa_docker_db HOST = db PORT = 5432 -USER = wcoa_docker_user -PASSWORD = wcoa_docker_pass +USER = postgres [EMAIL] HOST = localhost PORT = 25 -HOST_USER = mail user -HOST_PASSWORD = mail password -DEFAULT_FROM_EMAIL = Mid-Atlantic Portal -SERVER_EMAIL = MidA Site Errors +HOST_USER = +HOST_PASSWORD = +DEFAULT_FROM_EMAIL = WCOA Portal +SERVER_EMAIL = WCOA Site Errors [AWS] AWS_ACCESS_KEY_ID = @@ -68,12 +66,14 @@ AWS_SES_REGION_NAME = us-east-1 AWS_SES_REGION_ENDPOINT = email.us-east-1.amazonaws.com [SOCIAL_AUTH] -FACEBOOK_KEY = You forgot to set the facebook key -FACEBOOK_SECRET = You forgot to set the facebook secret -TWITTER_KEY = You forgot to set the twitter key -TWITTER_SECRET = You forgot to set the twitter secret -GOOGLE_KEY = You forgot to set the google key -GOOGLE_SECRET = You forgot to set the google secret +# Supply these via env vars: FACEBOOK_KEY, FACEBOOK_SECRET, etc. +FACEBOOK_KEY = +FACEBOOK_SECRET = +TWITTER_KEY = +TWITTER_SECRET = +GOOGLE_KEY = +GOOGLE_SECRET = + [CATALOG] DATA_CATALOG_ENABLED = False From 8f9a93e3835a8f7eee8d27ca3a4ec5b57e948a78 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Thu, 9 Apr 2026 19:54:22 -0700 Subject: [PATCH 031/152] Update configuration file names for WCOA to include 'docker' suffix --- docker/AWS_DEPLOY.md | 2 +- docker/README.md | 11 +++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/docker/AWS_DEPLOY.md b/docker/AWS_DEPLOY.md index a5d9fbb..091ca9a 100644 --- a/docker/AWS_DEPLOY.md +++ b/docker/AWS_DEPLOY.md @@ -288,7 +288,7 @@ Elasticsearch credentials later. ```bash cd ~/portals/madrona_portal/marco -cp config.docker.ini.template config.docker.wcoa.ini +cp config.docker.ini.template config.wcoa.docker.ini ``` diff --git a/docker/README.md b/docker/README.md index 817204f..8af4048 100644 --- a/docker/README.md +++ b/docker/README.md @@ -81,10 +81,10 @@ Everything else has working defaults for local development. ```bash cd marco -cp config.docker.ini.template config.docker.wcoa.ini +cp config.docker.ini.template config.wcoa.docker.ini ``` -Edit `config.docker.wcoa.ini` : +Edit `config.wcoa.docker.ini` : ```ini LOCATION = redis://tasks:6379/1 @@ -143,8 +143,15 @@ On first boot the entrypoint automatically: Open: http://localhost:8000/ (or whatever `APP_PORT` is set to in `.env`) +### Step 7 - Import the Database +To import a database dump, copy the file into the ... + +```bash + --- +# Untested instructions below this line — will update after testing + ## Deploy to fully containerized live instance 1. Set up your `.env` From 84b4c226d73ca7df7a0dcaff66ebd3dd78ab6fe8 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Fri, 10 Apr 2026 09:45:53 -0700 Subject: [PATCH 032/152] Add db-restore.sh script for restoring PostgreSQL dumps in Docker --- scripts/db-restore.sh | 123 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100755 scripts/db-restore.sh diff --git a/scripts/db-restore.sh b/scripts/db-restore.sh new file mode 100755 index 0000000..f16bd25 --- /dev/null +++ b/scripts/db-restore.sh @@ -0,0 +1,123 @@ +#!/usr/bin/env bash +# ----------------------------------------------------------------------------- +# db-restore.sh — Restore a PostgreSQL dump into the Dockerized dev database. +# +# Usage: +# ./scripts/db-restore.sh +# ./scripts/db-restore.sh --drop # drop & recreate DB first +# +# Run from anywhere — this script always operates relative to madrona_portal/. +# +# Prerequisites: +# 1. Docker Compose stack is running: +# docker compose -f docker/docker-compose.yml --env-file .env --profile full up -d +# 2. madrona_portal/.env exists and contains DB_NAME, DB_USER, DB_PASSWORD. +# +# Options: +# --drop Terminate all active connections, drop, and recreate the target +# database before restoring. Required for a clean import from prod. +# Without this flag the dump is applied on top of existing data. +# +# Notes: +# - The dump is streamed directly into the container — no temp files on disk. +# - psql warnings (e.g. "already exists") are normal when importing a dump +# produced on a different Postgres version (12 → 16) and are not fatal. +# - After a --drop restore, run migrations to pick up any schema drift: +# docker compose exec app python marco/manage.py migrate +# ----------------------------------------------------------------------------- +set -euo pipefail + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- +die() { echo "[db-restore] ERROR: $*" >&2; exit 1; } +info() { echo "[db-restore] $*"; } + +# --------------------------------------------------------------------------- +# Parse arguments +# --------------------------------------------------------------------------- +DROP_FIRST=false +DUMP_FILE="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --drop) DROP_FIRST=true; shift ;; + -*) die "Unknown option: '$1'. Usage: $0 [--drop] " ;; + *) [[ -z "$DUMP_FILE" ]] || die "Unexpected argument: '$1'" + DUMP_FILE="$1"; shift ;; + esac +done + +[[ -n "$DUMP_FILE" ]] || die "Usage: $0 [--drop] " + +# Resolve dump path before we cd away. +DUMP_ABS="$(cd "$(dirname "$DUMP_FILE")" && pwd)/$(basename "$DUMP_FILE")" +[[ -f "$DUMP_ABS" ]] || die "Dump file not found: $DUMP_FILE" + +# --------------------------------------------------------------------------- +# Always operate from madrona_portal/ regardless of where the script is called +# --------------------------------------------------------------------------- +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$SCRIPT_DIR/.." + +# --------------------------------------------------------------------------- +# Load .env for DB credentials +# --------------------------------------------------------------------------- +[[ -f .env ]] || die ".env not found in $(pwd). Copy .env.example and fill in values." + +set -a +# shellcheck source=/dev/null +source .env +set +a + +DB_NAME="${DB_NAME:-wcoa_docker_db}" +DB_USER="${DB_USER:-postgres}" +DB_PASSWORD="${DB_PASSWORD:?DB_PASSWORD must be set in .env}" + +COMPOSE="docker compose -f docker/docker-compose.yml --env-file .env" +PSQL="$COMPOSE exec -T -e PGPASSWORD=$DB_PASSWORD db psql -U $DB_USER" + +# --------------------------------------------------------------------------- +# Verify the db container is healthy before doing anything +# --------------------------------------------------------------------------- +info "Checking db service health..." +$COMPOSE ps db | grep -q "healthy" \ + || die "db container is not healthy. Is the stack running? Try: $COMPOSE up -d" + +# --------------------------------------------------------------------------- +# Optional: terminate connections, drop, and recreate the database +# --------------------------------------------------------------------------- +if [[ "$DROP_FIRST" == true ]]; then + info "Terminating active connections to '$DB_NAME'..." + $PSQL -d postgres -c \ + "SELECT pg_terminate_backend(pid) + FROM pg_stat_activity + WHERE datname = '$DB_NAME' AND pid <> pg_backend_pid();" \ + > /dev/null + + info "Dropping database '$DB_NAME'..." + $PSQL -d postgres -c "DROP DATABASE IF EXISTS \"$DB_NAME\";" + + info "Creating database '$DB_NAME'..." + $PSQL -d postgres -c "CREATE DATABASE \"$DB_NAME\";" + + info "Enabling PostGIS extension..." + $PSQL -d "$DB_NAME" -c "CREATE EXTENSION IF NOT EXISTS postgis;" +fi + +# --------------------------------------------------------------------------- +# Stream the dump into the database +# --------------------------------------------------------------------------- +DUMP_SIZE="$(du -sh "$DUMP_ABS" | cut -f1)" +info "Restoring '$DUMP_FILE' (${DUMP_SIZE}) → '$DB_NAME'..." +info "psql warnings about existing objects are expected and non-fatal." + +$PSQL -d "$DB_NAME" \ + --set ON_ERROR_STOP=off \ + < "$DUMP_ABS" + +info "Restore complete." +info "" +info "Next steps:" +info " Apply any pending migrations:" +info " docker compose exec app python marco/manage.py migrate" From cec8965629db129582b3417f553f6958b81ef5f8 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Fri, 10 Apr 2026 09:47:58 -0700 Subject: [PATCH 033/152] Update README.md to include instructions for importing production SQL dumps --- docker/README.md | 52 ++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/docker/README.md b/docker/README.md index 8af4048..c4e2adb 100644 --- a/docker/README.md +++ b/docker/README.md @@ -143,10 +143,58 @@ On first boot the entrypoint automatically: Open: http://localhost:8000/ (or whatever `APP_PORT` is set to in `.env`) -### Step 7 - Import the Database -To import a database dump, copy the file into the ... +### Step 7 - Importing a Production SQL Dump into the Dockerized Database +#### Prerequisites +- Docker Compose stack is running — `docker compose -f docker/docker-compose.yml --env-file .env --profile full up -d` +- `madrona_portal/.env` exists with `DB_NAME`, `DB_USER`, and `DB_PASSWORD` set + + +#### Step 7.1 — Ensure you have the db-restore script + +`madrona_portal/scripts/db-restore.sh` has the following behaviour: +- Loads DB credentials from `.env` +- Verifies the `db` container is healthy before proceeding +- With `--drop`: terminates active connections, drops and recreates the database, and enables the PostGIS extension +- Streams the dump file directly into the container via `docker compose exec` (no temp files) +- Prints next-step instructions on completion + +Made sure it is executable: +```bash +chmod +x madrona_portal/scripts/db-restore.sh +``` + + +#### Step 7.2 — Run the restore + +From `madrona_portal/`: +```bash +./scripts/db-restore.sh --drop +``` +*example:* ```bash +./scripts/db-restore.sh --drop ../madrona-apps/wcoa/wcodp_prod_dump_20260320.sql +``` + +The `--drop` flag was used to ensure a clean import. The script: +1. Terminated all active connections to `wcoa_docker_db` +2. Dropped and recreated the database +3. Enabled the `postgis` extension +4. Streamed the sql dump into the container via `psql` + +**Expected warnings (non-fatal):** +- `ERROR: relation "..." does not exist` — pg_dump tries to drop constraints before creating them; safe to ignore on a fresh DB +- `ERROR: role "wcoa_user" does not exist` — prod uses a dedicated app role; dev uses `postgres` which has full access + + +#### Step 7.3 — Apply migrations + +```bash +docker compose -f docker/docker-compose.yml --env-file .env --profile full exec app python marco/manage.py migrate +``` + +*Please note:* on 4/10/2026 a 130+ migrations were applied to bring the schema from the prod dump (PostgreSQL 12) up to date with the current codebase (PostgreSQL 16). + --- From 5d1e3dd68e43c06a7f9395c8f1255efd0c557440 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Fri, 10 Apr 2026 10:10:28 -0700 Subject: [PATCH 034/152] Add instructions for importing production media files into Dockerized application --- docker/README.md | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/docker/README.md b/docker/README.md index c4e2adb..f39ed4b 100644 --- a/docker/README.md +++ b/docker/README.md @@ -195,6 +195,52 @@ docker compose -f docker/docker-compose.yml --env-file .env --profile full exec *Please note:* on 4/10/2026 a 130+ migrations were applied to bring the schema from the prod dump (PostgreSQL 12) up to date with the current codebase (PostgreSQL 16). +--- + +### Step 8 — Importing production media files into the Dockerized Application + +#### Prerequisites +- `madrona_portal/.env` exists with `MEDIA_ROOT` set to a valid directory +- Production media files are available + +#### Step 8.1a - Copy the media files into Docker + +Get your docker container name or ID for the `app` service: +```bash +docker ps +``` + +Then use `docker cp` to copy media files from the production location to the local directory specified by `MEDIA_ROOT` in `.env`. +```bash +docker cp /. :/vol/web/media/ +``` + +Example `docker cp` command: +```bash +docker cp madrona-apps/wcoa/media/. docker-app-1:/vol/web/media/ +``` + +#### Step 8.1b — Sync media files +Use `rsync` or a similar tool to copy media files from the production location to the local directory specified by `MEDIA_ROOT` in `.env`. +Using `rsync` command: +```bash +rsync -avz @:/path/to/remote/media/ / +``` + +Then use `docker cp` to copy media files from the local directory to the Docker container: +```bash +docker cp /. :/vol/web/media/ +``` + +Make sure to include the trailing slashes to sync the contents correctly. +`-avz` flags preserve permissions, show progress, and compress data during transfer. + +#### Step 8.2 — Verify media access + +```bash +docker exec du -sh /vol/web/media/ +docker exec ls /vol/web/media/ +``` --- From 35e875b84c14f7e4ec68d64bd81703e01ae513ec Mon Sep 17 00:00:00 2001 From: David Pollard Date: Fri, 10 Apr 2026 16:01:21 -0700 Subject: [PATCH 035/152] Add migration instructions for mp-layers to README.md --- docker/README.md | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/docker/README.md b/docker/README.md index f39ed4b..3561c4c 100644 --- a/docker/README.md +++ b/docker/README.md @@ -195,6 +195,45 @@ docker compose -f docker/docker-compose.yml --env-file .env --profile full exec *Please note:* on 4/10/2026 a 130+ migrations were applied to bring the schema from the prod dump (PostgreSQL 12) up to date with the current codebase (PostgreSQL 16). +#### Step 7.4 - Migration to mp-layers + +```bash +docker compose -f docker/docker-compose.yml --env-file .env --profile full exec app python marco/manage.py migrate migration_to_layers +``` + +```bash +docker exec -it bash +``` + +Then inside the container: + +```bash +python marco/manage.py shell +``` + +```python +from layers.models import Theme +from data_manager.models import Theme as Dm_theme + + +for theme in Dm_theme.all_objects.all(): + try: + new_theme = Theme.all_objects.get(pk=theme.pk) + if new_theme.parent == None and new_theme.name != 'companion': + new_theme.is_top_theme = True + new_theme.save() + except Exception: + pass +``` + +exit the shell + +```bash +python marco/manage.py collectstatic +python marco/manage.py compress +``` + + --- ### Step 8 — Importing production media files into the Dockerized Application From bf3a6671ee49bd92d48d9cf888e46263929b1bb4 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Fri, 10 Apr 2026 14:46:26 -0700 Subject: [PATCH 036/152] Add API URL patterns for visualize, drawing, and mapgroups modules --- marco/marco/urls.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/marco/marco/urls.py b/marco/marco/urls.py index 96f21a0..f012640 100644 --- a/marco/marco/urls.py +++ b/marco/marco/urls.py @@ -20,6 +20,10 @@ import accounts.urls import explore.urls +from visualize.urls import api_urlpatterns as visualize_api_urlpatterns +from drawing.urls import api_urlpatterns as drawing_api_urlpatterns +from mapgroups.urls import api_urlpatterns as mapgroups_api_urlpatterns + from portal.base import views as base_views from portal.data_catalog import views as data_catalog_views from marco_site import views as marco_site_views @@ -51,7 +55,10 @@ re_path(r'^django-admin/', admin.site.urls), re_path(r'^admin/', include(wagtailadmin_urls)), - # /rpc endpoint removed — see each sub-app's api.py for DRF replacements + # /rpc endpoint removed — DRF replacements from each sub-app's api.py + re_path(r'^api/', include(visualize_api_urlpatterns)), + re_path(r'^api/', include(drawing_api_urlpatterns)), + re_path(r'^api/', include(mapgroups_api_urlpatterns)), re_path(r'^auth/', include('social_django.urls', namespace='social')), re_path(r'^account/', include('accounts.urls'), name='account'), From 47d0594c24399502f0310411707ed01946c246e6 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Fri, 10 Apr 2026 14:58:48 -0700 Subject: [PATCH 037/152] Fix JSON-RPC URL by adding trailing slash to the endpoint --- marco/marco_site/static/js/jsonrpc.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/marco/marco_site/static/js/jsonrpc.js b/marco/marco_site/static/js/jsonrpc.js index 97d63c0..65e57cc 100644 --- a/marco/marco_site/static/js/jsonrpc.js +++ b/marco/marco_site/static/js/jsonrpc.js @@ -60,7 +60,7 @@ function jsonrpc_call(method, args, options) { request_encoded = JSON.stringify(request); $.ajax({ - url: '/rpc', + url: '/rpc/', method: 'POST', data: request_encoded, dataType: 'json', From 4c42277da5becbfa6315d46f4d77698bef288ef8 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Fri, 10 Apr 2026 16:01:58 -0700 Subject: [PATCH 038/152] Add JSON-RPC 2.0 compatibility shim and update URL routing --- marco/marco/rpc_compat.py | 375 ++++++++++++++++++++++++++++++++++++++ marco/marco/urls.py | 5 +- 2 files changed, 379 insertions(+), 1 deletion(-) create mode 100644 marco/marco/rpc_compat.py diff --git a/marco/marco/rpc_compat.py b/marco/marco/rpc_compat.py new file mode 100644 index 0000000..a175700 --- /dev/null +++ b/marco/marco/rpc_compat.py @@ -0,0 +1,375 @@ +"""JSON-RPC 2.0 compatibility shim — POST /rpc/ + +Accepts JSON-RPC 2.0 requests from the legacy jsonrpc.js frontend and +dispatches them to the same business logic as the new DRF REST API views. + +This shim allows the frontend JavaScript to keep using $.jsonrpc() without +modification while the backend has moved to REST endpoints at /api/. + +rpc4django required @csrf_exempt on the /rpc/ view; this shim preserves +that behaviour so the JS can send application/json without a CSRF token. +""" +from __future__ import annotations + +import json +from typing import Any + +from django.conf import settings +from django.contrib.auth.models import Group +from django.http import JsonResponse +from django.views.decorators.csrf import csrf_exempt +from django.views.decorators.http import require_POST + + +# --------------------------------------------------------------------------- +# JSON-RPC 2.0 response helpers +# --------------------------------------------------------------------------- + +def _ok(result: Any, rpc_id: Any) -> JsonResponse: + return JsonResponse({'jsonrpc': '2.0', 'id': rpc_id, 'result': result}) + + +def _err(message: str, rpc_id: Any, code: int = -32000) -> JsonResponse: + return JsonResponse({'jsonrpc': '2.0', 'id': rpc_id, 'error': {'code': code, 'message': message}}) + + +# --------------------------------------------------------------------------- +# Bookmark handlers +# --------------------------------------------------------------------------- + +def _get_bookmarks(request: Any) -> list: + from visualize.models import Bookmark + + content: list[dict] = [] + bookmark_list = Bookmark.objects.filter(user=request.user) + + for bookmark in bookmark_list: + sharing_groups = [ + g.mapgroup_set.get().name for g in bookmark.sharing_groups.all() + ] + content.append({ + 'uid': bookmark.uid, + 'name': bookmark.name, + 'description': bookmark.description, + 'hash': bookmark.url_hash, + 'sharing_groups': sharing_groups, + 'json': bookmark.json, + }) + + shared_bookmarks = Bookmark.objects.shared_with_user(request.user) + for bookmark in shared_bookmarks: + if bookmark not in bookmark_list: + groups = bookmark.sharing_groups.filter(user__in=[request.user]) + shared_groups = [g.mapgroup_set.get().name for g in groups] + content.append({ + 'uid': bookmark.uid, + 'name': bookmark.name, + 'description': bookmark.description, + 'hash': bookmark.url_hash, + 'shared': True, + 'shared_by_user': bookmark.user.id, + 'shared_to_groups': shared_groups, + 'shared_by_name': bookmark.user.get_short_name(), + 'json': bookmark.json, + }) + + return content + + +def _add_bookmark(request: Any, params: list) -> bool: + from visualize.models import Bookmark + + name, description, url_hash, json_str = params[0], params[1], params[2], params[3] + bookmark = Bookmark( + user=request.user, + name=name, + description=description, + url_hash=url_hash, + json=json_str, + ) + bookmark.save() + return True + + +def _load_bookmark(params: list) -> list: + from visualize.models import Bookmark + + bookmark = Bookmark.objects.get(pk=int(params[0])) + return [{'uid': bookmark.uid, 'hash': bookmark.url_hash, 'json': bookmark.json}] + + +def _remove_bookmark(request: Any, params: list) -> bool: + from features.registry import get_feature_by_uid + + bookmark = get_feature_by_uid(params[0]) + viewable, _ = bookmark.is_viewable(request.user) + if viewable: + bookmark.delete() + return True + + +def _share_bookmark(request: Any, params: list) -> bool: + from features.registry import get_feature_by_uid + + uid, group_names = params[0], params[1] + bookmark = get_feature_by_uid(uid) + viewable, _ = bookmark.is_viewable(request.user) + if not viewable: + return False + bookmark.share_with(None) + groups = [Group.objects.get(mapgroup__name=gname) for gname in group_names] + bookmark.share_with(groups, append=False) + return True + + +# --------------------------------------------------------------------------- +# User Layer handlers +# --------------------------------------------------------------------------- + +def _get_user_layers(request: Any) -> list: + from visualize.models import UserLayer + + content: list[dict] = [] + + try: + user_layer_list = UserLayer.objects.filter(user=request.user) + except TypeError: + user_layer_list = [] + + for ul in user_layer_list: + sharing_groups = [ + g.mapgroup_set.get().name + for g in ul.sharing_groups.all() + if g.mapgroup_set.exists() + ] + public_groups = [ + g.name + for g in Group.objects.filter(name__in=settings.SHARING_TO_PUBLIC_GROUPS) + if g in ul.sharing_groups.all() + ] + content.append({ + 'id': ul.id, + 'uid': ul.uid, + 'name': ul.name, + 'description': ul.description, + 'url': ul.url, + 'layer_type': ul.layer_type, + 'password_protected': ul.password_protected, + 'arcgis_layers': ul.arcgis_layers, + 'sharing_groups': sharing_groups + public_groups, + 'shared_to_groups': sharing_groups, + 'owned_by_user': True, + 'wms_slug': ul.wms_slug, + 'wms_srs': ul.wms_srs, + 'wms_params': ul.wms_params, + 'wms_version': ul.wms_version, + 'wms_format': ul.wms_format, + 'wms_styles': ul.wms_styles, + }) + + try: + shared_layers = UserLayer.objects.shared_with_user(request.user) + except TypeError: + shared_layers = UserLayer.objects.filter(pk=-1) + + for ul in shared_layers: + if ul not in user_layer_list: + try: + permission_groups = [ + x.map_group.permission_group + for x in request.user.mapgroupmember_set.all() + ] + except TypeError: + permission_groups = [] + + sharing_groups = [ + g.mapgroup_set.get().name + for g in ul.sharing_groups.all() + if g.mapgroup_set.exists() and g in permission_groups + ] + public_groups = [ + g.name + for g in Group.objects.filter(name__in=settings.SHARING_TO_PUBLIC_GROUPS) + if g in ul.sharing_groups.all() + ] + content.append({ + 'id': ul.id, + 'uid': ul.uid, + 'name': ul.name, + 'description': ul.description, + 'url': ul.url, + 'layer_type': ul.layer_type, + 'password_protected': ul.password_protected, + 'arcgis_layers': ul.arcgis_layers, + 'shared': True, + 'shared_by_user': ul.user.id, + 'sharing_groups': sharing_groups + public_groups, + 'shared_to_groups': sharing_groups, + 'shared_by_name': ul.user.get_short_name(), + 'owned_by_user': len(sharing_groups) > 0, + 'wms_slug': ul.wms_slug, + 'wms_srs': ul.wms_srs, + 'wms_params': ul.wms_params, + 'wms_version': ul.wms_version, + 'wms_format': ul.wms_format, + 'wms_styles': ul.wms_styles, + }) + + return content + + +def _add_user_layer(request: Any, params: list) -> bool: + from visualize.models import UserLayer + + (name, description, layer_type, url, arcgis_layers, + wms_slug, wms_srs, wms_params, wms_version, wms_format, wms_styles) = params + + ul = UserLayer( + user=request.user, + name=name, + description=description or '', + url=url, + layer_type=layer_type, + arcgis_layers=arcgis_layers or '', + wms_slug=wms_slug, + wms_srs=wms_srs, + wms_params=wms_params, + wms_version=wms_version, + wms_format=wms_format, + wms_styles=wms_styles, + ) + ul.save() + return True + + +def _remove_user_layer(request: Any, params: list) -> bool: + from features.registry import get_feature_by_uid + + ul = get_feature_by_uid(params[0]) + viewable, _ = ul.is_viewable(request.user) + if viewable: + ul.delete() + return True + + +def _share_user_layer(request: Any, params: list) -> bool: + from features.registry import get_feature_by_uid + + uid, group_names = params[0], params[1] + ul = get_feature_by_uid(uid) + viewable, _ = ul.is_viewable(request.user) + if not viewable: + return False + ul.share_with(None) + groups = [Group.objects.get(mapgroup__name=gname) for gname in group_names] + ul.share_with(groups, append=False) + return True + + +# --------------------------------------------------------------------------- +# Sharing Groups handler +# --------------------------------------------------------------------------- + +def _get_sharing_groups(request: Any) -> list: + data: list[dict] = [] + + for membership in request.user.mapgroupmember_set.all(): + group = membership.map_group + members = sorted( + member.user_name_for_group() + for member in group.mapgroupmember_set.all() + ) + data.append({ + 'group_name': group.name, + 'group_slug': group.permission_group.name, + 'members': members, + 'is_mapgroup': True, + }) + + for public_group in Group.objects.filter(name__in=settings.SHARING_TO_PUBLIC_GROUPS): + data.append({ + 'group_name': public_group.name, + 'group_slug': public_group.name, + 'members': [], + 'is_mapgroup': False, + }) + + return data + + +# --------------------------------------------------------------------------- +# Drawing handler +# --------------------------------------------------------------------------- + +def _delete_drawing(request: Any, params: list) -> bool: + from features.registry import get_feature_by_uid + + drawing = get_feature_by_uid(params[0]) + viewable, _ = drawing.is_viewable(request.user) + if viewable: + drawing.delete() + return True + + +# --------------------------------------------------------------------------- +# Dispatcher +# --------------------------------------------------------------------------- + +_AUTH_REQUIRED = frozenset({ + 'get_bookmarks', 'add_bookmark', 'remove_bookmark', 'share_bookmark', + 'add_user_layer', 'remove_user_layer', 'share_user_layer', + 'get_sharing_groups', 'delete_drawing', +}) + + +def _dispatch(request: Any, method: str, params: list, rpc_id: Any) -> JsonResponse: + if method in _AUTH_REQUIRED and not request.user.is_authenticated: + return _err('Authentication required.', rpc_id, code=-32001) + + try: + if method == 'get_bookmarks': + return _ok(_get_bookmarks(request), rpc_id) + elif method == 'add_bookmark': + return _ok(_add_bookmark(request, params), rpc_id) + elif method == 'load_bookmark': + return _ok(_load_bookmark(params), rpc_id) + elif method == 'remove_bookmark': + return _ok(_remove_bookmark(request, params), rpc_id) + elif method == 'share_bookmark': + return _ok(_share_bookmark(request, params), rpc_id) + elif method == 'get_user_layers': + return _ok(_get_user_layers(request), rpc_id) + elif method == 'add_user_layer': + return _ok(_add_user_layer(request, params), rpc_id) + elif method == 'remove_user_layer': + return _ok(_remove_user_layer(request, params), rpc_id) + elif method == 'share_user_layer': + return _ok(_share_user_layer(request, params), rpc_id) + elif method == 'get_sharing_groups': + return _ok(_get_sharing_groups(request), rpc_id) + elif method == 'delete_drawing': + return _ok(_delete_drawing(request, params), rpc_id) + else: + return _err(f'Method not found: {method}', rpc_id, code=-32601) + except Exception as exc: + return _err(str(exc), rpc_id) + + +@csrf_exempt +@require_POST +def rpc_view(request): + """JSON-RPC 2.0 endpoint — POST /rpc/ + + Accepts legacy jsonrpc.js requests and dispatches them to the same + business logic as the REST API views at /api/. + """ + try: + body = json.loads(request.body) + except (json.JSONDecodeError, ValueError): + return _err('Parse error.', None, code=-32700) + + method = body.get('method', '') + params = body.get('params', []) + rpc_id = body.get('id', 7) + + return _dispatch(request, method, params, rpc_id) diff --git a/marco/marco/urls.py b/marco/marco/urls.py index f012640..25710a2 100644 --- a/marco/marco/urls.py +++ b/marco/marco/urls.py @@ -23,6 +23,7 @@ from visualize.urls import api_urlpatterns as visualize_api_urlpatterns from drawing.urls import api_urlpatterns as drawing_api_urlpatterns from mapgroups.urls import api_urlpatterns as mapgroups_api_urlpatterns +from marco.rpc_compat import rpc_view from portal.base import views as base_views from portal.data_catalog import views as data_catalog_views @@ -55,7 +56,9 @@ re_path(r'^django-admin/', admin.site.urls), re_path(r'^admin/', include(wagtailadmin_urls)), - # /rpc endpoint removed — DRF replacements from each sub-app's api.py + # /rpc — JSON-RPC 2.0 compat shim for legacy frontend JS (see rpc_compat.py) + re_path(r'^rpc/', rpc_view), + # DRF REST replacements from each sub-app's api.py re_path(r'^api/', include(visualize_api_urlpatterns)), re_path(r'^api/', include(drawing_api_urlpatterns)), re_path(r'^api/', include(mapgroups_api_urlpatterns)), From 4f1afa8060a7fa64e940486c32acbb1dcc364f33 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Wed, 15 Apr 2026 12:55:48 -0700 Subject: [PATCH 039/152] Set DEBUG to False in core settings for production readiness --- marco/marco/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/marco/marco/settings.py b/marco/marco/settings.py index a39ffc2..220f941 100644 --- a/marco/marco/settings.py +++ b/marco/marco/settings.py @@ -67,7 +67,7 @@ def _env(env_key: str, cfg_section: configparser.SectionProxy, cfg_key: str, # --------------------------------------------------------------------------- # Core settings # --------------------------------------------------------------------------- -DEBUG = app_cfg.getboolean('DEBUG', True) +DEBUG = app_cfg.getboolean('DEBUG', False) APP_NAME = app_cfg.get('APP_NAME', 'Marine Planner') APP_URL = app_cfg.get('APP_URL', '') From 854a9b7b333505221795a3c8b10e6ecbfaea7563 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Wed, 15 Apr 2026 14:45:03 -0700 Subject: [PATCH 040/152] Remove rpc4django from Docker requirements and replace with djangorestframework --- docker/docker-requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/docker/docker-requirements.txt b/docker/docker-requirements.txt index 0af3df6..7837218 100644 --- a/docker/docker-requirements.txt +++ b/docker/docker-requirements.txt @@ -73,7 +73,6 @@ elasticsearch-dsl>=7.0,<8.0 # APIs # --------------------------------------------------------------------------- # rpc4django removed — replaced by djangorestframework API views -rpc4django djangorestframework>=3.14,<4.0 # --------------------------------------------------------------------------- From b558ce653f02e0d440bf295b315f0943c8273079 Mon Sep 17 00:00:00 2001 From: Ryan Hodges Date: Wed, 15 Apr 2026 17:46:02 -0700 Subject: [PATCH 041/152] first stab at streamlining docker build and toggling database manipulation --- .env.example => docker/.env.example | 0 docker/AWS_DEPLOY.md | 34 ++++---- Dockerfile => docker/Dockerfile | 15 ++-- docker/README.md | 124 ++++++++++++---------------- docker/docker-compose.yml | 40 +++++++-- docker/entrypoint.sh | 41 +++++---- marco/config.docker.ini.template | 3 +- 7 files changed, 141 insertions(+), 116 deletions(-) rename .env.example => docker/.env.example (100%) rename Dockerfile => docker/Dockerfile (90%) diff --git a/.env.example b/docker/.env.example similarity index 100% rename from .env.example rename to docker/.env.example diff --git a/docker/AWS_DEPLOY.md b/docker/AWS_DEPLOY.md index 091ca9a..91f9806 100644 --- a/docker/AWS_DEPLOY.md +++ b/docker/AWS_DEPLOY.md @@ -161,7 +161,7 @@ sudo systemctl enable containerd ## Phase 3 — Clone the Repositories The Dockerfile build context must be the **workspace root** — a parent -directory containing both `madrona_portal/` and `madrona-apps/` as siblings. +directory containing both `madrona-portal/` and `madrona-apps/` as siblings. This layout is required; it is not optional. ### 3.1 Set up SSH access to GitHub (on the server) @@ -198,7 +198,7 @@ mkdir ~/portals && cd ~/portals Clone the main portal: ```bash -git clone -b docker git@github.com:Ecotrust/madrona-portal.git madrona_portal +git clone -b docker git@github.com:Ecotrust/madrona-portal.git madrona-portal ``` Clone all sub-apps: @@ -229,7 +229,7 @@ Verify the layout: ```bash ls ~/portals/ -# madrona_portal/ madrona-apps/ +# madrona-portal/ madrona-apps/ ``` --- @@ -239,7 +239,7 @@ ls ~/portals/ ### 4.1 Create the `.env` file ```bash -cd ~/portals/madrona_portal +cd ~/portals/madrona-portal cp .env.example .env ``` @@ -287,7 +287,7 @@ Elasticsearch credentials later. ### 4.4 Create ini file ```bash -cd ~/portals/madrona_portal/marco +cd ~/portals/madrona-portal/marco cp config.docker.ini.template config.wcoa.docker.ini ``` @@ -307,8 +307,8 @@ cd ~/portals docker buildx build \ --load \ - -f madrona_portal/Dockerfile \ - -t madrona_portal-app:latest \ + -f madrona-portal/Dockerfile \ + -t madrona-portal-app:latest \ . ``` @@ -323,10 +323,10 @@ Watch for any errors. Common first-build issues: ## Phase 6 — Start the Stack -From `~/portals/madrona_portal/`: +From `~/portals/madrona-portal/`: ```bash -cd ~/portals/madrona_portal +cd ~/portals/madrona-portal docker compose -f docker/docker-compose.yml \ --env-file .env \ @@ -444,7 +444,7 @@ ALLOWED_HOSTS=portal.yourdomain.com,,localhost Then restart the app container to pick up the change: ```bash -cd ~/portals/madrona_portal +cd ~/portals/madrona-portal docker compose -f docker/docker-compose.yml --env-file .env --profile full \ up -d --force-recreate app ``` @@ -471,7 +471,7 @@ After=docker.service network-online.target [Service] Type=oneshot RemainAfterExit=yes -WorkingDirectory=/home/ubuntu/portals/madrona_portal +WorkingDirectory=/home/ubuntu/portals/madrona-portal ExecStart=/usr/bin/docker compose \ -f docker/docker-compose.yml \ --env-file .env \ @@ -515,22 +515,22 @@ be included in the image. **2. On the server — pull and rebuild:** ```bash -cd ~/portals/madrona_portal && git pull +cd ~/portals/madrona-portal && git pull cd ~/portals/madrona-apps/ && git pull # repeat for each changed sub-app cd ~/portals docker buildx build \ --load \ - -f madrona_portal/Dockerfile \ - -t madrona_portal-app:latest \ + -f madrona-portal/Dockerfile \ + -t madrona-portal-app:latest \ . ``` **3. Recreate the app container:** ```bash -cd ~/portals/madrona_portal +cd ~/portals/madrona-portal docker compose -f docker/docker-compose.yml --env-file .env --profile full \ up -d --force-recreate app @@ -543,7 +543,7 @@ container restart (~5–10 seconds). ## Useful Commands (on the server) -All `docker compose` commands run from `~/portals/madrona_portal/`. +All `docker compose` commands run from `~/portals/madrona-portal/`. ```bash # Tail app logs @@ -578,7 +578,7 @@ docker compose -f docker/docker-compose.yml --env-file .env --profile full down | Service | Image | Internal port | Exposed to host | |---|---|---|---| -| `app` | `madrona_portal-app:latest` | 8000 | Yes — proxied by Nginx | +| `app` | `madrona-portal-app:latest` | 8000 | Yes — proxied by Nginx | | `db` | `postgis/postgis:16-3.4` | 5432 | Yes (restrict in security group) | | `tasks` | `redis:7-alpine` | 6379 | Yes (restrict in security group) | | `geoportal` | built from `wcoa/docker` | 8080 | Yes (add Nginx location if needed) | diff --git a/Dockerfile b/docker/Dockerfile similarity index 90% rename from Dockerfile rename to docker/Dockerfile index ad02b68..031d749 100644 --- a/Dockerfile +++ b/docker/Dockerfile @@ -41,13 +41,13 @@ RUN python3 -m venv /opt/venv # --------------------------------------------------------------------------- # Copy application source # --------------------------------------------------------------------------- -COPY madrona_portal/marco ./marco -COPY madrona_portal/apps/__init__.py ./apps/__init__.py -COPY madrona_portal/assets ./assets -COPY madrona_portal/bower_components ./bower_components -COPY madrona_portal/docker/entrypoint.sh /entrypoint.sh -COPY madrona_portal/docker/docker-requirements.txt /requirements.txt -COPY madrona_portal/backups ./backups +COPY madrona-portal/marco ./marco +COPY madrona-portal/apps/__init__.py ./apps/__init__.py +COPY madrona-portal/assets ./assets +COPY madrona-portal/bower_components ./bower_components +COPY madrona-portal/docker/entrypoint.sh /entrypoint.sh +COPY madrona-portal/docker/docker-requirements.txt /requirements.txt +COPY madrona-portal/backups ./backups COPY madrona-apps/django_url_shortener ./apps/django_url_shortener COPY madrona-apps/madrona-analysistools ./apps/madrona-analysistools @@ -90,5 +90,6 @@ RUN chmod 755 /entrypoint.sh && \ USER madrona_user EXPOSE 8000 +EXPOSE 8008 CMD ["/entrypoint.sh"] diff --git a/docker/README.md b/docker/README.md index 3561c4c..3184c68 100644 --- a/docker/README.md +++ b/docker/README.md @@ -17,7 +17,7 @@ cd portals ### Step 2 — Clone madrona-portal ```bash -git clone -b docker https://github.com/Ecotrust/madrona-portal.git madrona_portal +git clone -b docker https://github.com/Ecotrust/madrona-portal.git madrona-portal ``` ### Step 3 — Clone the sub-app packages @@ -51,7 +51,7 @@ Your workspace should now look like: ``` portals/ -├── madrona_portal/ ← cloned from Ecotrust/madrona-portal, branch: docker +├── madrona-portal/ ← cloned from Ecotrust/madrona-portal, branch: docker └── madrona-apps/ ├── wcoa/ ← branch: vagrant2docker ├── mp-layers/ @@ -60,10 +60,10 @@ portals/ ### Step 4 — Configure environment -From `madrona_portal/`: +From `madrona-portal/`: ```bash -cd madrona_portal +cd madrona-portal/docker cp .env.example .env ``` @@ -80,7 +80,7 @@ Everything else has working defaults for local development. ### Step 4.1 - Create ini file ```bash -cd marco +cd ../marco cp config.docker.ini.template config.wcoa.docker.ini ``` @@ -94,18 +94,25 @@ CELERY_BROKER_URL = redis://tasks:6379/0 ### Step 5 — Build the image -Run this from the **workspace root** (`madrona_portal/`), not from -inside `madrona_portal/`. The build context must include both repos. +Run this from the **workspace root** (`madrona-portal/`), not from +inside `madrona-portal/`. The build context must include both repos. ```bash -cd ../../ # back to portals/ +cd ../docker + +docker compose build + +docker buildx build --load -f ./Dockerfile ../../ +``` +When building a tagged image for deployment (use `builder desktop-linux` if on Mac): +``` docker buildx build \ --builder desktop-linux \ --load \ - -f madrona_portal/Dockerfile \ - -t madrona_portal-app:latest \ - . + -f ./Dockerfile \ + -t madrona-portal-app:latest \ + ../../ ``` This takes several minutes on a first build (compiling GDAL, installing @@ -116,22 +123,17 @@ Python packages). Subsequent builds are fast thanks to layer caching. > inside the build context, BuildKit reads files from the git object store > (committed versions) rather than the filesystem. If you forget to commit > a change, the old version is silently baked into the image. The same -> restriction applies — always commit changes to `madrona_portal/` or +> restriction applies — always commit changes to `madrona-portal/` or > `madrona-apps/` before rebuilding. -### Step 6 — Start the full stack +### Step 6 — Start the full stack; Populate testing DB -From `madrona_portal/`: +From `madrona-portal/docker`: ```bash -cd madrona_portal - -docker compose -f docker/docker-compose.yml --env-file .env --profile full up -d +DB_INIT=1 docker compose up ``` -The `--profile full` flag is required to start the `app` container. -Without it only `db` (PostGIS) and `tasks` (Redis) start. - On first boot the entrypoint automatically: 1. Waits for PostgreSQL to accept connections @@ -143,16 +145,22 @@ On first boot the entrypoint automatically: Open: http://localhost:8000/ (or whatever `APP_PORT` is set to in `.env`) +Once you have a populated DB (either dummy or with migrated data) omit the `DB_INIT=1`: +```bash +docker compose up +``` + + ### Step 7 - Importing a Production SQL Dump into the Dockerized Database #### Prerequisites -- Docker Compose stack is running — `docker compose -f docker/docker-compose.yml --env-file .env --profile full up -d` -- `madrona_portal/.env` exists with `DB_NAME`, `DB_USER`, and `DB_PASSWORD` set +- Docker Compose stack is running — `docker compose up` +- `madrona-portal/docker/.env` exists with `DB_NAME`, `DB_USER`, and `DB_PASSWORD` set #### Step 7.1 — Ensure you have the db-restore script -`madrona_portal/scripts/db-restore.sh` has the following behaviour: +`madrona-portal/scripts/db-restore.sh` has the following behaviour: - Loads DB credentials from `.env` - Verifies the `db` container is healthy before proceeding - With `--drop`: terminates active connections, drops and recreates the database, and enables the PostGIS extension @@ -160,20 +168,22 @@ Open: http://localhost:8000/ (or whatever `APP_PORT` is set to in `.env`) - Prints next-step instructions on completion Made sure it is executable: + +From `madrona-portal/docker`: ```bash -chmod +x madrona_portal/scripts/db-restore.sh +chmod +x ../scripts/db-restore.sh ``` #### Step 7.2 — Run the restore -From `madrona_portal/`: +From `madrona-portal/docker`: ```bash -./scripts/db-restore.sh --drop +../scripts/db-restore.sh --drop ``` *example:* ```bash -./scripts/db-restore.sh --drop ../madrona-apps/wcoa/wcodp_prod_dump_20260320.sql +../scripts/db-restore.sh --drop ../../madrona-apps/wcoa/wcodp_prod_dump_20260320.sql ``` The `--drop` flag was used to ensure a clean import. The script: @@ -190,7 +200,7 @@ The `--drop` flag was used to ensure a clean import. The script: #### Step 7.3 — Apply migrations ```bash -docker compose -f docker/docker-compose.yml --env-file .env --profile full exec app python marco/manage.py migrate +docker compose exec app python marco/manage.py migrate ``` *Please note:* on 4/10/2026 a 130+ migrations were applied to bring the schema from the prod dump (PostgreSQL 12) up to date with the current codebase (PostgreSQL 16). @@ -198,7 +208,7 @@ docker compose -f docker/docker-compose.yml --env-file .env --profile full exec #### Step 7.4 - Migration to mp-layers ```bash -docker compose -f docker/docker-compose.yml --env-file .env --profile full exec app python marco/manage.py migrate migration_to_layers +docker compose exec app python marco/manage.py migration_to_layers ``` ```bash @@ -239,41 +249,17 @@ python marco/manage.py compress ### Step 8 — Importing production media files into the Dockerized Application #### Prerequisites -- `madrona_portal/.env` exists with `MEDIA_ROOT` set to a valid directory +- `madrona-portal/marco/marco/config.wcoa.docker.ini` exists with `MEDIA_ROOT` set to a valid directory +- That valid directory should match the volume location is docker-compose.yml + - `portals/madrona-portal/media` - Production media files are available -#### Step 8.1a - Copy the media files into Docker - -Get your docker container name or ID for the `app` service: +#### Step 8.1 - Copy the media files into Docker +From `madrona-portal/docker`: ```bash -docker ps +cp -r {your_media_dir}/* ../media/ ``` -Then use `docker cp` to copy media files from the production location to the local directory specified by `MEDIA_ROOT` in `.env`. -```bash -docker cp /. :/vol/web/media/ -``` - -Example `docker cp` command: -```bash -docker cp madrona-apps/wcoa/media/. docker-app-1:/vol/web/media/ -``` - -#### Step 8.1b — Sync media files -Use `rsync` or a similar tool to copy media files from the production location to the local directory specified by `MEDIA_ROOT` in `.env`. -Using `rsync` command: -```bash -rsync -avz @:/path/to/remote/media/ / -``` - -Then use `docker cp` to copy media files from the local directory to the Docker container: -```bash -docker cp /. :/vol/web/media/ -``` - -Make sure to include the trailing slashes to sync the contents correctly. -`-avz` flags preserve permissions, show progress, and compress data during transfer. - #### Step 8.2 — Verify media access ```bash @@ -294,14 +280,14 @@ docker exec ls /vol/web/media/ docker buildx build \ --builder desktop-linux \ --load \ - -f madrona_portal/Dockerfile \ - -t madrona_portal-app:latest \ + -f madrona-portal/Dockerfile \ + -t madrona-portal-app:latest \ . ``` ### Redeploying after code changes -Then from `madrona_portal/`: +Then from `madrona-portal/`: ```bash docker compose -f docker/docker-compose.yml --env-file .env --profile full \ @@ -315,7 +301,7 @@ Add `--no-cache` to the buildx command to force a full dependency reinstall ## Everyday usage -All `docker compose` commands below are run from **`madrona_portal/`**. +All `docker compose` commands below are run from **`madrona-portal/`**. ### View logs @@ -359,7 +345,7 @@ To run Django locally against Docker-managed PostGIS and Redis (no app container # Start only db and tasks (omit --profile full) docker compose -f docker/docker-compose.yml --env-file .env up -d -# Then in a separate terminal, from madrona_portal/: +# Then in a separate terminal, from madrona-portal/: cd marco python manage.py runserver ``` @@ -377,12 +363,12 @@ From the **workspace root** (`portals/`): docker buildx build \ --builder desktop-linux \ --load \ - -f madrona_portal/Dockerfile \ - -t madrona_portal-app:latest \ + -f madrona-portal/Dockerfile \ + -t madrona-portal-app:latest \ . ``` -Then from `madrona_portal/`: +Then from `madrona-portal/`: ```bash docker compose -f docker/docker-compose.yml --env-file .env --profile full \ @@ -397,7 +383,7 @@ Add `--no-cache` to the buildx command to force a full dependency reinstall ## Reset to a clean state ```bash -# From madrona_portal/ +# From madrona-portal/ docker compose -f docker/docker-compose.yml --env-file .env --profile full down -v ``` @@ -410,7 +396,7 @@ migrations and reload fixtures from scratch. | Service | Image | Default host port | Override via | |---|---|---|---| -| `app` | `madrona_portal-app:latest` | `8000` | `APP_PORT` in `.env` | +| `app` | `madrona-portal-app:latest` | `8000` | `APP_PORT` in `.env` | | `db` | `postgis/postgis:16-3.4` | `5432` | `DB_PORT` in `.env` | | `tasks` | `redis:7-alpine` | `6379` | `REDIS_PORT` in `.env` | diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 574db3b..e8568ff 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -11,16 +11,44 @@ services: app: - image: madrona_portal-app:latest + # image: madrona-portal-app:latest build: context: ../../ - dockerfile: madrona_portal/Dockerfile + dockerfile: madrona-portal/docker/Dockerfile volumes: - static_data:/vol/web - - media_data:/vol/web + - ./media:/usr/local/apps/madrona-portal/media + # Mount the main Django project tree from the host. + # Changes to Python, templates, and config files are live immediately. + - ../marco:/usr/local/apps/madrona-portal/marco + + # Mount the WCOA app package from the host. + - ../../madrona-apps/wcoa:/usr/local/apps/madrona-portal/apps/wcoa + + # Mount other madrona-apps packages you are actively developing. + # Comment out any you are NOT changing — using the baked image copy + # for those packages is faster and avoids unnecessary inotify watches. + - ../../madrona-apps/mp-data-manager:/usr/local/apps/madrona-portal/apps/mp-data-manager + - ../../madrona-apps/mp-layers:/usr/local/apps/madrona-portal/apps/mp-layers + - ../../madrona-apps/mp-accounts:/usr/local/apps/madrona-portal/apps/mp-accounts + - ../../madrona-apps/mp-drawing:/usr/local/apps/madrona-portal/apps/mp-drawing + - ../../madrona-apps/mp-visualize:/usr/local/apps/madrona-portal/apps/mp-visualize + - ../../madrona-apps/madrona-features:/usr/local/apps/madrona-portal/apps/madrona-features + - ../../madrona-apps/madrona-manipulators:/usr/local/apps/madrona-portal/apps/madrona-manipulators + - ../../madrona-apps/madrona-scenarios:/usr/local/apps/madrona-portal/apps/madrona-scenarios + - ../../madrona-apps/mp-map-groups:/usr/local/apps/madrona-portal/apps/mp-map-groups + - ../../madrona-apps/mp-explore:/usr/local/apps/madrona-portal/apps/mp-explore + - ../../madrona-apps/mp-proxy:/usr/local/apps/madrona-portal/apps/mp-proxy + - ../../madrona-apps/p97-nursery:/usr/local/apps/madrona-portal/apps/p97-nursery + - ../../madrona-apps/django_url_shortener:/usr/local/apps/madrona-portal/apps/django_url_shortener + - ../../madrona-apps/madrona-analysistools:/usr/local/apps/madrona-portal/apps/madrona-analysistools env_file: - - ../.env # load all secrets from the project-root .env file + - ./.env # load all secrets from the project-root .env file environment: + # DB_INIT=1 runs migrations, fixtures, and superuser creation on startup. + # Defaults to 0 (skip) to protect existing databases. + - DB_INIT=${DB_INIT:-0} + # Config file selection — override specific values via env vars below. - MP_PROJECT_CONFIG=${MP_PROJECT_CONFIG:-config.wcoa.docker.ini} @@ -58,10 +86,9 @@ services: condition: service_healthy ports: - "${APP_PORT:-8000}:8000" + - "${APP_PORT:-8008}:8008" networks: - djangonetwork - profiles: - - full restart: unless-stopped db: @@ -104,7 +131,6 @@ services: volumes: static_data: - media_data: postgis_data: redis_data: diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 77921bc..a59c44e 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -1,6 +1,10 @@ #!/bin/sh # Madrona Portal — Docker entrypoint -# Waits for the database, runs migrations, seeds a fresh DB, then starts the server. +# Waits for the database, then starts the application server. +# +# By default only step 1 (DB wait) and step 5 (server start) run. +# Set DB_INIT=1 to also run steps 2-4 (migrate, seed fixtures, create superuser). +# This is intentionally opt-in to protect existing databases. set -e @@ -29,6 +33,13 @@ while True: print("Database is up.", flush=True) PY +# --------------------------------------------------------------------------- +# 2-4. Database initialisation (opt-in via DB_INIT=1) +# --------------------------------------------------------------------------- +if [ "${DB_INIT:-0}" != "1" ]; then + echo "DB_INIT not set — skipping migrations, fixtures, and superuser creation." +else + # --------------------------------------------------------------------------- # 2. Migrate and collect static files # --------------------------------------------------------------------------- @@ -166,6 +177,8 @@ else: PY fi +fi # end DB_INIT block + # --------------------------------------------------------------------------- # 5. Start the application server # @@ -185,16 +198,16 @@ print("true" if settings.DEBUG else "false") PY ) -if [ "${DJANGO_ENV:-}" = "production" ] || [ "${DJANGO_DEBUG}" = "false" ]; then - echo "Starting gunicorn (production mode)..." - exec gunicorn marco.wsgi:application \ - --bind 0.0.0.0:8000 \ - --workers "${GUNICORN_WORKERS:-3}" \ - --timeout "${GUNICORN_TIMEOUT:-120}" \ - --chdir marco \ - --access-logfile - \ - --error-logfile - -else - echo "Starting Django development server..." - exec python marco/manage.py runserver 0.0.0.0:8000 -fi +# if [ "${DJANGO_ENV:-}" = "production" ] || [ "${DJANGO_DEBUG}" = "false" ]; then +# echo "Starting gunicorn (production mode)..." +# exec gunicorn marco.wsgi:application \ +# --bind 0.0.0.0:8008 \ +# --workers "${GUNICORN_WORKERS:-3}" \ +# --timeout "${GUNICORN_TIMEOUT:-120}" \ +# --chdir marco \ +# --access-logfile - \ +# --error-logfile - +# else +echo "Starting Django development server..." +exec python marco/manage.py runserver 0.0.0.0:8000 +# fi diff --git a/marco/config.docker.ini.template b/marco/config.docker.ini.template index fdfd641..f23f516 100644 --- a/marco/config.docker.ini.template +++ b/marco/config.docker.ini.template @@ -11,8 +11,7 @@ DEBUG = True TEMPLATE_DEBUG = True ALLOWED_HOSTS = ["localhost", "127.0.0.1", "::1"] SECRET_KEY = -# TODO: Does the MEDIA_ROOT need to be updated here? potentially /usr/local/apps/madrona-portal/apps/wcoa/media/ -MEDIA_ROOT = /vol/web/media +MEDIA_ROOT = /usr/local/apps/mardona-portal/media MEDIA_URL = /media/ TIME_ZONE = UTC GA_ACCOUNT = From 89876e0f564ed88770994be9c81d3a28efa58dec Mon Sep 17 00:00:00 2001 From: Ryan Hodges Date: Thu, 16 Apr 2026 08:55:37 -0700 Subject: [PATCH 042/152] fixing configs for default development work --- docker/docker-compose.yml | 3 ++- marco/config.docker.ini.template | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index e8568ff..7c46ccb 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -49,6 +49,7 @@ services: # Defaults to 0 (skip) to protect existing databases. - DB_INIT=${DB_INIT:-0} + - DEBUG=True # Config file selection — override specific values via env vars below. - MP_PROJECT_CONFIG=${MP_PROJECT_CONFIG:-config.wcoa.docker.ini} @@ -86,7 +87,7 @@ services: condition: service_healthy ports: - "${APP_PORT:-8000}:8000" - - "${APP_PORT:-8008}:8008" + - "${GUNICORN_PORT:-8008}:8008" networks: - djangonetwork restart: unless-stopped diff --git a/marco/config.docker.ini.template b/marco/config.docker.ini.template index f23f516..61849e7 100644 --- a/marco/config.docker.ini.template +++ b/marco/config.docker.ini.template @@ -11,7 +11,7 @@ DEBUG = True TEMPLATE_DEBUG = True ALLOWED_HOSTS = ["localhost", "127.0.0.1", "::1"] SECRET_KEY = -MEDIA_ROOT = /usr/local/apps/mardona-portal/media +MEDIA_ROOT = /usr/local/apps/madrona-portal/media MEDIA_URL = /media/ TIME_ZONE = UTC GA_ACCOUNT = From 1f2942e4d3965cb1fc7794954c3afa396d597c46 Mon Sep 17 00:00:00 2001 From: Ryan Hodges Date: Thu, 16 Apr 2026 10:13:39 -0700 Subject: [PATCH 043/152] finishing streamline effort --- .gitignore | 1 + docker/README.md | 47 ++++++++-------------------------------- docker/media/__init__.py | 0 scripts/db-restore.sh | 40 ++++++++++++++++++++-------------- 4 files changed, 34 insertions(+), 54 deletions(-) create mode 100644 docker/media/__init__.py diff --git a/.gitignore b/.gitignore index 96ef017..6b5f549 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ vagrant .sass-cache node_modules +docker/media/ docker/entrypoint.sh docker/docker-requirements.txt diff --git a/docker/README.md b/docker/README.md index 3184c68..a7cc689 100644 --- a/docker/README.md +++ b/docker/README.md @@ -97,15 +97,15 @@ CELERY_BROKER_URL = redis://tasks:6379/0 Run this from the **workspace root** (`madrona-portal/`), not from inside `madrona-portal/`. The build context must include both repos. +If running this build from a MAC, add `--builder desktop-linux` to the `buildx build` command. + ```bash cd ../docker -docker compose build - docker buildx build --load -f ./Dockerfile ../../ ``` -When building a tagged image for deployment (use `builder desktop-linux` if on Mac): +When building a tagged image for deployment, add `-t madrona-portal-app:latest`: ``` docker buildx build \ --builder desktop-linux \ @@ -192,6 +192,8 @@ The `--drop` flag was used to ensure a clean import. The script: 3. Enabled the `postgis` extension 4. Streamed the sql dump into the container via `psql` +There is an optional `--env-file ` if you place your `.env` file in a non-standard location. + **Expected warnings (non-fatal):** - `ERROR: relation "..." does not exist` — pg_dump tries to drop constraints before creating them; safe to ignore on a fresh DB - `ERROR: role "wcoa_user" does not exist` — prod uses a dedicated app role; dev uses `postgres` which has full access @@ -203,47 +205,16 @@ The `--drop` flag was used to ensure a clean import. The script: docker compose exec app python marco/manage.py migrate ``` -*Please note:* on 4/10/2026 a 130+ migrations were applied to bring the schema from the prod dump (PostgreSQL 12) up to date with the current codebase (PostgreSQL 16). +*Please note:* on 4/10/2026 a 130+ migrations were applied to bring the schema from the prod dump up to date with the current codebase, largely driven by migrating from Wagtail v2 to v7, adding mp-layers, and adding the WCOA OHI indicators (for WCOA installs). #### Step 7.4 - Migration to mp-layers -```bash -docker compose exec app python marco/manage.py migration_to_layers -``` - -```bash -docker exec -it bash -``` - -Then inside the container: +*If migrating from a server that has not migrated to mp-layers from mp-data-manager*: ```bash -python marco/manage.py shell -``` - -```python -from layers.models import Theme -from data_manager.models import Theme as Dm_theme - - -for theme in Dm_theme.all_objects.all(): - try: - new_theme = Theme.all_objects.get(pk=theme.pk) - if new_theme.parent == None and new_theme.name != 'companion': - new_theme.is_top_theme = True - new_theme.save() - except Exception: - pass -``` - -exit the shell - -```bash -python marco/manage.py collectstatic -python marco/manage.py compress +docker compose exec app python marco/manage.py migration_to_layers ``` - --- ### Step 8 — Importing production media files into the Dockerized Application @@ -257,7 +228,7 @@ python marco/manage.py compress #### Step 8.1 - Copy the media files into Docker From `madrona-portal/docker`: ```bash -cp -r {your_media_dir}/* ../media/ +cp -r {your_media_dir}/* ./media/ ``` #### Step 8.2 — Verify media access diff --git a/docker/media/__init__.py b/docker/media/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/scripts/db-restore.sh b/scripts/db-restore.sh index f16bd25..7bf29be 100755 --- a/scripts/db-restore.sh +++ b/scripts/db-restore.sh @@ -4,19 +4,22 @@ # # Usage: # ./scripts/db-restore.sh -# ./scripts/db-restore.sh --drop # drop & recreate DB first +# ./scripts/db-restore.sh --drop # drop & recreate DB first +# ./scripts/db-restore.sh --env-file # # Run from anywhere — this script always operates relative to madrona_portal/. # # Prerequisites: -# 1. Docker Compose stack is running: -# docker compose -f docker/docker-compose.yml --env-file .env --profile full up -d -# 2. madrona_portal/.env exists and contains DB_NAME, DB_USER, DB_PASSWORD. +# 1. Docker Compose stack is running from madrona-portal/docker: +# docker compose up +# 2. madrona_portal/docker/.env exists and contains DB_NAME, DB_USER, DB_PASSWORD. # # Options: -# --drop Terminate all active connections, drop, and recreate the target -# database before restoring. Required for a clean import from prod. -# Without this flag the dump is applied on top of existing data. +# --drop Terminate all active connections, drop, and recreate the +# target database before restoring. Required for a clean +# import from prod. Without this flag the dump is applied +# on top of existing data. +# --env-file Path to the .env file (default: ./docker/.env). # # Notes: # - The dump is streamed directly into the container — no temp files on disk. @@ -38,22 +41,29 @@ info() { echo "[db-restore] $*"; } # --------------------------------------------------------------------------- DROP_FIRST=false DUMP_FILE="" +ENV_FILE=../docker/.env while [[ $# -gt 0 ]]; do case "$1" in - --drop) DROP_FIRST=true; shift ;; - -*) die "Unknown option: '$1'. Usage: $0 [--drop] " ;; - *) [[ -z "$DUMP_FILE" ]] || die "Unexpected argument: '$1'" - DUMP_FILE="$1"; shift ;; + --drop) DROP_FIRST=true; shift ;; + --env-file) [[ -n "${2:-}" ]] || die "--env-file requires a path argument" + ENV_FILE="$2"; shift 2 ;; + -*) die "Unknown option: '$1'. Usage: $0 [--drop] [--env-file ] " ;; + *) [[ -z "$DUMP_FILE" ]] || die "Unexpected argument: '$1'" + DUMP_FILE="$1"; shift ;; esac done -[[ -n "$DUMP_FILE" ]] || die "Usage: $0 [--drop] " +[[ -n "$DUMP_FILE" ]] || die "Usage: $0 [--drop] [--env-file ] " # Resolve dump path before we cd away. DUMP_ABS="$(cd "$(dirname "$DUMP_FILE")" && pwd)/$(basename "$DUMP_FILE")" [[ -f "$DUMP_ABS" ]] || die "Dump file not found: $DUMP_FILE" +# Resolve ENV_FILE to an absolute path before we cd away. +ENV_FILE_ABS="$(cd "$(dirname "$ENV_FILE")" && pwd)/$(basename "$ENV_FILE")" +[[ -f "$ENV_FILE_ABS" ]] || die "Env file not found: $ENV_FILE" + # --------------------------------------------------------------------------- # Always operate from madrona_portal/ regardless of where the script is called # --------------------------------------------------------------------------- @@ -63,18 +73,16 @@ cd "$SCRIPT_DIR/.." # --------------------------------------------------------------------------- # Load .env for DB credentials # --------------------------------------------------------------------------- -[[ -f .env ]] || die ".env not found in $(pwd). Copy .env.example and fill in values." - set -a # shellcheck source=/dev/null -source .env +source "$ENV_FILE_ABS" set +a DB_NAME="${DB_NAME:-wcoa_docker_db}" DB_USER="${DB_USER:-postgres}" DB_PASSWORD="${DB_PASSWORD:?DB_PASSWORD must be set in .env}" -COMPOSE="docker compose -f docker/docker-compose.yml --env-file .env" +COMPOSE="docker compose -f docker/docker-compose.yml --env-file $ENV_FILE_ABS" PSQL="$COMPOSE exec -T -e PGPASSWORD=$DB_PASSWORD db psql -U $DB_USER" # --------------------------------------------------------------------------- From c7577db22d8e86e0fea2e97bc0d83b7ebe82b9d3 Mon Sep 17 00:00:00 2001 From: Ryan Hodges Date: Thu, 16 Apr 2026 10:56:44 -0700 Subject: [PATCH 044/152] Apply suggestion from @Copilot Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- docker/AWS_DEPLOY.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/AWS_DEPLOY.md b/docker/AWS_DEPLOY.md index 91f9806..67a035e 100644 --- a/docker/AWS_DEPLOY.md +++ b/docker/AWS_DEPLOY.md @@ -307,7 +307,7 @@ cd ~/portals docker buildx build \ --load \ - -f madrona-portal/Dockerfile \ + -f madrona-portal/docker/Dockerfile \ -t madrona-portal-app:latest \ . ``` From e28979d10f337a4d722fc820bc293d83cc4dd156 Mon Sep 17 00:00:00 2001 From: Ryan Hodges Date: Thu, 16 Apr 2026 10:57:40 -0700 Subject: [PATCH 045/152] Apply suggestion from @Copilot minor comment clarification Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- docker/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 7c46ccb..18bcbc1 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -43,7 +43,7 @@ services: - ../../madrona-apps/django_url_shortener:/usr/local/apps/madrona-portal/apps/django_url_shortener - ../../madrona-apps/madrona-analysistools:/usr/local/apps/madrona-portal/apps/madrona-analysistools env_file: - - ./.env # load all secrets from the project-root .env file + - ./.env # load all secrets from the docker/.env file environment: # DB_INIT=1 runs migrations, fixtures, and superuser creation on startup. # Defaults to 0 (skip) to protect existing databases. From 6e5435c0dc3228ef244bed390c27cb917ae597b9 Mon Sep 17 00:00:00 2001 From: Ryan Hodges Date: Thu, 16 Apr 2026 10:59:13 -0700 Subject: [PATCH 046/152] Apply suggestion from @Copilot Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- docker/docker-compose.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 18bcbc1..2fd059e 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -49,7 +49,6 @@ services: # Defaults to 0 (skip) to protect existing databases. - DB_INIT=${DB_INIT:-0} - - DEBUG=True # Config file selection — override specific values via env vars below. - MP_PROJECT_CONFIG=${MP_PROJECT_CONFIG:-config.wcoa.docker.ini} From 0c223573d018236c7585abde8acab3d3aed763bd Mon Sep 17 00:00:00 2001 From: Ryan Hodges Date: Thu, 16 Apr 2026 11:00:55 -0700 Subject: [PATCH 047/152] Apply suggestion from @Copilot Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- docker/docker-compose.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 2fd059e..a32fd85 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -86,7 +86,6 @@ services: condition: service_healthy ports: - "${APP_PORT:-8000}:8000" - - "${GUNICORN_PORT:-8008}:8008" networks: - djangonetwork restart: unless-stopped From d9177a68848412ff9f62e1fbc8b9a81877cec007 Mon Sep 17 00:00:00 2001 From: Ryan Hodges Date: Thu, 16 Apr 2026 11:05:34 -0700 Subject: [PATCH 048/152] Apply suggestion from @Copilot documentation clarification. A bit verbose. Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- docker/README.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/docker/README.md b/docker/README.md index a7cc689..d58a3c2 100644 --- a/docker/README.md +++ b/docker/README.md @@ -94,8 +94,11 @@ CELERY_BROKER_URL = redis://tasks:6379/0 ### Step 5 — Build the image -Run this from the **workspace root** (`madrona-portal/`), not from -inside `madrona-portal/`. The build context must include both repos. +Run this command from `madrona-portal/docker` (the previous step leaves +you in `madrona-portal/marco`, so `cd ../docker` gets you there). The +Docker build context for this command is `../../`, which resolves to the +parent workspace directory `portals/` that contains both +`madrona-portal/` and `madrona-apps/`. If running this build from a MAC, add `--builder desktop-linux` to the `buildx build` command. From 037cec193f8c5a1cb8215b19df3daef1127f74e6 Mon Sep 17 00:00:00 2001 From: Ryan Hodges Date: Thu, 16 Apr 2026 11:08:37 -0700 Subject: [PATCH 049/152] removing REDME.md reference to /vol/web/media, which is now kept external to the container --- docker/README.md | 7 ------- 1 file changed, 7 deletions(-) diff --git a/docker/README.md b/docker/README.md index d58a3c2..a5d4cd3 100644 --- a/docker/README.md +++ b/docker/README.md @@ -234,13 +234,6 @@ From `madrona-portal/docker`: cp -r {your_media_dir}/* ./media/ ``` -#### Step 8.2 — Verify media access - -```bash -docker exec du -sh /vol/web/media/ -docker exec ls /vol/web/media/ -``` - --- # Untested instructions below this line — will update after testing From fdc074dec7643170f5f843a1ffc529fc4f94d8bb Mon Sep 17 00:00:00 2001 From: Ryan Hodges Date: Thu, 16 Apr 2026 11:15:09 -0700 Subject: [PATCH 050/152] purge remaining references to 'madrona_portal' --- backups/dump_fixtures.sh | 4 ++-- backups/load_sql_dump.sh | 2 +- docker/docker-compose.dev.yml | 2 +- marco/marco/apps.py | 2 +- marco/marco/settings.py | 2 +- scripts/db-restore.sh | 6 +++--- 6 files changed, 9 insertions(+), 9 deletions(-) diff --git a/backups/dump_fixtures.sh b/backups/dump_fixtures.sh index 62363cf..8b72745 100644 --- a/backups/dump_fixtures.sh +++ b/backups/dump_fixtures.sh @@ -1,7 +1,7 @@ #!/bin/bash # Regenerate WCOA fixture files from the current running database. # -# Usage (run from madrona_portal/): +# Usage (run from madrona-portal/): # ./backups/dump_fixtures.sh # # Requires the full stack to be running: @@ -14,7 +14,7 @@ set -euo pipefail DC="docker compose --env-file docker/.env.dev -f docker/docker-compose.yml" -WCOA_FX="/usr/local/apps/madrona_portal/apps/wcoa/wcoa/fixtures" +WCOA_FX="/usr/local/apps/madrona-portal/apps/wcoa/wcoa/fixtures" echo "Exporting wcoa_init.json (base, wagtailcore, wagtailimages, wcoa) ..." $DC run --rm app \ diff --git a/backups/load_sql_dump.sh b/backups/load_sql_dump.sh index c46e621..2aed09f 100755 --- a/backups/load_sql_dump.sh +++ b/backups/load_sql_dump.sh @@ -1,7 +1,7 @@ #!/bin/bash # Import a PostgreSQL SQL dump into the running Docker database service. # -# Usage (run from madrona_portal/): +# Usage (run from madrona-portal/): # ./backups/load_sql_dump.sh /path/to/your_dump.sql # # The db container must already be running: diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml index 1211a36..1abca59 100644 --- a/docker/docker-compose.dev.yml +++ b/docker/docker-compose.dev.yml @@ -4,7 +4,7 @@ # Python's editable installs follow .pth files to the mounted directories, so # any file saved on the host is immediately visible inside the container. # -# Usage (from madrona_portal/): +# Usage (from madrona-portal/): # docker compose \ # -f docker/docker-compose.yml \ # -f docker/docker-compose.dev.yml \ diff --git a/marco/marco/apps.py b/marco/marco/apps.py index 499791e..fba0b1c 100644 --- a/marco/marco/apps.py +++ b/marco/marco/apps.py @@ -2,5 +2,5 @@ class MadronaPortalConfig(AppConfig): - # TODO: Rename this module to 'madrona' or 'madrona_portal' + # TODO: Rename this module to 'madrona' or 'madrona-portal' name = 'marco' diff --git a/marco/marco/settings.py b/marco/marco/settings.py index a39ffc2..8a87dfa 100644 --- a/marco/marco/settings.py +++ b/marco/marco/settings.py @@ -303,7 +303,7 @@ def _parse_hosts(raw: str | None) -> list[str]: ) DATABASES = {'default': default_db} -DB_CHANNEL = db_cfg.get('DB_CHANNEL', 'madrona_portal') +DB_CHANNEL = db_cfg.get('DB_CHANNEL', 'madrona-portal') DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' diff --git a/scripts/db-restore.sh b/scripts/db-restore.sh index 7bf29be..5e7449e 100755 --- a/scripts/db-restore.sh +++ b/scripts/db-restore.sh @@ -7,12 +7,12 @@ # ./scripts/db-restore.sh --drop # drop & recreate DB first # ./scripts/db-restore.sh --env-file # -# Run from anywhere — this script always operates relative to madrona_portal/. +# Run from anywhere — this script always operates relative to madrona-portal/. # # Prerequisites: # 1. Docker Compose stack is running from madrona-portal/docker: # docker compose up -# 2. madrona_portal/docker/.env exists and contains DB_NAME, DB_USER, DB_PASSWORD. +# 2. madrona-portal/docker/.env exists and contains DB_NAME, DB_USER, DB_PASSWORD. # # Options: # --drop Terminate all active connections, drop, and recreate the @@ -65,7 +65,7 @@ ENV_FILE_ABS="$(cd "$(dirname "$ENV_FILE")" && pwd)/$(basename "$ENV_FILE")" [[ -f "$ENV_FILE_ABS" ]] || die "Env file not found: $ENV_FILE" # --------------------------------------------------------------------------- -# Always operate from madrona_portal/ regardless of where the script is called +# Always operate from madrona-portal/ regardless of where the script is called # --------------------------------------------------------------------------- SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" cd "$SCRIPT_DIR/.." From a8e489d24a9f06fce914eed5f51154e55b3748c6 Mon Sep 17 00:00:00 2001 From: Ryan Hodges Date: Thu, 16 Apr 2026 11:23:24 -0700 Subject: [PATCH 051/152] smarter default relative addressing of .env file in db-restore.sh script --- scripts/db-restore.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/db-restore.sh b/scripts/db-restore.sh index 5e7449e..8505a91 100755 --- a/scripts/db-restore.sh +++ b/scripts/db-restore.sh @@ -41,7 +41,7 @@ info() { echo "[db-restore] $*"; } # --------------------------------------------------------------------------- DROP_FIRST=false DUMP_FILE="" -ENV_FILE=../docker/.env +ENV_FILE="$(dirname "${BASH_SOURCE[0]}")/../docker/.env" while [[ $# -gt 0 ]]; do case "$1" in From 12dbf463264ac7413320cfb3298fb46d2185ee72 Mon Sep 17 00:00:00 2001 From: Ryan Hodges Date: Thu, 16 Apr 2026 11:29:04 -0700 Subject: [PATCH 052/152] handle leading ~ in env-file for db-restore.sh --- scripts/db-restore.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/db-restore.sh b/scripts/db-restore.sh index 8505a91..dbd48b4 100755 --- a/scripts/db-restore.sh +++ b/scripts/db-restore.sh @@ -60,6 +60,10 @@ done DUMP_ABS="$(cd "$(dirname "$DUMP_FILE")" && pwd)/$(basename "$DUMP_FILE")" [[ -f "$DUMP_ABS" ]] || die "Dump file not found: $DUMP_FILE" +# Expand a leading ~ in ENV_FILE before resolving it to an absolute path. +if [[ "$ENV_FILE" == ~* ]]; then + ENV_FILE="${ENV_FILE/#\~/$HOME}" +fi # Resolve ENV_FILE to an absolute path before we cd away. ENV_FILE_ABS="$(cd "$(dirname "$ENV_FILE")" && pwd)/$(basename "$ENV_FILE")" [[ -f "$ENV_FILE_ABS" ]] || die "Env file not found: $ENV_FILE" From 06f658609ed654d1bfa3005ad2e9cbe8da54b70d Mon Sep 17 00:00:00 2001 From: Ryan Hodges Date: Thu, 16 Apr 2026 15:54:41 -0700 Subject: [PATCH 053/152] integrating default Geoportal deployment with portal --- .gitignore | 1 + docker/.env.example | 65 +++++++ docker/AWS_DEPLOY.md | 2 +- docker/docker-compose.yml | 56 +++++- docker/geoportal-entrypoint.sh | 194 ++++++++++++++++++++ docker/templates/authentication-simple.xml | 21 +++ docker/templates/catalog-app-security.xml | 80 ++++++++ docker/templates/harvester-app-security.xml | 42 +++++ docker/wars/__init__.py | 0 9 files changed, 456 insertions(+), 5 deletions(-) create mode 100755 docker/geoportal-entrypoint.sh create mode 100644 docker/templates/authentication-simple.xml create mode 100644 docker/templates/catalog-app-security.xml create mode 100644 docker/templates/harvester-app-security.xml create mode 100644 docker/wars/__init__.py diff --git a/.gitignore b/.gitignore index 6b5f549..630e633 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,7 @@ node_modules docker/media/ docker/entrypoint.sh docker/docker-requirements.txt +docker/wars/ marco_site/static/bundles/ marco_site/static/css/ diff --git a/docker/.env.example b/docker/.env.example index f1d1f26..88609e9 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -73,3 +73,68 @@ GOOGLE_SECRET= DJANGO_SUPERUSER_USERNAME=admin DJANGO_SUPERUSER_EMAIL=admin@example.com DJANGO_SUPERUSER_PASSWORD= + +# ================================================= +# Catalog settings: Elasticsearch and Geoportal +# ================================================= + +# Project namespace (defaults to the current folder name if not set) +#COMPOSE_PROJECT_NAME=myproject + +# Password for the 'elastic' user (at least 6 characters) +ELASTIC_PASSWORD=changeme + +# Password for the 'kibana_system' user (at least 6 characters) +KIBANA_PASSWORD=changeme + +# Version of Elastic products +STACK_VERSION=8.8.2 + +# Set the cluster name +CLUSTER_NAME=elasticsearch + +# Port to expose Elasticsearch HTTP API to the host +ES_PORT=9200 +ES_REINDEX_REMOTE_WHITELIST=[] + +# Port to expose Kibana to the host +KIBANA_PORT=5601 + +# Increase or decrease based on the available host memory (in bytes) +ES_MEM_LIMIT=1073741824 +KB_MEM_LIMIT=1073741824 +LS_MEM_LIMIT=1073741824 + +# SAMPLE Predefined Key only to be used in POC environments +ENCRYPTION_KEY=abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz01 + +# ================================================= + +# Geoportal Authentication Configuration +# Override these values in your Docker deployment + +# Admin User (Full Access) +gpt_admin_username=admin +gpt_admin_password=admin + +# Publisher User (Can publish metadata) +gpt_publisher_username=publisher +gpt_publisher_password=publisher + +# Regular User (Read-only access) +gpt_user_username=user +gpt_user_password=user + +gpt_wcoa_username=wcoa +gpt_wcoa_password=changeme + +gpt_esri_username=esri +gpt_esri_password=changeme + +ES_NODE=elastic + +gpt_catalog_war=./wars/geoportal.war +gpt_harvester_war=./wars/harvester.war + +gpt_frame_options=DENY +gpt_allowed_origin="localhost localhost:*" \ No newline at end of file diff --git a/docker/AWS_DEPLOY.md b/docker/AWS_DEPLOY.md index 67a035e..acade8a 100644 --- a/docker/AWS_DEPLOY.md +++ b/docker/AWS_DEPLOY.md @@ -586,4 +586,4 @@ docker compose -f docker/docker-compose.yml --env-file .env --profile full down > After go-live, update your security group inbound rules to remove public > access to ports 5432, 6379, 9200, and 9300. These are only needed -> internally between containers on `djangonetwork`. +> internally between containers on `madronanetwork`. diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index a32fd85..98c3256 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -87,7 +87,7 @@ services: ports: - "${APP_PORT:-8000}:8000" networks: - - djangonetwork + - madronanetwork restart: unless-stopped db: @@ -101,7 +101,7 @@ services: ports: - "${DB_PORT:-5432}:5432" networks: - - djangonetwork + - madronanetwork healthcheck: test: ["CMD-SHELL", "pg_isready -U ${DB_USER:-postgres} -d ${DB_NAME:-wcoa_docker_db}"] interval: 10s @@ -119,7 +119,7 @@ services: volumes: - redis_data:/data networks: - - djangonetwork + - madronanetwork healthcheck: # -a flag is only passed when a password is configured. test: ["CMD-SHELL", "redis-cli ${REDIS_PASSWORD:+-a ${REDIS_PASSWORD}} ping"] @@ -128,11 +128,59 @@ services: retries: 5 restart: unless-stopped + geoportal: + image: tomcat:9-jdk11 + ports: + - 8080:8080 + volumes: + - gp-volume:/usr/local/tomcat/webapps/ + - ${gpt_catalog_war}:/usr/local/tomcat/webapps/geoportal.war + - ${gpt_harvester_war}:/usr/local/tomcat/webapps/harvester.war + # Configuration override setup + - ./templates:/templates:ro + - ./geoportal-entrypoint.sh:/usr/local/bin/entrypoint.sh:ro + entrypoint: ["/usr/local/bin/entrypoint.sh"] + networks: + - madronanetwork + restart: always + env_file: + - ./.env + depends_on: + elastic: + condition: service_healthy + + elastic: + image: elasticsearch:8.19.12 + volumes: + - es-volume:/usr/share/elasticsearch/data + environment: + - discovery.type=single-node + - ES_JAVA_OPTS=-Xms512m -Xmx512m + - cluster.name=${CLUSTER_NAME} + - ELASTIC_PASSWORD=${ELASTIC_PASSWORD} + - bootstrap.memory_lock=true + - reindex.remote.whitelist=${ES_REINDEX_REMOTE_WHITELIST} + - xpack.security.enabled=false + + ports: + - 9200:9200 + - 9300:9300 + networks: + - madronanetwork + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9200/_cluster/health?wait_for_status=yellow"] + interval: 30s + timeout: 10s + retries: 10 + restart: always + volumes: static_data: postgis_data: redis_data: + gp-volume: + es-volume: networks: - djangonetwork: + madronanetwork: driver: bridge diff --git a/docker/geoportal-entrypoint.sh b/docker/geoportal-entrypoint.sh new file mode 100755 index 0000000..cbdc495 --- /dev/null +++ b/docker/geoportal-entrypoint.sh @@ -0,0 +1,194 @@ +#!/bin/bash +################################### +# This file 100% written by Copilot +################################### +set -e + +echo "Starting Geoportal with configuration override..." + +# Install gettext for envsubst command +echo "Installing gettext package for environment variable substitution..." +apt-get update -qq && apt-get install -y gettext-base && apt-get clean && rm -rf /var/lib/apt/lists/* + +# Cleanup function for graceful shutdown +cleanup() { + if [ ! -z "$TOMCAT_PID" ] && kill -0 $TOMCAT_PID 2>/dev/null; then + echo "Cleaning up Tomcat process (PID: $TOMCAT_PID)..." + kill $TOMCAT_PID 2>/dev/null + sleep 2 + if kill -0 $TOMCAT_PID 2>/dev/null; then + kill -9 $TOMCAT_PID 2>/dev/null + fi + fi +} +trap cleanup EXIT INT TERM + +# Function to wait for WAR deployment +wait_for_deployment() { + local app_name=$1 + local max_wait=120 + local wait_time=0 + + echo "Waiting for $app_name to deploy..." + while [ ! -d "/usr/local/tomcat/webapps/$app_name" ] && [ $wait_time -lt $max_wait ]; do + sleep 2 + wait_time=$((wait_time + 2)) + echo "Waiting... ${wait_time}s" + done + + if [ $wait_time -ge $max_wait ]; then + echo "ERROR: $app_name failed to deploy within ${max_wait} seconds" + return 1 + fi + + echo "$app_name deployed successfully" + return 0 +} + +# Function to substitute environment variables in templates +substitute_variables() { + local template_file=$1 + local output_file=$2 + + echo "Processing template: $template_file -> $output_file" + + # Validate required environment variables + local missing_vars=() + if [ -z "$gpt_frame_options" ]; then + missing_vars+=("gpt_frame_options") + fi + if [ -z "$gpt_allowed_origin" ]; then + missing_vars+=("gpt_allowed_origin") + fi + + if [ ${#missing_vars[@]} -gt 0 ]; then + echo "WARNING: Missing required environment variables: ${missing_vars[*]}" + echo "Check your .env file and ensure these variables are set" + fi + + # Display current values for debugging + echo " gpt_frame_options = '$gpt_frame_options'" + echo " gpt_allowed_origin = '$gpt_allowed_origin'" + + # Validate CSP format (check for problematic characters) + if echo "$gpt_allowed_origin" | grep -q ":.*\*"; then + echo " WARNING: Port wildcards (*) in CSP frame-ancestors may not be supported by all browsers" + echo " Consider using specific ports or removing wildcards if you encounter issues" + fi + + # Use envsubst to replace environment variables + envsubst < "$template_file" > "$output_file" + + if [ $? -eq 0 ]; then + echo "Successfully processed $template_file" + + # Show a sample of the processed content for verification + echo "Sample of processed content:" + grep -E "(frame-options|Content-Security-Policy)" "$output_file" | head -2 | sed 's/^/ /' + else + echo "ERROR: Failed to process $template_file" + return 1 + fi +} + +# Start Tomcat in background to deploy WARs +echo "Starting Tomcat to deploy applications..." +catalina.sh run & +TOMCAT_PID=$! +echo "Tomcat started with PID: $TOMCAT_PID" + +# Wait for both applications to deploy +wait_for_deployment "geoportal" || exit 1 +wait_for_deployment "harvester" || exit 1 + +# Additional wait to ensure full extraction +echo "Waiting for full application extraction..." +sleep 10 + +# Create config directory if it doesn't exist +CATALOG_CONFIG_DIR="/usr/local/tomcat/webapps/geoportal/WEB-INF/classes/config" +mkdir -p "$CATALOG_CONFIG_DIR" +HARVESTER_CONFIG_DIR="/usr/local/tomcat/webapps/harvester/WEB-INF/classes/config" +mkdir -p "$HARVESTER_CONFIG_DIR" + +# Process and copy authentication configuration +if [ -f "/templates/authentication-simple.xml" ]; then + substitute_variables "/templates/authentication-simple.xml" "$CATALOG_CONFIG_DIR/authentication-simple.xml" + substitute_variables "/templates/authentication-simple.xml" "$HARVESTER_CONFIG_DIR/authentication-simple.xml" +else + echo "WARNING: authentication-simple.xml template not found" +fi + +# Process and copy security configuration +if [ -f "/templates/catalog-app-security.xml" ] && [ -f "/templates/harvester-app-security.xml" ]; then + substitute_variables "/templates/catalog-app-security.xml" "$CATALOG_CONFIG_DIR/app-security.xml" + substitute_variables "/templates/harvester-app-security.xml" "$HARVESTER_CONFIG_DIR/app-security.xml" +else + echo "WARNING: app-security.xml templates not found" + if [ ! -f "/templates/catalog-app-security.xml" ]; then + echo " Missing: /templates/catalog-app-security.xml" + fi + if [ ! -f "/templates/harvester-app-security.xml" ]; then + echo " Missing: /templates/harvester-app-security.xml" + fi +fi + +# Verify the configuration files were created +echo "Verifying configuration files..." +if [ -f "$CATALOG_CONFIG_DIR/authentication-simple.xml" ]; then + echo "✓ authentication-simple.xml configured" +else + echo "✗ authentication-simple.xml missing" +fi + +if [ -f "$CATALOG_CONFIG_DIR/app-security.xml" ]; then + echo "✓ CATALOG app-security.xml configured" +else + echo "✗ CATALOG app-security.xml missing" +fi + +if [ -f "$HARVESTER_CONFIG_DIR/authentication-simple.xml" ]; then + echo "✓ HARVESTER authentication-simple.xml configured" +else + echo "✗ HARVESTER authentication-simple.xml missing" +fi + +if [ -f "$HARVESTER_CONFIG_DIR/app-security.xml" ]; then + echo "✓ app-security.xml configured" +else + echo "✗ app-security.xml missing" +fi + +# Stop background Tomcat +echo "Stopping background Tomcat (PID: $TOMCAT_PID)..." + +# Check if the process is still running +if kill -0 $TOMCAT_PID 2>/dev/null; then + echo "Sending TERM signal to Tomcat..." + kill $TOMCAT_PID + + # Wait for graceful shutdown (up to 10 seconds) + for i in {1..10}; do + if ! kill -0 $TOMCAT_PID 2>/dev/null; then + echo "Tomcat stopped gracefully" + break + fi + echo "Waiting for shutdown... ${i}/10" + sleep 1 + done + + # Force kill if still running + if kill -0 $TOMCAT_PID 2>/dev/null; then + echo "Force stopping Tomcat..." + kill -9 $TOMCAT_PID + sleep 2 + fi +else + echo "Tomcat process was not running (PID $TOMCAT_PID)" +fi + +echo "Tomcat stopped successfully" + +# Start Tomcat in foreground +echo "Starting Tomcat with updated configuration..." +exec catalina.sh run \ No newline at end of file diff --git a/docker/templates/authentication-simple.xml b/docker/templates/authentication-simple.xml new file mode 100644 index 0000000..eada6db --- /dev/null +++ b/docker/templates/authentication-simple.xml @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/docker/templates/catalog-app-security.xml b/docker/templates/catalog-app-security.xml new file mode 100644 index 0000000..4b09d8c --- /dev/null +++ b/docker/templates/catalog-app-security.xml @@ -0,0 +1,80 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docker/templates/harvester-app-security.xml b/docker/templates/harvester-app-security.xml new file mode 100644 index 0000000..b93047f --- /dev/null +++ b/docker/templates/harvester-app-security.xml @@ -0,0 +1,42 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/docker/wars/__init__.py b/docker/wars/__init__.py new file mode 100644 index 0000000..e69de29 From 9270041f2de4675699ef3f2326ba9587447ab15d Mon Sep 17 00:00:00 2001 From: Ryan Hodges Date: Thu, 16 Apr 2026 15:55:18 -0700 Subject: [PATCH 054/152] more madrona-portal vs madrona_portal cleanup --- Vagrantfile | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index f124c84..cc2a2f0 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -58,12 +58,12 @@ Vagrant.configure("2") do |config| # Automatically detect the SMB host IP smb_host_ip = get_host_ip - config.vm.synced_folder "./", "/usr/local/apps/madrona_portal", + config.vm.synced_folder "./", "/usr/local/apps/madrona-portal", type: "smb", smb_host: smb_host_ip, mount_options: ["sec=ntlmssp", "nounix", "noperm", "vers=3.0"] - config.vm.synced_folder "../madrona-apps", "/usr/local/apps/madrona_portal/apps", + config.vm.synced_folder "../madrona-apps", "/usr/local/apps/madrona-portal/apps", type: "smb", smb_host: smb_host_ip, mount_options: ["sec=ntlmssp", "nounix", "noperm", "vers=3.0"] @@ -109,8 +109,8 @@ Vagrant.configure("2") do |config| # an identifier, the second is the path on the guest to mount the # folder, and the third is the path on the host to the actual folder. # config.vm.share_folder "project", "/home/vagrant/marco_portal2", "." - config.vm.synced_folder "./", "/usr/local/apps/madrona_portal" - config.vm.synced_folder "../madrona-apps/", "/usr/local/apps/madrona_portal/apps" + config.vm.synced_folder "./", "/usr/local/apps/madrona-portal" + config.vm.synced_folder "../madrona-apps/", "/usr/local/apps/madrona-portal/apps" # Enable provisioning with a shell script. # config.vm.provision :shell, :path => "scripts/vagrant_provision.sh", :args => "'marco_portal2' 'marco' 'marco_portal'", :privileged => false @@ -131,12 +131,12 @@ Vagrant.configure("2") do |config| # Default synced folder setup for Windows smb_host_ip = "192.168.1.1" # Fallback IP for Windows - config.vm.synced_folder "./", "/usr/local/apps/madrona_portal", + config.vm.synced_folder "./", "/usr/local/apps/madrona-portal", type: "smb", smb_host: smb_host_ip, mount_options: ["sec=ntlmssp", "nounix", "noperm", "vers=3.0"] - config.vm.synced_folder "../madrona-apps", "/usr/local/apps/madrona_portal/apps", + config.vm.synced_folder "../madrona-apps", "/usr/local/apps/madrona-portal/apps", type: "smb", smb_host: smb_host_ip, mount_options: ["sec=ntlmssp", "nounix", "noperm", "vers=3.0"] From 1ea2747230fc4742a6f9d42281848afcd8043695 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Fri, 17 Apr 2026 15:05:44 -0700 Subject: [PATCH 055/152] Add API URL auto-discovery and tests for mounted routes --- marco/marco/tests/__init__.py | 0 marco/marco/tests/test_api_url_discovery.py | 18 ++++++++++++++++++ 2 files changed, 18 insertions(+) create mode 100644 marco/marco/tests/__init__.py create mode 100644 marco/marco/tests/test_api_url_discovery.py diff --git a/marco/marco/tests/__init__.py b/marco/marco/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/marco/marco/tests/test_api_url_discovery.py b/marco/marco/tests/test_api_url_discovery.py new file mode 100644 index 0000000..0690d83 --- /dev/null +++ b/marco/marco/tests/test_api_url_discovery.py @@ -0,0 +1,18 @@ +from django.test import SimpleTestCase +from django.urls import resolve + + +class ApiUrlDiscoveryTests(SimpleTestCase): + """Ensure API routes from installed apps are auto-mounted under /api/.""" + + def test_visualize_api_route_resolves(self): + match = resolve('/api/bookmarks/') + self.assertEqual(match.func.view_class.__name__, 'BookmarkListView') + + def test_drawing_api_route_resolves(self): + match = resolve('/api/drawings/testuid123/') + self.assertEqual(match.func.view_class.__name__, 'DrawingDeleteView') + + def test_mapgroups_api_route_resolves(self): + match = resolve('/api/sharing-groups/') + self.assertEqual(match.func.view_class.__name__, 'SharingGroupListView') From a11c20080fd0137cfc16bf15a5c4ab40b670d5fe Mon Sep 17 00:00:00 2001 From: David Pollard Date: Fri, 17 Apr 2026 15:05:54 -0700 Subject: [PATCH 056/152] Add API URL auto-discovery and new conventions for sub-apps --- MODERNIZATION.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/MODERNIZATION.md b/MODERNIZATION.md index 4ff6b46..c0cca3b 100644 --- a/MODERNIZATION.md +++ b/MODERNIZATION.md @@ -39,6 +39,8 @@ These changes have been applied to the codebase. - Removed trailing `/?` optional slashes on most routes (ambiguous in Django URL routing). - Replaced `re_path(r'^django-admin/?', ...)` with `re_path(r'^django-admin/', ...)` — `admin.site.urls` already handles trailing slash. - Added `warnings.warn` instead of silent `except Exception: pass` when `PROJECT_APP` URL import fails. +- Added API URL auto-discovery: for each entry in `INSTALLED_APPS`, if `.urls` exists and defines `api_urlpatterns`, those routes are automatically mounted under `/api/`. +- New convention for sub-apps: define REST endpoints in `/api.py`, expose them from `/urls.py` as `api_urlpatterns`, and avoid hard-coding app-specific API imports in the project URLConf. ### Migrations From c5b312496acc2c9b50abd60aa9aeff18c2c8fa03 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Fri, 17 Apr 2026 15:06:03 -0700 Subject: [PATCH 057/152] Refactor URL configuration to dynamically discover API patterns from installed apps --- marco/marco/urls.py | 45 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 36 insertions(+), 9 deletions(-) diff --git a/marco/marco/urls.py b/marco/marco/urls.py index 25710a2..13aae6a 100644 --- a/marco/marco/urls.py +++ b/marco/marco/urls.py @@ -3,6 +3,10 @@ Requires: Django 4.2+, Wagtail 7.0+ """ +import warnings +from importlib import import_module +from importlib.util import find_spec + from django.conf import settings from django.conf.urls.static import static from django.contrib import admin @@ -20,9 +24,6 @@ import accounts.urls import explore.urls -from visualize.urls import api_urlpatterns as visualize_api_urlpatterns -from drawing.urls import api_urlpatterns as drawing_api_urlpatterns -from mapgroups.urls import api_urlpatterns as mapgroups_api_urlpatterns from marco.rpc_compat import rpc_view from portal.base import views as base_views @@ -32,6 +33,32 @@ admin.autodiscover() wagtailsearch_register_signal_handlers() + +def _iter_discovered_api_includes(): + """Yield include() patterns for apps exposing urls.api_urlpatterns.""" + for installed_app in settings.INSTALLED_APPS: + app_module = installed_app.split('.apps.', 1)[0] + urls_module_name = f"{app_module}.urls" + + try: + if find_spec(urls_module_name) is None: + continue + except (ImportError, ValueError, AttributeError): + continue + + try: + app_urls = import_module(urls_module_name) + except Exception as exc: # pragma: no cover - defensive runtime warning + warnings.warn(f"Could not import '{urls_module_name}' for api_urlpatterns: {exc}") + continue + + app_api_urlpatterns = getattr(app_urls, 'api_urlpatterns', None) + if app_api_urlpatterns: + yield re_path(r'^api/', include(app_api_urlpatterns)) + + +api_url_includes = list(_iter_discovered_api_includes()) + # --------------------------------------------------------------------------- # Project-specific URL patterns # Optional: a portal variant (wcoa, mida, etc.) can prepend its own patterns. @@ -40,11 +67,9 @@ if settings.PROJECT_APP: try: - from importlib import import_module portal_app_urls = import_module(f"{settings.PROJECT_APP}.urls") urlpatterns = list(getattr(portal_app_urls, 'urlpatterns', [])) except (ImportError, AttributeError) as e: - import warnings warnings.warn(f"Could not load URL patterns from PROJECT_APP '{settings.PROJECT_APP}': {e}") # --------------------------------------------------------------------------- @@ -58,10 +83,12 @@ # /rpc — JSON-RPC 2.0 compat shim for legacy frontend JS (see rpc_compat.py) re_path(r'^rpc/', rpc_view), - # DRF REST replacements from each sub-app's api.py - re_path(r'^api/', include(visualize_api_urlpatterns)), - re_path(r'^api/', include(drawing_api_urlpatterns)), - re_path(r'^api/', include(mapgroups_api_urlpatterns)), +] + +urlpatterns += api_url_includes + +urlpatterns += [ + # DRF REST replacements discovered from each sub-app's urls.py api_urlpatterns re_path(r'^auth/', include('social_django.urls', namespace='social')), re_path(r'^account/', include('accounts.urls'), name='account'), From a98382c61e83538212315df7ba54b51af75ebb57 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Mon, 20 Apr 2026 13:05:32 -0700 Subject: [PATCH 058/152] Update environment variable handling for ReCAPTCHA keys and clean up config files --- .env.example | 4 ++-- marco/config.docker.ini.template | 27 ++++++++++----------------- marco/marco/settings.py | 4 ++-- 3 files changed, 14 insertions(+), 21 deletions(-) diff --git a/.env.example b/.env.example index f1d1f26..410466d 100644 --- a/.env.example +++ b/.env.example @@ -63,8 +63,8 @@ GOOGLE_SECRET= # --------------------------------------------------------------------------- # ReCAPTCHA (set in config.ini [APP] section or here) # --------------------------------------------------------------------------- -# RECAPTCHA_PUBLIC_KEY= -# RECAPTCHA_PRIVATE_KEY= +RECAPTCHA_PUBLIC_KEY= +RECAPTCHA_PRIVATE_KEY= # --------------------------------------------------------------------------- # Dev superuser bootstrap (entrypoint creates this user on first start) diff --git a/marco/config.docker.ini.template b/marco/config.docker.ini.template index fdfd641..341859e 100644 --- a/marco/config.docker.ini.template +++ b/marco/config.docker.ini.template @@ -10,14 +10,13 @@ PROJECT_SETTINGS_FILE = True DEBUG = True TEMPLATE_DEBUG = True ALLOWED_HOSTS = ["localhost", "127.0.0.1", "::1"] -SECRET_KEY = +# SECRET_KEY is loaded from environment variable: SECRET_KEY # TODO: Does the MEDIA_ROOT need to be updated here? potentially /usr/local/apps/madrona-portal/apps/wcoa/media/ MEDIA_ROOT = /vol/web/media MEDIA_URL = /media/ TIME_ZONE = UTC GA_ACCOUNT = -RECAPTCHA_PUBLIC_KEY = -RECAPTCHA_PRIVATE_KEY = +# ReCAPTCHA keys are loaded from env vars: RECAPTCHA_PUBLIC_KEY, RECAPTCHA_PRIVATE_KEY STATIC_ROOT = /vol/web/static EMAIL_SUBJECT_PREFIX = [WCOA] MAP_LIBRARY = ol8 @@ -35,12 +34,12 @@ MAP = ocean [CACHES] BACKEND = django_redis.cache.RedisCache -LOCATION = redis://:sOmE_sEcUrE_pAsS@tasks:6379/1 +LOCATION = redis://tasks:6379/1 CLIENT_CLASS = django_redis.client.DefaultClient [CELERY] -CELERY_RESULT_BACKEND = redis://:sOmE_sEcUrE_pAsS@tasks:6379/1 -CELERY_BROKER_URL = redis://:sOmE_sEcUrE_pAsS@tasks:6379 +CELERY_RESULT_BACKEND = redis://tasks:6379/1 +CELERY_BROKER_URL = redis://tasks:6379 CELERY_ALWAYS_EAGER = False CELERY_DISABLE_RATE_LIMITS = True @@ -50,29 +49,23 @@ NAME = wcoa_docker_db HOST = db PORT = 5432 USER = postgres +# DB password is loaded from environment variable: DB_PASSWORD [EMAIL] HOST = localhost PORT = 25 -HOST_USER = -HOST_PASSWORD = +# SMTP credentials are loaded from env vars: EMAIL_HOST_USER, EMAIL_HOST_PASSWORD DEFAULT_FROM_EMAIL = WCOA Portal SERVER_EMAIL = WCOA Site Errors [AWS] -AWS_ACCESS_KEY_ID = -AWS_SECRET_ACCESS_KEY = +# AWS credentials are loaded from env vars: AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY AWS_SES_REGION_NAME = us-east-1 AWS_SES_REGION_ENDPOINT = email.us-east-1.amazonaws.com [SOCIAL_AUTH] -# Supply these via env vars: FACEBOOK_KEY, FACEBOOK_SECRET, etc. -FACEBOOK_KEY = -FACEBOOK_SECRET = -TWITTER_KEY = -TWITTER_SECRET = -GOOGLE_KEY = -GOOGLE_SECRET = +# Social OAuth credentials are loaded from env vars: +# FACEBOOK_KEY, FACEBOOK_SECRET, TWITTER_KEY, TWITTER_SECRET, GOOGLE_KEY, GOOGLE_SECRET [CATALOG] diff --git a/marco/marco/settings.py b/marco/marco/settings.py index 220f941..ad06b2a 100644 --- a/marco/marco/settings.py +++ b/marco/marco/settings.py @@ -571,8 +571,8 @@ def _parse_hosts(raw: str | None) -> list[str]: # ReCAPTCHA # --------------------------------------------------------------------------- NOCAPTCHA = True -RECAPTCHA_PUBLIC_KEY = app_cfg.get('RECAPTCHA_PUBLIC_KEY', '') -RECAPTCHA_PRIVATE_KEY = app_cfg.get('RECAPTCHA_PRIVATE_KEY', '') +RECAPTCHA_PUBLIC_KEY = _env('RECAPTCHA_PUBLIC_KEY', app_cfg, 'RECAPTCHA_PUBLIC_KEY', '') +RECAPTCHA_PRIVATE_KEY = _env('RECAPTCHA_PRIVATE_KEY', app_cfg, 'RECAPTCHA_PRIVATE_KEY', '') # --------------------------------------------------------------------------- # Analytics From 2f6a814c95e2ebdfb3e019744d90b5463ef29c13 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Mon, 20 Apr 2026 16:18:41 -0700 Subject: [PATCH 059/152] Update docker/README.md Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- docker/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/README.md b/docker/README.md index a5d4cd3..9e35785 100644 --- a/docker/README.md +++ b/docker/README.md @@ -247,7 +247,7 @@ cp -r {your_media_dir}/* ./media/ docker buildx build \ --builder desktop-linux \ --load \ - -f madrona-portal/Dockerfile \ + -f madrona-portal/docker/Dockerfile \ -t madrona-portal-app:latest \ . ``` From bfb35ef67562117a8aa5447a90db86cb4fd2a179 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Mon, 20 Apr 2026 16:31:37 -0700 Subject: [PATCH 060/152] Add platform-specific Docker build instructions for MAC and LINUX --- docker/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker/README.md b/docker/README.md index 9e35785..8ba71ca 100644 --- a/docker/README.md +++ b/docker/README.md @@ -104,7 +104,9 @@ If running this build from a MAC, add `--builder desktop-linux` to the `buildx b ```bash cd ../docker - +# MAC OS +docker buildx build --builder desktop-linux --load -f ./Dockerfile ../../ +# LINUX docker buildx build --load -f ./Dockerfile ../../ ``` From 9419d6da4f8255c42b13c6ff1eb894fbce6a6ee3 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Mon, 20 Apr 2026 16:31:42 -0700 Subject: [PATCH 061/152] Refactor entrypoint script to enable production mode with Gunicorn and retain development server option --- docker/entrypoint.sh | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index a59c44e..85f122b 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -198,16 +198,16 @@ print("true" if settings.DEBUG else "false") PY ) -# if [ "${DJANGO_ENV:-}" = "production" ] || [ "${DJANGO_DEBUG}" = "false" ]; then -# echo "Starting gunicorn (production mode)..." -# exec gunicorn marco.wsgi:application \ -# --bind 0.0.0.0:8008 \ -# --workers "${GUNICORN_WORKERS:-3}" \ -# --timeout "${GUNICORN_TIMEOUT:-120}" \ -# --chdir marco \ -# --access-logfile - \ -# --error-logfile - -# else -echo "Starting Django development server..." -exec python marco/manage.py runserver 0.0.0.0:8000 -# fi +if [ "${DJANGO_ENV:-}" = "production" ] || [ "${DJANGO_DEBUG}" = "false" ]; then + echo "Starting gunicorn (production mode)..." + exec gunicorn marco.wsgi:application \ + --bind 0.0.0.0:8008 \ + --workers "${GUNICORN_WORKERS:-3}" \ + --timeout "${GUNICORN_TIMEOUT:-120}" \ + --chdir marco \ + --access-logfile - \ + --error-logfile - +else + echo "Starting Django development server..." + exec python marco/manage.py runserver 0.0.0.0:8000 +fi From 82f9d797e6c0e13104ee1a4520200e07a8efdf1f Mon Sep 17 00:00:00 2001 From: David Pollard Date: Tue, 21 Apr 2026 15:06:17 -0700 Subject: [PATCH 062/152] Add GitHub Container Registry configuration to .env.example --- docker/.env.example | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docker/.env.example b/docker/.env.example index 2cf0ab7..964bfd4 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -6,6 +6,15 @@ # Priority for every setting: env var > config.ini > built-in default # ============================================================================= +# --------------------------------------------------------------------------- +# GitHub Container Registry (production only) +# GHCR_PAT: read-only fine-grained PAT used to pull the image from GHCR. +# Run: echo $GHCR_PAT | docker login ghcr.io -u --password-stdin +# IMAGE_TAG: pin to a specific build SHA for rollback (default: latest) +# --------------------------------------------------------------------------- +GHCR_PAT= +IMAGE_TAG=latest + # --------------------------------------------------------------------------- # Django core # --------------------------------------------------------------------------- From 9fe797e900b63d6e9d617fd8912c1807b50a5a85 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Tue, 21 Apr 2026 15:06:23 -0700 Subject: [PATCH 063/152] Add GitHub Actions workflow to create and publish Docker images for WCODP --- .../create-and-publish-docker-images.yml | 200 ++++++++++++++++++ 1 file changed, 200 insertions(+) create mode 100644 .github/workflows/create-and-publish-docker-images.yml diff --git a/.github/workflows/create-and-publish-docker-images.yml b/.github/workflows/create-and-publish-docker-images.yml new file mode 100644 index 0000000..60eb47a --- /dev/null +++ b/.github/workflows/create-and-publish-docker-images.yml @@ -0,0 +1,200 @@ +# source: https://docs.github.com/en/actions/tutorials/publish-packages/publish-docker-images#publishing-images-to-docker-hub-and-github-packages +name: Create and publish West Coast Ocean Data Portal (WCODP) Docker image + +# Builds a single Docker image from the full workspace (madrona-portal + +# all madrona-apps sub-repos) and pushes it to GitHub Container Registry. +# +# Image: ghcr.io/ecotrust/madrona-portal: (and :latest) + +on: + push: + branches: ['docker'] + # TODO: switch to main branch once we're ready to build from there + workflow_dispatch: + +# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu. +jobs: + build-and-push: + runs-on: ubuntu-latest + # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job. + permissions: + contents: read + packages: write + attestations: write + id-token: write + + steps: + # ----------------------------------------------------------------------- + # Checkout madrona-portal into madrona-portal/ so the Dockerfile's COPY + # paths (e.g. COPY madrona-portal/marco ...) resolve correctly. + # ----------------------------------------------------------------------- + - name: Checkout madrona-portal + uses: actions/checkout@v5 + with: + path: madrona-portal + + # ----------------------------------------------------------------------- + # Checkout all sub-apps into madrona-apps// — mirrors the local + # workspace layout the Dockerfile expects. + # GH_PAT must have read access to all sub-app repos in the Ecotrust org. + # ----------------------------------------------------------------------- + - name: Checkout django_url_shortener + uses: actions/checkout@v5 + with: + repository: Ecotrust/django_url_shortener + token: ${{ secrets.GH_PAT }} + path: madrona-apps/django_url_shortener + + - name: Checkout madrona-analysistools + uses: actions/checkout@v5 + with: + repository: Ecotrust/madrona-analysistools + token: ${{ secrets.GH_PAT }} + path: madrona-apps/madrona-analysistools + + - name: Checkout madrona-features + uses: actions/checkout@v5 + with: + repository: Ecotrust/madrona-features + token: ${{ secrets.GH_PAT }} + path: madrona-apps/madrona-features + + - name: Checkout madrona-manipulators + uses: actions/checkout@v5 + with: + repository: Ecotrust/madrona-manipulators + token: ${{ secrets.GH_PAT }} + path: madrona-apps/madrona-manipulators + + - name: Checkout madrona-scenarios + uses: actions/checkout@v5 + with: + repository: Ecotrust/madrona-scenarios + token: ${{ secrets.GH_PAT }} + path: madrona-apps/madrona-scenarios + + - name: Checkout mp-accounts + uses: actions/checkout@v5 + with: + repository: Ecotrust/mp-accounts + token: ${{ secrets.GH_PAT }} + path: madrona-apps/mp-accounts + + - name: Checkout mp-data-manager + uses: actions/checkout@v5 + with: + repository: Ecotrust/mp-data-manager + token: ${{ secrets.GH_PAT }} + path: madrona-apps/mp-data-manager + + - name: Checkout mp-drawing + uses: actions/checkout@v5 + with: + repository: Ecotrust/mp-drawing + token: ${{ secrets.GH_PAT }} + path: madrona-apps/mp-drawing + + - name: Checkout mp-explore + uses: actions/checkout@v5 + with: + repository: Ecotrust/mp-explore + token: ${{ secrets.GH_PAT }} + path: madrona-apps/mp-explore + + - name: Checkout mp-layers + uses: actions/checkout@v5 + with: + repository: Ecotrust/mp-layers + token: ${{ secrets.GH_PAT }} + path: madrona-apps/mp-layers + + - name: Checkout mp-map-groups + uses: actions/checkout@v5 + with: + repository: Ecotrust/mp-map-groups + token: ${{ secrets.GH_PAT }} + path: madrona-apps/mp-map-groups + + - name: Checkout mp-proxy + uses: actions/checkout@v5 + with: + repository: Ecotrust/mp-proxy + token: ${{ secrets.GH_PAT }} + path: madrona-apps/mp-proxy + + - name: Checkout mp-visualize + uses: actions/checkout@v5 + with: + repository: Ecotrust/mp-visualize + token: ${{ secrets.GH_PAT }} + path: madrona-apps/mp-visualize + + - name: Checkout p97-nursery + uses: actions/checkout@v5 + with: + repository: Ecotrust/p97-nursery + token: ${{ secrets.GH_PAT }} + path: madrona-apps/p97-nursery + + # wcoa uses the vagrant2docker branch (contains Docker-specific config) + - name: Checkout wcoa + uses: actions/checkout@v5 + with: + repository: Ecotrust/wcoa + # TODO: change to main branch once Docker-specific config is merged there + ref: vagrant2docker + token: ${{ secrets.GH_PAT }} + path: madrona-apps/wcoa + + # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels. + - name: Set lowercase image name + run: | + IMAGE_NAME=$(echo "${GITHUB_REPOSITORY}" | tr '[:upper:]' '[:lower:]') + echo "IMAGE_NAME=${IMAGE_NAME}" >> $GITHUB_ENV + + # ----------------------------------------------------------------------- + # Docker setup + # ----------------------------------------------------------------------- + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract short SHA + id: meta + run: | + cd madrona-portal + echo "sha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT + + # ----------------------------------------------------------------------- + # Build from workspace root — matches the layout the Dockerfile expects. + # Pushing :sha and :latest together so EC2 can pin a specific build or + # always pull the newest with :latest. + # ----------------------------------------------------------------------- + - name: Build and push + uses: docker/build-push-action@v5 + with: + context: . + file: madrona-portal/docker/Dockerfile + push: true + tags: | + ghcr.io/${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.sha }} + ghcr.io/${{ env.IMAGE_NAME }}:latest + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Image digest summary + run: | + echo "### Image pushed to GHCR" >> $GITHUB_STEP_SUMMARY + echo "- \`ghcr.io/${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.sha }}\`" >> $GITHUB_STEP_SUMMARY + echo "- \`ghcr.io/${{ env.IMAGE_NAME }}:latest\`" >> $GITHUB_STEP_SUMMARY From 751e545b8acd7a1f7568cbe32c1e70e6d299d8b2 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Tue, 21 Apr 2026 15:37:03 -0700 Subject: [PATCH 064/152] Add production Docker Compose configuration for Madrona Portal --- docker/docker-compose.prod.yml | 152 +++++++++++++++++++++++++++++++++ 1 file changed, 152 insertions(+) create mode 100644 docker/docker-compose.prod.yml diff --git a/docker/docker-compose.prod.yml b/docker/docker-compose.prod.yml new file mode 100644 index 0000000..d30a05f --- /dev/null +++ b/docker/docker-compose.prod.yml @@ -0,0 +1,152 @@ +# Madrona Portal — Production Docker Compose (WCOA) +# +# Uses the pre-built image from GitHub Container Registry instead of building +# from source. All madrona-apps code is baked into the image — no source +# volume mounts. +# +# Usage: +# docker compose -f docker/docker-compose.prod.yml \ +# --env-file .env \ +# --profile full \ +# up -d +# +# To pull the latest image before starting: +# docker pull ghcr.io/ecotrust/madrona-portal:latest +# +# To pin to a specific build (for rollback): +# IMAGE_TAG=abc1234 docker compose -f docker/docker-compose.prod.yml ... + +services: + + app: + image: ghcr.io/ecotrust/madrona-portal:${IMAGE_TAG:-latest} + volumes: + - static_data:/vol/web + # User-uploaded media files — persisted on the host across deploys. + - ./media:/usr/local/apps/madrona-portal/media + # Config file — lets you update portal settings without rebuilding the image. + - ../marco/config.wcoa.docker.ini:/usr/local/apps/madrona-portal/marco/config.wcoa.docker.ini:ro + env_file: + - ./.env + environment: + - DB_INIT=${DB_INIT:-0} + - MP_PROJECT_CONFIG=${MP_PROJECT_CONFIG:-config.wcoa.docker.ini} + - SECRET_KEY=${SECRET_KEY} + - ALLOWED_HOSTS=${ALLOWED_HOSTS:-localhost,127.0.0.1,::1} + - DEBUG=${DEBUG:-False} + - DB_ENGINE=${DB_ENGINE:-django.contrib.gis.db.backends.postgis} + - DB_NAME=${DB_NAME:-wcoa_docker_db} + - DB_USER=${DB_USER:-postgres} + - DB_PASSWORD=${DB_PASSWORD} + - DB_HOST=db + - DB_PORT=5432 + - REDIS_URL=redis://${REDIS_PASSWORD:+:${REDIS_PASSWORD}@}tasks:6379/1 + - CELERY_BROKER_URL=redis://${REDIS_PASSWORD:+:${REDIS_PASSWORD}@}tasks:6379/0 + - DJANGO_SUPERUSER_USERNAME=${DJANGO_SUPERUSER_USERNAME:-admin} + - DJANGO_SUPERUSER_EMAIL=${DJANGO_SUPERUSER_EMAIL:-admin@example.com} + - DJANGO_SUPERUSER_PASSWORD=${DJANGO_SUPERUSER_PASSWORD:-} + - DJANGO_ENV=${DJANGO_ENV:-production} + - GUNICORN_WORKERS=${GUNICORN_WORKERS:-3} + depends_on: + db: + condition: service_healthy + tasks: + condition: service_healthy + ports: + - "${APP_PORT:-8000}:8000" + networks: + - madronanetwork + restart: unless-stopped + profiles: [full] + + db: + image: postgis/postgis:16-3.4 + volumes: + - postgis_data:/var/lib/postgresql + environment: + - POSTGRES_USER=${DB_USER:-postgres} + - POSTGRES_PASSWORD=${DB_PASSWORD} + - POSTGRES_DB=${DB_NAME:-wcoa_docker_db} + ports: + - "${DB_PORT:-5432}:5432" + networks: + - madronanetwork + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${DB_USER:-postgres} -d ${DB_NAME:-wcoa_docker_db}"] + interval: 10s + timeout: 5s + retries: 5 + restart: unless-stopped + + tasks: + image: redis:7-alpine + command: redis-server ${REDIS_PASSWORD:+--requirepass ${REDIS_PASSWORD}} + ports: + - "${REDIS_PORT:-6379}:6379" + volumes: + - redis_data:/data + networks: + - madronanetwork + healthcheck: + test: ["CMD-SHELL", "redis-cli ${REDIS_PASSWORD:+-a ${REDIS_PASSWORD}} ping"] + interval: 10s + timeout: 5s + retries: 5 + restart: unless-stopped + + geoportal: + image: tomcat:9-jdk11 + ports: + - 8080:8080 + volumes: + - gp-volume:/usr/local/tomcat/webapps/ + - ${gpt_catalog_war}:/usr/local/tomcat/webapps/geoportal.war + - ${gpt_harvester_war}:/usr/local/tomcat/webapps/harvester.war + - ./templates:/templates:ro + - ./geoportal-entrypoint.sh:/usr/local/bin/entrypoint.sh:ro + entrypoint: ["/usr/local/bin/entrypoint.sh"] + networks: + - madronanetwork + restart: always + env_file: + - ./.env + depends_on: + elastic: + condition: service_healthy + profiles: [full] + + elastic: + image: elasticsearch:8.19.12 + volumes: + - es-volume:/usr/share/elasticsearch/data + environment: + - discovery.type=single-node + - ES_JAVA_OPTS=-Xms512m -Xmx512m + - cluster.name=${CLUSTER_NAME} + - ELASTIC_PASSWORD=${ELASTIC_PASSWORD} + - bootstrap.memory_lock=true + - reindex.remote.whitelist=${ES_REINDEX_REMOTE_WHITELIST} + - xpack.security.enabled=false + ports: + - 9200:9200 + - 9300:9300 + networks: + - madronanetwork + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9200/_cluster/health?wait_for_status=yellow"] + interval: 30s + timeout: 10s + retries: 10 + restart: always + profiles: [full] + +volumes: + static_data: + postgis_data: + redis_data: + gp-volume: + es-volume: + +networks: + madronanetwork: + driver: bridge From 3a4f8b427f5b6bfbeea5ce3967d881f4fd4e5c19 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Tue, 21 Apr 2026 15:37:11 -0700 Subject: [PATCH 065/152] Update AWS deployment guide and README for production setup and usage --- docker/AWS_DEPLOY.md | 404 ++++++++++++++++++++++++++----------------- docker/README.md | 68 +------- 2 files changed, 253 insertions(+), 219 deletions(-) diff --git a/docker/AWS_DEPLOY.md b/docker/AWS_DEPLOY.md index acade8a..d3c780b 100644 --- a/docker/AWS_DEPLOY.md +++ b/docker/AWS_DEPLOY.md @@ -4,15 +4,77 @@ This guide takes you from a blank AWS account to a running production stack. All services (Django, PostGIS, Redis, Elasticsearch, Geoportal) run as Docker containers on a single EC2 instance using Docker Compose. +The Docker image is built by GitHub Actions and stored in the GitHub Container +Registry (GHCR). The EC2 server pulls a pre-built image — no source code +checkout or build step is needed on the server. + --- ## Prerequisites - An AWS account with billing enabled -- Your local machine has the AWS CLI installed and configured, - or you are comfortable using the AWS Console - SSH client on your local machine -- A GitHub account with access to all required repos +- A GitHub account with admin access to the `Ecotrust` organization + +--- + +## Phase 0 — One-time GHCR Setup + +These steps are done once by a GitHub org admin. Skip to Phase 1 if the +`madrona-portal` package already exists in GHCR. + +### 0.1 Create a PAT for GitHub Actions (CI push) + +This token lets GitHub Actions push images to GHCR on behalf of the org. + +**GitHub → Settings → Developer settings → Personal access tokens → Fine-grained tokens → Generate new token** + +| Setting | Value | +|---|---| +| Token name | `madrona-portal-ci` | +| Expiration | 1 year (set a reminder to rotate) | +| Resource owner | `Ecotrust` | +| Repository access | All repositories (needed to read all sub-app repos) | +| Permissions → Contents | Read-only | +| Permissions → Packages | Read and write | + +Copy the token. Add it as an Actions secret in `madrona-portal`: + +**GitHub → `Ecotrust/madrona-portal` → Settings → Secrets and variables → Actions → New repository secret** + +| Name | Value | +|---|---| +| `GH_PAT` | `` | + +### 0.2 Create a read-only PAT for the EC2 server (pull only) + +**GitHub → Settings → Developer settings → Personal access tokens → Fine-grained tokens → Generate new token** + +| Setting | Value | +|---|---| +| Token name | `madrona-portal-ec2` | +| Expiration | 1 year | +| Resource owner | `Ecotrust` | +| Repository access | Public repositories only | +| Permissions → Packages | Read-only | + +Copy this token — you will add it to the server `.env` in Phase 4. + +### 0.3 Trigger the first image build + +Push a commit to the `docker` branch of `madrona-portal`, or run the workflow +manually: + +**GitHub → `Ecotrust/madrona-portal` → Actions → "Build and Push to GHCR" → Run workflow** + +The first build takes 15–25 minutes (compiling GDAL, installing all Python +packages). Subsequent builds are faster thanks to GitHub Actions layer caching. + +Confirm the image appears at: +**GitHub → `Ecotrust` org → Packages → `madrona-portal`** + +Note the short SHA from the workflow summary — you can use it to pin a specific +build on EC2 instead of always pulling `:latest`. --- @@ -20,8 +82,6 @@ containers on a single EC2 instance using Docker Compose. ### 1.1 Create a key pair -You need this before launching the instance. - **AWS Console → EC2 → Key Pairs → Create key pair** | Setting | Value | @@ -55,8 +115,8 @@ chmod 400 ~/.ssh/madrona-portal.pem | HTTP | 80 | `0.0.0.0/0` | Web traffic (Nginx) | | HTTPS | 443 | `0.0.0.0/0` | Web traffic (Nginx + SSL) | -> Do **not** open port 8000 to the public. Nginx (added in Phase 5) will -> proxy traffic to gunicorn on port 8000 internally. +> Do **not** open port 8000 to the public. Nginx proxies traffic to Gunicorn +> on port 8000 internally. **Outbound rules:** leave the default (all traffic allowed). @@ -74,13 +134,13 @@ chmod 400 ~/.ssh/madrona-portal.pem | Storage | 60 GB gp3 — expand the default 8 GB root volume | > **Why t3.large?** Elasticsearch alone reserves 1 GB of heap -> (`-Xms512m -Xmx512m` in docker-compose.yml) plus JVM overhead. -> Add PostGIS, gunicorn workers, Redis, and Tomcat (Geoportal) and you need -> at least 6–7 GB free. A `t3.large` (8 GB) is the practical minimum; -> `t3.xlarge` (16 GB) gives comfortable headroom. +> (`-Xms512m -Xmx512m`) plus JVM overhead. Add PostGIS, Gunicorn workers, +> Redis, and Tomcat (Geoportal) and you need at least 6–7 GB free. A +> `t3.large` (8 GB) is the practical minimum; `t3.xlarge` (16 GB) gives +> comfortable headroom. -> **Why 60 GB?** The Docker image is ~3–4 GB after build. PostGIS data, -> Elasticsearch indices, and Docker's build cache add up quickly. +> **Why 60 GB?** The Docker image is ~3–4 GB after pull. PostGIS data, +> Elasticsearch indices, and Docker's image cache add up quickly. Launch the instance and wait for it to reach **Running** state. @@ -156,30 +216,77 @@ sudo systemctl enable docker sudo systemctl enable containerd ``` +### 2.6 Log in to GitHub Container Registry + +```bash +# You will add GHCR_PAT to .env in Phase 4. +# Run this now using the read-only PAT you created in Phase 0.2: +echo "" | docker login ghcr.io -u --password-stdin +``` + +The login token is saved to `~/.docker/config.json` and persists across +reboots. You only need to re-run this if the PAT expires. + +--- + +## Phase 3 — Transfer Geoportal WAR Files + +The Geoportal service requires two Java WAR files that are not in any Git +repository. Copy them from the old production server. + +**On the old server**, find the WAR files: + +```bash +# Common locations on the old server: +find / -name "geoportal.war" -o -name "harvester.war" 2>/dev/null +``` + +**On your local machine**, SCP them to the new EC2 instance: + +```bash +# Replace OLD_SERVER_IP and paths as appropriate +scp ubuntu@:/path/to/geoportal.war \ + ubuntu@:/tmp/geoportal.war + +scp ubuntu@:/path/to/harvester.war \ + ubuntu@:/tmp/harvester.war +``` + +**On the new EC2 instance**, move them into the expected location: + +```bash +mkdir -p ~/portals/madrona-portal/docker/wars +mv /tmp/geoportal.war ~/portals/madrona-portal/docker/wars/ +mv /tmp/harvester.war ~/portals/madrona-portal/docker/wars/ +``` + +The `.env.example` defaults point to `./wars/geoportal.war` and +`./wars/harvester.war` (relative to the `docker/` directory), so these paths +will work without any further changes. + --- -## Phase 3 — Clone the Repositories +## Phase 4 — Clone the Portal Configuration -The Dockerfile build context must be the **workspace root** — a parent -directory containing both `madrona-portal/` and `madrona-apps/` as siblings. -This layout is required; it is not optional. +The EC2 server only needs the `madrona-portal` repository — for the Compose +file, Nginx templates, entrypoint scripts, and environment configuration. +No sub-app repos are needed; all application code is baked into the image. -### 3.1 Set up SSH access to GitHub (on the server) +### 4.1 Set up SSH access to GitHub (on the server) ```bash ssh-keygen -t ed25519 -C "madrona-portal-server" -f ~/.ssh/github -N "" cat ~/.ssh/github.pub ``` -Copy the output and add it as a **Deploy Key** in each GitHub repository -(or as an SSH key on your GitHub account if you have access to all repos): +Copy the output and add it as a **Deploy Key** in `madrona-portal`: -**GitHub repo → Settings → Deploy keys → Add deploy key** -- Paste the public key +**GitHub → `Ecotrust/madrona-portal` → Settings → Deploy keys → Add deploy key** - Title: `madrona-portal EC2` -- Enable "Allow write access": No (read-only is sufficient) +- Paste the public key +- Allow write access: No -Configure SSH to use this key for GitHub: +Configure SSH to use this key: ```bash cat >> ~/.ssh/config << 'EOF' @@ -189,72 +296,41 @@ Host github.com EOF ``` -### 3.2 Create the workspace and clone +### 4.2 Clone madrona-portal ```bash mkdir ~/portals && cd ~/portals -``` - -Clone the main portal: - -```bash git clone -b docker git@github.com:Ecotrust/madrona-portal.git madrona-portal ``` -Clone all sub-apps: - -```bash -mkdir madrona-apps && cd madrona-apps - -git clone git@github.com:Ecotrust/django_url_shortener.git -git clone git@github.com:Ecotrust/madrona-analysistools.git -git clone git@github.com:Ecotrust/madrona-features.git -git clone git@github.com:Ecotrust/madrona-manipulators.git -git clone git@github.com:Ecotrust/madrona-scenarios.git -git clone git@github.com:Ecotrust/mp-accounts.git -git clone git@github.com:Ecotrust/mp-data-manager.git -git clone git@github.com:Ecotrust/mp-drawing.git -git clone git@github.com:Ecotrust/mp-explore.git -git clone git@github.com:Ecotrust/mp-layers.git -git clone git@github.com:Ecotrust/mp-map-groups.git -git clone git@github.com:Ecotrust/mp-proxy.git -git clone git@github.com:Ecotrust/mp-visualize.git -git clone git@github.com:Ecotrust/p97-nursery.git -git clone -b vagrant2docker git@github.com:Ecotrust/wcoa.git - -cd .. -``` - -Verify the layout: +Verify: ```bash ls ~/portals/ -# madrona-portal/ madrona-apps/ +# madrona-portal/ ``` --- -## Phase 4 — Configure the Environment +## Phase 5 — Configure the Environment -### 4.1 Create the `.env` file +### 5.1 Create the `.env` file ```bash cd ~/portals/madrona-portal -cp .env.example .env +cp docker/.env.example docker/.env ``` -### 4.2 Generate a secret key +### 5.2 Generate a secret key ```bash python3 -c "import secrets; print(secrets.token_urlsafe(50))" ``` -Copy the output — you will paste it as `SECRET_KEY` below. - -### 4.3 Edit `.env` +### 5.3 Edit `.env` ```bash -nano .env +nano docker/.env ``` Set these values at minimum: @@ -272,87 +348,86 @@ DB_PASSWORD= # Redis REDIS_PASSWORD= -# Superuser (created automatically on first boot) +# Superuser (created automatically on first boot if DB_INIT=1) DJANGO_SUPERUSER_USERNAME=admin DJANGO_SUPERUSER_EMAIL=your@email.com DJANGO_SUPERUSER_PASSWORD= -# Gunicorn workers (set to 2× vCPU count; t3.large has 2 vCPUs → 4 workers) +# Gunicorn workers (2× vCPU count; t3.large has 2 vCPUs → 4 workers) GUNICORN_WORKERS=4 + +# GHCR — the read-only PAT from Phase 0.2 (documents what token was used to +# log in; docker login stores credentials in ~/.docker/config.json) +GHCR_PAT= + +# Geoportal WAR paths (defaults match Phase 3 location — no change needed) +gpt_catalog_war=./wars/geoportal.war +gpt_harvester_war=./wars/harvester.war ``` Leave everything else at its default for now. You can add email, OAuth, and Elasticsearch credentials later. -### 4.4 Create ini file +### 5.4 Create the ini config file ```bash cd ~/portals/madrona-portal/marco cp config.docker.ini.template config.wcoa.docker.ini ``` - --- -## Phase 5 — Build the Docker Image - -> **Important:** The `--builder desktop-linux` flag in the local development -> guide is specific to Docker Desktop on Mac. On Linux EC2 you omit it — -> BuildKit is the default builder. +## Phase 6 — Pull and Start the Stack -From the **workspace root** (`~/portals/`): +### 6.1 Pull the image from GHCR ```bash -cd ~/portals - -docker buildx build \ - --load \ - -f madrona-portal/docker/Dockerfile \ - -t madrona-portal-app:latest \ - . +docker pull ghcr.io/ecotrust/madrona-portal:latest ``` -This will take 10–20 minutes on first build (compiling GDAL, installing all -Python packages). Subsequent builds are fast thanks to layer caching. - -Watch for any errors. Common first-build issues: -- Out of disk space → increase EBS volume or run `docker system prune -f` first -- Network timeouts fetching packages → re-run the command (layer cache resumes) - ---- +> To use a specific build instead of `latest`, note the short SHA from the +> GitHub Actions workflow summary and pull by tag: +> `docker pull ghcr.io/ecotrust/madrona-portal:abc1234` -## Phase 6 — Start the Stack +### 6.2 Start the stack From `~/portals/madrona-portal/`: ```bash cd ~/portals/madrona-portal -docker compose -f docker/docker-compose.yml \ - --env-file .env \ +docker compose -f docker/docker-compose.prod.yml \ + --env-file docker/.env \ --profile full \ up -d ``` -### 6.1 Watch the startup logs +### 6.3 Watch the startup logs ```bash -docker compose -f docker/docker-compose.yml --env-file .env --profile full \ +docker compose -f docker/docker-compose.prod.yml --env-file docker/.env --profile full \ logs -f app ``` On first boot the entrypoint automatically: -1. Waits for PostgreSQL +1. Waits for PostgreSQL to be healthy 2. Runs `migrate` 3. Runs `collectstatic` and `compress` 4. Detects fresh database → loads initial fixtures -5. Creates the superuser from `.env` -6. Starts gunicorn (because `DEBUG=False`) +5. Creates the superuser from `.env` (if `DB_INIT=1`) +6. Starts Gunicorn + +Startup takes 2–5 minutes. Look for `Booting worker` lines from Gunicorn. -Startup takes 2–5 minutes. You should see `Booting worker` lines from -gunicorn when it is ready. +> For a fresh database, run with `DB_INIT=1` the first time: +> ```bash +> DB_INIT=1 docker compose -f docker/docker-compose.prod.yml --env-file docker/.env \ +> --profile full up -d +> ``` +> On subsequent starts, leave `DB_INIT` at its default (`0`) to skip the +> fixture and superuser steps. -### 6.2 Smoke test +### 6.4 Smoke test ```bash curl -I http://localhost:8000/ @@ -381,9 +456,14 @@ In your DNS provider (Route 53, Cloudflare, etc.): | Type | Name | Value | |---|---|---| -| A | `portal.yourdomain.com` | `` | +| A | `portal.westcoastoceans.org` | `` | -Wait for DNS to propagate before continuing (check with `dig portal.yourdomain.com`). +Wait for DNS to propagate before continuing: + +```bash +dig portal.westcoastoceans.org +# Should return your Elastic IP +``` ### 7.3 Configure Nginx @@ -396,12 +476,7 @@ Paste: ```nginx server { listen 80; - server_name portal.yourdomain.com; - - # Static and media files served directly by Nginx from the Docker volume. - # The static_data volume is mounted at /vol/web inside the container but - # is not accessible from the host — gunicorn serves these for now. - # See note below about a future Nginx-native static setup. + server_name portal.westcoastoceans.org; location / { proxy_pass http://127.0.0.1:8000; @@ -427,25 +502,25 @@ sudo systemctl restart nginx ### 7.4 Obtain an SSL certificate ```bash -sudo certbot --nginx -d portal.yourdomain.com +sudo certbot --nginx -d portal.westcoastoceans.org ``` Certbot edits your Nginx config automatically to add SSL and redirect HTTP -to HTTPS. It also installs a cron job to renew the certificate automatically. +to HTTPS. It installs a cron job to renew the certificate automatically. ### 7.5 Update `ALLOWED_HOSTS` -Add your domain to `.env`: +Add the domain to `docker/.env`: ```ini -ALLOWED_HOSTS=portal.yourdomain.com,,localhost +ALLOWED_HOSTS=portal.westcoastoceans.org,,localhost ``` -Then restart the app container to pick up the change: +Restart the app container to pick up the change: ```bash cd ~/portals/madrona-portal -docker compose -f docker/docker-compose.yml --env-file .env --profile full \ +docker compose -f docker/docker-compose.prod.yml --env-file docker/.env --profile full \ up -d --force-recreate app ``` @@ -453,8 +528,7 @@ docker compose -f docker/docker-compose.yml --env-file .env --profile full \ ## Phase 8 — Keep the Stack Running Across Reboots -Docker Compose does not automatically restart after the EC2 instance reboots. -Set up a systemd service to handle this. +### 8.1 Create a systemd service ```bash sudo nano /etc/systemd/system/madrona-portal.service @@ -473,13 +547,13 @@ Type=oneshot RemainAfterExit=yes WorkingDirectory=/home/ubuntu/portals/madrona-portal ExecStart=/usr/bin/docker compose \ - -f docker/docker-compose.yml \ - --env-file .env \ + -f docker/docker-compose.prod.yml \ + --env-file docker/.env \ --profile full \ up -d ExecStop=/usr/bin/docker compose \ - -f docker/docker-compose.yml \ - --env-file .env \ + -f docker/docker-compose.prod.yml \ + --env-file docker/.env \ --profile full \ down TimeoutStartSec=300 @@ -488,56 +562,69 @@ TimeoutStartSec=300 WantedBy=multi-user.target ``` -Enable it: +### 8.2 Enable the service ```bash sudo systemctl daemon-reload sudo systemctl enable madrona-portal ``` -Test it (optional — simulates a reboot): +### 8.3 Test it (simulates a reboot) ```bash sudo systemctl stop madrona-portal sudo systemctl start madrona-portal +docker compose -f docker/docker-compose.prod.yml --env-file docker/.env --profile full \ + ps # all services should show "running" ``` --- -## Redeploying After Code Changes +## Deploying a New Release -When you push new code and want to redeploy: +When code changes are merged to the `docker` branch, GitHub Actions +automatically builds and pushes a new image to GHCR. To deploy it: -**1. On your local machine — commit and push all changes first.** -BuildKit reads from the git object store, so uncommitted changes will not -be included in the image. - -**2. On the server — pull and rebuild:** +### On the server ```bash -cd ~/portals/madrona-portal && git pull -cd ~/portals/madrona-apps/ && git pull # repeat for each changed sub-app +cd ~/portals/madrona-portal -cd ~/portals +# Pull the latest image (or a specific SHA tag for a pinned deploy) +docker pull ghcr.io/ecotrust/madrona-portal:latest -docker buildx build \ - --load \ - -f madrona-portal/Dockerfile \ - -t madrona-portal-app:latest \ - . +# Recreate only the app container — db, Redis, and Elasticsearch are untouched +docker compose -f docker/docker-compose.prod.yml --env-file docker/.env --profile full \ + up -d --force-recreate app ``` -**3. Recreate the app container:** +Downtime is limited to the container restart (~5–10 seconds). + +### Rolling back to a previous build ```bash -cd ~/portals/madrona-portal +# List available tags in GHCR (or check the GitHub Actions workflow summaries +# for the SHA of any previous build) -docker compose -f docker/docker-compose.yml --env-file .env --profile full \ +# Pull the specific SHA tag +docker pull ghcr.io/ecotrust/madrona-portal: + +# Update IMAGE_TAG in docker/.env, then recreate: +# IMAGE_TAG= in docker/.env +docker compose -f docker/docker-compose.prod.yml --env-file docker/.env --profile full \ up -d --force-recreate app ``` -The database and Redis containers are untouched. Downtime is limited to the -container restart (~5–10 seconds). +### If portal configuration changes (config.wcoa.docker.ini) + +The ini file is bind-mounted into the container (read-only), so changes take +effect immediately on the next container restart — no image rebuild needed: + +```bash +nano ~/portals/madrona-portal/marco/config.wcoa.docker.ini +docker compose -f docker/docker-compose.prod.yml --env-file docker/.env --profile full \ + up -d --force-recreate app +``` --- @@ -547,43 +634,48 @@ All `docker compose` commands run from `~/portals/madrona-portal/`. ```bash # Tail app logs -docker compose -f docker/docker-compose.yml --env-file .env --profile full logs -f app +docker compose -f docker/docker-compose.prod.yml --env-file docker/.env --profile full \ + logs -f app # Run a Django management command -docker compose -f docker/docker-compose.yml --env-file .env --profile full \ +docker compose -f docker/docker-compose.prod.yml --env-file docker/.env --profile full \ run --rm app python marco/manage.py # Open a Django shell -docker compose -f docker/docker-compose.yml --env-file .env --profile full \ +docker compose -f docker/docker-compose.prod.yml --env-file docker/.env --profile full \ run --rm app python marco/manage.py shell # Open a database shell -docker exec -it $(docker compose -f docker/docker-compose.yml --env-file .env ps -q db) \ +docker exec -it \ + $(docker compose -f docker/docker-compose.prod.yml --env-file docker/.env ps -q db) \ psql -U postgres wcoa_docker_db # Check disk and Docker space usage df -h docker system df -# Stop the stack (data preserved) -docker compose -f docker/docker-compose.yml --env-file .env --profile full down +# Remove old/unused images to free space +docker image prune -f + +# Stop the stack (data preserved in named volumes) +docker compose -f docker/docker-compose.prod.yml --env-file docker/.env --profile full down # Full reset — DESTROYS ALL DATA -docker compose -f docker/docker-compose.yml --env-file .env --profile full down -v +docker compose -f docker/docker-compose.prod.yml --env-file docker/.env --profile full down -v ``` --- ## Services and Ports -| Service | Image | Internal port | Exposed to host | +| Service | Image | Internal port | Notes | |---|---|---|---| -| `app` | `madrona-portal-app:latest` | 8000 | Yes — proxied by Nginx | -| `db` | `postgis/postgis:16-3.4` | 5432 | Yes (restrict in security group) | -| `tasks` | `redis:7-alpine` | 6379 | Yes (restrict in security group) | -| `geoportal` | built from `wcoa/docker` | 8080 | Yes (add Nginx location if needed) | -| `elastic` | `elasticsearch:8.19.12` | 9200, 9300 | Yes (restrict in security group) | +| `app` | `ghcr.io/ecotrust/madrona-portal:latest` | 8000 | Proxied by Nginx | +| `db` | `postgis/postgis:16-3.4` | 5432 | Restrict in security group after go-live | +| `tasks` | `redis:7-alpine` | 6379 | Restrict in security group after go-live | +| `geoportal` | `tomcat:9-jdk11` | 8080 | Add Nginx location if public access needed | +| `elastic` | `elasticsearch:8.19.12` | 9200, 9300 | Restrict in security group after go-live | > After go-live, update your security group inbound rules to remove public -> access to ports 5432, 6379, 9200, and 9300. These are only needed +> access to ports 5432, 6379, 9200, and 9300. These ports are only needed > internally between containers on `madronanetwork`. diff --git a/docker/README.md b/docker/README.md index 8ba71ca..835649c 100644 --- a/docker/README.md +++ b/docker/README.md @@ -238,73 +238,15 @@ cp -r {your_media_dir}/* ./media/ --- -# Untested instructions below this line — will update after testing - -## Deploy to fully containerized live instance - -1. Set up your `.env` -2. Build and run (from `portals/`): - -```bash -docker buildx build \ - --builder desktop-linux \ - --load \ - -f madrona-portal/docker/Dockerfile \ - -t madrona-portal-app:latest \ - . -``` - -### Redeploying after code changes +# Deploy to fully containerized production environment -Then from `madrona-portal/`: - -```bash -docker compose -f docker/docker-compose.yml --env-file .env --profile full \ - up -d --force-recreate app -``` - -Add `--no-cache` to the buildx command to force a full dependency reinstall -(needed when `docker-requirements.txt` changes). - ---- +## AWS EC2 -## Everyday usage +See [AWS_DEPLOY.md](AWS_DEPLOY.md) -All `docker compose` commands below are run from **`madrona-portal/`**. - -### View logs - -```bash -docker compose -f docker/docker-compose.yml --env-file .env --profile full logs -f app -``` - -### Run a management command - -```bash -docker compose -f docker/docker-compose.yml --env-file .env --profile full \ - run --rm app python marco/manage.py -``` - -Examples: - -```bash -# Django shell -... run --rm app python marco/manage.py shell - -# Create superuser manually -... run --rm app python marco/manage.py createsuperuser - -# Load a fixture -... run --rm app python marco/manage.py loaddata /path/to/fixture.json -``` - -### Open a database shell - -```bash -docker exec -it docker-db-1 psql -U postgres wcoa_docker_db -``` +--- ---- +# Untested instructions below this line — will update after testing ## Dev infrastructure only (local Django server) From c860d77bd5e9e5effc0b22001c1604c6c9c5dc3a Mon Sep 17 00:00:00 2001 From: David Pollard Date: Wed, 22 Apr 2026 15:14:13 -0700 Subject: [PATCH 066/152] Fix mismatched app ports in prod compose configuration Co-authored-by: Copilot --- docker/docker-compose.prod.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/docker-compose.prod.yml b/docker/docker-compose.prod.yml index d30a05f..6e1cc84 100644 --- a/docker/docker-compose.prod.yml +++ b/docker/docker-compose.prod.yml @@ -53,7 +53,7 @@ services: tasks: condition: service_healthy ports: - - "${APP_PORT:-8000}:8000" + - "${APP_PORT:-8008}:8008" networks: - madronanetwork restart: unless-stopped From 944d6b8eb9c65d7514b9b97574c57d383185ad74 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Wed, 22 Apr 2026 15:16:03 -0700 Subject: [PATCH 067/152] Change location of AWS deployment guide for Madrona Portal setup and configuration Co-authored-by: Copilot --- {docker => docs}/AWS_DEPLOY.md | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) rename {docker => docs}/AWS_DEPLOY.md (99%) diff --git a/docker/AWS_DEPLOY.md b/docs/AWS_DEPLOY.md similarity index 99% rename from docker/AWS_DEPLOY.md rename to docs/AWS_DEPLOY.md index d3c780b..773e287 100644 --- a/docker/AWS_DEPLOY.md +++ b/docs/AWS_DEPLOY.md @@ -252,18 +252,6 @@ scp ubuntu@:/path/to/harvester.war \ ubuntu@:/tmp/harvester.war ``` -**On the new EC2 instance**, move them into the expected location: - -```bash -mkdir -p ~/portals/madrona-portal/docker/wars -mv /tmp/geoportal.war ~/portals/madrona-portal/docker/wars/ -mv /tmp/harvester.war ~/portals/madrona-portal/docker/wars/ -``` - -The `.env.example` defaults point to `./wars/geoportal.war` and -`./wars/harvester.war` (relative to the `docker/` directory), so these paths -will work without any further changes. - --- ## Phase 4 — Clone the Portal Configuration @@ -310,6 +298,19 @@ ls ~/portals/ # madrona-portal/ ``` +### 4.3 Move the WAR files into place + +**On the new EC2 instance**, move them into the expected location: + +```bash +mv /tmp/geoportal.war ~/portals/madrona-portal/docker/wars/ +mv /tmp/harvester.war ~/portals/madrona-portal/docker/wars/ +``` + +The `.env.example` defaults point to `./wars/geoportal.war` and +`./wars/harvester.war` (relative to the `docker/` directory), so these paths +will work without any further changes. + --- ## Phase 5 — Configure the Environment @@ -479,7 +480,7 @@ server { server_name portal.westcoastoceans.org; location / { - proxy_pass http://127.0.0.1:8000; + proxy_pass http://127.0.0.1:8008; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; From 257f0f26b85589d0ef8e1219598a641abb774300 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Wed, 22 Apr 2026 15:16:21 -0700 Subject: [PATCH 068/152] Add production environment variables to .env.example for GitHub Container Registry --- docker/.env.example | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docker/.env.example b/docker/.env.example index 964bfd4..e98b2a2 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -6,6 +6,14 @@ # Priority for every setting: env var > config.ini > built-in default # ============================================================================= +# ---------------------------------------------------------------------------- +# GitHub Container Registry (production only) +DJANGO_ENV=production + +# ---------------------------------------------------------------------------- +# Gunicorn workers (2× vCPU count; t3.large has 2 vCPUs → 4 workers) +GUNICORN_WORKERS=4 + # --------------------------------------------------------------------------- # GitHub Container Registry (production only) # GHCR_PAT: read-only fine-grained PAT used to pull the image from GHCR. From 0d2c48cc74e32e5b497f0694a1a8d859f6007f58 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Wed, 22 Apr 2026 15:18:25 -0700 Subject: [PATCH 069/152] Update smoke test URL in AWS deployment guide to reflect correct port --- docs/AWS_DEPLOY.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/AWS_DEPLOY.md b/docs/AWS_DEPLOY.md index 773e287..93678e1 100644 --- a/docs/AWS_DEPLOY.md +++ b/docs/AWS_DEPLOY.md @@ -431,7 +431,7 @@ Startup takes 2–5 minutes. Look for `Booting worker` lines from Gunicorn. ### 6.4 Smoke test ```bash -curl -I http://localhost:8000/ +curl -I http://localhost:8008/ # Expected: HTTP/1.1 200 OK (or 301/302 redirect) ``` From a256324c9352739ca41eedad33b3db797efa126a Mon Sep 17 00:00:00 2001 From: David Pollard Date: Wed, 22 Apr 2026 15:20:07 -0700 Subject: [PATCH 070/152] Fix smoke test URL in AWS deployment guide to use correct port --- docs/AWS_DEPLOY.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/AWS_DEPLOY.md b/docs/AWS_DEPLOY.md index 93678e1..cc4cb3c 100644 --- a/docs/AWS_DEPLOY.md +++ b/docs/AWS_DEPLOY.md @@ -431,7 +431,7 @@ Startup takes 2–5 minutes. Look for `Booting worker` lines from Gunicorn. ### 6.4 Smoke test ```bash -curl -I http://localhost:8008/ +curl -I http://localhost:8000 # Expected: HTTP/1.1 200 OK (or 301/302 redirect) ``` From 2d387212c8b4b647ff8653b3aff134a7e2d4146d Mon Sep 17 00:00:00 2001 From: David Pollard Date: Wed, 22 Apr 2026 15:36:33 -0700 Subject: [PATCH 071/152] Refactor db-restore.sh to improve environment detection and update usage instructions Co-authored-by: Copilot --- scripts/db-restore.sh | 73 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 64 insertions(+), 9 deletions(-) diff --git a/scripts/db-restore.sh b/scripts/db-restore.sh index dbd48b4..a0ce9ad 100755 --- a/scripts/db-restore.sh +++ b/scripts/db-restore.sh @@ -1,24 +1,35 @@ #!/usr/bin/env bash # ----------------------------------------------------------------------------- -# db-restore.sh — Restore a PostgreSQL dump into the Dockerized dev database. +# db-restore.sh — Restore a PostgreSQL dump into the Dockerized database. # # Usage: # ./scripts/db-restore.sh # ./scripts/db-restore.sh --drop # drop & recreate DB first +# ./scripts/db-restore.sh --prod # force production compose +# ./scripts/db-restore.sh --dev # force development compose # ./scripts/db-restore.sh --env-file # # Run from anywhere — this script always operates relative to madrona-portal/. # # Prerequisites: -# 1. Docker Compose stack is running from madrona-portal/docker: -# docker compose up +# 1. Docker Compose stack is running (dev or prod): +# docker compose -f docker/docker-compose.yml up # dev +# docker compose -f docker/docker-compose.prod.yml up # prod # 2. madrona-portal/docker/.env exists and contains DB_NAME, DB_USER, DB_PASSWORD. # +# Environment detection (applied in order, first match wins): +# 1. --prod / --dev CLI flag +# 2. Running Docker Compose stack detected via `docker compose ls` +# 3. DJANGO_ENV variable in the loaded .env file +# 4. Default: development +# # Options: # --drop Terminate all active connections, drop, and recreate the # target database before restoring. Required for a clean # import from prod. Without this flag the dump is applied # on top of existing data. +# --prod Force the production compose file (docker-compose.prod.yml). +# --dev Force the development compose file (docker-compose.yml). # --env-file Path to the .env file (default: ./docker/.env). # # Notes: @@ -26,7 +37,7 @@ # - psql warnings (e.g. "already exists") are normal when importing a dump # produced on a different Postgres version (12 → 16) and are not fatal. # - After a --drop restore, run migrations to pick up any schema drift: -# docker compose exec app python marco/manage.py migrate +# docker compose -f exec app python marco/manage.py migrate # ----------------------------------------------------------------------------- set -euo pipefail @@ -36,25 +47,31 @@ set -euo pipefail die() { echo "[db-restore] ERROR: $*" >&2; exit 1; } info() { echo "[db-restore] $*"; } +COMPOSE_DEV="docker/docker-compose.yml" +COMPOSE_PROD="docker/docker-compose.prod.yml" + # --------------------------------------------------------------------------- # Parse arguments # --------------------------------------------------------------------------- DROP_FIRST=false DUMP_FILE="" ENV_FILE="$(dirname "${BASH_SOURCE[0]}")/../docker/.env" +FORCE_ENV="" # "prod" | "dev" | "" (auto-detect) while [[ $# -gt 0 ]]; do case "$1" in --drop) DROP_FIRST=true; shift ;; + --prod) FORCE_ENV="prod"; shift ;; + --dev) FORCE_ENV="dev"; shift ;; --env-file) [[ -n "${2:-}" ]] || die "--env-file requires a path argument" ENV_FILE="$2"; shift 2 ;; - -*) die "Unknown option: '$1'. Usage: $0 [--drop] [--env-file ] " ;; + -*) die "Unknown option: '$1'. Usage: $0 [--drop] [--prod|--dev] [--env-file ] " ;; *) [[ -z "$DUMP_FILE" ]] || die "Unexpected argument: '$1'" DUMP_FILE="$1"; shift ;; esac done -[[ -n "$DUMP_FILE" ]] || die "Usage: $0 [--drop] [--env-file ] " +[[ -n "$DUMP_FILE" ]] || die "Usage: $0 [--drop] [--prod|--dev] [--env-file ] " # Resolve dump path before we cd away. DUMP_ABS="$(cd "$(dirname "$DUMP_FILE")" && pwd)/$(basename "$DUMP_FILE")" @@ -75,7 +92,7 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" cd "$SCRIPT_DIR/.." # --------------------------------------------------------------------------- -# Load .env for DB credentials +# Load .env for DB credentials and DJANGO_ENV # --------------------------------------------------------------------------- set -a # shellcheck source=/dev/null @@ -86,9 +103,47 @@ DB_NAME="${DB_NAME:-wcoa_docker_db}" DB_USER="${DB_USER:-postgres}" DB_PASSWORD="${DB_PASSWORD:?DB_PASSWORD must be set in .env}" -COMPOSE="docker compose -f docker/docker-compose.yml --env-file $ENV_FILE_ABS" +# --------------------------------------------------------------------------- +# Detect which compose file to target +# --------------------------------------------------------------------------- +# detect_compose_file: inspects `docker compose ls` for a running stack whose +# config file matches one of our known compose files, preferring prod. +# Falls back to DJANGO_ENV from .env, then defaults to dev. +detect_compose_file() { + # 1. CLI flag takes priority. + if [[ "$FORCE_ENV" == "prod" ]]; then + echo "$COMPOSE_PROD"; return + elif [[ "$FORCE_ENV" == "dev" ]]; then + echo "$COMPOSE_DEV"; return + fi + + # 2. Inspect running stacks. `docker compose ls` lists all projects with + # their config file paths — grep for our known filenames. + local ls_out + if ls_out=$(docker compose ls 2>/dev/null); then + if echo "$ls_out" | grep -q "docker-compose\.prod\.yml"; then + echo "$COMPOSE_PROD"; return + fi + if echo "$ls_out" | grep -q "docker-compose\.yml"; then + echo "$COMPOSE_DEV"; return + fi + fi + + # 3. Fall back to DJANGO_ENV from the loaded .env. + if [[ "${DJANGO_ENV:-}" == "production" ]]; then + echo "$COMPOSE_PROD"; return + fi + + # 4. Default to development. + echo "$COMPOSE_DEV" +} + +COMPOSE_FILE="$(detect_compose_file)" +COMPOSE="docker compose -f $COMPOSE_FILE --env-file $ENV_FILE_ABS" PSQL="$COMPOSE exec -T -e PGPASSWORD=$DB_PASSWORD db psql -U $DB_USER" +info "Using compose file: $COMPOSE_FILE" + # --------------------------------------------------------------------------- # Verify the db container is healthy before doing anything # --------------------------------------------------------------------------- @@ -132,4 +187,4 @@ info "Restore complete." info "" info "Next steps:" info " Apply any pending migrations:" -info " docker compose exec app python marco/manage.py migrate" +info " docker compose -f $COMPOSE_FILE exec app python marco/manage.py migrate" From 8c8c809125307bfa21f27e9565173d69745a4841 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Thu, 23 Apr 2026 11:45:24 -0700 Subject: [PATCH 072/152] Refactor AWS deployment guide to streamline Docker commands and add database migration steps Co-authored-by: Copilot --- docs/AWS_DEPLOY.md | 69 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 55 insertions(+), 14 deletions(-) diff --git a/docs/AWS_DEPLOY.md b/docs/AWS_DEPLOY.md index cc4cb3c..f702c2a 100644 --- a/docs/AWS_DEPLOY.md +++ b/docs/AWS_DEPLOY.md @@ -397,17 +397,13 @@ From `~/portals/madrona-portal/`: ```bash cd ~/portals/madrona-portal -docker compose -f docker/docker-compose.prod.yml \ - --env-file docker/.env \ - --profile full \ - up -d +docker compose -f docker/docker-compose.prod.yml --profile full up -d ``` ### 6.3 Watch the startup logs ```bash -docker compose -f docker/docker-compose.prod.yml --env-file docker/.env --profile full \ - logs -f app +docker compose -f docker/docker-compose.prod.yml --profile full logs -f app ``` On first boot the entrypoint automatically: @@ -422,16 +418,53 @@ Startup takes 2–5 minutes. Look for `Booting worker` lines from Gunicorn. > For a fresh database, run with `DB_INIT=1` the first time: > ```bash -> DB_INIT=1 docker compose -f docker/docker-compose.prod.yml --env-file docker/.env \ -> --profile full up -d +> DB_INIT=1 docker compose -f docker/docker-compose.prod.yml --profile full up -d > ``` > On subsequent starts, leave `DB_INIT` at its default (`0`) to skip the > fixture and superuser steps. -### 6.4 Smoke test +### Apply migrations (if needed) + +```bash +docker compose -f docker/docker-compose.prod.yml --profile full exec app python marco/manage.py migrate +``` + +### Import database + +From `madrona-portal/docker`: +```bash +../scripts/db-restore.sh --drop +``` +*example:* +```bash +../scripts/db-restore.sh --drop ../../madrona-apps/wcoa/wcodp_prod_dump_20260320.sql +``` + +### Apply migrations again (if needed) + +```bash +docker compose -f docker/docker-compose.prod.yml --profile full exec app python marco/manage.py migrate +``` + +### Migration to Layers ```bash -curl -I http://localhost:8000 +docker compose -f docker/docker-compose.prod.yml --profile full exec app python marco/manage.py migration_to_layers +``` + + + +### Collect static and compress assets (if needed) + +```bash +docker compose -f docker/docker-compose.prod.yml --profile full exec app python marco/manage.py collectstatic --noinput +docker compose -f docker/docker-compose.prod.yml --profile full exec app python marco/manage.py compress +``` + +### Smoke test + +```bash +curl -I http://localhost:8000/ # Expected: HTTP/1.1 200 OK (or 301/302 redirect) ``` @@ -477,10 +510,10 @@ Paste: ```nginx server { listen 80; - server_name portal.westcoastoceans.org; + server_name or ; location / { - proxy_pass http://127.0.0.1:8008; + proxy_pass http://127.0.0.1:8000; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; @@ -521,8 +554,16 @@ Restart the app container to pick up the change: ```bash cd ~/portals/madrona-portal -docker compose -f docker/docker-compose.prod.yml --env-file docker/.env --profile full \ - up -d --force-recreate app +docker compose -f docker/docker-compose.prod.yml --profile full up -d --force-recreate app +``` + +--- + +## Import media + +#### Copy the media files +```bash +scp -r {your_media_dir} ubuntu@:/home/ubuntu/portals/madrona-portal/docker/media/ ``` --- From c10d4e1077f346568e9cf5b4df83f341603dfa40 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Thu, 23 Apr 2026 11:45:42 -0700 Subject: [PATCH 073/152] Remove services and ports section from Docker README --- docker/README.md | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/docker/README.md b/docker/README.md index 835649c..a095d2c 100644 --- a/docker/README.md +++ b/docker/README.md @@ -303,16 +303,6 @@ migrations and reload fixtures from scratch. --- -## Services and ports - -| Service | Image | Default host port | Override via | -|---|---|---|---| -| `app` | `madrona-portal-app:latest` | `8000` | `APP_PORT` in `.env` | -| `db` | `postgis/postgis:16-3.4` | `5432` | `DB_PORT` in `.env` | -| `tasks` | `redis:7-alpine` | `6379` | `REDIS_PORT` in `.env` | - ---- - ## Disk space Docker's build cache can grow large over time: From ebe71763a9d12f8746e9da0e59dae711bd147039 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Thu, 23 Apr 2026 16:36:49 -0700 Subject: [PATCH 074/152] Enhance AWS deployment guide with Nginx logging, CORS configuration, and GeoPortal migration instructions Co-authored-by: Copilot --- docs/AWS_DEPLOY.md | 160 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 160 insertions(+) diff --git a/docs/AWS_DEPLOY.md b/docs/AWS_DEPLOY.md index f702c2a..c975ebf 100644 --- a/docs/AWS_DEPLOY.md +++ b/docs/AWS_DEPLOY.md @@ -512,6 +512,9 @@ server { listen 80; server_name or ; + access_log /var/log/nginx/wcoa.access.log; + error_log /var/log/nginx/wcoa.error.log; + location / { proxy_pass http://127.0.0.1:8000; proxy_set_header Host $host; @@ -521,6 +524,69 @@ server { proxy_read_timeout 120s; client_max_body_size 50M; } + + location /geospatial/ { + alias /var/www/html/geospatial/; + autoindex on; + } + + location /munin/static/ { + alias /etc/munin/static/; + } + + location /munin { + alias /var/cache/munin/www; + } + + # Shared CORS policy for these proxied endpoints + # (if you only want specific origins, replace "*" with your domain) + set $cors_allow_origin "*"; + + location ~ ^/(?:_search/|_doc/|metadata).*$ { + proxy_pass http://:9200; + proxy_redirect off; + proxy_connect_timeout 5s; + proxy_read_timeout 60s; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + add_header Access-Control-Allow-Origin $cors_allow_origin always; + add_header Access-Control-Allow-Methods "GET, POST, OPTIONS" always; + add_header Access-Control-Allow-Headers "DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range" always; + add_header Access-Control-Expose-Headers "Content-Length,Content-Range" always; + + if ($request_method = OPTIONS) { + add_header Access-Control-Max-Age 1728000 always; + add_header Content-Type "text/plain; charset=utf-8" always; + add_header Content-Length 0 always; + return 204; + } + } + + location ~ ^/(?:manager|host-manager|semantix|solr|gc|geoportal|harvester).*$ { + proxy_pass http://:8080; + proxy_redirect off; + proxy_connect_timeout 5s; + proxy_read_timeout 60s; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + add_header Access-Control-Allow-Origin $cors_allow_origin always; + add_header Access-Control-Allow-Methods "GET, POST, OPTIONS" always; + add_header Access-Control-Allow-Headers "DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range" always; + add_header Access-Control-Expose-Headers "Content-Length,Content-Range" always; + + if ($request_method = OPTIONS) { + add_header Access-Control-Max-Age 1728000 always; + add_header Content-Type "text/plain; charset=utf-8" always; + add_header Content-Length 0 always; + return 204; + } + } } ``` @@ -566,6 +632,13 @@ docker compose -f docker/docker-compose.prod.yml --profile full up -d --force-re scp -r {your_media_dir} ubuntu@:/home/ubuntu/portals/madrona-portal/docker/media/ ``` +## Migrate existing GeoPortal records + +Once the stack is running, you can migrate existing GeoPortal records with the following command (replace the source host, username, and password with your old GeoPortal's Elasticsearch credentials): +```bash +time curl -X POST "http://52.33.200.130:9200/_reindex" -H 'Content-Type: application/json' -d'{"conflicts": "proceed", "max_docs": 51000, "source": {"remote": { "host":"http://elastic.prod.wcoa.ecotrust.org:80/geoportal/elastic/", "username": "[[USERNAME]]", "password": "[[PASSWORD]]" }, "index": "metadata", "query": {"match_all": {} }, "size": 100 }, "dest": { "index": "metadata" } }'' +``` + --- ## Phase 8 — Keep the Stack Running Across Reboots @@ -620,6 +693,92 @@ docker compose -f docker/docker-compose.prod.yml --env-file docker/.env --profil ps # all services should show "running" ``` +--- + +## AWS Email (SES) Setup + +### Verify your domain in AWS SES (Simple Email Service) + +1. Go to SES Console → us-west-2 → Create identity → choose Domain → enter: +`portal.westcoastoceans.org` +2. Leave defaults +3. Click "Create identity" + +### Add the provided DNS records to your DNS provider +1. In the SES Console, click on your new identity → DNS records tab +2. Add the provided records to your DNS provider (e.g., Hover) +3. Wait for AWS to verify the domain (minute to hours) + +### Create SMTP Credentials +1. In SES Console → SMTP Settings → Create SMTP credentials → note the username and password (only shown once). + +### Update the portal configuration +1. SSH into the server +2. Open the `.env` file +3. Add the following values (replace with your SMTP credentials): +``` +EMAIL_HOST=email-smtp.us-west-2.amazonaws.com +EMAIL_PORT=587 +EMAIL_HOST_USER= +EMAIL_HOST_PASSWORD= +EMAIL_USE_TLS=true +DEFAULT_FROM_EMAIL=noreply@prod.mail.ecotrust.org +``` + +### Request SES production access +Submit a production access request in SES Console → Account dashboard → Request production access. Takes 24hrs typically. + +--- + +## Automatic Security Updates + +Install unattended-upgrades and update-notifier-common to automatically apply security updates to the server OS: +```bash +sudo apt-get install unattended-upgrades update-notifier-common -y +``` + +Enable automatic updates: +```bash +sudo dpkg-reconfigure --priority=low unattended-upgrades +``` + +Edit the configuration to allow automatic reboots and set the time for reboots to occur: +```bash +// Open the config file +sudo vim /etc/apt/apt.conf.d/50unattended-upgrades + +// Find, uncomment, and set "true" the line that contains "Unattended-Upgrade::Automatic-Reboot" +Unattended-Upgrade::Automatic-Reboot "true"; + +// Find and uncomment the line that contains "Unattended-Upgrade::Automatic-Reboot-Time" +Unattended-Upgrade::Automatic-Reboot-Time "02:00"; +``` + +--- + +## Install Munin + +```bash +sudo apt-get install munin -y +``` + +--- + +## Set Up Swap Space + +```bash +sudo fallocate -l 1G /swapfile +sudo chmod 600 /swapfile +sudo mkswap /swapfile +sudo swapon /swapfile +sudo cp /etc/fstab /etc/fstab.bak +echo '/swapfile none swap sw 0 0' | sudo tee -a /etc/fstab +``` + +--- + + + --- ## Deploying a New Release @@ -627,6 +786,7 @@ docker compose -f docker/docker-compose.prod.yml --env-file docker/.env --profil When code changes are merged to the `docker` branch, GitHub Actions automatically builds and pushes a new image to GHCR. To deploy it: + ### On the server ```bash From 727eff1ca654532626aad14c6175b4c2ab90a7fc Mon Sep 17 00:00:00 2001 From: David Pollard Date: Thu, 23 Apr 2026 16:54:48 -0700 Subject: [PATCH 075/152] Add Google Analytics configuration and custom mail domain setup instructions to deployment guide Co-authored-by: Copilot --- docker/.env.example | 5 +++++ docs/AWS_DEPLOY.md | 14 +++++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/docker/.env.example b/docker/.env.example index e98b2a2..5a7bb04 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -67,6 +67,11 @@ AWS_SECRET_ACCESS_KEY= AWS_SES_REGION_NAME=us-east-1 AWS_SES_REGION_ENDPOINT=email.us-east-1.amazonaws.com +# --------------------------------------------------------------------------- +# Google Analytics +# --------------------------------------------------------------------------- +GA_ACCOUNT=G-XXXXXXXXXX + # --------------------------------------------------------------------------- # Social Auth OAuth keys # --------------------------------------------------------------------------- diff --git a/docs/AWS_DEPLOY.md b/docs/AWS_DEPLOY.md index c975ebf..8cb0384 100644 --- a/docs/AWS_DEPLOY.md +++ b/docs/AWS_DEPLOY.md @@ -725,6 +725,12 @@ EMAIL_USE_TLS=true DEFAULT_FROM_EMAIL=noreply@prod.mail.ecotrust.org ``` +### Create custom mail from domain +1. In SES Console → Domains → click on your domain → Create mail from domain +2. Enter a subdomain (e.g., `mail`) → Create +3. Add the provided DNS records to your DNS provider +4. Wait for AWS to verify the mail from domain + ### Request SES production access Submit a production access request in SES Console → Account dashboard → Request production access. Takes 24hrs typically. @@ -777,7 +783,13 @@ echo '/swapfile none swap sw 0 0' | sudo tee -a /etc/fstab --- - +## Add Google Analytics Key +1. Get the GA tracking ID (e.g., `G-XXXXXXXXXX`) +2. SSH into the server and edit the `.env` file +3. Edit or add the following line in the `.env` file: +```bash +GA_ACCOUNT=G-XXXXXXXXXX +``` --- From 5425aa0a9bb71e6fca4688e9c3e24308fe81d62e Mon Sep 17 00:00:00 2001 From: David Pollard Date: Fri, 24 Apr 2026 15:16:02 -0700 Subject: [PATCH 076/152] Update GeoPortal migration instructions to include environment setup and reindexing steps Co-authored-by: Copilot --- docs/AWS_DEPLOY.md | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/docs/AWS_DEPLOY.md b/docs/AWS_DEPLOY.md index 8cb0384..696144c 100644 --- a/docs/AWS_DEPLOY.md +++ b/docs/AWS_DEPLOY.md @@ -634,9 +634,27 @@ scp -r {your_media_dir} ubuntu@:/home/ubuntu/portals/madrona-por ## Migrate existing GeoPortal records -Once the stack is running, you can migrate existing GeoPortal records with the following command (replace the source host, username, and password with your old GeoPortal's Elasticsearch credentials): +1. Update .env with the reindex remote whitelist and port. You can find this info by looking at the old server's configuration or by asking the previous admin. + +2. Edit the hosts file to allow the server to resolve the old Elastic IP of the GeoPortal instance to the new internal Docker network: + +```bash +sudo vim /etc/hosts +# Add the following line, replacing and : + +``` + +3. Ensure the app is running, you can migrate existing GeoPortal records with the following command (replace the username and password): + +```bash +time curl -X POST "http://localhost:9200/_reindex" -H 'Content-Type: application/json' -d'{"conflicts": "proceed", "max_docs": 51000, "source": {"remote": { "host":"http://elastic.prod.wcoa.ecotrust.org:80/geoportal/elastic/", "username": "[[USERNAME]]", "password": "[[PASSWORD]]" }, "index": "metadata", "size": 100 }, "dest": { "index": "metadata" } }' +``` + +4. Do a down, including volumes, and up for the elasticsearch container and geoportal to pick up the new records: + ```bash -time curl -X POST "http://52.33.200.130:9200/_reindex" -H 'Content-Type: application/json' -d'{"conflicts": "proceed", "max_docs": 51000, "source": {"remote": { "host":"http://elastic.prod.wcoa.ecotrust.org:80/geoportal/elastic/", "username": "[[USERNAME]]", "password": "[[PASSWORD]]" }, "index": "metadata", "query": {"match_all": {} }, "size": 100 }, "dest": { "index": "metadata" } }'' +docker compose -f docker/docker-compose.prod.yml down elasticsearch geoportal -v +docker compose -f docker/docker-compose.prod.yml up -d elasticsearch geoportal ``` --- From a72d9b93038632c017a4b0abadb9bc4dac8c875f Mon Sep 17 00:00:00 2001 From: David Pollard Date: Fri, 24 Apr 2026 15:31:36 -0700 Subject: [PATCH 077/152] Refactor db_dump.sh to enhance error handling, support custom Docker compose and environment files, and improve usage instructions Co-authored-by: Copilot --- backups/db_dump.sh | 110 +++++++++++++++++++++++++++++++++++++-------- 1 file changed, 92 insertions(+), 18 deletions(-) diff --git a/backups/db_dump.sh b/backups/db_dump.sh index b295029..9339cdd 100755 --- a/backups/db_dump.sh +++ b/backups/db_dump.sh @@ -1,21 +1,95 @@ -#!/bin/bash - -DBNAME=dbname -DBOWNER=dbowner -DBPASSWORD=password -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -OUTFILE=$DBNAME'_dump.sql' -OUTDIR=$DIR - -while getopts n:o:p:d:f: flag -do - case "${flag}" in - n) DBNAME=${OPTARG};; # (n)ame of the database - o) DBOWNER=${OPTARG};; # Database (o)wner - p) DBPASSWORD=${OPTARG};; # Database owner's (p)assword - d) OUTDIR=${OPTARG};; # Output dump file (d)irectory - f) OUTFILE=${OPTARG};; # Output dump (f)ilename +#!/usr/bin/env bash + +set -euo pipefail + +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +ROOT_DIR="$(cd "$DIR/.." >/dev/null 2>&1 && pwd)" + +COMPOSE_FILE="$ROOT_DIR/docker/docker-compose.prod.yml" +ENV_FILE="$ROOT_DIR/docker/.env" +SERVICE_NAME="db" + +DBNAME="" +DBOWNER="" +DBPASSWORD="" +OUTDIR="$DIR" +OUTFILE="" + +usage() { + cat < Database name (default: DB_NAME from env file) + -o Database user (default: DB_USER from env file) + -p Database password (default: DB_PASSWORD from env file) + -d Output directory (default: backups/) + -f Output filename (default: _dump_.sql) + -c Docker compose file path + -e Environment file path + -s Docker service name (default: db) + -h Show this help +EOF +} + +while getopts ":n:o:p:d:f:c:e:s:h" flag; do + case "$flag" in + n) DBNAME="$OPTARG" ;; + o) DBOWNER="$OPTARG" ;; + p) DBPASSWORD="$OPTARG" ;; + d) OUTDIR="$OPTARG" ;; + f) OUTFILE="$OPTARG" ;; + c) COMPOSE_FILE="$OPTARG" ;; + e) ENV_FILE="$OPTARG" ;; + s) SERVICE_NAME="$OPTARG" ;; + h) + usage + exit 0 + ;; + :) echo "Error: Option -$OPTARG requires an argument." >&2; usage; exit 1 ;; + \?) echo "Error: Invalid option -$OPTARG" >&2; usage; exit 1 ;; esac done -PGPASSWORD=$DBPASSWORD /usr/bin/pg_dump -b -c -n public -O --quote-all-identifiers --no-acl -w -U $DBOWNER -f $OUTDIR/$OUTFILE $DBNAME +if [[ ! -f "$COMPOSE_FILE" ]]; then + echo "Error: Docker compose file not found at $COMPOSE_FILE" >&2 + exit 1 +fi + +if [[ -f "$ENV_FILE" ]]; then + set -a + # shellcheck disable=SC1090 + source "$ENV_FILE" + set +a +fi + +DBNAME="${DBNAME:-${DB_NAME:-}}" +DBOWNER="${DBOWNER:-${DB_USER:-}}" +DBPASSWORD="${DBPASSWORD:-${DB_PASSWORD:-}}" + +if [[ -z "$DBNAME" || -z "$DBOWNER" || -z "$DBPASSWORD" ]]; then + echo "Error: DB credentials are incomplete. Provide -n/-o/-p or set DB_NAME/DB_USER/DB_PASSWORD in env file." >&2 + exit 1 +fi + +mkdir -p "$OUTDIR" + +if [[ -z "$OUTFILE" ]]; then + OUTFILE="${DBNAME}_dump_$(date +%F_%H-%M-%S).sql" +fi + +OUTPATH="$OUTDIR/$OUTFILE" + +CONTAINER_ID="$(docker compose -f "$COMPOSE_FILE" --env-file "$ENV_FILE" ps -q "$SERVICE_NAME")" + +if [[ -z "$CONTAINER_ID" ]]; then + echo "Error: Service '$SERVICE_NAME' is not running." >&2 + exit 1 +fi + +docker compose -f "$COMPOSE_FILE" --env-file "$ENV_FILE" exec -T \ + -e PGPASSWORD="$DBPASSWORD" \ + "$SERVICE_NAME" \ + pg_dump -b -c -n public -O --quote-all-identifiers --no-acl -w -U "$DBOWNER" -d "$DBNAME" > "$OUTPATH" + +echo "Database dump created: $OUTPATH" From 5eed7806fc0020da95ff2f9a37971a8f60682a13 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Mon, 27 Apr 2026 15:46:13 -0700 Subject: [PATCH 078/152] Add mp-survey integration to Docker setup and update documentation Co-authored-by: Copilot --- .github/workflows/create-and-publish-docker-images.yml | 7 +++++++ docker/Dockerfile | 1 + docker/README.md | 3 ++- docker/docker-requirements.txt | 1 + marco/marco/settings.py | 1 + 5 files changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/create-and-publish-docker-images.yml b/.github/workflows/create-and-publish-docker-images.yml index 60eb47a..01f46f9 100644 --- a/.github/workflows/create-and-publish-docker-images.yml +++ b/.github/workflows/create-and-publish-docker-images.yml @@ -127,6 +127,13 @@ jobs: token: ${{ secrets.GH_PAT }} path: madrona-apps/mp-proxy + - name: Checkout mp-survey + uses: actions/checkout@v5 + with: + repository: Ecotrust/mp-survey + token: ${{ secrets.GH_PAT }} + path: madrona-apps/mp-survey + - name: Checkout mp-visualize uses: actions/checkout@v5 with: diff --git a/docker/Dockerfile b/docker/Dockerfile index 031d749..669d808 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -61,6 +61,7 @@ COPY madrona-apps/mp-explore ./apps/mp-explore COPY madrona-apps/mp-layers ./apps/mp-layers COPY madrona-apps/mp-map-groups ./apps/mp-map-groups COPY madrona-apps/mp-proxy ./apps/mp-proxy +COPY madrona-apps/mp-survey ./apps/mp-survey COPY madrona-apps/mp-visualize ./apps/mp-visualize COPY madrona-apps/p97-nursery ./apps/p97-nursery COPY madrona-apps/wcoa ./apps/wcoa diff --git a/docker/README.md b/docker/README.md index a095d2c..3540319 100644 --- a/docker/README.md +++ b/docker/README.md @@ -40,6 +40,7 @@ git clone https://github.com/Ecotrust/mp-explore.git git clone https://github.com/Ecotrust/mp-layers.git git clone https://github.com/Ecotrust/mp-map-groups.git git clone https://github.com/Ecotrust/mp-proxy.git +git clone https://github.com/Ecotrust/mp-survey.git git clone https://github.com/Ecotrust/mp-visualize.git git clone https://github.com/Ecotrust/p97-nursery.git git clone -b vagrant2docker https://github.com/Ecotrust/wcoa.git @@ -110,7 +111,7 @@ docker buildx build --builder desktop-linux --load -f ./Dockerfile ../../ docker buildx build --load -f ./Dockerfile ../../ ``` -When building a tagged image for deployment, add `-t madrona-portal-app:latest`: +If you want to build a tagged image, add `-t madrona-portal-app:latest`: ``` docker buildx build \ --builder desktop-linux \ diff --git a/docker/docker-requirements.txt b/docker/docker-requirements.txt index 7837218..70cccac 100644 --- a/docker/docker-requirements.txt +++ b/docker/docker-requirements.txt @@ -92,6 +92,7 @@ djangorestframework>=3.14,<4.0 -e /usr/local/apps/madrona-portal/apps/mp-map-groups -e /usr/local/apps/madrona-portal/apps/p97-nursery -e /usr/local/apps/madrona-portal/apps/mp-proxy +-e /usr/local/apps/madrona-portal/apps/mp-survey # --------------------------------------------------------------------------- # Portal variant — choose one: diff --git a/marco/marco/settings.py b/marco/marco/settings.py index 2b858e7..a813eb6 100644 --- a/marco/marco/settings.py +++ b/marco/marco/settings.py @@ -218,6 +218,7 @@ def _parse_hosts(raw: str | None) -> list[str]: 'accounts.apps.AccountsAppConfig', 'django_social_share', 'mapgroups', + 'survey', ] # Optional apps — installed when available From d065b2a3092824a3ddb31f23f5cf606ccd2c8b1a Mon Sep 17 00:00:00 2001 From: David Pollard Date: Mon, 27 Apr 2026 16:22:25 -0700 Subject: [PATCH 079/152] Add mp-survey service to Docker Compose configuration Co-authored-by: Copilot --- docker/docker-compose.dev.yml | 1 + docker/docker-compose.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml index 1abca59..b0955fb 100644 --- a/docker/docker-compose.dev.yml +++ b/docker/docker-compose.dev.yml @@ -50,3 +50,4 @@ services: - ../../madrona-apps/p97-nursery:/usr/local/apps/madrona-portal/apps/p97-nursery - ../../madrona-apps/django_url_shortener:/usr/local/apps/madrona-portal/apps/django_url_shortener - ../../madrona-apps/madrona-analysistools:/usr/local/apps/madrona-portal/apps/madrona-analysistools + - ../../madrona-apps/mp-survey:/usr/local/apps/madrona-portal/apps/mp-survey diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 98c3256..0475231 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -42,6 +42,7 @@ services: - ../../madrona-apps/p97-nursery:/usr/local/apps/madrona-portal/apps/p97-nursery - ../../madrona-apps/django_url_shortener:/usr/local/apps/madrona-portal/apps/django_url_shortener - ../../madrona-apps/madrona-analysistools:/usr/local/apps/madrona-portal/apps/madrona-analysistools + - ../../madrona-apps/mp-survey:/usr/local/apps/madrona-portal/apps/mp-survey env_file: - ./.env # load all secrets from the docker/.env file environment: From 0514f1c1266702017a92f04da309d1ceb8d293fb Mon Sep 17 00:00:00 2001 From: David Pollard Date: Mon, 27 Apr 2026 17:07:37 -0700 Subject: [PATCH 080/152] Update Docker build instructions to simplify commands and improve clarity for Mac and Linux users Co-authored-by: Copilot --- docker/README.md | 39 ++++++++++----------------------------- 1 file changed, 10 insertions(+), 29 deletions(-) diff --git a/docker/README.md b/docker/README.md index 3540319..363f5bc 100644 --- a/docker/README.md +++ b/docker/README.md @@ -97,8 +97,7 @@ CELERY_BROKER_URL = redis://tasks:6379/0 Run this command from `madrona-portal/docker` (the previous step leaves you in `madrona-portal/marco`, so `cd ../docker` gets you there). The -Docker build context for this command is `../../`, which resolves to the -parent workspace directory `portals/` that contains both +Docker build context for this command is `../../`, which resolves to the parent workspace directory `portals/` that contains both `madrona-portal/` and `madrona-apps/`. If running this build from a MAC, add `--builder desktop-linux` to the `buildx build` command. @@ -106,23 +105,12 @@ If running this build from a MAC, add `--builder desktop-linux` to the `buildx b ```bash cd ../docker # MAC OS -docker buildx build --builder desktop-linux --load -f ./Dockerfile ../../ +docker compose build --no-cache app # LINUX -docker buildx build --load -f ./Dockerfile ../../ +docker compose build --no-cache app ``` -If you want to build a tagged image, add `-t madrona-portal-app:latest`: -``` -docker buildx build \ - --builder desktop-linux \ - --load \ - -f ./Dockerfile \ - -t madrona-portal-app:latest \ - ../../ -``` - -This takes several minutes on a first build (compiling GDAL, installing -Python packages). Subsequent builds are fast thanks to layer caching. +Previously we recommended `docker buildx build --builder desktop-linux --no-cache --load -f ./Dockerfile ../../` for all platforms, but BuildKit caching on Linux does not appear to have the same git object store issue as on Mac, so the simpler `docker compose build --no-cache app` is sufficient on Linux. > **Why `docker buildx build` and not `docker compose build`?** > `docker compose build` has a caching bug: when a `.git` directory exists @@ -269,27 +257,20 @@ python manage.py runserver > **Commit first.** BuildKit reads from the git object store — uncommitted > changes are invisible to the build. -From the **workspace root** (`portals/`): +From the docker directory (`madrona-portal/docker`): ```bash -docker buildx build \ - --builder desktop-linux \ - --load \ - -f madrona-portal/Dockerfile \ - -t madrona-portal-app:latest \ - . +docker buildx build --builder desktop-linux --no-cache --load -f ./Dockerfile ../../ ``` -Then from `madrona-portal/`: +| Note on `--no-cache`: Use it when dependencies have changed; without it, Docker reuses the cached pip install layer (needed when `docker-requirements.txt` changes). + +Then bring the stack up: ```bash -docker compose -f docker/docker-compose.yml --env-file .env --profile full \ - up -d --force-recreate app +docker compose up --force-recreate ``` -Add `--no-cache` to the buildx command to force a full dependency reinstall -(needed when `docker-requirements.txt` changes). - --- ## Reset to a clean state From 3f9aed839db0ff3a3c4fdefa17ae6aee873975cd Mon Sep 17 00:00:00 2001 From: David Pollard Date: Mon, 27 Apr 2026 17:09:17 -0700 Subject: [PATCH 081/152] Update Docker README with simplified local development instructions and rebuild commands Co-authored-by: Copilot --- docker/README.md | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/docker/README.md b/docker/README.md index 363f5bc..64a5186 100644 --- a/docker/README.md +++ b/docker/README.md @@ -235,15 +235,13 @@ See [AWS_DEPLOY.md](AWS_DEPLOY.md) --- -# Untested instructions below this line — will update after testing - ## Dev infrastructure only (local Django server) To run Django locally against Docker-managed PostGIS and Redis (no app container): ```bash -# Start only db and tasks (omit --profile full) -docker compose -f docker/docker-compose.yml --env-file .env up -d +# Start only db and tasks +docker compose up -d db tasks # Then in a separate terminal, from madrona-portal/: cd marco @@ -254,13 +252,10 @@ python manage.py runserver ## Rebuilding after code changes -> **Commit first.** BuildKit reads from the git object store — uncommitted -> changes are invisible to the build. - From the docker directory (`madrona-portal/docker`): ```bash -docker buildx build --builder desktop-linux --no-cache --load -f ./Dockerfile ../../ +docker compose build --no-cache app ``` | Note on `--no-cache`: Use it when dependencies have changed; without it, Docker reuses the cached pip install layer (needed when `docker-requirements.txt` changes). From d05fde7e6d2a28f5db97da884ce4822206681b51 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Thu, 30 Apr 2026 11:46:48 -0700 Subject: [PATCH 082/152] Remove polyfill script for compatibility with older environments in extra_js.html --- .../portal/ocean_stories/templates/ocean_stories/extra_js.html | 2 -- 1 file changed, 2 deletions(-) diff --git a/marco/portal/ocean_stories/templates/ocean_stories/extra_js.html b/marco/portal/ocean_stories/templates/ocean_stories/extra_js.html index bc8f47b..ebec7d1 100644 --- a/marco/portal/ocean_stories/templates/ocean_stories/extra_js.html +++ b/marco/portal/ocean_stories/templates/ocean_stories/extra_js.html @@ -27,8 +27,6 @@ {% if MAP_LIBRARY == 'ol8' %} - - {% endif %} From b0bcc9d824ed5f739e18f7e813e8c1e8b7ce2b1f Mon Sep 17 00:00:00 2001 From: David Pollard Date: Thu, 30 Apr 2026 12:40:54 -0700 Subject: [PATCH 083/152] Add NATIVE_LAND_API_KEY to environment variables in .env.example --- docker/.env.example | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docker/.env.example b/docker/.env.example index 5a7bb04..b342c8f 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -72,6 +72,11 @@ AWS_SES_REGION_ENDPOINT=email.us-east-1.amazonaws.com # --------------------------------------------------------------------------- GA_ACCOUNT=G-XXXXXXXXXX +# --------------------------------------------------------------------------- +# Native Lands +# --------------------------------------------------------------------------- +NATIVE_LAND_API_KEY= + # --------------------------------------------------------------------------- # Social Auth OAuth keys # --------------------------------------------------------------------------- From 262d399d57d2cdc77fcc532b8dc1826fbc7f166e Mon Sep 17 00:00:00 2001 From: David Pollard Date: Thu, 30 Apr 2026 12:41:00 -0700 Subject: [PATCH 084/152] Add instructions for migrating existing GeoPortal records to README --- docker/README.md | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/docker/README.md b/docker/README.md index 64a5186..2599449 100644 --- a/docker/README.md +++ b/docker/README.md @@ -227,6 +227,40 @@ cp -r {your_media_dir}/* ./media/ --- +## Migrate existing GeoPortal records + +1. Update .env with the reindex remote whitelist and port. You can find this info by looking at the old server's configuration or by asking the previous admin. + +2. Edit the hosts file to allow the server to resolve the old Elastic IP of the GeoPortal instance to the new internal Docker network: + +```bash +sudo vim /etc/hosts +# Add the following line, replacing and : + +``` + +3. Restart the elastic container to apply the .env and hosts file change: + +```bash +docker compose down -v elastic +docker compose up -d elastic +``` + +4. Ensure the app is running, you can migrate existing GeoPortal records with the following command (replace the username and password): + +```bash +time curl -X POST "http://localhost:9200/_reindex" -H 'Content-Type: application/json' -d'{"conflicts": "proceed", "max_docs": 51000, "source": {"remote": { "host":"http://elastic.prod.wcoa.ecotrust.org:80/geoportal/elastic/", "username": "[[USERNAME]]", "password": "[[PASSWORD]]" }, "index": "metadata", "size": 100 }, "dest": { "index": "metadata" } }' +``` + +1. Do a down, including volumes, and up for the geoportal container to pick up the new records: + +```bash +docker compose down -v geoportal +docker compose up -d geoportal +``` + +--- + # Deploy to fully containerized production environment ## AWS EC2 From 29f53f2ad49af4b08e2bf2ae358ca4da36bde859 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Thu, 30 Apr 2026 12:41:19 -0700 Subject: [PATCH 085/152] Add NATIVE_LAND_API_KEY to settings for environment variable configuration --- marco/marco/settings.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/marco/marco/settings.py b/marco/marco/settings.py index a813eb6..b797259 100644 --- a/marco/marco/settings.py +++ b/marco/marco/settings.py @@ -580,6 +580,11 @@ def _parse_hosts(raw: str | None) -> list[str]: # --------------------------------------------------------------------------- GA_ACCOUNT = app_cfg.get('GA_ACCOUNT', '') +# --------------------------------------------------------------------------- +# NATIVE LANDS API KEY +# --------------------------------------------------------------------------- +NATIVE_LAND_API_KEY = _env('NATIVE_LAND_API_KEY', app_cfg, 'NATIVE_LAND_API_KEY', '') + # --------------------------------------------------------------------------- # Project-level settings overrides # (Optional app + settings file specified in config.ini) From 65d1d67c42aa4c9f93b0053c9a3a9613405d6328 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Fri, 1 May 2026 10:13:25 -0700 Subject: [PATCH 086/152] Update AWS deployment instructions to use 'apt' instead of 'apt-get' for package management --- docs/AWS_DEPLOY.md | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/docs/AWS_DEPLOY.md b/docs/AWS_DEPLOY.md index 696144c..9a9ad02 100644 --- a/docs/AWS_DEPLOY.md +++ b/docs/AWS_DEPLOY.md @@ -170,7 +170,7 @@ ssh -i ~/.ssh/madrona-portal.pem ubuntu@ ### 2.2 Update the system ```bash -sudo apt-get update && sudo apt-get upgrade -y +sudo apt update && sudo apt upgrade -y ``` ### 2.3 Install Docker Engine @@ -179,26 +179,26 @@ AWS's Ubuntu AMI does not include Docker. Install the official Docker Engine (not the snap package — it has permission issues with volumes). ```bash -# Install prerequisites -sudo apt-get install -y ca-certificates curl gnupg - # Add Docker's official GPG key +sudo apt install ca-certificates curl sudo install -m 0755 -d /etc/apt/keyrings -curl -fsSL https://download.docker.com/linux/ubuntu/gpg \ - | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg -sudo chmod a+r /etc/apt/keyrings/docker.gpg +sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc +sudo chmod a+r /etc/apt/keyrings/docker.asc + +# Add the repository to Apt sources: +sudo tee /etc/apt/sources.list.d/docker.sources < /dev/null +sudo apt update # Install Docker Engine + Compose plugin + BuildKit -sudo apt-get update -sudo apt-get install -y docker-ce docker-ce-cli containerd.io \ - docker-buildx-plugin docker-compose-plugin +sudo apt install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin ``` ### 2.4 Allow your user to run Docker without sudo From c11adba7b7a4250a9722a277d09d1939319e965f Mon Sep 17 00:00:00 2001 From: David Pollard Date: Fri, 1 May 2026 10:35:50 -0700 Subject: [PATCH 087/152] Update environment variable documentation and add Docker service verification steps --- docker/.env.example | 5 ++++- docs/AWS_DEPLOY.md | 13 ++++++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/docker/.env.example b/docker/.env.example index b342c8f..86404b6 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -7,11 +7,14 @@ # ============================================================================= # ---------------------------------------------------------------------------- -# GitHub Container Registry (production only) +# Django environment +# production, development +# ---------------------------------------------------------------------------- DJANGO_ENV=production # ---------------------------------------------------------------------------- # Gunicorn workers (2× vCPU count; t3.large has 2 vCPUs → 4 workers) +# ---------------------------------------------------------------------------- GUNICORN_WORKERS=4 # --------------------------------------------------------------------------- diff --git a/docs/AWS_DEPLOY.md b/docs/AWS_DEPLOY.md index 9a9ad02..778bd4f 100644 --- a/docs/AWS_DEPLOY.md +++ b/docs/AWS_DEPLOY.md @@ -201,6 +201,15 @@ sudo apt update sudo apt install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin ``` +*Verify Docker is running:* + +```bash +sudo systemctl status docker + +# if not running, start the service +sudo systemctl start docker +``` + ### 2.4 Allow your user to run Docker without sudo ```bash @@ -337,11 +346,13 @@ nano docker/.env Set these values at minimum: ```ini +# Environment +DJANGO_ENV=production + # Django SECRET_KEY= ALLOWED_HOSTS=,localhost DEBUG=False -DJANGO_ENV=production # Database DB_PASSWORD= From 993967bac8be8718ed223afa81e4dfd4ea00affb Mon Sep 17 00:00:00 2001 From: David Pollard Date: Fri, 1 May 2026 11:31:15 -0700 Subject: [PATCH 088/152] Remove 'full' profile from app, geoportal, and elastic services in Docker Compose --- docker/docker-compose.prod.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/docker/docker-compose.prod.yml b/docker/docker-compose.prod.yml index 6e1cc84..86a85c8 100644 --- a/docker/docker-compose.prod.yml +++ b/docker/docker-compose.prod.yml @@ -57,7 +57,6 @@ services: networks: - madronanetwork restart: unless-stopped - profiles: [full] db: image: postgis/postgis:16-3.4 @@ -113,7 +112,6 @@ services: depends_on: elastic: condition: service_healthy - profiles: [full] elastic: image: elasticsearch:8.19.12 @@ -138,7 +136,6 @@ services: timeout: 10s retries: 10 restart: always - profiles: [full] volumes: static_data: From df50bdb58aca62f41625cc9a47dffa365a9247c1 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Fri, 1 May 2026 15:30:13 -0700 Subject: [PATCH 089/152] Update README and AWS deployment instructions for database restoration and environment variables --- docker/README.md | 17 +++++++---- docs/AWS_DEPLOY.md | 71 ++++++++++++++++++++++++++++++++++++---------- 2 files changed, 68 insertions(+), 20 deletions(-) diff --git a/docker/README.md b/docker/README.md index 2599449..47db89e 100644 --- a/docker/README.md +++ b/docker/README.md @@ -168,13 +168,13 @@ From `madrona-portal/docker`: chmod +x ../scripts/db-restore.sh ``` - #### Step 7.2 — Run the restore From `madrona-portal/docker`: ```bash ../scripts/db-restore.sh --drop ``` + *example:* ```bash ../scripts/db-restore.sh --drop ../../madrona-apps/wcoa/wcodp_prod_dump_20260320.sql @@ -199,8 +199,6 @@ There is an optional `--env-file ` if you place your `.en docker compose exec app python marco/manage.py migrate ``` -*Please note:* on 4/10/2026 a 130+ migrations were applied to bring the schema from the prod dump up to date with the current codebase, largely driven by migrating from Wagtail v2 to v7, adding mp-layers, and adding the WCOA OHI indicators (for WCOA installs). - #### Step 7.4 - Migration to mp-layers *If migrating from a server that has not migrated to mp-layers from mp-data-manager*: @@ -217,11 +215,20 @@ docker compose exec app python marco/manage.py migration_to_layers - `madrona-portal/marco/marco/config.wcoa.docker.ini` exists with `MEDIA_ROOT` set to a valid directory - That valid directory should match the volume location is docker-compose.yml - `portals/madrona-portal/media` -- Production media files are available +- Production media files are available #### Step 8.1 - Copy the media files into Docker -From `madrona-portal/docker`: + +If media files need to be copied to the server, you can use `scp`: + +```bash +scp -r /path/to/your_media_dir ubuntu@:/home/ubuntu/portals/madrona-portal/media +``` + +If media files are somewhere on EC2: + ```bash +cd ~/portals/madrona-portal/docker cp -r {your_media_dir}/* ./media/ ``` diff --git a/docs/AWS_DEPLOY.md b/docs/AWS_DEPLOY.md index 778bd4f..49fdae7 100644 --- a/docs/AWS_DEPLOY.md +++ b/docs/AWS_DEPLOY.md @@ -349,6 +349,13 @@ Set these values at minimum: # Environment DJANGO_ENV=production +# Gunicorn workers (2× vCPU count; t3.large has 2 vCPUs → 4 workers) +GUNICORN_WORKERS=4 + +# GHCR — the read-only PAT from Phase 0.2 (documents what token was used to +# log in; docker login stores credentials in ~/.docker/config.json) +GHCR_PAT= + # Django SECRET_KEY= ALLOWED_HOSTS=,localhost @@ -364,17 +371,37 @@ REDIS_PASSWORD= DJANGO_SUPERUSER_USERNAME=admin DJANGO_SUPERUSER_EMAIL=your@email.com DJANGO_SUPERUSER_PASSWORD= +``` -# Gunicorn workers (2× vCPU count; t3.large has 2 vCPUs → 4 workers) -GUNICORN_WORKERS=4 +Can be added now or later: -# GHCR — the read-only PAT from Phase 0.2 (documents what token was used to -# log in; docker login stores credentials in ~/.docker/config.json) -GHCR_PAT= +```ini +# Password for the 'elastic' user +ELASTIC_PASSWORD= + +# Password for the 'kibana_system' user +KIBANA_PASSWORD= -# Geoportal WAR paths (defaults match Phase 3 location — no change needed) -gpt_catalog_war=./wars/geoportal.war -gpt_harvester_war=./wars/harvester.war +# Admin User (Full Access) +gpt_admin_username= +gpt_admin_password= + +# Publisher User (Can publish metadata) +gpt_publisher_username= +gpt_publisher_password= + +# Regular User (Read-only access) +gpt_user_username= +gpt_user_password= + +gpt_wcoa_username= +gpt_wcoa_password= + +gpt_esri_username= +gpt_esri_password= + +gpt_frame_options=DENY +gpt_allowed_origin="localhost localhost:* " ``` Leave everything else at its default for now. You can add email, OAuth, and @@ -403,18 +430,26 @@ docker pull ghcr.io/ecotrust/madrona-portal:latest ### 6.2 Start the stack -From `~/portals/madrona-portal/`: +:warning: For a fresh database, run with `DB_INIT=1` the first time to load initial fixtures and create the superuser:** +```bash +cd ~/portals/madrona-portal/docker + +DB_INIT=1 docker compose up -d +``` +*On subsequent starts, leave `DB_INIT` at its default (`0`) to skip the fixture and superuser steps.* + +For an existing database: ```bash -cd ~/portals/madrona-portal +cd ~/portals/madrona-portal/docker -docker compose -f docker/docker-compose.prod.yml --profile full up -d +docker compose up -d ``` ### 6.3 Watch the startup logs ```bash -docker compose -f docker/docker-compose.prod.yml --profile full logs -f app +docker compose logs -f app ``` On first boot the entrypoint automatically: @@ -437,11 +472,17 @@ Startup takes 2–5 minutes. Look for `Booting worker` lines from Gunicorn. ### Apply migrations (if needed) ```bash -docker compose -f docker/docker-compose.prod.yml --profile full exec app python marco/manage.py migrate +docker compose exec app python marco/manage.py migrate ``` ### Import database +Copy your SQL dump to the server (e.g., using `scp`): + +```bash +scp /path/to/your_dump.sql ubuntu@:/home/ubuntu/your_dump.sql +``` + From `madrona-portal/docker`: ```bash ../scripts/db-restore.sh --drop @@ -468,7 +509,7 @@ docker compose -f docker/docker-compose.prod.yml --profile full exec app python ### Collect static and compress assets (if needed) ```bash -docker compose -f docker/docker-compose.prod.yml --profile full exec app python marco/manage.py collectstatic --noinput +docker compose --profile full exec app python marco/manage.py collectstatic --noinput docker compose -f docker/docker-compose.prod.yml --profile full exec app python marco/manage.py compress ``` @@ -492,7 +533,7 @@ termination, compression, and static file serving. ### 7.1 Install Nginx and Certbot ```bash -sudo apt-get install -y nginx certbot python3-certbot-nginx +sudo apt install -y nginx certbot python3-certbot-nginx ``` ### 7.2 Create a DNS A record From 54068c4648c4e1ef9368334f5d4b4cffd7af1f73 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Fri, 1 May 2026 16:05:56 -0700 Subject: [PATCH 090/152] Remove 'full' profile usage from Docker Compose usage instructions --- docker/docker-compose.prod.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/docker/docker-compose.prod.yml b/docker/docker-compose.prod.yml index 86a85c8..0b55988 100644 --- a/docker/docker-compose.prod.yml +++ b/docker/docker-compose.prod.yml @@ -7,7 +7,6 @@ # Usage: # docker compose -f docker/docker-compose.prod.yml \ # --env-file .env \ -# --profile full \ # up -d # # To pull the latest image before starting: From 916ebf00586e6684c80ee2bbefe8206e0e7bc6ba Mon Sep 17 00:00:00 2001 From: David Pollard Date: Tue, 5 May 2026 16:03:46 -0700 Subject: [PATCH 091/152] Update AWS deployment instructions to add back use of prod compose file and remove need to use 'full' profile in Docker commands --- docs/AWS_DEPLOY.md | 70 ++++++++++++++++++++++------------------------ 1 file changed, 34 insertions(+), 36 deletions(-) diff --git a/docs/AWS_DEPLOY.md b/docs/AWS_DEPLOY.md index 49fdae7..5a27b3b 100644 --- a/docs/AWS_DEPLOY.md +++ b/docs/AWS_DEPLOY.md @@ -434,7 +434,7 @@ docker pull ghcr.io/ecotrust/madrona-portal:latest ```bash cd ~/portals/madrona-portal/docker -DB_INIT=1 docker compose up -d +DB_INIT=1 docker compose -f docker-compose.prod.yml up -d ``` *On subsequent starts, leave `DB_INIT` at its default (`0`) to skip the fixture and superuser steps.* @@ -443,13 +443,13 @@ For an existing database: ```bash cd ~/portals/madrona-portal/docker -docker compose up -d +docker compose -f docker-compose.prod.yml up -d ``` ### 6.3 Watch the startup logs ```bash -docker compose logs -f app +docker compose -f docker-compose.prod.yml logs -f app ``` On first boot the entrypoint automatically: @@ -464,7 +464,7 @@ Startup takes 2–5 minutes. Look for `Booting worker` lines from Gunicorn. > For a fresh database, run with `DB_INIT=1` the first time: > ```bash -> DB_INIT=1 docker compose -f docker/docker-compose.prod.yml --profile full up -d +> DB_INIT=1 docker compose -f docker/docker-compose.prod.yml up -d > ``` > On subsequent starts, leave `DB_INIT` at its default (`0`) to skip the > fixture and superuser steps. @@ -472,7 +472,7 @@ Startup takes 2–5 minutes. Look for `Booting worker` lines from Gunicorn. ### Apply migrations (if needed) ```bash -docker compose exec app python marco/manage.py migrate +docker compose -f docker/docker-compose.prod.yml exec app python marco/manage.py migrate ``` ### Import database @@ -495,13 +495,13 @@ From `madrona-portal/docker`: ### Apply migrations again (if needed) ```bash -docker compose -f docker/docker-compose.prod.yml --profile full exec app python marco/manage.py migrate +docker compose -f docker/docker-compose.prod.yml exec app python marco/manage.py migrate ``` ### Migration to Layers ```bash -docker compose -f docker/docker-compose.prod.yml --profile full exec app python marco/manage.py migration_to_layers +docker compose -f docker/docker-compose.prod.yml exec app python marco/manage.py migration_to_layers ``` @@ -509,8 +509,8 @@ docker compose -f docker/docker-compose.prod.yml --profile full exec app python ### Collect static and compress assets (if needed) ```bash -docker compose --profile full exec app python marco/manage.py collectstatic --noinput -docker compose -f docker/docker-compose.prod.yml --profile full exec app python marco/manage.py compress +docker compose -f docker/docker-compose.prod.yml exec app python marco/manage.py collectstatic --noinput +docker compose -f docker/docker-compose.prod.yml exec app python marco/manage.py compress ``` ### Smoke test @@ -527,8 +527,7 @@ and try again. ## Phase 7 — Nginx + SSL (Production Hardening) -Gunicorn should not be exposed directly to the internet. Nginx handles SSL -termination, compression, and static file serving. +Gunicorn should not be exposed directly to the internet. Nginx handles SSL termination, compression, and static file serving. ### 7.1 Install Nginx and Certbot @@ -590,12 +589,21 @@ server { alias /var/cache/munin/www; } + location /nativeland { + proxy_pass https://native-land.ca/; + resolver 8.8.8.8; + resolver_timeout 10s; + proxy_redirect off; + proxy_pass_request_headers on; + proxy_ssl_server_name on; + } + # Shared CORS policy for these proxied endpoints # (if you only want specific origins, replace "*" with your domain) set $cors_allow_origin "*"; location ~ ^/(?:_search/|_doc/|metadata).*$ { - proxy_pass http://:9200; + proxy_pass http://127.0.0.1:9200; proxy_redirect off; proxy_connect_timeout 5s; proxy_read_timeout 60s; @@ -654,7 +662,7 @@ sudo systemctl restart nginx ### 7.4 Obtain an SSL certificate ```bash -sudo certbot --nginx -d portal.westcoastoceans.org +sudo certbot --nginx -d ``` Certbot edits your Nginx config automatically to add SSL and redirect HTTP @@ -672,7 +680,7 @@ Restart the app container to pick up the change: ```bash cd ~/portals/madrona-portal -docker compose -f docker/docker-compose.prod.yml --profile full up -d --force-recreate app +docker compose -f docker/docker-compose.prod.yml up -d --force-recreate app ``` --- @@ -734,12 +742,10 @@ WorkingDirectory=/home/ubuntu/portals/madrona-portal ExecStart=/usr/bin/docker compose \ -f docker/docker-compose.prod.yml \ --env-file docker/.env \ - --profile full \ up -d ExecStop=/usr/bin/docker compose \ -f docker/docker-compose.prod.yml \ --env-file docker/.env \ - --profile full \ down TimeoutStartSec=300 @@ -759,8 +765,7 @@ sudo systemctl enable madrona-portal ```bash sudo systemctl stop madrona-portal sudo systemctl start madrona-portal -docker compose -f docker/docker-compose.prod.yml --env-file docker/.env --profile full \ - ps # all services should show "running" +docker compose -f docker/docker-compose.prod.yml ps # all services should show "running" ``` --- @@ -810,7 +815,7 @@ Submit a production access request in SES Console → Account dashboard → Requ Install unattended-upgrades and update-notifier-common to automatically apply security updates to the server OS: ```bash -sudo apt-get install unattended-upgrades update-notifier-common -y +sudo apt install unattended-upgrades update-notifier-common -y ``` Enable automatic updates: @@ -835,7 +840,7 @@ Unattended-Upgrade::Automatic-Reboot-Time "02:00"; ## Install Munin ```bash -sudo apt-get install munin -y +sudo apt install munin -y ``` --- @@ -878,8 +883,7 @@ cd ~/portals/madrona-portal docker pull ghcr.io/ecotrust/madrona-portal:latest # Recreate only the app container — db, Redis, and Elasticsearch are untouched -docker compose -f docker/docker-compose.prod.yml --env-file docker/.env --profile full \ - up -d --force-recreate app +docker compose -f docker/docker-compose.prod.yml up -d --force-recreate app ``` Downtime is limited to the container restart (~5–10 seconds). @@ -895,19 +899,16 @@ docker pull ghcr.io/ecotrust/madrona-portal: # Update IMAGE_TAG in docker/.env, then recreate: # IMAGE_TAG= in docker/.env -docker compose -f docker/docker-compose.prod.yml --env-file docker/.env --profile full \ - up -d --force-recreate app +docker compose -f docker/docker-compose.prod.yml up -d --force-recreate app ``` ### If portal configuration changes (config.wcoa.docker.ini) -The ini file is bind-mounted into the container (read-only), so changes take -effect immediately on the next container restart — no image rebuild needed: +The ini file is bind-mounted into the container (read-only), so changes take effect immediately on the next container restart — no image rebuild needed: ```bash nano ~/portals/madrona-portal/marco/config.wcoa.docker.ini -docker compose -f docker/docker-compose.prod.yml --env-file docker/.env --profile full \ - up -d --force-recreate app +docker compose -f docker/docker-compose.prod.yml up -d --force-recreate app ``` --- @@ -918,16 +919,13 @@ All `docker compose` commands run from `~/portals/madrona-portal/`. ```bash # Tail app logs -docker compose -f docker/docker-compose.prod.yml --env-file docker/.env --profile full \ - logs -f app +docker compose -f docker/docker-compose.prod.yml logs -f app # Run a Django management command -docker compose -f docker/docker-compose.prod.yml --env-file docker/.env --profile full \ - run --rm app python marco/manage.py +docker compose -f docker/docker-compose.prod.yml --env-file docker/.env run --rm app python marco/manage.py # Open a Django shell -docker compose -f docker/docker-compose.prod.yml --env-file docker/.env --profile full \ - run --rm app python marco/manage.py shell +docker compose -f docker/docker-compose.prod.yml --env-file docker/.env run --rm app python marco/manage.py shell # Open a database shell docker exec -it \ @@ -942,10 +940,10 @@ docker system df docker image prune -f # Stop the stack (data preserved in named volumes) -docker compose -f docker/docker-compose.prod.yml --env-file docker/.env --profile full down +docker compose -f docker/docker-compose.prod.yml --env-file docker/.env down # Full reset — DESTROYS ALL DATA -docker compose -f docker/docker-compose.prod.yml --env-file docker/.env --profile full down -v +docker compose -f docker/docker-compose.prod.yml --env-file docker/.env down -v ``` --- From 2890d5af7469bc1ef18af60dd9c450a237f89c2b Mon Sep 17 00:00:00 2001 From: David Pollard Date: Tue, 5 May 2026 16:13:14 -0700 Subject: [PATCH 092/152] Add optional app info variables to .env.example for templates and emails --- docker/.env.example | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docker/.env.example b/docker/.env.example index 86404b6..f8f0816 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -34,6 +34,12 @@ ALLOWED_HOSTS=localhost,127.0.0.1 MP_PROJECT_CONFIG=config.wcoa.docker.ini DEBUG=False +# --------------------------------------------------------------------------- +# App info (optional, used in templates and emails) +# --------------------------------------------------------------------------- +APP_NAME= +APP_TEAM_NAME= + # --------------------------------------------------------------------------- # PostgreSQL / PostGIS # DB_* is preferred; SQL_* aliases are accepted for legacy docker-compose files. From 32c475761e50fb83bfec0b081ca08ad7cbd71339 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Tue, 5 May 2026 16:13:32 -0700 Subject: [PATCH 093/152] Implement configuration standard with typed helpers for environment settings Co-authored-by: Copilot --- docs/CONFIGURATION_STANDARD.md | 50 +++++++++++++++++ marco/marco/config_helpers.py | 65 ++++++++++++++++++++++ marco/marco/settings.py | 70 ++++++++++-------------- marco/marco/tests/test_config_helpers.py | 66 ++++++++++++++++++++++ 4 files changed, 211 insertions(+), 40 deletions(-) create mode 100644 docs/CONFIGURATION_STANDARD.md create mode 100644 marco/marco/config_helpers.py create mode 100644 marco/marco/tests/test_config_helpers.py diff --git a/docs/CONFIGURATION_STANDARD.md b/docs/CONFIGURATION_STANDARD.md new file mode 100644 index 0000000..9e26dd5 --- /dev/null +++ b/docs/CONFIGURATION_STANDARD.md @@ -0,0 +1,50 @@ +# Configuration Standard + +This project uses a layered configuration model with strict precedence and typed parsing. + +## Precedence + +For runtime settings, use this priority order: + +1. Environment variable +2. config.ini value +3. Safe default in code + +This keeps deployments flexible while preserving stable local defaults. + +## Required Standard + +Use typed helper functions from marco.config_helpers for app settings. + +- env_str for string settings +- env_bool for boolean settings +- env_int for integer settings + +Do not parse booleans ad hoc in settings code. + +## Why This Standard Exists + +- Avoid inconsistent precedence across settings +- Prevent string-typed booleans from silently behaving incorrectly +- Make behavior testable and explicit + +## Approved Patterns + +Use typed helper access for standard settings: + +- DEBUG +- EMAIL_USE_TLS +- EMAIL_PORT +- SECRET_KEY and other credentials +- STATIC and MEDIA path settings + +Direct environment lookups are still acceptable for special alias chains where multiple environment variable names must be supported for compatibility, such as DB_* and SQL_* overrides. + +## Test Coverage + +The helper contract is enforced by tests in marco/marco/tests/test_config_helpers.py: + +- env over config precedence +- config over default fallback +- strict boolean parsing and invalid-value failures +- integer parsing behavior diff --git a/marco/marco/config_helpers.py b/marco/marco/config_helpers.py new file mode 100644 index 0000000..d1a943d --- /dev/null +++ b/marco/marco/config_helpers.py @@ -0,0 +1,65 @@ +import configparser +import os + + +_TRUE_VALUES = {"1", "true", "yes", "on"} +_FALSE_VALUES = {"0", "false", "no", "off"} + + +def env_str( + env_key: str, + cfg_section: configparser.SectionProxy, + cfg_key: str, + default: str = "", +) -> str: + """Resolve a string setting using env > config.ini > default precedence.""" + return os.environ.get(env_key) or cfg_section.get(cfg_key, default) + + +def parse_bool(value: object, *, setting_name: str = "setting") -> bool: + """Parse a bool from common string values, raising on invalid input.""" + if isinstance(value, bool): + return value + if value is None: + raise ValueError(f"{setting_name} cannot be None") + + normalized = str(value).strip().lower() + if normalized in _TRUE_VALUES: + return True + if normalized in _FALSE_VALUES: + return False + + raise ValueError( + f"Invalid boolean value for {setting_name}: {value!r}. " + "Use one of: 1/0, true/false, yes/no, on/off." + ) + + +def env_bool( + env_key: str, + cfg_section: configparser.SectionProxy, + cfg_key: str, + default: bool = False, +) -> bool: + """Resolve and parse a bool setting using env > config.ini > default.""" + raw = os.environ.get(env_key) + if raw is None: + raw = cfg_section.get(cfg_key) + if raw is None: + return default + return parse_bool(raw, setting_name=env_key) + + +def env_int( + env_key: str, + cfg_section: configparser.SectionProxy, + cfg_key: str, + default: int, +) -> int: + """Resolve and parse an int setting using env > config.ini > default.""" + raw = os.environ.get(env_key) + if raw is None: + raw = cfg_section.get(cfg_key) + if raw is None: + return default + return int(str(raw).strip()) diff --git a/marco/marco/settings.py b/marco/marco/settings.py index b797259..c1d2cd8 100644 --- a/marco/marco/settings.py +++ b/marco/marco/settings.py @@ -19,6 +19,8 @@ from os.path import abspath, dirname from typing import Any +from .config_helpers import env_bool, env_int, env_str + # --------------------------------------------------------------------------- # Path helpers # --------------------------------------------------------------------------- @@ -52,29 +54,17 @@ social_cfg = cfg['SOCIAL_AUTH'] region_cfg = cfg['REGION'] -# --------------------------------------------------------------------------- -# Secret resolution helper -# --------------------------------------------------------------------------- -def _env(env_key: str, cfg_section: configparser.SectionProxy, cfg_key: str, - default: Any = '') -> str: - """Return a setting value, checking the environment first. - - Priority: env var > config.ini > default. - This allows Docker / CI to override secrets without touching config files. - """ - return os.environ.get(env_key) or cfg_section.get(cfg_key, default) - # --------------------------------------------------------------------------- # Core settings # --------------------------------------------------------------------------- -DEBUG = app_cfg.getboolean('DEBUG', False) +DEBUG = env_bool('DEBUG', app_cfg, 'DEBUG', False) -APP_NAME = app_cfg.get('APP_NAME', 'Marine Planner') -APP_URL = app_cfg.get('APP_URL', '') -APP_TEAM_NAME = app_cfg.get('APP_TEAM_NAME', f"{APP_NAME} Team") +APP_NAME = env_str('APP_NAME', app_cfg, 'APP_NAME', 'Marine Planner') +APP_URL = env_str('APP_URL', app_cfg, 'APP_URL', '') +APP_TEAM_NAME = env_str('APP_TEAM_NAME', app_cfg, 'APP_TEAM_NAME', f"{APP_NAME} Team") # env var takes priority so Docker / CI can inject secrets without touching config.ini -SECRET_KEY = _env('SECRET_KEY', app_cfg, 'SECRET_KEY', '') +SECRET_KEY = env_str('SECRET_KEY', app_cfg, 'SECRET_KEY', '') _placeholder_phrases = ('forgot', 'change me', 'changeme', 'placeholder', 'you forgot') if not SECRET_KEY or any(p in SECRET_KEY.lower() for p in _placeholder_phrases): raise RuntimeError( @@ -101,7 +91,7 @@ def _parse_hosts(raw: str | None) -> list[str]: return [h.strip() for h in raw.split(',') if h.strip()] return [raw] -_raw_hosts = os.environ.get('ALLOWED_HOSTS', app_cfg.get('ALLOWED_HOSTS', '')) +_raw_hosts = env_str('ALLOWED_HOSTS', app_cfg, 'ALLOWED_HOSTS', '') ALLOWED_HOSTS = _parse_hosts(_raw_hosts) # Normalise bracketed IPv6 forms like [::1] → ::1 for Django host checks @@ -344,12 +334,12 @@ def _parse_hosts(raw: str | None) -> list[str]: # --------------------------------------------------------------------------- # Static & media files # --------------------------------------------------------------------------- -STATIC_ROOT = _env('STATIC_ROOT', app_cfg, 'STATIC_ROOT', os.path.join(BASE_DIR, 'static')) -STATIC_URL = _env('STATIC_URL', app_cfg, 'STATIC_URL', '/static/') +STATIC_ROOT = env_str('STATIC_ROOT', app_cfg, 'STATIC_ROOT', os.path.join(BASE_DIR, 'static')) +STATIC_URL = env_str('STATIC_URL', app_cfg, 'STATIC_URL', '/static/') STATIC_CORE = app_cfg.get('STATIC_CORE', '') -MEDIA_ROOT = _env('MEDIA_ROOT', app_cfg, 'MEDIA_ROOT', os.path.join(BASE_DIR, 'media')) -MEDIA_URL = _env('MEDIA_URL', app_cfg, 'MEDIA_URL', '/media/') +MEDIA_ROOT = env_str('MEDIA_ROOT', app_cfg, 'MEDIA_ROOT', os.path.join(BASE_DIR, 'media')) +MEDIA_URL = env_str('MEDIA_URL', app_cfg, 'MEDIA_URL', '/media/') _static_root_abs = os.path.abspath(STATIC_ROOT) _staticfiles_dirs: list[str] = [] @@ -491,15 +481,15 @@ def _parse_hosts(raw: str | None) -> list[str]: # Env var overrides: FACEBOOK_KEY, FACEBOOK_SECRET, TWITTER_KEY, # TWITTER_SECRET, GOOGLE_KEY, GOOGLE_SECRET -SOCIAL_AUTH_FACEBOOK_KEY = _env('FACEBOOK_KEY', social_cfg, 'FACEBOOK_KEY', '') -SOCIAL_AUTH_FACEBOOK_SECRET = _env('FACEBOOK_SECRET', social_cfg, 'FACEBOOK_SECRET', '') +SOCIAL_AUTH_FACEBOOK_KEY = env_str('FACEBOOK_KEY', social_cfg, 'FACEBOOK_KEY', '') +SOCIAL_AUTH_FACEBOOK_SECRET = env_str('FACEBOOK_SECRET', social_cfg, 'FACEBOOK_SECRET', '') SOCIAL_AUTH_FACEBOOK_SCOPE = ['public_profile,email'] -SOCIAL_AUTH_TWITTER_KEY = _env('TWITTER_KEY', social_cfg, 'TWITTER_KEY', '') -SOCIAL_AUTH_TWITTER_SECRET = _env('TWITTER_SECRET', social_cfg, 'TWITTER_SECRET', '') +SOCIAL_AUTH_TWITTER_KEY = env_str('TWITTER_KEY', social_cfg, 'TWITTER_KEY', '') +SOCIAL_AUTH_TWITTER_SECRET = env_str('TWITTER_SECRET', social_cfg, 'TWITTER_SECRET', '') -SOCIAL_AUTH_GOOGLE_OAUTH2_KEY = _env('GOOGLE_KEY', social_cfg, 'GOOGLE_KEY', '') -SOCIAL_AUTH_GOOGLE_OAUTH2_SECRET = _env('GOOGLE_SECRET', social_cfg, 'GOOGLE_SECRET', '') +SOCIAL_AUTH_GOOGLE_OAUTH2_KEY = env_str('GOOGLE_KEY', social_cfg, 'GOOGLE_KEY', '') +SOCIAL_AUTH_GOOGLE_OAUTH2_SECRET = env_str('GOOGLE_SECRET', social_cfg, 'GOOGLE_SECRET', '') SOCIAL_AUTH_DISCONNECT_REDIRECT_URL = '/' SOCIAL_AUTH_JSONFIELD_ENABLED = True @@ -529,14 +519,14 @@ def _parse_hosts(raw: str | None) -> list[str]: # Env var overrides: EMAIL_HOST, EMAIL_PORT, EMAIL_HOST_USER, # EMAIL_HOST_PASSWORD, EMAIL_USE_TLS # --------------------------------------------------------------------------- -EMAIL_HOST = _env('EMAIL_HOST', email_cfg, 'HOST', 'localhost') -EMAIL_PORT = int(_env('EMAIL_PORT', email_cfg, 'PORT', '25')) -EMAIL_HOST_USER = _env('EMAIL_HOST_USER', email_cfg, 'HOST_USER', '') -EMAIL_HOST_PASSWORD = _env('EMAIL_HOST_PASSWORD', email_cfg, 'HOST_PASSWORD', '') +EMAIL_HOST = env_str('EMAIL_HOST', email_cfg, 'HOST', 'localhost') +EMAIL_PORT = env_int('EMAIL_PORT', email_cfg, 'PORT', 25) +EMAIL_HOST_USER = env_str('EMAIL_HOST_USER', email_cfg, 'HOST_USER', '') +EMAIL_HOST_PASSWORD = env_str('EMAIL_HOST_PASSWORD', email_cfg, 'HOST_PASSWORD', '') EMAIL_BACKEND = email_cfg.get('EMAIL_BACKEND', 'django.core.mail.backends.smtp.EmailBackend') DEFAULT_FROM_EMAIL = email_cfg.get('DEFAULT_FROM_EMAIL', "MARCO Portal Team ") SERVER_EMAIL = email_cfg.get('SERVER_EMAIL', "MARCO Site Errors ") -EMAIL_USE_TLS = bool(os.environ.get('EMAIL_USE_TLS', email_cfg.get('EMAIL_USE_TLS', 'false')).lower() in ('1', 'true', 'yes')) +EMAIL_USE_TLS = env_bool('EMAIL_USE_TLS', email_cfg, 'EMAIL_USE_TLS', False) EMAIL_SUBJECT_PREFIX = app_cfg.get('EMAIL_SUBJECT_PREFIX', '[MARCO]') + ' ' ADMINS = (('KSDev', 'ksdev@ecotrust.org'),) @@ -546,10 +536,10 @@ def _parse_hosts(raw: str | None) -> list[str]: # Env var overrides: AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, # AWS_SES_REGION_NAME, AWS_SES_REGION_ENDPOINT # --------------------------------------------------------------------------- -AWS_ACCESS_KEY_ID = _env('AWS_ACCESS_KEY_ID', aws_cfg, 'AWS_ACCESS_KEY_ID', '') -AWS_SECRET_ACCESS_KEY = _env('AWS_SECRET_ACCESS_KEY', aws_cfg, 'AWS_SECRET_ACCESS_KEY', '') -AWS_SES_REGION_NAME = _env('AWS_SES_REGION_NAME', aws_cfg, 'AWS_SES_REGION_NAME', 'us-east-1') -AWS_SES_REGION_ENDPOINT = _env('AWS_SES_REGION_ENDPOINT', aws_cfg, 'AWS_SES_REGION_ENDPOINT', 'email.us-east-1.amazonaws.com') +AWS_ACCESS_KEY_ID = env_str('AWS_ACCESS_KEY_ID', aws_cfg, 'AWS_ACCESS_KEY_ID', '') +AWS_SECRET_ACCESS_KEY = env_str('AWS_SECRET_ACCESS_KEY', aws_cfg, 'AWS_SECRET_ACCESS_KEY', '') +AWS_SES_REGION_NAME = env_str('AWS_SES_REGION_NAME', aws_cfg, 'AWS_SES_REGION_NAME', 'us-east-1') +AWS_SES_REGION_ENDPOINT = env_str('AWS_SES_REGION_ENDPOINT', aws_cfg, 'AWS_SES_REGION_ENDPOINT', 'email.us-east-1.amazonaws.com') # --------------------------------------------------------------------------- # Celery (Celery 5+ settings) @@ -572,8 +562,8 @@ def _parse_hosts(raw: str | None) -> list[str]: # ReCAPTCHA # --------------------------------------------------------------------------- NOCAPTCHA = True -RECAPTCHA_PUBLIC_KEY = _env('RECAPTCHA_PUBLIC_KEY', app_cfg, 'RECAPTCHA_PUBLIC_KEY', '') -RECAPTCHA_PRIVATE_KEY = _env('RECAPTCHA_PRIVATE_KEY', app_cfg, 'RECAPTCHA_PRIVATE_KEY', '') +RECAPTCHA_PUBLIC_KEY = env_str('RECAPTCHA_PUBLIC_KEY', app_cfg, 'RECAPTCHA_PUBLIC_KEY', '') +RECAPTCHA_PRIVATE_KEY = env_str('RECAPTCHA_PRIVATE_KEY', app_cfg, 'RECAPTCHA_PRIVATE_KEY', '') # --------------------------------------------------------------------------- # Analytics @@ -583,7 +573,7 @@ def _parse_hosts(raw: str | None) -> list[str]: # --------------------------------------------------------------------------- # NATIVE LANDS API KEY # --------------------------------------------------------------------------- -NATIVE_LAND_API_KEY = _env('NATIVE_LAND_API_KEY', app_cfg, 'NATIVE_LAND_API_KEY', '') +NATIVE_LAND_API_KEY = env_str('NATIVE_LAND_API_KEY', app_cfg, 'NATIVE_LAND_API_KEY', '') # --------------------------------------------------------------------------- # Project-level settings overrides diff --git a/marco/marco/tests/test_config_helpers.py b/marco/marco/tests/test_config_helpers.py new file mode 100644 index 0000000..0f8aed5 --- /dev/null +++ b/marco/marco/tests/test_config_helpers.py @@ -0,0 +1,66 @@ +import configparser + +import pytest + +from marco.config_helpers import env_bool, env_int, env_str, parse_bool + + +def _section(values: dict[str, str]) -> configparser.SectionProxy: + cfg = configparser.ConfigParser() + cfg["APP"] = values + return cfg["APP"] + + +def test_env_str_prefers_environment(monkeypatch: pytest.MonkeyPatch) -> None: + section = _section({"APP_NAME": "FromConfig"}) + monkeypatch.setenv("APP_NAME", "FromEnv") + assert env_str("APP_NAME", section, "APP_NAME", "Default") == "FromEnv" + + +def test_env_str_falls_back_to_config_then_default(monkeypatch: pytest.MonkeyPatch) -> None: + section = _section({"APP_NAME": "FromConfig"}) + monkeypatch.delenv("APP_NAME", raising=False) + assert env_str("APP_NAME", section, "APP_NAME", "Default") == "FromConfig" + assert env_str("MISSING", section, "MISSING", "Default") == "Default" + + +def test_env_bool_prefers_environment(monkeypatch: pytest.MonkeyPatch) -> None: + section = _section({"DEBUG": "false"}) + monkeypatch.setenv("DEBUG", "true") + assert env_bool("DEBUG", section, "DEBUG", False) is True + + +def test_env_bool_falls_back_to_default(monkeypatch: pytest.MonkeyPatch) -> None: + section = _section({}) + monkeypatch.delenv("DEBUG", raising=False) + assert env_bool("DEBUG", section, "DEBUG", False) is False + assert env_bool("DEBUG", section, "DEBUG", True) is True + + +def test_parse_bool_accepts_common_values() -> None: + assert parse_bool("1") is True + assert parse_bool("yes") is True + assert parse_bool("ON") is True + assert parse_bool("0") is False + assert parse_bool("no") is False + assert parse_bool("off") is False + + +def test_parse_bool_rejects_invalid_values() -> None: + with pytest.raises(ValueError): + parse_bool("maybe", setting_name="DEBUG") + + +def test_env_int_prefers_environment_and_parses(monkeypatch: pytest.MonkeyPatch) -> None: + section = _section({"PORT": "25"}) + monkeypatch.setenv("EMAIL_PORT", "587") + assert env_int("EMAIL_PORT", section, "PORT", 25) == 587 + + +def test_env_int_falls_back_to_config_then_default(monkeypatch: pytest.MonkeyPatch) -> None: + section = _section({"PORT": "2525"}) + monkeypatch.delenv("EMAIL_PORT", raising=False) + assert env_int("EMAIL_PORT", section, "PORT", 25) == 2525 + + empty_section = _section({}) + assert env_int("EMAIL_PORT", empty_section, "PORT", 25) == 25 From 90882fd676cb741708d9c186cd1c68bc8a549455 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Tue, 5 May 2026 16:45:02 -0700 Subject: [PATCH 094/152] Refactor env_str, env_bool, and env_int functions for improved readability and consistency in handling default values Co-authored-by: Copilot --- marco/marco/config_helpers.py | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/marco/marco/config_helpers.py b/marco/marco/config_helpers.py index d1a943d..898ecf8 100644 --- a/marco/marco/config_helpers.py +++ b/marco/marco/config_helpers.py @@ -13,7 +13,13 @@ def env_str( default: str = "", ) -> str: """Resolve a string setting using env > config.ini > default precedence.""" - return os.environ.get(env_key) or cfg_section.get(cfg_key, default) + raw = os.environ.get(env_key) + if raw is not None: + return raw + raw = cfg_section.get(cfg_key) + if raw is not None: + return raw + return default def parse_bool(value: object, *, setting_name: str = "setting") -> bool: @@ -43,11 +49,12 @@ def env_bool( ) -> bool: """Resolve and parse a bool setting using env > config.ini > default.""" raw = os.environ.get(env_key) - if raw is None: - raw = cfg_section.get(cfg_key) - if raw is None: - return default - return parse_bool(raw, setting_name=env_key) + if raw is not None: + return parse_bool(raw, setting_name=env_key) + raw = cfg_section.get(cfg_key) + if raw is not None: + return parse_bool(raw, setting_name=cfg_key) + return default def env_int( @@ -58,8 +65,9 @@ def env_int( ) -> int: """Resolve and parse an int setting using env > config.ini > default.""" raw = os.environ.get(env_key) - if raw is None: - raw = cfg_section.get(cfg_key) - if raw is None: - return default - return int(str(raw).strip()) + if raw is not None: + return int(str(raw).strip()) + raw = cfg_section.get(cfg_key) + if raw is not None: + return int(str(raw).strip()) + return default From 500eb84c6b0caa0739ada9eb950107468cc8d6ad Mon Sep 17 00:00:00 2001 From: David Pollard Date: Tue, 5 May 2026 16:45:16 -0700 Subject: [PATCH 095/152] Add MEDIA_ROOT and MEDIA_URL to .env.example; update media file path in README --- docker/.env.example | 2 ++ docker/README.md | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docker/.env.example b/docker/.env.example index f8f0816..4cdac7f 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -39,6 +39,8 @@ DEBUG=False # --------------------------------------------------------------------------- APP_NAME= APP_TEAM_NAME= +MEDIA_ROOT=/usr/local/apps/madrona-portal/media +MEDIA_URL=/media/ # --------------------------------------------------------------------------- # PostgreSQL / PostGIS diff --git a/docker/README.md b/docker/README.md index 47db89e..51c9c36 100644 --- a/docker/README.md +++ b/docker/README.md @@ -222,7 +222,7 @@ docker compose exec app python marco/manage.py migration_to_layers If media files need to be copied to the server, you can use `scp`: ```bash -scp -r /path/to/your_media_dir ubuntu@:/home/ubuntu/portals/madrona-portal/media +scp -r /path/to/your_media_dir ubuntu@:/home/ubuntu/portals/madrona-portal/docker/media ``` If media files are somewhere on EC2: From 50f21591005275f44c81ba11114b7f8b9a2fcd58 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Wed, 6 May 2026 17:55:50 -0700 Subject: [PATCH 096/152] Add STATIC_ROOT to .env.example; update Docker Compose files to mount static directory and modify entrypoint script for asset management --- docker/.env.example | 1 + docker/docker-compose.prod.yml | 3 +-- docker/docker-compose.yml | 3 +-- docker/entrypoint.sh | 22 ++++++++++++++-------- docs/AWS_DEPLOY.md | 23 ++++++++++++++++++----- 5 files changed, 35 insertions(+), 17 deletions(-) diff --git a/docker/.env.example b/docker/.env.example index 4cdac7f..6651cf8 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -41,6 +41,7 @@ APP_NAME= APP_TEAM_NAME= MEDIA_ROOT=/usr/local/apps/madrona-portal/media MEDIA_URL=/media/ +STATIC_ROOT=/vol/web/static # --------------------------------------------------------------------------- # PostgreSQL / PostGIS diff --git a/docker/docker-compose.prod.yml b/docker/docker-compose.prod.yml index 0b55988..04fc50d 100644 --- a/docker/docker-compose.prod.yml +++ b/docker/docker-compose.prod.yml @@ -20,7 +20,7 @@ services: app: image: ghcr.io/ecotrust/madrona-portal:${IMAGE_TAG:-latest} volumes: - - static_data:/vol/web + - ./static:/vol/web/static # User-uploaded media files — persisted on the host across deploys. - ./media:/usr/local/apps/madrona-portal/media # Config file — lets you update portal settings without rebuilding the image. @@ -137,7 +137,6 @@ services: restart: always volumes: - static_data: postgis_data: redis_data: gp-volume: diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 0475231..0d88ec3 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -16,7 +16,7 @@ services: context: ../../ dockerfile: madrona-portal/docker/Dockerfile volumes: - - static_data:/vol/web + - ./static:/vol/web/static - ./media:/usr/local/apps/madrona-portal/media # Mount the main Django project tree from the host. # Changes to Python, templates, and config files are live immediately. @@ -176,7 +176,6 @@ services: restart: always volumes: - static_data: postgis_data: redis_data: gp-volume: diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 85f122b..5982a7b 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -2,8 +2,8 @@ # Madrona Portal — Docker entrypoint # Waits for the database, then starts the application server. # -# By default only step 1 (DB wait) and step 5 (server start) run. -# Set DB_INIT=1 to also run steps 2-4 (migrate, seed fixtures, create superuser). +# By default steps 1 (DB wait), 2 (collectstatic + compress), and 6 (server +# start) run. Set DB_INIT=1 to also run steps 3-5 (migrate, fixtures, superuser). # This is intentionally opt-in to protect existing databases. set -e @@ -34,21 +34,27 @@ print("Database is up.", flush=True) PY # --------------------------------------------------------------------------- -# 2-4. Database initialisation (opt-in via DB_INIT=1) +# 2. Collect static files and compress assets (always runs) +# --------------------------------------------------------------------------- +echo "Collecting static files..." +python marco/manage.py collectstatic --noinput +echo "Compressing assets..." +python marco/manage.py compress --force + +# --------------------------------------------------------------------------- +# 3-5. Database initialisation (opt-in via DB_INIT=1) # --------------------------------------------------------------------------- if [ "${DB_INIT:-0}" != "1" ]; then echo "DB_INIT not set — skipping migrations, fixtures, and superuser creation." else # --------------------------------------------------------------------------- -# 2. Migrate and collect static files +# 3. Migrate # --------------------------------------------------------------------------- python marco/manage.py migrate --noinput -python marco/manage.py collectstatic --noinput -python marco/manage.py compress --force # --------------------------------------------------------------------------- -# 3. Seed a fresh database with initial fixture data +# 4. Seed a fresh database with initial fixture data # # A brand-new PostGIS install contains exactly one Wagtail Page row (the # Wagtail root page, depth=1). We count pages at depth > 1 — if none exist, @@ -154,7 +160,7 @@ else fi # --------------------------------------------------------------------------- -# 4. Create superuser (only when DJANGO_SUPERUSER_PASSWORD is set and the +# 5. Create superuser (only when DJANGO_SUPERUSER_PASSWORD is set and the # username does not already exist — safe to run on every restart) # --------------------------------------------------------------------------- if [ -n "${DJANGO_SUPERUSER_PASSWORD:-}" ]; then diff --git a/docs/AWS_DEPLOY.md b/docs/AWS_DEPLOY.md index 5a27b3b..74ce998 100644 --- a/docs/AWS_DEPLOY.md +++ b/docs/AWS_DEPLOY.md @@ -454,10 +454,10 @@ docker compose -f docker-compose.prod.yml logs -f app On first boot the entrypoint automatically: 1. Waits for PostgreSQL to be healthy -2. Runs `migrate` -3. Runs `collectstatic` and `compress` -4. Detects fresh database → loads initial fixtures -5. Creates the superuser from `.env` (if `DB_INIT=1`) +2. Runs `collectstatic` and `compress` (always) +3. Runs `migrate` (only when `DB_INIT=1`) +4. Detects fresh database → loads initial fixtures (only when `DB_INIT=1`) +5. Creates the superuser from `.env` (only when `DB_INIT=1`) 6. Starts Gunicorn Startup takes 2–5 minutes. Look for `Booting worker` lines from Gunicorn. @@ -506,7 +506,10 @@ docker compose -f docker/docker-compose.prod.yml exec app python marco/manage.py -### Collect static and compress assets (if needed) +### Collect static and compress assets + +Static files are collected automatically on every container startup. To force +a manual re-run without restarting the container: ```bash docker compose -f docker/docker-compose.prod.yml exec app python marco/manage.py collectstatic --noinput @@ -566,6 +569,16 @@ server { access_log /var/log/nginx/wcoa.access.log; error_log /var/log/nginx/wcoa.error.log; + location /static/ { + alias /home/ubuntu/portals/madrona-portal/docker/static/; + expires 30d; + add_header Cache-Control "public, no-transform"; + } + + location /media/ { + alias /home/ubuntu/portals/madrona-portal/docker/media/; + } + location / { proxy_pass http://127.0.0.1:8000; proxy_set_header Host $host; From 51e4a19971c01d5b20e3ccc8e84a9af831190d89 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Wed, 6 May 2026 18:23:14 -0700 Subject: [PATCH 097/152] Add docker/static/ to .gitignore to exclude static files from version control --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 630e633..e122a0b 100644 --- a/.gitignore +++ b/.gitignore @@ -17,6 +17,7 @@ vagrant node_modules docker/media/ +docker/static/ docker/entrypoint.sh docker/docker-requirements.txt docker/wars/ From 2f16d71dd457aa178c2e02aa829cb10d7c31102f Mon Sep 17 00:00:00 2001 From: David Pollard Date: Wed, 6 May 2026 18:24:01 -0700 Subject: [PATCH 098/152] Update environment variables in .env.example and adjust Dockerfile permissions for media and static directories --- docker/.env.example | 6 +++--- docker/Dockerfile | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docker/.env.example b/docker/.env.example index 6651cf8..9d88466 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -35,10 +35,10 @@ MP_PROJECT_CONFIG=config.wcoa.docker.ini DEBUG=False # --------------------------------------------------------------------------- -# App info (optional, used in templates and emails) +# App info and configuration # --------------------------------------------------------------------------- -APP_NAME= -APP_TEAM_NAME= +APP_NAME="Madrona Portal" +APP_TEAM_NAME="Marine Planner Team" MEDIA_ROOT=/usr/local/apps/madrona-portal/media MEDIA_URL=/media/ STATIC_ROOT=/vol/web/static diff --git a/docker/Dockerfile b/docker/Dockerfile index 669d808..1f8262b 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -83,10 +83,10 @@ RUN pip install "GDAL==$(gdal-config --version)" --no-cache-dir # Runtime setup # --------------------------------------------------------------------------- RUN chmod 755 /entrypoint.sh && \ - mkdir -p /vol/web/media /vol/web/static && \ + mkdir -p /vol/web/static && \ useradd --create-home --shell /bin/sh madrona_user && \ chown -R madrona_user:madrona_user /vol /usr/local/apps/madrona-portal && \ - chmod -R 755 /vol/web + chmod -R 775 /vol/web USER madrona_user From 201e70bc7bd35d52e3325486490fa10c2b459379 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Thu, 7 May 2026 10:48:55 -0700 Subject: [PATCH 099/152] Add CSRF trusted origins configuration to settings --- marco/marco/settings.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/marco/marco/settings.py b/marco/marco/settings.py index c1d2cd8..ec2627e 100644 --- a/marco/marco/settings.py +++ b/marco/marco/settings.py @@ -63,6 +63,13 @@ APP_URL = env_str('APP_URL', app_cfg, 'APP_URL', '') APP_TEAM_NAME = env_str('APP_TEAM_NAME', app_cfg, 'APP_TEAM_NAME', f"{APP_NAME} Team") +# CSRF trusted origins: accepts a comma-separated string, a JSON array string, or a plain string. +SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https") +CSRF_TRUSTED_ORIGINS_ENV = env_str('CSRF_TRUSTED_ORIGINS', app_cfg, 'CSRF_TRUSTED_ORIGINS', '') +if CSRF_TRUSTED_ORIGINS_ENV: + CSRF_TRUSTED_ORIGINS = CSRF_TRUSTED_ORIGINS_ENV.split(",") + + # env var takes priority so Docker / CI can inject secrets without touching config.ini SECRET_KEY = env_str('SECRET_KEY', app_cfg, 'SECRET_KEY', '') _placeholder_phrases = ('forgot', 'change me', 'changeme', 'placeholder', 'you forgot') From 816437d22187b98e2066111f909da8716430ea3d Mon Sep 17 00:00:00 2001 From: David Pollard Date: Thu, 7 May 2026 11:59:10 -0700 Subject: [PATCH 100/152] Add gosu for user switching in Docker entrypoint and update permissions for static files --- docker/Dockerfile | 3 +-- docker/entrypoint.sh | 12 ++++++++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 1f8262b..b005e73 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -31,6 +31,7 @@ RUN apt-get update && apt-get upgrade -y && \ perl libxml2-dev \ libproj-dev proj-bin \ libffi-dev openssl \ + gosu \ && rm -rf /var/lib/apt/lists/* # --------------------------------------------------------------------------- @@ -88,8 +89,6 @@ RUN chmod 755 /entrypoint.sh && \ chown -R madrona_user:madrona_user /vol /usr/local/apps/madrona-portal && \ chmod -R 775 /vol/web -USER madrona_user - EXPOSE 8000 EXPOSE 8008 diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 5982a7b..34da985 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -36,10 +36,14 @@ PY # --------------------------------------------------------------------------- # 2. Collect static files and compress assets (always runs) # --------------------------------------------------------------------------- +# Ensure the bind-mounted static dir is writable by madrona_user regardless +# of how Docker created it on the host (often root:root on Linux). +chown madrona_user:madrona_user /vol/web/static 2>/dev/null || true + echo "Collecting static files..." -python marco/manage.py collectstatic --noinput +gosu madrona_user python marco/manage.py collectstatic --noinput echo "Compressing assets..." -python marco/manage.py compress --force +gosu madrona_user python marco/manage.py compress --force # --------------------------------------------------------------------------- # 3-5. Database initialisation (opt-in via DB_INIT=1) @@ -206,7 +210,7 @@ PY if [ "${DJANGO_ENV:-}" = "production" ] || [ "${DJANGO_DEBUG}" = "false" ]; then echo "Starting gunicorn (production mode)..." - exec gunicorn marco.wsgi:application \ + exec gosu madrona_user gunicorn marco.wsgi:application \ --bind 0.0.0.0:8008 \ --workers "${GUNICORN_WORKERS:-3}" \ --timeout "${GUNICORN_TIMEOUT:-120}" \ @@ -215,5 +219,5 @@ if [ "${DJANGO_ENV:-}" = "production" ] || [ "${DJANGO_DEBUG}" = "false" ]; then --error-logfile - else echo "Starting Django development server..." - exec python marco/manage.py runserver 0.0.0.0:8000 + exec gosu madrona_user python marco/manage.py runserver 0.0.0.0:8000 fi From 6a810820adbc2cfd4fea1ede36a1bedf5a19953c Mon Sep 17 00:00:00 2001 From: David Pollard Date: Thu, 7 May 2026 12:50:55 -0700 Subject: [PATCH 101/152] Add instructions for setting static file permissions for Nginx --- docs/AWS_DEPLOY.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/AWS_DEPLOY.md b/docs/AWS_DEPLOY.md index 74ce998..730912f 100644 --- a/docs/AWS_DEPLOY.md +++ b/docs/AWS_DEPLOY.md @@ -672,6 +672,18 @@ sudo nginx -t # verify config sudo systemctl restart nginx ``` +> **Static file permissions:** Nginx runs as `www-data`, which must be able to +> traverse every directory in the path to `docker/static/`. Ubuntu home +> directories default to `750` (group-only execute), which blocks `www-data`. +> Fix it once after cloning: +> +> ```bash +> chmod o+x /home/ubuntu +> ``` +> +> Verify with `namei -l /home/ubuntu/portals/madrona-portal/docker/static/` — +> every component in the path needs at least `o+x`. + ### 7.4 Obtain an SSL certificate ```bash From 9b51f3ad437f03f02b70aa3c13ef284adfe1057d Mon Sep 17 00:00:00 2001 From: David Pollard Date: Fri, 8 May 2026 11:18:02 -0700 Subject: [PATCH 102/152] Update .env.example and settings.py to move most settings from .ini to .env --- docker/.env.example | 13 ++++++++++--- marco/marco/settings.py | 22 +++++++++++----------- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/docker/.env.example b/docker/.env.example index 9d88466..0c28d35 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -30,18 +30,24 @@ IMAGE_TAG=latest # Django core # --------------------------------------------------------------------------- SECRET_KEY=change-me-to-a-long-random-string -ALLOWED_HOSTS=localhost,127.0.0.1 +ALLOWED_HOSTS=localhost,127.0.0.1,::1 MP_PROJECT_CONFIG=config.wcoa.docker.ini DEBUG=False +CSRF_TRUSTED_ORIGINS="http://localhost,https://*.ecotrust.org" # --------------------------------------------------------------------------- # App info and configuration # --------------------------------------------------------------------------- APP_NAME="Madrona Portal" APP_TEAM_NAME="Marine Planner Team" +PROJECT_APP ="wcoa" +TIME_ZONE=UTC +COMPRESS_ENABLED=True +MAP_LIBRARY=ol8 MEDIA_ROOT=/usr/local/apps/madrona-portal/media MEDIA_URL=/media/ STATIC_ROOT=/vol/web/static +STATIC_CORE=/vol/web/static/ # --------------------------------------------------------------------------- # PostgreSQL / PostGIS @@ -58,7 +64,7 @@ APP_PORT=8000 # Redis (used for Django cache + Celery broker + result backend) # docker-compose builds REDIS_URL from REDIS_PASSWORD automatically. # --------------------------------------------------------------------------- -REDIS_PASSWORD= +REDIS_PASSWORD=changeme REDIS_PORT=6379 # REDIS_URL and CELERY_BROKER_URL are assembled in docker-compose.yml. @@ -70,6 +76,7 @@ EMAIL_PORT=587 EMAIL_HOST_USER=noreply@example.com EMAIL_HOST_PASSWORD=change-me EMAIL_USE_TLS=true +DEFAULT_FROM_EMAIL=noreply@prod.mail.ecotrust.org # --------------------------------------------------------------------------- # AWS SES (optional — only needed if EMAIL_BACKEND uses SES) @@ -111,7 +118,7 @@ RECAPTCHA_PRIVATE_KEY= # --------------------------------------------------------------------------- DJANGO_SUPERUSER_USERNAME=admin DJANGO_SUPERUSER_EMAIL=admin@example.com -DJANGO_SUPERUSER_PASSWORD= +DJANGO_SUPERUSER_PASSWORD=changeme # ================================================= # Catalog settings: Elasticsearch and Geoportal diff --git a/marco/marco/settings.py b/marco/marco/settings.py index ec2627e..8dd6a64 100644 --- a/marco/marco/settings.py +++ b/marco/marco/settings.py @@ -332,7 +332,7 @@ def _parse_hosts(raw: str | None) -> list[str]: # Internationalisation # --------------------------------------------------------------------------- LANGUAGE_CODE = 'en-us' -TIME_ZONE = app_cfg.get('TIME_ZONE', 'UTC') +TIME_ZONE = env_str('TIME_ZONE', app_cfg, 'TIME_ZONE', 'UTC') USE_I18N = True USE_TZ = True WAGTAIL_I18N_ENABLED = False @@ -343,7 +343,7 @@ def _parse_hosts(raw: str | None) -> list[str]: # --------------------------------------------------------------------------- STATIC_ROOT = env_str('STATIC_ROOT', app_cfg, 'STATIC_ROOT', os.path.join(BASE_DIR, 'static')) STATIC_URL = env_str('STATIC_URL', app_cfg, 'STATIC_URL', '/static/') -STATIC_CORE = app_cfg.get('STATIC_CORE', '') +STATIC_CORE = env_str('STATIC_CORE', app_cfg, 'STATIC_CORE', '') MEDIA_ROOT = env_str('MEDIA_ROOT', app_cfg, 'MEDIA_ROOT', os.path.join(BASE_DIR, 'media')) MEDIA_URL = env_str('MEDIA_URL', app_cfg, 'MEDIA_URL', '/media/') @@ -371,7 +371,7 @@ def _parse_hosts(raw: str | None) -> list[str]: COMPRESS_PRECOMPILERS = ( ('text/x-scss', 'django_libsass.SassCompiler'), ) -COMPRESS_ENABLED = app_cfg.getboolean('COMPRESS_ENABLED', True) +COMPRESS_ENABLED = env_bool('COMPRESS_ENABLED', app_cfg, 'COMPRESS_ENABLED', True) COMPRESS_OFFLINE = True # --------------------------------------------------------------------------- @@ -407,7 +407,7 @@ def _parse_hosts(raw: str | None) -> list[str]: # --------------------------------------------------------------------------- # Map / Geospatial # --------------------------------------------------------------------------- -MAP_LIBRARY = app_cfg.get('MAP_LIBRARY', 'ol6') +MAP_LIBRARY = env_str('MAP_LIBRARY', app_cfg, 'MAP_LIBRARY', 'ol6') GEOMETRY_DB_SRID = 3857 GEOMETRY_CLIENT_SRID = 3857 GEOJSON_SRID = 3857 @@ -530,11 +530,11 @@ def _parse_hosts(raw: str | None) -> list[str]: EMAIL_PORT = env_int('EMAIL_PORT', email_cfg, 'PORT', 25) EMAIL_HOST_USER = env_str('EMAIL_HOST_USER', email_cfg, 'HOST_USER', '') EMAIL_HOST_PASSWORD = env_str('EMAIL_HOST_PASSWORD', email_cfg, 'HOST_PASSWORD', '') -EMAIL_BACKEND = email_cfg.get('EMAIL_BACKEND', 'django.core.mail.backends.smtp.EmailBackend') -DEFAULT_FROM_EMAIL = email_cfg.get('DEFAULT_FROM_EMAIL', "MARCO Portal Team ") -SERVER_EMAIL = email_cfg.get('SERVER_EMAIL', "MARCO Site Errors ") +EMAIL_BACKEND = env_str('EMAIL_BACKEND', email_cfg, 'EMAIL_BACKEND', 'django.core.mail.backends.smtp.EmailBackend') +DEFAULT_FROM_EMAIL = env_str('DEFAULT_FROM_EMAIL', email_cfg, 'DEFAULT_FROM_EMAIL', "MARCO Portal Team ") +SERVER_EMAIL = env_str('SERVER_EMAIL', email_cfg, 'SERVER_EMAIL', "MARCO Site Errors ") EMAIL_USE_TLS = env_bool('EMAIL_USE_TLS', email_cfg, 'EMAIL_USE_TLS', False) -EMAIL_SUBJECT_PREFIX = app_cfg.get('EMAIL_SUBJECT_PREFIX', '[MARCO]') + ' ' +EMAIL_SUBJECT_PREFIX = env_str('EMAIL_SUBJECT_PREFIX', app_cfg, 'EMAIL_SUBJECT_PREFIX', '[MARCO]') + ' ' ADMINS = (('KSDev', 'ksdev@ecotrust.org'),) @@ -575,7 +575,7 @@ def _parse_hosts(raw: str | None) -> list[str]: # --------------------------------------------------------------------------- # Analytics # --------------------------------------------------------------------------- -GA_ACCOUNT = app_cfg.get('GA_ACCOUNT', '') +GA_ACCOUNT = env_str('GA_ACCOUNT', app_cfg, 'GA_ACCOUNT', '') # --------------------------------------------------------------------------- # NATIVE LANDS API KEY @@ -586,7 +586,7 @@ def _parse_hosts(raw: str | None) -> list[str]: # Project-level settings overrides # (Optional app + settings file specified in config.ini) # --------------------------------------------------------------------------- -PROJECT_APP = app_cfg.get('PROJECT_APP', '') +PROJECT_APP = env_str('PROJECT_APP', app_cfg, 'PROJECT_APP', '') if PROJECT_APP: INSTALLED_APPS.append(PROJECT_APP) @@ -595,7 +595,7 @@ def _parse_hosts(raw: str | None) -> list[str]: if 'data_manager' in INSTALLED_APPS: from data_manager.settings import * # noqa: F401, F403 -PROJECT_SETTINGS_FILE = app_cfg.get('PROJECT_SETTINGS_FILE', '') +PROJECT_SETTINGS_FILE = env_bool('PROJECT_SETTINGS_FILE', app_cfg, 'PROJECT_SETTINGS_FILE', '') if PROJECT_SETTINGS_FILE: try: from importlib import import_module From 5614cb2c82f0aa4259dd5486a272efad4b5d93c4 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Fri, 8 May 2026 14:25:11 -0700 Subject: [PATCH 103/152] Remove branch reference from WCOA clone command in README and update Docker workflow to use main branch for checkout --- .github/workflows/create-and-publish-docker-images.yml | 2 -- docker/README.md | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/create-and-publish-docker-images.yml b/.github/workflows/create-and-publish-docker-images.yml index 01f46f9..afd159b 100644 --- a/.github/workflows/create-and-publish-docker-images.yml +++ b/.github/workflows/create-and-publish-docker-images.yml @@ -153,8 +153,6 @@ jobs: uses: actions/checkout@v5 with: repository: Ecotrust/wcoa - # TODO: change to main branch once Docker-specific config is merged there - ref: vagrant2docker token: ${{ secrets.GH_PAT }} path: madrona-apps/wcoa diff --git a/docker/README.md b/docker/README.md index 51c9c36..fe57471 100644 --- a/docker/README.md +++ b/docker/README.md @@ -43,7 +43,7 @@ git clone https://github.com/Ecotrust/mp-proxy.git git clone https://github.com/Ecotrust/mp-survey.git git clone https://github.com/Ecotrust/mp-visualize.git git clone https://github.com/Ecotrust/p97-nursery.git -git clone -b vagrant2docker https://github.com/Ecotrust/wcoa.git +git clone https://github.com/Ecotrust/wcoa.git cd .. ``` From f5ead213c068df28df5ec9034fde03b2fc599fdc Mon Sep 17 00:00:00 2001 From: David Pollard Date: Mon, 11 May 2026 15:38:31 -0700 Subject: [PATCH 104/152] Update AWS deployment guide to use environment variable for Elasticsearch remote host and fix Docker commands --- docs/AWS_DEPLOY.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/AWS_DEPLOY.md b/docs/AWS_DEPLOY.md index 730912f..a8222b0 100644 --- a/docs/AWS_DEPLOY.md +++ b/docs/AWS_DEPLOY.md @@ -732,14 +732,14 @@ sudo vim /etc/hosts 3. Ensure the app is running, you can migrate existing GeoPortal records with the following command (replace the username and password): ```bash -time curl -X POST "http://localhost:9200/_reindex" -H 'Content-Type: application/json' -d'{"conflicts": "proceed", "max_docs": 51000, "source": {"remote": { "host":"http://elastic.prod.wcoa.ecotrust.org:80/geoportal/elastic/", "username": "[[USERNAME]]", "password": "[[PASSWORD]]" }, "index": "metadata", "size": 100 }, "dest": { "index": "metadata" } }' +time curl -X POST "http://localhost:9200/_reindex" -H 'Content-Type: application/json' -d'{"conflicts": "proceed", "max_docs": 51000, "source": {"remote": { "host":"http://[ES_REINDEX_REMOTE_WHITELIST]:80/geoportal/elastic/", "username": "[[USERNAME]]", "password": "[[PASSWORD]]" }, "index": "metadata", "size": 100 }, "dest": { "index": "metadata" } }' ``` 4. Do a down, including volumes, and up for the elasticsearch container and geoportal to pick up the new records: ```bash -docker compose -f docker/docker-compose.prod.yml down elasticsearch geoportal -v -docker compose -f docker/docker-compose.prod.yml up -d elasticsearch geoportal +docker compose -f docker/docker-compose.prod.yml down elastic geoportal -v +docker compose -f docker/docker-compose.prod.yml up -d elastic geoportal ``` --- From d290253a7f2e67f5025bd26459adf4eedafbd78d Mon Sep 17 00:00:00 2001 From: David Pollard Date: Wed, 13 May 2026 13:28:23 -0700 Subject: [PATCH 105/152] Move location of APP_PORT variable in .env.example to more logical line --- docker/.env.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/.env.example b/docker/.env.example index 0c28d35..855941e 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -48,6 +48,7 @@ MEDIA_ROOT=/usr/local/apps/madrona-portal/media MEDIA_URL=/media/ STATIC_ROOT=/vol/web/static STATIC_CORE=/vol/web/static/ +APP_PORT=8008 # --------------------------------------------------------------------------- # PostgreSQL / PostGIS @@ -58,7 +59,6 @@ DB_NAME=wcoa_docker_db DB_USER=postgres DB_PASSWORD=change-me # DB_HOST and DB_PORT are set inside docker-compose.yml (always "db" and 5432) -APP_PORT=8000 # --------------------------------------------------------------------------- # Redis (used for Django cache + Celery broker + result backend) From 4ddfaa7832fcec3366260ff768cec124e5c1c3f7 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Wed, 13 May 2026 16:35:18 -0700 Subject: [PATCH 106/152] Fix formatting of PROJECT_APP variable and add PROJECT_SETTINGS_FILE to .env.example --- docker/.env.example | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/.env.example b/docker/.env.example index 855941e..275ac47 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -40,7 +40,8 @@ CSRF_TRUSTED_ORIGINS="http://localhost,https://*.ecotrust.org" # --------------------------------------------------------------------------- APP_NAME="Madrona Portal" APP_TEAM_NAME="Marine Planner Team" -PROJECT_APP ="wcoa" +PROJECT_APP="wcoa" +PROJECT_SETTINGS_FILE=True TIME_ZONE=UTC COMPRESS_ENABLED=True MAP_LIBRARY=ol8 From 0d60d112d9f73931a402119deb309027e6085f8e Mon Sep 17 00:00:00 2001 From: David Pollard Date: Wed, 13 May 2026 17:14:56 -0700 Subject: [PATCH 107/152] Update link to AWS deployment guide in README for improved navigation --- docker/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/README.md b/docker/README.md index fe57471..d079414 100644 --- a/docker/README.md +++ b/docker/README.md @@ -272,7 +272,7 @@ docker compose up -d geoportal ## AWS EC2 -See [AWS_DEPLOY.md](AWS_DEPLOY.md) +See [AWS_DEPLOY.md](../docs/AWS_DEPLOY.md) --- From 93cbcf067369dcf95743a7b05d9b272641c2dbfa Mon Sep 17 00:00:00 2001 From: David Pollard Date: Thu, 14 May 2026 14:25:13 -0700 Subject: [PATCH 108/152] Trigger workflow on release publication --- .github/workflows/create-and-publish-docker-images.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/create-and-publish-docker-images.yml b/.github/workflows/create-and-publish-docker-images.yml index afd159b..ccd890a 100644 --- a/.github/workflows/create-and-publish-docker-images.yml +++ b/.github/workflows/create-and-publish-docker-images.yml @@ -7,6 +7,8 @@ name: Create and publish West Coast Ocean Data Portal (WCODP) Docker image # Image: ghcr.io/ecotrust/madrona-portal: (and :latest) on: + release: + types: [published] push: branches: ['docker'] # TODO: switch to main branch once we're ready to build from there From c2b4ab1ca5b9dbf93bf6cd85880d094e537508f9 Mon Sep 17 00:00:00 2001 From: Ryan Hodges Date: Fri, 15 May 2026 09:10:18 -0700 Subject: [PATCH 109/152] enable ES snapshotting and repo creation to local storage --- docker/backups/elasticsearch/blank.txt | 0 docker/docker-compose.prod.yml | 6 +++++- 2 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 docker/backups/elasticsearch/blank.txt diff --git a/docker/backups/elasticsearch/blank.txt b/docker/backups/elasticsearch/blank.txt new file mode 100644 index 0000000..e69de29 diff --git a/docker/docker-compose.prod.yml b/docker/docker-compose.prod.yml index 04fc50d..5b8b6bf 100644 --- a/docker/docker-compose.prod.yml +++ b/docker/docker-compose.prod.yml @@ -116,6 +116,7 @@ services: image: elasticsearch:8.19.12 volumes: - es-volume:/usr/share/elasticsearch/data + - ./backups/elasticsearch:/usr/share/elasticsearch/backups environment: - discovery.type=single-node - ES_JAVA_OPTS=-Xms512m -Xmx512m @@ -123,12 +124,15 @@ services: - ELASTIC_PASSWORD=${ELASTIC_PASSWORD} - bootstrap.memory_lock=true - reindex.remote.whitelist=${ES_REINDEX_REMOTE_WHITELIST} - - xpack.security.enabled=false + - xpack.security.enabled=false # DO NOT EXPOSE ELASTIC TO THE INTERNET!!! + - path.repo=/usr/share/elasticsearch/backups ports: - 9200:9200 - 9300:9300 networks: - madronanetwork + group_add: + - "0" # Add the elasticsearch user to the root group to allow backup permissions healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9200/_cluster/health?wait_for_status=yellow"] interval: 30s From b8911bef16a98fc6738409f2e553623e6321d2eb Mon Sep 17 00:00:00 2001 From: Ryan Hodges Date: Fri, 15 May 2026 09:17:12 -0700 Subject: [PATCH 110/152] adding 'dev' profile to docker prod to test integration with NGINX and kibana --- docker/docker-compose.prod.yml | 30 +++++++ docker/nginx.conf | 141 +++++++++++++++++++++++++++++++++ 2 files changed, 171 insertions(+) create mode 100644 docker/nginx.conf diff --git a/docker/docker-compose.prod.yml b/docker/docker-compose.prod.yml index 5b8b6bf..a0cb35d 100644 --- a/docker/docker-compose.prod.yml +++ b/docker/docker-compose.prod.yml @@ -140,6 +140,36 @@ services: retries: 10 restart: always + kibana: + image: kibana:8.19.12 + profiles: ["dev"] # Only start Kibana in dev profile + ports: + - 5601:5601 + environment: + ELASTICSEARCH_HOSTS: http://elastic:9200 + networks: + - madronanetwork + depends_on: + elastic: + condition: service_healthy + restart: always + + nginx: + image: nginx:alpine + profiles: ["dev"] # Only start nginx in dev profile + ports: + - "8081:80" + volumes: + - ./nginx.conf:/etc/nginx/conf.d/default.conf:ro + - ./static:/vol/web/static:ro + - ./media:/usr/local/apps/madrona-portal/media:ro + networks: + - madronanetwork + depends_on: + - app + - geoportal + restart: unless-stopped + volumes: postgis_data: redis_data: diff --git a/docker/nginx.conf b/docker/nginx.conf new file mode 100644 index 0000000..aca0841 --- /dev/null +++ b/docker/nginx.conf @@ -0,0 +1,141 @@ +server { + listen 80; + server_name _; + + # Use Docker's internal DNS so upstream hostnames are resolved at + # request time, not at nginx startup (avoids "host not found" errors + # when a backend container hasn't started yet). + resolver 127.0.0.11 valid=30s; + # TODO: consider writing logs to specific file + # access_log /var/log/nginx/wcoa.access.log; + # error_log /var/log/nginx/wcoa.error.log; + + # Increase client body size for file uploads + client_max_body_size 100M; + + # TODO - this was on prod - what is it? + # location /geospatial/ { + # alias /var/www/html/geopatial/; + # autoindex on; + # } + + # TODO: Add munin! + # location /munin/static/ { + # alias /etc/munin/static/; + # } + + # location /munin { + # alias /var/cache/munin/www; + # } + + location /static { + # Static files served from Docker volume + alias /vol/web/static/; + # prevent caching + add_header Last-Modified $date_gmt; + add_header Cache-Control 'no-store, no-cache, must-revalidate'; + add_header Pragma 'no-cache'; + add_header Expires 0; + if_modified_since off; + expires off; + etag off; + } + + location /media { + # Media files served from Docker volume + alias /usr/local/apps/madrona-portal/media/; + } + + location /favicon.ico { + # Favicon served from static volume + alias /vol/web/favicon.ico; + } + + # Elasticsearch routing - route search/doc/metadata requests to Elasticsearch + location ~ ^(/_search/|/_doc/|/metadata).*$ { + set $elastic_backend http://elastic:9200; + proxy_pass $elastic_backend; + proxy_redirect off; + proxy_connect_timeout 5s; + proxy_read_timeout 60s; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + # ENABLE CORS + if ($request_method = 'OPTIONS') { + add_header 'Access-Control-Allow-Origin' '*'; + add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; + # Custom headers and headers various browsers *should* be OK with but are not + add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range'; + # Tell client that this pre-flight info is valid for 20 days + add_header 'Access-Control-Max-Age' 1728000; + add_header 'Content-Type' 'text/plain; charset=utf-8'; + add_header 'Content-Length' 0; + return 204; + } + if ($request_method = 'POST') { + add_header 'Access-Control-Allow-Origin' '*'; + add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; + add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range'; + add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range'; + } + if ($request_method = 'GET') { + add_header 'Access-Control-Allow-Origin' '*'; + add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; + add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range'; + add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range'; + } + } + + # Geoportal management interfaces routing + location ~ ^/(manager|host-manager|semantix|solr|gc|geoportal|harvester).*$ { + set $geoportal_backend http://geoportal:8080; + proxy_pass $geoportal_backend; + proxy_redirect off; + proxy_connect_timeout 5s; + proxy_read_timeout 60s; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + # ENABLE CORS + if ($request_method = 'OPTIONS') { + add_header 'Access-Control-Allow-Origin' '*'; + add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; + # Custom headers and headers various browsers *should* be OK with but are not + add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range'; + # Tell client that this pre-flight info is valid for 20 days + add_header 'Access-Control-Max-Age' 1728000; + add_header 'Content-Type' 'text/plain; charset=utf-8'; + add_header 'Content-Length' 0; + return 204; + } + if ($request_method = 'POST') { + add_header 'Access-Control-Allow-Origin' '*'; + add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; + add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range'; + add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range'; + } + if ($request_method = 'GET') { + add_header 'Access-Control-Allow-Origin' '*'; + add_header 'Access-Control-Allow-Methods' 'GET, POST, OPTIONS'; + add_header 'Access-Control-Allow-Headers' 'DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range'; + add_header 'Access-Control-Expose-Headers' 'Content-Length,Content-Range'; + } + } + + # Default routing - everything else goes to Django app + location / { + set $app_backend http://app:8008; + proxy_pass $app_backend; + proxy_redirect off; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } +} + From 8a28d6762c113305cdd474b152f5b093a34c313f Mon Sep 17 00:00:00 2001 From: Ryan Hodges Date: Fri, 15 May 2026 11:23:13 -0700 Subject: [PATCH 111/152] don't risk committing backup files --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index e122a0b..d91f333 100644 --- a/.gitignore +++ b/.gitignore @@ -16,6 +16,7 @@ vagrant .sass-cache node_modules +docker/backups/ docker/media/ docker/static/ docker/entrypoint.sh From 464c2897060eac1bb0916d3b632cab7287c03118 Mon Sep 17 00:00:00 2001 From: Ryan Hodges Date: Fri, 15 May 2026 11:24:06 -0700 Subject: [PATCH 112/152] limit Kibana to localhost access only for security --- docker/docker-compose.prod.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/docker-compose.prod.yml b/docker/docker-compose.prod.yml index a0cb35d..026a1b2 100644 --- a/docker/docker-compose.prod.yml +++ b/docker/docker-compose.prod.yml @@ -144,7 +144,7 @@ services: image: kibana:8.19.12 profiles: ["dev"] # Only start Kibana in dev profile ports: - - 5601:5601 + - 127.0.0.1:5601:5601 environment: ELASTICSEARCH_HOSTS: http://elastic:9200 networks: From 8b34d0d3ed7ab27e1bd24d26308361b38d5e4263 Mon Sep 17 00:00:00 2001 From: Ryan Hodges Date: Fri, 15 May 2026 11:30:16 -0700 Subject: [PATCH 113/152] fix typo in commented-out nginx config placeholder WAF --- docker/nginx.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/nginx.conf b/docker/nginx.conf index aca0841..b4209fc 100644 --- a/docker/nginx.conf +++ b/docker/nginx.conf @@ -15,7 +15,7 @@ server { # TODO - this was on prod - what is it? # location /geospatial/ { - # alias /var/www/html/geopatial/; + # alias /var/www/html/geospatial/; # autoindex on; # } From a6f692651aa98474c3ef58d7ac916be7da93cc59 Mon Sep 17 00:00:00 2001 From: Ryan Hodges Date: Fri, 15 May 2026 11:37:31 -0700 Subject: [PATCH 114/152] don't auto-restart dev-profile services --- docker/docker-compose.prod.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/docker-compose.prod.yml b/docker/docker-compose.prod.yml index 026a1b2..29cc077 100644 --- a/docker/docker-compose.prod.yml +++ b/docker/docker-compose.prod.yml @@ -152,7 +152,7 @@ services: depends_on: elastic: condition: service_healthy - restart: always + restart: no nginx: image: nginx:alpine @@ -168,7 +168,7 @@ services: depends_on: - app - geoportal - restart: unless-stopped + restart: no volumes: postgis_data: From 824ccae6211e20d13115d2adce306e8a2ee795b8 Mon Sep 17 00:00:00 2001 From: Ryan Hodges Date: Fri, 15 May 2026 11:40:27 -0700 Subject: [PATCH 115/152] clean up some ambiguous comments --- docker/nginx.conf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/nginx.conf b/docker/nginx.conf index b4209fc..7d37469 100644 --- a/docker/nginx.conf +++ b/docker/nginx.conf @@ -13,13 +13,13 @@ server { # Increase client body size for file uploads client_max_body_size 100M; - # TODO - this was on prod - what is it? + # For reference: SCCWRP server also hosted a WAF - we may need to restore this on AWS. # location /geospatial/ { # alias /var/www/html/geospatial/; # autoindex on; # } - # TODO: Add munin! + # For reference only: add Munin and its static files # location /munin/static/ { # alias /etc/munin/static/; # } From 61ca0565b8a2bd3bd391837677c513a5b96615ad Mon Sep 17 00:00:00 2001 From: Ryan Hodges Date: Fri, 15 May 2026 13:12:13 -0700 Subject: [PATCH 116/152] clarifying comment regarding xpack-security config for elastic service --- docker/docker-compose.prod.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker/docker-compose.prod.yml b/docker/docker-compose.prod.yml index 29cc077..dc423f2 100644 --- a/docker/docker-compose.prod.yml +++ b/docker/docker-compose.prod.yml @@ -124,7 +124,9 @@ services: - ELASTIC_PASSWORD=${ELASTIC_PASSWORD} - bootstrap.memory_lock=true - reindex.remote.whitelist=${ES_REINDEX_REMOTE_WHITELIST} - - xpack.security.enabled=false # DO NOT EXPOSE ELASTIC TO THE INTERNET!!! + # Disable security so that GeoPortal can write to Elasticsearch. + # DO NOT EXPOSE ELASTIC TO THE INTERNET until we solve for this and enable xpack.security. + - xpack.security.enabled=false - path.repo=/usr/share/elasticsearch/backups ports: - 9200:9200 From 276db5f1365585869aa0036095b46a0c20761d21 Mon Sep 17 00:00:00 2001 From: Ryan Hodges Date: Fri, 15 May 2026 17:26:59 -0700 Subject: [PATCH 117/152] Adding a reference template for building a crontab --- deployment/crontab.template | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 deployment/crontab.template diff --git a/deployment/crontab.template b/deployment/crontab.template new file mode 100644 index 0000000..8695d22 --- /dev/null +++ b/deployment/crontab.template @@ -0,0 +1,32 @@ + Edit this file to introduce tasks to be run by cron. +# +# Each task to run has to be defined through a single line +# indicating with different fields when the task will be run +# and what command to run for the task +# +# To define the time you can provide concrete values for +# minute (m), hour (h), day of month (dom), month (mon), +# and day of week (dow) or use '*' in these fields (for 'any'). +# +# Notice that tasks will be started based on the cron's system +# daemon's notion of time and timezones. +# +# Output of the crontab jobs (including errors) is sent through +# email to the user the crontab file belongs to (unless redirected). +# +# For example, you can run a backup of all your user accounts +# at 5 a.m every week with: +# 0 5 * * 1 tar -zcf /var/backups/home.tgz /home/ +# +# For more information see the manual pages of crontab(5) and cron(8) +# +# NOTE: EBS Backup is scheduled for XX:XX every XXX +# +# m h dom mon dow command +# 02:15 daily (18:15 PST) - Dump PostgreSQL DB to dumpfile +15 2 * * * cd /home/ubuntu/portals/madrona-portal && /bin/bash -lc './backups/db_dump.sh -d ./backups/sql && find ./backups/sql -type f -n> +# 03:15 daily (19:15 PST) - Snapshot Elasticsearch to FS +15 3 * * * /usr/bin/curl -X PUT "localhost:9200/_snapshot/gp_es_stage_snap/snapshot_$(date +'%Y%m%d_%H%M')" -H 'Content-Type: application/json' -d '{"indices": "metadata_v1", "ignore_unavailable": true, "include_global_state": false}' +# 05:31 daily (21:31 PST) - Refresh NativeLand JSON layers +31 5 * * * cd /home/ubuntu/portals/madrona-portal/docker && docker compose exec app marco/manage.py import_nativeland + From 66ce86edd5ec0f63808728f2a5b1abb82f34e4bd Mon Sep 17 00:00:00 2001 From: Ryan Hodges Date: Fri, 15 May 2026 17:29:48 -0700 Subject: [PATCH 118/152] Updating AWS Deploy doc: WARs, WAFs, Backups, and Cron --- docs/AWS_DEPLOY.md | 127 +++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 118 insertions(+), 9 deletions(-) diff --git a/docs/AWS_DEPLOY.md b/docs/AWS_DEPLOY.md index a8222b0..598e827 100644 --- a/docs/AWS_DEPLOY.md +++ b/docs/AWS_DEPLOY.md @@ -238,17 +238,15 @@ reboots. You only need to re-run this if the PAT expires. --- -## Phase 3 — Transfer Geoportal WAR Files +## Phase 3 — Locate/Create and upload Geoportal WAR Files The Geoportal service requires two Java WAR files that are not in any Git -repository. Copy them from the old production server. - -**On the old server**, find the WAR files: - -```bash -# Common locations on the old server: -find / -name "geoportal.war" -o -name "harvester.war" 2>/dev/null -``` +repository. Since these files include passwords (even if encrypted) +Ecotrust keeps custom, private builds in our shared drive. If you do +not have access to those, you can build your own using Maven (beyond the +scope of this document) on the repos cloned from: +* https://github.com/Ecotrust/geoportal-server-catalog (to create `geoportal.war`) +* https://github.com/Ecotrust/geoportal-server-harvester (to create `harvester.war`) **On your local machine**, SCP them to the new EC2 instance: @@ -891,6 +889,117 @@ echo '/swapfile none swap sw 0 0' | sudo tee -a /etc/fstab GA_ACCOUNT=G-XXXXXXXXXX ``` +--- + +## Create local metadata Web Accessible Folder +You may have noticed we created a `/geospatial` location in the Nginx config. +This is used to serve metadata records used by input brokers in the GeoPortal +Harvester as well as referenced by layers stored in mp-layers' Layers records. + +Referring to the Nginx config, you'll see it refers to /var/www/html/geospatial. +All you need to do is create this folder, ensure www-data has read privileges, +and the endpoint will work (it will be empty). If migrating from an existing +server, you can copy those folders and existing metadata records over using scp +or wget (they should be public-readable). + +--- + +## Backups and Snaptshots + +AWS Lifecycle Management allows you to easily create snapshots of your EBS volumes. +This is great for data stored as files (media, staticfiles, WAFs, etc...) but is +not sufficient for capturing data in databases as the snapshot is not atomic, and +we have not scheduled our databases to stop for these snapshots. Instead, it's +important to use each database's built-in tools to create file-system-level +backups that will be accurately represented in any EBS volume snapshot. + +Those dumps are best managed by Cron jobs (covered below) but in some cases, some +prep work is required. + +### Elasticsearch Snapshot Repository + +Elasticsearch uses a built-in API-based tool called 'Snapshot' to create file-system +level backups that are appropriate for restoring your database to prior states or +populating from scratch. Before you can create a snapshot, you must first create a +Snapshot Repository. + +Below: +* REPO_NAME - the name you wish to use for your repository. + * This will need to be recorded and set again in your cron job that creates the snapshots +* /usr/share/elasticsearch/backups + * This is the assumed location of the backups folder from the perspective of the elasticsearch container + * This value should be correct if you did not edit docker-compose.prod.yml + * Docker maps this folder to your local `~/portals/madrona-portal/docker/backups/elasticsearch` + * This way the files representing your snapshots are exposed during the EBS backup + +``` +curl -X PUT "localhost:9200/_snapshot/{REPO_NAME}" -H "Content-Type: application/json" -d '{"type": "fs", "settings": {"location": "/usr/share/elasticsearch/backups", "compress": true}}' + +``` + +You can confirm the creation/existence of this repository +``` +curl -X GET "localhost:9200/_cat/repositories?v" +``` + +### EBS Snapshots (lifecycle) + +Be sure to set up regularly recurring snapshots of your EBS volume(s) so that +data may be recovered in a disaster. This is easily managed in the EC2 dashboard. + +--- + +## Cron Jobs +There are several tasks that should run regularly to ensure your server is +serving up-to-date data and keeping file-system-level backup dumps to +ensure AWS Lifecycle snapshots can capture accurate representations of the +database contents/state. + +Please see [crontab.template](../deployment/crontab.template) for an up-to-date +list of recommended jobs with examples of how to implement them. + +### PostgreSQL/Django dump +``` +cd /home/ubuntu/portals/madrona-portal && /bin/bash -lc './backups/db_dump.sh -d ./backups/sql && find ./backups/sql -type f -n> +``` +Uses the built-in `db_dump.sh` script to dump the database to a local SQL file + +### Elasticsearch/GeoPortal snapshot +``` +/usr/bin/curl -X PUT "localhost:9200/_snapshot/{REPO_NAME}/snapshot_$(date +'%Y%m%d_%H%M')" -H 'Content-Type: application/json' -d '{"indices": "{INDEX_NAME}", "ignore_unavailable": true, "include_global_state": false}' +``` +* REPO_NAME + * You should have set this in the section on Elasticsearch Snapshot Repositories above +* INDEX_NAME + * This will most likely be `metadata_v1` + * You can get a list of index names from the Elasticsearch API: + * `curl -X GET "localhost:9200/_cat/indices?v"` + +You can review the name of your snapshot(s) with: +``` +curl -X GET "localhost:9200/_cat/snapshots?v" +``` + +You can review the status of your snapshot(s) with: +``` +curl -X GET "localhost:9200/_snapshot/{REPO_NAME}/{SNAPSHOT_NAME}" +``` +* SNAPSHOT_NAME + * You should get this from the previous `_cat/snapshots` query + + +### NativeLands Digital JSON files +``` +cd /home/ubuntu/portals/madrona-portal/docker && docker compose exec app marco/manage.py import_nativeland +``` + +Assuming you have an API key for https://native-land.ca/ and configured your .env correctly, this management +command should pull 3 layers in locally as GeoJSONs to be served statically for vector layers in the Portal: +* Languages +* Historic Territories +* Treaties + + --- ## Deploying a New Release From 6a5a8e2c40c0ecf0ba54e466c24dc0b65f0ba1ae Mon Sep 17 00:00:00 2001 From: Ryan Hodges Date: Fri, 15 May 2026 17:38:31 -0700 Subject: [PATCH 119/152] renaming the docker nginx config file for clarity --- docker/docker-compose.prod.yml | 2 +- docker/{nginx.conf => nginx-dev.conf} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename docker/{nginx.conf => nginx-dev.conf} (100%) diff --git a/docker/docker-compose.prod.yml b/docker/docker-compose.prod.yml index dc423f2..ee183ff 100644 --- a/docker/docker-compose.prod.yml +++ b/docker/docker-compose.prod.yml @@ -162,7 +162,7 @@ services: ports: - "8081:80" volumes: - - ./nginx.conf:/etc/nginx/conf.d/default.conf:ro + - ./nginx-dev.conf:/etc/nginx/conf.d/default.conf:ro - ./static:/vol/web/static:ro - ./media:/usr/local/apps/madrona-portal/media:ro networks: diff --git a/docker/nginx.conf b/docker/nginx-dev.conf similarity index 100% rename from docker/nginx.conf rename to docker/nginx-dev.conf From 5b7b8c18680494d08274f1b47dc7506c98b0f351 Mon Sep 17 00:00:00 2001 From: Ryan Hodges Date: Fri, 15 May 2026 18:49:17 -0700 Subject: [PATCH 120/152] making elastic snapshot cron job work --- backups/create_elastic_snapshot.sh | 18 ++++++++++++++++++ deployment/crontab.template | 2 +- 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100755 backups/create_elastic_snapshot.sh diff --git a/backups/create_elastic_snapshot.sh b/backups/create_elastic_snapshot.sh new file mode 100755 index 0000000..df5ec4b --- /dev/null +++ b/backups/create_elastic_snapshot.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +REPOSITORY="" + +while getopts "r:" opt; do + case $opt in + r) REPOSITORY="$OPTARG" ;; + *) echo "Usage: $0 -r "; exit 1 ;; + esac +done + +if [[ -z "$REPOSITORY" ]]; then + echo "Error: -r is required" + exit 1 +fi + +DATETIME_VAR=$(date +%Y%m%d_%H%M) +/usr/bin/curl -X PUT "localhost:9200/_snapshot/${REPOSITORY}/snapshot_${DATETIME_VAR}" -H 'Content-Type: application/json' -d '{"indices": "metadata_v1", "ignore_unavailable": true, "include_global_state": false}' diff --git a/deployment/crontab.template b/deployment/crontab.template index 8695d22..b0364f0 100644 --- a/deployment/crontab.template +++ b/deployment/crontab.template @@ -26,7 +26,7 @@ # 02:15 daily (18:15 PST) - Dump PostgreSQL DB to dumpfile 15 2 * * * cd /home/ubuntu/portals/madrona-portal && /bin/bash -lc './backups/db_dump.sh -d ./backups/sql && find ./backups/sql -type f -n> # 03:15 daily (19:15 PST) - Snapshot Elasticsearch to FS -15 3 * * * /usr/bin/curl -X PUT "localhost:9200/_snapshot/gp_es_stage_snap/snapshot_$(date +'%Y%m%d_%H%M')" -H 'Content-Type: application/json' -d '{"indices": "metadata_v1", "ignore_unavailable": true, "include_global_state": false}' +15 3 * * * /usr/bin/bash /home/ubuntu/portals/madrona-portal/backups/create_elastic_snapshot.sh -r gp_es_snap # 05:31 daily (21:31 PST) - Refresh NativeLand JSON layers 31 5 * * * cd /home/ubuntu/portals/madrona-portal/docker && docker compose exec app marco/manage.py import_nativeland From 313ada1b3adde1012a9c13d5d3dca4a96ed336d4 Mon Sep 17 00:00:00 2001 From: Ryan Hodges Date: Fri, 15 May 2026 18:55:32 -0700 Subject: [PATCH 121/152] fixing db dump cron to replace truncated content --- deployment/crontab.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment/crontab.template b/deployment/crontab.template index b0364f0..03daf04 100644 --- a/deployment/crontab.template +++ b/deployment/crontab.template @@ -24,7 +24,7 @@ # # m h dom mon dow command # 02:15 daily (18:15 PST) - Dump PostgreSQL DB to dumpfile -15 2 * * * cd /home/ubuntu/portals/madrona-portal && /bin/bash -lc './backups/db_dump.sh -d ./backups/sql && find ./backups/sql -type f -n> +15 2 * * * cd /home/ubuntu/portals/madrona-portal && /bin/bash -lc './backups/db_dump.sh -d ./backups/sql && find ./backups/sql -type f -name "*.sql" -mtime +10 -delete' >> /home/ubuntu/portals/madrona-portal/backups/db_dump.log 2>&1 # 03:15 daily (19:15 PST) - Snapshot Elasticsearch to FS 15 3 * * * /usr/bin/bash /home/ubuntu/portals/madrona-portal/backups/create_elastic_snapshot.sh -r gp_es_snap # 05:31 daily (21:31 PST) - Refresh NativeLand JSON layers From 7a5fbbb195c75965de1b8d3168dffe7e9c27dcda Mon Sep 17 00:00:00 2001 From: Ryan Hodges Date: Fri, 15 May 2026 18:57:28 -0700 Subject: [PATCH 122/152] fixing missing comment in crontab.template --- deployment/crontab.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deployment/crontab.template b/deployment/crontab.template index 03daf04..9528bbf 100644 --- a/deployment/crontab.template +++ b/deployment/crontab.template @@ -1,4 +1,4 @@ - Edit this file to introduce tasks to be run by cron. +# Edit this file to introduce tasks to be run by cron. # # Each task to run has to be defined through a single line # indicating with different fields when the task will be run From 6e245638149d2adef92bd8fbcce17581ac962ecd Mon Sep 17 00:00:00 2001 From: Ryan Hodges Date: Fri, 15 May 2026 19:00:51 -0700 Subject: [PATCH 123/152] fixing cron job notes in AWS deploy --- docs/AWS_DEPLOY.md | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/docs/AWS_DEPLOY.md b/docs/AWS_DEPLOY.md index 598e827..c315294 100644 --- a/docs/AWS_DEPLOY.md +++ b/docs/AWS_DEPLOY.md @@ -960,21 +960,17 @@ list of recommended jobs with examples of how to implement them. ### PostgreSQL/Django dump ``` -cd /home/ubuntu/portals/madrona-portal && /bin/bash -lc './backups/db_dump.sh -d ./backups/sql && find ./backups/sql -type f -n> +cd /home/ubuntu/portals/madrona-portal && /bin/bash -lc './backups/db_dump.sh -d ./backups/sql && find ./backups/sql -type f -name "*.sql" -mtime +10 -delete' >> /home/ubuntu/portals/madrona-portal/backups/db_dump.log 2>&1 ``` Uses the built-in `db_dump.sh` script to dump the database to a local SQL file ### Elasticsearch/GeoPortal snapshot ``` -/usr/bin/curl -X PUT "localhost:9200/_snapshot/{REPO_NAME}/snapshot_$(date +'%Y%m%d_%H%M')" -H 'Content-Type: application/json' -d '{"indices": "{INDEX_NAME}", "ignore_unavailable": true, "include_global_state": false}' +/usr/bin/bash /home/ubuntu/portals/madrona-portal/backups/create_elastic_snapshot.sh -r {REPO_NAME} ``` * REPO_NAME * You should have set this in the section on Elasticsearch Snapshot Repositories above -* INDEX_NAME - * This will most likely be `metadata_v1` - * You can get a list of index names from the Elasticsearch API: - * `curl -X GET "localhost:9200/_cat/indices?v"` - + You can review the name of your snapshot(s) with: ``` curl -X GET "localhost:9200/_cat/snapshots?v" From cfc6645cf66bd5801f97794372c0f57f860f4705 Mon Sep 17 00:00:00 2001 From: Ryan Hodges Date: Fri, 15 May 2026 19:01:26 -0700 Subject: [PATCH 124/152] fixing minor type in AWS Deploy --- docs/AWS_DEPLOY.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/AWS_DEPLOY.md b/docs/AWS_DEPLOY.md index c315294..eb698e0 100644 --- a/docs/AWS_DEPLOY.md +++ b/docs/AWS_DEPLOY.md @@ -904,7 +904,7 @@ or wget (they should be public-readable). --- -## Backups and Snaptshots +## Backups and Snapshots AWS Lifecycle Management allows you to easily create snapshots of your EBS volumes. This is great for data stored as files (media, staticfiles, WAFs, etc...) but is @@ -970,7 +970,7 @@ Uses the built-in `db_dump.sh` script to dump the database to a local SQL file ``` * REPO_NAME * You should have set this in the section on Elasticsearch Snapshot Repositories above - + You can review the name of your snapshot(s) with: ``` curl -X GET "localhost:9200/_cat/snapshots?v" From ac8acdb1996ed29f74967f77f312fddbb1405eea Mon Sep 17 00:00:00 2001 From: Ryan Hodges Date: Thu, 21 May 2026 17:16:50 -0700 Subject: [PATCH 125/152] adding url-shortener and layers dependencies to README --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 2a76fe6..d25d766 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,8 @@ The following is the **_recommended_** folder structure for the **entire** MARCO git clone https://github.com/Ecotrust/mp-data-manager.git git clone https://github.com/Ecotrust/mp-drawing.git git clone https://github.com/Ecotrust/mp-explore.git + git clone https://github.com/Ecotrust/mp-layers.git + git clone https://github.com/Ecotrust/mp-map-groups.git git clone https://github.com/Ecotrust/mp-proxy.git git clone https://github.com/Ecotrust/mp-visualize.git git clone https://github.com/Ecotrust/p97-nursery.git From 454f112de774a00fbb4c6b1b0e8bce5d5081744a Mon Sep 17 00:00:00 2001 From: David Pollard Date: Wed, 27 May 2026 14:17:37 -0700 Subject: [PATCH 126/152] correct SES Console navigation for creating mail from domain --- docs/AWS_DEPLOY.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/AWS_DEPLOY.md b/docs/AWS_DEPLOY.md index a8222b0..c323828 100644 --- a/docs/AWS_DEPLOY.md +++ b/docs/AWS_DEPLOY.md @@ -826,7 +826,7 @@ DEFAULT_FROM_EMAIL=noreply@prod.mail.ecotrust.org ``` ### Create custom mail from domain -1. In SES Console → Domains → click on your domain → Create mail from domain +1. In SES Console → Identities → click on your domain → Create mail from domain 2. Enter a subdomain (e.g., `mail`) → Create 3. Add the provided DNS records to your DNS provider 4. Wait for AWS to verify the mail from domain From 7dcc75f6db1cd7092a0dc10b128760409845b856 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Wed, 27 May 2026 17:16:34 -0700 Subject: [PATCH 127/152] Move docker/README.md to docs/Docker_ Development_Guide.md --- docker/README.md => docs/Docker_Development_Guide.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docker/README.md => docs/Docker_Development_Guide.md (100%) diff --git a/docker/README.md b/docs/Docker_Development_Guide.md similarity index 100% rename from docker/README.md rename to docs/Docker_Development_Guide.md From 30fd309ca533626e67edbd6489da92b2ef581303 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Fri, 29 May 2026 16:15:42 -0700 Subject: [PATCH 128/152] Remove deprecated documentation files: ROADMAP.md, Docker Development Guide.md, and logs_and_config.md --- ROADMAP.md | 91 ------ docs/Docker_Development_Guide.md | 332 ---------------------- MODERNIZATION.md => docs/MODERNIZATION.md | 0 logs_and_config.md | 3 - 4 files changed, 426 deletions(-) delete mode 100644 ROADMAP.md delete mode 100644 docs/Docker_Development_Guide.md rename MODERNIZATION.md => docs/MODERNIZATION.md (100%) delete mode 100644 logs_and_config.md diff --git a/ROADMAP.md b/ROADMAP.md deleted file mode 100644 index 6dde9ab..0000000 --- a/ROADMAP.md +++ /dev/null @@ -1,91 +0,0 @@ -Bootstrap base components: - - - Use this in the base template as a block: http://getbootstrap.com/components/#page-header - - Use these as basis for cards: http://getbootstrap.com/components/#thumbnails-custom-content - - how to park elements? http://stackoverflow.com/questions/21301316/how-to-bootstrap-navbar-static-to-fixed-on-scroll - -Done: - - - Bootstrap in stack - - bower install - - gulp, scss - - vendor build (not source) - - hard code top nav - -Major task groups - - - Styles - - Base typog - - Heading typog - - Color swatches - - Logo - - Topbar styles - - - CMS - - add groups - - Stubbed UI (stock bootstrap, hard coded content) - - Scaffolding (real data) - - skeleton content templates - - Production - - Basic deploy - - Search (elasticsearch) - - Redis caching - - cron tasks (or similar): - - search: update_index - - publish_scheduled_pages - - ocean story functionality/prototype (real data) - - pull top nav menu items from CMS (?) - - figure out how placing pages in the menus should work (with Jenny) - - filter/search/view switch for cards - - coupled Marine Planner instance - -Lay out incremental goals for next week and the rest of Nov - - - ol3 map on OS pages - - maybe render some data? - - - first pass: - - stock OL3 (http://docs.openlayers.org/library/introduction.html) - - openlayers 3 - - hosted, initially - - will need to setup build env, likely - - https://github.com/openlayers/ol3/blob/master/CONTRIBUTING.md - - http://boundlessgeo.com/2014/02/openlayers-3-custom-builds/ - - bower - - build - -How to have full width media elements inside the content area? -http://stackoverflow.com/questions/24049467/how-to-create-a-100-screen-width-div-inside-a-container-in-bootstrap - - -Top nav Menu - - 2nd level only? - - How to designate which menu a page goes in? (E) - -Login item? - - "My MARCO +ICON" - -Grid/list view - - what metadata fields in CMS to support filtering? (E) - -Event page: look at comp, make sure CMS has all the fields (E) - - esp Address - -groups need to be in beta in some form - -summary: for beta, implement Join a group/public groups page, but not "My groups" - - dependency ordering with portal update? - - public group profiles, centrally administered for beta - - "this group elsewhere" field (with help text suggesting fb, goog, mailing lists, etc) - - linked to a marine planner group? (URL or id or something?) - - request to join (manual addition) diff --git a/docs/Docker_Development_Guide.md b/docs/Docker_Development_Guide.md deleted file mode 100644 index d079414..0000000 --- a/docs/Docker_Development_Guide.md +++ /dev/null @@ -1,332 +0,0 @@ -# Docker Development Guide — Madrona Portal (WCOA) - -## Quick start - -These steps take a fresh machine from nothing to a running portal. - -### Step 1 — Create the workspace directory - -All repos live inside a single parent directory. The Dockerfile build -context is the parent, so the layout is not optional. - -```bash -mkdir portals -cd portals -``` - -### Step 2 — Clone madrona-portal - -```bash -git clone -b docker https://github.com/Ecotrust/madrona-portal.git madrona-portal -``` - -### Step 3 — Clone the sub-app packages - -The Dockerfile copies all of these at build time. Clone them into a -`madrona-apps/` sibling directory: - -```bash -mkdir madrona-apps && cd madrona-apps - -git clone https://github.com/Ecotrust/django_url_shortener.git -git clone https://github.com/Ecotrust/madrona-analysistools.git -git clone https://github.com/Ecotrust/madrona-features.git -git clone https://github.com/Ecotrust/madrona-manipulators.git -git clone https://github.com/Ecotrust/madrona-scenarios.git -git clone https://github.com/Ecotrust/mp-accounts.git -git clone https://github.com/Ecotrust/mp-data-manager.git -git clone https://github.com/Ecotrust/mp-drawing.git -git clone https://github.com/Ecotrust/mp-explore.git -git clone https://github.com/Ecotrust/mp-layers.git -git clone https://github.com/Ecotrust/mp-map-groups.git -git clone https://github.com/Ecotrust/mp-proxy.git -git clone https://github.com/Ecotrust/mp-survey.git -git clone https://github.com/Ecotrust/mp-visualize.git -git clone https://github.com/Ecotrust/p97-nursery.git -git clone https://github.com/Ecotrust/wcoa.git - -cd .. -``` - -Your workspace should now look like: - -``` -portals/ -├── madrona-portal/ ← cloned from Ecotrust/madrona-portal, branch: docker -└── madrona-apps/ - ├── wcoa/ ← branch: vagrant2docker - ├── mp-layers/ - └── ... ← all others on main -``` - -### Step 4 — Configure environment - -From `madrona-portal/`: - -```bash -cd madrona-portal/docker -cp .env.example .env -``` - -Edit `.env` and set at minimum: - -```ini -SECRET_KEY= -DB_PASSWORD= -DJANGO_SUPERUSER_PASSWORD= -``` - -Everything else has working defaults for local development. - -### Step 4.1 - Create ini file - -```bash -cd ../marco -cp config.docker.ini.template config.wcoa.docker.ini -``` - -Edit `config.wcoa.docker.ini` : - -```ini -LOCATION = redis://tasks:6379/1 -CELERY_RESULT_BACKEND = redis://tasks:6379/1 -CELERY_BROKER_URL = redis://tasks:6379/0 -``` - -### Step 5 — Build the image - -Run this command from `madrona-portal/docker` (the previous step leaves -you in `madrona-portal/marco`, so `cd ../docker` gets you there). The -Docker build context for this command is `../../`, which resolves to the parent workspace directory `portals/` that contains both -`madrona-portal/` and `madrona-apps/`. - -If running this build from a MAC, add `--builder desktop-linux` to the `buildx build` command. - -```bash -cd ../docker -# MAC OS -docker compose build --no-cache app -# LINUX -docker compose build --no-cache app -``` - -Previously we recommended `docker buildx build --builder desktop-linux --no-cache --load -f ./Dockerfile ../../` for all platforms, but BuildKit caching on Linux does not appear to have the same git object store issue as on Mac, so the simpler `docker compose build --no-cache app` is sufficient on Linux. - -> **Why `docker buildx build` and not `docker compose build`?** -> `docker compose build` has a caching bug: when a `.git` directory exists -> inside the build context, BuildKit reads files from the git object store -> (committed versions) rather than the filesystem. If you forget to commit -> a change, the old version is silently baked into the image. The same -> restriction applies — always commit changes to `madrona-portal/` or -> `madrona-apps/` before rebuilding. - -### Step 6 — Start the full stack; Populate testing DB - -From `madrona-portal/docker`: - -```bash -DB_INIT=1 docker compose up -``` - -On first boot the entrypoint automatically: - -1. Waits for PostgreSQL to accept connections -2. Runs `migrate` -3. Runs `collectstatic` and `compress` -4. Detects a fresh database and loads initial fixtures (1,782 + 22 objects) -5. Creates the superuser defined in `.env` (if `DJANGO_SUPERUSER_PASSWORD` is set) -6. Starts the application server - -Open: http://localhost:8000/ (or whatever `APP_PORT` is set to in `.env`) - -Once you have a populated DB (either dummy or with migrated data) omit the `DB_INIT=1`: -```bash -docker compose up -``` - - -### Step 7 - Importing a Production SQL Dump into the Dockerized Database - -#### Prerequisites -- Docker Compose stack is running — `docker compose up` -- `madrona-portal/docker/.env` exists with `DB_NAME`, `DB_USER`, and `DB_PASSWORD` set - - -#### Step 7.1 — Ensure you have the db-restore script - -`madrona-portal/scripts/db-restore.sh` has the following behaviour: -- Loads DB credentials from `.env` -- Verifies the `db` container is healthy before proceeding -- With `--drop`: terminates active connections, drops and recreates the database, and enables the PostGIS extension -- Streams the dump file directly into the container via `docker compose exec` (no temp files) -- Prints next-step instructions on completion - -Made sure it is executable: - -From `madrona-portal/docker`: -```bash -chmod +x ../scripts/db-restore.sh -``` - -#### Step 7.2 — Run the restore - -From `madrona-portal/docker`: -```bash -../scripts/db-restore.sh --drop -``` - -*example:* -```bash -../scripts/db-restore.sh --drop ../../madrona-apps/wcoa/wcodp_prod_dump_20260320.sql -``` - -The `--drop` flag was used to ensure a clean import. The script: -1. Terminated all active connections to `wcoa_docker_db` -2. Dropped and recreated the database -3. Enabled the `postgis` extension -4. Streamed the sql dump into the container via `psql` - -There is an optional `--env-file ` if you place your `.env` file in a non-standard location. - -**Expected warnings (non-fatal):** -- `ERROR: relation "..." does not exist` — pg_dump tries to drop constraints before creating them; safe to ignore on a fresh DB -- `ERROR: role "wcoa_user" does not exist` — prod uses a dedicated app role; dev uses `postgres` which has full access - - -#### Step 7.3 — Apply migrations - -```bash -docker compose exec app python marco/manage.py migrate -``` - -#### Step 7.4 - Migration to mp-layers - -*If migrating from a server that has not migrated to mp-layers from mp-data-manager*: - -```bash -docker compose exec app python marco/manage.py migration_to_layers -``` - ---- - -### Step 8 — Importing production media files into the Dockerized Application - -#### Prerequisites -- `madrona-portal/marco/marco/config.wcoa.docker.ini` exists with `MEDIA_ROOT` set to a valid directory -- That valid directory should match the volume location is docker-compose.yml - - `portals/madrona-portal/media` -- Production media files are available - -#### Step 8.1 - Copy the media files into Docker - -If media files need to be copied to the server, you can use `scp`: - -```bash -scp -r /path/to/your_media_dir ubuntu@:/home/ubuntu/portals/madrona-portal/docker/media -``` - -If media files are somewhere on EC2: - -```bash -cd ~/portals/madrona-portal/docker -cp -r {your_media_dir}/* ./media/ -``` - ---- - -## Migrate existing GeoPortal records - -1. Update .env with the reindex remote whitelist and port. You can find this info by looking at the old server's configuration or by asking the previous admin. - -2. Edit the hosts file to allow the server to resolve the old Elastic IP of the GeoPortal instance to the new internal Docker network: - -```bash -sudo vim /etc/hosts -# Add the following line, replacing and : - -``` - -3. Restart the elastic container to apply the .env and hosts file change: - -```bash -docker compose down -v elastic -docker compose up -d elastic -``` - -4. Ensure the app is running, you can migrate existing GeoPortal records with the following command (replace the username and password): - -```bash -time curl -X POST "http://localhost:9200/_reindex" -H 'Content-Type: application/json' -d'{"conflicts": "proceed", "max_docs": 51000, "source": {"remote": { "host":"http://elastic.prod.wcoa.ecotrust.org:80/geoportal/elastic/", "username": "[[USERNAME]]", "password": "[[PASSWORD]]" }, "index": "metadata", "size": 100 }, "dest": { "index": "metadata" } }' -``` - -1. Do a down, including volumes, and up for the geoportal container to pick up the new records: - -```bash -docker compose down -v geoportal -docker compose up -d geoportal -``` - ---- - -# Deploy to fully containerized production environment - -## AWS EC2 - -See [AWS_DEPLOY.md](../docs/AWS_DEPLOY.md) - ---- - -## Dev infrastructure only (local Django server) - -To run Django locally against Docker-managed PostGIS and Redis (no app container): - -```bash -# Start only db and tasks -docker compose up -d db tasks - -# Then in a separate terminal, from madrona-portal/: -cd marco -python manage.py runserver -``` - ---- - -## Rebuilding after code changes - -From the docker directory (`madrona-portal/docker`): - -```bash -docker compose build --no-cache app -``` - -| Note on `--no-cache`: Use it when dependencies have changed; without it, Docker reuses the cached pip install layer (needed when `docker-requirements.txt` changes). - -Then bring the stack up: - -```bash -docker compose up --force-recreate -``` - ---- - -## Reset to a clean state - -```bash -# From madrona-portal/ -docker compose -f docker/docker-compose.yml --env-file .env --profile full down -v -``` - -`-v` removes the PostGIS and Redis volumes. The next `up` will re-run -migrations and reload fixtures from scratch. - ---- - -## Disk space - -Docker's build cache can grow large over time: - -```bash -docker system df # show usage breakdown -docker system prune -f # remove stopped containers, dangling images, unused networks, build cache -docker volume prune -f # remove unused volumes — only run when all containers are stopped -``` diff --git a/MODERNIZATION.md b/docs/MODERNIZATION.md similarity index 100% rename from MODERNIZATION.md rename to docs/MODERNIZATION.md diff --git a/logs_and_config.md b/logs_and_config.md deleted file mode 100644 index f48fa6b..0000000 --- a/logs_and_config.md +++ /dev/null @@ -1,3 +0,0 @@ -**Logs**: `/home/midatlantic/logs/user/` - -**Config**: `/home/midatlantic/webapps/marco_portal/apache2/conf/httpd.conf` From 374a9493393db723bf2fb92072d95756eb4c6b82 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Fri, 29 May 2026 16:15:51 -0700 Subject: [PATCH 129/152] Add data_manager directory creation step to Docker setup guide --- docs/AWS_DEPLOY.md | 5 + docs/DOCKER_README.md | 337 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 342 insertions(+) create mode 100644 docs/DOCKER_README.md diff --git a/docs/AWS_DEPLOY.md b/docs/AWS_DEPLOY.md index c323828..47c39dc 100644 --- a/docs/AWS_DEPLOY.md +++ b/docs/AWS_DEPLOY.md @@ -717,6 +717,11 @@ docker compose -f docker/docker-compose.prod.yml up -d --force-recreate app scp -r {your_media_dir} ubuntu@:/home/ubuntu/portals/madrona-portal/docker/media/ ``` +#### Create a directory for data_manager +```bash +mkdir ~/portals/madrona-portal/docker/data_manager +``` + ## Migrate existing GeoPortal records 1. Update .env with the reindex remote whitelist and port. You can find this info by looking at the old server's configuration or by asking the previous admin. diff --git a/docs/DOCKER_README.md b/docs/DOCKER_README.md new file mode 100644 index 0000000..b9b8470 --- /dev/null +++ b/docs/DOCKER_README.md @@ -0,0 +1,337 @@ +# Docker Development Guide — Madrona Portal (WCOA) + +## Quick start + +These steps take a fresh machine from nothing to a running portal. + +### Step 1 — Create the workspace directory + +All repos live inside a single parent directory. The Dockerfile build +context is the parent, so the layout is not optional. + +```bash +mkdir portals +cd portals +``` + +### Step 2 — Clone madrona-portal + +```bash +git clone -b docker https://github.com/Ecotrust/madrona-portal.git madrona-portal +``` + +### Step 3 — Clone the sub-app packages + +The Dockerfile copies all of these at build time. Clone them into a +`madrona-apps/` sibling directory: + +```bash +mkdir madrona-apps && cd madrona-apps + +git clone https://github.com/Ecotrust/django_url_shortener.git +git clone https://github.com/Ecotrust/madrona-analysistools.git +git clone https://github.com/Ecotrust/madrona-features.git +git clone https://github.com/Ecotrust/madrona-manipulators.git +git clone https://github.com/Ecotrust/madrona-scenarios.git +git clone https://github.com/Ecotrust/mp-accounts.git +git clone https://github.com/Ecotrust/mp-data-manager.git +git clone https://github.com/Ecotrust/mp-drawing.git +git clone https://github.com/Ecotrust/mp-explore.git +git clone https://github.com/Ecotrust/mp-layers.git +git clone https://github.com/Ecotrust/mp-map-groups.git +git clone https://github.com/Ecotrust/mp-proxy.git +git clone https://github.com/Ecotrust/mp-survey.git +git clone https://github.com/Ecotrust/mp-visualize.git +git clone https://github.com/Ecotrust/p97-nursery.git +git clone https://github.com/Ecotrust/wcoa.git + +cd .. +``` + +Your workspace should now look like: + +``` +portals/ +├── madrona-portal/ ← cloned from Ecotrust/madrona-portal, branch: docker +└── madrona-apps/ + ├── wcoa/ ← branch: vagrant2docker + ├── mp-layers/ + └── ... ← all others on main +``` + +### Step 4 — Configure environment + +From `madrona-portal/`: + +```bash +cd madrona-portal/docker +cp .env.example .env +``` + +Edit `.env` and set at minimum: + +```ini +SECRET_KEY= +DB_PASSWORD= +DJANGO_SUPERUSER_PASSWORD= +``` + +Everything else has working defaults for local development. + +### Step 4.1 - Create ini file + +```bash +cd ../marco +cp config.docker.ini.template config.wcoa.docker.ini +``` + +Edit `config.wcoa.docker.ini` : + +```ini +LOCATION = redis://tasks:6379/1 +CELERY_RESULT_BACKEND = redis://tasks:6379/1 +CELERY_BROKER_URL = redis://tasks:6379/0 +``` + +### Step 5 — Build the image + +Run this command from `madrona-portal/docker` (the previous step leaves +you in `madrona-portal/marco`, so `cd ../docker` gets you there). The +Docker build context for this command is `../../`, which resolves to the parent workspace directory `portals/` that contains both +`madrona-portal/` and `madrona-apps/`. + +If running this build from a MAC, add `--builder desktop-linux` to the `buildx build` command. + +```bash +cd ../docker +# MAC OS +docker compose build --no-cache app +# LINUX +docker compose build --no-cache app +``` + +Previously we recommended `docker buildx build --builder desktop-linux --no-cache --load -f ./Dockerfile ../../` for all platforms, but BuildKit caching on Linux does not appear to have the same git object store issue as on Mac, so the simpler `docker compose build --no-cache app` is sufficient on Linux. + +> **Why `docker buildx build` and not `docker compose build`?** +> `docker compose build` has a caching bug: when a `.git` directory exists +> inside the build context, BuildKit reads files from the git object store +> (committed versions) rather than the filesystem. If you forget to commit +> a change, the old version is silently baked into the image. The same +> restriction applies — always commit changes to `madrona-portal/` or +> `madrona-apps/` before rebuilding. + +### Step 6 — Start the full stack; Populate testing DB + +From `madrona-portal/docker`: + +```bash +DB_INIT=1 docker compose up +``` + +On first boot the entrypoint automatically: + +1. Waits for PostgreSQL to accept connections +2. Runs `migrate` +3. Runs `collectstatic` and `compress` +4. Detects a fresh database and loads initial fixtures (1,782 + 22 objects) +5. Creates the superuser defined in `.env` (if `DJANGO_SUPERUSER_PASSWORD` is set) +6. Starts the application server + +Open: http://localhost:8000/ (or whatever `APP_PORT` is set to in `.env`) + +Once you have a populated DB (either dummy or with migrated data) omit the `DB_INIT=1`: +```bash +docker compose up +``` + + +### Step 7 - Importing a Production SQL Dump into the Dockerized Database + +#### Prerequisites +- Docker Compose stack is running — `docker compose up` +- `madrona-portal/docker/.env` exists with `DB_NAME`, `DB_USER`, and `DB_PASSWORD` set + + +#### Step 7.1 — Ensure you have the db-restore script + +`madrona-portal/scripts/db-restore.sh` has the following behaviour: +- Loads DB credentials from `.env` +- Verifies the `db` container is healthy before proceeding +- With `--drop`: terminates active connections, drops and recreates the database, and enables the PostGIS extension +- Streams the dump file directly into the container via `docker compose exec` (no temp files) +- Prints next-step instructions on completion + +Made sure it is executable: + +From `madrona-portal/docker`: +```bash +chmod +x ../scripts/db-restore.sh +``` + +#### Step 7.2 — Run the restore + +From `madrona-portal/docker`: +```bash +../scripts/db-restore.sh --drop +``` + +*example:* +```bash +../scripts/db-restore.sh --drop ../../madrona-apps/wcoa/wcodp_prod_dump_20260320.sql +``` + +The `--drop` flag was used to ensure a clean import. The script: +1. Terminated all active connections to `wcoa_docker_db` +2. Dropped and recreated the database +3. Enabled the `postgis` extension +4. Streamed the sql dump into the container via `psql` + +There is an optional `--env-file ` if you place your `.env` file in a non-standard location. + +**Expected warnings (non-fatal):** +- `ERROR: relation "..." does not exist` — pg_dump tries to drop constraints before creating them; safe to ignore on a fresh DB +- `ERROR: role "wcoa_user" does not exist` — prod uses a dedicated app role; dev uses `postgres` which has full access + + +#### Step 7.3 — Apply migrations + +```bash +docker compose exec app python marco/manage.py migrate +``` + +#### Step 7.4 - Migration to mp-layers + +*If migrating from a server that has not migrated to mp-layers from mp-data-manager*: + +```bash +docker compose exec app python marco/manage.py migration_to_layers +``` + +--- + +### Step 8 — Importing production media files into the Dockerized Application + +#### Prerequisites +- `madrona-portal/marco/marco/config.wcoa.docker.ini` exists with `MEDIA_ROOT` set to a valid directory +- That valid directory should match the volume location is docker-compose.yml + - `portals/madrona-portal/media` +- Production media files are available + +#### Step 8.1 - Copy the media files into Docker + +If media files need to be copied to the server, you can use `scp`: + +```bash +scp -r /path/to/your_media_dir ubuntu@:/home/ubuntu/portals/madrona-portal/docker/media +``` + +If media files are somewhere on EC2: + +```bash +cd ~/portals/madrona-portal/docker +cp -r {your_media_dir}/* ./media/ +``` + +#### Create a directory for data_manager +```bash +mkdir ~/portals/madrona-portal/docker/data_manager +``` + +--- + +## Migrate existing GeoPortal records + +1. Update .env with the reindex remote whitelist and port. You can find this info by looking at the old server's configuration or by asking the previous admin. + +2. Edit the hosts file to allow the server to resolve the old Elastic IP of the GeoPortal instance to the new internal Docker network: + +```bash +sudo vim /etc/hosts +# Add the following line, replacing and : + +``` + +3. Restart the elastic container to apply the .env and hosts file change: + +```bash +docker compose down -v elastic +docker compose up -d elastic +``` + +4. Ensure the app is running, you can migrate existing GeoPortal records with the following command (replace the username and password): + +```bash +time curl -X POST "http://localhost:9200/_reindex" -H 'Content-Type: application/json' -d'{"conflicts": "proceed", "max_docs": 51000, "source": {"remote": { "host":"http://elastic.prod.wcoa.ecotrust.org:80/geoportal/elastic/", "username": "[[USERNAME]]", "password": "[[PASSWORD]]" }, "index": "metadata", "size": 100 }, "dest": { "index": "metadata" } }' +``` + +1. Do a down, including volumes, and up for the geoportal container to pick up the new records: + +```bash +docker compose down -v geoportal +docker compose up -d geoportal +``` + +--- + +# Deploy to fully containerized production environment + +## AWS EC2 + +See [AWS_DEPLOY.md](../docs/AWS_DEPLOY.md) + +--- + +## Dev infrastructure only (local Django server) + +To run Django locally against Docker-managed PostGIS and Redis (no app container): + +```bash +# Start only db and tasks +docker compose up -d db tasks + +# Then in a separate terminal, from madrona-portal/: +cd marco +python manage.py runserver +``` + +--- + +## Rebuilding after code changes + +From the docker directory (`madrona-portal/docker`): + +```bash +docker compose build --no-cache app +``` + +| Note on `--no-cache`: Use it when dependencies have changed; without it, Docker reuses the cached pip install layer (needed when `docker-requirements.txt` changes). + +Then bring the stack up: + +```bash +docker compose up --force-recreate +``` + +--- + +## Reset to a clean state + +```bash +# From madrona-portal/ +docker compose -f docker/docker-compose.yml --env-file .env --profile full down -v +``` + +`-v` removes the PostGIS and Redis volumes. The next `up` will re-run +migrations and reload fixtures from scratch. + +--- + +## Disk space + +Docker's build cache can grow large over time: + +```bash +docker system df # show usage breakdown +docker system prune -f # remove stopped containers, dangling images, unused networks, build cache +docker volume prune -f # remove unused volumes — only run when all containers are stopped +``` From a9c3f6ba516554b6c8520710d87430096fe0668a Mon Sep 17 00:00:00 2001 From: David Pollard Date: Wed, 3 Jun 2026 15:57:49 -0700 Subject: [PATCH 130/152] Update PostgreSQL volume path in Docker Compose files --- docker/docker-compose.prod.yml | 2 +- docker/docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/docker-compose.prod.yml b/docker/docker-compose.prod.yml index dc423f2..8fcf699 100644 --- a/docker/docker-compose.prod.yml +++ b/docker/docker-compose.prod.yml @@ -60,7 +60,7 @@ services: db: image: postgis/postgis:16-3.4 volumes: - - postgis_data:/var/lib/postgresql + - postgis_data:/var/lib/postgresql/data environment: - POSTGRES_USER=${DB_USER:-postgres} - POSTGRES_PASSWORD=${DB_PASSWORD} diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 0d88ec3..89a11c9 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -94,7 +94,7 @@ services: db: image: postgis/postgis:16-3.4 volumes: - - postgis_data:/var/lib/postgresql + - postgis_data:/var/lib/postgresql/data environment: - POSTGRES_USER=${DB_USER:-postgres} - POSTGRES_PASSWORD=${DB_PASSWORD} From 0438ee880f3247a34a939e75152e21a910055ddf Mon Sep 17 00:00:00 2001 From: Ryan Hodges Date: Mon, 8 Jun 2026 10:37:53 -0700 Subject: [PATCH 131/152] preserve harvester's H2 DB across container events --- docker/docker-compose.prod.yml | 2 ++ docker/geoportal-entrypoint.sh | 22 ++++++++++++---------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/docker/docker-compose.prod.yml b/docker/docker-compose.prod.yml index 8fcf699..5147162 100644 --- a/docker/docker-compose.prod.yml +++ b/docker/docker-compose.prod.yml @@ -98,6 +98,7 @@ services: - 8080:8080 volumes: - gp-volume:/usr/local/tomcat/webapps/ + - harvester_data:/root - ${gpt_catalog_war}:/usr/local/tomcat/webapps/geoportal.war - ${gpt_harvester_war}:/usr/local/tomcat/webapps/harvester.war - ./templates:/templates:ro @@ -177,6 +178,7 @@ volumes: redis_data: gp-volume: es-volume: + harvester_data: networks: madronanetwork: diff --git a/docker/geoportal-entrypoint.sh b/docker/geoportal-entrypoint.sh index cbdc495..692dd88 100755 --- a/docker/geoportal-entrypoint.sh +++ b/docker/geoportal-entrypoint.sh @@ -159,25 +159,22 @@ else echo "✗ app-security.xml missing" fi -# Stop background Tomcat +# Stop background Tomcat gracefully so H2 can flush and release its lock echo "Stopping background Tomcat (PID: $TOMCAT_PID)..." -# Check if the process is still running if kill -0 $TOMCAT_PID 2>/dev/null; then - echo "Sending TERM signal to Tomcat..." - kill $TOMCAT_PID - - # Wait for graceful shutdown (up to 10 seconds) - for i in {1..10}; do + echo "Requesting graceful Tomcat shutdown via catalina.sh stop..." + catalina.sh stop 30 -force + # Wait for the background process to exit + for i in {1..35}; do if ! kill -0 $TOMCAT_PID 2>/dev/null; then echo "Tomcat stopped gracefully" break fi - echo "Waiting for shutdown... ${i}/10" + echo "Waiting for shutdown... ${i}/35" sleep 1 done - - # Force kill if still running + # Final safety net if kill -0 $TOMCAT_PID 2>/dev/null; then echo "Force stopping Tomcat..." kill -9 $TOMCAT_PID @@ -189,6 +186,11 @@ fi echo "Tomcat stopped successfully" +# Remove any stale H2 lock/trace files left by the background Tomcat run. +# These persist in the named volume and prevent the DB from opening on restart. +echo "Cleaning up stale H2 artifacts in /root..." +rm -f /root/harvester.lock.db /root/harvester.trace.db + # Start Tomcat in foreground echo "Starting Tomcat with updated configuration..." exec catalina.sh run \ No newline at end of file From 0696cd20f7dbac24ecc9da4c2cdce9a577ef2261 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Wed, 8 Jul 2026 13:58:14 -0700 Subject: [PATCH 132/152] Update README.md for Docker development setup and instructions --- README.md | 462 +++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 315 insertions(+), 147 deletions(-) diff --git a/README.md b/README.md index d25d766..b9b8470 100644 --- a/README.md +++ b/README.md @@ -1,169 +1,337 @@ -# MARCO Portal Redesign +# Docker Development Guide — Madrona Portal (WCOA) -### This is the top level project for the Mid-Atlantic Ocean Data Portal +## Quick start -### ~Development Installation +These steps take a fresh machine from nothing to a running portal. -##### Initial Setup using Vagrant: -The following is the **_recommended_** folder structure for the **entire** MARCO project and the customized provisioning script is inherently dependent on it. Altering the folder and naming structure will require modifications to the provisioning script, so please be aware! The provisioning script is designed to be a **one-step** install after initial setup. +### Step 1 — Create the workspace directory +All repos live inside a single parent directory. The Dockerfile build +context is the parent, so the layout is not optional. + +```bash +mkdir portals +cd portals +``` + +### Step 2 — Clone madrona-portal + +```bash +git clone -b docker https://github.com/Ecotrust/madrona-portal.git madrona-portal +``` + +### Step 3 — Clone the sub-app packages + +The Dockerfile copies all of these at build time. Clone them into a +`madrona-apps/` sibling directory: + +```bash +mkdir madrona-apps && cd madrona-apps + +git clone https://github.com/Ecotrust/django_url_shortener.git +git clone https://github.com/Ecotrust/madrona-analysistools.git +git clone https://github.com/Ecotrust/madrona-features.git +git clone https://github.com/Ecotrust/madrona-manipulators.git +git clone https://github.com/Ecotrust/madrona-scenarios.git +git clone https://github.com/Ecotrust/mp-accounts.git +git clone https://github.com/Ecotrust/mp-data-manager.git +git clone https://github.com/Ecotrust/mp-drawing.git +git clone https://github.com/Ecotrust/mp-explore.git +git clone https://github.com/Ecotrust/mp-layers.git +git clone https://github.com/Ecotrust/mp-map-groups.git +git clone https://github.com/Ecotrust/mp-proxy.git +git clone https://github.com/Ecotrust/mp-survey.git +git clone https://github.com/Ecotrust/mp-visualize.git +git clone https://github.com/Ecotrust/p97-nursery.git +git clone https://github.com/Ecotrust/wcoa.git + +cd .. +``` + +Your workspace should now look like: + +``` +portals/ +├── madrona-portal/ ← cloned from Ecotrust/madrona-portal, branch: docker +└── madrona-apps/ + ├── wcoa/ ← branch: vagrant2docker + ├── mp-layers/ + └── ... ← all others on main +``` + +### Step 4 — Configure environment + +From `madrona-portal/`: + +```bash +cd madrona-portal/docker +cp .env.example .env +``` + +Edit `.env` and set at minimum: + +```ini +SECRET_KEY= +DB_PASSWORD= +DJANGO_SUPERUSER_PASSWORD= +``` + +Everything else has working defaults for local development. + +### Step 4.1 - Create ini file + +```bash +cd ../marco +cp config.docker.ini.template config.wcoa.docker.ini +``` + +Edit `config.wcoa.docker.ini` : + +```ini +LOCATION = redis://tasks:6379/1 +CELERY_RESULT_BACKEND = redis://tasks:6379/1 +CELERY_BROKER_URL = redis://tasks:6379/0 +``` + +### Step 5 — Build the image + +Run this command from `madrona-portal/docker` (the previous step leaves +you in `madrona-portal/marco`, so `cd ../docker` gets you there). The +Docker build context for this command is `../../`, which resolves to the parent workspace directory `portals/` that contains both +`madrona-portal/` and `madrona-apps/`. + +If running this build from a MAC, add `--builder desktop-linux` to the `buildx build` command. + +```bash +cd ../docker +# MAC OS +docker compose build --no-cache app +# LINUX +docker compose build --no-cache app +``` + +Previously we recommended `docker buildx build --builder desktop-linux --no-cache --load -f ./Dockerfile ../../` for all platforms, but BuildKit caching on Linux does not appear to have the same git object store issue as on Mac, so the simpler `docker compose build --no-cache app` is sufficient on Linux. + +> **Why `docker buildx build` and not `docker compose build`?** +> `docker compose build` has a caching bug: when a `.git` directory exists +> inside the build context, BuildKit reads files from the git object store +> (committed versions) rather than the filesystem. If you forget to commit +> a change, the old version is silently baked into the image. The same +> restriction applies — always commit changes to `madrona-portal/` or +> `madrona-apps/` before rebuilding. + +### Step 6 — Start the full stack; Populate testing DB + +From `madrona-portal/docker`: + +```bash +DB_INIT=1 docker compose up +``` + +On first boot the entrypoint automatically: + +1. Waits for PostgreSQL to accept connections +2. Runs `migrate` +3. Runs `collectstatic` and `compress` +4. Detects a fresh database and loads initial fixtures (1,782 + 22 objects) +5. Creates the superuser defined in `.env` (if `DJANGO_SUPERUSER_PASSWORD` is set) +6. Starts the application server + +Open: http://localhost:8000/ (or whatever `APP_PORT` is set to in `.env`) + +Once you have a populated DB (either dummy or with migrated data) omit the `DB_INIT=1`: +```bash +docker compose up +``` + + +### Step 7 - Importing a Production SQL Dump into the Dockerized Database + +#### Prerequisites +- Docker Compose stack is running — `docker compose up` +- `madrona-portal/docker/.env` exists with `DB_NAME`, `DB_USER`, and `DB_PASSWORD` set + + +#### Step 7.1 — Ensure you have the db-restore script + +`madrona-portal/scripts/db-restore.sh` has the following behaviour: +- Loads DB credentials from `.env` +- Verifies the `db` container is healthy before proceeding +- With `--drop`: terminates active connections, drops and recreates the database, and enables the PostGIS extension +- Streams the dump file directly into the container via `docker compose exec` (no temp files) +- Prints next-step instructions on completion + +Made sure it is executable: + +From `madrona-portal/docker`: +```bash +chmod +x ../scripts/db-restore.sh +``` + +#### Step 7.2 — Run the restore + +From `madrona-portal/docker`: +```bash +../scripts/db-restore.sh --drop ``` - -- madrona-portal - -- apps (all remaining repositories within Madrona Portal) - -- mardona-analysistools - -- madrona-features - -- etc. + +*example:* +```bash +../scripts/db-restore.sh --drop ../../madrona-apps/wcoa/wcodp_prod_dump_20260320.sql +``` + +The `--drop` flag was used to ensure a clean import. The script: +1. Terminated all active connections to `wcoa_docker_db` +2. Dropped and recreated the database +3. Enabled the `postgis` extension +4. Streamed the sql dump into the container via `psql` + +There is an optional `--env-file ` if you place your `.env` file in a non-standard location. + +**Expected warnings (non-fatal):** +- `ERROR: relation "..." does not exist` — pg_dump tries to drop constraints before creating them; safe to ignore on a fresh DB +- `ERROR: role "wcoa_user" does not exist` — prod uses a dedicated app role; dev uses `postgres` which has full access + + +#### Step 7.3 — Apply migrations + +```bash +docker compose exec app python marco/manage.py migrate +``` + +#### Step 7.4 - Migration to mp-layers + +*If migrating from a server that has not migrated to mp-layers from mp-data-manager*: + +```bash +docker compose exec app python marco/manage.py migration_to_layers +``` + +--- + +### Step 8 — Importing production media files into the Dockerized Application + +#### Prerequisites +- `madrona-portal/marco/marco/config.wcoa.docker.ini` exists with `MEDIA_ROOT` set to a valid directory +- That valid directory should match the volume location is docker-compose.yml + - `portals/madrona-portal/media` +- Production media files are available + +#### Step 8.1 - Copy the media files into Docker + +If media files need to be copied to the server, you can use `scp`: + +```bash +scp -r /path/to/your_media_dir ubuntu@:/home/ubuntu/portals/madrona-portal/docker/media +``` + +If media files are somewhere on EC2: + +```bash +cd ~/portals/madrona-portal/docker +cp -r {your_media_dir}/* ./media/ +``` + +#### Create a directory for data_manager +```bash +mkdir ~/portals/madrona-portal/docker/data_manager +``` + +--- + +## Migrate existing GeoPortal records + +1. Update .env with the reindex remote whitelist and port. You can find this info by looking at the old server's configuration or by asking the previous admin. + +2. Edit the hosts file to allow the server to resolve the old Elastic IP of the GeoPortal instance to the new internal Docker network: + +```bash +sudo vim /etc/hosts +# Add the following line, replacing and : + ``` -1. Download the required code and dependencies: +3. Restart the elastic container to apply the .env and hosts file change: + +```bash +docker compose down -v elastic +docker compose up -d elastic ``` - git clone https://github.com/Ecotrust/madrona-portal.git - mv madrona-portal madrona-portal - cd madrona-portal - mkdir apps - cd apps - git clone https://github.com/Ecotrust/madrona-analysistools.git - git clone https://github.com/Ecotrust/madrona-features.git - git clone https://github.com/Ecotrust/madrona-manipulators.git - git clone https://github.com/Ecotrust/madrona-scenarios.git - git clone https://github.com/Ecotrust/mp-map-groups.git - git clone https://github.com/Ecotrust/mp-accounts.git - git clone https://github.com/Ecotrust/mp-data-manager.git - git clone https://github.com/Ecotrust/mp-drawing.git - git clone https://github.com/Ecotrust/mp-explore.git - git clone https://github.com/Ecotrust/mp-layers.git - git clone https://github.com/Ecotrust/mp-map-groups.git - git clone https://github.com/Ecotrust/mp-proxy.git - git clone https://github.com/Ecotrust/mp-visualize.git - git clone https://github.com/Ecotrust/p97-nursery.git + +4. Ensure the app is running, you can migrate existing GeoPortal records with the following command (replace the username and password): + +```bash +time curl -X POST "http://localhost:9200/_reindex" -H 'Content-Type: application/json' -d'{"conflicts": "proceed", "max_docs": 51000, "source": {"remote": { "host":"http://elastic.prod.wcoa.ecotrust.org:80/geoportal/elastic/", "username": "[[USERNAME]]", "password": "[[PASSWORD]]" }, "index": "metadata", "size": 100 }, "dest": { "index": "metadata" } }' ``` -2. Once your folder structure is set up, create a `config.ini` file by making a copy of the `config.ini.template` located at `madrona-portal/marco` and modify the following - * **SECRET_KEY** = [Punch in some random gibberish] - * **MEDIA_ROOT** = /home/vagrant/marco_portal2/media - * **STATIC_ROOT** = /home/vagrant/marco_portal2/static - * **LOCATION** = /var/run/redis/redis.sock - * **RESULT_BACKEND** = redis+socket:///var/run/redis/redis.sock - * **BROKER_URL** = redis+socket:///var/run/redis/redis.sock +1. Do a down, including volumes, and up for the geoportal container to pick up the new records: + +```bash +docker compose down -v geoportal +docker compose up -d geoportal +``` + +--- + +# Deploy to fully containerized production environment + +## AWS EC2 + +See [AWS_DEPLOY.md](../docs/AWS_DEPLOY.md) -3. Create a `/static/` directory at the root level and move the `/bower_components/` directory (also found at the root level) within it +--- -4. Create a `/media/` directory at the root level and retrieve the live server's media folder via ssh/sftp located at `/webapps/marco_portal_media/` and add it to the `/media/` path. Refer to your team's technical documentation for server login (username and password) credentials - * Of note - you may want to exclude the `data_manager` folder within the media directory - unless you're interested in several GBs of utfgrid layers. +## Dev infrastructure only (local Django server) + +To run Django locally against Docker-managed PostGIS and Redis (no app container): + +```bash +# Start only db and tasks +docker compose up -d db tasks + +# Then in a separate terminal, from madrona-portal/: +cd marco +python manage.py runserver ``` -mkdir media -scp -r user@live_server:~/webapps/marco_portal_media/documents ./media/ #11s -- RDH 7/27/2017 -scp -r user@live_server:~/webapps/marco_portal_media/group_images ./media/ #11s -scp -r user@live_server:~/webapps/marco_portal_media/images ./media/ #3m19s -scp -r user@live_server:~/webapps/marco_portal_media/original_images ./media/ #3m57s -scp user@live_server:~/webapps/marco_portal_media/index.html ./media/ #12s + +--- + +## Rebuilding after code changes + +From the docker directory (`madrona-portal/docker`): + +```bash +docker compose build --no-cache app ``` -5. Retrieve the data & content fixture from `~/fixtures/dev_fixture.json` via ssh/sftp and place it at the root level of `madrona-portal` +| Note on `--no-cache`: Use it when dependencies have changed; without it, Docker reuses the cached pip install layer (needed when `docker-requirements.txt` changes). + +Then bring the stack up: + +```bash +docker compose up --force-recreate ``` -cd [working dir]/madrona-portal -scp user@live_server:~/fixtures/dev_fixture.json ./ #25s + +--- + +## Reset to a clean state + +```bash +# From madrona-portal/ +docker compose -f docker/docker-compose.yml --env-file .env --profile full down -v ``` -6. Download and install [vagrant](https://www.vagrantup.com/downloads.html) and [virtual box](https://www.virtualbox.org/wiki/Downloads) (if you haven't already done so already) - -7. At the root of `madrona-portal`, run `vagrant up` and let it install ALL of dependencies MARCO relies upon - -8. At this point, you should be completely setup! - * Note: At this point, there still seem to be issues with Wagtail Pages, and therefore Ocean Stories. - * This is due to needing to configure for Redis to run on a socket, and Celery to point to that socket. - * This is likely not the only way, but it's what I have working and how it runs on production. --RDH - -9. You probably want to create a superuser once you're in your VM, so that you have access to both the Django and Wagtail backend - -##### Using Vagrant -* Access your VM by running `vagrant ssh`. This will automatically log you into your virtual machine with your virtual environment activated at the project root level. - - -* **Shortcuts** - * To use `/manage.py` with normal django administrative tasks , use the keyword `dj` - - ``` - dj makemigrations - dj migrate - dj createsuperuser - dj dumpdata - etc. - ``` - - * Typing `djrun` will run your dev server - remember to add your sample data first (see #5): - - -* **NOTE:** The provisioning script is designed for a fresh install and will completely wipe the database and any associated content - IF you decide to shutdown your VM! Outside of halting your vagrant machine, running `vagrant up` or `vagrant provision` will cause the provisioning script to re-run. Adding the flag `--no-provision` to `vagrant up` will ignore the script. - -#### **** OPTIONAL *** -If you decide to use pgAdmin3 for database management rather than using the command line, you'll need to allow/enable access to your virtual machine. -* Enter into `postgres.conf` and change `listen_addresses`: - ``` - sudo nano /etc/postgresql/9.3/main/postgresql.conf - listen_addresses = '*' - ``` - -* Enter into `pg_hba.conf` and add the `host` line: - ``` - sudo nano /etc/postgresql/9.3/main/pg_hba.conf - host all all 10.0.0.0/16 md5 - ``` - -* Restart postgresql - ``` - sudo /etc/init.d/postgresql restart - ``` - -* Within pgAdmin3, modify your settings: - * **Name:** marco_portal - * **Host:** localhost - * **Port:** 65432 - * **Username:** vagrant - - -### ~Code Deployment -Since this project is modularized, changes to a submodule only requires server updates to that specific submodule - rather than the entire code base. - -1. SSH into the server -2. Activate your virtual env - `source ~/env/marco_portal2/bin/activate` -3. Navigate to the submodule that you're updating. Submodules are located at: - * **Sandbox** - `cd /home/midatlantic/env/marco_portal2/src/[THE-NAME-OF-YOUR-SUBMODULE]` - * **Production** - `cd ~/webapps/marco_portal/marco/src/` -4. Once you're at that path - `git fetch && git reset -q --hard origin/master` - * `origin/master` pertains to the main master branch - you can change that to whatever your branch you'd like - * Of note, the master *madrona-portal* branch runs as `origin/prototype` -5. Navigate to `cd ~/webapps/marco_portal/marco` -6. Run `python manage.py collectstatic` to collect all the neccessary static (js/css) files - * you can use the -i flag to ignore utfgrids in the rare chance that those files seems to be "collecting" - * `python manage.py collectstatic -i utfgrid` -7. Run `python manage.py compress` to compress -8. Restart the server - `~/webapps/marco_portal/apache2/bin/restart` - -### ~Adding a new module to the apps directory and deployment -Adding a new module to marco requires a few additional steps for both local/development setup and deployment. +`-v` removes the PostGIS and Redis volumes. The next `up` will re-run +migrations and reload fixtures from scratch. -**Local/Development setup**: - -1. create directory within `madrona-portal/apps` - * Use `git clone` for exisiting module or create a new direcotry and use `git init` to set up your new repository. - * If this is a new git repository create a new remote origin repo within the [MidAtlanticPortal](https://github.com/MidAtlanticPortal) orgainization. *Next steps assume your new module is ready to use.* -2. open `madrona-portal/requirements.txt` and add the newly created git remote repository (*e.g.* `-e git+https://github.com/MidAtlanticPortal/your_new_repo.git@master#egg=an_alias`). *the `@master#egg=` assigns an alias (simple name) for your module* -3. open `madrona-portal/marco/marco/settings.py` and add your new module's alias as an `INSTALLED_APPS`. (*e.g.*, `INSTALLED_APPS = [ 'an_alias']`) -4. run `vagrant provision` +--- +## Disk space -**Deployment**: +Docker's build cache can grow large over time: -1. ssh into the server (sandbox or production) -2. activate your virtual env - `source ~/env/marco_portal2/bin/activate` -3. navigate to the submodule directory `cd /home/midatlantic/env/marco_portal2/src/` -4. `git clone` your new module repository -5. navigate to the marco-portal repo `cd ~/code/marco_portal2/prototype/` -6. run `git fetch && git reset -q --hard origin/prototype` -7. open the `requirements.txt` file and copy the line you added for your repo (*e.g.*, `-e git+https://github.com/MidAtlanticPortal/new_repo.git@master#egg=an_alias`) -8. enter `pip install` and paste (*e.g.*,`pip install -e git+https://github.com/MidAtlanticPortal/new_repo.git@master#egg=an_alias` ) and run -9. navigate to `cd ~/webapps/marco_portal/marco` -10. run `python manage.py collectstatic -i utfgrid` -11. run `python manage.py compress` -12. restart the server - `~/webapps/marco_portal/apache2/bin/restart` +```bash +docker system df # show usage breakdown +docker system prune -f # remove stopped containers, dangling images, unused networks, build cache +docker volume prune -f # remove unused volumes — only run when all containers are stopped +``` From 816a6e8d724f9dad1d42f6d2172efe21d43d9fe6 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Thu, 9 Jul 2026 13:41:42 -0700 Subject: [PATCH 133/152] Allow MP_PROJECT_CONFIG handling to support absolute paths for modular configuration --- marco/marco/settings.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/marco/marco/settings.py b/marco/marco/settings.py index 8dd6a64..521d1ac 100644 --- a/marco/marco/settings.py +++ b/marco/marco/settings.py @@ -35,7 +35,14 @@ # Configuration file # --------------------------------------------------------------------------- MP_PROJECT_CONFIG = os.environ.get("MP_PROJECT_CONFIG", "config.ini") -CONFIG_FILE = os.path.normpath(os.path.join(BASE_DIR, MP_PROJECT_CONFIG)) + +# Allow MP_PROJECT_CONFIG to be an absolute path. Enabling modularity, +# because a config.ini can live outside the madrona portal project directory. +# Otherwise use the config.ini in madrona portal. +if os.path.isabs(MP_PROJECT_CONFIG): + CONFIG_FILE = MP_PROJECT_CONFIG +else: + CONFIG_FILE = os.path.normpath(os.path.join(BASE_DIR, MP_PROJECT_CONFIG)) cfg = configparser.ConfigParser() cfg.read(CONFIG_FILE) From 4eb64d849c52fe9df256c856680b87758d71d0a5 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Mon, 13 Jul 2026 14:49:37 -0700 Subject: [PATCH 134/152] Add initial Docker Compose base configuration file --- docker/compose.base.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 docker/compose.base.yml diff --git a/docker/compose.base.yml b/docker/compose.base.yml new file mode 100644 index 0000000..e69de29 From 4bae146a4a4ef068ea4045502e74e444e692c40d Mon Sep 17 00:00:00 2001 From: David Pollard Date: Mon, 13 Jul 2026 15:01:11 -0700 Subject: [PATCH 135/152] Add Docker Compose configuration for app, db, and tasks services --- docker/compose.base.yml | 60 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/docker/compose.base.yml b/docker/compose.base.yml index e69de29..762a7c6 100644 --- a/docker/compose.base.yml +++ b/docker/compose.base.yml @@ -0,0 +1,60 @@ +services: + app: + environment: + - DB_INIT=${DB_INIT:-0} + - SECRET_KEY=${SECRET_KEY} + - DEBUG=${DEBUG:-True} + - ALLOWED_HOSTS=${ALLOWED_HOSTS:-localhost,127.0.0.1,::1} + - DB_ENGINE=${DB_ENGINE:-django.contrib.gis.db.backends.postgis} + - DB_NAME=${DB_NAME:-madrona_portal_db} + - DB_USER=${DB_USER:-postgres} + - DB_PASSWORD=${DB_PASSWORD} + - DB_HOST=db + - DB_PORT=5432 + - REDIS_URL=redis://${REDIS_PASSWORD:+:${REDIS_PASSWORD}@}tasks:6379/1 + - CELERY_BROKER_URL=redis://${REDIS_PASSWORD:+:${REDIS_PASSWORD}@}tasks:6379/0 + - DJANGO_SUPERUSER_USERNAME=${DJANGO_SUPERUSER_USERNAME:-admin} + - DJANGO_SUPERUSER_EMAIL=${DJANGO_SUPERUSER_EMAIL:-admin@example.com} + - DJANGO_SUPERUSER_PASSWORD=${DJANGO_SUPERUSER_PASSWORD:-} + - DJANGO_ENV=${DJANGO_ENV:-development} + - GUNICORN_WORKERS=${GUNICORN_WORKERS:-3} + ports: + - "${APP_PORT:-8000}:8000" + depends_on: + db: + condition: service_healthy + tasks: + condition: service_healthy + restart: unless-stopped + + db: + image: postgis/postgis:16-3.4 + volumes: + - postgis_data:/var/lib/postgresql/data + environment: + - POSTGRES_DB=${DB_NAME:-madrona_portal_db} + - POSTGRES_USER=${DB_USER:-postgres} + - POSTGRES_PASSWORD=${DB_PASSWORD} + ports: + - "${DB_PORT:-5432}:5432" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${DB_USER:-postgres} -d ${DB_NAME}"] + interval: 10s + timeout: 5s + retries: 5 + restart: unless-stopped + + tasks: + image: redis:7-alpine + command: redis-server ${REDIS_PASSWORD:+--requirepass ${REDIS_PASSWORD}} + volumes: [ "redis_data:/data" ] + healthcheck: + test: ["CMD-SHELL", "redis-cli ${REDIS_PASSWORD:+-a ${REDIS_PASSWORD}} ping"] + interval: 10s + timeout: 5s + retries: 5 + restart: unless-stopped + +volumes: + postgis_data: + redis_data: From 09ddf8187616b30903c21da735f0d76b5260107e Mon Sep 17 00:00:00 2001 From: David Pollard Date: Mon, 13 Jul 2026 17:12:54 -0700 Subject: [PATCH 136/152] Use dockerdecouple branch to build decoupled ghcr images --- .github/workflows/create-and-publish-docker-images.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/create-and-publish-docker-images.yml b/.github/workflows/create-and-publish-docker-images.yml index ccd890a..3f0708c 100644 --- a/.github/workflows/create-and-publish-docker-images.yml +++ b/.github/workflows/create-and-publish-docker-images.yml @@ -10,7 +10,7 @@ on: release: types: [published] push: - branches: ['docker'] + branches: ['dockerdecouple'] # TODO: switch to main branch once we're ready to build from there workflow_dispatch: From e27d5efce489fe0aa2730afc942d75617cd6651e Mon Sep 17 00:00:00 2001 From: David Pollard Date: Mon, 13 Jul 2026 17:14:21 -0700 Subject: [PATCH 137/152] Update tag for image to dockerdecouple --- .github/workflows/create-and-publish-docker-images.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/create-and-publish-docker-images.yml b/.github/workflows/create-and-publish-docker-images.yml index 3f0708c..3d6138a 100644 --- a/.github/workflows/create-and-publish-docker-images.yml +++ b/.github/workflows/create-and-publish-docker-images.yml @@ -196,7 +196,7 @@ jobs: push: true tags: | ghcr.io/${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.sha }} - ghcr.io/${{ env.IMAGE_NAME }}:latest + ghcr.io/${{ env.IMAGE_NAME }}:dockerdecouple cache-from: type=gha cache-to: type=gha,mode=max @@ -204,4 +204,4 @@ jobs: run: | echo "### Image pushed to GHCR" >> $GITHUB_STEP_SUMMARY echo "- \`ghcr.io/${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.sha }}\`" >> $GITHUB_STEP_SUMMARY - echo "- \`ghcr.io/${{ env.IMAGE_NAME }}:latest\`" >> $GITHUB_STEP_SUMMARY + echo "- \`ghcr.io/${{ env.IMAGE_NAME }}:dockerdecouple\`" >> $GITHUB_STEP_SUMMARY From ec6bd08e55cf6d3df871d48aa7b99addac4bfa56 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Tue, 14 Jul 2026 14:25:07 -0700 Subject: [PATCH 138/152] Refactor Dockerfile and requirements: remove unused environment variable and app copy --- docker/Dockerfile | 4 +--- docker/docker-requirements.txt | 7 ------- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index b005e73..da6e0ae 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -10,8 +10,7 @@ ENV DEBIAN_FRONTEND=noninteractive # Python & app environment ENV PYTHONDONTWRITEBYTECODE=1 \ PYTHONUNBUFFERED=1 \ - PATH="/opt/venv/bin:$PATH" \ - MP_PROJECT_CONFIG=config.wcoa.docker.ini + PATH="/opt/venv/bin:$PATH" WORKDIR /usr/local/apps/madrona-portal @@ -65,7 +64,6 @@ COPY madrona-apps/mp-proxy ./apps/mp-proxy COPY madrona-apps/mp-survey ./apps/mp-survey COPY madrona-apps/mp-visualize ./apps/mp-visualize COPY madrona-apps/p97-nursery ./apps/p97-nursery -COPY madrona-apps/wcoa ./apps/wcoa # --------------------------------------------------------------------------- # Python dependencies diff --git a/docker/docker-requirements.txt b/docker/docker-requirements.txt index 70cccac..dd0a2dd 100644 --- a/docker/docker-requirements.txt +++ b/docker/docker-requirements.txt @@ -93,10 +93,3 @@ djangorestframework>=3.14,<4.0 -e /usr/local/apps/madrona-portal/apps/p97-nursery -e /usr/local/apps/madrona-portal/apps/mp-proxy -e /usr/local/apps/madrona-portal/apps/mp-survey - -# --------------------------------------------------------------------------- -# Portal variant — choose one: -# --------------------------------------------------------------------------- -# -e /usr/local/apps/madrona-portal/apps/mida-portal --e /usr/local/apps/madrona-portal/apps/wcoa -# -e /usr/local/apps/madrona-portal/apps/wc-offshore-portal From 8cc398a866c4b339405f7d42cbb79e730b4c58b0 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Thu, 16 Jul 2026 12:24:44 -0700 Subject: [PATCH 139/152] Update README.md: remove branch annotations from directory structure and other references to wcoa --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index b9b8470..f282b5a 100644 --- a/README.md +++ b/README.md @@ -52,11 +52,11 @@ Your workspace should now look like: ``` portals/ -├── madrona-portal/ ← cloned from Ecotrust/madrona-portal, branch: docker +├── madrona-portal/ └── madrona-apps/ - ├── wcoa/ ← branch: vagrant2docker + ├── wcoa/ ├── mp-layers/ - └── ... ← all others on main + └── ... ``` ### Step 4 — Configure environment @@ -82,10 +82,10 @@ Everything else has working defaults for local development. ```bash cd ../marco -cp config.docker.ini.template config.wcoa.docker.ini +cp config.docker.ini.template config.docker.ini ``` -Edit `config.wcoa.docker.ini` : +Edit `config.docker.ini` : ```ini LOCATION = redis://tasks:6379/1 From dbd4340358d0e0c03492a39d6afa71eb7323b276 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Tue, 21 Jul 2026 12:53:30 -0700 Subject: [PATCH 140/152] Update email settings: change default from email address to noreply@example.com and clarify URL patterns comment --- marco/marco/settings.py | 2 +- marco/marco/urls.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/marco/marco/settings.py b/marco/marco/settings.py index 521d1ac..ba38a87 100644 --- a/marco/marco/settings.py +++ b/marco/marco/settings.py @@ -538,7 +538,7 @@ def _parse_hosts(raw: str | None) -> list[str]: EMAIL_HOST_USER = env_str('EMAIL_HOST_USER', email_cfg, 'HOST_USER', '') EMAIL_HOST_PASSWORD = env_str('EMAIL_HOST_PASSWORD', email_cfg, 'HOST_PASSWORD', '') EMAIL_BACKEND = env_str('EMAIL_BACKEND', email_cfg, 'EMAIL_BACKEND', 'django.core.mail.backends.smtp.EmailBackend') -DEFAULT_FROM_EMAIL = env_str('DEFAULT_FROM_EMAIL', email_cfg, 'DEFAULT_FROM_EMAIL', "MARCO Portal Team ") +DEFAULT_FROM_EMAIL = env_str('DEFAULT_FROM_EMAIL', email_cfg, 'DEFAULT_FROM_EMAIL', 'MARCO Portal Team ') SERVER_EMAIL = env_str('SERVER_EMAIL', email_cfg, 'SERVER_EMAIL', "MARCO Site Errors ") EMAIL_USE_TLS = env_bool('EMAIL_USE_TLS', email_cfg, 'EMAIL_USE_TLS', False) EMAIL_SUBJECT_PREFIX = env_str('EMAIL_SUBJECT_PREFIX', app_cfg, 'EMAIL_SUBJECT_PREFIX', '[MARCO]') + ' ' diff --git a/marco/marco/urls.py b/marco/marco/urls.py index 13aae6a..24ef506 100644 --- a/marco/marco/urls.py +++ b/marco/marco/urls.py @@ -61,7 +61,7 @@ def _iter_discovered_api_includes(): # --------------------------------------------------------------------------- # Project-specific URL patterns -# Optional: a portal variant (wcoa, mida, etc.) can prepend its own patterns. +# Optional: a project app can prepend its own patterns. # --------------------------------------------------------------------------- urlpatterns: list = [] From ebe1bc7a736fb1416cd49290b0774377bc9625cb Mon Sep 17 00:00:00 2001 From: David Pollard Date: Tue, 21 Jul 2026 13:01:35 -0700 Subject: [PATCH 141/152] Remove Geoportal specific templates, entrypoint, and wars init. These have all been moved to the `wcoa` project app repo. --- docker/geoportal-entrypoint.sh | 196 -------------------- docker/templates/authentication-simple.xml | 21 --- docker/templates/catalog-app-security.xml | 80 -------- docker/templates/harvester-app-security.xml | 42 ----- docker/wars/__init__.py | 0 5 files changed, 339 deletions(-) delete mode 100755 docker/geoportal-entrypoint.sh delete mode 100644 docker/templates/authentication-simple.xml delete mode 100644 docker/templates/catalog-app-security.xml delete mode 100644 docker/templates/harvester-app-security.xml delete mode 100644 docker/wars/__init__.py diff --git a/docker/geoportal-entrypoint.sh b/docker/geoportal-entrypoint.sh deleted file mode 100755 index 692dd88..0000000 --- a/docker/geoportal-entrypoint.sh +++ /dev/null @@ -1,196 +0,0 @@ -#!/bin/bash -################################### -# This file 100% written by Copilot -################################### -set -e - -echo "Starting Geoportal with configuration override..." - -# Install gettext for envsubst command -echo "Installing gettext package for environment variable substitution..." -apt-get update -qq && apt-get install -y gettext-base && apt-get clean && rm -rf /var/lib/apt/lists/* - -# Cleanup function for graceful shutdown -cleanup() { - if [ ! -z "$TOMCAT_PID" ] && kill -0 $TOMCAT_PID 2>/dev/null; then - echo "Cleaning up Tomcat process (PID: $TOMCAT_PID)..." - kill $TOMCAT_PID 2>/dev/null - sleep 2 - if kill -0 $TOMCAT_PID 2>/dev/null; then - kill -9 $TOMCAT_PID 2>/dev/null - fi - fi -} -trap cleanup EXIT INT TERM - -# Function to wait for WAR deployment -wait_for_deployment() { - local app_name=$1 - local max_wait=120 - local wait_time=0 - - echo "Waiting for $app_name to deploy..." - while [ ! -d "/usr/local/tomcat/webapps/$app_name" ] && [ $wait_time -lt $max_wait ]; do - sleep 2 - wait_time=$((wait_time + 2)) - echo "Waiting... ${wait_time}s" - done - - if [ $wait_time -ge $max_wait ]; then - echo "ERROR: $app_name failed to deploy within ${max_wait} seconds" - return 1 - fi - - echo "$app_name deployed successfully" - return 0 -} - -# Function to substitute environment variables in templates -substitute_variables() { - local template_file=$1 - local output_file=$2 - - echo "Processing template: $template_file -> $output_file" - - # Validate required environment variables - local missing_vars=() - if [ -z "$gpt_frame_options" ]; then - missing_vars+=("gpt_frame_options") - fi - if [ -z "$gpt_allowed_origin" ]; then - missing_vars+=("gpt_allowed_origin") - fi - - if [ ${#missing_vars[@]} -gt 0 ]; then - echo "WARNING: Missing required environment variables: ${missing_vars[*]}" - echo "Check your .env file and ensure these variables are set" - fi - - # Display current values for debugging - echo " gpt_frame_options = '$gpt_frame_options'" - echo " gpt_allowed_origin = '$gpt_allowed_origin'" - - # Validate CSP format (check for problematic characters) - if echo "$gpt_allowed_origin" | grep -q ":.*\*"; then - echo " WARNING: Port wildcards (*) in CSP frame-ancestors may not be supported by all browsers" - echo " Consider using specific ports or removing wildcards if you encounter issues" - fi - - # Use envsubst to replace environment variables - envsubst < "$template_file" > "$output_file" - - if [ $? -eq 0 ]; then - echo "Successfully processed $template_file" - - # Show a sample of the processed content for verification - echo "Sample of processed content:" - grep -E "(frame-options|Content-Security-Policy)" "$output_file" | head -2 | sed 's/^/ /' - else - echo "ERROR: Failed to process $template_file" - return 1 - fi -} - -# Start Tomcat in background to deploy WARs -echo "Starting Tomcat to deploy applications..." -catalina.sh run & -TOMCAT_PID=$! -echo "Tomcat started with PID: $TOMCAT_PID" - -# Wait for both applications to deploy -wait_for_deployment "geoportal" || exit 1 -wait_for_deployment "harvester" || exit 1 - -# Additional wait to ensure full extraction -echo "Waiting for full application extraction..." -sleep 10 - -# Create config directory if it doesn't exist -CATALOG_CONFIG_DIR="/usr/local/tomcat/webapps/geoportal/WEB-INF/classes/config" -mkdir -p "$CATALOG_CONFIG_DIR" -HARVESTER_CONFIG_DIR="/usr/local/tomcat/webapps/harvester/WEB-INF/classes/config" -mkdir -p "$HARVESTER_CONFIG_DIR" - -# Process and copy authentication configuration -if [ -f "/templates/authentication-simple.xml" ]; then - substitute_variables "/templates/authentication-simple.xml" "$CATALOG_CONFIG_DIR/authentication-simple.xml" - substitute_variables "/templates/authentication-simple.xml" "$HARVESTER_CONFIG_DIR/authentication-simple.xml" -else - echo "WARNING: authentication-simple.xml template not found" -fi - -# Process and copy security configuration -if [ -f "/templates/catalog-app-security.xml" ] && [ -f "/templates/harvester-app-security.xml" ]; then - substitute_variables "/templates/catalog-app-security.xml" "$CATALOG_CONFIG_DIR/app-security.xml" - substitute_variables "/templates/harvester-app-security.xml" "$HARVESTER_CONFIG_DIR/app-security.xml" -else - echo "WARNING: app-security.xml templates not found" - if [ ! -f "/templates/catalog-app-security.xml" ]; then - echo " Missing: /templates/catalog-app-security.xml" - fi - if [ ! -f "/templates/harvester-app-security.xml" ]; then - echo " Missing: /templates/harvester-app-security.xml" - fi -fi - -# Verify the configuration files were created -echo "Verifying configuration files..." -if [ -f "$CATALOG_CONFIG_DIR/authentication-simple.xml" ]; then - echo "✓ authentication-simple.xml configured" -else - echo "✗ authentication-simple.xml missing" -fi - -if [ -f "$CATALOG_CONFIG_DIR/app-security.xml" ]; then - echo "✓ CATALOG app-security.xml configured" -else - echo "✗ CATALOG app-security.xml missing" -fi - -if [ -f "$HARVESTER_CONFIG_DIR/authentication-simple.xml" ]; then - echo "✓ HARVESTER authentication-simple.xml configured" -else - echo "✗ HARVESTER authentication-simple.xml missing" -fi - -if [ -f "$HARVESTER_CONFIG_DIR/app-security.xml" ]; then - echo "✓ app-security.xml configured" -else - echo "✗ app-security.xml missing" -fi - -# Stop background Tomcat gracefully so H2 can flush and release its lock -echo "Stopping background Tomcat (PID: $TOMCAT_PID)..." - -if kill -0 $TOMCAT_PID 2>/dev/null; then - echo "Requesting graceful Tomcat shutdown via catalina.sh stop..." - catalina.sh stop 30 -force - # Wait for the background process to exit - for i in {1..35}; do - if ! kill -0 $TOMCAT_PID 2>/dev/null; then - echo "Tomcat stopped gracefully" - break - fi - echo "Waiting for shutdown... ${i}/35" - sleep 1 - done - # Final safety net - if kill -0 $TOMCAT_PID 2>/dev/null; then - echo "Force stopping Tomcat..." - kill -9 $TOMCAT_PID - sleep 2 - fi -else - echo "Tomcat process was not running (PID $TOMCAT_PID)" -fi - -echo "Tomcat stopped successfully" - -# Remove any stale H2 lock/trace files left by the background Tomcat run. -# These persist in the named volume and prevent the DB from opening on restart. -echo "Cleaning up stale H2 artifacts in /root..." -rm -f /root/harvester.lock.db /root/harvester.trace.db - -# Start Tomcat in foreground -echo "Starting Tomcat with updated configuration..." -exec catalina.sh run \ No newline at end of file diff --git a/docker/templates/authentication-simple.xml b/docker/templates/authentication-simple.xml deleted file mode 100644 index eada6db..0000000 --- a/docker/templates/authentication-simple.xml +++ /dev/null @@ -1,21 +0,0 @@ - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/docker/templates/catalog-app-security.xml b/docker/templates/catalog-app-security.xml deleted file mode 100644 index 4b09d8c..0000000 --- a/docker/templates/catalog-app-security.xml +++ /dev/null @@ -1,80 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/docker/templates/harvester-app-security.xml b/docker/templates/harvester-app-security.xml deleted file mode 100644 index b93047f..0000000 --- a/docker/templates/harvester-app-security.xml +++ /dev/null @@ -1,42 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/docker/wars/__init__.py b/docker/wars/__init__.py deleted file mode 100644 index e69de29..0000000 From 4eea40584b91a0a098dede0d161156e2eefa0fe2 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Tue, 21 Jul 2026 13:04:22 -0700 Subject: [PATCH 142/152] Refactor Docker setup: update workflows, remove legacy configurations, and enhance documentation for madrona-portal base image --- .../create-and-publish-docker-images.yml | 53 +---- README.md | 47 ++++- docker/.env.example | 149 ++------------ docker/docker-compose.dev.yml | 53 ----- docker/docker-compose.prod.yml | 185 ----------------- docker/docker-compose.yml | 186 ------------------ docker/entrypoint.sh | 45 +---- docker/nginx-dev.conf | 4 +- 8 files changed, 72 insertions(+), 650 deletions(-) delete mode 100644 docker/docker-compose.dev.yml delete mode 100644 docker/docker-compose.prod.yml delete mode 100644 docker/docker-compose.yml diff --git a/.github/workflows/create-and-publish-docker-images.yml b/.github/workflows/create-and-publish-docker-images.yml index 3d6138a..0351a70 100644 --- a/.github/workflows/create-and-publish-docker-images.yml +++ b/.github/workflows/create-and-publish-docker-images.yml @@ -1,23 +1,13 @@ -# source: https://docs.github.com/en/actions/tutorials/publish-packages/publish-docker-images#publishing-images-to-docker-hub-and-github-packages -name: Create and publish West Coast Ocean Data Portal (WCODP) Docker image - -# Builds a single Docker image from the full workspace (madrona-portal + -# all madrona-apps sub-repos) and pushes it to GitHub Container Registry. -# -# Image: ghcr.io/ecotrust/madrona-portal: (and :latest) +name: Build and publish madrona-portal base image on: - release: - types: [published] push: - branches: ['dockerdecouple'] - # TODO: switch to main branch once we're ready to build from there + branches: [main, dockerdecouple] workflow_dispatch: # Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. env: - REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} + IMAGE_NAME: ghcr.io/ecotrust/madrona-portal # There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu. jobs: @@ -31,20 +21,11 @@ jobs: id-token: write steps: - # ----------------------------------------------------------------------- - # Checkout madrona-portal into madrona-portal/ so the Dockerfile's COPY - # paths (e.g. COPY madrona-portal/marco ...) resolve correctly. - # ----------------------------------------------------------------------- - name: Checkout madrona-portal uses: actions/checkout@v5 with: path: madrona-portal - # ----------------------------------------------------------------------- - # Checkout all sub-apps into madrona-apps// — mirrors the local - # workspace layout the Dockerfile expects. - # GH_PAT must have read access to all sub-app repos in the Ecotrust org. - # ----------------------------------------------------------------------- - name: Checkout django_url_shortener uses: actions/checkout@v5 with: @@ -149,20 +130,6 @@ jobs: repository: Ecotrust/p97-nursery token: ${{ secrets.GH_PAT }} path: madrona-apps/p97-nursery - - # wcoa uses the vagrant2docker branch (contains Docker-specific config) - - name: Checkout wcoa - uses: actions/checkout@v5 - with: - repository: Ecotrust/wcoa - token: ${{ secrets.GH_PAT }} - path: madrona-apps/wcoa - - # This step uses [docker/metadata-action](https://github.com/docker/metadata-action#about) to extract tags and labels that will be applied to the specified image. The `id` "meta" allows the output of this step to be referenced in a subsequent step. The `images` value provides the base name for the tags and labels. - - name: Set lowercase image name - run: | - IMAGE_NAME=$(echo "${GITHUB_REPOSITORY}" | tr '[:upper:]' '[:lower:]') - echo "IMAGE_NAME=${IMAGE_NAME}" >> $GITHUB_ENV # ----------------------------------------------------------------------- # Docker setup @@ -180,14 +147,8 @@ jobs: - name: Extract short SHA id: meta run: | - cd madrona-portal echo "sha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT - # ----------------------------------------------------------------------- - # Build from workspace root — matches the layout the Dockerfile expects. - # Pushing :sha and :latest together so EC2 can pin a specific build or - # always pull the newest with :latest. - # ----------------------------------------------------------------------- - name: Build and push uses: docker/build-push-action@v5 with: @@ -195,13 +156,13 @@ jobs: file: madrona-portal/docker/Dockerfile push: true tags: | - ghcr.io/${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.sha }} - ghcr.io/${{ env.IMAGE_NAME }}:dockerdecouple + ${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.sha }} + ${{ env.IMAGE_NAME }}:latest cache-from: type=gha cache-to: type=gha,mode=max - name: Image digest summary run: | echo "### Image pushed to GHCR" >> $GITHUB_STEP_SUMMARY - echo "- \`ghcr.io/${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.sha }}\`" >> $GITHUB_STEP_SUMMARY - echo "- \`ghcr.io/${{ env.IMAGE_NAME }}:dockerdecouple\`" >> $GITHUB_STEP_SUMMARY + echo "- \`${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.sha }}\`" >> $GITHUB_STEP_SUMMARY + echo "- \`${{ env.IMAGE_NAME }}:latest\`" >> $GITHUB_STEP_SUMMARY diff --git a/README.md b/README.md index f282b5a..923a4ba 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,43 @@ # Docker Development Guide — Madrona Portal (WCOA) +## Docker architecture + +The madrona-portal repository is a core project-agnostic platform image: + +- `ghcr.io/ecotrust/madrona-portal:{sha,latest}` +- contains shared Madrona/MP apps and runtime tooling +- does not include `wcoa` or `mida` project code/configuration + +Project overlays now live in each project repository and build from this base image. + +### Build and publish core base image + +Core CI publishes on merge to `main`: + +```bash +ghcr.io/ecotrust/madrona-portal: +ghcr.io/ecotrust/madrona-portal:latest +``` + +### Dockerizing a new portal checklist + +1. Create `/docker/Dockerfile` that does: + - `FROM ghcr.io/ecotrust/madrona-portal:` + - `COPY . ./apps/` + - `pip install --no-deps -e ./apps/` + - `ENV MP_PROJECT_CONFIG=/usr/local/apps/madrona-portal/apps//docker/config..docker.ini` +2. Add `/docker/config..docker.ini`. +3. Add `/docker/compose.yml` overlay with project env/ports/volumes/services. +4. Add `/docker/.env.example` and local `docker/.env` (gitignored). +5. Add project CI workflow to build/push: + - `ghcr.io/ecotrust/:` + - `ghcr.io/ecotrust/:latest` + - pass a pinned `BASE_TAG` build argument. +6. Add a quickstart section to the project README. + +*Please note:* The first portal to be Dockerized was `wcoa`. You may come across legacy WCOA-coupled Docker notes. Use the project repos for current portal-specific Docker workflows. + + ## Quick start These steps take a fresh machine from nothing to a running portal. @@ -33,6 +71,7 @@ git clone https://github.com/Ecotrust/madrona-analysistools.git git clone https://github.com/Ecotrust/madrona-features.git git clone https://github.com/Ecotrust/madrona-manipulators.git git clone https://github.com/Ecotrust/madrona-scenarios.git +git clone https://github.com/Ecotrust/mida-portal.git git clone https://github.com/Ecotrust/mp-accounts.git git clone https://github.com/Ecotrust/mp-data-manager.git git clone https://github.com/Ecotrust/mp-drawing.git @@ -54,9 +93,10 @@ Your workspace should now look like: portals/ ├── madrona-portal/ └── madrona-apps/ - ├── wcoa/ + ├── wcoa/ + ├── mida-portal/ ├── mp-layers/ - └── ... + └── ... ``` ### Step 4 — Configure environment @@ -212,10 +252,7 @@ docker compose exec app python marco/manage.py migration_to_layers ### Step 8 — Importing production media files into the Dockerized Application #### Prerequisites -- `madrona-portal/marco/marco/config.wcoa.docker.ini` exists with `MEDIA_ROOT` set to a valid directory -- That valid directory should match the volume location is docker-compose.yml - `portals/madrona-portal/media` -- Production media files are available #### Step 8.1 - Copy the media files into Docker diff --git a/docker/.env.example b/docker/.env.example index 275ac47..a6c9fa6 100644 --- a/docker/.env.example +++ b/docker/.env.example @@ -1,47 +1,24 @@ # ============================================================================= -# Madrona Portal — Environment Variables -# Copy this file to .env and fill in real values. -# NEVER commit the real .env file to version control. -# -# Priority for every setting: env var > config.ini > built-in default +# Madrona Portal core Docker environment template +# This file defines generic defaults for the project-agnostic core image. # ============================================================================= -# ---------------------------------------------------------------------------- -# Django environment -# production, development -# ---------------------------------------------------------------------------- -DJANGO_ENV=production - -# ---------------------------------------------------------------------------- -# Gunicorn workers (2× vCPU count; t3.large has 2 vCPUs → 4 workers) -# ---------------------------------------------------------------------------- +DJANGO_ENV=development GUNICORN_WORKERS=4 -# --------------------------------------------------------------------------- -# GitHub Container Registry (production only) -# GHCR_PAT: read-only fine-grained PAT used to pull the image from GHCR. -# Run: echo $GHCR_PAT | docker login ghcr.io -u --password-stdin -# IMAGE_TAG: pin to a specific build SHA for rollback (default: latest) -# --------------------------------------------------------------------------- GHCR_PAT= IMAGE_TAG=latest -# --------------------------------------------------------------------------- -# Django core -# --------------------------------------------------------------------------- -SECRET_KEY=change-me-to-a-long-random-string +DEBUG=True +SECRET_KEY=change-me ALLOWED_HOSTS=localhost,127.0.0.1,::1 -MP_PROJECT_CONFIG=config.wcoa.docker.ini -DEBUG=False -CSRF_TRUSTED_ORIGINS="http://localhost,https://*.ecotrust.org" +CSRF_TRUSTED_ORIGINS=http://localhost +MP_PROJECT_CONFIG=config.ini -# --------------------------------------------------------------------------- -# App info and configuration -# --------------------------------------------------------------------------- APP_NAME="Madrona Portal" APP_TEAM_NAME="Marine Planner Team" -PROJECT_APP="wcoa" -PROJECT_SETTINGS_FILE=True +PROJECT_APP="" +PROJECT_SETTINGS_FILE=False TIME_ZONE=UTC COMPRESS_ENABLED=True MAP_LIBRARY=ol8 @@ -49,57 +26,31 @@ MEDIA_ROOT=/usr/local/apps/madrona-portal/media MEDIA_URL=/media/ STATIC_ROOT=/vol/web/static STATIC_CORE=/vol/web/static/ -APP_PORT=8008 +APP_PORT=8000 -# --------------------------------------------------------------------------- -# PostgreSQL / PostGIS -# DB_* is preferred; SQL_* aliases are accepted for legacy docker-compose files. -# --------------------------------------------------------------------------- DB_ENGINE=django.contrib.gis.db.backends.postgis -DB_NAME=wcoa_docker_db +DB_NAME=madrona_docker_db DB_USER=postgres DB_PASSWORD=change-me -# DB_HOST and DB_PORT are set inside docker-compose.yml (always "db" and 5432) -# --------------------------------------------------------------------------- -# Redis (used for Django cache + Celery broker + result backend) -# docker-compose builds REDIS_URL from REDIS_PASSWORD automatically. -# --------------------------------------------------------------------------- REDIS_PASSWORD=changeme REDIS_PORT=6379 -# REDIS_URL and CELERY_BROKER_URL are assembled in docker-compose.yml. -# --------------------------------------------------------------------------- -# Email (SMTP) -# --------------------------------------------------------------------------- EMAIL_HOST=smtp.example.com EMAIL_PORT=587 EMAIL_HOST_USER=noreply@example.com EMAIL_HOST_PASSWORD=change-me EMAIL_USE_TLS=true -DEFAULT_FROM_EMAIL=noreply@prod.mail.ecotrust.org +DEFAULT_FROM_EMAIL=noreply@example.com -# --------------------------------------------------------------------------- -# AWS SES (optional — only needed if EMAIL_BACKEND uses SES) -# --------------------------------------------------------------------------- AWS_ACCESS_KEY_ID= AWS_SECRET_ACCESS_KEY= AWS_SES_REGION_NAME=us-east-1 AWS_SES_REGION_ENDPOINT=email.us-east-1.amazonaws.com -# --------------------------------------------------------------------------- -# Google Analytics -# --------------------------------------------------------------------------- -GA_ACCOUNT=G-XXXXXXXXXX - -# --------------------------------------------------------------------------- -# Native Lands -# --------------------------------------------------------------------------- +GA_ACCOUNT= NATIVE_LAND_API_KEY= -# --------------------------------------------------------------------------- -# Social Auth OAuth keys -# --------------------------------------------------------------------------- FACEBOOK_KEY= FACEBOOK_SECRET= TWITTER_KEY= @@ -107,81 +58,9 @@ TWITTER_SECRET= GOOGLE_KEY= GOOGLE_SECRET= -# --------------------------------------------------------------------------- -# ReCAPTCHA (set in config.ini [APP] section or here) -# --------------------------------------------------------------------------- RECAPTCHA_PUBLIC_KEY= RECAPTCHA_PRIVATE_KEY= -# --------------------------------------------------------------------------- -# Dev superuser bootstrap (entrypoint creates this user on first start) -# Leave DJANGO_SUPERUSER_PASSWORD empty to skip superuser creation. -# --------------------------------------------------------------------------- DJANGO_SUPERUSER_USERNAME=admin DJANGO_SUPERUSER_EMAIL=admin@example.com -DJANGO_SUPERUSER_PASSWORD=changeme - -# ================================================= -# Catalog settings: Elasticsearch and Geoportal -# ================================================= - -# Project namespace (defaults to the current folder name if not set) -#COMPOSE_PROJECT_NAME=myproject - -# Password for the 'elastic' user (at least 6 characters) -ELASTIC_PASSWORD=changeme - -# Password for the 'kibana_system' user (at least 6 characters) -KIBANA_PASSWORD=changeme - -# Version of Elastic products -STACK_VERSION=8.8.2 - -# Set the cluster name -CLUSTER_NAME=elasticsearch - -# Port to expose Elasticsearch HTTP API to the host -ES_PORT=9200 -ES_REINDEX_REMOTE_WHITELIST=[] - -# Port to expose Kibana to the host -KIBANA_PORT=5601 - -# Increase or decrease based on the available host memory (in bytes) -ES_MEM_LIMIT=1073741824 -KB_MEM_LIMIT=1073741824 -LS_MEM_LIMIT=1073741824 - -# SAMPLE Predefined Key only to be used in POC environments -ENCRYPTION_KEY=abcdefghijklmnopqrstuvwxyz0123456789abcdefghijklmnopqrstuvwxyz01 - -# ================================================= - -# Geoportal Authentication Configuration -# Override these values in your Docker deployment - -# Admin User (Full Access) -gpt_admin_username=admin -gpt_admin_password=admin - -# Publisher User (Can publish metadata) -gpt_publisher_username=publisher -gpt_publisher_password=publisher - -# Regular User (Read-only access) -gpt_user_username=user -gpt_user_password=user - -gpt_wcoa_username=wcoa -gpt_wcoa_password=changeme - -gpt_esri_username=esri -gpt_esri_password=changeme - -ES_NODE=elastic - -gpt_catalog_war=./wars/geoportal.war -gpt_harvester_war=./wars/harvester.war - -gpt_frame_options=DENY -gpt_allowed_origin="localhost localhost:*" \ No newline at end of file +DJANGO_SUPERUSER_PASSWORD= diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml deleted file mode 100644 index b0955fb..0000000 --- a/docker/docker-compose.dev.yml +++ /dev/null @@ -1,53 +0,0 @@ -# Madrona Portal — Development Compose Override -# -# Layers on top of docker-compose.yml to enable live code sync from the host. -# Python's editable installs follow .pth files to the mounted directories, so -# any file saved on the host is immediately visible inside the container. -# -# Usage (from madrona-portal/): -# docker compose \ -# -f docker/docker-compose.yml \ -# -f docker/docker-compose.dev.yml \ -# --env-file .env --profile full up -# -# Django's runserver (active when DEBUG=True) auto-reloads on .py changes. -# Template changes are picked up per-request — no restart needed. -# Static file changes are served directly by runserver — no collectstatic needed. -# -# What still requires a container restart: -# - New pip packages (pip install must run inside the venv) -# - config.ini changes -# -# What still requires manage.py migrate: -# - New migration files (run: docker exec docker-app-1 python marco/manage.py migrate) - -services: - app: - environment: - - DEBUG=True - volumes: - # Mount the main Django project tree from the host. - # Changes to Python, templates, and config files are live immediately. - - ../marco:/usr/local/apps/madrona-portal/marco - - # Mount the WCOA app package from the host. - - ../../madrona-apps/wcoa:/usr/local/apps/madrona-portal/apps/wcoa - - # Mount other madrona-apps packages you are actively developing. - # Comment out any you are NOT changing — using the baked image copy - # for those packages is faster and avoids unnecessary inotify watches. - - ../../madrona-apps/mp-data-manager:/usr/local/apps/madrona-portal/apps/mp-data-manager - - ../../madrona-apps/mp-layers:/usr/local/apps/madrona-portal/apps/mp-layers - - ../../madrona-apps/mp-accounts:/usr/local/apps/madrona-portal/apps/mp-accounts - - ../../madrona-apps/mp-drawing:/usr/local/apps/madrona-portal/apps/mp-drawing - - ../../madrona-apps/mp-visualize:/usr/local/apps/madrona-portal/apps/mp-visualize - - ../../madrona-apps/madrona-features:/usr/local/apps/madrona-portal/apps/madrona-features - - ../../madrona-apps/madrona-manipulators:/usr/local/apps/madrona-portal/apps/madrona-manipulators - - ../../madrona-apps/madrona-scenarios:/usr/local/apps/madrona-portal/apps/madrona-scenarios - - ../../madrona-apps/mp-map-groups:/usr/local/apps/madrona-portal/apps/mp-map-groups - - ../../madrona-apps/mp-explore:/usr/local/apps/madrona-portal/apps/mp-explore - - ../../madrona-apps/mp-proxy:/usr/local/apps/madrona-portal/apps/mp-proxy - - ../../madrona-apps/p97-nursery:/usr/local/apps/madrona-portal/apps/p97-nursery - - ../../madrona-apps/django_url_shortener:/usr/local/apps/madrona-portal/apps/django_url_shortener - - ../../madrona-apps/madrona-analysistools:/usr/local/apps/madrona-portal/apps/madrona-analysistools - - ../../madrona-apps/mp-survey:/usr/local/apps/madrona-portal/apps/mp-survey diff --git a/docker/docker-compose.prod.yml b/docker/docker-compose.prod.yml deleted file mode 100644 index 87e4e47..0000000 --- a/docker/docker-compose.prod.yml +++ /dev/null @@ -1,185 +0,0 @@ -# Madrona Portal — Production Docker Compose (WCOA) -# -# Uses the pre-built image from GitHub Container Registry instead of building -# from source. All madrona-apps code is baked into the image — no source -# volume mounts. -# -# Usage: -# docker compose -f docker/docker-compose.prod.yml \ -# --env-file .env \ -# up -d -# -# To pull the latest image before starting: -# docker pull ghcr.io/ecotrust/madrona-portal:latest -# -# To pin to a specific build (for rollback): -# IMAGE_TAG=abc1234 docker compose -f docker/docker-compose.prod.yml ... - -services: - - app: - image: ghcr.io/ecotrust/madrona-portal:${IMAGE_TAG:-latest} - volumes: - - ./static:/vol/web/static - # User-uploaded media files — persisted on the host across deploys. - - ./media:/usr/local/apps/madrona-portal/media - # Config file — lets you update portal settings without rebuilding the image. - - ../marco/config.wcoa.docker.ini:/usr/local/apps/madrona-portal/marco/config.wcoa.docker.ini:ro - env_file: - - ./.env - environment: - - DB_INIT=${DB_INIT:-0} - - MP_PROJECT_CONFIG=${MP_PROJECT_CONFIG:-config.wcoa.docker.ini} - - SECRET_KEY=${SECRET_KEY} - - ALLOWED_HOSTS=${ALLOWED_HOSTS:-localhost,127.0.0.1,::1} - - DEBUG=${DEBUG:-False} - - DB_ENGINE=${DB_ENGINE:-django.contrib.gis.db.backends.postgis} - - DB_NAME=${DB_NAME:-wcoa_docker_db} - - DB_USER=${DB_USER:-postgres} - - DB_PASSWORD=${DB_PASSWORD} - - DB_HOST=db - - DB_PORT=5432 - - REDIS_URL=redis://${REDIS_PASSWORD:+:${REDIS_PASSWORD}@}tasks:6379/1 - - CELERY_BROKER_URL=redis://${REDIS_PASSWORD:+:${REDIS_PASSWORD}@}tasks:6379/0 - - DJANGO_SUPERUSER_USERNAME=${DJANGO_SUPERUSER_USERNAME:-admin} - - DJANGO_SUPERUSER_EMAIL=${DJANGO_SUPERUSER_EMAIL:-admin@example.com} - - DJANGO_SUPERUSER_PASSWORD=${DJANGO_SUPERUSER_PASSWORD:-} - - DJANGO_ENV=${DJANGO_ENV:-production} - - GUNICORN_WORKERS=${GUNICORN_WORKERS:-3} - depends_on: - db: - condition: service_healthy - tasks: - condition: service_healthy - ports: - - "${APP_PORT:-8008}:8008" - networks: - - madronanetwork - restart: unless-stopped - - db: - image: postgis/postgis:16-3.4 - volumes: - - postgis_data:/var/lib/postgresql/data - environment: - - POSTGRES_USER=${DB_USER:-postgres} - - POSTGRES_PASSWORD=${DB_PASSWORD} - - POSTGRES_DB=${DB_NAME:-wcoa_docker_db} - ports: - - "${DB_PORT:-5432}:5432" - networks: - - madronanetwork - healthcheck: - test: ["CMD-SHELL", "pg_isready -U ${DB_USER:-postgres} -d ${DB_NAME:-wcoa_docker_db}"] - interval: 10s - timeout: 5s - retries: 5 - restart: unless-stopped - - tasks: - image: redis:7-alpine - command: redis-server ${REDIS_PASSWORD:+--requirepass ${REDIS_PASSWORD}} - ports: - - "${REDIS_PORT:-6379}:6379" - volumes: - - redis_data:/data - networks: - - madronanetwork - healthcheck: - test: ["CMD-SHELL", "redis-cli ${REDIS_PASSWORD:+-a ${REDIS_PASSWORD}} ping"] - interval: 10s - timeout: 5s - retries: 5 - restart: unless-stopped - - geoportal: - image: tomcat:9-jdk11 - ports: - - 8080:8080 - volumes: - - gp-volume:/usr/local/tomcat/webapps/ - - harvester_data:/root - - ${gpt_catalog_war}:/usr/local/tomcat/webapps/geoportal.war - - ${gpt_harvester_war}:/usr/local/tomcat/webapps/harvester.war - - ./templates:/templates:ro - - ./geoportal-entrypoint.sh:/usr/local/bin/entrypoint.sh:ro - entrypoint: ["/usr/local/bin/entrypoint.sh"] - networks: - - madronanetwork - restart: always - env_file: - - ./.env - depends_on: - elastic: - condition: service_healthy - - elastic: - image: elasticsearch:8.19.12 - volumes: - - es-volume:/usr/share/elasticsearch/data - - ./backups/elasticsearch:/usr/share/elasticsearch/backups - environment: - - discovery.type=single-node - - ES_JAVA_OPTS=-Xms512m -Xmx512m - - cluster.name=${CLUSTER_NAME} - - ELASTIC_PASSWORD=${ELASTIC_PASSWORD} - - bootstrap.memory_lock=true - - reindex.remote.whitelist=${ES_REINDEX_REMOTE_WHITELIST} - # Disable security so that GeoPortal can write to Elasticsearch. - # DO NOT EXPOSE ELASTIC TO THE INTERNET until we solve for this and enable xpack.security. - - xpack.security.enabled=false - - path.repo=/usr/share/elasticsearch/backups - ports: - - 9200:9200 - - 9300:9300 - networks: - - madronanetwork - group_add: - - "0" # Add the elasticsearch user to the root group to allow backup permissions - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:9200/_cluster/health?wait_for_status=yellow"] - interval: 30s - timeout: 10s - retries: 10 - restart: always - - kibana: - image: kibana:8.19.12 - profiles: ["dev"] # Only start Kibana in dev profile - ports: - - 127.0.0.1:5601:5601 - environment: - ELASTICSEARCH_HOSTS: http://elastic:9200 - networks: - - madronanetwork - depends_on: - elastic: - condition: service_healthy - restart: no - - nginx: - image: nginx:alpine - profiles: ["dev"] # Only start nginx in dev profile - ports: - - "8081:80" - volumes: - - ./nginx-dev.conf:/etc/nginx/conf.d/default.conf:ro - - ./static:/vol/web/static:ro - - ./media:/usr/local/apps/madrona-portal/media:ro - networks: - - madronanetwork - depends_on: - - app - - geoportal - restart: no - -volumes: - postgis_data: - redis_data: - gp-volume: - es-volume: - harvester_data: - -networks: - madronanetwork: - driver: bridge diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml deleted file mode 100644 index 89a11c9..0000000 --- a/docker/docker-compose.yml +++ /dev/null @@ -1,186 +0,0 @@ -# Madrona Portal — Docker Compose (WCOA) -# -# Quick start (full Docker stack): -# cp ../.env.example ../.env # then fill in real values -# docker compose -f docker/docker-compose.yml --profile full up --build -# -# Dev infrastructure only (db + Redis, for use with local Django dev server): -# docker compose -f docker/docker-compose.yml up -# python marco/manage.py runserver # in a separate terminal - -services: - - app: - # image: madrona-portal-app:latest - build: - context: ../../ - dockerfile: madrona-portal/docker/Dockerfile - volumes: - - ./static:/vol/web/static - - ./media:/usr/local/apps/madrona-portal/media - # Mount the main Django project tree from the host. - # Changes to Python, templates, and config files are live immediately. - - ../marco:/usr/local/apps/madrona-portal/marco - - # Mount the WCOA app package from the host. - - ../../madrona-apps/wcoa:/usr/local/apps/madrona-portal/apps/wcoa - - # Mount other madrona-apps packages you are actively developing. - # Comment out any you are NOT changing — using the baked image copy - # for those packages is faster and avoids unnecessary inotify watches. - - ../../madrona-apps/mp-data-manager:/usr/local/apps/madrona-portal/apps/mp-data-manager - - ../../madrona-apps/mp-layers:/usr/local/apps/madrona-portal/apps/mp-layers - - ../../madrona-apps/mp-accounts:/usr/local/apps/madrona-portal/apps/mp-accounts - - ../../madrona-apps/mp-drawing:/usr/local/apps/madrona-portal/apps/mp-drawing - - ../../madrona-apps/mp-visualize:/usr/local/apps/madrona-portal/apps/mp-visualize - - ../../madrona-apps/madrona-features:/usr/local/apps/madrona-portal/apps/madrona-features - - ../../madrona-apps/madrona-manipulators:/usr/local/apps/madrona-portal/apps/madrona-manipulators - - ../../madrona-apps/madrona-scenarios:/usr/local/apps/madrona-portal/apps/madrona-scenarios - - ../../madrona-apps/mp-map-groups:/usr/local/apps/madrona-portal/apps/mp-map-groups - - ../../madrona-apps/mp-explore:/usr/local/apps/madrona-portal/apps/mp-explore - - ../../madrona-apps/mp-proxy:/usr/local/apps/madrona-portal/apps/mp-proxy - - ../../madrona-apps/p97-nursery:/usr/local/apps/madrona-portal/apps/p97-nursery - - ../../madrona-apps/django_url_shortener:/usr/local/apps/madrona-portal/apps/django_url_shortener - - ../../madrona-apps/madrona-analysistools:/usr/local/apps/madrona-portal/apps/madrona-analysistools - - ../../madrona-apps/mp-survey:/usr/local/apps/madrona-portal/apps/mp-survey - env_file: - - ./.env # load all secrets from the docker/.env file - environment: - # DB_INIT=1 runs migrations, fixtures, and superuser creation on startup. - # Defaults to 0 (skip) to protect existing databases. - - DB_INIT=${DB_INIT:-0} - - # Config file selection — override specific values via env vars below. - - MP_PROJECT_CONFIG=${MP_PROJECT_CONFIG:-config.wcoa.docker.ini} - - # Core Django - - SECRET_KEY=${SECRET_KEY} - - ALLOWED_HOSTS=${ALLOWED_HOSTS:-localhost,127.0.0.1,::1} - - DEBUG=${DEBUG:-True} - - # Database (DB_* preferred; SQL_* aliases kept for legacy compatibility) - - DB_ENGINE=${DB_ENGINE:-django.contrib.gis.db.backends.postgis} - - DB_NAME=${DB_NAME:-wcoa_docker_db} - - DB_USER=${DB_USER:-postgres} - - DB_PASSWORD=${DB_PASSWORD} - - DB_HOST=db - - DB_PORT=5432 - - # Redis — single URL used for cache, Celery broker, and result backend. - # Auth segment (:password@) is included only when REDIS_PASSWORD is set. - - REDIS_URL=redis://${REDIS_PASSWORD:+:${REDIS_PASSWORD}@}tasks:6379/1 - - CELERY_BROKER_URL=redis://${REDIS_PASSWORD:+:${REDIS_PASSWORD}@}tasks:6379/0 - - # Superuser bootstrap — only creates if username doesn't already exist. - # Leave DJANGO_SUPERUSER_PASSWORD empty to skip (safe for production). - - DJANGO_SUPERUSER_USERNAME=${DJANGO_SUPERUSER_USERNAME:-admin} - - DJANGO_SUPERUSER_EMAIL=${DJANGO_SUPERUSER_EMAIL:-admin@example.com} - - DJANGO_SUPERUSER_PASSWORD=${DJANGO_SUPERUSER_PASSWORD:-} - - # Application server mode - - DJANGO_ENV=${DJANGO_ENV:-development} - - GUNICORN_WORKERS=${GUNICORN_WORKERS:-3} - depends_on: - db: - condition: service_healthy - tasks: - condition: service_healthy - ports: - - "${APP_PORT:-8000}:8000" - networks: - - madronanetwork - restart: unless-stopped - - db: - image: postgis/postgis:16-3.4 - volumes: - - postgis_data:/var/lib/postgresql/data - environment: - - POSTGRES_USER=${DB_USER:-postgres} - - POSTGRES_PASSWORD=${DB_PASSWORD} - - POSTGRES_DB=${DB_NAME:-wcoa_docker_db} - ports: - - "${DB_PORT:-5432}:5432" - networks: - - madronanetwork - healthcheck: - test: ["CMD-SHELL", "pg_isready -U ${DB_USER:-postgres} -d ${DB_NAME:-wcoa_docker_db}"] - interval: 10s - timeout: 5s - retries: 5 - restart: unless-stopped - - tasks: - image: redis:7-alpine - # Only pass --requirepass when REDIS_PASSWORD is non-empty. - # An empty REDIS_PASSWORD causes "wrong number of arguments" in Redis 7. - command: redis-server ${REDIS_PASSWORD:+--requirepass ${REDIS_PASSWORD}} - ports: - - "${REDIS_PORT:-6379}:6379" - volumes: - - redis_data:/data - networks: - - madronanetwork - healthcheck: - # -a flag is only passed when a password is configured. - test: ["CMD-SHELL", "redis-cli ${REDIS_PASSWORD:+-a ${REDIS_PASSWORD}} ping"] - interval: 10s - timeout: 5s - retries: 5 - restart: unless-stopped - - geoportal: - image: tomcat:9-jdk11 - ports: - - 8080:8080 - volumes: - - gp-volume:/usr/local/tomcat/webapps/ - - ${gpt_catalog_war}:/usr/local/tomcat/webapps/geoportal.war - - ${gpt_harvester_war}:/usr/local/tomcat/webapps/harvester.war - # Configuration override setup - - ./templates:/templates:ro - - ./geoportal-entrypoint.sh:/usr/local/bin/entrypoint.sh:ro - entrypoint: ["/usr/local/bin/entrypoint.sh"] - networks: - - madronanetwork - restart: always - env_file: - - ./.env - depends_on: - elastic: - condition: service_healthy - - elastic: - image: elasticsearch:8.19.12 - volumes: - - es-volume:/usr/share/elasticsearch/data - environment: - - discovery.type=single-node - - ES_JAVA_OPTS=-Xms512m -Xmx512m - - cluster.name=${CLUSTER_NAME} - - ELASTIC_PASSWORD=${ELASTIC_PASSWORD} - - bootstrap.memory_lock=true - - reindex.remote.whitelist=${ES_REINDEX_REMOTE_WHITELIST} - - xpack.security.enabled=false - - ports: - - 9200:9200 - - 9300:9300 - networks: - - madronanetwork - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:9200/_cluster/health?wait_for_status=yellow"] - interval: 30s - timeout: 10s - retries: 10 - restart: always - -volumes: - postgis_data: - redis_data: - gp-volume: - es-volume: - -networks: - madronanetwork: - driver: bridge diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index 34da985..58cc997 100755 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -2,8 +2,7 @@ # Madrona Portal — Docker entrypoint # Waits for the database, then starts the application server. # -# By default steps 1 (DB wait), 2 (collectstatic + compress), and 6 (server -# start) run. Set DB_INIT=1 to also run steps 3-5 (migrate, fixtures, superuser). +# By default steps 1 (DB wait), 2 (collectstatic + compress), and 6 (server start) run. Set DB_INIT=1 to also run steps 3-5 (migrate, fixtures, superuser). # This is intentionally opt-in to protect existing databases. set -e @@ -36,8 +35,7 @@ PY # --------------------------------------------------------------------------- # 2. Collect static files and compress assets (always runs) # --------------------------------------------------------------------------- -# Ensure the bind-mounted static dir is writable by madrona_user regardless -# of how Docker created it on the host (often root:root on Linux). +# Ensure the bind-mounted static dir is writable by madrona_user regardless of how Docker created it on the host (often root:root on Linux). chown madrona_user:madrona_user /vol/web/static 2>/dev/null || true echo "Collecting static files..." @@ -60,13 +58,9 @@ python marco/manage.py migrate --noinput # --------------------------------------------------------------------------- # 4. Seed a fresh database with initial fixture data # -# A brand-new PostGIS install contains exactly one Wagtail Page row (the -# Wagtail root page, depth=1). We count pages at depth > 1 — if none exist, -# this is a fresh database and we load the initial fixture. +# A brand-new PostGIS install contains exactly one Wagtail Page row (the Wagtail root page, depth=1). We count pages at depth > 1 — if none exist, this is a fresh database and we load the initial fixture. # -# IMPORTANT: We never wipe content on an existing database. That would -# destroy real data. Set FORCE_RELOAD_FIXTURES=1 only in CI or dev reset -# scenarios where wiping the database is intentional. +# IMPORTANT: Be careful not to wipe content on an existing database. Set FORCE_RELOAD_FIXTURES=1 only in CI or dev reset scenarios where wiping the database is intentional. # --------------------------------------------------------------------------- CONTENT_PAGES=$(python - 2>/dev/null <<'PY' || echo "unknown" import sys, os @@ -107,11 +101,7 @@ except Exception: pass PY - # Load fixtures in a single Python process so that ContentTypes created - # here are guaranteed to be visible when loaddata deserializes FK natural - # keys. Wagtail Page records reference content types by natural key - # (e.g. ["wcoa", "ctapage"]); if the ContentType row is absent Django's - # deserializer defers the FK and the INSERT fails with a NOT NULL violation. + # Load fixtures in a single Python process so ContentTypes created here are guaranteed to be visible when loaddata deserializes FK natural keys. python - <<'PY' import sys, os sys.path.insert(0, 'marco') @@ -119,8 +109,7 @@ os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'marco.settings') import django django.setup() -# Step 1: ensure every installed app's ContentTypes exist before loading -# fixture data. create_contenttypes is idempotent. +# Step 1: ensure every installed app's ContentTypes exist before loading fixture data. create_contenttypes is idempotent. from django.apps import apps as django_apps from django.contrib.contenttypes.management import create_contenttypes from django.contrib.contenttypes.models import ContentType @@ -128,30 +117,10 @@ from django.contrib.contenttypes.models import ContentType for app_config in django_apps.get_app_configs(): create_contenttypes(app_config, verbosity=0) -# Confirm the wcoa types that the fixture depends on are present. -wcoa_models = [ - 'ctapage', 'connectpage', 'catalogiframepage', - 'catalogthemegridpage', 'catalogthemegridpagedetail', - 'ohidashboard', 'wcoaoceanstories', 'wcoaoceanstory', -] -missing = [] -for model in wcoa_models: - if not ContentType.objects.filter(app_label='wcoa', model=model).exists(): - # Force-create it so loaddata can resolve the natural key. - ContentType.objects.get_or_create(app_label='wcoa', model=model) - missing.append(model) -if missing: - print(f'WARNING: had to force-create ContentTypes: {missing}', flush=True) -else: - print('All wcoa ContentTypes verified.', flush=True) +print('ContentTypes synchronized for all installed apps.', flush=True) # Step 2: load fixtures — same process, same DB session. from django.core.management import call_command -call_command( - 'loaddata', - 'apps/wcoa/wcoa/fixtures/initial_data_prod.json', - verbosity=1, -) call_command( 'loaddata', 'apps/madrona-scenarios/scenarios/fixtures/initial_data.json', diff --git a/docker/nginx-dev.conf b/docker/nginx-dev.conf index 7d37469..981d789 100644 --- a/docker/nginx-dev.conf +++ b/docker/nginx-dev.conf @@ -7,8 +7,8 @@ server { # when a backend container hasn't started yet). resolver 127.0.0.11 valid=30s; # TODO: consider writing logs to specific file - # access_log /var/log/nginx/wcoa.access.log; - # error_log /var/log/nginx/wcoa.error.log; + # access_log /var/log/nginx/app.access.log; + # error_log /var/log/nginx/app.error.log; # Increase client body size for file uploads client_max_body_size 100M; From 5909ddb40d5ef91ea315533d9e978c86fd7bba9d Mon Sep 17 00:00:00 2001 From: David Pollard Date: Tue, 21 Jul 2026 15:14:40 -0700 Subject: [PATCH 143/152] Update README.md: remove WCOA reference from Docker Development Guide title --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 923a4ba..5e0bf80 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Docker Development Guide — Madrona Portal (WCOA) +# Docker Development Guide — Madrona Portal ## Docker architecture From 648a1afec214bfec21ee62e2333a3e16ef29553d Mon Sep 17 00:00:00 2001 From: David Pollard Date: Tue, 21 Jul 2026 15:24:53 -0700 Subject: [PATCH 144/152] Update Docker workflow: set working directory for SHA extraction and improve error handling --- .github/workflows/create-and-publish-docker-images.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/create-and-publish-docker-images.yml b/.github/workflows/create-and-publish-docker-images.yml index 0351a70..105898c 100644 --- a/.github/workflows/create-and-publish-docker-images.yml +++ b/.github/workflows/create-and-publish-docker-images.yml @@ -146,9 +146,11 @@ jobs: - name: Extract short SHA id: meta + working-directory: madrona-portal run: | - echo "sha=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT - + SHA=$(git rev-parse --short HEAD 2>/dev/null || echo "${GITHUB_SHA::7}") + echo "sha=$SHA" >> "$GITHUB_OUTPUT" + - name: Build and push uses: docker/build-push-action@v5 with: From 02ef9e72e7fdeb997c8509f78456ad31aea640ae Mon Sep 17 00:00:00 2001 From: David Pollard Date: Wed, 22 Jul 2026 16:38:50 -0700 Subject: [PATCH 145/152] Remove MODERNIZATION.md: eliminate outdated roadmap and modernization log --- docs/MODERNIZATION.md | 154 ------------------------------------------ 1 file changed, 154 deletions(-) delete mode 100644 docs/MODERNIZATION.md diff --git a/docs/MODERNIZATION.md b/docs/MODERNIZATION.md deleted file mode 100644 index c0cca3b..0000000 --- a/docs/MODERNIZATION.md +++ /dev/null @@ -1,154 +0,0 @@ -# Madrona Portal — Modernization Log & Roadmap - -> **Last updated:** March 2026 -> **Stack target:** Python 3.10+, Django 4.2 LTS, Wagtail 7.x - ---- - -## Phase 1 — Completed (March 2026) - -These changes have been applied to the codebase. - -### Dependencies - -| File | Change | -|---|---| -| `requirements.txt` | Added version bounds to all packages; removed duplicate `django-colorfield`; resolved `social-auth-app-django` conflict (`<5.0` vs `>5.4`); updated `django-taggit` to `>=5.0,<7.0`; widened Django constraint to `>=4.2,<5.0` to allow patch updates | -| `dev_requirements.txt` | **Completely replaced.** Old file pinned Django <1.10, Wagtail 1.3.1 (2015-era). New file contains `pytest`, `pytest-django`, `pytest-cov`, `factory-boy`, `ruff`, `mypy`, `django-stubs`, and `django-debug-toolbar` | -| `docker/docker-requirements.txt` | No changes — already modern. Production reference file. | - -### `settings.py` - -- Removed **Wagtail v1 / v2 / v3 runtime detection** (nested try/except over `INSTALLED_APPS`). Locked to Wagtail 7+ with a clean, single `INSTALLED_APPS` list. -- Removed **`REDIS_PACKAGE_NAME` / `redis_cache` fallback** — `django_redis` is the only supported cache backend. -- Removed **dead `if False:` debug-toolbar block** — enable via `dev_requirements.txt` and `ADDITIONAL_APPS` in config. -- Removed **commented-out Wagtail v1/v2 middleware blocks**. -- Replaced **`eval()` calls** for `ADDITIONAL_APPS` / `ADDITIONAL_MIDDLEWARE` with `json.loads()` + `ast.literal_eval()` fallback. `eval()` on config-file values is a remote-code execution risk. -- Replaced **`exec("from %s.settings import *")` pattern** with a proper `import_module` + namespace merge loop. -- Removed **deprecated `BROKER_URL`** — Celery 5 uses `CELERY_BROKER_URL` only. -- Removed **`SOCIAL_AUTH_GOOGLE_OAUTH2_USE_DEPRECATED_API = True`** (deprecated). -- Replaced **`try: VAR except NameError`** patterns for `FEEDBACK_IFRAME_URL`, `DISCLAIMER_BUTTON_DEFAULT`, `DATA_MANAGER_ADMIN`, `PROJECT_REGION` with direct assignment. -- Updated docstring reference from Django 1.7 to 4.2. -- Added **`SECRET_KEY` guard** — raises `RuntimeError` at startup if key is unset, rather than silently running with `'you forgot to set the secret key'`. -- Consolidated all `cfg.sections()` existence checks into a single loop at the top. - -### `urls.py` - -- Removed **Django 1.x `from django.conf.urls import url` try/except** — `django.urls.re_path` (Django 2.0+) is now imported directly. -- Removed **`WAGTAIL_VERSION > 1` branch** — both branches were identical (Wagtail v1 `wagtail.docs` vs v2+ `wagtail.documents`). The v2+ import is used directly. -- Removed trailing `/?` optional slashes on most routes (ambiguous in Django URL routing). -- Replaced `re_path(r'^django-admin/?', ...)` with `re_path(r'^django-admin/', ...)` — `admin.site.urls` already handles trailing slash. -- Added `warnings.warn` instead of silent `except Exception: pass` when `PROJECT_APP` URL import fails. -- Added API URL auto-discovery: for each entry in `INSTALLED_APPS`, if `.urls` exists and defines `api_urlpatterns`, those routes are automatically mounted under `/api/`. -- New convention for sub-apps: define REST endpoints in `/api.py`, expose them from `/urls.py` as `api_urlpatterns`, and avoid hard-coding app-specific API imports in the project URLConf. - -### Migrations - -- Stripped **`from __future__ import unicode_literals`** from **72 migration files** — this Python 2 compatibility import is a no-op in Python 3 and adds noise. - -### Docker & DevOps - -| File | Change | -|---|---| -| `Dockerfile` | Fixed **indentation bug**: three `RUN` statements inside the `apt-get` block were indented as if part of it, but only the first `RUN` was correctly associated. Moved venv creation, pip install, and GDAL install to separate top-level `RUN` layers for correct caching. Consolidated final `RUN` commands (chmod, mkdir, useradd, chown) into one layer. | -| `docker/docker-compose.yml` | Added **`healthcheck`** blocks for `db` (pg_isready) and `tasks` (redis ping). Replaced `links:` with `depends_on: condition: service_healthy`. Added `restart: unless-stopped`. Removed stale Vagrant-era volume name `redis.conf`. Set sensible `:-default` values for env vars. | -| `.env.example` | **New file** — documents every required environment variable with safe placeholder values. Committed to repo so developers know what to configure. | -| `.gitignore` | Added `.env` entry to prevent real credentials from being committed. | - -### Tooling - -| File | Change | -|---|---| -| `pyproject.toml` | **New file** — central config for `pytest`, `coverage`, `ruff`, and `mypy`. Replaces ad-hoc tool configs scattered across the project. | - ---- - -## Phase 2 — Completed (March 2026) - -### Secret Management - -- **Extended env var support** throughout `settings.py` via a new `_env(env_key, cfg_section, cfg_key, default)` helper. Every credential-bearing setting now checks an environment variable *first*, falls back to `config.ini`, then to a safe default. -- **Database** — supports `DB_*` env vars (`DB_ENGINE`, `DB_NAME`, `DB_USER`, `DB_PASSWORD`, `DB_HOST`, `DB_PORT`) as well as legacy `SQL_*` aliases for docker-compose compatibility. -- **Redis** — a single `REDIS_URL` env var configures the Django cache location, `CELERY_BROKER_URL`, and `CELERY_RESULT_BACKEND` simultaneously. -- **Email** — `EMAIL_HOST`, `EMAIL_PORT`, `EMAIL_HOST_USER`, `EMAIL_HOST_PASSWORD`, `EMAIL_USE_TLS` all respect env vars. -- **AWS SES** — `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_SES_REGION_NAME`, `AWS_SES_REGION_ENDPOINT`. -- **Social auth** — `FACEBOOK_KEY`, `FACEBOOK_SECRET`, `TWITTER_KEY`, `TWITTER_SECRET`, `GOOGLE_KEY`, `GOOGLE_SECRET`. -- **`.env.example`** — updated to document every env var with safe placeholder values, organized by category. -- **`config.wcoa.ini` / `config.mida.ini`** — these still contain real credentials and should be removed from git history using `git filter-repo --path config.wcoa.ini --path config.mida.ini --invert-paths`. That step requires a git client and is left for the team to execute. - -### Social Auth Pipeline - -- Renamed all `social.pipeline.*` strings in `SOCIAL_AUTH_PIPELINE` to `social_core.pipeline.*` — the correct module path for `social-auth-core ≥ 4.x`. The old `social` namespace was a legacy alias that has been dropped. - -### rpc4django Replacement - -- **Removed** `rpc4django` from `INSTALLED_APPS`, `requirements.txt`, and `urls.py`. -- The single `/rpc` XML-RPC endpoint served **11 methods** across 3 sub-apps. Each has been replaced with a typed DRF `APIView`: - -| Old RPC method | New endpoint | App | -|---|---|---| -| `get_bookmarks` | `GET /api/bookmarks/` | visualize | -| `add_bookmark` | `POST /api/bookmarks/` | visualize | -| `load_bookmark` | `GET /api/bookmarks//` | visualize | -| `remove_bookmark` | `DELETE /api/bookmarks//` | visualize | -| `share_bookmark` | `POST /api/bookmarks//share/` | visualize | -| `get_user_layers` | `GET /api/user-layers/` | visualize | -| `add_user_layer` | `POST /api/user-layers/` | visualize | -| `load_user_layer` | `GET /api/user-layers//` | visualize | -| `remove_user_layer` | `DELETE /api/user-layers//` | visualize | -| `share_user_layer` | `POST /api/user-layers//share/` | visualize | -| `delete_drawing` | `DELETE /api/drawings//` | drawing | -| `get_sharing_groups` | `GET /api/sharing-groups/` | mapgroups | -| `update_map_group` | `PATCH /api/map-groups//` | mapgroups | - -- New files: `visualize/api.py`, `drawing/api.py`, `mapgroups/api.py`. Each app's `urls.py` updated accordingly. -- All new views carry full **type annotations** and proper DRF permission classes (`IsAuthenticated` / `AllowAny`). - -### accounts/pipeline.py - -- Removed Python 2 compatibility shims: `try/except ImportError` for `django.urls.reverse`, `try/except ImportError` for `urllib.parse`, and `import urlparse` (Python 2 stdlib). -- Replaced `urlparse.urlsplit` / `urlparse.urlunsplit` with `urllib.parse.urlsplit` / `urllib.parse.urlunsplit`. -- Removed dead `from django.core.context_processors import request` import (removed in Django 1.10). -- Removed dead `from django.conf.urls import include, url` fallback. -- Added proper type hints and a complete `send_validation_email` stub (was missing from the pipeline). - -### wagtail_migrations/ Directory - -- The directory contains 30+ step-by-step upgrade shell scripts (Wagtail 1.4 → 2.11), Python 2 view backups, and ancient requirements snapshots. None are needed at Wagtail 7. -- **The files are OS-level read-only in this environment.** Run this from the project root to remove them: - ```bash - git rm -rf wagtail_migrations/ - git commit -m "Remove historical wagtail_migrations upgrade scripts" - ``` - ---- - -## Phase 3 — Recommended Next Steps - -### High Priority - -- **Frontend build tooling** — Replace Bower + Gulp with `npm` + Vite. Bower has been deprecated since 2017. Add `/bower_components/` to `.gitignore` and drive dependencies through `package.json`. -- **Test coverage** — Only 2 test files exist. Add `pytest-django` suites targeting 60%+ coverage for models and views across the portal sub-apps. -- **CI / CD pipeline** — GitHub Actions: lint (`ruff`), test (`pytest`), Docker build, tag-based image push to registry. - -### Medium Priority - -- **Django 5.x upgrade** — Django 4.2 LTS support ends April 2026. Evaluate Django 5.1 once sub-app compatibility is confirmed. -- **Consolidate config.ini variants** — 6 config files remain. Migrate to a single `.env`-driven approach and retire the `.ini` files. -- **Expand type coverage** — Run `mypy --strict` against `portal/`, `marco_site/`, and all sub-app `views.py` files; address errors incrementally. - ---- - -## Appendix — Technical Debt Removed - -| Category | Count / Description | -|---|---| -| Python 2 imports removed | 72 migration files | -| Wagtail version branches removed | 3 (v1, v2, v5 detection) | -| `eval()` calls on config data removed | 2 (`ADDITIONAL_APPS`, `ADDITIONAL_MIDDLEWARE`) | -| `exec()` for dynamic import removed | 1 | -| Deprecated Celery settings removed | 1 (`BROKER_URL`) | -| Dead code blocks removed | 2 (`if False:`, commented middleware) | -| Dockerfile layer ordering bugs fixed | 3 mis-indented `RUN` commands | -| Docker Compose healthchecks added | 2 services (`db`, `tasks`) | -| Secret key runtime guard added | 1 (was silently `'you forgot...'`) | From 5b182606771f02a960b002b0301fc767e70f3329 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Wed, 22 Jul 2026 18:15:46 -0700 Subject: [PATCH 146/152] Update README.md: revise Docker Development Guide to clarify architecture and project overlays --- README.md | 103 +++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 79 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 5e0bf80..a2beebb 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,14 @@ -# Docker Development Guide — Madrona Portal +# Madrona Portal -## Docker architecture - -The madrona-portal repository is a core project-agnostic platform image: +The madrona-portal repository builds a **generic base image** that contains the platform and shared apps. -- `ghcr.io/ecotrust/madrona-portal:{sha,latest}` -- contains shared Madrona/MP apps and runtime tooling -- does not include `wcoa` or `mida` project code/configuration +## Docker architecture -Project overlays now live in each project repository and build from this base image. +The madrona-portal core base images are published to GitHub Container Registry (GHCR) on merge to `main` using github actions. -### Build and publish core base image +The base image is built from the `docker/Dockerfile` in this repo [view Dockerfile](docker/Dockerfile) and contains: + - image on GHCR `ghcr.io/ecotrust/madrona-portal:{sha,latest}` + - shared Madrona/MP apps and runtime tooling Core CI publishes on merge to `main`: @@ -19,24 +17,82 @@ ghcr.io/ecotrust/madrona-portal: ghcr.io/ecotrust/madrona-portal:latest ``` -### Dockerizing a new portal checklist +### Portal overlays + +Each portal overlay builds a thin image on top of the base image, adding its own code, templates, static assets, and settings. + +The madrona-portal supports the following portal overlays: + - [West Coast Ocean Data Portal (WCOA)](https://github.com/Ecotrust/wcoa) + - [Mid-Atlantic Data Portal (MidA)](https://github.com/Ecotrust/mida-portal) + +Each overlay is built from its own `docker/Dockerfile` and published to GHCR on merge to `main` using github actions. + +> *Please note:* The first portal to be Dockerized was `wcoa`. You may come across legacy WCOA-coupled Docker notes. Use the project repos for current portal-specific Docker workflows. + +### Portal overlay structure +Two-layer image hierarchy, mirroring the code architecture (platform + customization): + +``` +┌────────────────────────────────────────────────────────────┐ +│ madrona-portal (base image) │ +│ ghcr.io/ecotrust/madrona-portal: │ +│ • Ubuntu, Python venv, GDAL/GEOS/PostGIS libs. │ +│ • marco/ │ +│ • all shared mp-* & madrona-* apps (pip -e installed) │ +│ • entrypoint.sh │ +└──────────────┬─────────────────────────┬───────────────────┘ + │ FROM │ FROM +┌──────────────▼───────────┐ ┌───────────▼──────────────────┐ +│ mida-portal image │ │ wcoa image │ +│ • COPY mida repo │ │ • COPY wcoa repo │ +│ • pip -e install mida │ │ • pip -e install wcoa │ +│ • config.mida.docker.ini│ │ • config.wcoa.docker.ini │ +│ • ENV MP_PROJECT_CONFIG │ │ • ENV MP_PROJECT_CONFIG │ +│ │ │ + geoportal/elastic compose │ +│ │ │ overlay │ +└──────────────────────────┘ └──────────────────────────────┘ +``` -1. Create `/docker/Dockerfile` that does: - - `FROM ghcr.io/ecotrust/madrona-portal:` - - `COPY . ./apps/` - - `pip install --no-deps -e ./apps/` - - `ENV MP_PROJECT_CONFIG=/usr/local/apps/madrona-portal/apps//docker/config..docker.ini` -2. Add `/docker/config..docker.ini`. -3. Add `/docker/compose.yml` overlay with project env/ports/volumes/services. -4. Add `/docker/.env.example` and local `docker/.env` (gitignored). +### Portal ownership boundaries + +| Concern | Lives in core (`madrona-portal`) | Lives in project repo (`mida-portal`, `wcoa`) | +| --------------------------------------------------------------------- | -------------------------------- | --------------------------------------------- | +| System deps (GDAL, PostGIS libs, build toolchain) | ✔ | | +| Shared Python deps (`docker-requirements.txt`) | ✔ | | +| Shared mp-* apps | ✔ (COPY + editable install) | | +| Entrypoint (connect to db, collectstatic, compress, DB_INIT, server select) | ✔ | | +| Base compose (db, redis, app skeleton) | ✔ | | +| Project app code + editable install | | ✔ | +| Project `config..docker.ini` | | ✔ | +| Project-only Python deps | | ✔ (`docker/requirements.txt`) | +| Project-only services (geoportal, elastic) | | ✔ (compose overlay) | +| `.env`, ports, DB name, volume names | | ✔ | +| `MP_PROJECT_CONFIG` value | | ✔ (set in project Dockerfile/compose) | + +### Portal architecture guidelines +- Projects (mida, wcoa) are decoupled from the core platform (madrona-portal) and own their own Docker overlay, config, and compose. +- Projects reference the core; the core never references a project. + +### Steps for adding a new portal +It is recommended to use an existing portal overlay as a template for creating a new portal overlay. The following steps generalize the process: + +1. Create `/docker/Dockerfile` with the following contents: + ```dockerfile + FROM ghcr.io/ecotrust/madrona-portal: + COPY . ./apps/ + pip install --no-deps -e ./apps/ + ENV MP_PROJECT_CONFIG=/usr/local/apps/madrona-portal/apps//docker/config..docker.ini + ``` +2. Add `/docker/config..docker.ini` +3. Add `/docker/compose.yml` overlay with project env/ports/volumes/services +4. Add `/docker/.env.example` and local `docker/.env` 5. Add project CI workflow to build/push: - `ghcr.io/ecotrust/:` - `ghcr.io/ecotrust/:latest` - - pass a pinned `BASE_TAG` build argument. -6. Add a quickstart section to the project README. -*Please note:* The first portal to be Dockerized was `wcoa`. You may come across legacy WCOA-coupled Docker notes. Use the project repos for current portal-specific Docker workflows. +---- +# :alert: Documentation beyond this point is a work in progress and may be out of date. ## Quick start @@ -44,8 +100,7 @@ These steps take a fresh machine from nothing to a running portal. ### Step 1 — Create the workspace directory -All repos live inside a single parent directory. The Dockerfile build -context is the parent, so the layout is not optional. +All repos live inside a single parent directory. ```bash mkdir portals @@ -55,7 +110,7 @@ cd portals ### Step 2 — Clone madrona-portal ```bash -git clone -b docker https://github.com/Ecotrust/madrona-portal.git madrona-portal +git clone https://github.com/Ecotrust/madrona-portal.git madrona-portal ``` ### Step 3 — Clone the sub-app packages From 4cdb112a3f6cf7ef7f663771a57e464bed7d9120 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Wed, 22 Jul 2026 18:17:50 -0700 Subject: [PATCH 147/152] Remove development configuration file for MARCO Portal --- marco/config.ini.dev | 54 -------------------------------------------- 1 file changed, 54 deletions(-) delete mode 100644 marco/config.ini.dev diff --git a/marco/config.ini.dev b/marco/config.ini.dev deleted file mode 100644 index 5f10712..0000000 --- a/marco/config.ini.dev +++ /dev/null @@ -1,54 +0,0 @@ -# Configuration file for MARCO Portal deployments. -# Server-specific configuration goes here. - -[APP] -DEBUG = True -TEMPLATE_DEBUG = True -ALLOWED_HOSTS = * -SECRET_KEY = You_forgot_to_set_the_secret_key -MEDIA_ROOT = /home/vagrant/marco_portal2/media -MEDIA_URL = /media/ -TIME_ZONE = UTC -GA_ACCOUNT = You forgot to set the google analytics account -STATIC_ROOT = /home/vagrant/marco_portal2/static -EMAIL_SUBJECT_PREFIX = [Marco] -STATIC_URL = /static/ - -[CACHES] -BACKEND = redis_cache.RedisCache -LOCATION = /var/run/redis/redis.sock - -[CELERY] -RESULT_BACKEND = redis+socket:///var/run/redis/redis.sock -BROKER_URL = redis+socket:///var/run/redis/redis.sock - -[DATABASE] -ENGINE = django.contrib.gis.db.backends.postgis -NAME = marco_portal -HOST = localhost -PORT = 5432 -USER = vagrant -PASSWORD = None - -[EMAIL] -HOST = localhost -PORT = 8025 -HOST_USER = mail user -HOST_PASSWORD = mail password - -[SOCIAL_AUTH] -FACEBOOK_KEY = You forgot to set the facebook key -FACEBOOK_SECRET = You forgot to set the facebook secret -TWITTER_KEY = You forgot to set the twitter key -TWITTER_SECRET = You forgot to set the twitter secret -GOOGLE_KEY = You forgot to set the google key -GOOGLE_SECRET = You forgot to set the google secret - -[CATALOG] -DATA_CATALOG_ENABLED = False -CATALOG_TECHNOLOGY = default -#CATALOG_TECHNOLOGY = GeoPortal2 -CATALOG_PROXY = -#CATALOG_SOURCE = http://127.0.0.1:9200 -CATALOG_QUERY_ENDPOINT = /geoportal/elastic/metadata/item/_search/ - From fe93fe13ea5b1a59ba2185c9fd063750dd49be4a Mon Sep 17 00:00:00 2001 From: David Pollard Date: Wed, 22 Jul 2026 18:18:33 -0700 Subject: [PATCH 148/152] Update workflow trigger: restrict to main branch only --- .github/workflows/create-and-publish-docker-images.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/create-and-publish-docker-images.yml b/.github/workflows/create-and-publish-docker-images.yml index 105898c..d7dd44e 100644 --- a/.github/workflows/create-and-publish-docker-images.yml +++ b/.github/workflows/create-and-publish-docker-images.yml @@ -2,7 +2,7 @@ name: Build and publish madrona-portal base image on: push: - branches: [main, dockerdecouple] + branches: [main] workflow_dispatch: # Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. From 75bf73a6da5449f9c7e8d346ab2d4d30f85afce0 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Thu, 30 Jul 2026 16:09:34 -0700 Subject: [PATCH 149/152] Update workflow to trigger on dockerdecouple branch and add multi-platform support for Docker images --- .github/workflows/create-and-publish-docker-images.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/create-and-publish-docker-images.yml b/.github/workflows/create-and-publish-docker-images.yml index d7dd44e..8bc7818 100644 --- a/.github/workflows/create-and-publish-docker-images.yml +++ b/.github/workflows/create-and-publish-docker-images.yml @@ -2,7 +2,7 @@ name: Build and publish madrona-portal base image on: push: - branches: [main] + branches: [main, dockerdecouple] workflow_dispatch: # Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. @@ -157,6 +157,7 @@ jobs: context: . file: madrona-portal/docker/Dockerfile push: true + platforms: linux/amd64,linux/arm64 tags: | ${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.sha }} ${{ env.IMAGE_NAME }}:latest From e43f2c06ef6ea1b7f90fc0759472a7f662837560 Mon Sep 17 00:00:00 2001 From: David Pollard Date: Mon, 3 Aug 2026 14:30:07 -0700 Subject: [PATCH 150/152] Remove multi-platform support from Docker image build configuration --- .github/workflows/create-and-publish-docker-images.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/create-and-publish-docker-images.yml b/.github/workflows/create-and-publish-docker-images.yml index 8bc7818..105898c 100644 --- a/.github/workflows/create-and-publish-docker-images.yml +++ b/.github/workflows/create-and-publish-docker-images.yml @@ -157,7 +157,6 @@ jobs: context: . file: madrona-portal/docker/Dockerfile push: true - platforms: linux/amd64,linux/arm64 tags: | ${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.sha }} ${{ env.IMAGE_NAME }}:latest From 00a915342fc10a1792a53c23fb5a97d45e3afe7e Mon Sep 17 00:00:00 2001 From: David Pollard Date: Mon, 3 Aug 2026 14:34:39 -0700 Subject: [PATCH 151/152] Bump version to 4.0.2 in pyproject.toml --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 5258227..8d9c9fe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.backends.legacy:build" [project] name = "madrona-portal" -version = "2.0.0" +version = "4.0.2" description = "MARCO Mid-Atlantic Ocean Data Portal" requires-python = ">=3.10" readme = "README.md" From 6879cd1bfbe83a36c7fb20693f12d97b86110f9a Mon Sep 17 00:00:00 2001 From: David Pollard Date: Mon, 3 Aug 2026 15:47:57 -0700 Subject: [PATCH 152/152] Add version tag 4.0.3 to Docker image build configuration --- .github/workflows/create-and-publish-docker-images.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/create-and-publish-docker-images.yml b/.github/workflows/create-and-publish-docker-images.yml index 105898c..f301dcd 100644 --- a/.github/workflows/create-and-publish-docker-images.yml +++ b/.github/workflows/create-and-publish-docker-images.yml @@ -160,6 +160,7 @@ jobs: tags: | ${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.sha }} ${{ env.IMAGE_NAME }}:latest + ${{ env.IMAGE_NAME }}:4.0.3 cache-from: type=gha cache-to: type=gha,mode=max