diff --git a/.github/workflows/create-and-publish-docker-images.yml b/.github/workflows/create-and-publish-docker-images.yml new file mode 100644 index 00000000..f301dcd5 --- /dev/null +++ b/.github/workflows/create-and-publish-docker-images.yml @@ -0,0 +1,171 @@ +name: Build and publish madrona-portal base image + +on: + push: + branches: [main, dockerdecouple] + workflow_dispatch: + +# Defines two custom environment variables for the workflow. These are used for the Container registry domain, and a name for the Docker image that this workflow builds. +env: + IMAGE_NAME: ghcr.io/ecotrust/madrona-portal + +# There is a single job in this workflow. It's configured to run on the latest available version of Ubuntu. +jobs: + build-and-push: + runs-on: ubuntu-latest + # Sets the permissions granted to the `GITHUB_TOKEN` for the actions in this job. + permissions: + contents: read + packages: write + attestations: write + id-token: write + + steps: + - name: Checkout madrona-portal + uses: actions/checkout@v5 + with: + path: madrona-portal + + - name: Checkout django_url_shortener + uses: actions/checkout@v5 + with: + repository: Ecotrust/django_url_shortener + token: ${{ secrets.GH_PAT }} + path: madrona-apps/django_url_shortener + + - name: Checkout madrona-analysistools + uses: actions/checkout@v5 + with: + repository: Ecotrust/madrona-analysistools + token: ${{ secrets.GH_PAT }} + path: madrona-apps/madrona-analysistools + + - name: Checkout madrona-features + uses: actions/checkout@v5 + with: + repository: Ecotrust/madrona-features + token: ${{ secrets.GH_PAT }} + path: madrona-apps/madrona-features + + - name: Checkout madrona-manipulators + uses: actions/checkout@v5 + with: + repository: Ecotrust/madrona-manipulators + token: ${{ secrets.GH_PAT }} + path: madrona-apps/madrona-manipulators + + - name: Checkout madrona-scenarios + uses: actions/checkout@v5 + with: + repository: Ecotrust/madrona-scenarios + token: ${{ secrets.GH_PAT }} + path: madrona-apps/madrona-scenarios + + - name: Checkout mp-accounts + uses: actions/checkout@v5 + with: + repository: Ecotrust/mp-accounts + token: ${{ secrets.GH_PAT }} + path: madrona-apps/mp-accounts + + - name: Checkout mp-data-manager + uses: actions/checkout@v5 + with: + repository: Ecotrust/mp-data-manager + token: ${{ secrets.GH_PAT }} + path: madrona-apps/mp-data-manager + + - name: Checkout mp-drawing + uses: actions/checkout@v5 + with: + repository: Ecotrust/mp-drawing + token: ${{ secrets.GH_PAT }} + path: madrona-apps/mp-drawing + + - name: Checkout mp-explore + uses: actions/checkout@v5 + with: + repository: Ecotrust/mp-explore + token: ${{ secrets.GH_PAT }} + path: madrona-apps/mp-explore + + - name: Checkout mp-layers + uses: actions/checkout@v5 + with: + repository: Ecotrust/mp-layers + token: ${{ secrets.GH_PAT }} + path: madrona-apps/mp-layers + + - name: Checkout mp-map-groups + uses: actions/checkout@v5 + with: + repository: Ecotrust/mp-map-groups + token: ${{ secrets.GH_PAT }} + path: madrona-apps/mp-map-groups + + - name: Checkout mp-proxy + uses: actions/checkout@v5 + with: + repository: Ecotrust/mp-proxy + token: ${{ secrets.GH_PAT }} + path: madrona-apps/mp-proxy + + - name: Checkout mp-survey + uses: actions/checkout@v5 + with: + repository: Ecotrust/mp-survey + token: ${{ secrets.GH_PAT }} + path: madrona-apps/mp-survey + + - name: Checkout mp-visualize + uses: actions/checkout@v5 + with: + repository: Ecotrust/mp-visualize + token: ${{ secrets.GH_PAT }} + path: madrona-apps/mp-visualize + + - name: Checkout p97-nursery + uses: actions/checkout@v5 + with: + repository: Ecotrust/p97-nursery + token: ${{ secrets.GH_PAT }} + path: madrona-apps/p97-nursery + + # ----------------------------------------------------------------------- + # Docker setup + # ----------------------------------------------------------------------- + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract short SHA + id: meta + working-directory: madrona-portal + run: | + SHA=$(git rev-parse --short HEAD 2>/dev/null || echo "${GITHUB_SHA::7}") + echo "sha=$SHA" >> "$GITHUB_OUTPUT" + + - name: Build and push + uses: docker/build-push-action@v5 + with: + context: . + file: madrona-portal/docker/Dockerfile + push: true + tags: | + ${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.sha }} + ${{ env.IMAGE_NAME }}:latest + ${{ env.IMAGE_NAME }}:4.0.3 + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Image digest summary + run: | + echo "### Image pushed to GHCR" >> $GITHUB_STEP_SUMMARY + echo "- \`${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.sha }}\`" >> $GITHUB_STEP_SUMMARY + echo "- \`${{ env.IMAGE_NAME }}:latest\`" >> $GITHUB_STEP_SUMMARY diff --git a/.gitignore b/.gitignore index cb9e35f3..d91f333f 100644 --- a/.gitignore +++ b/.gitignore @@ -16,8 +16,12 @@ vagrant .sass-cache node_modules +docker/backups/ +docker/media/ +docker/static/ docker/entrypoint.sh docker/docker-requirements.txt +docker/wars/ marco_site/static/bundles/ marco_site/static/css/ @@ -29,9 +33,12 @@ htmlcov .coverage config.ini config.mida.ini +config.mida.docker.ini config.wcoa.ini +config.wcoa.docker.ini pytest.ini marco.db +.env .bashrc dev_fixture.json diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index c12da9cc..00000000 --- a/Dockerfile +++ /dev/null @@ -1,58 +0,0 @@ -# pull official base image -FROM python:3.9.6-alpine -#FROM alpine:3.14 -#FROM python:3.8.10-alpine - -# set environment variables -ENV PYTHONDONTWRITEBYTECODE 1 -ENV PYTHONUNBUFFERED 1 - -# set work directory -WORKDIR /usr/local/apps/madrona-portal - -# copy project -COPY ./marco /usr/local/apps/madrona-portal/marco -COPY ./apps /usr/local/apps/madrona-portal/apps -COPY ./assets /usr/local/apps/madrona-portal/assets -COPY ./bower_components /usr/local/apps/madrona-portal/bower_components -COPY ./docker/entrypoint.sh /entrypoint.sh -COPY ./docker/docker-requirements.txt /requirements.txt - -COPY ./backups/ /usr/local/apps/madrona-portal/backups - -# install dependencies -RUN \ - apk update &&\ - apk add --no-cache --virtual .build-deps \ - autoconf automake make \ - musl-dev gcc binutils g++ \ - libffi-dev \ - pkgconfig openssl \ - &&\ - apk add --no-cache --update \ - python3-dev \ - postgresql postgresql-contrib \ - postgresql-dev postgresql-libs \ - jpeg-dev libjpeg zlib-dev libtool \ - libpq gdal gdal-tools geos-dev gdal-dev \ - protobuf-c-dev json-c-dev perl libxml2-dev \ - proj proj-dev proj-util \ - && \ - pip install --upgrade pip &&\ - pip install -r /requirements.txt - #&&\ - #apk --purge del .build-deps - -RUN chmod +x /entrypoint.sh - -RUN mkdir -p /vol/web/media -RUN mkdir -p /vol/web/static - -RUN adduser -D madrona_user -RUN chown -R madrona_user:madrona_user /vol -RUN chown -R madrona_user:madrona_user /usr/local/apps/madrona-portal -RUN chmod -R 755 /vol/web - -USER madrona_user - -CMD ["/entrypoint.sh"] diff --git a/README.md b/README.md index 2a76fe63..a2beebb0 100644 --- a/README.md +++ b/README.md @@ -1,167 +1,429 @@ -# MARCO Portal Redesign +# Madrona Portal -### This is the top level project for the Mid-Atlantic Ocean Data Portal +The madrona-portal repository builds a **generic base image** that contains the platform and shared apps. -### ~Development Installation +## Docker architecture -##### Initial Setup using Vagrant: -The following is the **_recommended_** folder structure for the **entire** MARCO project and the customized provisioning script is inherently dependent on it. Altering the folder and naming structure will require modifications to the provisioning script, so please be aware! The provisioning script is designed to be a **one-step** install after initial setup. +The madrona-portal core base images are published to GitHub Container Registry (GHCR) on merge to `main` using github actions. + +The base image is built from the `docker/Dockerfile` in this repo [view Dockerfile](docker/Dockerfile) and contains: + - image on GHCR `ghcr.io/ecotrust/madrona-portal:{sha,latest}` + - shared Madrona/MP apps and runtime tooling + +Core CI publishes on merge to `main`: + +```bash +ghcr.io/ecotrust/madrona-portal: +ghcr.io/ecotrust/madrona-portal:latest +``` + +### Portal overlays + +Each portal overlay builds a thin image on top of the base image, adding its own code, templates, static assets, and settings. + +The madrona-portal supports the following portal overlays: + - [West Coast Ocean Data Portal (WCOA)](https://github.com/Ecotrust/wcoa) + - [Mid-Atlantic Data Portal (MidA)](https://github.com/Ecotrust/mida-portal) + +Each overlay is built from its own `docker/Dockerfile` and published to GHCR on merge to `main` using github actions. + +> *Please note:* The first portal to be Dockerized was `wcoa`. You may come across legacy WCOA-coupled Docker notes. Use the project repos for current portal-specific Docker workflows. + +### Portal overlay structure +Two-layer image hierarchy, mirroring the code architecture (platform + customization): ``` - -- madrona-portal - -- apps (all remaining repositories within Madrona Portal) - -- mardona-analysistools - -- madrona-features - -- etc. +┌────────────────────────────────────────────────────────────┐ +│ madrona-portal (base image) │ +│ ghcr.io/ecotrust/madrona-portal: │ +│ • Ubuntu, Python venv, GDAL/GEOS/PostGIS libs. │ +│ • marco/ │ +│ • all shared mp-* & madrona-* apps (pip -e installed) │ +│ • entrypoint.sh │ +└──────────────┬─────────────────────────┬───────────────────┘ + │ FROM │ FROM +┌──────────────▼───────────┐ ┌───────────▼──────────────────┐ +│ mida-portal image │ │ wcoa image │ +│ • COPY mida repo │ │ • COPY wcoa repo │ +│ • pip -e install mida │ │ • pip -e install wcoa │ +│ • config.mida.docker.ini│ │ • config.wcoa.docker.ini │ +│ • ENV MP_PROJECT_CONFIG │ │ • ENV MP_PROJECT_CONFIG │ +│ │ │ + geoportal/elastic compose │ +│ │ │ overlay │ +└──────────────────────────┘ └──────────────────────────────┘ ``` -1. Download the required code and dependencies: +### Portal ownership boundaries + +| Concern | Lives in core (`madrona-portal`) | Lives in project repo (`mida-portal`, `wcoa`) | +| --------------------------------------------------------------------- | -------------------------------- | --------------------------------------------- | +| System deps (GDAL, PostGIS libs, build toolchain) | ✔ | | +| Shared Python deps (`docker-requirements.txt`) | ✔ | | +| Shared mp-* apps | ✔ (COPY + editable install) | | +| Entrypoint (connect to db, collectstatic, compress, DB_INIT, server select) | ✔ | | +| Base compose (db, redis, app skeleton) | ✔ | | +| Project app code + editable install | | ✔ | +| Project `config..docker.ini` | | ✔ | +| Project-only Python deps | | ✔ (`docker/requirements.txt`) | +| Project-only services (geoportal, elastic) | | ✔ (compose overlay) | +| `.env`, ports, DB name, volume names | | ✔ | +| `MP_PROJECT_CONFIG` value | | ✔ (set in project Dockerfile/compose) | + +### Portal architecture guidelines +- Projects (mida, wcoa) are decoupled from the core platform (madrona-portal) and own their own Docker overlay, config, and compose. +- Projects reference the core; the core never references a project. + +### Steps for adding a new portal +It is recommended to use an existing portal overlay as a template for creating a new portal overlay. The following steps generalize the process: + +1. Create `/docker/Dockerfile` with the following contents: + ```dockerfile + FROM ghcr.io/ecotrust/madrona-portal: + COPY . ./apps/ + pip install --no-deps -e ./apps/ + ENV MP_PROJECT_CONFIG=/usr/local/apps/madrona-portal/apps//docker/config..docker.ini + ``` +2. Add `/docker/config..docker.ini` +3. Add `/docker/compose.yml` overlay with project env/ports/volumes/services +4. Add `/docker/.env.example` and local `docker/.env` +5. Add project CI workflow to build/push: + - `ghcr.io/ecotrust/:` + - `ghcr.io/ecotrust/:latest` + +---- + +# :alert: Documentation beyond this point is a work in progress and may be out of date. + +## Quick start + +These steps take a fresh machine from nothing to a running portal. + +### Step 1 — Create the workspace directory + +All repos live inside a single parent directory. + +```bash +mkdir portals +cd portals ``` - git clone https://github.com/Ecotrust/madrona-portal.git - mv madrona-portal madrona-portal - cd madrona-portal - mkdir apps - cd apps - git clone https://github.com/Ecotrust/madrona-analysistools.git - git clone https://github.com/Ecotrust/madrona-features.git - git clone https://github.com/Ecotrust/madrona-manipulators.git - git clone https://github.com/Ecotrust/madrona-scenarios.git - git clone https://github.com/Ecotrust/mp-map-groups.git - git clone https://github.com/Ecotrust/mp-accounts.git - git clone https://github.com/Ecotrust/mp-data-manager.git - git clone https://github.com/Ecotrust/mp-drawing.git - git clone https://github.com/Ecotrust/mp-explore.git - git clone https://github.com/Ecotrust/mp-proxy.git - git clone https://github.com/Ecotrust/mp-visualize.git - git clone https://github.com/Ecotrust/p97-nursery.git + +### Step 2 — Clone madrona-portal + +```bash +git clone https://github.com/Ecotrust/madrona-portal.git madrona-portal ``` -2. Once your folder structure is set up, create a `config.ini` file by making a copy of the `config.ini.template` located at `madrona-portal/marco` and modify the following - * **SECRET_KEY** = [Punch in some random gibberish] - * **MEDIA_ROOT** = /home/vagrant/marco_portal2/media - * **STATIC_ROOT** = /home/vagrant/marco_portal2/static - * **LOCATION** = /var/run/redis/redis.sock - * **RESULT_BACKEND** = redis+socket:///var/run/redis/redis.sock - * **BROKER_URL** = redis+socket:///var/run/redis/redis.sock +### Step 3 — Clone the sub-app packages + +The Dockerfile copies all of these at build time. Clone them into a +`madrona-apps/` sibling directory: + +```bash +mkdir madrona-apps && cd madrona-apps + +git clone https://github.com/Ecotrust/django_url_shortener.git +git clone https://github.com/Ecotrust/madrona-analysistools.git +git clone https://github.com/Ecotrust/madrona-features.git +git clone https://github.com/Ecotrust/madrona-manipulators.git +git clone https://github.com/Ecotrust/madrona-scenarios.git +git clone https://github.com/Ecotrust/mida-portal.git +git clone https://github.com/Ecotrust/mp-accounts.git +git clone https://github.com/Ecotrust/mp-data-manager.git +git clone https://github.com/Ecotrust/mp-drawing.git +git clone https://github.com/Ecotrust/mp-explore.git +git clone https://github.com/Ecotrust/mp-layers.git +git clone https://github.com/Ecotrust/mp-map-groups.git +git clone https://github.com/Ecotrust/mp-proxy.git +git clone https://github.com/Ecotrust/mp-survey.git +git clone https://github.com/Ecotrust/mp-visualize.git +git clone https://github.com/Ecotrust/p97-nursery.git +git clone https://github.com/Ecotrust/wcoa.git + +cd .. +``` -3. Create a `/static/` directory at the root level and move the `/bower_components/` directory (also found at the root level) within it +Your workspace should now look like: -4. Create a `/media/` directory at the root level and retrieve the live server's media folder via ssh/sftp located at `/webapps/marco_portal_media/` and add it to the `/media/` path. Refer to your team's technical documentation for server login (username and password) credentials - * Of note - you may want to exclude the `data_manager` folder within the media directory - unless you're interested in several GBs of utfgrid layers. ``` -mkdir media -scp -r user@live_server:~/webapps/marco_portal_media/documents ./media/ #11s -- RDH 7/27/2017 -scp -r user@live_server:~/webapps/marco_portal_media/group_images ./media/ #11s -scp -r user@live_server:~/webapps/marco_portal_media/images ./media/ #3m19s -scp -r user@live_server:~/webapps/marco_portal_media/original_images ./media/ #3m57s -scp user@live_server:~/webapps/marco_portal_media/index.html ./media/ #12s +portals/ +├── madrona-portal/ +└── madrona-apps/ + ├── wcoa/ + ├── mida-portal/ + ├── mp-layers/ + └── ... ``` -5. Retrieve the data & content fixture from `~/fixtures/dev_fixture.json` via ssh/sftp and place it at the root level of `madrona-portal` +### Step 4 — Configure environment + +From `madrona-portal/`: + +```bash +cd madrona-portal/docker +cp .env.example .env ``` -cd [working dir]/madrona-portal -scp user@live_server:~/fixtures/dev_fixture.json ./ #25s + +Edit `.env` and set at minimum: + +```ini +SECRET_KEY= +DB_PASSWORD= +DJANGO_SUPERUSER_PASSWORD= ``` -6. Download and install [vagrant](https://www.vagrantup.com/downloads.html) and [virtual box](https://www.virtualbox.org/wiki/Downloads) (if you haven't already done so already) +Everything else has working defaults for local development. -7. At the root of `madrona-portal`, run `vagrant up` and let it install ALL of dependencies MARCO relies upon +### Step 4.1 - Create ini file -8. At this point, you should be completely setup! - * Note: At this point, there still seem to be issues with Wagtail Pages, and therefore Ocean Stories. - * This is due to needing to configure for Redis to run on a socket, and Celery to point to that socket. - * This is likely not the only way, but it's what I have working and how it runs on production. --RDH +```bash +cd ../marco +cp config.docker.ini.template config.docker.ini +``` -9. You probably want to create a superuser once you're in your VM, so that you have access to both the Django and Wagtail backend +Edit `config.docker.ini` : -##### Using Vagrant -* Access your VM by running `vagrant ssh`. This will automatically log you into your virtual machine with your virtual environment activated at the project root level. +```ini +LOCATION = redis://tasks:6379/1 +CELERY_RESULT_BACKEND = redis://tasks:6379/1 +CELERY_BROKER_URL = redis://tasks:6379/0 +``` + +### Step 5 — Build the image +Run this command from `madrona-portal/docker` (the previous step leaves +you in `madrona-portal/marco`, so `cd ../docker` gets you there). The +Docker build context for this command is `../../`, which resolves to the parent workspace directory `portals/` that contains both +`madrona-portal/` and `madrona-apps/`. -* **Shortcuts** - * To use `/manage.py` with normal django administrative tasks , use the keyword `dj` +If running this build from a MAC, add `--builder desktop-linux` to the `buildx build` command. - ``` - dj makemigrations - dj migrate - dj createsuperuser - dj dumpdata - etc. - ``` +```bash +cd ../docker +# MAC OS +docker compose build --no-cache app +# LINUX +docker compose build --no-cache app +``` + +Previously we recommended `docker buildx build --builder desktop-linux --no-cache --load -f ./Dockerfile ../../` for all platforms, but BuildKit caching on Linux does not appear to have the same git object store issue as on Mac, so the simpler `docker compose build --no-cache app` is sufficient on Linux. + +> **Why `docker buildx build` and not `docker compose build`?** +> `docker compose build` has a caching bug: when a `.git` directory exists +> inside the build context, BuildKit reads files from the git object store +> (committed versions) rather than the filesystem. If you forget to commit +> a change, the old version is silently baked into the image. The same +> restriction applies — always commit changes to `madrona-portal/` or +> `madrona-apps/` before rebuilding. + +### Step 6 — Start the full stack; Populate testing DB + +From `madrona-portal/docker`: + +```bash +DB_INIT=1 docker compose up +``` - * Typing `djrun` will run your dev server - remember to add your sample data first (see #5): +On first boot the entrypoint automatically: +1. Waits for PostgreSQL to accept connections +2. Runs `migrate` +3. Runs `collectstatic` and `compress` +4. Detects a fresh database and loads initial fixtures (1,782 + 22 objects) +5. Creates the superuser defined in `.env` (if `DJANGO_SUPERUSER_PASSWORD` is set) +6. Starts the application server -* **NOTE:** The provisioning script is designed for a fresh install and will completely wipe the database and any associated content - IF you decide to shutdown your VM! Outside of halting your vagrant machine, running `vagrant up` or `vagrant provision` will cause the provisioning script to re-run. Adding the flag `--no-provision` to `vagrant up` will ignore the script. +Open: http://localhost:8000/ (or whatever `APP_PORT` is set to in `.env`) -#### **** OPTIONAL *** -If you decide to use pgAdmin3 for database management rather than using the command line, you'll need to allow/enable access to your virtual machine. -* Enter into `postgres.conf` and change `listen_addresses`: - ``` - sudo nano /etc/postgresql/9.3/main/postgresql.conf - listen_addresses = '*' - ``` - -* Enter into `pg_hba.conf` and add the `host` line: - ``` - sudo nano /etc/postgresql/9.3/main/pg_hba.conf - host all all 10.0.0.0/16 md5 - ``` - -* Restart postgresql - ``` - sudo /etc/init.d/postgresql restart - ``` - -* Within pgAdmin3, modify your settings: - * **Name:** marco_portal - * **Host:** localhost - * **Port:** 65432 - * **Username:** vagrant - - -### ~Code Deployment -Since this project is modularized, changes to a submodule only requires server updates to that specific submodule - rather than the entire code base. - -1. SSH into the server -2. Activate your virtual env - `source ~/env/marco_portal2/bin/activate` -3. Navigate to the submodule that you're updating. Submodules are located at: - * **Sandbox** - `cd /home/midatlantic/env/marco_portal2/src/[THE-NAME-OF-YOUR-SUBMODULE]` - * **Production** - `cd ~/webapps/marco_portal/marco/src/` -4. Once you're at that path - `git fetch && git reset -q --hard origin/master` - * `origin/master` pertains to the main master branch - you can change that to whatever your branch you'd like - * Of note, the master *madrona-portal* branch runs as `origin/prototype` -5. Navigate to `cd ~/webapps/marco_portal/marco` -6. Run `python manage.py collectstatic` to collect all the neccessary static (js/css) files - * you can use the -i flag to ignore utfgrids in the rare chance that those files seems to be "collecting" - * `python manage.py collectstatic -i utfgrid` -7. Run `python manage.py compress` to compress -8. Restart the server - `~/webapps/marco_portal/apache2/bin/restart` - -### ~Adding a new module to the apps directory and deployment -Adding a new module to marco requires a few additional steps for both local/development setup and deployment. +Once you have a populated DB (either dummy or with migrated data) omit the `DB_INIT=1`: +```bash +docker compose up +``` + + +### Step 7 - Importing a Production SQL Dump into the Dockerized Database + +#### Prerequisites +- Docker Compose stack is running — `docker compose up` +- `madrona-portal/docker/.env` exists with `DB_NAME`, `DB_USER`, and `DB_PASSWORD` set + + +#### Step 7.1 — Ensure you have the db-restore script + +`madrona-portal/scripts/db-restore.sh` has the following behaviour: +- Loads DB credentials from `.env` +- Verifies the `db` container is healthy before proceeding +- With `--drop`: terminates active connections, drops and recreates the database, and enables the PostGIS extension +- Streams the dump file directly into the container via `docker compose exec` (no temp files) +- Prints next-step instructions on completion + +Made sure it is executable: + +From `madrona-portal/docker`: +```bash +chmod +x ../scripts/db-restore.sh +``` + +#### Step 7.2 — Run the restore + +From `madrona-portal/docker`: +```bash +../scripts/db-restore.sh --drop +``` + +*example:* +```bash +../scripts/db-restore.sh --drop ../../madrona-apps/wcoa/wcodp_prod_dump_20260320.sql +``` + +The `--drop` flag was used to ensure a clean import. The script: +1. Terminated all active connections to `wcoa_docker_db` +2. Dropped and recreated the database +3. Enabled the `postgis` extension +4. Streamed the sql dump into the container via `psql` + +There is an optional `--env-file ` if you place your `.env` file in a non-standard location. + +**Expected warnings (non-fatal):** +- `ERROR: relation "..." does not exist` — pg_dump tries to drop constraints before creating them; safe to ignore on a fresh DB +- `ERROR: role "wcoa_user" does not exist` — prod uses a dedicated app role; dev uses `postgres` which has full access + + +#### Step 7.3 — Apply migrations + +```bash +docker compose exec app python marco/manage.py migrate +``` + +#### Step 7.4 - Migration to mp-layers + +*If migrating from a server that has not migrated to mp-layers from mp-data-manager*: + +```bash +docker compose exec app python marco/manage.py migration_to_layers +``` -**Local/Development setup**: - -1. create directory within `madrona-portal/apps` - * Use `git clone` for exisiting module or create a new direcotry and use `git init` to set up your new repository. - * If this is a new git repository create a new remote origin repo within the [MidAtlanticPortal](https://github.com/MidAtlanticPortal) orgainization. *Next steps assume your new module is ready to use.* -2. open `madrona-portal/requirements.txt` and add the newly created git remote repository (*e.g.* `-e git+https://github.com/MidAtlanticPortal/your_new_repo.git@master#egg=an_alias`). *the `@master#egg=` assigns an alias (simple name) for your module* -3. open `madrona-portal/marco/marco/settings.py` and add your new module's alias as an `INSTALLED_APPS`. (*e.g.*, `INSTALLED_APPS = [ 'an_alias']`) -4. run `vagrant provision` +--- +### Step 8 — Importing production media files into the Dockerized Application -**Deployment**: +#### Prerequisites + - `portals/madrona-portal/media` -1. ssh into the server (sandbox or production) -2. activate your virtual env - `source ~/env/marco_portal2/bin/activate` -3. navigate to the submodule directory `cd /home/midatlantic/env/marco_portal2/src/` -4. `git clone` your new module repository -5. navigate to the marco-portal repo `cd ~/code/marco_portal2/prototype/` -6. run `git fetch && git reset -q --hard origin/prototype` -7. open the `requirements.txt` file and copy the line you added for your repo (*e.g.*, `-e git+https://github.com/MidAtlanticPortal/new_repo.git@master#egg=an_alias`) -8. enter `pip install` and paste (*e.g.*,`pip install -e git+https://github.com/MidAtlanticPortal/new_repo.git@master#egg=an_alias` ) and run -9. navigate to `cd ~/webapps/marco_portal/marco` -10. run `python manage.py collectstatic -i utfgrid` -11. run `python manage.py compress` -12. restart the server - `~/webapps/marco_portal/apache2/bin/restart` +#### Step 8.1 - Copy the media files into Docker + +If media files need to be copied to the server, you can use `scp`: + +```bash +scp -r /path/to/your_media_dir ubuntu@:/home/ubuntu/portals/madrona-portal/docker/media +``` + +If media files are somewhere on EC2: + +```bash +cd ~/portals/madrona-portal/docker +cp -r {your_media_dir}/* ./media/ +``` + +#### Create a directory for data_manager +```bash +mkdir ~/portals/madrona-portal/docker/data_manager +``` + +--- + +## Migrate existing GeoPortal records + +1. Update .env with the reindex remote whitelist and port. You can find this info by looking at the old server's configuration or by asking the previous admin. + +2. Edit the hosts file to allow the server to resolve the old Elastic IP of the GeoPortal instance to the new internal Docker network: + +```bash +sudo vim /etc/hosts +# Add the following line, replacing and : + +``` + +3. Restart the elastic container to apply the .env and hosts file change: + +```bash +docker compose down -v elastic +docker compose up -d elastic +``` + +4. Ensure the app is running, you can migrate existing GeoPortal records with the following command (replace the username and password): + +```bash +time curl -X POST "http://localhost:9200/_reindex" -H 'Content-Type: application/json' -d'{"conflicts": "proceed", "max_docs": 51000, "source": {"remote": { "host":"http://elastic.prod.wcoa.ecotrust.org:80/geoportal/elastic/", "username": "[[USERNAME]]", "password": "[[PASSWORD]]" }, "index": "metadata", "size": 100 }, "dest": { "index": "metadata" } }' +``` + +1. Do a down, including volumes, and up for the geoportal container to pick up the new records: + +```bash +docker compose down -v geoportal +docker compose up -d geoportal +``` + +--- + +# Deploy to fully containerized production environment + +## AWS EC2 + +See [AWS_DEPLOY.md](../docs/AWS_DEPLOY.md) + +--- + +## Dev infrastructure only (local Django server) + +To run Django locally against Docker-managed PostGIS and Redis (no app container): + +```bash +# Start only db and tasks +docker compose up -d db tasks + +# Then in a separate terminal, from madrona-portal/: +cd marco +python manage.py runserver +``` + +--- + +## Rebuilding after code changes + +From the docker directory (`madrona-portal/docker`): + +```bash +docker compose build --no-cache app +``` + +| Note on `--no-cache`: Use it when dependencies have changed; without it, Docker reuses the cached pip install layer (needed when `docker-requirements.txt` changes). + +Then bring the stack up: + +```bash +docker compose up --force-recreate +``` + +--- + +## Reset to a clean state + +```bash +# From madrona-portal/ +docker compose -f docker/docker-compose.yml --env-file .env --profile full down -v +``` + +`-v` removes the PostGIS and Redis volumes. The next `up` will re-run +migrations and reload fixtures from scratch. + +--- + +## Disk space + +Docker's build cache can grow large over time: + +```bash +docker system df # show usage breakdown +docker system prune -f # remove stopped containers, dangling images, unused networks, build cache +docker volume prune -f # remove unused volumes — only run when all containers are stopped +``` diff --git a/ROADMAP.md b/ROADMAP.md deleted file mode 100644 index 6dde9ab0..00000000 --- a/ROADMAP.md +++ /dev/null @@ -1,91 +0,0 @@ -Bootstrap base components: - - - Use this in the base template as a block: http://getbootstrap.com/components/#page-header - - Use these as basis for cards: http://getbootstrap.com/components/#thumbnails-custom-content - - how to park elements? http://stackoverflow.com/questions/21301316/how-to-bootstrap-navbar-static-to-fixed-on-scroll - -Done: - - - Bootstrap in stack - - bower install - - gulp, scss - - vendor build (not source) - - hard code top nav - -Major task groups - - - Styles - - Base typog - - Heading typog - - Color swatches - - Logo - - Topbar styles - - - CMS - - add groups - - Stubbed UI (stock bootstrap, hard coded content) - - Scaffolding (real data) - - skeleton content templates - - Production - - Basic deploy - - Search (elasticsearch) - - Redis caching - - cron tasks (or similar): - - search: update_index - - publish_scheduled_pages - - ocean story functionality/prototype (real data) - - pull top nav menu items from CMS (?) - - figure out how placing pages in the menus should work (with Jenny) - - filter/search/view switch for cards - - coupled Marine Planner instance - -Lay out incremental goals for next week and the rest of Nov - - - ol3 map on OS pages - - maybe render some data? - - - first pass: - - stock OL3 (http://docs.openlayers.org/library/introduction.html) - - openlayers 3 - - hosted, initially - - will need to setup build env, likely - - https://github.com/openlayers/ol3/blob/master/CONTRIBUTING.md - - http://boundlessgeo.com/2014/02/openlayers-3-custom-builds/ - - bower - - build - -How to have full width media elements inside the content area? -http://stackoverflow.com/questions/24049467/how-to-create-a-100-screen-width-div-inside-a-container-in-bootstrap - - -Top nav Menu - - 2nd level only? - - How to designate which menu a page goes in? (E) - -Login item? - - "My MARCO +ICON" - -Grid/list view - - what metadata fields in CMS to support filtering? (E) - -Event page: look at comp, make sure CMS has all the fields (E) - - esp Address - -groups need to be in beta in some form - -summary: for beta, implement Join a group/public groups page, but not "My groups" - - dependency ordering with portal update? - - public group profiles, centrally administered for beta - - "this group elsewhere" field (with help text suggesting fb, goog, mailing lists, etc) - - linked to a marine planner group? (URL or id or something?) - - request to join (manual addition) diff --git a/Vagrantfile b/Vagrantfile index f124c84c..cc2a2f00 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -58,12 +58,12 @@ Vagrant.configure("2") do |config| # Automatically detect the SMB host IP smb_host_ip = get_host_ip - config.vm.synced_folder "./", "/usr/local/apps/madrona_portal", + config.vm.synced_folder "./", "/usr/local/apps/madrona-portal", type: "smb", smb_host: smb_host_ip, mount_options: ["sec=ntlmssp", "nounix", "noperm", "vers=3.0"] - config.vm.synced_folder "../madrona-apps", "/usr/local/apps/madrona_portal/apps", + config.vm.synced_folder "../madrona-apps", "/usr/local/apps/madrona-portal/apps", type: "smb", smb_host: smb_host_ip, mount_options: ["sec=ntlmssp", "nounix", "noperm", "vers=3.0"] @@ -109,8 +109,8 @@ Vagrant.configure("2") do |config| # an identifier, the second is the path on the guest to mount the # folder, and the third is the path on the host to the actual folder. # config.vm.share_folder "project", "/home/vagrant/marco_portal2", "." - config.vm.synced_folder "./", "/usr/local/apps/madrona_portal" - config.vm.synced_folder "../madrona-apps/", "/usr/local/apps/madrona_portal/apps" + config.vm.synced_folder "./", "/usr/local/apps/madrona-portal" + config.vm.synced_folder "../madrona-apps/", "/usr/local/apps/madrona-portal/apps" # Enable provisioning with a shell script. # config.vm.provision :shell, :path => "scripts/vagrant_provision.sh", :args => "'marco_portal2' 'marco' 'marco_portal'", :privileged => false @@ -131,12 +131,12 @@ Vagrant.configure("2") do |config| # Default synced folder setup for Windows smb_host_ip = "192.168.1.1" # Fallback IP for Windows - config.vm.synced_folder "./", "/usr/local/apps/madrona_portal", + config.vm.synced_folder "./", "/usr/local/apps/madrona-portal", type: "smb", smb_host: smb_host_ip, mount_options: ["sec=ntlmssp", "nounix", "noperm", "vers=3.0"] - config.vm.synced_folder "../madrona-apps", "/usr/local/apps/madrona_portal/apps", + config.vm.synced_folder "../madrona-apps", "/usr/local/apps/madrona-portal/apps", type: "smb", smb_host: smb_host_ip, mount_options: ["sec=ntlmssp", "nounix", "noperm", "vers=3.0"] diff --git a/backups/create_elastic_snapshot.sh b/backups/create_elastic_snapshot.sh new file mode 100755 index 00000000..df5ec4b5 --- /dev/null +++ b/backups/create_elastic_snapshot.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +REPOSITORY="" + +while getopts "r:" opt; do + case $opt in + r) REPOSITORY="$OPTARG" ;; + *) echo "Usage: $0 -r "; exit 1 ;; + esac +done + +if [[ -z "$REPOSITORY" ]]; then + echo "Error: -r is required" + exit 1 +fi + +DATETIME_VAR=$(date +%Y%m%d_%H%M) +/usr/bin/curl -X PUT "localhost:9200/_snapshot/${REPOSITORY}/snapshot_${DATETIME_VAR}" -H 'Content-Type: application/json' -d '{"indices": "metadata_v1", "ignore_unavailable": true, "include_global_state": false}' diff --git a/backups/db_dump.sh b/backups/db_dump.sh index b2950297..9339cdd0 100755 --- a/backups/db_dump.sh +++ b/backups/db_dump.sh @@ -1,21 +1,95 @@ -#!/bin/bash - -DBNAME=dbname -DBOWNER=dbowner -DBPASSWORD=password -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -OUTFILE=$DBNAME'_dump.sql' -OUTDIR=$DIR - -while getopts n:o:p:d:f: flag -do - case "${flag}" in - n) DBNAME=${OPTARG};; # (n)ame of the database - o) DBOWNER=${OPTARG};; # Database (o)wner - p) DBPASSWORD=${OPTARG};; # Database owner's (p)assword - d) OUTDIR=${OPTARG};; # Output dump file (d)irectory - f) OUTFILE=${OPTARG};; # Output dump (f)ilename +#!/usr/bin/env bash + +set -euo pipefail + +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)" +ROOT_DIR="$(cd "$DIR/.." >/dev/null 2>&1 && pwd)" + +COMPOSE_FILE="$ROOT_DIR/docker/docker-compose.prod.yml" +ENV_FILE="$ROOT_DIR/docker/.env" +SERVICE_NAME="db" + +DBNAME="" +DBOWNER="" +DBPASSWORD="" +OUTDIR="$DIR" +OUTFILE="" + +usage() { + cat < Database name (default: DB_NAME from env file) + -o Database user (default: DB_USER from env file) + -p Database password (default: DB_PASSWORD from env file) + -d Output directory (default: backups/) + -f Output filename (default: _dump_.sql) + -c Docker compose file path + -e Environment file path + -s Docker service name (default: db) + -h Show this help +EOF +} + +while getopts ":n:o:p:d:f:c:e:s:h" flag; do + case "$flag" in + n) DBNAME="$OPTARG" ;; + o) DBOWNER="$OPTARG" ;; + p) DBPASSWORD="$OPTARG" ;; + d) OUTDIR="$OPTARG" ;; + f) OUTFILE="$OPTARG" ;; + c) COMPOSE_FILE="$OPTARG" ;; + e) ENV_FILE="$OPTARG" ;; + s) SERVICE_NAME="$OPTARG" ;; + h) + usage + exit 0 + ;; + :) echo "Error: Option -$OPTARG requires an argument." >&2; usage; exit 1 ;; + \?) echo "Error: Invalid option -$OPTARG" >&2; usage; exit 1 ;; esac done -PGPASSWORD=$DBPASSWORD /usr/bin/pg_dump -b -c -n public -O --quote-all-identifiers --no-acl -w -U $DBOWNER -f $OUTDIR/$OUTFILE $DBNAME +if [[ ! -f "$COMPOSE_FILE" ]]; then + echo "Error: Docker compose file not found at $COMPOSE_FILE" >&2 + exit 1 +fi + +if [[ -f "$ENV_FILE" ]]; then + set -a + # shellcheck disable=SC1090 + source "$ENV_FILE" + set +a +fi + +DBNAME="${DBNAME:-${DB_NAME:-}}" +DBOWNER="${DBOWNER:-${DB_USER:-}}" +DBPASSWORD="${DBPASSWORD:-${DB_PASSWORD:-}}" + +if [[ -z "$DBNAME" || -z "$DBOWNER" || -z "$DBPASSWORD" ]]; then + echo "Error: DB credentials are incomplete. Provide -n/-o/-p or set DB_NAME/DB_USER/DB_PASSWORD in env file." >&2 + exit 1 +fi + +mkdir -p "$OUTDIR" + +if [[ -z "$OUTFILE" ]]; then + OUTFILE="${DBNAME}_dump_$(date +%F_%H-%M-%S).sql" +fi + +OUTPATH="$OUTDIR/$OUTFILE" + +CONTAINER_ID="$(docker compose -f "$COMPOSE_FILE" --env-file "$ENV_FILE" ps -q "$SERVICE_NAME")" + +if [[ -z "$CONTAINER_ID" ]]; then + echo "Error: Service '$SERVICE_NAME' is not running." >&2 + exit 1 +fi + +docker compose -f "$COMPOSE_FILE" --env-file "$ENV_FILE" exec -T \ + -e PGPASSWORD="$DBPASSWORD" \ + "$SERVICE_NAME" \ + pg_dump -b -c -n public -O --quote-all-identifiers --no-acl -w -U "$DBOWNER" -d "$DBNAME" > "$OUTPATH" + +echo "Database dump created: $OUTPATH" diff --git a/backups/dump_fixtures.sh b/backups/dump_fixtures.sh new file mode 100644 index 00000000..8b727452 --- /dev/null +++ b/backups/dump_fixtures.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# Regenerate WCOA fixture files from the current running database. +# +# Usage (run from madrona-portal/): +# ./backups/dump_fixtures.sh +# +# Requires the full stack to be running: +# docker compose --env-file docker/.env.dev -f docker/docker-compose.yml up -d +# +# After running, review changes and commit the updated fixture files: +# git diff ../madrona-apps/wcoa/wcoa/fixtures/ +# git add ../madrona-apps/wcoa/wcoa/fixtures/ && git commit -m "Update fixtures from db" + +set -euo pipefail + +DC="docker compose --env-file docker/.env.dev -f docker/docker-compose.yml" +WCOA_FX="/usr/local/apps/madrona-portal/apps/wcoa/wcoa/fixtures" + +echo "Exporting wcoa_init.json (base, wagtailcore, wagtailimages, wcoa) ..." +$DC run --rm app \ + python marco/manage.py dumpdata --verbosity 0 \ + base wagtailcore wagtailimages wagtailredirects wcoa \ + --natural-foreign --indent 2 \ + > "$WCOA_FX/wcoa_init.json" +echo " -> $WCOA_FX/wcoa_init.json ($(wc -l < "$WCOA_FX/wcoa_init.json") lines)" + +echo "Exporting wcoa_init_layers.json (data_manager, sites) ..." +$DC run --rm app \ + python marco/manage.py dumpdata --verbosity 0 \ + data_manager sites \ + --natural-foreign --indent 2 \ + > "$WCOA_FX/wcoa_init_layers.json" +echo " -> $WCOA_FX/wcoa_init_layers.json ($(wc -l < "$WCOA_FX/wcoa_init_layers.json") lines)" + +echo "Exporting wagtail_menus.json (menu) ..." +$DC run --rm app \ + python marco/manage.py dumpdata --verbosity 0 \ + menu \ + --natural-foreign --indent 2 \ + > "$WCOA_FX/wagtail_menus.json" +echo " -> $WCOA_FX/wagtail_menus.json ($(wc -l < "$WCOA_FX/wagtail_menus.json") lines)" + +echo "" +echo "Fixtures updated. Review with:" +echo " git diff $WCOA_FX/" diff --git a/backups/load_sql_dump.sh b/backups/load_sql_dump.sh new file mode 100755 index 00000000..2aed09f2 --- /dev/null +++ b/backups/load_sql_dump.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# Import a PostgreSQL SQL dump into the running Docker database service. +# +# Usage (run from madrona-portal/): +# ./backups/load_sql_dump.sh /path/to/your_dump.sql +# +# The db container must already be running: +# docker compose --env-file docker/.env.dev -f docker/docker-compose.yml up -d db + +set -euo pipefail + +DUMP_FILE="${1:-}" + +if [ -z "$DUMP_FILE" ] || [ ! -f "$DUMP_FILE" ]; then + echo "Usage: $0 " + echo " Example: $0 ~/wcoa_prod.sql" + exit 1 +fi + +# Load credentials from the Docker env file (same directory as docker-compose.yml) +set -a +# shellcheck source=/dev/null +source "$(dirname "$0")/../docker/.env.dev" +set +a + +echo "Importing $(basename "$DUMP_FILE") into database '$SQL_DATABASE' ..." + +docker compose --env-file docker/.env.dev -f docker/docker-compose.yml exec -T db \ + psql -U "$SQL_USER" -d "$SQL_DATABASE" < "$DUMP_FILE" + +echo "" +echo "Import complete." +echo "" +echo "Next: restart the app to run Django migrations on top of the imported data:" +echo " docker compose --env-file docker/.env.dev -f docker/docker-compose.yml restart app" diff --git a/deployment/crontab.template b/deployment/crontab.template new file mode 100644 index 00000000..9528bbfb --- /dev/null +++ b/deployment/crontab.template @@ -0,0 +1,32 @@ +# Edit this file to introduce tasks to be run by cron. +# +# Each task to run has to be defined through a single line +# indicating with different fields when the task will be run +# and what command to run for the task +# +# To define the time you can provide concrete values for +# minute (m), hour (h), day of month (dom), month (mon), +# and day of week (dow) or use '*' in these fields (for 'any'). +# +# Notice that tasks will be started based on the cron's system +# daemon's notion of time and timezones. +# +# Output of the crontab jobs (including errors) is sent through +# email to the user the crontab file belongs to (unless redirected). +# +# For example, you can run a backup of all your user accounts +# at 5 a.m every week with: +# 0 5 * * 1 tar -zcf /var/backups/home.tgz /home/ +# +# For more information see the manual pages of crontab(5) and cron(8) +# +# NOTE: EBS Backup is scheduled for XX:XX every XXX +# +# m h dom mon dow command +# 02:15 daily (18:15 PST) - Dump PostgreSQL DB to dumpfile +15 2 * * * cd /home/ubuntu/portals/madrona-portal && /bin/bash -lc './backups/db_dump.sh -d ./backups/sql && find ./backups/sql -type f -name "*.sql" -mtime +10 -delete' >> /home/ubuntu/portals/madrona-portal/backups/db_dump.log 2>&1 +# 03:15 daily (19:15 PST) - Snapshot Elasticsearch to FS +15 3 * * * /usr/bin/bash /home/ubuntu/portals/madrona-portal/backups/create_elastic_snapshot.sh -r gp_es_snap +# 05:31 daily (21:31 PST) - Refresh NativeLand JSON layers +31 5 * * * cd /home/ubuntu/portals/madrona-portal/docker && docker compose exec app marco/manage.py import_nativeland + diff --git a/dev_requirements.txt b/dev_requirements.txt deleted file mode 100644 index 374ecc69..00000000 --- a/dev_requirements.txt +++ /dev/null @@ -1,60 +0,0 @@ -# Minimal requirements -Django<1.10 -djangorestframework==3.3.2 -django-picklefield==0.3.2 - - -# django-compressor scss hack --e git+https://github.com/MidAtlanticPortal/django-libsass.git@master#egg=django_libsass - -libsass==0.13.4 -#django-libsass==0.7 - -django-modelcluster==1.1 -wagtail==1.3.1 - -# Recommended components (require additional setup): -psycopg2==2.5.2 -elasticsearch==1.3.0 -Embedly==0.5.0 - -# Recommended components to improve performance in production: -django-redis-cache==2.0.0 -# django-celery==3.1.10 - -django-email-log==0.2.0 -celery==3.1.25 -django-celery==3.1.16 -django-celery-email==1.1.1 -kombu==3.0.37 - -requests==2.21.0 - -django-apptemplates==1.4 - -django-social-share==0.3.0 - -rpc4django==0.5.0 - -django-compressor==1.6 - -diff-match-patch==20121119 -tablib==0.11.2 -django-import-export==0.7.0 - -pyshp==1.2.3 - -urllib3==1.25.2 -social-auth-core==3.1.0 -social-auth-app-django==3.1.0 -django-tinymce==2.0.4 -django-nested-admin==3.0.21 -django-taggit==0.21.6 - -# JSONB issues on PG 9.3 -django-pgjsonb==0.0.32 -Pillow==4.3.0 -six==1.12.0 -django-appconf==1.0.2 -South==1.0 -Unidecode==0.04.21 diff --git a/docker/.env b/docker/.env deleted file mode 100644 index b45d7822..00000000 --- a/docker/.env +++ /dev/null @@ -1,100 +0,0 @@ -##################### -# REQUIRED SETTINGS # -##################### - -# These settings must absolutely be changed for any production deployment. -# Most of these are critical to the security of your database application. -# Many of these settings should be changed even for development environments. - -## SECRET_KEY: -## Used to secure the app. Can be anything -- feel free to hammer out -## a long line of numbers, letters, caps, and symbols. You will not need -## to remember or retype this ever. -SECRET_KEY=changeme - -## PROXY_PORT: -## The port the proxy server (NGINX) will serve the application on. Use 80 in -## production for HTTP or 443 for HTTPS. For dev you may prefer 80xx. -PROXY_PORT=8000 - -## ALLOWED_HOSTS: -## List of web addresses server will accept traffic from. This can be an -## IP address (127.0.0.1) or a URL (your.site.com). Separate addresses with a -## comma (no spaces). Leave the current default addresses in place unless -## you know what you're doing. -ALLOWED_HOSTS=localhost,127.0.0.1,[::1] - -## SQL_DATABASE: -## The name of your database. This can be any word (no spaces). You can leave -## the default in place, but you will get extra security by making it unique. -SQL_DATABASE=ocean_portal - -## SQL_USER: -## A username for the owner of your database. Can be any word (no spaces). -## It is recommended that you change this for security purposes -SQL_USER=madrona_user - -## SQL_PASSWORD: -## A password for your database user. Can be any word (no spaces). You -## absolutely MUST change this password before you put any sensitive -## information in your database -SQL_PASSWORD=madrona_password - -################### -# SERVER SETTINGS # -################### - -# These are settings that may impact how the application is served. These should -# only be important for development or highly-customized deployments. - -## DEBUG: -## Set to 1 for 'true' if you are actively modifying the code base -## Leave as 0 for 'false' (default) for running in production -DEBUG=1 - -## TIME_ZONE: -## To see all Timezone options, see -## https://en.wikipedia.org/wiki/List_of_tz_database_time_zones#List -TIME_ZONE=America/Los_Angeles - -##################### -# DATABASE SETTINGS # -##################### - -# These are the default settings for connecting the application to the -# database. They most likely should not be changed, and only then in -# experimental development of highly-customized deployments. The descriptions -# will assume a high-familiarity with the subject matter, particularly of -# Django, RDBMSs, networking, and Docker. - -## SQL_ENGINE: -## The django database backend to use to connect to the database. -SQL_ENGINE=django.contrib.gis.db.backends.postgis - -## SQL_HOST: -## The network address of the database. The default 'db' is the variable name -## assigned and recognized by Docker from your docker-compose.yml file. -SQL_HOST=db - -## SQL_PORT: -## The port your database is accepting connections on. Default for PostgreSQL -## is 5432. -SQL_PORT=65432 - -################# -# TASK SETTINGS # -################# - -## TASK_PORT: -## The port your task queue is accepting connections on. Default for Redis -## is 6379. -TASK_PORT=8379 -REDIS_PASSWORD=sOmE_sEcUrE_pAsS - -####################### -# DEPENDENCY SETTINGS # -####################### - -## PROJ_DIR: -## The location of the PROJ.4 executable -PROJ_DIR=/usr diff --git a/docker/.env.example b/docker/.env.example new file mode 100644 index 00000000..a6c9fa69 --- /dev/null +++ b/docker/.env.example @@ -0,0 +1,66 @@ +# ============================================================================= +# Madrona Portal core Docker environment template +# This file defines generic defaults for the project-agnostic core image. +# ============================================================================= + +DJANGO_ENV=development +GUNICORN_WORKERS=4 + +GHCR_PAT= +IMAGE_TAG=latest + +DEBUG=True +SECRET_KEY=change-me +ALLOWED_HOSTS=localhost,127.0.0.1,::1 +CSRF_TRUSTED_ORIGINS=http://localhost +MP_PROJECT_CONFIG=config.ini + +APP_NAME="Madrona Portal" +APP_TEAM_NAME="Marine Planner Team" +PROJECT_APP="" +PROJECT_SETTINGS_FILE=False +TIME_ZONE=UTC +COMPRESS_ENABLED=True +MAP_LIBRARY=ol8 +MEDIA_ROOT=/usr/local/apps/madrona-portal/media +MEDIA_URL=/media/ +STATIC_ROOT=/vol/web/static +STATIC_CORE=/vol/web/static/ +APP_PORT=8000 + +DB_ENGINE=django.contrib.gis.db.backends.postgis +DB_NAME=madrona_docker_db +DB_USER=postgres +DB_PASSWORD=change-me + +REDIS_PASSWORD=changeme +REDIS_PORT=6379 + +EMAIL_HOST=smtp.example.com +EMAIL_PORT=587 +EMAIL_HOST_USER=noreply@example.com +EMAIL_HOST_PASSWORD=change-me +EMAIL_USE_TLS=true +DEFAULT_FROM_EMAIL=noreply@example.com + +AWS_ACCESS_KEY_ID= +AWS_SECRET_ACCESS_KEY= +AWS_SES_REGION_NAME=us-east-1 +AWS_SES_REGION_ENDPOINT=email.us-east-1.amazonaws.com + +GA_ACCOUNT= +NATIVE_LAND_API_KEY= + +FACEBOOK_KEY= +FACEBOOK_SECRET= +TWITTER_KEY= +TWITTER_SECRET= +GOOGLE_KEY= +GOOGLE_SECRET= + +RECAPTCHA_PUBLIC_KEY= +RECAPTCHA_PRIVATE_KEY= + +DJANGO_SUPERUSER_USERNAME=admin +DJANGO_SUPERUSER_EMAIL=admin@example.com +DJANGO_SUPERUSER_PASSWORD= diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 00000000..da6e0ae5 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,93 @@ +# ============================================================================= +# Madrona Portal — Production Dockerfile +# Base: Ubuntu 24.04 LTS | Django 4.2+ | Wagtail 7.0+ | Python 3.10+ +# ============================================================================= +FROM ubuntu:24.04 + +# Prevent apt from blocking on interactive prompts during build +ENV DEBIAN_FRONTEND=noninteractive + +# Python & app environment +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PATH="/opt/venv/bin:$PATH" + +WORKDIR /usr/local/apps/madrona-portal + +# --------------------------------------------------------------------------- +# System dependencies +# --------------------------------------------------------------------------- +RUN apt-get update && apt-get upgrade -y && \ + apt-get install -y --no-install-recommends \ + python3 python3-pip python3-dev python3-venv \ + python-is-python3 \ + build-essential pkg-config \ + libpq-dev \ + gdal-bin libgdal-dev \ + libgeos-dev \ + libjpeg-dev zlib1g-dev libtool \ + libprotobuf-c-dev libjson-c-dev \ + perl libxml2-dev \ + libproj-dev proj-bin \ + libffi-dev openssl \ + gosu \ + && rm -rf /var/lib/apt/lists/* + +# --------------------------------------------------------------------------- +# Python virtual environment +# --------------------------------------------------------------------------- +RUN python3 -m venv /opt/venv + +# --------------------------------------------------------------------------- +# Copy application source +# --------------------------------------------------------------------------- +COPY madrona-portal/marco ./marco +COPY madrona-portal/apps/__init__.py ./apps/__init__.py +COPY madrona-portal/assets ./assets +COPY madrona-portal/bower_components ./bower_components +COPY madrona-portal/docker/entrypoint.sh /entrypoint.sh +COPY madrona-portal/docker/docker-requirements.txt /requirements.txt +COPY madrona-portal/backups ./backups + +COPY madrona-apps/django_url_shortener ./apps/django_url_shortener +COPY madrona-apps/madrona-analysistools ./apps/madrona-analysistools +COPY madrona-apps/madrona-features ./apps/madrona-features +COPY madrona-apps/madrona-manipulators ./apps/madrona-manipulators +COPY madrona-apps/madrona-scenarios ./apps/madrona-scenarios +COPY madrona-apps/mp-accounts ./apps/mp-accounts +COPY madrona-apps/mp-data-manager ./apps/mp-data-manager +COPY madrona-apps/mp-drawing ./apps/mp-drawing +COPY madrona-apps/mp-explore ./apps/mp-explore +COPY madrona-apps/mp-layers ./apps/mp-layers +COPY madrona-apps/mp-map-groups ./apps/mp-map-groups +COPY madrona-apps/mp-proxy ./apps/mp-proxy +COPY madrona-apps/mp-survey ./apps/mp-survey +COPY madrona-apps/mp-visualize ./apps/mp-visualize +COPY madrona-apps/p97-nursery ./apps/p97-nursery + +# --------------------------------------------------------------------------- +# Python dependencies +# Install mp-layers first (no-deps) so later resolution can satisfy any +# local dependency on its distribution before installing the rest. +# --------------------------------------------------------------------------- +RUN pip install --upgrade pip setuptools wheel && \ + pip install --no-deps -e ./apps/mp-layers && \ + pip install -r /requirements.txt + +# GDAL Python bindings — installed in a separate layer to preserve Docker +# cache when the rest of requirements.txt changes. +RUN pip install "GDAL==$(gdal-config --version)" --no-cache-dir + +# --------------------------------------------------------------------------- +# Runtime setup +# --------------------------------------------------------------------------- +RUN chmod 755 /entrypoint.sh && \ + mkdir -p /vol/web/static && \ + useradd --create-home --shell /bin/sh madrona_user && \ + chown -R madrona_user:madrona_user /vol /usr/local/apps/madrona-portal && \ + chmod -R 775 /vol/web + +EXPOSE 8000 +EXPOSE 8008 + +CMD ["/entrypoint.sh"] diff --git a/docker/backups/elasticsearch/blank.txt b/docker/backups/elasticsearch/blank.txt new file mode 100644 index 00000000..e69de29b diff --git a/docker/compose.base.yml b/docker/compose.base.yml new file mode 100644 index 00000000..762a7c66 --- /dev/null +++ b/docker/compose.base.yml @@ -0,0 +1,60 @@ +services: + app: + environment: + - DB_INIT=${DB_INIT:-0} + - SECRET_KEY=${SECRET_KEY} + - DEBUG=${DEBUG:-True} + - ALLOWED_HOSTS=${ALLOWED_HOSTS:-localhost,127.0.0.1,::1} + - DB_ENGINE=${DB_ENGINE:-django.contrib.gis.db.backends.postgis} + - DB_NAME=${DB_NAME:-madrona_portal_db} + - DB_USER=${DB_USER:-postgres} + - DB_PASSWORD=${DB_PASSWORD} + - DB_HOST=db + - DB_PORT=5432 + - REDIS_URL=redis://${REDIS_PASSWORD:+:${REDIS_PASSWORD}@}tasks:6379/1 + - CELERY_BROKER_URL=redis://${REDIS_PASSWORD:+:${REDIS_PASSWORD}@}tasks:6379/0 + - DJANGO_SUPERUSER_USERNAME=${DJANGO_SUPERUSER_USERNAME:-admin} + - DJANGO_SUPERUSER_EMAIL=${DJANGO_SUPERUSER_EMAIL:-admin@example.com} + - DJANGO_SUPERUSER_PASSWORD=${DJANGO_SUPERUSER_PASSWORD:-} + - DJANGO_ENV=${DJANGO_ENV:-development} + - GUNICORN_WORKERS=${GUNICORN_WORKERS:-3} + ports: + - "${APP_PORT:-8000}:8000" + depends_on: + db: + condition: service_healthy + tasks: + condition: service_healthy + restart: unless-stopped + + db: + image: postgis/postgis:16-3.4 + volumes: + - postgis_data:/var/lib/postgresql/data + environment: + - POSTGRES_DB=${DB_NAME:-madrona_portal_db} + - POSTGRES_USER=${DB_USER:-postgres} + - POSTGRES_PASSWORD=${DB_PASSWORD} + ports: + - "${DB_PORT:-5432}:5432" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${DB_USER:-postgres} -d ${DB_NAME}"] + interval: 10s + timeout: 5s + retries: 5 + restart: unless-stopped + + tasks: + image: redis:7-alpine + command: redis-server ${REDIS_PASSWORD:+--requirepass ${REDIS_PASSWORD}} + volumes: [ "redis_data:/data" ] + healthcheck: + test: ["CMD-SHELL", "redis-cli ${REDIS_PASSWORD:+-a ${REDIS_PASSWORD}} ping"] + interval: 10s + timeout: 5s + retries: 5 + restart: unless-stopped + +volumes: + postgis_data: + redis_data: diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml deleted file mode 100644 index 87e0d3a3..00000000 --- a/docker/docker-compose.yml +++ /dev/null @@ -1,79 +0,0 @@ -version: '3.7' - -services: - app: - build: - context: ../ - # command: dockerize -wait tcp://db:5432 sh -c "python manage.py migrate --noinput" - # command: dockerize -wait tcp://db:5432 sh -c "python manage.py loaddata /usr/local/apps/TEKDB/TEKDB/TEKDB/fixtures/all_dummy_data.json" - volumes: - - static_data:/vol/web - environment: - - SECRET_KEY=${SECRET_KEY} - - ALLOWED_HOSTS=${ALLOWED_HOSTS} - - SQL_ENGINE=${SQL_ENGINE} - - SQL_DATABASE=${SQL_DATABASE} - - SQL_USER=${SQL_USER} - - SQL_PASSWORD=${SQL_PASSWORD} - - SQL_HOST=${SQL_HOST} - - SQL_PORT=${SQL_PORT} - - PROJ_DIR=${PROJ_DIR} - depends_on: - - ${SQL_HOST} - links: - - ${SQL_HOST} - ports: - - "8000:8000" - networks: - - djangonetwork - - # proxy: - # build: - # context: ../../proxy - # volumes: - # - static_data:/vol/static - # # - media_data:/vol/media - # ports: - # - "${PROXY_PORT}:8080" - # depends_on: - # - app - # networks: - # - djangonetwork - - db: - image: postgis/postgis:14-3.1-alpine - volumes: - - postgis-data:/var/lib/postgresql - environment: - - POSTGRES_USER=${SQL_USER} - - POSTGRES_PASSWORD=${SQL_PASSWORD} - - POSTGRES_DB=${SQL_DATABASE} - ports: - - ${SQL_PORT}:5432 - networks: - - djangonetwork - - tasks: - image: redis:alpine3.14 - command: redis-server --requirepass ${REDIS_PASSWORD} - ports: - - ${TASK_PORT}:6379 - volumes: - - redis-data:/var/lib/redis - - redis.conf:/usr/local/etc/redis/redis.conf - - environment: - - REDIS_REPLICATION_MODE=master - networks: - - djangonetwork - -volumes: - postgis-data: - static_data: - redis-data: - redis.conf: - # media_data: - -networks: - djangonetwork: - driver: bridge diff --git a/docker/docker-requirements.txt b/docker/docker-requirements.txt index eae373c4..dd0a2dde 100644 --- a/docker/docker-requirements.txt +++ b/docker/docker-requirements.txt @@ -1,84 +1,95 @@ -# Minimal requirements -Django>=3.2,<3.3 -wagtail +# ============================================================================= +# Madrona Portal — Docker / Production Requirements +# This file is used by the Dockerfile. Pin only what is needed here; +# version constraints live in the top-level requirements.txt. +# ============================================================================= -# Tentative additions: -wagtail-import-export -python-social-auth -social-auth-app-django -python-jose -pyjwt -django-social-share -django-email-log -django-compressor -django-tinymce -django-wysiwyg -django-recaptcha -django-flatblocks -django-nested-admin -django-redis -rpc4django +# --------------------------------------------------------------------------- +# Core framework +# --------------------------------------------------------------------------- +Django>=4.2,<5.0 +wagtail>=7.0,<8.0 -# 11/13/2021 alpine default -pygdal<3.2.4 +# --------------------------------------------------------------------------- +# Authentication & social auth +# --------------------------------------------------------------------------- +social-auth-app-django>=5.4,<6.0 +social-auth-core>=4.5,<5.0 +python-jose>=3.3,<4.0 +pyjwt>=2.8,<3.0 +django-social-share>=2.3,<3.0 -################################## -#-e git+https://github.com/Ecotrust/mp-data-manager.git@main#egg=mp_data_manager -### OR #### -#-e git+https://github.com/Ecotrust/mp-data-manager.git@gp2#egg=data-manager --e /usr/local/apps/madrona-portal/apps/mp-data-manager/ -################################## +# --------------------------------------------------------------------------- +# Admin & CMS +# --------------------------------------------------------------------------- +django-autocomplete-light>=3.9,<4.0 +django-nested-admin>=4.0,<5.0 +django-tinymce>=3.6,<4.0 +django-recaptcha>=4.0,<5.0 +django-flatblocks>=1.0,<2.0 +django-querysetsequence>=0.14,<1.0 +wagtailcharts>=0.4,<1.0 +django-import-export>=3.3,<4.0 +django-colorfield>=0.11,<1.0 + +# --------------------------------------------------------------------------- +# Content & tagging +# --------------------------------------------------------------------------- +django-taggit>=5.0,<7.0 +django-email-log>=1.0,<2.0 +django-compressor>=4.4,<5.0 +django-libsass>=0.9,<1.0 +libsass>=0.23,<1.0 + +# --------------------------------------------------------------------------- +# Async tasks & caching +# --------------------------------------------------------------------------- +celery>=5.3,<6.0 +django-celery-email>=3.0,<4.0 +django-redis>=5.4,<6.0 + +# --------------------------------------------------------------------------- +# Application server (production) +# --------------------------------------------------------------------------- +gunicorn>=22.0,<24.0 + +# --------------------------------------------------------------------------- +# Database & geospatial +# --------------------------------------------------------------------------- +# pg2 2.9 breaks migrations: https://stackoverflow.com/a/68025007/706797 +psycopg2-binary>=2.9.9,<3.0 +# GDAL Python bindings are installed separately in the Dockerfile via: +# pip install "GDAL==$(gdal-config --version)" --no-cache-dir +pyshp>=2.3,<3.0 +owslib>=0.29,<1.0 + +# --------------------------------------------------------------------------- +# Search +# --------------------------------------------------------------------------- +elasticsearch>=7.0,<8.0 +elasticsearch-dsl>=7.0,<8.0 + +# --------------------------------------------------------------------------- +# APIs +# --------------------------------------------------------------------------- +# rpc4django removed — replaced by djangorestframework API views +djangorestframework>=3.14,<4.0 -#-e git+https://github.com/Ecotrust/madrona-analysistools.git@main#egg=analysistools +# --------------------------------------------------------------------------- +# Ecotrust / Madrona sub-apps (local editable installs via Dockerfile COPY) +# --------------------------------------------------------------------------- +-e /usr/local/apps/madrona-portal/apps/mp-layers +-e /usr/local/apps/madrona-portal/apps/mp-data-manager/ +-e /usr/local/apps/madrona-portal/apps/django_url_shortener/url_short -e /usr/local/apps/madrona-portal/apps/madrona-analysistools -#-e git+https://github.com/Ecotrust/mp-visualize.git@main#egg=visualize -e /usr/local/apps/madrona-portal/apps/mp-visualize -#-e git+https://github.com/Ecotrust/madrona-features.git@main#egg=features -e /usr/local/apps/madrona-portal/apps/madrona-features -#-e git+https://github.com/Ecotrust/mp-accounts.git@main#egg=accounts -e /usr/local/apps/madrona-portal/apps/mp-accounts -#-e git+https://github.com/Ecotrust/madrona-scenarios.git@main#egg=scenarios -e /usr/local/apps/madrona-portal/apps/madrona-scenarios -#-e git+https://github.com/Ecotrust/madrona-manipulators.git@main#egg=manipulators -e /usr/local/apps/madrona-portal/apps/madrona-manipulators -#-e git+https://github.com/Ecotrust/mp-drawing.git@main#egg=drawing -e /usr/local/apps/madrona-portal/apps/mp-drawing -#-e git+https://github.com/Ecotrust/mp-explore.git@main#egg=explore -e /usr/local/apps/madrona-portal/apps/mp-explore -#-e git+https://github.com/Ecotrust/mp-map-groups.git@main#egg=map_groups -e /usr/local/apps/madrona-portal/apps/mp-map-groups -#-e git+https://github.com/Ecotrust/p97-nursery.git@main#egg=nursery -e /usr/local/apps/madrona-portal/apps/p97-nursery -#-e git+https://github.com/Ecotrust/mp-proxy.git@main#egg=proxy -e /usr/local/apps/madrona-portal/apps/mp-proxy - -################################## -#-e git+https://github.com/Ecotrust/mida-portal.git@main#egg=mida-portal -### OR ### -#-e git+https://github.com/Ecotrust/wcoa.git@migration_2021#egg=wcoa --e /usr/local/apps/madrona-portal/apps/wcoa -### OR ### -#-e git+https://github.com/Ecotrust/wc-offshore-portal.git@master#egg=wc-offshore-portal -################################## - - -celery -# Django-Celery is incompatible with py3/Django2 -#django-celery -django-celery-email - - -# Note: django-libsass needs libsass>0.4 -#libsass -#django-libsass - -# Recommended components (require additional setup): -# pg2 2.9 breaks migrations: https://stackoverflow.com/a/68025007/706797 -psycopg2-binary<2.9 -elasticsearch - -django-import-export - -pyshp - -owslib +-e /usr/local/apps/madrona-portal/apps/mp-survey diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh old mode 100644 new mode 100755 index bd790d1f..58cc997d --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -1,9 +1,192 @@ #!/bin/sh +# Madrona Portal — Docker entrypoint +# Waits for the database, then starts the application server. +# +# By default steps 1 (DB wait), 2 (collectstatic + compress), and 6 (server start) run. Set DB_INIT=1 to also run steps 3-5 (migrate, fixtures, superuser). +# This is intentionally opt-in to protect existing databases. -#set -e -python marco/manage.py collectstatic --noinput +set -e + +# --------------------------------------------------------------------------- +# 1. Wait for the database to accept connections +# --------------------------------------------------------------------------- +python - <<'PY' +import os, socket, time, sys + +# Accept both DB_HOST (preferred) and legacy SQL_HOST +host = os.environ.get("DB_HOST") or os.environ.get("SQL_HOST", "db") +port = int(os.environ.get("DB_PORT") or os.environ.get("SQL_PORT", "5432")) +timeout = int(os.environ.get("DB_WAIT_TIMEOUT", "90")) + +print(f"Waiting for database at {host}:{port} (timeout {timeout}s)...", flush=True) +start = time.time() +while True: + try: + with socket.create_connection((host, port), timeout=2): + break + except OSError: + if time.time() - start > timeout: + sys.exit(f"Timed out waiting for database at {host}:{port}") + time.sleep(1) + +print("Database is up.", flush=True) +PY + +# --------------------------------------------------------------------------- +# 2. Collect static files and compress assets (always runs) +# --------------------------------------------------------------------------- +# Ensure the bind-mounted static dir is writable by madrona_user regardless of how Docker created it on the host (often root:root on Linux). +chown madrona_user:madrona_user /vol/web/static 2>/dev/null || true + +echo "Collecting static files..." +gosu madrona_user python marco/manage.py collectstatic --noinput +echo "Compressing assets..." +gosu madrona_user python marco/manage.py compress --force + +# --------------------------------------------------------------------------- +# 3-5. Database initialisation (opt-in via DB_INIT=1) +# --------------------------------------------------------------------------- +if [ "${DB_INIT:-0}" != "1" ]; then + echo "DB_INIT not set — skipping migrations, fixtures, and superuser creation." +else + +# --------------------------------------------------------------------------- +# 3. Migrate +# --------------------------------------------------------------------------- python marco/manage.py migrate --noinput -python marco/manage.py runserver 0:8000 -#uwsgi --socket :8000 --master --enable-threads --module marco.marco.wsgi -#exec "$@" +# --------------------------------------------------------------------------- +# 4. Seed a fresh database with initial fixture data +# +# A brand-new PostGIS install contains exactly one Wagtail Page row (the Wagtail root page, depth=1). We count pages at depth > 1 — if none exist, this is a fresh database and we load the initial fixture. +# +# IMPORTANT: Be careful not to wipe content on an existing database. Set FORCE_RELOAD_FIXTURES=1 only in CI or dev reset scenarios where wiping the database is intentional. +# --------------------------------------------------------------------------- +CONTENT_PAGES=$(python - 2>/dev/null <<'PY' || echo "unknown" +import sys, os +sys.path.insert(0, 'marco') +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'marco.settings') +import django +django.setup() +from wagtail.models import Page +print(Page.objects.filter(depth__gt=1).count()) +PY +) + +echo "Content pages in database: ${CONTENT_PAGES}" + +if [ "${CONTENT_PAGES:-0}" -lt "5" ] || [ "${FORCE_RELOAD_FIXTURES:-0}" = "1" ]; then + echo "Fresh database detected — loading initial fixtures..." + + # Clear rows created by migrations that would conflict with fixture data. + python - <<'PY' +import sys, os +sys.path.insert(0, 'marco') +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'marco.settings') +import django +django.setup() + +# The sites migration creates a default site and initial_data migration +# creates placeholder pages — both conflict with fixture data. +from django.contrib.sites.models import Site +Site.objects.all().delete() + +from wagtail.models import Page +Page.objects.filter(depth__gt=1).delete() + +try: + from portal.base.models import PortalRendition + PortalRendition.objects.all().delete() +except Exception: + pass +PY + + # Load fixtures in a single Python process so ContentTypes created here are guaranteed to be visible when loaddata deserializes FK natural keys. + python - <<'PY' +import sys, os +sys.path.insert(0, 'marco') +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'marco.settings') +import django +django.setup() + +# Step 1: ensure every installed app's ContentTypes exist before loading fixture data. create_contenttypes is idempotent. +from django.apps import apps as django_apps +from django.contrib.contenttypes.management import create_contenttypes +from django.contrib.contenttypes.models import ContentType + +for app_config in django_apps.get_app_configs(): + create_contenttypes(app_config, verbosity=0) + +print('ContentTypes synchronized for all installed apps.', flush=True) + +# Step 2: load fixtures — same process, same DB session. +from django.core.management import call_command +call_command( + 'loaddata', + 'apps/madrona-scenarios/scenarios/fixtures/initial_data.json', + verbosity=1, +) +PY + echo "Initial fixtures loaded." +else + echo "Existing database — skipping fixture load." +fi + +# --------------------------------------------------------------------------- +# 5. Create superuser (only when DJANGO_SUPERUSER_PASSWORD is set and the +# username does not already exist — safe to run on every restart) +# --------------------------------------------------------------------------- +if [ -n "${DJANGO_SUPERUSER_PASSWORD:-}" ]; then + python - <` | + +### 0.2 Create a read-only PAT for the EC2 server (pull only) + +**GitHub → Settings → Developer settings → Personal access tokens → Fine-grained tokens → Generate new token** + +| Setting | Value | +|---|---| +| Token name | `madrona-portal-ec2` | +| Expiration | 1 year | +| Resource owner | `Ecotrust` | +| Repository access | Public repositories only | +| Permissions → Packages | Read-only | + +Copy this token — you will add it to the server `.env` in Phase 4. + +### 0.3 Trigger the first image build + +Push a commit to the `docker` branch of `madrona-portal`, or run the workflow +manually: + +**GitHub → `Ecotrust/madrona-portal` → Actions → "Build and Push to GHCR" → Run workflow** + +The first build takes 15–25 minutes (compiling GDAL, installing all Python +packages). Subsequent builds are faster thanks to GitHub Actions layer caching. + +Confirm the image appears at: +**GitHub → `Ecotrust` org → Packages → `madrona-portal`** + +Note the short SHA from the workflow summary — you can use it to pin a specific +build on EC2 instead of always pulling `:latest`. + +--- + +## Phase 1 — AWS Infrastructure + +### 1.1 Create a key pair + +**AWS Console → EC2 → Key Pairs → Create key pair** + +| Setting | Value | +|---|---| +| Name | `madrona-portal` | +| Key pair type | RSA | +| Private key format | `.pem` (Linux/Mac) or `.ppk` (PuTTY/Windows) | + +Download the `.pem` file and move it somewhere safe: + +```bash +mv ~/Downloads/madrona-portal.pem ~/.ssh/ +chmod 400 ~/.ssh/madrona-portal.pem +``` + +### 1.2 Create a security group + +**AWS Console → EC2 → Security Groups → Create security group** + +| Setting | Value | +|---|---| +| Name | `madrona-portal-sg` | +| Description | Madrona Portal web server | +| VPC | Default VPC (or your own) | + +**Inbound rules:** + +| Type | Port | Source | Purpose | +|---|---|---|---| +| SSH | 22 | Your IP only (`x.x.x.x/32`) | Server access | +| HTTP | 80 | `0.0.0.0/0` | Web traffic (Nginx) | +| HTTPS | 443 | `0.0.0.0/0` | Web traffic (Nginx + SSL) | + +> Do **not** open port 8000 to the public. Nginx proxies traffic to Gunicorn +> on port 8000 internally. + +**Outbound rules:** leave the default (all traffic allowed). + +### 1.3 Launch an EC2 instance + +**AWS Console → EC2 → Instances → Launch instances** + +| Setting | Value | +|---|---| +| Name | `madrona-portal` | +| AMI | Ubuntu Server 24.04 LTS (64-bit x86) | +| Instance type | `t3.large` (8 GB RAM) — minimum. See note below. | +| Key pair | `madrona-portal` (created above) | +| Security group | `madrona-portal-sg` (created above) | +| Storage | 60 GB gp3 — expand the default 8 GB root volume | + +> **Why t3.large?** Elasticsearch alone reserves 1 GB of heap +> (`-Xms512m -Xmx512m`) plus JVM overhead. Add PostGIS, Gunicorn workers, +> Redis, and Tomcat (Geoportal) and you need at least 6–7 GB free. A +> `t3.large` (8 GB) is the practical minimum; `t3.xlarge` (16 GB) gives +> comfortable headroom. + +> **Why 60 GB?** The Docker image is ~3–4 GB after pull. PostGIS data, +> Elasticsearch indices, and Docker's image cache add up quickly. + +Launch the instance and wait for it to reach **Running** state. + +### 1.4 Allocate an Elastic IP + +Without an Elastic IP, AWS reassigns your public IP every time the instance +stops. An Elastic IP is free while attached to a running instance. + +**AWS Console → EC2 → Elastic IPs → Allocate Elastic IP address** + +- Click **Allocate** +- Select the new IP → **Actions → Associate Elastic IP address** +- Choose your `madrona-portal` instance → **Associate** + +Note the Elastic IP — you will use it in your `.env` and DNS records. + +--- + +## Phase 2 — Server Setup + +### 2.1 SSH into the instance + +```bash +ssh -i ~/.ssh/madrona-portal.pem ubuntu@ +``` + +### 2.2 Update the system + +```bash +sudo apt update && sudo apt upgrade -y +``` + +### 2.3 Install Docker Engine + +AWS's Ubuntu AMI does not include Docker. Install the official Docker Engine +(not the snap package — it has permission issues with volumes). + +```bash +# Add Docker's official GPG key +sudo apt install ca-certificates curl +sudo install -m 0755 -d /etc/apt/keyrings +sudo curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc +sudo chmod a+r /etc/apt/keyrings/docker.asc + +# Add the repository to Apt sources: +sudo tee /etc/apt/sources.list.d/docker.sources <" | docker login ghcr.io -u --password-stdin +``` + +The login token is saved to `~/.docker/config.json` and persists across +reboots. You only need to re-run this if the PAT expires. + +--- + +## Phase 3 — Locate/Create and upload Geoportal WAR Files + +The Geoportal service requires two Java WAR files that are not in any Git +repository. Since these files include passwords (even if encrypted) +Ecotrust keeps custom, private builds in our shared drive. If you do +not have access to those, you can build your own using Maven (beyond the +scope of this document) on the repos cloned from: +* https://github.com/Ecotrust/geoportal-server-catalog (to create `geoportal.war`) +* https://github.com/Ecotrust/geoportal-server-harvester (to create `harvester.war`) + +**On your local machine**, SCP them to the new EC2 instance: + +```bash +# Replace OLD_SERVER_IP and paths as appropriate +scp ubuntu@:/path/to/geoportal.war \ + ubuntu@:/tmp/geoportal.war + +scp ubuntu@:/path/to/harvester.war \ + ubuntu@:/tmp/harvester.war +``` + +--- + +## Phase 4 — Clone the Portal Configuration + +The EC2 server only needs the `madrona-portal` repository — for the Compose +file, Nginx templates, entrypoint scripts, and environment configuration. +No sub-app repos are needed; all application code is baked into the image. + +### 4.1 Set up SSH access to GitHub (on the server) + +```bash +ssh-keygen -t ed25519 -C "madrona-portal-server" -f ~/.ssh/github -N "" +cat ~/.ssh/github.pub +``` + +Copy the output and add it as a **Deploy Key** in `madrona-portal`: + +**GitHub → `Ecotrust/madrona-portal` → Settings → Deploy keys → Add deploy key** +- Title: `madrona-portal EC2` +- Paste the public key +- Allow write access: No + +Configure SSH to use this key: + +```bash +cat >> ~/.ssh/config << 'EOF' +Host github.com + IdentityFile ~/.ssh/github + StrictHostKeyChecking no +EOF +``` + +### 4.2 Clone madrona-portal + +```bash +mkdir ~/portals && cd ~/portals +git clone -b docker git@github.com:Ecotrust/madrona-portal.git madrona-portal +``` + +Verify: + +```bash +ls ~/portals/ +# madrona-portal/ +``` + +### 4.3 Move the WAR files into place + +**On the new EC2 instance**, move them into the expected location: + +```bash +mv /tmp/geoportal.war ~/portals/madrona-portal/docker/wars/ +mv /tmp/harvester.war ~/portals/madrona-portal/docker/wars/ +``` + +The `.env.example` defaults point to `./wars/geoportal.war` and +`./wars/harvester.war` (relative to the `docker/` directory), so these paths +will work without any further changes. + +--- + +## Phase 5 — Configure the Environment + +### 5.1 Create the `.env` file + +```bash +cd ~/portals/madrona-portal +cp docker/.env.example docker/.env +``` + +### 5.2 Generate a secret key + +```bash +python3 -c "import secrets; print(secrets.token_urlsafe(50))" +``` + +### 5.3 Edit `.env` + +```bash +nano docker/.env +``` + +Set these values at minimum: + +```ini +# Environment +DJANGO_ENV=production + +# Gunicorn workers (2× vCPU count; t3.large has 2 vCPUs → 4 workers) +GUNICORN_WORKERS=4 + +# GHCR — the read-only PAT from Phase 0.2 (documents what token was used to +# log in; docker login stores credentials in ~/.docker/config.json) +GHCR_PAT= + +# Django +SECRET_KEY= +ALLOWED_HOSTS=,localhost +DEBUG=False + +# Database +DB_PASSWORD= + +# Redis +REDIS_PASSWORD= + +# Superuser (created automatically on first boot if DB_INIT=1) +DJANGO_SUPERUSER_USERNAME=admin +DJANGO_SUPERUSER_EMAIL=your@email.com +DJANGO_SUPERUSER_PASSWORD= +``` + +Can be added now or later: + +```ini +# Password for the 'elastic' user +ELASTIC_PASSWORD= + +# Password for the 'kibana_system' user +KIBANA_PASSWORD= + +# Admin User (Full Access) +gpt_admin_username= +gpt_admin_password= + +# Publisher User (Can publish metadata) +gpt_publisher_username= +gpt_publisher_password= + +# Regular User (Read-only access) +gpt_user_username= +gpt_user_password= + +gpt_wcoa_username= +gpt_wcoa_password= + +gpt_esri_username= +gpt_esri_password= + +gpt_frame_options=DENY +gpt_allowed_origin="localhost localhost:* " +``` + +Leave everything else at its default for now. You can add email, OAuth, and +Elasticsearch credentials later. + +### 5.4 Create the ini config file + +```bash +cd ~/portals/madrona-portal/marco +cp config.docker.ini.template config.wcoa.docker.ini +``` + +--- + +## Phase 6 — Pull and Start the Stack + +### 6.1 Pull the image from GHCR + +```bash +docker pull ghcr.io/ecotrust/madrona-portal:latest +``` + +> To use a specific build instead of `latest`, note the short SHA from the +> GitHub Actions workflow summary and pull by tag: +> `docker pull ghcr.io/ecotrust/madrona-portal:abc1234` + +### 6.2 Start the stack + +:warning: For a fresh database, run with `DB_INIT=1` the first time to load initial fixtures and create the superuser:** +```bash +cd ~/portals/madrona-portal/docker + +DB_INIT=1 docker compose -f docker-compose.prod.yml up -d +``` +*On subsequent starts, leave `DB_INIT` at its default (`0`) to skip the fixture and superuser steps.* + +For an existing database: + +```bash +cd ~/portals/madrona-portal/docker + +docker compose -f docker-compose.prod.yml up -d +``` + +### 6.3 Watch the startup logs + +```bash +docker compose -f docker-compose.prod.yml logs -f app +``` + +On first boot the entrypoint automatically: +1. Waits for PostgreSQL to be healthy +2. Runs `collectstatic` and `compress` (always) +3. Runs `migrate` (only when `DB_INIT=1`) +4. Detects fresh database → loads initial fixtures (only when `DB_INIT=1`) +5. Creates the superuser from `.env` (only when `DB_INIT=1`) +6. Starts Gunicorn + +Startup takes 2–5 minutes. Look for `Booting worker` lines from Gunicorn. + +> For a fresh database, run with `DB_INIT=1` the first time: +> ```bash +> DB_INIT=1 docker compose -f docker/docker-compose.prod.yml up -d +> ``` +> On subsequent starts, leave `DB_INIT` at its default (`0`) to skip the +> fixture and superuser steps. + +### Apply migrations (if needed) + +```bash +docker compose -f docker/docker-compose.prod.yml exec app python marco/manage.py migrate +``` + +### Import database + +Copy your SQL dump to the server (e.g., using `scp`): + +```bash +scp /path/to/your_dump.sql ubuntu@:/home/ubuntu/your_dump.sql +``` + +From `madrona-portal/docker`: +```bash +../scripts/db-restore.sh --drop +``` +*example:* +```bash +../scripts/db-restore.sh --drop ../../madrona-apps/wcoa/wcodp_prod_dump_20260320.sql +``` + +### Apply migrations again (if needed) + +```bash +docker compose -f docker/docker-compose.prod.yml exec app python marco/manage.py migrate +``` + +### Migration to Layers + +```bash +docker compose -f docker/docker-compose.prod.yml exec app python marco/manage.py migration_to_layers +``` + + + +### Collect static and compress assets + +Static files are collected automatically on every container startup. To force +a manual re-run without restarting the container: + +```bash +docker compose -f docker/docker-compose.prod.yml exec app python marco/manage.py collectstatic --noinput +docker compose -f docker/docker-compose.prod.yml exec app python marco/manage.py compress +``` + +### Smoke test + +```bash +curl -I http://localhost:8000/ +# Expected: HTTP/1.1 200 OK (or 301/302 redirect) +``` + +If you get a connection refused, the app is still starting. Wait 30 seconds +and try again. + +--- + +## Phase 7 — Nginx + SSL (Production Hardening) + +Gunicorn should not be exposed directly to the internet. Nginx handles SSL termination, compression, and static file serving. + +### 7.1 Install Nginx and Certbot + +```bash +sudo apt install -y nginx certbot python3-certbot-nginx +``` + +### 7.2 Create a DNS A record + +In your DNS provider (Route 53, Cloudflare, etc.): + +| Type | Name | Value | +|---|---|---| +| A | `portal.westcoastoceans.org` | `` | + +Wait for DNS to propagate before continuing: + +```bash +dig portal.westcoastoceans.org +# Should return your Elastic IP +``` + +### 7.3 Configure Nginx + +```bash +sudo nano /etc/nginx/sites-available/madrona-portal +``` + +Paste: + +```nginx +server { + listen 80; + server_name or ; + + access_log /var/log/nginx/wcoa.access.log; + error_log /var/log/nginx/wcoa.error.log; + + location /static/ { + alias /home/ubuntu/portals/madrona-portal/docker/static/; + expires 30d; + add_header Cache-Control "public, no-transform"; + } + + location /media/ { + alias /home/ubuntu/portals/madrona-portal/docker/media/; + } + + location / { + proxy_pass http://127.0.0.1:8000; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_read_timeout 120s; + client_max_body_size 50M; + } + + location /geospatial/ { + alias /var/www/html/geospatial/; + autoindex on; + } + + location /munin/static/ { + alias /etc/munin/static/; + } + + location /munin { + alias /var/cache/munin/www; + } + + location /nativeland { + proxy_pass https://native-land.ca/; + resolver 8.8.8.8; + resolver_timeout 10s; + proxy_redirect off; + proxy_pass_request_headers on; + proxy_ssl_server_name on; + } + + # Shared CORS policy for these proxied endpoints + # (if you only want specific origins, replace "*" with your domain) + set $cors_allow_origin "*"; + + location ~ ^/(?:_search/|_doc/|metadata).*$ { + proxy_pass http://127.0.0.1:9200; + proxy_redirect off; + proxy_connect_timeout 5s; + proxy_read_timeout 60s; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + add_header Access-Control-Allow-Origin $cors_allow_origin always; + add_header Access-Control-Allow-Methods "GET, POST, OPTIONS" always; + add_header Access-Control-Allow-Headers "DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range" always; + add_header Access-Control-Expose-Headers "Content-Length,Content-Range" always; + + if ($request_method = OPTIONS) { + add_header Access-Control-Max-Age 1728000 always; + add_header Content-Type "text/plain; charset=utf-8" always; + add_header Content-Length 0 always; + return 204; + } + } + + location ~ ^/(?:manager|host-manager|semantix|solr|gc|geoportal|harvester).*$ { + proxy_pass http://:8080; + proxy_redirect off; + proxy_connect_timeout 5s; + proxy_read_timeout 60s; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + add_header Access-Control-Allow-Origin $cors_allow_origin always; + add_header Access-Control-Allow-Methods "GET, POST, OPTIONS" always; + add_header Access-Control-Allow-Headers "DNT,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Range" always; + add_header Access-Control-Expose-Headers "Content-Length,Content-Range" always; + + if ($request_method = OPTIONS) { + add_header Access-Control-Max-Age 1728000 always; + add_header Content-Type "text/plain; charset=utf-8" always; + add_header Content-Length 0 always; + return 204; + } + } +} +``` + +Enable the site: + +```bash +sudo ln -s /etc/nginx/sites-available/madrona-portal \ + /etc/nginx/sites-enabled/madrona-portal +sudo nginx -t # verify config +sudo systemctl restart nginx +``` + +> **Static file permissions:** Nginx runs as `www-data`, which must be able to +> traverse every directory in the path to `docker/static/`. Ubuntu home +> directories default to `750` (group-only execute), which blocks `www-data`. +> Fix it once after cloning: +> +> ```bash +> chmod o+x /home/ubuntu +> ``` +> +> Verify with `namei -l /home/ubuntu/portals/madrona-portal/docker/static/` — +> every component in the path needs at least `o+x`. + +### 7.4 Obtain an SSL certificate + +```bash +sudo certbot --nginx -d +``` + +Certbot edits your Nginx config automatically to add SSL and redirect HTTP +to HTTPS. It installs a cron job to renew the certificate automatically. + +### 7.5 Update `ALLOWED_HOSTS` + +Add the domain to `docker/.env`: + +```ini +ALLOWED_HOSTS=portal.westcoastoceans.org,,localhost +``` + +Restart the app container to pick up the change: + +```bash +cd ~/portals/madrona-portal +docker compose -f docker/docker-compose.prod.yml up -d --force-recreate app +``` + +--- + +## Import media + +#### Copy the media files +```bash +scp -r {your_media_dir} ubuntu@:/home/ubuntu/portals/madrona-portal/docker/media/ +``` + +#### Create a directory for data_manager +```bash +mkdir ~/portals/madrona-portal/docker/data_manager +``` + +## Migrate existing GeoPortal records + +1. Update .env with the reindex remote whitelist and port. You can find this info by looking at the old server's configuration or by asking the previous admin. + +2. Edit the hosts file to allow the server to resolve the old Elastic IP of the GeoPortal instance to the new internal Docker network: + +```bash +sudo vim /etc/hosts +# Add the following line, replacing and : + +``` + +3. Ensure the app is running, you can migrate existing GeoPortal records with the following command (replace the username and password): + +```bash +time curl -X POST "http://localhost:9200/_reindex" -H 'Content-Type: application/json' -d'{"conflicts": "proceed", "max_docs": 51000, "source": {"remote": { "host":"http://[ES_REINDEX_REMOTE_WHITELIST]:80/geoportal/elastic/", "username": "[[USERNAME]]", "password": "[[PASSWORD]]" }, "index": "metadata", "size": 100 }, "dest": { "index": "metadata" } }' +``` + +4. Do a down, including volumes, and up for the elasticsearch container and geoportal to pick up the new records: + +```bash +docker compose -f docker/docker-compose.prod.yml down elastic geoportal -v +docker compose -f docker/docker-compose.prod.yml up -d elastic geoportal +``` + +--- + +## Phase 8 — Keep the Stack Running Across Reboots + +### 8.1 Create a systemd service + +```bash +sudo nano /etc/systemd/system/madrona-portal.service +``` + +Paste: + +```ini +[Unit] +Description=Madrona Portal Docker Compose Stack +Requires=docker.service +After=docker.service network-online.target + +[Service] +Type=oneshot +RemainAfterExit=yes +WorkingDirectory=/home/ubuntu/portals/madrona-portal +ExecStart=/usr/bin/docker compose \ + -f docker/docker-compose.prod.yml \ + --env-file docker/.env \ + up -d +ExecStop=/usr/bin/docker compose \ + -f docker/docker-compose.prod.yml \ + --env-file docker/.env \ + down +TimeoutStartSec=300 + +[Install] +WantedBy=multi-user.target +``` + +### 8.2 Enable the service + +```bash +sudo systemctl daemon-reload +sudo systemctl enable madrona-portal +``` + +### 8.3 Test it (simulates a reboot) + +```bash +sudo systemctl stop madrona-portal +sudo systemctl start madrona-portal +docker compose -f docker/docker-compose.prod.yml ps # all services should show "running" +``` + +--- + +## AWS Email (SES) Setup + +### Verify your domain in AWS SES (Simple Email Service) + +1. Go to SES Console → us-west-2 → Create identity → choose Domain → enter: +`portal.westcoastoceans.org` +2. Leave defaults +3. Click "Create identity" + +### Add the provided DNS records to your DNS provider +1. In the SES Console, click on your new identity → DNS records tab +2. Add the provided records to your DNS provider (e.g., Hover) +3. Wait for AWS to verify the domain (minute to hours) + +### Create SMTP Credentials +1. In SES Console → SMTP Settings → Create SMTP credentials → note the username and password (only shown once). + +### Update the portal configuration +1. SSH into the server +2. Open the `.env` file +3. Add the following values (replace with your SMTP credentials): +``` +EMAIL_HOST=email-smtp.us-west-2.amazonaws.com +EMAIL_PORT=587 +EMAIL_HOST_USER= +EMAIL_HOST_PASSWORD= +EMAIL_USE_TLS=true +DEFAULT_FROM_EMAIL=noreply@prod.mail.ecotrust.org +``` + +### Create custom mail from domain +1. In SES Console → Identities → click on your domain → Create mail from domain +2. Enter a subdomain (e.g., `mail`) → Create +3. Add the provided DNS records to your DNS provider +4. Wait for AWS to verify the mail from domain + +### Request SES production access +Submit a production access request in SES Console → Account dashboard → Request production access. Takes 24hrs typically. + +--- + +## Automatic Security Updates + +Install unattended-upgrades and update-notifier-common to automatically apply security updates to the server OS: +```bash +sudo apt install unattended-upgrades update-notifier-common -y +``` + +Enable automatic updates: +```bash +sudo dpkg-reconfigure --priority=low unattended-upgrades +``` + +Edit the configuration to allow automatic reboots and set the time for reboots to occur: +```bash +// Open the config file +sudo vim /etc/apt/apt.conf.d/50unattended-upgrades + +// Find, uncomment, and set "true" the line that contains "Unattended-Upgrade::Automatic-Reboot" +Unattended-Upgrade::Automatic-Reboot "true"; + +// Find and uncomment the line that contains "Unattended-Upgrade::Automatic-Reboot-Time" +Unattended-Upgrade::Automatic-Reboot-Time "02:00"; +``` + +--- + +## Install Munin + +```bash +sudo apt install munin -y +``` + +--- + +## Set Up Swap Space + +```bash +sudo fallocate -l 1G /swapfile +sudo chmod 600 /swapfile +sudo mkswap /swapfile +sudo swapon /swapfile +sudo cp /etc/fstab /etc/fstab.bak +echo '/swapfile none swap sw 0 0' | sudo tee -a /etc/fstab +``` + +--- + +## Add Google Analytics Key +1. Get the GA tracking ID (e.g., `G-XXXXXXXXXX`) +2. SSH into the server and edit the `.env` file +3. Edit or add the following line in the `.env` file: +```bash +GA_ACCOUNT=G-XXXXXXXXXX +``` + +--- + +## Create local metadata Web Accessible Folder +You may have noticed we created a `/geospatial` location in the Nginx config. +This is used to serve metadata records used by input brokers in the GeoPortal +Harvester as well as referenced by layers stored in mp-layers' Layers records. + +Referring to the Nginx config, you'll see it refers to /var/www/html/geospatial. +All you need to do is create this folder, ensure www-data has read privileges, +and the endpoint will work (it will be empty). If migrating from an existing +server, you can copy those folders and existing metadata records over using scp +or wget (they should be public-readable). + +--- + +## Backups and Snapshots + +AWS Lifecycle Management allows you to easily create snapshots of your EBS volumes. +This is great for data stored as files (media, staticfiles, WAFs, etc...) but is +not sufficient for capturing data in databases as the snapshot is not atomic, and +we have not scheduled our databases to stop for these snapshots. Instead, it's +important to use each database's built-in tools to create file-system-level +backups that will be accurately represented in any EBS volume snapshot. + +Those dumps are best managed by Cron jobs (covered below) but in some cases, some +prep work is required. + +### Elasticsearch Snapshot Repository + +Elasticsearch uses a built-in API-based tool called 'Snapshot' to create file-system +level backups that are appropriate for restoring your database to prior states or +populating from scratch. Before you can create a snapshot, you must first create a +Snapshot Repository. + +Below: +* REPO_NAME - the name you wish to use for your repository. + * This will need to be recorded and set again in your cron job that creates the snapshots +* /usr/share/elasticsearch/backups + * This is the assumed location of the backups folder from the perspective of the elasticsearch container + * This value should be correct if you did not edit docker-compose.prod.yml + * Docker maps this folder to your local `~/portals/madrona-portal/docker/backups/elasticsearch` + * This way the files representing your snapshots are exposed during the EBS backup + +``` +curl -X PUT "localhost:9200/_snapshot/{REPO_NAME}" -H "Content-Type: application/json" -d '{"type": "fs", "settings": {"location": "/usr/share/elasticsearch/backups", "compress": true}}' + +``` + +You can confirm the creation/existence of this repository +``` +curl -X GET "localhost:9200/_cat/repositories?v" +``` + +### EBS Snapshots (lifecycle) + +Be sure to set up regularly recurring snapshots of your EBS volume(s) so that +data may be recovered in a disaster. This is easily managed in the EC2 dashboard. + +--- + +## Cron Jobs +There are several tasks that should run regularly to ensure your server is +serving up-to-date data and keeping file-system-level backup dumps to +ensure AWS Lifecycle snapshots can capture accurate representations of the +database contents/state. + +Please see [crontab.template](../deployment/crontab.template) for an up-to-date +list of recommended jobs with examples of how to implement them. + +### PostgreSQL/Django dump +``` +cd /home/ubuntu/portals/madrona-portal && /bin/bash -lc './backups/db_dump.sh -d ./backups/sql && find ./backups/sql -type f -name "*.sql" -mtime +10 -delete' >> /home/ubuntu/portals/madrona-portal/backups/db_dump.log 2>&1 +``` +Uses the built-in `db_dump.sh` script to dump the database to a local SQL file + +### Elasticsearch/GeoPortal snapshot +``` +/usr/bin/bash /home/ubuntu/portals/madrona-portal/backups/create_elastic_snapshot.sh -r {REPO_NAME} +``` +* REPO_NAME + * You should have set this in the section on Elasticsearch Snapshot Repositories above + +You can review the name of your snapshot(s) with: +``` +curl -X GET "localhost:9200/_cat/snapshots?v" +``` + +You can review the status of your snapshot(s) with: +``` +curl -X GET "localhost:9200/_snapshot/{REPO_NAME}/{SNAPSHOT_NAME}" +``` +* SNAPSHOT_NAME + * You should get this from the previous `_cat/snapshots` query + + +### NativeLands Digital JSON files +``` +cd /home/ubuntu/portals/madrona-portal/docker && docker compose exec app marco/manage.py import_nativeland +``` + +Assuming you have an API key for https://native-land.ca/ and configured your .env correctly, this management +command should pull 3 layers in locally as GeoJSONs to be served statically for vector layers in the Portal: +* Languages +* Historic Territories +* Treaties + + +--- + +## Deploying a New Release + +When code changes are merged to the `docker` branch, GitHub Actions +automatically builds and pushes a new image to GHCR. To deploy it: + + +### On the server + +```bash +cd ~/portals/madrona-portal + +# Pull the latest image (or a specific SHA tag for a pinned deploy) +docker pull ghcr.io/ecotrust/madrona-portal:latest + +# Recreate only the app container — db, Redis, and Elasticsearch are untouched +docker compose -f docker/docker-compose.prod.yml up -d --force-recreate app +``` + +Downtime is limited to the container restart (~5–10 seconds). + +### Rolling back to a previous build + +```bash +# List available tags in GHCR (or check the GitHub Actions workflow summaries +# for the SHA of any previous build) + +# Pull the specific SHA tag +docker pull ghcr.io/ecotrust/madrona-portal: + +# Update IMAGE_TAG in docker/.env, then recreate: +# IMAGE_TAG= in docker/.env +docker compose -f docker/docker-compose.prod.yml up -d --force-recreate app +``` + +### If portal configuration changes (config.wcoa.docker.ini) + +The ini file is bind-mounted into the container (read-only), so changes take effect immediately on the next container restart — no image rebuild needed: + +```bash +nano ~/portals/madrona-portal/marco/config.wcoa.docker.ini +docker compose -f docker/docker-compose.prod.yml up -d --force-recreate app +``` + +--- + +## Useful Commands (on the server) + +All `docker compose` commands run from `~/portals/madrona-portal/`. + +```bash +# Tail app logs +docker compose -f docker/docker-compose.prod.yml logs -f app + +# Run a Django management command +docker compose -f docker/docker-compose.prod.yml --env-file docker/.env run --rm app python marco/manage.py + +# Open a Django shell +docker compose -f docker/docker-compose.prod.yml --env-file docker/.env run --rm app python marco/manage.py shell + +# Open a database shell +docker exec -it \ + $(docker compose -f docker/docker-compose.prod.yml --env-file docker/.env ps -q db) \ + psql -U postgres wcoa_docker_db + +# Check disk and Docker space usage +df -h +docker system df + +# Remove old/unused images to free space +docker image prune -f + +# Stop the stack (data preserved in named volumes) +docker compose -f docker/docker-compose.prod.yml --env-file docker/.env down + +# Full reset — DESTROYS ALL DATA +docker compose -f docker/docker-compose.prod.yml --env-file docker/.env down -v +``` + +--- + +## Services and Ports + +| Service | Image | Internal port | Notes | +|---|---|---|---| +| `app` | `ghcr.io/ecotrust/madrona-portal:latest` | 8000 | Proxied by Nginx | +| `db` | `postgis/postgis:16-3.4` | 5432 | Restrict in security group after go-live | +| `tasks` | `redis:7-alpine` | 6379 | Restrict in security group after go-live | +| `geoportal` | `tomcat:9-jdk11` | 8080 | Add Nginx location if public access needed | +| `elastic` | `elasticsearch:8.19.12` | 9200, 9300 | Restrict in security group after go-live | + +> After go-live, update your security group inbound rules to remove public +> access to ports 5432, 6379, 9200, and 9300. These ports are only needed +> internally between containers on `madronanetwork`. diff --git a/docs/CONFIGURATION_STANDARD.md b/docs/CONFIGURATION_STANDARD.md new file mode 100644 index 00000000..9e26dd58 --- /dev/null +++ b/docs/CONFIGURATION_STANDARD.md @@ -0,0 +1,50 @@ +# Configuration Standard + +This project uses a layered configuration model with strict precedence and typed parsing. + +## Precedence + +For runtime settings, use this priority order: + +1. Environment variable +2. config.ini value +3. Safe default in code + +This keeps deployments flexible while preserving stable local defaults. + +## Required Standard + +Use typed helper functions from marco.config_helpers for app settings. + +- env_str for string settings +- env_bool for boolean settings +- env_int for integer settings + +Do not parse booleans ad hoc in settings code. + +## Why This Standard Exists + +- Avoid inconsistent precedence across settings +- Prevent string-typed booleans from silently behaving incorrectly +- Make behavior testable and explicit + +## Approved Patterns + +Use typed helper access for standard settings: + +- DEBUG +- EMAIL_USE_TLS +- EMAIL_PORT +- SECRET_KEY and other credentials +- STATIC and MEDIA path settings + +Direct environment lookups are still acceptable for special alias chains where multiple environment variable names must be supported for compatibility, such as DB_* and SQL_* overrides. + +## Test Coverage + +The helper contract is enforced by tests in marco/marco/tests/test_config_helpers.py: + +- env over config precedence +- config over default fallback +- strict boolean parsing and invalid-value failures +- integer parsing behavior diff --git a/docs/DOCKER_README.md b/docs/DOCKER_README.md new file mode 100644 index 00000000..b9b84701 --- /dev/null +++ b/docs/DOCKER_README.md @@ -0,0 +1,337 @@ +# Docker Development Guide — Madrona Portal (WCOA) + +## Quick start + +These steps take a fresh machine from nothing to a running portal. + +### Step 1 — Create the workspace directory + +All repos live inside a single parent directory. The Dockerfile build +context is the parent, so the layout is not optional. + +```bash +mkdir portals +cd portals +``` + +### Step 2 — Clone madrona-portal + +```bash +git clone -b docker https://github.com/Ecotrust/madrona-portal.git madrona-portal +``` + +### Step 3 — Clone the sub-app packages + +The Dockerfile copies all of these at build time. Clone them into a +`madrona-apps/` sibling directory: + +```bash +mkdir madrona-apps && cd madrona-apps + +git clone https://github.com/Ecotrust/django_url_shortener.git +git clone https://github.com/Ecotrust/madrona-analysistools.git +git clone https://github.com/Ecotrust/madrona-features.git +git clone https://github.com/Ecotrust/madrona-manipulators.git +git clone https://github.com/Ecotrust/madrona-scenarios.git +git clone https://github.com/Ecotrust/mp-accounts.git +git clone https://github.com/Ecotrust/mp-data-manager.git +git clone https://github.com/Ecotrust/mp-drawing.git +git clone https://github.com/Ecotrust/mp-explore.git +git clone https://github.com/Ecotrust/mp-layers.git +git clone https://github.com/Ecotrust/mp-map-groups.git +git clone https://github.com/Ecotrust/mp-proxy.git +git clone https://github.com/Ecotrust/mp-survey.git +git clone https://github.com/Ecotrust/mp-visualize.git +git clone https://github.com/Ecotrust/p97-nursery.git +git clone https://github.com/Ecotrust/wcoa.git + +cd .. +``` + +Your workspace should now look like: + +``` +portals/ +├── madrona-portal/ ← cloned from Ecotrust/madrona-portal, branch: docker +└── madrona-apps/ + ├── wcoa/ ← branch: vagrant2docker + ├── mp-layers/ + └── ... ← all others on main +``` + +### Step 4 — Configure environment + +From `madrona-portal/`: + +```bash +cd madrona-portal/docker +cp .env.example .env +``` + +Edit `.env` and set at minimum: + +```ini +SECRET_KEY= +DB_PASSWORD= +DJANGO_SUPERUSER_PASSWORD= +``` + +Everything else has working defaults for local development. + +### Step 4.1 - Create ini file + +```bash +cd ../marco +cp config.docker.ini.template config.wcoa.docker.ini +``` + +Edit `config.wcoa.docker.ini` : + +```ini +LOCATION = redis://tasks:6379/1 +CELERY_RESULT_BACKEND = redis://tasks:6379/1 +CELERY_BROKER_URL = redis://tasks:6379/0 +``` + +### Step 5 — Build the image + +Run this command from `madrona-portal/docker` (the previous step leaves +you in `madrona-portal/marco`, so `cd ../docker` gets you there). The +Docker build context for this command is `../../`, which resolves to the parent workspace directory `portals/` that contains both +`madrona-portal/` and `madrona-apps/`. + +If running this build from a MAC, add `--builder desktop-linux` to the `buildx build` command. + +```bash +cd ../docker +# MAC OS +docker compose build --no-cache app +# LINUX +docker compose build --no-cache app +``` + +Previously we recommended `docker buildx build --builder desktop-linux --no-cache --load -f ./Dockerfile ../../` for all platforms, but BuildKit caching on Linux does not appear to have the same git object store issue as on Mac, so the simpler `docker compose build --no-cache app` is sufficient on Linux. + +> **Why `docker buildx build` and not `docker compose build`?** +> `docker compose build` has a caching bug: when a `.git` directory exists +> inside the build context, BuildKit reads files from the git object store +> (committed versions) rather than the filesystem. If you forget to commit +> a change, the old version is silently baked into the image. The same +> restriction applies — always commit changes to `madrona-portal/` or +> `madrona-apps/` before rebuilding. + +### Step 6 — Start the full stack; Populate testing DB + +From `madrona-portal/docker`: + +```bash +DB_INIT=1 docker compose up +``` + +On first boot the entrypoint automatically: + +1. Waits for PostgreSQL to accept connections +2. Runs `migrate` +3. Runs `collectstatic` and `compress` +4. Detects a fresh database and loads initial fixtures (1,782 + 22 objects) +5. Creates the superuser defined in `.env` (if `DJANGO_SUPERUSER_PASSWORD` is set) +6. Starts the application server + +Open: http://localhost:8000/ (or whatever `APP_PORT` is set to in `.env`) + +Once you have a populated DB (either dummy or with migrated data) omit the `DB_INIT=1`: +```bash +docker compose up +``` + + +### Step 7 - Importing a Production SQL Dump into the Dockerized Database + +#### Prerequisites +- Docker Compose stack is running — `docker compose up` +- `madrona-portal/docker/.env` exists with `DB_NAME`, `DB_USER`, and `DB_PASSWORD` set + + +#### Step 7.1 — Ensure you have the db-restore script + +`madrona-portal/scripts/db-restore.sh` has the following behaviour: +- Loads DB credentials from `.env` +- Verifies the `db` container is healthy before proceeding +- With `--drop`: terminates active connections, drops and recreates the database, and enables the PostGIS extension +- Streams the dump file directly into the container via `docker compose exec` (no temp files) +- Prints next-step instructions on completion + +Made sure it is executable: + +From `madrona-portal/docker`: +```bash +chmod +x ../scripts/db-restore.sh +``` + +#### Step 7.2 — Run the restore + +From `madrona-portal/docker`: +```bash +../scripts/db-restore.sh --drop +``` + +*example:* +```bash +../scripts/db-restore.sh --drop ../../madrona-apps/wcoa/wcodp_prod_dump_20260320.sql +``` + +The `--drop` flag was used to ensure a clean import. The script: +1. Terminated all active connections to `wcoa_docker_db` +2. Dropped and recreated the database +3. Enabled the `postgis` extension +4. Streamed the sql dump into the container via `psql` + +There is an optional `--env-file ` if you place your `.env` file in a non-standard location. + +**Expected warnings (non-fatal):** +- `ERROR: relation "..." does not exist` — pg_dump tries to drop constraints before creating them; safe to ignore on a fresh DB +- `ERROR: role "wcoa_user" does not exist` — prod uses a dedicated app role; dev uses `postgres` which has full access + + +#### Step 7.3 — Apply migrations + +```bash +docker compose exec app python marco/manage.py migrate +``` + +#### Step 7.4 - Migration to mp-layers + +*If migrating from a server that has not migrated to mp-layers from mp-data-manager*: + +```bash +docker compose exec app python marco/manage.py migration_to_layers +``` + +--- + +### Step 8 — Importing production media files into the Dockerized Application + +#### Prerequisites +- `madrona-portal/marco/marco/config.wcoa.docker.ini` exists with `MEDIA_ROOT` set to a valid directory +- That valid directory should match the volume location is docker-compose.yml + - `portals/madrona-portal/media` +- Production media files are available + +#### Step 8.1 - Copy the media files into Docker + +If media files need to be copied to the server, you can use `scp`: + +```bash +scp -r /path/to/your_media_dir ubuntu@:/home/ubuntu/portals/madrona-portal/docker/media +``` + +If media files are somewhere on EC2: + +```bash +cd ~/portals/madrona-portal/docker +cp -r {your_media_dir}/* ./media/ +``` + +#### Create a directory for data_manager +```bash +mkdir ~/portals/madrona-portal/docker/data_manager +``` + +--- + +## Migrate existing GeoPortal records + +1. Update .env with the reindex remote whitelist and port. You can find this info by looking at the old server's configuration or by asking the previous admin. + +2. Edit the hosts file to allow the server to resolve the old Elastic IP of the GeoPortal instance to the new internal Docker network: + +```bash +sudo vim /etc/hosts +# Add the following line, replacing and : + +``` + +3. Restart the elastic container to apply the .env and hosts file change: + +```bash +docker compose down -v elastic +docker compose up -d elastic +``` + +4. Ensure the app is running, you can migrate existing GeoPortal records with the following command (replace the username and password): + +```bash +time curl -X POST "http://localhost:9200/_reindex" -H 'Content-Type: application/json' -d'{"conflicts": "proceed", "max_docs": 51000, "source": {"remote": { "host":"http://elastic.prod.wcoa.ecotrust.org:80/geoportal/elastic/", "username": "[[USERNAME]]", "password": "[[PASSWORD]]" }, "index": "metadata", "size": 100 }, "dest": { "index": "metadata" } }' +``` + +1. Do a down, including volumes, and up for the geoportal container to pick up the new records: + +```bash +docker compose down -v geoportal +docker compose up -d geoportal +``` + +--- + +# Deploy to fully containerized production environment + +## AWS EC2 + +See [AWS_DEPLOY.md](../docs/AWS_DEPLOY.md) + +--- + +## Dev infrastructure only (local Django server) + +To run Django locally against Docker-managed PostGIS and Redis (no app container): + +```bash +# Start only db and tasks +docker compose up -d db tasks + +# Then in a separate terminal, from madrona-portal/: +cd marco +python manage.py runserver +``` + +--- + +## Rebuilding after code changes + +From the docker directory (`madrona-portal/docker`): + +```bash +docker compose build --no-cache app +``` + +| Note on `--no-cache`: Use it when dependencies have changed; without it, Docker reuses the cached pip install layer (needed when `docker-requirements.txt` changes). + +Then bring the stack up: + +```bash +docker compose up --force-recreate +``` + +--- + +## Reset to a clean state + +```bash +# From madrona-portal/ +docker compose -f docker/docker-compose.yml --env-file .env --profile full down -v +``` + +`-v` removes the PostGIS and Redis volumes. The next `up` will re-run +migrations and reload fixtures from scratch. + +--- + +## Disk space + +Docker's build cache can grow large over time: + +```bash +docker system df # show usage breakdown +docker system prune -f # remove stopped containers, dangling images, unused networks, build cache +docker volume prune -f # remove unused volumes — only run when all containers are stopped +``` diff --git a/logs_and_config.md b/logs_and_config.md deleted file mode 100644 index f48fa6b5..00000000 --- a/logs_and_config.md +++ /dev/null @@ -1,3 +0,0 @@ -**Logs**: `/home/midatlantic/logs/user/` - -**Config**: `/home/midatlantic/webapps/marco_portal/apache2/conf/httpd.conf` diff --git a/marco/config.docker.ini.template b/marco/config.docker.ini.template new file mode 100644 index 00000000..6cea02f6 --- /dev/null +++ b/marco/config.docker.ini.template @@ -0,0 +1,75 @@ +# Docker-focused configuration for WCOA local development. +# Use with MP_PROJECT_CONFIG=config.wcoa.docker.ini + +[APP] +APP_NAME = WCOA Portal +APP_URL = +APP_TEAM_NAME = Marine Planner Team +PROJECT_APP = wcoa +PROJECT_SETTINGS_FILE = True +DEBUG = True +TEMPLATE_DEBUG = True +ALLOWED_HOSTS = ["localhost", "127.0.0.1", "::1"] +SECRET_KEY = +MEDIA_ROOT = /usr/local/apps/madrona-portal/media +MEDIA_URL = /media/ +TIME_ZONE = UTC +GA_ACCOUNT = +# ReCAPTCHA keys are loaded from env vars: RECAPTCHA_PUBLIC_KEY, RECAPTCHA_PRIVATE_KEY +STATIC_ROOT = /vol/web/static +EMAIL_SUBJECT_PREFIX = [WCOA] +MAP_LIBRARY = ol8 +COMPRESS_ENABLED = True +STATIC_CORE = /vol/web/static/ +ADDITIONAL_APPS = [] +ADDITIONAL_MIDDLEWARE = [] + +[REGION] +NAME = West Coast Ocean +INIT_ZOOM = 6 +INIT_LAT = 39 +INIT_LON = -120 +MAP = ocean + +[CACHES] +BACKEND = django_redis.cache.RedisCache +LOCATION = redis://tasks:6379/1 +CLIENT_CLASS = django_redis.client.DefaultClient + +[CELERY] +CELERY_RESULT_BACKEND = redis://tasks:6379/1 +CELERY_BROKER_URL = redis://tasks:6379 +CELERY_ALWAYS_EAGER = False +CELERY_DISABLE_RATE_LIMITS = True + +[DATABASE] +ENGINE = django.contrib.gis.db.backends.postgis +NAME = wcoa_docker_db +HOST = db +PORT = 5432 +USER = postgres +# DB password is loaded from environment variable: DB_PASSWORD + +[EMAIL] +HOST = localhost +PORT = 25 +# SMTP credentials are loaded from env vars: EMAIL_HOST_USER, EMAIL_HOST_PASSWORD +DEFAULT_FROM_EMAIL = WCOA Portal +SERVER_EMAIL = WCOA Site Errors + +[AWS] +# AWS credentials are loaded from env vars: AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY +AWS_SES_REGION_NAME = us-east-1 +AWS_SES_REGION_ENDPOINT = email.us-east-1.amazonaws.com + +[SOCIAL_AUTH] +# Social OAuth credentials are loaded from env vars: +# FACEBOOK_KEY, FACEBOOK_SECRET, TWITTER_KEY, TWITTER_SECRET, GOOGLE_KEY, GOOGLE_SECRET + + +[CATALOG] +DATA_CATALOG_ENABLED = False +CATALOG_TECHNOLOGY = GeoPortal2 +CATALOG_PROXY = +CATALOG_SOURCE = http://192.168.0.40:9200 +CATALOG_QUERY_ENDPOINT = /geoportal/elastic/metadata/item/_search/ diff --git a/marco/config.ini.dev b/marco/config.ini.dev deleted file mode 100644 index 5f107129..00000000 --- a/marco/config.ini.dev +++ /dev/null @@ -1,54 +0,0 @@ -# Configuration file for MARCO Portal deployments. -# Server-specific configuration goes here. - -[APP] -DEBUG = True -TEMPLATE_DEBUG = True -ALLOWED_HOSTS = * -SECRET_KEY = You_forgot_to_set_the_secret_key -MEDIA_ROOT = /home/vagrant/marco_portal2/media -MEDIA_URL = /media/ -TIME_ZONE = UTC -GA_ACCOUNT = You forgot to set the google analytics account -STATIC_ROOT = /home/vagrant/marco_portal2/static -EMAIL_SUBJECT_PREFIX = [Marco] -STATIC_URL = /static/ - -[CACHES] -BACKEND = redis_cache.RedisCache -LOCATION = /var/run/redis/redis.sock - -[CELERY] -RESULT_BACKEND = redis+socket:///var/run/redis/redis.sock -BROKER_URL = redis+socket:///var/run/redis/redis.sock - -[DATABASE] -ENGINE = django.contrib.gis.db.backends.postgis -NAME = marco_portal -HOST = localhost -PORT = 5432 -USER = vagrant -PASSWORD = None - -[EMAIL] -HOST = localhost -PORT = 8025 -HOST_USER = mail user -HOST_PASSWORD = mail password - -[SOCIAL_AUTH] -FACEBOOK_KEY = You forgot to set the facebook key -FACEBOOK_SECRET = You forgot to set the facebook secret -TWITTER_KEY = You forgot to set the twitter key -TWITTER_SECRET = You forgot to set the twitter secret -GOOGLE_KEY = You forgot to set the google key -GOOGLE_SECRET = You forgot to set the google secret - -[CATALOG] -DATA_CATALOG_ENABLED = False -CATALOG_TECHNOLOGY = default -#CATALOG_TECHNOLOGY = GeoPortal2 -CATALOG_PROXY = -#CATALOG_SOURCE = http://127.0.0.1:9200 -CATALOG_QUERY_ENDPOINT = /geoportal/elastic/metadata/item/_search/ - diff --git a/marco/marco/apps.py b/marco/marco/apps.py index 1f63bb49..fba0b1c1 100644 --- a/marco/marco/apps.py +++ b/marco/marco/apps.py @@ -1,10 +1,6 @@ -import sys from django.apps import AppConfig + class MadronaPortalConfig(AppConfig): - #TODO: Rename this module to 'madrona' or 'madrona_portal' + # TODO: Rename this module to 'madrona' or 'madrona-portal' name = 'marco' - - def ready(self): - from .tasks import start_dbwatch - watch_db = start_dbwatch.delay() \ No newline at end of file diff --git a/marco/marco/celery.py b/marco/marco/celery.py index 9e07129b..6f673ce8 100644 --- a/marco/marco/celery.py +++ b/marco/marco/celery.py @@ -1,13 +1,24 @@ import os from celery import Celery +from celery.signals import worker_ready from django.conf import settings os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'marco.settings') -BASE_REDIS_URL = os.environ.get('REDIS_URL', 'redis://localhost:6379') app = Celery('marco', include=['marco.tasks']) +# Pull Celery config from Django settings (keys prefixed with CELERY_). +# CELERY_BROKER_URL and CELERY_RESULT_BACKEND are set in settings.py from +# the CELERY_BROKER_URL / CELERY_RESULT_BACKEND env vars, or from the +# [CELERY] section of the project .ini config file. app.config_from_object('django.conf:settings', namespace='CELERY') app.autodiscover_tasks(lambda: settings.INSTALLED_APPS) -app.conf.broker_url = BASE_REDIS_URL -app.conf.result_backend = BASE_REDIS_URL + + +@worker_ready.connect +def on_worker_ready(sender, **kwargs): + """Start the long-running db-notify → cache-invalidation listener task + once a Celery worker is live. Keeping this out of AppConfig.ready() + means Django can start without a Redis connection being required.""" + from marco.tasks import start_dbwatch + start_dbwatch.delay() diff --git a/marco/marco/config_helpers.py b/marco/marco/config_helpers.py new file mode 100644 index 00000000..898ecf8e --- /dev/null +++ b/marco/marco/config_helpers.py @@ -0,0 +1,73 @@ +import configparser +import os + + +_TRUE_VALUES = {"1", "true", "yes", "on"} +_FALSE_VALUES = {"0", "false", "no", "off"} + + +def env_str( + env_key: str, + cfg_section: configparser.SectionProxy, + cfg_key: str, + default: str = "", +) -> str: + """Resolve a string setting using env > config.ini > default precedence.""" + raw = os.environ.get(env_key) + if raw is not None: + return raw + raw = cfg_section.get(cfg_key) + if raw is not None: + return raw + return default + + +def parse_bool(value: object, *, setting_name: str = "setting") -> bool: + """Parse a bool from common string values, raising on invalid input.""" + if isinstance(value, bool): + return value + if value is None: + raise ValueError(f"{setting_name} cannot be None") + + normalized = str(value).strip().lower() + if normalized in _TRUE_VALUES: + return True + if normalized in _FALSE_VALUES: + return False + + raise ValueError( + f"Invalid boolean value for {setting_name}: {value!r}. " + "Use one of: 1/0, true/false, yes/no, on/off." + ) + + +def env_bool( + env_key: str, + cfg_section: configparser.SectionProxy, + cfg_key: str, + default: bool = False, +) -> bool: + """Resolve and parse a bool setting using env > config.ini > default.""" + raw = os.environ.get(env_key) + if raw is not None: + return parse_bool(raw, setting_name=env_key) + raw = cfg_section.get(cfg_key) + if raw is not None: + return parse_bool(raw, setting_name=cfg_key) + return default + + +def env_int( + env_key: str, + cfg_section: configparser.SectionProxy, + cfg_key: str, + default: int, +) -> int: + """Resolve and parse an int setting using env > config.ini > default.""" + raw = os.environ.get(env_key) + if raw is not None: + return int(str(raw).strip()) + raw = cfg_section.get(cfg_key) + if raw is not None: + return int(str(raw).strip()) + return default diff --git a/marco/marco/rpc_compat.py b/marco/marco/rpc_compat.py new file mode 100644 index 00000000..a1757003 --- /dev/null +++ b/marco/marco/rpc_compat.py @@ -0,0 +1,375 @@ +"""JSON-RPC 2.0 compatibility shim — POST /rpc/ + +Accepts JSON-RPC 2.0 requests from the legacy jsonrpc.js frontend and +dispatches them to the same business logic as the new DRF REST API views. + +This shim allows the frontend JavaScript to keep using $.jsonrpc() without +modification while the backend has moved to REST endpoints at /api/. + +rpc4django required @csrf_exempt on the /rpc/ view; this shim preserves +that behaviour so the JS can send application/json without a CSRF token. +""" +from __future__ import annotations + +import json +from typing import Any + +from django.conf import settings +from django.contrib.auth.models import Group +from django.http import JsonResponse +from django.views.decorators.csrf import csrf_exempt +from django.views.decorators.http import require_POST + + +# --------------------------------------------------------------------------- +# JSON-RPC 2.0 response helpers +# --------------------------------------------------------------------------- + +def _ok(result: Any, rpc_id: Any) -> JsonResponse: + return JsonResponse({'jsonrpc': '2.0', 'id': rpc_id, 'result': result}) + + +def _err(message: str, rpc_id: Any, code: int = -32000) -> JsonResponse: + return JsonResponse({'jsonrpc': '2.0', 'id': rpc_id, 'error': {'code': code, 'message': message}}) + + +# --------------------------------------------------------------------------- +# Bookmark handlers +# --------------------------------------------------------------------------- + +def _get_bookmarks(request: Any) -> list: + from visualize.models import Bookmark + + content: list[dict] = [] + bookmark_list = Bookmark.objects.filter(user=request.user) + + for bookmark in bookmark_list: + sharing_groups = [ + g.mapgroup_set.get().name for g in bookmark.sharing_groups.all() + ] + content.append({ + 'uid': bookmark.uid, + 'name': bookmark.name, + 'description': bookmark.description, + 'hash': bookmark.url_hash, + 'sharing_groups': sharing_groups, + 'json': bookmark.json, + }) + + shared_bookmarks = Bookmark.objects.shared_with_user(request.user) + for bookmark in shared_bookmarks: + if bookmark not in bookmark_list: + groups = bookmark.sharing_groups.filter(user__in=[request.user]) + shared_groups = [g.mapgroup_set.get().name for g in groups] + content.append({ + 'uid': bookmark.uid, + 'name': bookmark.name, + 'description': bookmark.description, + 'hash': bookmark.url_hash, + 'shared': True, + 'shared_by_user': bookmark.user.id, + 'shared_to_groups': shared_groups, + 'shared_by_name': bookmark.user.get_short_name(), + 'json': bookmark.json, + }) + + return content + + +def _add_bookmark(request: Any, params: list) -> bool: + from visualize.models import Bookmark + + name, description, url_hash, json_str = params[0], params[1], params[2], params[3] + bookmark = Bookmark( + user=request.user, + name=name, + description=description, + url_hash=url_hash, + json=json_str, + ) + bookmark.save() + return True + + +def _load_bookmark(params: list) -> list: + from visualize.models import Bookmark + + bookmark = Bookmark.objects.get(pk=int(params[0])) + return [{'uid': bookmark.uid, 'hash': bookmark.url_hash, 'json': bookmark.json}] + + +def _remove_bookmark(request: Any, params: list) -> bool: + from features.registry import get_feature_by_uid + + bookmark = get_feature_by_uid(params[0]) + viewable, _ = bookmark.is_viewable(request.user) + if viewable: + bookmark.delete() + return True + + +def _share_bookmark(request: Any, params: list) -> bool: + from features.registry import get_feature_by_uid + + uid, group_names = params[0], params[1] + bookmark = get_feature_by_uid(uid) + viewable, _ = bookmark.is_viewable(request.user) + if not viewable: + return False + bookmark.share_with(None) + groups = [Group.objects.get(mapgroup__name=gname) for gname in group_names] + bookmark.share_with(groups, append=False) + return True + + +# --------------------------------------------------------------------------- +# User Layer handlers +# --------------------------------------------------------------------------- + +def _get_user_layers(request: Any) -> list: + from visualize.models import UserLayer + + content: list[dict] = [] + + try: + user_layer_list = UserLayer.objects.filter(user=request.user) + except TypeError: + user_layer_list = [] + + for ul in user_layer_list: + sharing_groups = [ + g.mapgroup_set.get().name + for g in ul.sharing_groups.all() + if g.mapgroup_set.exists() + ] + public_groups = [ + g.name + for g in Group.objects.filter(name__in=settings.SHARING_TO_PUBLIC_GROUPS) + if g in ul.sharing_groups.all() + ] + content.append({ + 'id': ul.id, + 'uid': ul.uid, + 'name': ul.name, + 'description': ul.description, + 'url': ul.url, + 'layer_type': ul.layer_type, + 'password_protected': ul.password_protected, + 'arcgis_layers': ul.arcgis_layers, + 'sharing_groups': sharing_groups + public_groups, + 'shared_to_groups': sharing_groups, + 'owned_by_user': True, + 'wms_slug': ul.wms_slug, + 'wms_srs': ul.wms_srs, + 'wms_params': ul.wms_params, + 'wms_version': ul.wms_version, + 'wms_format': ul.wms_format, + 'wms_styles': ul.wms_styles, + }) + + try: + shared_layers = UserLayer.objects.shared_with_user(request.user) + except TypeError: + shared_layers = UserLayer.objects.filter(pk=-1) + + for ul in shared_layers: + if ul not in user_layer_list: + try: + permission_groups = [ + x.map_group.permission_group + for x in request.user.mapgroupmember_set.all() + ] + except TypeError: + permission_groups = [] + + sharing_groups = [ + g.mapgroup_set.get().name + for g in ul.sharing_groups.all() + if g.mapgroup_set.exists() and g in permission_groups + ] + public_groups = [ + g.name + for g in Group.objects.filter(name__in=settings.SHARING_TO_PUBLIC_GROUPS) + if g in ul.sharing_groups.all() + ] + content.append({ + 'id': ul.id, + 'uid': ul.uid, + 'name': ul.name, + 'description': ul.description, + 'url': ul.url, + 'layer_type': ul.layer_type, + 'password_protected': ul.password_protected, + 'arcgis_layers': ul.arcgis_layers, + 'shared': True, + 'shared_by_user': ul.user.id, + 'sharing_groups': sharing_groups + public_groups, + 'shared_to_groups': sharing_groups, + 'shared_by_name': ul.user.get_short_name(), + 'owned_by_user': len(sharing_groups) > 0, + 'wms_slug': ul.wms_slug, + 'wms_srs': ul.wms_srs, + 'wms_params': ul.wms_params, + 'wms_version': ul.wms_version, + 'wms_format': ul.wms_format, + 'wms_styles': ul.wms_styles, + }) + + return content + + +def _add_user_layer(request: Any, params: list) -> bool: + from visualize.models import UserLayer + + (name, description, layer_type, url, arcgis_layers, + wms_slug, wms_srs, wms_params, wms_version, wms_format, wms_styles) = params + + ul = UserLayer( + user=request.user, + name=name, + description=description or '', + url=url, + layer_type=layer_type, + arcgis_layers=arcgis_layers or '', + wms_slug=wms_slug, + wms_srs=wms_srs, + wms_params=wms_params, + wms_version=wms_version, + wms_format=wms_format, + wms_styles=wms_styles, + ) + ul.save() + return True + + +def _remove_user_layer(request: Any, params: list) -> bool: + from features.registry import get_feature_by_uid + + ul = get_feature_by_uid(params[0]) + viewable, _ = ul.is_viewable(request.user) + if viewable: + ul.delete() + return True + + +def _share_user_layer(request: Any, params: list) -> bool: + from features.registry import get_feature_by_uid + + uid, group_names = params[0], params[1] + ul = get_feature_by_uid(uid) + viewable, _ = ul.is_viewable(request.user) + if not viewable: + return False + ul.share_with(None) + groups = [Group.objects.get(mapgroup__name=gname) for gname in group_names] + ul.share_with(groups, append=False) + return True + + +# --------------------------------------------------------------------------- +# Sharing Groups handler +# --------------------------------------------------------------------------- + +def _get_sharing_groups(request: Any) -> list: + data: list[dict] = [] + + for membership in request.user.mapgroupmember_set.all(): + group = membership.map_group + members = sorted( + member.user_name_for_group() + for member in group.mapgroupmember_set.all() + ) + data.append({ + 'group_name': group.name, + 'group_slug': group.permission_group.name, + 'members': members, + 'is_mapgroup': True, + }) + + for public_group in Group.objects.filter(name__in=settings.SHARING_TO_PUBLIC_GROUPS): + data.append({ + 'group_name': public_group.name, + 'group_slug': public_group.name, + 'members': [], + 'is_mapgroup': False, + }) + + return data + + +# --------------------------------------------------------------------------- +# Drawing handler +# --------------------------------------------------------------------------- + +def _delete_drawing(request: Any, params: list) -> bool: + from features.registry import get_feature_by_uid + + drawing = get_feature_by_uid(params[0]) + viewable, _ = drawing.is_viewable(request.user) + if viewable: + drawing.delete() + return True + + +# --------------------------------------------------------------------------- +# Dispatcher +# --------------------------------------------------------------------------- + +_AUTH_REQUIRED = frozenset({ + 'get_bookmarks', 'add_bookmark', 'remove_bookmark', 'share_bookmark', + 'add_user_layer', 'remove_user_layer', 'share_user_layer', + 'get_sharing_groups', 'delete_drawing', +}) + + +def _dispatch(request: Any, method: str, params: list, rpc_id: Any) -> JsonResponse: + if method in _AUTH_REQUIRED and not request.user.is_authenticated: + return _err('Authentication required.', rpc_id, code=-32001) + + try: + if method == 'get_bookmarks': + return _ok(_get_bookmarks(request), rpc_id) + elif method == 'add_bookmark': + return _ok(_add_bookmark(request, params), rpc_id) + elif method == 'load_bookmark': + return _ok(_load_bookmark(params), rpc_id) + elif method == 'remove_bookmark': + return _ok(_remove_bookmark(request, params), rpc_id) + elif method == 'share_bookmark': + return _ok(_share_bookmark(request, params), rpc_id) + elif method == 'get_user_layers': + return _ok(_get_user_layers(request), rpc_id) + elif method == 'add_user_layer': + return _ok(_add_user_layer(request, params), rpc_id) + elif method == 'remove_user_layer': + return _ok(_remove_user_layer(request, params), rpc_id) + elif method == 'share_user_layer': + return _ok(_share_user_layer(request, params), rpc_id) + elif method == 'get_sharing_groups': + return _ok(_get_sharing_groups(request), rpc_id) + elif method == 'delete_drawing': + return _ok(_delete_drawing(request, params), rpc_id) + else: + return _err(f'Method not found: {method}', rpc_id, code=-32601) + except Exception as exc: + return _err(str(exc), rpc_id) + + +@csrf_exempt +@require_POST +def rpc_view(request): + """JSON-RPC 2.0 endpoint — POST /rpc/ + + Accepts legacy jsonrpc.js requests and dispatches them to the same + business logic as the REST API views at /api/. + """ + try: + body = json.loads(request.body) + except (json.JSONDecodeError, ValueError): + return _err('Parse error.', None, code=-32700) + + method = body.get('method', '') + params = body.get('params', []) + rpc_id = body.get('id', 7) + + return _dispatch(request, method, params, rpc_id) diff --git a/marco/marco/settings.py b/marco/marco/settings.py index 357d76a7..ba38a87d 100644 --- a/marco/marco/settings.py +++ b/marco/marco/settings.py @@ -1,167 +1,173 @@ """ -Django settings for marco_portal project. +Django settings for the Madrona Portal project. -For more information on this file, see -https://docs.djangoproject.com/en/1.7/topics/settings/ +References: + https://docs.djangoproject.com/en/4.2/topics/settings/ + https://docs.djangoproject.com/en/4.2/ref/settings/ -For the full list of settings and their values, see -https://docs.djangoproject.com/en/1.7/ref/settings/ +Requires: Django 4.2+, Wagtail 7.0+, Python 3.10+ + +Secret / credential precedence (highest → lowest): + 1. Environment variable (e.g. export SECRET_KEY=...) + 2. config.ini value (e.g. [APP]\nSECRET_KEY = ...) + 3. Hard-coded default (safe defaults only — never real secrets) """ -import sys +import ast +import json import os import configparser from os.path import abspath, dirname -# from social.backends.google import GooglePlusAuth +from typing import Any -# Absolute filesystem path to the Django project directory: -PROJECT_ROOT = dirname(dirname(dirname(abspath(__file__)))) +from .config_helpers import env_bool, env_int, env_str +# --------------------------------------------------------------------------- +# Path helpers +# --------------------------------------------------------------------------- +PROJECT_ROOT = dirname(dirname(dirname(abspath(__file__)))) BASE_DIR = os.path.dirname(os.path.dirname(__file__)) -ASSETS_DIR = os.path.realpath(os.path.join(BASE_DIR, '..', 'assets')) # assets directory -COMPONENTS_DIR = os.path.realpath(os.path.join(BASE_DIR, '..', 'bower_components')) # Bower components directory -STYLES_DIR = os.path.realpath(os.path.join(ASSETS_DIR, 'styles')) # SCSS files under assets/styles +ASSETS_DIR = os.path.realpath(os.path.join(BASE_DIR, '..', 'assets')) +COMPONENTS_DIR = os.path.realpath(os.path.join(BASE_DIR, '..', 'bower_components')) +STYLES_DIR = os.path.realpath(os.path.join(ASSETS_DIR, 'styles')) -MP_PROJECT_CONFIG = os.environ.get("MP_PROJECT_CONFIG", default='config.ini') -CONFIG_FILE = os.path.normpath(os.path.join(BASE_DIR, MP_PROJECT_CONFIG)) +# --------------------------------------------------------------------------- +# Configuration file +# --------------------------------------------------------------------------- +MP_PROJECT_CONFIG = os.environ.get("MP_PROJECT_CONFIG", "config.ini") + +# Allow MP_PROJECT_CONFIG to be an absolute path. Enabling modularity, +# because a config.ini can live outside the madrona portal project directory. +# Otherwise use the config.ini in madrona portal. +if os.path.isabs(MP_PROJECT_CONFIG): + CONFIG_FILE = MP_PROJECT_CONFIG +else: + CONFIG_FILE = os.path.normpath(os.path.join(BASE_DIR, MP_PROJECT_CONFIG)) cfg = configparser.ConfigParser() cfg.read(CONFIG_FILE) -if 'APP' not in cfg.sections(): - cfg['APP'] = {} +for section in ('APP', 'CATALOG', 'DATABASE', 'CACHES', 'CELERY', 'EMAIL', 'AWS', 'SOCIAL_AUTH', 'REGION'): + if section not in cfg.sections(): + cfg[section] = {} app_cfg = cfg['APP'] +catalog_cfg = cfg['CATALOG'] +db_cfg = cfg['DATABASE'] +cache_cfg = cfg['CACHES'] +celery_cfg = cfg['CELERY'] +email_cfg = cfg['EMAIL'] +aws_cfg = cfg['AWS'] +social_cfg = cfg['SOCIAL_AUTH'] +region_cfg = cfg['REGION'] -DEBUG = app_cfg.getboolean('DEBUG', True) - -APP_NAME = app_cfg.get('APP_NAME', 'Marine Planner') -APP_URL = app_cfg.get('APP_URL', '') -APP_TEAM_NAME = app_cfg.get('APP_TEAM_NAME', "{} Team".format(APP_NAME)) - -TEMPLATE_DEBUG = app_cfg.getboolean('TEMPLATE_DEBUG', True) - -SECRET_KEY = app_cfg.get('SECRET_KEY', 'you forgot to set the secret key') -host_list = app_cfg.get('ALLOWED_HOSTS') -if type(host_list) == str: - if '[' in host_list and ']' in host_list: - import ast - ALLOWED_HOSTS = ast.literal_eval(host_list) - elif ',' in host_list: - ALLOWED_HOSTS = host_list.split(',') - else: - ALLOWED_HOSTS = [host_list] -elif type(host_list) == list: - ALLOWED_HOSTS = host_list -else: - ALLOWED_HOSTS = [str(host_list)] +# --------------------------------------------------------------------------- +# Core settings +# --------------------------------------------------------------------------- +DEBUG = env_bool('DEBUG', app_cfg, 'DEBUG', False) + +APP_NAME = env_str('APP_NAME', app_cfg, 'APP_NAME', 'Marine Planner') +APP_URL = env_str('APP_URL', app_cfg, 'APP_URL', '') +APP_TEAM_NAME = env_str('APP_TEAM_NAME', app_cfg, 'APP_TEAM_NAME', f"{APP_NAME} Team") + +# CSRF trusted origins: accepts a comma-separated string, a JSON array string, or a plain string. +SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https") +CSRF_TRUSTED_ORIGINS_ENV = env_str('CSRF_TRUSTED_ORIGINS', app_cfg, 'CSRF_TRUSTED_ORIGINS', '') +if CSRF_TRUSTED_ORIGINS_ENV: + CSRF_TRUSTED_ORIGINS = CSRF_TRUSTED_ORIGINS_ENV.split(",") + + +# env var takes priority so Docker / CI can inject secrets without touching config.ini +SECRET_KEY = env_str('SECRET_KEY', app_cfg, 'SECRET_KEY', '') +_placeholder_phrases = ('forgot', 'change me', 'changeme', 'placeholder', 'you forgot') +if not SECRET_KEY or any(p in SECRET_KEY.lower() for p in _placeholder_phrases): + raise RuntimeError( + "SECRET_KEY is not set or still contains a placeholder value.\n" + "Set it via the SECRET_KEY environment variable or in config.ini [APP].\n" + f"Current value: {SECRET_KEY!r}" + ) + +# ALLOWED_HOSTS: accepts a comma-separated string, a JSON array string, or a plain string. +def _parse_hosts(raw: str | None) -> list[str]: + if not raw: + return [] + raw = raw.strip() + if raw.startswith('['): + try: + parsed = ast.literal_eval(raw) + if isinstance(parsed, (list, tuple)): + return [str(h).strip() for h in parsed if str(h).strip()] + except (SyntaxError, ValueError): + pass + # Fall back: strip brackets and split on comma + return [h.strip() for h in raw[1:-1].split(',') if h.strip()] + if ',' in raw: + return [h.strip() for h in raw.split(',') if h.strip()] + return [raw] + +_raw_hosts = env_str('ALLOWED_HOSTS', app_cfg, 'ALLOWED_HOSTS', '') +ALLOWED_HOSTS = _parse_hosts(_raw_hosts) + +# Normalise bracketed IPv6 forms like [::1] → ::1 for Django host checks +ALLOWED_HOSTS = [ + h[1:-1] if h.startswith('[') and h.endswith(']') and ':' in h else h + for h in ALLOWED_HOSTS +] -# Set logging to default, and then make admin error emails come through as HTML +# --------------------------------------------------------------------------- +# Logging +# --------------------------------------------------------------------------- from django.utils.log import DEFAULT_LOGGING LOGGING = DEFAULT_LOGGING LOGGING['handlers']['mail_admins']['include_html'] = True -if 'CATALOG' not in cfg.sections(): - cfg['CATALOG'] = {} -catalog_cfg = cfg['CATALOG'] +# --------------------------------------------------------------------------- +# Catalog settings +# --------------------------------------------------------------------------- DATA_CATALOG_ENABLED = catalog_cfg.getboolean('DATA_CATALOG_ENABLED', True) -#CATALOG_TECHNOLOGY: Current support for 'default' (built in catalog) and 'GeoPortal2' +# Options: 'default' (built-in) or 'GeoPortal2' CATALOG_TECHNOLOGY = catalog_cfg.get('CATALOG_TECHNOLOGY', 'default') CATALOG_PROXY = catalog_cfg.get('CATALOG_PROXY', '') CATALOG_SOURCE = catalog_cfg.get('CATALOG_SOURCE', 'http://127.0.0.1:9200') -CATALOG_QUERY_ENDPOINT = catalog_cfg.get('CATALOG_QUERY_ENDPOINT', '/geoportal/elastic/metadata/item/_search/') - -try: - # Wagtail v5 - INSTALLED_APPS = [ - 'wagtail.contrib.forms', - 'wagtail.contrib.redirects', - 'wagtail.contrib.sitemaps', - 'wagtail.contrib.styleguide', - 'wagtail.contrib.table_block', - 'wagtail.embeds', - 'wagtail.sites', - 'wagtail.users', - 'wagtail.snippets', - 'wagtail.documents', - 'wagtail.images', - 'wagtail.search', - 'wagtail.admin', - 'wagtail', - ] - - import wagtail - WAGTAIL_VERSION = wagtail.VERSION[0] - -except ImportError as e: - # Application definition - try: - # Thanks to tgandor for this inspiration to handle two different wagtail - # versions conditionally while performing this terrible merge: - # https://djangosnippets.org/snippets/3048/ - __import__('wagtail.contrib.forms') - # Wagtail v2 - WAGTAIL_VERSION = 2 - - INSTALLED_APPS = [ - 'wagtail.contrib.forms', - 'wagtail.contrib.redirects', - 'wagtail.embeds', - 'wagtail.sites', - 'wagtail.users', - 'wagtail.snippets', - 'wagtail.documents', - 'wagtail.images', - 'wagtail.search', - 'wagtail.admin', - 'wagtail', - 'wagtail.contrib.styleguide', - 'wagtail.contrib.sitemaps', - 'wagtail.locales', - 'wagtail.contrib.table_block', - 'wagtail.redirects', - ] - - except ImportError as e: - # print(e) - # Wagtail v1 for merging in old MidA Portal - WAGTAIL_VERSION = 1 - INSTALLED_APPS = [ - 'wagtail', - 'wagtail.admin', - 'wagtail.docs', - 'wagtail.snippets', - 'wagtail.users', - 'wagtail.sites', - 'wagtail.images', - 'wagtail.embeds', - 'wagtail.search', - 'wagtail.redirects', - 'wagtail.forms', - 'wagtail.contrib.sitemaps', - ] - -try: - __import__('django_redis') - REDIS_PACKAGE_NAME = 'django_redis' - INSTALLED_APPS += ['django_redis',] -except ImportError as e: - REDIS_PACKAGE_NAME = 'redis_cache' - - -INSTALLED_APPS += [ - 'marco_site', - 'marco.apps.MadronaPortalConfig', - # DLP 2025.06.11: This is where favoring marco/marco static files over other apps happens. - # INSTALLED_APPS sets precedence from top to bottom (e.g., 0 indexed INSTALLED_APPS static files are chosen over other apps). +CATALOG_QUERY_ENDPOINT = catalog_cfg.get( + 'CATALOG_QUERY_ENDPOINT', + '/geoportal/elastic/metadata/item/_search/', +) - # 'kombu.transport.django', +# --------------------------------------------------------------------------- +# Installed Applications (Wagtail 7+) +# --------------------------------------------------------------------------- +import wagtail +WAGTAIL_VERSION = wagtail.VERSION[0] + +INSTALLED_APPS = [ + # Wagtail contrib modules + 'wagtail.contrib.forms', + 'wagtail.contrib.redirects', + 'wagtail.contrib.sitemaps', + 'wagtail.contrib.styleguide', + 'wagtail.contrib.table_block', + # Wagtail core + 'wagtail.embeds', + 'wagtail.sites', + 'wagtail.users', + 'wagtail.snippets', + 'wagtail.documents', + 'wagtail.images', + 'wagtail.search', + 'wagtail.admin', + 'wagtail', + + # Portal application + 'marco_site', + 'marco.apps.MadronaPortalConfig', - # Django-autocomplete-light + # Django Autocomplete Light 'dal', 'dal_select2', 'dal_queryset_sequence', + # Django core 'django.contrib.admin', 'django.contrib.auth', 'django.contrib.contenttypes', @@ -171,35 +177,24 @@ 'django.contrib.sites', 'django.contrib.sitemaps', 'django.contrib.staticfiles', - # 'django.contrib.webdesign', 'django.contrib.humanize', - # 'p97settings', - + # Third-party + 'django_redis', 'email_log', 'djcelery_email', 'compressor', 'taggit', 'modelcluster', - 'rpc4django', + # rpc4django removed — replaced by DRF API views (visualize/api.py, drawing/api.py, mapgroups/api.py) 'tinymce', -] - -# RDH 20240315: this is getting really messy, but django-recaptcha changed from calling itself 'captcha' when it hit v4 -# Newer Wagtail versions use v4, earlier are stuck on v2 or 3, so swapping based on WAGTAIL_VERSION works for now... -if WAGTAIL_VERSION > 4: - INSTALLED_APPS += [ - 'django_recaptcha', - ] -else: - INSTALLED_APPS += [ - 'captcha', - ] - -INSTALLED_APPS += [ + 'django_recaptcha', # Wagtail 7+ uses django-recaptcha v4 (app name: django_recaptcha) 'social_django', - # 'django_redis', + 'flatblocks', + 'import_export', + 'rest_framework', + # Portal sub-apps 'portal.base', 'portal.menu', 'portal.home', @@ -213,11 +208,8 @@ 'portal.initial_data', 'portal.welcome_snippet', 'portal.news', - 'rest_framework', - - 'flatblocks', - # 'wagtailimportexport', + # Ecotrust / Madrona sub-apps 'data_manager', 'layers', 'url_short', @@ -227,36 +219,31 @@ 'drawing', 'manipulators', 'explore', - # 'survey', - - # Account management - 'social.apps.django_app.default', 'accounts.apps.AccountsAppConfig', 'django_social_share', 'mapgroups', - 'import_export', - + 'survey', ] -try: - __import__('nested_admin') - INSTALLED_APPS += ['nested_admin',] -except ImportError as e: - pass - -try: - __import__('colorfield') - INSTALLED_APPS += ['colorfield',] -except ImportError as e: - pass +# Optional apps — installed when available +for _optional_app in ('nested_admin', 'colorfield', 'wagtailcharts'): + try: + __import__(_optional_app) + if _optional_app not in INSTALLED_APPS: + INSTALLED_APPS.append(_optional_app) + except ImportError: + pass + +# --------------------------------------------------------------------------- +# Authentication +# --------------------------------------------------------------------------- AUTHENTICATION_BACKENDS = ( - # 'social.backends.google.GoogleOAuth2', - # 'social.backends.google.GoogleOpenId', - # 'social.backends.facebook.FacebookOAuth2', - # 'social.backends.twitter.TwitterOAuth', 'django.contrib.auth.backends.ModelBackend', ) +# --------------------------------------------------------------------------- +# Middleware +# --------------------------------------------------------------------------- MIDDLEWARE = [ 'django.contrib.sessions.middleware.SessionMiddleware', 'django.middleware.common.CommonMiddleware', @@ -264,200 +251,139 @@ 'django.contrib.auth.middleware.AuthenticationMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware', - - # 'wagtail.middleware.SiteMiddleware', - # 'wagtail.contrib.redirects.middleware.RedirectMiddleware', 'marco.host_site_middleware.HostSiteMiddleware', + 'wagtail.contrib.redirects.middleware.RedirectMiddleware', ] X_FRAME_OPTIONS = 'SAMEORIGIN' -# FILE_UPLOAD_HANDLERS = [ -# 'django.core.files.uploadhandler.MemoryFileUploadHandler', -# 'django.core.files.uploadhandler.TemporaryFileUploadHandler' -# ] - - -# if WAGTAIL_VERSION > 1: -# try: -# __import__('wagtail.middleware.SiteMiddleware') -# MIDDLEWARE += [ -# 'wagtail.middleware.SiteMiddleware', -# ] -# except ImportError as e: -# # https://docs.wagtail.io/en/stable/releases/2.11.html#sitemiddleware-moved-to-wagtail-contrib-legacy -# MIDDLEWARE += [ -# 'wagtail.contrib.legacy.sitemiddleware.SiteMiddleware', -# ] -# WAGTAIL_VERSION = 2.11 -# MIDDLEWARE += [ -# 'wagtail.contrib.redirects.middleware.RedirectMiddleware', -# ] -# else: -MIDDLEWARE += [ - # 'wagtail.middleware.SiteMiddleware', - 'wagtail.contrib.redirects.middleware.RedirectMiddleware', - # 'wagtail.redirects.middleware.RedirectMiddleware', -] - -# Valid site IDs are 1 and 2, corresponding to the primary site(1) and the -# test site(2) +# --------------------------------------------------------------------------- +# URLs / WSGI +# --------------------------------------------------------------------------- SITE_ID = 1 - INTERNAL_IPS = ('127.0.0.1',) - ROOT_URLCONF = 'marco.urls' WSGI_APPLICATION = 'marco.wsgi.application' - - -if 'DATABASE' not in cfg.sections(): - cfg['DATABASE'] = {} - -db_cfg = cfg['DATABASE'] - -default = { - 'ENGINE': db_cfg.get('ENGINE', - 'django.contrib.gis.db.backends.postgis'), -} - -if default['ENGINE'].endswith('spatialite'): - SPATIALITE_LIBRARY_PATH = db_cfg.get('SPATIALITE_LIBRARY_PATH') - default['NAME'] = db_cfg.get('NAME', os.path.join(BASE_DIR, 'marco.db')) +APPEND_SLASH=True + +# --------------------------------------------------------------------------- +# Database (PostGIS by default) +# Env var overrides (Docker / CI): DB_ENGINE, DB_NAME, DB_USER, DB_PASSWORD, +# DB_HOST, DB_PORT. SQL_* aliases are also accepted for legacy docker-compose +# compatibility. +# --------------------------------------------------------------------------- +_db_engine = ( + os.environ.get('DB_ENGINE') + or os.environ.get('SQL_ENGINE') + or db_cfg.get('ENGINE', 'django.contrib.gis.db.backends.postgis') +) +default_db: dict[str, Any] = {'ENGINE': _db_engine} + +if _db_engine.endswith('spatialite'): + default_db['SPATIALITE_LIBRARY_PATH'] = db_cfg.get('SPATIALITE_LIBRARY_PATH') + default_db['NAME'] = ( + os.environ.get('DB_NAME') + or db_cfg.get('NAME', os.path.join(BASE_DIR, 'marco.db')) + ) else: - default['NAME'] = db_cfg.get('NAME') - # default['NAME'] = os.environ.get("SQL_DATABASE", "ocean_portal"), - if cfg.has_option('DATABASE', 'USER'): - default['USER'] = db_cfg.get('USER') - if cfg.has_option('DATABASE', 'HOST'): - default['HOST'] = db_cfg.get('HOST', 'localhost') - default['PORT'] = db_cfg.getint('PORT', 5432) - if cfg.has_option('DATABASE', 'PASSWORD'): - default['PASSWORD'] = db_cfg.get('PASSWORD') - -DATABASES = {'default': default} - + default_db['NAME'] = ( + os.environ.get('DB_NAME') or os.environ.get('SQL_DATABASE') + or db_cfg.get('NAME', '') + ) + default_db['USER'] = ( + os.environ.get('DB_USER') or os.environ.get('SQL_USER') + or db_cfg.get('USER', '') + ) + default_db['PASSWORD'] = ( + os.environ.get('DB_PASSWORD') or os.environ.get('SQL_PASSWORD') + or db_cfg.get('PASSWORD', '') + ) + default_db['HOST'] = ( + os.environ.get('DB_HOST') or os.environ.get('SQL_HOST') + or db_cfg.get('HOST', 'localhost') + ) + default_db['PORT'] = int( + os.environ.get('DB_PORT') or os.environ.get('SQL_PORT') + or db_cfg.get('PORT', '5432') + ) + +DATABASES = {'default': default_db} +DB_CHANNEL = db_cfg.get('DB_CHANNEL', 'madrona-portal') -DB_CHANNEL = db_cfg.get('DB_CHANNEL', 'madrona_portal') - -if 'CACHES' not in cfg.sections(): - cfg['CACHES'] = {} - -cache_cfg = cfg['DATABASE'] +DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' -# ------------------------------------------------------------------------------ -# Redis sessions and caching -# ------------------------------------------------------------------------------ -# SESSION_ENGINE = 'redis_sessions.session' +# --------------------------------------------------------------------------- +# Caching (Redis via django-redis) +# Env var override: REDIS_URL (e.g. redis://:password@tasks:6379/1) +# --------------------------------------------------------------------------- SESSION_ENGINE = "django.contrib.sessions.backends.cache" SESSION_CACHE_ALIAS = "default" -SESSION_REDIS_HOST = 'localhost' -SESSION_REDIS_PORT = 6379 -SESSION_REDIS_DB = 0 -if REDIS_PACKAGE_NAME == 'redis_cache': - { - 'default': { - 'BACKEND': 'redis_cache.RedisCache', - 'LOCATION': '/home/midatlantic/run/redis.sock', - 'KEY_PREFIX': 'marco_portal', - 'OPTIONS': { - 'CLIENT_CLASS': 'redis_cache.client.DefaultClient' - }, - } - } -else: - CACHES = { - 'default': { - 'BACKEND': cache_cfg.get('BACKEND', 'django_redis.cache.RedisCache'), - 'LOCATION': cache_cfg.get('LOCATION', 'redis://127.0.0.1:6379/1'), - 'KEY_PREFIX': 'marco_portal', - 'OPTIONS': { - 'CLIENT_CLASS': cache_cfg.get('CLIENT_CLASS', 'django_redis.client.DefaultClient'), - } - } - } +_redis_location = ( + os.environ.get('REDIS_URL') + or cache_cfg.get('LOCATION', 'redis://127.0.0.1:6379/1') +) -# Internationalization -# https://docs.djangoproject.com/en/1.7/topics/i18n/ +CACHES = { + 'default': { + 'BACKEND': cache_cfg.get('BACKEND', 'django_redis.cache.RedisCache'), + 'LOCATION': _redis_location, + 'KEY_PREFIX': 'marco_portal', + 'OPTIONS': { + 'CLIENT_CLASS': cache_cfg.get('CLIENT_CLASS', 'django_redis.client.DefaultClient'), + }, + } +} +# --------------------------------------------------------------------------- +# Internationalisation +# --------------------------------------------------------------------------- LANGUAGE_CODE = 'en-us' -TIME_ZONE = app_cfg.get('TIME_ZONE', 'UTC') +TIME_ZONE = env_str('TIME_ZONE', app_cfg, 'TIME_ZONE', 'UTC') USE_I18N = True USE_TZ = True WAGTAIL_I18N_ENABLED = False -WAGTAIL_CONTENT_LANGUAGES = LANGUAGES = [ - ('en', "English"), -] - -# Static files (CSS, JavaScript, Images) -# https://docs.djangoproject.com/en/1.7/howto/static-files/ - -STATIC_ROOT = app_cfg.get('STATIC_ROOT', os.path.join(BASE_DIR, 'static')) -STATIC_URL = app_cfg.get('STATIC_URL', '/static/') -STATIC_CORE = app_cfg.get('STATIC_CORE', '/usr/local/apps/marco_portal_static/') - -STATICFILES_DIRS = ( - STYLES_DIR, - COMPONENTS_DIR, - ASSETS_DIR, - STATIC_CORE, -) -# Precedence for static files in STATICFILES_DIRS is determined by the order of the directories in STATICFILES_DIRS +WAGTAIL_CONTENT_LANGUAGES = LANGUAGES = [('en', "English")] + +# --------------------------------------------------------------------------- +# Static & media files +# --------------------------------------------------------------------------- +STATIC_ROOT = env_str('STATIC_ROOT', app_cfg, 'STATIC_ROOT', os.path.join(BASE_DIR, 'static')) +STATIC_URL = env_str('STATIC_URL', app_cfg, 'STATIC_URL', '/static/') +STATIC_CORE = env_str('STATIC_CORE', app_cfg, 'STATIC_CORE', '') + +MEDIA_ROOT = env_str('MEDIA_ROOT', app_cfg, 'MEDIA_ROOT', os.path.join(BASE_DIR, 'media')) +MEDIA_URL = env_str('MEDIA_URL', app_cfg, 'MEDIA_URL', '/media/') + +_static_root_abs = os.path.abspath(STATIC_ROOT) +_staticfiles_dirs: list[str] = [] +for _dir in (STYLES_DIR, COMPONENTS_DIR, ASSETS_DIR, STATIC_CORE): + if not _dir: + continue + _abs = os.path.abspath(_dir) + if _abs == _static_root_abs or _abs in [os.path.abspath(d) for d in _staticfiles_dirs]: + continue + _staticfiles_dirs.append(_dir) + +STATICFILES_DIRS = tuple(_staticfiles_dirs) STATICFILES_FINDERS = ( 'django.contrib.staticfiles.finders.FileSystemFinder', 'django.contrib.staticfiles.finders.AppDirectoriesFinder', 'django.contrib.staticfiles.finders.DefaultStorageFinder', 'compressor.finders.CompressorFinder', -) -# Precedence for static files for in App (i.e., AppDirectoriesFinder) is determined by the order of INSTALLED_APPS - -MEDIA_ROOT = app_cfg.get('MEDIA_ROOT', os.path.join(BASE_DIR, 'media')) -MEDIA_URL = app_cfg.get('MEDIA_URL', '/media/') - +) -# Django compressor settings +# Django Compressor / SASS COMPRESS_PRECOMPILERS = ( - ('text/x-scss', 'django_libsass.SassCompiler'), # for wagtail + ('text/x-scss', 'django_libsass.SassCompiler'), ) - -COMPRESS_ENABLED = app_cfg.getboolean('COMPRESS_ENABLED', True) +COMPRESS_ENABLED = env_bool('COMPRESS_ENABLED', app_cfg, 'COMPRESS_ENABLED', True) COMPRESS_OFFLINE = True -try: - # Test is DATA_MANAGER_ADMIN was already defined by PROJECT settings. - DATA_MANAGER_ADMIN -except NameError: - DATA_MANAGER_ADMIN = False - -LAYER_TYPE_CHOICES = ( - ('XYZ', 'XYZ'), - ('WMS', 'WMS'), - ('ArcRest', 'ArcRest'), - ('ArcFeatureServer', 'ArcFeatureServer'), - ('radio', 'radio'), - ('checkbox', 'checkbox'), - ('Vector', 'Vector'), - ('VectorTile', 'VectorTile'), - ('placeholder', 'placeholder'), -) - -# Template configuration - -from django.conf import global_settings - -# Removed due to this: https://stackoverflow.com/a/39315587 - RDH (WCOA) 7/8/2019 -# # RDH (MARCO) 20191114 - if tuple, make into a list -# TEMPLATE_CONTEXT_PROCESSORS = [x for x in global_settings.TEMPLATE_CONTEXT_PROCESSORS] + [ -# 'django.core.context_processors.request', -# 'social_django.context_processors.backends', -# 'portal.base.context_processors.search_disabled', -# ] -# -# TEMPLATE_LOADERS = [x for x in global_settings.TEMPLATE_LOADERS] + [ -# 'apptemplates.Loader', -# ] - +# --------------------------------------------------------------------------- +# Templates +# --------------------------------------------------------------------------- TEMPLATES = [ { 'BACKEND': 'django.template.backends.django.DjangoTemplates', @@ -470,314 +396,248 @@ 'django.template.context_processors.debug', 'django.template.context_processors.request', 'django.contrib.auth.context_processors.auth', - 'django.contrib.messages.context_processors.messages' + 'django.contrib.messages.context_processors.messages', ] }, }, ] -# Wagtail settings - +# --------------------------------------------------------------------------- +# Wagtail +# --------------------------------------------------------------------------- LOGIN_URL = 'account:index' -# LOGIN_REDIRECT_URL = 'wagtailadmin_home' - WAGTAIL_SITE_NAME = 'MARCO Portal' - WAGTAILSEARCH_RESULTS_TEMPLATE = 'portal/search_results.html' - - -# WAGTAILSEARCH_BACKENDS = { -# 'default': { -# 'BACKEND': 'wagtail.search.backends.elasticsearch.ElasticSearch', -# # 'URLS': ['https://iu20e5efzd:dibenj5fn5@point-97-elasticsear-6230081365.us-east-1.bonsai.io'], -# 'URLS': ['https://site:a379ac680e6aaa45f0c129c2cd28d064@bofur-us-east-1.searchly.com'], -# 'INDEX': 'marco_portal', -# 'TIMEOUT': 5, -# } -# } - -# Whether to use face/feature detection to improve image cropping - requires OpenCV WAGTAILIMAGES_FEATURE_DETECTION_ENABLED = False - -# Override the Image class used by wagtailimages with a custom one WAGTAILIMAGES_IMAGE_MODEL = 'base.PortalImage' -try: - FEEDBACK_IFRAME_URL -except NameError as e: - FEEDBACK_IFRAME_URL = "//docs.google.com/forms/d/e/1FAIpQLSdi0nBoQK-3ia8rKtzh7cif0slzDCjA_ACH9Y_ryam-co6p8A/viewform?usp=sf_link" - -try: - DISCLAIMER_BUTTON_DEFAULT -except NameError as e: - DISCLAIMER_BUTTON_DEFAULT = False - -# madrona-features -SHARING_TO_PUBLIC_GROUPS = ['Share with Public'] -SHARING_TO_STAFF_GROUPS = ['Share with Staff'] +# --------------------------------------------------------------------------- +# Map / Geospatial +# --------------------------------------------------------------------------- +MAP_LIBRARY = env_str('MAP_LIBRARY', app_cfg, 'MAP_LIBRARY', 'ol6') +GEOMETRY_DB_SRID = 3857 +GEOMETRY_CLIENT_SRID = 3857 +GEOJSON_SRID = 3857 +GEOJSON_DOWNLOAD = True +SUPPORT_INVERTED_COORDINATES = False +SERVER_SRID = 4326 -# KML SETTINGS -KML_SIMPLIFY_TOLERANCE = 20 # meters -KML_SIMPLIFY_TOLERANCE_DEGREES = 0.0002 # Very roughly ~ 20 meters +KML_SIMPLIFY_TOLERANCE = 20 # metres +KML_SIMPLIFY_TOLERANCE_DEGREES = 0.0002 KML_EXTRUDE_HEIGHT = 100 KML_ALTITUDEMODE_DEFAULT = 'absolute' -# madrona-scenarios -GEOMETRY_DB_SRID = 3857 -GEOMETRY_CLIENT_SRID = 3857 #for latlon -GEOJSON_SRID = 3857 +LAYER_TYPE_CHOICES = ( + ('XYZ', 'XYZ'), + ('WMS', 'WMS'), + ('ArcRest', 'ArcRest'), + ('ArcFeatureServer', 'ArcFeatureServer'), + ('radio', 'radio'), + ('checkbox', 'checkbox'), + ('Vector', 'Vector'), + ('VectorTile', 'VectorTile'), + ('placeholder', 'placeholder'), +) -GEOJSON_DOWNLOAD = True # force headers to treat like an attachment -SUPPORT_INVERTED_COORDINATES = False +# Region defaults (can be overridden by PROJECT settings or config.ini) +PROJECT_REGION: dict = {} +PROJECT_REGION = { + 'name': region_cfg.get('NAME', PROJECT_REGION.get('name', 'Mid-Atlantic Ocean')), + 'init_zoom': region_cfg.getint('INIT_ZOOM', PROJECT_REGION.get('init_zoom', 7)), + 'init_lat': region_cfg.getfloat('INIT_LAT', PROJECT_REGION.get('init_lat', 39.0)), + 'init_lon': region_cfg.getfloat('INIT_LON', PROJECT_REGION.get('init_lon', -74.0)), + 'srid': region_cfg.getint('SRID', PROJECT_REGION.get('srid', 4326)), + 'map': region_cfg.get('MAP', PROJECT_REGION.get('map', 'ocean')), + 'max_zoom': region_cfg.getint('MAX_ZOOM', PROJECT_REGION.get('max_zoom', 13)), +} + +# WMS proxy settings +WMS_PROXY = 'http://tiles.ecotrust.org/mapserver/' +WMS_PROXY_MAPFILE_FIELD = 'map' +WMS_PROXY_MAPFILE = '/mapfiles/generic.map' +WMS_PROXY_LAYERNAME = 'LAYERNAME' +WMS_PROXY_CONNECTION = 'CONN' +WMS_PROXY_FORMAT = 'FORMAT' +WMS_PROXY_VERSION = 'VERSION' +WMS_PROXY_SOURCE_SRS = 'SOURCESRS' +WMS_PROXY_SOURCE_STYLE = 'SRCSTYLE' +WMS_PROXY_TIME_EXTENT = 'TIMEEXT' +WMS_PROXY_TIME = 'TIME' +WMS_PROXY_TIME_DEFAULT = 'TIMEDEF' +WMS_PROXY_TIME_ITEM = 'TIMEITEM' +WMS_PROXY_GENERIC_LAYER = 'generic' +WMS_PROXY_TIME_LAYER = 'time' -# authentication +# --------------------------------------------------------------------------- +# Sharing / features +# --------------------------------------------------------------------------- +DATA_MANAGER_ADMIN = False +SHARING_TO_PUBLIC_GROUPS = ['Share with Public'] +SHARING_TO_STAFF_GROUPS = ['Share with Staff'] +FEEDBACK_IFRAME_URL = ( + "//docs.google.com/forms/d/e/" + "1FAIpQLSdi0nBoQK-3ia8rKtzh7cif0slzDCjA_ACH9Y_ryam-co6p8A/viewform?usp=sf_link" +) +DISCLAIMER_BUTTON_DEFAULT = False + +# --------------------------------------------------------------------------- +# Social Authentication +# --------------------------------------------------------------------------- SOCIAL_AUTH_NEW_USER_URL = '/account/?new=true&login=django' -SOCIAL_AUTH_FACBEOOK_NEW_USER_URL = '/account/?new=true&login=facebook' -# SOCIAL_AUTH_GOOGLE_PLUS_NEW_USER_URL = '/account/?new=true&login=gplus' +SOCIAL_AUTH_FACEBOOK_NEW_USER_URL = '/account/?new=true&login=facebook' SOCIAL_AUTH_TWITTER_NEW_USER_URL = '/account/?new=true&login=twitter' SOCIAL_AUTH_GOOGLE_NEW_USER_URL = '/account/?new=true&login=google' SOCIAL_AUTH_LOGIN_REDIRECT_URL = '/account/?login=django' -# SOCIAL_AUTH_GOOGLE_PLUS_LOGIN_REDIRECT_URL = '/account/?login=gplus' SOCIAL_AUTH_FACEBOOK_LOGIN_REDIRECT_URL = '/account/?login=facebook' SOCIAL_AUTH_TWITTER_LOGIN_REDIRECT_URL = '/account/?login=twitter' SOCIAL_AUTH_GOOGLE_LOGIN_REDIRECT_URL = '/account/?login=google' -# SOCIAL_AUTH_GOOGLE_PLUS_KEY = '' -# SOCIAL_AUTH_GOOGLE_PLUS_SECRET = '' -# SOCIAL_AUTH_GOOGLE_PLUS_SCOPES = ( -# 'https://www.googleapis.com/auth/plus.login', # Minimum needed to login -# 'https://www.googleapis.com/auth/plus.profile.emails.read', # emails -# ) - -if 'SOCIAL_AUTH' not in cfg.sections(): - cfg['SOCIAL_AUTH'] = {} - -social_cfg = cfg['SOCIAL_AUTH'] - -SOCIAL_AUTH_FACEBOOK_KEY = social_cfg.get('FACEBOOK_KEY', '') -SOCIAL_AUTH_FACEBOOK_SECRET = social_cfg.get('FACEBOOK_SECRET', '') +# Env var overrides: FACEBOOK_KEY, FACEBOOK_SECRET, TWITTER_KEY, +# TWITTER_SECRET, GOOGLE_KEY, GOOGLE_SECRET +SOCIAL_AUTH_FACEBOOK_KEY = env_str('FACEBOOK_KEY', social_cfg, 'FACEBOOK_KEY', '') +SOCIAL_AUTH_FACEBOOK_SECRET = env_str('FACEBOOK_SECRET', social_cfg, 'FACEBOOK_SECRET', '') SOCIAL_AUTH_FACEBOOK_SCOPE = ['public_profile,email'] -SOCIAL_AUTH_TWITTER_KEY = social_cfg.get('TWITTER_KEY', '') -SOCIAL_AUTH_TWITTER_SECRET = social_cfg.get('TWITTER_SECRET', '') - -SOCIAL_AUTH_GOOGLE_OAUTH2_KEY = social_cfg.get('GOOGLE_KEY', '') -SOCIAL_AUTH_GOOGLE_OAUTH2_SECRET = social_cfg.get('GOOGLE_SECRET', '') -#SOCIAL_AUTH_GOOGLE_OAUTH2_SCOPE = [] -SOCIAL_AUTH_GOOGLE_OAUTH2_USE_DEPRECATED_API = True +SOCIAL_AUTH_TWITTER_KEY = env_str('TWITTER_KEY', social_cfg, 'TWITTER_KEY', '') +SOCIAL_AUTH_TWITTER_SECRET = env_str('TWITTER_SECRET', social_cfg, 'TWITTER_SECRET', '') -# SOCIAL_AUTH_EMAIL_FORCE_EMAIL_VALIDATION = True -SOCIAL_AUTH_EMAIL_VALIDATION_FUNCTION = 'accounts.pipeline.send_validation_email' -SOCIAL_AUTH_EMAIL_VALIDATION_URL = '/account/validate' +SOCIAL_AUTH_GOOGLE_OAUTH2_KEY = env_str('GOOGLE_KEY', social_cfg, 'GOOGLE_KEY', '') +SOCIAL_AUTH_GOOGLE_OAUTH2_SECRET = env_str('GOOGLE_SECRET', social_cfg, 'GOOGLE_SECRET', '') SOCIAL_AUTH_DISCONNECT_REDIRECT_URL = '/' - SOCIAL_AUTH_JSONFIELD_ENABLED = True +SOCIAL_AUTH_EMAIL_VALIDATION_FUNCTION = 'accounts.pipeline.send_validation_email' +SOCIAL_AUTH_EMAIL_VALIDATION_URL = '/account/validate' -# Our authentication pipeline SOCIAL_AUTH_PIPELINE = ( 'accounts.pipeline.clean_session', - - # Get the information we can about the user and return it in a simple - # format to create the user instance later. On some cases the details are - # already part of the auth response from the provider, but sometimes this - # could hit a provider API. - 'social.pipeline.social_auth.social_details', - - # Get the social uid from whichever service we're authing thru. The uid is - # the unique identifier of the given user in the provider. - 'social.pipeline.social_auth.social_uid', - - # Verifies that the current auth process is valid within the current - # project, this is were emails and domains whitelists are applied (if - # defined). - 'social.pipeline.social_auth.auth_allowed', - - # Checks if the current social-account is already associated in the site. - 'social.pipeline.social_auth.social_user', - - # Make up a username for this person, appends a random string at the end if - # there's any collision. - 'social.pipeline.user.get_username', - - # Confirm with the user that they really want to make an account, also - # make them enter an email address if they somehow didn't - # 'accounts.pipeline.confirm_account', - - # Send a validation email to the user to verify its email address. - 'social.pipeline.mail.mail_validation', - - # Associates the current social details with another user account with - # a similar email address. Disabled by default. - # 'social.pipeline.social_auth.associate_by_email', - - # Create a user account if we haven't found one yet. - 'social.pipeline.user.create_user', - - # Create the record that associated the social account with this user. - 'social.pipeline.social_auth.associate_user', - - # Populate the extra_data field in the social record with the values - # specified by settings (and the default ones like access_token, etc). - 'social.pipeline.social_auth.load_extra_data', - - # Update the user record with any changed info from the auth service. - 'social.pipeline.user.user_details', - - # Set up default django permission groups for new users. + # social-auth-core pipeline steps (social_core.pipeline.*) + 'social_core.pipeline.social_auth.social_details', + 'social_core.pipeline.social_auth.social_uid', + 'social_core.pipeline.social_auth.auth_allowed', + 'social_core.pipeline.social_auth.social_user', + 'social_core.pipeline.user.get_username', + 'social_core.pipeline.mail.mail_validation', + 'social_core.pipeline.user.create_user', + 'social_core.pipeline.social_auth.associate_user', + 'social_core.pipeline.social_auth.load_extra_data', + 'social_core.pipeline.user.user_details', 'accounts.pipeline.set_user_permissions', - - # Grab relevant information from the social provider (avatar) 'accounts.pipeline.get_social_details', - - # 'social.pipeline.debug.debug', 'accounts.pipeline.clean_session', ) -if 'EMAIL' not in cfg.sections(): - cfg['EMAIL'] = {} - -email_cfg = cfg['EMAIL'] - -EMAIL_HOST = email_cfg.get('HOST', 'localhost') -EMAIL_PORT = email_cfg.getint('PORT', 25) -if cfg.has_option('EMAIL', 'HOST_USER') and \ - cfg.has_option('EMAIL', 'HOST_PASSWORD'): - EMAIL_HOST_USER = email_cfg.get('HOST_USER') - EMAIL_HOST_PASSWORD = email_cfg.get('HOST_PASSWORD') -else: - EMAIL_HOST_USER = '' - EMAIL_HOST_PASSWORD = '' - -EMAIL_BACKEND = email_cfg.get('EMAIL_BACKEND', 'django.core.mail.backends.smtp.EmailBackend') - -DEFAULT_FROM_EMAIL = email_cfg.get('DEFAULT_FROM_EMAIL', "MARCO Portal Team ") -SERVER_EMAIL = email_cfg.get('SERVER_EMAIL', "MARCO Site Errors ") -EMAIL_USE_TLS = email_cfg.getboolean('EMAIL_USE_TLS', False) -# for mail to admins/managers only -EMAIL_SUBJECT_PREFIX = app_cfg.get('EMAIL_SUBJECT_PREFIX', '[MARCO]') + ' ' - -if 'AWS' not in cfg.sections(): - cfg['AWS'] = {} - -aws_cfg = cfg['AWS'] - -AWS_ACCESS_KEY_ID = aws_cfg.get('AWS_ACCESS_KEY_ID','') -AWS_SECRET_ACCESS_KEY = aws_cfg.get('AWS_SECRET_ACCESS_KEY','') -AWS_SES_REGION_NAME = aws_cfg.get('AWS_SES_REGION_NAME', 'us-east-1') -AWS_SES_REGION_ENDPOINT = aws_cfg.get('AWS_SES_REGION_ENDPOINT','email.us-east-1.amazonaws.com') - - -if 'CELERY' not in cfg.sections(): - cfg['CELERY'] = {} - -celery_cfg = cfg['CELERY'] - -CELERY_RESULT_BACKEND = celery_cfg.get('CELERY_RESULT_BACKEND', 'redis://localhost:6379') -BROKER_URL = celery_cfg.get('BROKER_URL', 'redis://localhost:6379/0') -CELERY_BROKER_URL = celery_cfg.get('CELERY_BROKER_URL', 'redis://localhost:6379') -CELERY_ALWAYS_EAGER = celery_cfg.get('CELERY_ALWAYS_EAGER', False) -CELERY_DISABLE_RATE_LIMITS = celery_cfg.get('CELERY_DISABLE_RATE_LIMITS', True) - -GA_ACCOUNT = app_cfg.get('GA_ACCOUNT', '') +# --------------------------------------------------------------------------- +# Email +# Env var overrides: EMAIL_HOST, EMAIL_PORT, EMAIL_HOST_USER, +# EMAIL_HOST_PASSWORD, EMAIL_USE_TLS +# --------------------------------------------------------------------------- +EMAIL_HOST = env_str('EMAIL_HOST', email_cfg, 'HOST', 'localhost') +EMAIL_PORT = env_int('EMAIL_PORT', email_cfg, 'PORT', 25) +EMAIL_HOST_USER = env_str('EMAIL_HOST_USER', email_cfg, 'HOST_USER', '') +EMAIL_HOST_PASSWORD = env_str('EMAIL_HOST_PASSWORD', email_cfg, 'HOST_PASSWORD', '') +EMAIL_BACKEND = env_str('EMAIL_BACKEND', email_cfg, 'EMAIL_BACKEND', 'django.core.mail.backends.smtp.EmailBackend') +DEFAULT_FROM_EMAIL = env_str('DEFAULT_FROM_EMAIL', email_cfg, 'DEFAULT_FROM_EMAIL', 'MARCO Portal Team ') +SERVER_EMAIL = env_str('SERVER_EMAIL', email_cfg, 'SERVER_EMAIL', "MARCO Site Errors ") +EMAIL_USE_TLS = env_bool('EMAIL_USE_TLS', email_cfg, 'EMAIL_USE_TLS', False) +EMAIL_SUBJECT_PREFIX = env_str('EMAIL_SUBJECT_PREFIX', app_cfg, 'EMAIL_SUBJECT_PREFIX', '[MARCO]') + ' ' ADMINS = (('KSDev', 'ksdev@ecotrust.org'),) -NOCAPTCHA = True -RECAPTCHA_PUBLIC_KEY = app_cfg.get('RECAPTCHA_PUBLIC_KEY', '') -RECAPTCHA_PRIVATE_KEY = app_cfg.get('RECAPTCHA_PRIVATE_KEY','') - -# OL2 doesn't support reprojecting rasters, so for WMS servers that don't provide -# EPSG:3857 we send it to a proxy to be re-projected. -WMS_PROXY = 'http://tiles.ecotrust.org/mapserver/' -WMS_PROXY_MAPFILE_FIELD = 'map' -WMS_PROXY_MAPFILE = '/mapfiles/generic.map' -WMS_PROXY_LAYERNAME = 'LAYERNAME' -WMS_PROXY_CONNECTION = 'CONN' -WMS_PROXY_FORMAT = 'FORMAT' -WMS_PROXY_VERSION = 'VERSION' -WMS_PROXY_SOURCE_SRS = 'SOURCESRS' -WMS_PROXY_SOURCE_STYLE = 'SRCSTYLE' -WMS_PROXY_TIME_EXTENT = 'TIMEEXT' -WMS_PROXY_TIME = 'TIME' -WMS_PROXY_TIME_DEFAULT = 'TIMEDEF' -WMS_PROXY_TIME_ITEM = 'TIMEITEM' -WMS_PROXY_GENERIC_LAYER = 'generic' -WMS_PROXY_TIME_LAYER = 'time' - -MAP_LIBRARY = app_cfg.get('MAP_LIBRARY', 'ol6') - -if 'REGION' not in cfg.sections(): - cfg['REGION'] = {} - -region_cfg = cfg['REGION'] - -try: - # Test is PROJECT_REGION was already defined by PROJECT settings. - PROJECT_REGION -except NameError: - PROJECT_REGION = {} - -PROJECT_REGION = { - 'name': region_cfg.get('NAME', PROJECT_REGION['name'] if PROJECT_REGION and 'name' in PROJECT_REGION.keys() else 'Mid-Atlantic Ocean'), - 'init_zoom': region_cfg.getint('INIT_ZOOM', PROJECT_REGION['init_zoom'] if PROJECT_REGION and 'init_zoom' in PROJECT_REGION.keys() else 7), - 'init_lat': region_cfg.getint('INIT_LAT', PROJECT_REGION['init_lat'] if PROJECT_REGION and 'init_lat' in PROJECT_REGION.keys() else 39), - 'init_lon': region_cfg.getint('INIT_LON', PROJECT_REGION['init_lon'] if PROJECT_REGION and 'init_lon' in PROJECT_REGION.keys() else -74), - 'srid': region_cfg.getint('SRID', PROJECT_REGION['srid'] if PROJECT_REGION and 'srid' in PROJECT_REGION.keys() else 4326), - 'map': region_cfg.get('MAP', PROJECT_REGION['map'] if PROJECT_REGION and 'map' in PROJECT_REGION.keys() else 'ocean'), - 'max_zoom': region_cfg.getint('MAX_ZOOM', PROJECT_REGION['max_zoom'] if PROJECT_REGION and 'max_zoom' in PROJECT_REGION.keys() else 13), -} +# --------------------------------------------------------------------------- +# AWS (SES) +# Env var overrides: AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, +# AWS_SES_REGION_NAME, AWS_SES_REGION_ENDPOINT +# --------------------------------------------------------------------------- +AWS_ACCESS_KEY_ID = env_str('AWS_ACCESS_KEY_ID', aws_cfg, 'AWS_ACCESS_KEY_ID', '') +AWS_SECRET_ACCESS_KEY = env_str('AWS_SECRET_ACCESS_KEY', aws_cfg, 'AWS_SECRET_ACCESS_KEY', '') +AWS_SES_REGION_NAME = env_str('AWS_SES_REGION_NAME', aws_cfg, 'AWS_SES_REGION_NAME', 'us-east-1') +AWS_SES_REGION_ENDPOINT = env_str('AWS_SES_REGION_ENDPOINT', aws_cfg, 'AWS_SES_REGION_ENDPOINT', 'email.us-east-1.amazonaws.com') + +# --------------------------------------------------------------------------- +# Celery (Celery 5+ settings) +# Env var overrides: CELERY_BROKER_URL, CELERY_RESULT_BACKEND +# --------------------------------------------------------------------------- +CELERY_BROKER_URL = ( + os.environ.get('CELERY_BROKER_URL') + or os.environ.get('REDIS_URL') + or celery_cfg.get('CELERY_BROKER_URL', 'redis://localhost:6379/0') +) +CELERY_RESULT_BACKEND = ( + os.environ.get('CELERY_RESULT_BACKEND') + or os.environ.get('REDIS_URL') + or celery_cfg.get('CELERY_RESULT_BACKEND', 'redis://localhost:6379') +) +CELERY_TASK_ALWAYS_EAGER = celery_cfg.getboolean('CELERY_ALWAYS_EAGER', False) +CELERY_TASK_RATE_LIMITS_DISABLED = celery_cfg.getboolean('CELERY_DISABLE_RATE_LIMITS', True) -PROJECT_APP = app_cfg.get('PROJECT_APP', False) -if PROJECT_APP and not PROJECT_APP == 'False': +# --------------------------------------------------------------------------- +# ReCAPTCHA +# --------------------------------------------------------------------------- +NOCAPTCHA = True +RECAPTCHA_PUBLIC_KEY = env_str('RECAPTCHA_PUBLIC_KEY', app_cfg, 'RECAPTCHA_PUBLIC_KEY', '') +RECAPTCHA_PRIVATE_KEY = env_str('RECAPTCHA_PRIVATE_KEY', app_cfg, 'RECAPTCHA_PRIVATE_KEY', '') + +# --------------------------------------------------------------------------- +# Analytics +# --------------------------------------------------------------------------- +GA_ACCOUNT = env_str('GA_ACCOUNT', app_cfg, 'GA_ACCOUNT', '') + +# --------------------------------------------------------------------------- +# NATIVE LANDS API KEY +# --------------------------------------------------------------------------- +NATIVE_LAND_API_KEY = env_str('NATIVE_LAND_API_KEY', app_cfg, 'NATIVE_LAND_API_KEY', '') + +# --------------------------------------------------------------------------- +# Project-level settings overrides +# (Optional app + settings file specified in config.ini) +# --------------------------------------------------------------------------- +PROJECT_APP = env_str('PROJECT_APP', app_cfg, 'PROJECT_APP', '') +if PROJECT_APP: INSTALLED_APPS.append(PROJECT_APP) if 'visualize' in INSTALLED_APPS: - from visualize.settings import * + from visualize.settings import * # noqa: F401, F403 if 'data_manager' in INSTALLED_APPS: - from data_manager.settings import * - -DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' + from data_manager.settings import * # noqa: F401, F403 -SERVER_SRID = 4326 - -PROJECT_SETTINGS_FILE = app_cfg.get('PROJECT_SETTINGS_FILE', False) -if PROJECT_SETTINGS_FILE and not PROJECT_SETTINGS_FILE == 'False': +PROJECT_SETTINGS_FILE = env_bool('PROJECT_SETTINGS_FILE', app_cfg, 'PROJECT_SETTINGS_FILE', '') +if PROJECT_SETTINGS_FILE: try: from importlib import import_module - APP_MODULE = import_module(PROJECT_APP) - exec("from %s.settings import *" % APP_MODULE.__package__) - except Exception as e: - print(e) - print('PROJECT APP (%s) settings not imported' % PROJECT_APP) -try: - ADDITIONAL_APPS = eval(app_cfg.get('ADDITIONAL_APPS', [])) -except Exception as e: - ADDITIONAL_APPS = [] -try: - ADDITIONAL_MIDDLEWARE = eval(app_cfg.get('ADDITIONAL_MIDDLEWARE', [])) -except Exception as e: - ADDITIONAL_MIDDLEWARE = [] + _project_module = import_module(PROJECT_APP) + _settings_module = import_module(f"{_project_module.__package__}.settings") + # Merge all public names into the current module's namespace + for _k, _v in vars(_settings_module).items(): + if not _k.startswith('_'): + globals()[_k] = _v + except Exception as _e: + import warnings + warnings.warn(f"PROJECT APP ({PROJECT_APP}) settings not imported: {_e}") + +# ADDITIONAL_APPS / ADDITIONAL_MIDDLEWARE — expected as JSON arrays in config.ini +# e.g.: ADDITIONAL_APPS = ["my_custom_app"] +def _parse_list_setting(raw: str) -> list: + """Safely parse a JSON or Python list literal from a config value.""" + if not raw: + return [] + try: + result = json.loads(raw) + if isinstance(result, list): + return result + except (json.JSONDecodeError, TypeError): + pass + try: + result = ast.literal_eval(raw) + if isinstance(result, list): + return result + except (SyntaxError, ValueError): + pass + return [] + +ADDITIONAL_APPS = _parse_list_setting(app_cfg.get('ADDITIONAL_APPS', '')) +ADDITIONAL_MIDDLEWARE = _parse_list_setting(app_cfg.get('ADDITIONAL_MIDDLEWARE', '')) INSTALLED_APPS += ADDITIONAL_APPS MIDDLEWARE += ADDITIONAL_MIDDLEWARE - -if False: - MIDDLEWARE += ['debug_toolbar.middleware.DebugToolbarMiddleware',] - INSTALLED_APPS += ['debug_toolbar',] - DEBUG_TOOLBAR_PANELS2 = [ - 'debug_toolbar.panels.cache.CachePanel', - 'debug_toolbar.panels.headers.HeadersPanel', - 'debug_toolbar.panels.logging.LoggingPanel', - 'debug_toolbar.panels.profiling.ProfilingPanel', - 'debug_toolbar.panels.redirects.RedirectsPanel', - 'debug_toolbar.panels.request.RequestPanel', - 'debug_toolbar.panels.settings.SettingsPanel', - 'debug_toolbar.panels.signals.SignalsPanel', - 'debug_toolbar.panels.staticfiles.StaticFilesPanel', - 'debug_toolbar.templates.panel.TemplatesPanel', - 'debug_toolbar.panels.timer.TimerPanel', - # 'debug_toolbar.sql.panel.SQLPanel', - ] - import debug_toolbar.panels diff --git a/marco/marco/tests/__init__.py b/marco/marco/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/marco/marco/tests/test_api_url_discovery.py b/marco/marco/tests/test_api_url_discovery.py new file mode 100644 index 00000000..0690d835 --- /dev/null +++ b/marco/marco/tests/test_api_url_discovery.py @@ -0,0 +1,18 @@ +from django.test import SimpleTestCase +from django.urls import resolve + + +class ApiUrlDiscoveryTests(SimpleTestCase): + """Ensure API routes from installed apps are auto-mounted under /api/.""" + + def test_visualize_api_route_resolves(self): + match = resolve('/api/bookmarks/') + self.assertEqual(match.func.view_class.__name__, 'BookmarkListView') + + def test_drawing_api_route_resolves(self): + match = resolve('/api/drawings/testuid123/') + self.assertEqual(match.func.view_class.__name__, 'DrawingDeleteView') + + def test_mapgroups_api_route_resolves(self): + match = resolve('/api/sharing-groups/') + self.assertEqual(match.func.view_class.__name__, 'SharingGroupListView') diff --git a/marco/marco/tests/test_config_helpers.py b/marco/marco/tests/test_config_helpers.py new file mode 100644 index 00000000..0f8aed5f --- /dev/null +++ b/marco/marco/tests/test_config_helpers.py @@ -0,0 +1,66 @@ +import configparser + +import pytest + +from marco.config_helpers import env_bool, env_int, env_str, parse_bool + + +def _section(values: dict[str, str]) -> configparser.SectionProxy: + cfg = configparser.ConfigParser() + cfg["APP"] = values + return cfg["APP"] + + +def test_env_str_prefers_environment(monkeypatch: pytest.MonkeyPatch) -> None: + section = _section({"APP_NAME": "FromConfig"}) + monkeypatch.setenv("APP_NAME", "FromEnv") + assert env_str("APP_NAME", section, "APP_NAME", "Default") == "FromEnv" + + +def test_env_str_falls_back_to_config_then_default(monkeypatch: pytest.MonkeyPatch) -> None: + section = _section({"APP_NAME": "FromConfig"}) + monkeypatch.delenv("APP_NAME", raising=False) + assert env_str("APP_NAME", section, "APP_NAME", "Default") == "FromConfig" + assert env_str("MISSING", section, "MISSING", "Default") == "Default" + + +def test_env_bool_prefers_environment(monkeypatch: pytest.MonkeyPatch) -> None: + section = _section({"DEBUG": "false"}) + monkeypatch.setenv("DEBUG", "true") + assert env_bool("DEBUG", section, "DEBUG", False) is True + + +def test_env_bool_falls_back_to_default(monkeypatch: pytest.MonkeyPatch) -> None: + section = _section({}) + monkeypatch.delenv("DEBUG", raising=False) + assert env_bool("DEBUG", section, "DEBUG", False) is False + assert env_bool("DEBUG", section, "DEBUG", True) is True + + +def test_parse_bool_accepts_common_values() -> None: + assert parse_bool("1") is True + assert parse_bool("yes") is True + assert parse_bool("ON") is True + assert parse_bool("0") is False + assert parse_bool("no") is False + assert parse_bool("off") is False + + +def test_parse_bool_rejects_invalid_values() -> None: + with pytest.raises(ValueError): + parse_bool("maybe", setting_name="DEBUG") + + +def test_env_int_prefers_environment_and_parses(monkeypatch: pytest.MonkeyPatch) -> None: + section = _section({"PORT": "25"}) + monkeypatch.setenv("EMAIL_PORT", "587") + assert env_int("EMAIL_PORT", section, "PORT", 25) == 587 + + +def test_env_int_falls_back_to_config_then_default(monkeypatch: pytest.MonkeyPatch) -> None: + section = _section({"PORT": "2525"}) + monkeypatch.delenv("EMAIL_PORT", raising=False) + assert env_int("EMAIL_PORT", section, "PORT", 25) == 2525 + + empty_section = _section({}) + assert env_int("EMAIL_PORT", empty_section, "PORT", 25) == 25 diff --git a/marco/marco/urls.py b/marco/marco/urls.py index 432d48af..24ef5068 100644 --- a/marco/marco/urls.py +++ b/marco/marco/urls.py @@ -1,115 +1,141 @@ -import os -try: - from django.urls import re_path, include -except (ModuleNotFoundError, ImportError): - from django.conf.urls import include, url as re_path -from django.conf.urls.static import static +""" +URL configuration for the Madrona Portal project. + +Requires: Django 4.2+, Wagtail 7.0+ +""" +import warnings +from importlib import import_module +from importlib.util import find_spec + from django.conf import settings +from django.conf.urls.static import static from django.contrib import admin +from django.urls import include, re_path +from django.views.generic.base import RedirectView -from django.views.generic.base import RedirectView, TemplateView - -if settings.WAGTAIL_VERSION > 1: - from wagtail.admin import urls as wagtailadmin_urls - from wagtail.documents import urls as wagtaildocs_urls - from wagtail import urls as wagtail_urls - from wagtail.contrib.sitemaps.views import sitemap - from wagtail.images import urls as wagtailimages_urls - # Register search signal handlers - from wagtail.search.signal_handlers import register_signal_handlers as wagtailsearch_register_signal_handlers -else: - from wagtail.admin import urls as wagtailadmin_urls - from wagtail.docs import urls as wagtaildocs_urls - from wagtail import urls as wagtail_urls - from wagtail.contrib.sitemaps.views import sitemap - from wagtail.images import urls as wagtailimages_urls - # Register search signal handlers - from wagtail.search.signal_handlers import register_signal_handlers as wagtailsearch_register_signal_handlers - -# from wagtailimportexport import urls as wagtailimportexport_urls +from wagtail.admin import urls as wagtailadmin_urls +from wagtail.documents import urls as wagtaildocs_urls +from wagtail import urls as wagtail_urls +from wagtail.contrib.sitemaps.views import sitemap +from wagtail.images import urls as wagtailimages_urls +from wagtail.search.signal_handlers import register_signal_handlers as wagtailsearch_register_signal_handlers import mapgroups.urls import accounts.urls import explore.urls -from rpc4django.views import serve_rpc_request -from social.apps import django_app +from marco.rpc_compat import rpc_view + from portal.base import views as base_views from portal.data_catalog import views as data_catalog_views from marco_site import views as marco_site_views admin.autodiscover() +wagtailsearch_register_signal_handlers() -wagtailsearch_register_signal_handlers() +def _iter_discovered_api_includes(): + """Yield include() patterns for apps exposing urls.api_urlpatterns.""" + for installed_app in settings.INSTALLED_APPS: + app_module = installed_app.split('.apps.', 1)[0] + urls_module_name = f"{app_module}.urls" + + try: + if find_spec(urls_module_name) is None: + continue + except (ImportError, ValueError, AttributeError): + continue + + try: + app_urls = import_module(urls_module_name) + except Exception as exc: # pragma: no cover - defensive runtime warning + warnings.warn(f"Could not import '{urls_module_name}' for api_urlpatterns: {exc}") + continue + + app_api_urlpatterns = getattr(app_urls, 'api_urlpatterns', None) + if app_api_urlpatterns: + yield re_path(r'^api/', include(app_api_urlpatterns)) -try: - from importlib import import_module - portal_app_urls = import_module(settings.PROJECT_APP + '.urls') - urlpatterns = portal_app_urls.urlpatterns -except Exception as e: - urlpatterns = [] +api_url_includes = list(_iter_discovered_api_includes()) +# --------------------------------------------------------------------------- +# Project-specific URL patterns +# Optional: a project app can prepend its own patterns. +# --------------------------------------------------------------------------- +urlpatterns: list = [] + +if settings.PROJECT_APP: + try: + portal_app_urls = import_module(f"{settings.PROJECT_APP}.urls") + urlpatterns = list(getattr(portal_app_urls, 'urlpatterns', [])) + except (ImportError, AttributeError) as e: + warnings.warn(f"Could not load URL patterns from PROJECT_APP '{settings.PROJECT_APP}': {e}") + +# --------------------------------------------------------------------------- +# Core URL patterns +# --------------------------------------------------------------------------- urlpatterns += [ - #'', re_path(r'^sitemap\.xml$', sitemap), - re_path(r'django-admin/?', admin.site.urls), - - re_path(r'^rpc$', serve_rpc_request), + re_path(r'^django-admin/', admin.site.urls), + re_path(r'^admin/', include(wagtailadmin_urls)), - # https://github.com/omab/python-social-auth/issues/399 - # I want the psa urls to be inside the account urls, but PSA doesn't allow - # nested namespaces. It will likely be fixed in 0.22 + # /rpc — JSON-RPC 2.0 compat shim for legacy frontend JS (see rpc_compat.py) + re_path(r'^rpc/', rpc_view), +] - # re_path(r'^account/auth/', include('social.apps.django_app.urls'), name='social'), - # url('^account/auth/', include('social_django.urls', namespace='social')), - re_path(r'^account/?', include('accounts.urls'), name='account'), - re_path(r'^collaborate/groups/?', include('mapgroups.urls'), name='groups'), - re_path(r'^groups/?', include('mapgroups.urls'), name='groups'), - re_path(r'^g/?', RedirectView.as_view(url='/groups/')), # 301 +urlpatterns += api_url_includes - re_path(r'^admin/?', include(wagtailadmin_urls)), - re_path(r'^search/?', base_views.search), - re_path(r'^documents/?', include(wagtaildocs_urls)), +urlpatterns += [ + # DRF REST replacements discovered from each sub-app's urls.py api_urlpatterns - # url(r'^data-catalog/', include('portal.data_catalog.urls')), - # TODO: we need to prevent Theme names with spaces or special characters. - re_path(r'^data-catalog/([\w\-\s\(\)]+)/?$', data_catalog_views.theme, name="portal.data_catalog.views.theme"), - re_path(r'^data-catalog/[\w\-\s\(\)]*/?', include('explore.urls')), - re_path(r'^data_manager/?', include('layers.urls')), - re_path(r'^old_manager/?', include('data_manager.urls')), - # re_path(r'^data_manager/', include('data_manager.urls')), - re_path(r'^url_shortener/?', include('url_short.urls')), - re_path(r'^layers/?', include('layers.urls')), - re_path(r'^styleguide/?$', marco_site_views.styleguide, name='styleguide'), - re_path(r'^planner/?', include('visualize.urls')), - re_path(r'^embed/?', include('visualize.urls')), - re_path(r'^visualize/?', include('visualize.urls')), - re_path(r'^features/?', include('features.urls')), - re_path(r'^scenario/?', include('scenarios.urls')), - re_path(r'^drawing/?', include('drawing.urls')), - re_path(r'^proxy/?', include('mp_proxy.urls')), - - re_path(r'^join/?', RedirectView.as_view(url='/account/register/')), + re_path(r'^auth/', include('social_django.urls', namespace='social')), + re_path(r'^account/', include('accounts.urls'), name='account'), + re_path(r'^collaborate/groups/', include('mapgroups.urls'), name='groups'), + re_path(r'^groups/', include('mapgroups.urls'), name='groups'), + re_path(r'^g/', RedirectView.as_view(url='/groups/')), # 301 legacy redirect + re_path(r'^search/', base_views.search), + re_path(r'^documents/', include(wagtaildocs_urls)), re_path(r'^images/', include(wagtailimages_urls)), + + # Data catalog: named theme pages then the explore SPA + # TODO (POR-206): Restrict theme slugs to prevent spaces and special characters. + re_path(r'^data-catalog/([\w\-\s\(\)]+)/?$', data_catalog_views.theme, name="portal.data_catalog.views.theme"), + re_path(r'^data-catalog/[\w\-\s\(\)]*/', include('explore.urls')), + + re_path(r'^data_manager/', include('layers.urls')), + re_path(r'^old_manager/', include('data_manager.urls')), + re_path(r'^url_shortener/', include('url_short.urls')), + re_path(r'^layers/', include('layers.urls')), + re_path(r'^styleguide/$', marco_site_views.styleguide, name='styleguide'), + re_path(r'^planner/', include('visualize.urls')), + re_path(r'^embed/', include('visualize.urls')), + re_path(r'^visualize/', include('visualize.urls')), + re_path(r'^features/', include('features.urls')), + re_path(r'^scenario/', include('scenarios.urls')), + re_path(r'^drawing/', include('drawing.urls')), + re_path(r'^proxy/', include('mp_proxy.urls')), + + re_path(r'^join/', RedirectView.as_view(url='/account/register/')), # 301 legacy redirect ] +# Optional survey module if 'survey' in settings.INSTALLED_APPS: urlpatterns += [ - re_path(r'^survey/?', include('survey.urls', namespace='survey')), + re_path(r'^survey/', include('survey.urls', namespace='survey')), ] +# Custom 404 handler if hasattr(settings, 'HANDLER_404'): handler404 = settings.HANDLER_404 - +# Development: serve static and media files through Django if settings.DEBUG: from django.contrib.staticfiles.urls import staticfiles_urlpatterns - urlpatterns += staticfiles_urlpatterns() urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) -urlpatterns += [re_path(r'', include(wagtail_urls)),] +# Wagtail page routing — must be last +urlpatterns += [re_path(r'', include(wagtail_urls))] diff --git a/marco/marco_site/static/js/jsonrpc.js b/marco/marco_site/static/js/jsonrpc.js index 97d63c04..65e57cc9 100644 --- a/marco/marco_site/static/js/jsonrpc.js +++ b/marco/marco_site/static/js/jsonrpc.js @@ -60,7 +60,7 @@ function jsonrpc_call(method, args, options) { request_encoded = JSON.stringify(request); $.ajax({ - url: '/rpc', + url: '/rpc/', method: 'POST', data: request_encoded, dataType: 'json', diff --git a/marco/portal/base/migrations/0002_auto_20200526_2354.py b/marco/portal/base/migrations/0002_auto_20200526_2354.py index ba83f55c..9ca031f8 100644 --- a/marco/portal/base/migrations/0002_auto_20200526_2354.py +++ b/marco/portal/base/migrations/0002_auto_20200526_2354.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- # Generated by Django 1.9.13 on 2020-05-26 23:54 -from __future__ import unicode_literals from django.db import migrations, models import django.db.models.deletion diff --git a/marco/portal/base/migrations/0003_auto_20200526_2357.py b/marco/portal/base/migrations/0003_auto_20200526_2357.py index 036f1a95..3e2c0fb1 100644 --- a/marco/portal/base/migrations/0003_auto_20200526_2357.py +++ b/marco/portal/base/migrations/0003_auto_20200526_2357.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- # Generated by Django 1.9.13 on 2020-05-26 23:57 -from __future__ import unicode_literals from django.db import migrations # from wagtail.images.utils import get_fill_filter_spec_migrations diff --git a/marco/portal/base/migrations/0004_auto_20200613_0023.py b/marco/portal/base/migrations/0004_auto_20200613_0023.py index 0252a04e..166bacea 100644 --- a/marco/portal/base/migrations/0004_auto_20200613_0023.py +++ b/marco/portal/base/migrations/0004_auto_20200613_0023.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- # Generated by Django 1.10.8 on 2020-06-13 00:23 -from __future__ import unicode_literals from django.db import migrations, models diff --git a/marco/portal/base/models.py b/marco/portal/base/models.py index 9037b221..2a5895bf 100644 --- a/marco/portal/base/models.py +++ b/marco/portal/base/models.py @@ -5,28 +5,11 @@ from django.utils.safestring import mark_safe from django.conf import settings -if settings.WAGTAIL_VERSION > 3: - from wagtail.models import Page - from wagtail.fields import RichTextField, StreamValue - from wagtail.search import index - from wagtail.admin.panels import FieldPanel,MultiFieldPanel,TitleFieldPanel - from wagtail.images.models import AbstractImage, AbstractRendition, Image -elif settings.WAGTAIL_VERSION > 1: - from wagtail.models import Page - from wagtail.fields import RichTextField, StreamValue - from wagtail.search import index - from wagtail.admin.panels import FieldPanel,MultiFieldPanel - from wagtail.images.edit_handlers import ImageChooserPanel - from wagtail.images.models import AbstractImage, AbstractRendition, Image - TitleFieldPanel = FieldPanel -else: - from wagtail.models import Page - from wagtail.fields import RichTextField, StreamValue - from wagtail.search import index - from wagtail.admin.panels import FieldPanel,MultiFieldPanel - from wagtail.images.edit_handlers import ImageChooserPanel - from wagtail.images.models import AbstractImage, AbstractRendition, Image - TitleFieldPanel = FieldPanel +from wagtail.models import Page +from wagtail.fields import RichTextField, StreamValue +from wagtail.search import index +from wagtail.admin.panels import FieldPanel, MultiFieldPanel, TitleFieldPanel +from wagtail.images.models import AbstractImage, AbstractRendition, Image @@ -50,7 +33,7 @@ class PortalImage(AbstractImage): @classmethod def creatable_subpage_models(cls): - print(cls) + pass # Receive the pre_delete signal and delete the file associated with the model instance. @receiver(pre_delete, sender=PortalImage) @@ -60,20 +43,11 @@ def image_delete(sender, instance, **kwargs): class PortalRendition(AbstractRendition): image = models.ForeignKey('PortalImage', related_name='renditions', on_delete=models.CASCADE) - # Wagtail 1.8 deviates drastically from Wagtail 1.7. We need to support both for - # the automated migration from wagtail 1.3 to 2.9 - # TODO: Check if support needed for Wagtail 4.2 https://docs.wagtail.org/en/stable/releases/4.2.html#upgrade-considerations - import wagtail - if hasattr(wagtail, 'VERSION') and wagtail.VERSION[0] > 0 and (wagtail.VERSION[0] > 1 or wagtail.VERSION[1] > 7): - class Meta: - unique_together = ( - ('image', 'filter_spec', 'focal_point_key'), - ) - else: - class Meta: - unique_together = ( - ('image', 'filter_spec', 'focal_point_key'), - ) + + class Meta: + unique_together = ( + ('image', 'filter_spec', 'focal_point_key'), + ) # Receive the pre_delete signal and delete the file associated with the model instance. @receiver(pre_delete, sender=PortalRendition) diff --git a/marco/portal/calendar/migrations/0001_initial.py b/marco/portal/calendar/migrations/0001_initial.py index c2db939f..6a37d023 100644 --- a/marco/portal/calendar/migrations/0001_initial.py +++ b/marco/portal/calendar/migrations/0001_initial.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations import django.db.models.deletion diff --git a/marco/portal/calendar/migrations/0002_auto_20141205_0036.py b/marco/portal/calendar/migrations/0002_auto_20141205_0036.py index d3ac3ad7..9d729776 100644 --- a/marco/portal/calendar/migrations/0002_auto_20141205_0036.py +++ b/marco/portal/calendar/migrations/0002_auto_20141205_0036.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations from django.conf import settings diff --git a/marco/portal/calendar/migrations/0003_auto_20141218_0154.py b/marco/portal/calendar/migrations/0003_auto_20141218_0154.py index 1e592614..faff65c9 100644 --- a/marco/portal/calendar/migrations/0003_auto_20141218_0154.py +++ b/marco/portal/calendar/migrations/0003_auto_20141218_0154.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations import django.db.models.deletion diff --git a/marco/portal/calendar/migrations/0004_event_location.py b/marco/portal/calendar/migrations/0004_event_location.py index 7da9a1bd..ca2275ec 100644 --- a/marco/portal/calendar/migrations/0004_event_location.py +++ b/marco/portal/calendar/migrations/0004_event_location.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations diff --git a/marco/portal/calendar/migrations/0005_auto_20150109_0053.py b/marco/portal/calendar/migrations/0005_auto_20150109_0053.py index 51393158..e1a576fc 100644 --- a/marco/portal/calendar/migrations/0005_auto_20150109_0053.py +++ b/marco/portal/calendar/migrations/0005_auto_20150109_0053.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations diff --git a/marco/portal/calendar/migrations/0006_auto_20150112_2303.py b/marco/portal/calendar/migrations/0006_auto_20150112_2303.py index 37b5343a..dd1ea3f0 100644 --- a/marco/portal/calendar/migrations/0006_auto_20150112_2303.py +++ b/marco/portal/calendar/migrations/0006_auto_20150112_2303.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations from django.conf import settings diff --git a/marco/portal/calendar/migrations/0007_auto_20150121_2313.py b/marco/portal/calendar/migrations/0007_auto_20150121_2313.py index eb929571..f19e4797 100644 --- a/marco/portal/calendar/migrations/0007_auto_20150121_2313.py +++ b/marco/portal/calendar/migrations/0007_auto_20150121_2313.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations import django.db.models.deletion diff --git a/marco/portal/data_catalog/migrations/0001_initial.py b/marco/portal/data_catalog/migrations/0001_initial.py index 0d0fe596..bdebf476 100644 --- a/marco/portal/data_catalog/migrations/0001_initial.py +++ b/marco/portal/data_catalog/migrations/0001_initial.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations import django.db.models.deletion diff --git a/marco/portal/data_catalog/migrations/0002_datacatalog_description.py b/marco/portal/data_catalog/migrations/0002_datacatalog_description.py index df6de020..3f0b858b 100644 --- a/marco/portal/data_catalog/migrations/0002_datacatalog_description.py +++ b/marco/portal/data_catalog/migrations/0002_datacatalog_description.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations from django.conf import settings diff --git a/marco/portal/data_gaps/migrations/0001_initial.py b/marco/portal/data_gaps/migrations/0001_initial.py index bf034f9b..f3f43290 100644 --- a/marco/portal/data_gaps/migrations/0001_initial.py +++ b/marco/portal/data_gaps/migrations/0001_initial.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations import django.db.models.deletion diff --git a/marco/portal/data_gaps/migrations/0002_auto_20141205_0024.py b/marco/portal/data_gaps/migrations/0002_auto_20141205_0024.py index 5890a17a..75cf2e31 100644 --- a/marco/portal/data_gaps/migrations/0002_auto_20141205_0024.py +++ b/marco/portal/data_gaps/migrations/0002_auto_20141205_0024.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations from django.conf import settings diff --git a/marco/portal/data_gaps/migrations/0003_auto_20141217_2301.py b/marco/portal/data_gaps/migrations/0003_auto_20141217_2301.py index 3237ab9d..186c9532 100644 --- a/marco/portal/data_gaps/migrations/0003_auto_20141217_2301.py +++ b/marco/portal/data_gaps/migrations/0003_auto_20141217_2301.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations import django.db.models.deletion diff --git a/marco/portal/data_gaps/migrations/0004_auto_20150112_2303.py b/marco/portal/data_gaps/migrations/0004_auto_20150112_2303.py index 30368e9d..3f291440 100644 --- a/marco/portal/data_gaps/migrations/0004_auto_20150112_2303.py +++ b/marco/portal/data_gaps/migrations/0004_auto_20150112_2303.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations from django.conf import settings diff --git a/marco/portal/data_gaps/migrations/0005_auto_20150121_2319.py b/marco/portal/data_gaps/migrations/0005_auto_20150121_2319.py index a267366f..1c4a5438 100644 --- a/marco/portal/data_gaps/migrations/0005_auto_20150121_2319.py +++ b/marco/portal/data_gaps/migrations/0005_auto_20150121_2319.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations import django.db.models.deletion diff --git a/marco/portal/grid_pages/migrations/0001_initial.py b/marco/portal/grid_pages/migrations/0001_initial.py index dec30349..41e20f91 100644 --- a/marco/portal/grid_pages/migrations/0001_initial.py +++ b/marco/portal/grid_pages/migrations/0001_initial.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations import django.db.models.deletion diff --git a/marco/portal/grid_pages/migrations/0002_auto_20150429_1843.py b/marco/portal/grid_pages/migrations/0002_auto_20150429_1843.py index 214b60a5..e007c734 100644 --- a/marco/portal/grid_pages/migrations/0002_auto_20150429_1843.py +++ b/marco/portal/grid_pages/migrations/0002_auto_20150429_1843.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations diff --git a/marco/portal/grid_pages/migrations/0003_auto_20150429_1844.py b/marco/portal/grid_pages/migrations/0003_auto_20150429_1844.py index 6ca18d48..fe231ce6 100644 --- a/marco/portal/grid_pages/migrations/0003_auto_20150429_1844.py +++ b/marco/portal/grid_pages/migrations/0003_auto_20150429_1844.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations diff --git a/marco/portal/grid_pages/migrations/0004_auto_20160225_0012.py b/marco/portal/grid_pages/migrations/0004_auto_20160225_0012.py index 680a44fe..78b55371 100644 --- a/marco/portal/grid_pages/migrations/0004_auto_20160225_0012.py +++ b/marco/portal/grid_pages/migrations/0004_auto_20160225_0012.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations diff --git a/marco/portal/home/migrations/0001_initial.py b/marco/portal/home/migrations/0001_initial.py index 7db80609..7e8b3b2a 100644 --- a/marco/portal/home/migrations/0001_initial.py +++ b/marco/portal/home/migrations/0001_initial.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations import django.db.models.deletion diff --git a/marco/portal/home/migrations/0002_create_homepage.py b/marco/portal/home/migrations/0002_create_homepage.py index 3649f5f9..179b3a2f 100644 --- a/marco/portal/home/migrations/0002_create_homepage.py +++ b/marco/portal/home/migrations/0002_create_homepage.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations diff --git a/marco/portal/home/migrations/0003_homepagecarousel.py b/marco/portal/home/migrations/0003_homepagecarousel.py index 0f1b905c..02bbc5d7 100644 --- a/marco/portal/home/migrations/0003_homepagecarousel.py +++ b/marco/portal/home/migrations/0003_homepagecarousel.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations from django.conf import settings diff --git a/marco/portal/home/migrations/0004_auto_20171118_0027.py b/marco/portal/home/migrations/0004_auto_20171118_0027.py index bf25a9c4..41bc57e9 100644 --- a/marco/portal/home/migrations/0004_auto_20171118_0027.py +++ b/marco/portal/home/migrations/0004_auto_20171118_0027.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations from django.conf import settings diff --git a/marco/portal/home/migrations/0005_remove_homepage_feature_image.py b/marco/portal/home/migrations/0005_remove_homepage_feature_image.py index daf198c3..c50d891c 100644 --- a/marco/portal/home/migrations/0005_remove_homepage_feature_image.py +++ b/marco/portal/home/migrations/0005_remove_homepage_feature_image.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations diff --git a/marco/portal/home/migrations/0006_auto_20171120_2017.py b/marco/portal/home/migrations/0006_auto_20171120_2017.py index 16e01581..669e6e90 100644 --- a/marco/portal/home/migrations/0006_auto_20171120_2017.py +++ b/marco/portal/home/migrations/0006_auto_20171120_2017.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations import django.db.models.deletion diff --git a/marco/portal/home/migrations/0007_auto_20171120_2023.py b/marco/portal/home/migrations/0007_auto_20171120_2023.py index 1b45b15b..23694511 100644 --- a/marco/portal/home/migrations/0007_auto_20171120_2023.py +++ b/marco/portal/home/migrations/0007_auto_20171120_2023.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations from django.conf import settings diff --git a/marco/portal/home/migrations/0008_auto_20171121_0042.py b/marco/portal/home/migrations/0008_auto_20171121_0042.py index bb702dbb..78374735 100644 --- a/marco/portal/home/migrations/0008_auto_20171121_0042.py +++ b/marco/portal/home/migrations/0008_auto_20171121_0042.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations diff --git a/marco/portal/home/migrations/0009_homestream.py b/marco/portal/home/migrations/0009_homestream.py index d42800f4..3ed082a8 100644 --- a/marco/portal/home/migrations/0009_homestream.py +++ b/marco/portal/home/migrations/0009_homestream.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations from django.conf import settings diff --git a/marco/portal/home/migrations/0010_auto_20171121_2246.py b/marco/portal/home/migrations/0010_auto_20171121_2246.py index 499bfe8b..9fdd0c3f 100644 --- a/marco/portal/home/migrations/0010_auto_20171121_2246.py +++ b/marco/portal/home/migrations/0010_auto_20171121_2246.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations from django.conf import settings diff --git a/marco/portal/home/migrations/0011_remove_homepagecarousel_link.py b/marco/portal/home/migrations/0011_remove_homepagecarousel_link.py index 839f170e..0c6c7924 100644 --- a/marco/portal/home/migrations/0011_remove_homepagecarousel_link.py +++ b/marco/portal/home/migrations/0011_remove_homepagecarousel_link.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations diff --git a/marco/portal/home/migrations/0012_homepagecards.py b/marco/portal/home/migrations/0012_homepagecards.py index 9e696a87..3ca439e3 100644 --- a/marco/portal/home/migrations/0012_homepagecards.py +++ b/marco/portal/home/migrations/0012_homepagecards.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations from django.conf import settings diff --git a/marco/portal/home/migrations/0013_auto_20171123_0013.py b/marco/portal/home/migrations/0013_auto_20171123_0013.py index 6cc0ff8d..a2bd8b16 100644 --- a/marco/portal/home/migrations/0013_auto_20171123_0013.py +++ b/marco/portal/home/migrations/0013_auto_20171123_0013.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations diff --git a/marco/portal/home/migrations/0014_auto_20171123_0025.py b/marco/portal/home/migrations/0014_auto_20171123_0025.py index 0c7aa115..3c6cb7b9 100644 --- a/marco/portal/home/migrations/0014_auto_20171123_0025.py +++ b/marco/portal/home/migrations/0014_auto_20171123_0025.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations from django.conf import settings diff --git a/marco/portal/home/migrations/0015_homepagecardset.py b/marco/portal/home/migrations/0015_homepagecardset.py index 81be97b6..cdc80dc7 100644 --- a/marco/portal/home/migrations/0015_homepagecardset.py +++ b/marco/portal/home/migrations/0015_homepagecardset.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations import modelcluster.fields diff --git a/marco/portal/home/migrations/0016_auto_20171130_1854.py b/marco/portal/home/migrations/0016_auto_20171130_1854.py index 28acaa80..0bca0611 100644 --- a/marco/portal/home/migrations/0016_auto_20171130_1854.py +++ b/marco/portal/home/migrations/0016_auto_20171130_1854.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations diff --git a/marco/portal/home/migrations/0017_auto_20171130_2030.py b/marco/portal/home/migrations/0017_auto_20171130_2030.py index 25ea6f96..34094f6d 100644 --- a/marco/portal/home/migrations/0017_auto_20171130_2030.py +++ b/marco/portal/home/migrations/0017_auto_20171130_2030.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations diff --git a/marco/portal/home/migrations/0018_auto_20171130_2057.py b/marco/portal/home/migrations/0018_auto_20171130_2057.py index 4ab34a5e..c0afad83 100644 --- a/marco/portal/home/migrations/0018_auto_20171130_2057.py +++ b/marco/portal/home/migrations/0018_auto_20171130_2057.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations diff --git a/marco/portal/home/models.py b/marco/portal/home/models.py index 14c7077e..7dcbd83c 100644 --- a/marco/portal/home/models.py +++ b/marco/portal/home/models.py @@ -6,28 +6,11 @@ from django.conf import settings -if settings.WAGTAIL_VERSION > 3: - from wagtail.models import Orderable, Page - from wagtail.fields import RichTextField - from wagtail.search import index - from wagtail.admin.panels import FieldPanel,InlinePanel,MultiFieldPanel,FieldRowPanel,PageChooserPanel,TitleFieldPanel - from wagtail.images.models import Image -elif settings.WAGTAIL_VERSION > 1: - from wagtail.models import Orderable, Page - from wagtail.fields import RichTextField - from wagtail.search import index - from wagtail.admin.panels import FieldPanel,InlinePanel,MultiFieldPanel,FieldRowPanel,PageChooserPanel - from wagtail.images.models import Image - from wagtail.images.edit_handlers import ImageChooserPanel - TitleFieldPanel = FieldPanel -else: - from wagtail.models import Orderable, Page - from wagtail.fields import RichTextField - from wagtail.search import index - from wagtail.admin.panels import FieldPanel,InlinePanel,MultiFieldPanel,FieldRowPanel,PageChooserPanel - from wagtail.images.models import Image - from wagtail.images.edit_handlers import ImageChooserPanel - TitleFieldPanel = FieldPanel +from wagtail.models import Orderable, Page +from wagtail.fields import RichTextField +from wagtail.search import index +from wagtail.admin.panels import FieldPanel, InlinePanel, MultiFieldPanel, FieldRowPanel, PageChooserPanel, TitleFieldPanel +from wagtail.images.models import Image from portal.ocean_stories.models import OceanStory diff --git a/marco/portal/initial_data/migrations/0001_initial_data.py b/marco/portal/initial_data/migrations/0001_initial_data.py index 2da493e0..24a9a75a 100644 --- a/marco/portal/initial_data/migrations/0001_initial_data.py +++ b/marco/portal/initial_data/migrations/0001_initial_data.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations from django.conf import settings diff --git a/marco/portal/initial_data/migrations/0002_create_pages.py b/marco/portal/initial_data/migrations/0002_create_pages.py index 2a61cac9..7bbd0429 100644 --- a/marco/portal/initial_data/migrations/0002_create_pages.py +++ b/marco/portal/initial_data/migrations/0002_create_pages.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations from django.conf import settings diff --git a/marco/portal/menu/migrations/0001_initial.py b/marco/portal/menu/migrations/0001_initial.py index 28062953..743f5c9c 100644 --- a/marco/portal/menu/migrations/0001_initial.py +++ b/marco/portal/menu/migrations/0001_initial.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations import modelcluster.fields diff --git a/marco/portal/menu/migrations/0002_menuentry_show_divider_underneath.py b/marco/portal/menu/migrations/0002_menuentry_show_divider_underneath.py index 026f7a5c..e80fa3b5 100644 --- a/marco/portal/menu/migrations/0002_menuentry_show_divider_underneath.py +++ b/marco/portal/menu/migrations/0002_menuentry_show_divider_underneath.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations diff --git a/marco/portal/menu/migrations/0003_menuentry_page.py b/marco/portal/menu/migrations/0003_menuentry_page.py index a3f0bce2..accfc0db 100644 --- a/marco/portal/menu/migrations/0003_menuentry_page.py +++ b/marco/portal/menu/migrations/0003_menuentry_page.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations import django.db.models.deletion diff --git a/marco/portal/menu/migrations/0004_auto_20150122_2150.py b/marco/portal/menu/migrations/0004_auto_20150122_2150.py index 836398e9..0ecbfb7d 100644 --- a/marco/portal/menu/migrations/0004_auto_20150122_2150.py +++ b/marco/portal/menu/migrations/0004_auto_20150122_2150.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations diff --git a/marco/portal/menu/migrations/0005_auto_20150518_2309.py b/marco/portal/menu/migrations/0005_auto_20150518_2309.py index 72cfa7fe..1927c07d 100644 --- a/marco/portal/menu/migrations/0005_auto_20150518_2309.py +++ b/marco/portal/menu/migrations/0005_auto_20150518_2309.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations diff --git a/marco/portal/menu/migrations/0006_auto_20150521_2053.py b/marco/portal/menu/migrations/0006_auto_20150521_2053.py index 6f97364e..bfc3638f 100644 --- a/marco/portal/menu/migrations/0006_auto_20150521_2053.py +++ b/marco/portal/menu/migrations/0006_auto_20150521_2053.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations diff --git a/marco/portal/menu/migrations/0007_auto_20171201_2126.py b/marco/portal/menu/migrations/0007_auto_20171201_2126.py index 9d434c4d..608d72a4 100644 --- a/marco/portal/menu/migrations/0007_auto_20171201_2126.py +++ b/marco/portal/menu/migrations/0007_auto_20171201_2126.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations diff --git a/marco/portal/news/migrations/0001_initial.py b/marco/portal/news/migrations/0001_initial.py index a7bce4ea..7db9fe67 100644 --- a/marco/portal/news/migrations/0001_initial.py +++ b/marco/portal/news/migrations/0001_initial.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations import django.db.models.deletion diff --git a/marco/portal/news/migrations/0002_auto_20150522_0047.py b/marco/portal/news/migrations/0002_auto_20150522_0047.py index 0b661ceb..06a585b9 100644 --- a/marco/portal/news/migrations/0002_auto_20150522_0047.py +++ b/marco/portal/news/migrations/0002_auto_20150522_0047.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations diff --git a/marco/portal/news/migrations/0003_auto_20160225_0012.py b/marco/portal/news/migrations/0003_auto_20160225_0012.py index 811c994b..9655f414 100644 --- a/marco/portal/news/migrations/0003_auto_20160225_0012.py +++ b/marco/portal/news/migrations/0003_auto_20160225_0012.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations diff --git a/marco/portal/ocean_stories/migrations/0001_initial.py b/marco/portal/ocean_stories/migrations/0001_initial.py index 5d65c969..020d6418 100644 --- a/marco/portal/ocean_stories/migrations/0001_initial.py +++ b/marco/portal/ocean_stories/migrations/0001_initial.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations from django.conf import settings diff --git a/marco/portal/ocean_stories/migrations/0002_auto_20141211_0121.py b/marco/portal/ocean_stories/migrations/0002_auto_20141211_0121.py index 2ff2e799..793b5657 100644 --- a/marco/portal/ocean_stories/migrations/0002_auto_20141211_0121.py +++ b/marco/portal/ocean_stories/migrations/0002_auto_20141211_0121.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations diff --git a/marco/portal/ocean_stories/migrations/0003_oceanstory_feature_image.py b/marco/portal/ocean_stories/migrations/0003_oceanstory_feature_image.py index d36fb0b9..6d26dea4 100644 --- a/marco/portal/ocean_stories/migrations/0003_oceanstory_feature_image.py +++ b/marco/portal/ocean_stories/migrations/0003_oceanstory_feature_image.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations import django.db.models.deletion diff --git a/marco/portal/ocean_stories/migrations/0004_auto_20141219_2132.py b/marco/portal/ocean_stories/migrations/0004_auto_20141219_2132.py index 96b9929c..120e302c 100644 --- a/marco/portal/ocean_stories/migrations/0004_auto_20141219_2132.py +++ b/marco/portal/ocean_stories/migrations/0004_auto_20141219_2132.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations from django.conf import settings diff --git a/marco/portal/ocean_stories/migrations/0005_auto_20150112_2302.py b/marco/portal/ocean_stories/migrations/0005_auto_20150112_2302.py index 333b3726..d98e35d7 100644 --- a/marco/portal/ocean_stories/migrations/0005_auto_20150112_2302.py +++ b/marco/portal/ocean_stories/migrations/0005_auto_20150112_2302.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations from django.conf import settings diff --git a/marco/portal/ocean_stories/migrations/0006_auto_20150121_2319.py b/marco/portal/ocean_stories/migrations/0006_auto_20150121_2319.py index 5bfbe3a3..6eb4b00b 100644 --- a/marco/portal/ocean_stories/migrations/0006_auto_20150121_2319.py +++ b/marco/portal/ocean_stories/migrations/0006_auto_20150121_2319.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations import django.db.models.deletion diff --git a/marco/portal/ocean_stories/migrations/0007_auto_20150122_2225.py b/marco/portal/ocean_stories/migrations/0007_auto_20150122_2225.py index ae5f42c6..0b3ae3d3 100644 --- a/marco/portal/ocean_stories/migrations/0007_auto_20150122_2225.py +++ b/marco/portal/ocean_stories/migrations/0007_auto_20150122_2225.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations import django.db.models.deletion diff --git a/marco/portal/ocean_stories/migrations/0008_auto_20150203_2337.py b/marco/portal/ocean_stories/migrations/0008_auto_20150203_2337.py index d02fc7ab..ec1a6d37 100644 --- a/marco/portal/ocean_stories/migrations/0008_auto_20150203_2337.py +++ b/marco/portal/ocean_stories/migrations/0008_auto_20150203_2337.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations diff --git a/marco/portal/ocean_stories/migrations/0009_oceanstorysection_map_legend.py b/marco/portal/ocean_stories/migrations/0009_oceanstorysection_map_legend.py index 3fa7eb74..9f8821f0 100644 --- a/marco/portal/ocean_stories/migrations/0009_oceanstorysection_map_legend.py +++ b/marco/portal/ocean_stories/migrations/0009_oceanstorysection_map_legend.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations diff --git a/marco/portal/ocean_stories/migrations/0010_auto_20150603_1721.py b/marco/portal/ocean_stories/migrations/0010_auto_20150603_1721.py index 0048a56f..3f0ea79d 100644 --- a/marco/portal/ocean_stories/migrations/0010_auto_20150603_1721.py +++ b/marco/portal/ocean_stories/migrations/0010_auto_20150603_1721.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations diff --git a/marco/portal/ocean_stories/migrations/0011_oceanstory_display_home_page.py b/marco/portal/ocean_stories/migrations/0011_oceanstory_display_home_page.py index c6b61bdb..3ef1dab2 100644 --- a/marco/portal/ocean_stories/migrations/0011_oceanstory_display_home_page.py +++ b/marco/portal/ocean_stories/migrations/0011_oceanstory_display_home_page.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations diff --git a/marco/portal/ocean_stories/migrations/0012_auto_20160225_0012.py b/marco/portal/ocean_stories/migrations/0012_auto_20160225_0012.py index 3096e62f..ad323257 100644 --- a/marco/portal/ocean_stories/migrations/0012_auto_20160225_0012.py +++ b/marco/portal/ocean_stories/migrations/0012_auto_20160225_0012.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations diff --git a/marco/portal/ocean_stories/templates/ocean_stories/extra_js.html b/marco/portal/ocean_stories/templates/ocean_stories/extra_js.html index bc8f47b6..ebec7d11 100644 --- a/marco/portal/ocean_stories/templates/ocean_stories/extra_js.html +++ b/marco/portal/ocean_stories/templates/ocean_stories/extra_js.html @@ -27,8 +27,6 @@ {% if MAP_LIBRARY == 'ol8' %} - - {% endif %} diff --git a/marco/portal/pages/migrations/0001_initial.py b/marco/portal/pages/migrations/0001_initial.py index 0630053c..e9fc0076 100644 --- a/marco/portal/pages/migrations/0001_initial.py +++ b/marco/portal/pages/migrations/0001_initial.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations from django.conf import settings diff --git a/marco/portal/pages/migrations/0002_page_description.py b/marco/portal/pages/migrations/0002_page_description.py index 4b60b6a2..04a70244 100644 --- a/marco/portal/pages/migrations/0002_page_description.py +++ b/marco/portal/pages/migrations/0002_page_description.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations from django.conf import settings diff --git a/marco/portal/pages/migrations/0003_auto_20150112_2308.py b/marco/portal/pages/migrations/0003_auto_20150112_2308.py index 7efd42c2..98f8b22a 100644 --- a/marco/portal/pages/migrations/0003_auto_20150112_2308.py +++ b/marco/portal/pages/migrations/0003_auto_20150112_2308.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations from django.conf import settings diff --git a/marco/portal/welcome_snippet/migrations/0001_initial.py b/marco/portal/welcome_snippet/migrations/0001_initial.py index 04d310f5..a260c228 100644 --- a/marco/portal/welcome_snippet/migrations/0001_initial.py +++ b/marco/portal/welcome_snippet/migrations/0001_initial.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations import django.db.models.deletion diff --git a/marco/portal/welcome_snippet/migrations/0002_welcomepage_use_on_site.py b/marco/portal/welcome_snippet/migrations/0002_welcomepage_use_on_site.py index b3a5ed43..a53b9f3b 100644 --- a/marco/portal/welcome_snippet/migrations/0002_welcomepage_use_on_site.py +++ b/marco/portal/welcome_snippet/migrations/0002_welcomepage_use_on_site.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations diff --git a/marco/portal/welcome_snippet/migrations/0003_auto_20150501_1850.py b/marco/portal/welcome_snippet/migrations/0003_auto_20150501_1850.py index 32f023c2..5e11fb25 100644 --- a/marco/portal/welcome_snippet/migrations/0003_auto_20150501_1850.py +++ b/marco/portal/welcome_snippet/migrations/0003_auto_20150501_1850.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations diff --git a/marco/portal/welcome_snippet/migrations/0004_auto_20150502_1649.py b/marco/portal/welcome_snippet/migrations/0004_auto_20150502_1649.py index 1e027d1b..248dde3e 100644 --- a/marco/portal/welcome_snippet/migrations/0004_auto_20150502_1649.py +++ b/marco/portal/welcome_snippet/migrations/0004_auto_20150502_1649.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations from django.conf import settings diff --git a/marco/portal/welcome_snippet/migrations/0005_welcomepage_body.py b/marco/portal/welcome_snippet/migrations/0005_welcomepage_body.py index bade9128..ff50f5b3 100644 --- a/marco/portal/welcome_snippet/migrations/0005_welcomepage_body.py +++ b/marco/portal/welcome_snippet/migrations/0005_welcomepage_body.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations from django.conf import settings diff --git a/marco/portal/welcome_snippet/migrations/0006_welcomepageentry_media_image.py b/marco/portal/welcome_snippet/migrations/0006_welcomepageentry_media_image.py index f8992271..5489bda0 100644 --- a/marco/portal/welcome_snippet/migrations/0006_welcomepageentry_media_image.py +++ b/marco/portal/welcome_snippet/migrations/0006_welcomepageentry_media_image.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals from django.db import models, migrations import django.db.models.deletion diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..8d9c9fef --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,84 @@ +[build-system] +requires = ["setuptools>=68", "wheel"] +build-backend = "setuptools.backends.legacy:build" + +[project] +name = "madrona-portal" +version = "4.0.2" +description = "MARCO Mid-Atlantic Ocean Data Portal" +requires-python = ">=3.10" +readme = "README.md" +license = { text = "MIT" } + +# Runtime dependencies are managed in requirements.txt / docker-requirements.txt. +# This section is intentionally minimal — the project is not distributed as a +# standalone package, so pip-installable metadata is kept lightweight. +dependencies = [] + +# --------------------------------------------------------------------------- +# Pytest +# --------------------------------------------------------------------------- +[tool.pytest.ini_options] +DJANGO_SETTINGS_MODULE = "marco.settings" +python_files = ["tests.py", "test_*.py", "*_test.py"] +addopts = "--strict-markers -q" + +# --------------------------------------------------------------------------- +# Coverage +# --------------------------------------------------------------------------- +[tool.coverage.run] +source = ["portal", "marco", "marco_site"] +omit = ["*/migrations/*", "*/tests/*", "manage.py"] + +[tool.coverage.report] +show_missing = true +skip_covered = false + +# --------------------------------------------------------------------------- +# Ruff (linting + formatting — replaces flake8, isort, pyupgrade) +# --------------------------------------------------------------------------- +[tool.ruff] +target-version = "py310" +line-length = 100 +exclude = [ + ".git", + ".venv", + "venv", + "bower_components", + "*/migrations/*", + "node_modules", + "static", + "media", +] + +[tool.ruff.lint] +select = [ + "E", # pycodestyle errors + "W", # pycodestyle warnings + "F", # pyflakes + "I", # isort + "UP", # pyupgrade + "B", # flake8-bugbear + "C4", # flake8-comprehensions + "SIM", # flake8-simplify + "DJ", # flake8-django +] +ignore = [ + "E501", # line too long — handled by formatter + "DJ001", # Avoid using null=True on string-based fields — existing models +] + +[tool.ruff.lint.isort] +known-first-party = ["marco", "marco_site", "portal"] + +# --------------------------------------------------------------------------- +# Mypy (optional static typing) +# --------------------------------------------------------------------------- +[tool.mypy] +python_version = "3.10" +plugins = ["mypy_django_plugin.main"] +ignore_missing_imports = true +exclude = ["migrations/", "bower_components/", "static/"] + +[tool.django-stubs] +django_settings_module = "marco.settings" diff --git a/requirements.txt b/requirements.txt index d4adcd24..d65840be 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,76 +1,93 @@ -# Minimal requirements -Django>=4.2,<4.3 -wagtail +# ============================================================================= +# Core Framework +# ============================================================================= +Django>=4.2,<5.0 +wagtail>=7.0,<8.0 -# Tentative additions: -wagtail-import-export -python-social-auth -# social-auth-app-django>5.4 -social-auth-app-django<5.0 -python-jose -pyjwt -django-autocomplete-light -django-social-share -django-email-log -django-compressor -django-tinymce -django-wysiwyg -django-recaptcha -django-flatblocks -django-nested-admin -django-querysetsequence -django-redis -django-taggit<5.0 -rpc4django +# ============================================================================= +# Authentication & Social Auth +# ============================================================================= +social-auth-app-django>=5.4,<6.0 +social-auth-core>=4.5,<5.0 +python-jose>=3.3,<4.0 +pyjwt>=2.8,<3.0 +django-social-share>=2.3,<3.0 -################################## -# -e git+https://github.com/Ecotrust/mp-data-manager.git@main#egg=mp_data_manager -### OR #### -#-e git+https://github.com/Ecotrust/mp-data-manager.git@gp2#egg=mp_data_manager-gp2 -################################## - -#-e git+https://github.com/Ecotrust/madrona-analysistools.git@main#egg=madrona_analysistools -#-e git+https://github.com/Ecotrust/mp-visualize.git@main#egg=mp_visualize -#-e git+https://github.com/Ecotrust/madrona-features.git@main#egg=madrona_features -#-e git+https://github.com/Ecotrust/mp-accounts.git@main#egg=mp_accounts -#-e git+https://github.com/Ecotrust/madrona-scenarios.git@main#egg=madrona_scenarios -#-e git+https://github.com/Ecotrust/madrona-manipulators.git@main#egg=madrona_manipulators -#-e git+https://github.com/Ecotrust/mp-drawing.git@main#egg=mp_drawing -#-e git+https://github.com/Ecotrust/mp-explore.git@main#egg=mp_explore -#-e git+https://github.com/Ecotrust/mp-map-groups.git@main#egg=mp-map-groups -#-e git+https://github.com/Ecotrust/p97-nursery.git@main#egg=p97-nursery -#-e git+https://github.com/Ecotrust/mp-proxy.git@main#egg=mp_proxy -#-e git+https://github.com/Ecotrust/mp-survey.git@main#egg=mp_survey +# ============================================================================= +# Admin & CMS Enhancements +# ============================================================================= +django-autocomplete-light>=3.9,<4.0 +django-nested-admin>=4.0,<5.0 +django-tinymce>=3.6,<4.0 +django-recaptcha>=4.0,<5.0 # django_recaptcha (Wagtail 7+ requires v4) +django-flatblocks>=1.0,<2.0 +django-querysetsequence>=0.14,<1.0 +wagtailcharts>=0.4,<1.0 +django-import-export>=3.3,<4.0 +django-colorfield>=0.11,<1.0 -################################## -#-e git+https://github.com/Ecotrust/mida-portal.git@main#egg=mida -### OR ### -#-e git+https://github.com/Ecotrust/wcoa.git@migration_2021#egg=wcoa-2021 -### OR ### -#-e git+https://github.com/Ecotrust/wc-offshore-portal.git@master#egg=offshore -################################## +# ============================================================================= +# Content & Tagging +# ============================================================================= +django-taggit>=5.0,<7.0 +django-email-log>=1.0,<2.0 +django-compressor>=4.4,<5.0 +django-libsass>=0.9,<1.0 +libsass>=0.23,<1.0 +django-picklefield>=3.1,<4.0 +xmltodict>=0.13,<1.0 +# ============================================================================= +# Async Tasks & Caching +# ============================================================================= +celery>=5.3,<6.0 +django-celery-email>=3.0,<4.0 +django-redis>=5.4,<6.0 -celery -# Django-Celery is incompatible with py3/Django2 -#django-celery -django-celery-email -django-colorfield -django-libsass +# ============================================================================= +# Database & Geospatial +# ============================================================================= +psycopg2-binary>=2.9.9,<3.0 +# Note: GDAL Python bindings are installed separately in the Dockerfile via: +# pip install "GDAL==$(gdal-config --version)" --no-cache-dir +pyshp>=2.3,<3.0 +owslib>=0.29,<1.0 -# Note: django-libsass needs libsass>0.4 -# 0.5.1 is the latest version that we can install on WF -libsass +# ============================================================================= +# Search +# ============================================================================= +elasticsearch>=7.0,<8.0 +elasticsearch-dsl>=7.0,<8.0 -# Recommended components (require additional setup): -# pg2 2.9 breaks migrations: https://stackoverflow.com/a/68025007/706797 -# psycopg2-binary>2.9.9 -psycopg2-binary<2.9 -elasticsearch +# ============================================================================= +# APIs & Protocols +# ============================================================================= +# rpc4django removed — replaced by djangorestframework API views +djangorestframework>=3.14,<4.0 -django-import-export - -pyshp +# ============================================================================= +# Ecotrust Sub-Apps +# (Uncomment the git source OR use local editable installs for development) +# ============================================================================= +# -e git+https://github.com/Ecotrust/mp-data-manager.git@main#egg=mp_data_manager +# -e git+https://github.com/Ecotrust/madrona-analysistools.git@main#egg=madrona_analysistools +# -e git+https://github.com/Ecotrust/mp-visualize.git@main#egg=mp_visualize +# -e git+https://github.com/Ecotrust/madrona-features.git@main#egg=madrona_features +# -e git+https://github.com/Ecotrust/mp-accounts.git@main#egg=mp_accounts +# -e git+https://github.com/Ecotrust/madrona-scenarios.git@main#egg=madrona_scenarios +# -e git+https://github.com/Ecotrust/madrona-manipulators.git@main#egg=madrona_manipulators +# -e git+https://github.com/Ecotrust/mp-drawing.git@main#egg=mp_drawing +# -e git+https://github.com/Ecotrust/mp-explore.git@main#egg=mp_explore +# -e git+https://github.com/Ecotrust/mp-layers.git@main#egg=mp_layers +# -e git+https://github.com/Ecotrust/mp-map-groups.git@main#egg=mp_map_groups +# -e git+https://github.com/Ecotrust/p97-nursery.git@main#egg=p97_nursery +# -e git+https://github.com/Ecotrust/mp-proxy.git@main#egg=mp_proxy +# -e git+https://github.com/Ecotrust/mp-survey.git@main#egg=mp_survey +# -e git+https://github.com/Ecotrust/django-url-shortener.git@main#egg=url_short -owslib -django-colorfield +# ============================================================================= +# Portal Variant (choose one): +# ============================================================================= +# -e git+https://github.com/Ecotrust/mida-portal.git@main#egg=mida +# -e git+https://github.com/Ecotrust/wcoa.git@main#egg=wcoa +# -e git+https://github.com/Ecotrust/wc-offshore-portal.git@master#egg=offshore diff --git a/scripts/db-restore.sh b/scripts/db-restore.sh new file mode 100755 index 00000000..a0ce9ad3 --- /dev/null +++ b/scripts/db-restore.sh @@ -0,0 +1,190 @@ +#!/usr/bin/env bash +# ----------------------------------------------------------------------------- +# db-restore.sh — Restore a PostgreSQL dump into the Dockerized database. +# +# Usage: +# ./scripts/db-restore.sh +# ./scripts/db-restore.sh --drop # drop & recreate DB first +# ./scripts/db-restore.sh --prod # force production compose +# ./scripts/db-restore.sh --dev # force development compose +# ./scripts/db-restore.sh --env-file +# +# Run from anywhere — this script always operates relative to madrona-portal/. +# +# Prerequisites: +# 1. Docker Compose stack is running (dev or prod): +# docker compose -f docker/docker-compose.yml up # dev +# docker compose -f docker/docker-compose.prod.yml up # prod +# 2. madrona-portal/docker/.env exists and contains DB_NAME, DB_USER, DB_PASSWORD. +# +# Environment detection (applied in order, first match wins): +# 1. --prod / --dev CLI flag +# 2. Running Docker Compose stack detected via `docker compose ls` +# 3. DJANGO_ENV variable in the loaded .env file +# 4. Default: development +# +# Options: +# --drop Terminate all active connections, drop, and recreate the +# target database before restoring. Required for a clean +# import from prod. Without this flag the dump is applied +# on top of existing data. +# --prod Force the production compose file (docker-compose.prod.yml). +# --dev Force the development compose file (docker-compose.yml). +# --env-file Path to the .env file (default: ./docker/.env). +# +# Notes: +# - The dump is streamed directly into the container — no temp files on disk. +# - psql warnings (e.g. "already exists") are normal when importing a dump +# produced on a different Postgres version (12 → 16) and are not fatal. +# - After a --drop restore, run migrations to pick up any schema drift: +# docker compose -f exec app python marco/manage.py migrate +# ----------------------------------------------------------------------------- +set -euo pipefail + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- +die() { echo "[db-restore] ERROR: $*" >&2; exit 1; } +info() { echo "[db-restore] $*"; } + +COMPOSE_DEV="docker/docker-compose.yml" +COMPOSE_PROD="docker/docker-compose.prod.yml" + +# --------------------------------------------------------------------------- +# Parse arguments +# --------------------------------------------------------------------------- +DROP_FIRST=false +DUMP_FILE="" +ENV_FILE="$(dirname "${BASH_SOURCE[0]}")/../docker/.env" +FORCE_ENV="" # "prod" | "dev" | "" (auto-detect) + +while [[ $# -gt 0 ]]; do + case "$1" in + --drop) DROP_FIRST=true; shift ;; + --prod) FORCE_ENV="prod"; shift ;; + --dev) FORCE_ENV="dev"; shift ;; + --env-file) [[ -n "${2:-}" ]] || die "--env-file requires a path argument" + ENV_FILE="$2"; shift 2 ;; + -*) die "Unknown option: '$1'. Usage: $0 [--drop] [--prod|--dev] [--env-file ] " ;; + *) [[ -z "$DUMP_FILE" ]] || die "Unexpected argument: '$1'" + DUMP_FILE="$1"; shift ;; + esac +done + +[[ -n "$DUMP_FILE" ]] || die "Usage: $0 [--drop] [--prod|--dev] [--env-file ] " + +# Resolve dump path before we cd away. +DUMP_ABS="$(cd "$(dirname "$DUMP_FILE")" && pwd)/$(basename "$DUMP_FILE")" +[[ -f "$DUMP_ABS" ]] || die "Dump file not found: $DUMP_FILE" + +# Expand a leading ~ in ENV_FILE before resolving it to an absolute path. +if [[ "$ENV_FILE" == ~* ]]; then + ENV_FILE="${ENV_FILE/#\~/$HOME}" +fi +# Resolve ENV_FILE to an absolute path before we cd away. +ENV_FILE_ABS="$(cd "$(dirname "$ENV_FILE")" && pwd)/$(basename "$ENV_FILE")" +[[ -f "$ENV_FILE_ABS" ]] || die "Env file not found: $ENV_FILE" + +# --------------------------------------------------------------------------- +# Always operate from madrona-portal/ regardless of where the script is called +# --------------------------------------------------------------------------- +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$SCRIPT_DIR/.." + +# --------------------------------------------------------------------------- +# Load .env for DB credentials and DJANGO_ENV +# --------------------------------------------------------------------------- +set -a +# shellcheck source=/dev/null +source "$ENV_FILE_ABS" +set +a + +DB_NAME="${DB_NAME:-wcoa_docker_db}" +DB_USER="${DB_USER:-postgres}" +DB_PASSWORD="${DB_PASSWORD:?DB_PASSWORD must be set in .env}" + +# --------------------------------------------------------------------------- +# Detect which compose file to target +# --------------------------------------------------------------------------- +# detect_compose_file: inspects `docker compose ls` for a running stack whose +# config file matches one of our known compose files, preferring prod. +# Falls back to DJANGO_ENV from .env, then defaults to dev. +detect_compose_file() { + # 1. CLI flag takes priority. + if [[ "$FORCE_ENV" == "prod" ]]; then + echo "$COMPOSE_PROD"; return + elif [[ "$FORCE_ENV" == "dev" ]]; then + echo "$COMPOSE_DEV"; return + fi + + # 2. Inspect running stacks. `docker compose ls` lists all projects with + # their config file paths — grep for our known filenames. + local ls_out + if ls_out=$(docker compose ls 2>/dev/null); then + if echo "$ls_out" | grep -q "docker-compose\.prod\.yml"; then + echo "$COMPOSE_PROD"; return + fi + if echo "$ls_out" | grep -q "docker-compose\.yml"; then + echo "$COMPOSE_DEV"; return + fi + fi + + # 3. Fall back to DJANGO_ENV from the loaded .env. + if [[ "${DJANGO_ENV:-}" == "production" ]]; then + echo "$COMPOSE_PROD"; return + fi + + # 4. Default to development. + echo "$COMPOSE_DEV" +} + +COMPOSE_FILE="$(detect_compose_file)" +COMPOSE="docker compose -f $COMPOSE_FILE --env-file $ENV_FILE_ABS" +PSQL="$COMPOSE exec -T -e PGPASSWORD=$DB_PASSWORD db psql -U $DB_USER" + +info "Using compose file: $COMPOSE_FILE" + +# --------------------------------------------------------------------------- +# Verify the db container is healthy before doing anything +# --------------------------------------------------------------------------- +info "Checking db service health..." +$COMPOSE ps db | grep -q "healthy" \ + || die "db container is not healthy. Is the stack running? Try: $COMPOSE up -d" + +# --------------------------------------------------------------------------- +# Optional: terminate connections, drop, and recreate the database +# --------------------------------------------------------------------------- +if [[ "$DROP_FIRST" == true ]]; then + info "Terminating active connections to '$DB_NAME'..." + $PSQL -d postgres -c \ + "SELECT pg_terminate_backend(pid) + FROM pg_stat_activity + WHERE datname = '$DB_NAME' AND pid <> pg_backend_pid();" \ + > /dev/null + + info "Dropping database '$DB_NAME'..." + $PSQL -d postgres -c "DROP DATABASE IF EXISTS \"$DB_NAME\";" + + info "Creating database '$DB_NAME'..." + $PSQL -d postgres -c "CREATE DATABASE \"$DB_NAME\";" + + info "Enabling PostGIS extension..." + $PSQL -d "$DB_NAME" -c "CREATE EXTENSION IF NOT EXISTS postgis;" +fi + +# --------------------------------------------------------------------------- +# Stream the dump into the database +# --------------------------------------------------------------------------- +DUMP_SIZE="$(du -sh "$DUMP_ABS" | cut -f1)" +info "Restoring '$DUMP_FILE' (${DUMP_SIZE}) → '$DB_NAME'..." +info "psql warnings about existing objects are expected and non-fatal." + +$PSQL -d "$DB_NAME" \ + --set ON_ERROR_STOP=off \ + < "$DUMP_ABS" + +info "Restore complete." +info "" +info "Next steps:" +info " Apply any pending migrations:" +info " docker compose -f $COMPOSE_FILE exec app python marco/manage.py migrate" diff --git a/scripts/vagrant_provision.sh b/scripts/vagrant_provision.sh index 1091b6eb..591d2de9 100755 --- a/scripts/vagrant_provision.sh +++ b/scripts/vagrant_provision.sh @@ -1,4 +1,6 @@ #!/bin/bash +# LEGACY — Vagrant-based provisioning. Primary dev environment is now Docker Compose. +# See docker/docker-compose.yml and README for current setup instructions. PROJECT_NAME=$1 APP_NAME=$2