From ab43d20b49543fa61b23d41dae2bf6a5e329b4c3 Mon Sep 17 00:00:00 2001 From: Aaron Madlon-Kay Date: Wed, 4 May 2022 23:55:22 +0900 Subject: [PATCH 1/3] Add Docker Compose-based setup --- .dockerignore | 5 +++++ .gitignore | 1 + docker-compose.yml | 36 ++++++++++++++++++++++++++++++++++++ docker_compose_setup.sh | 27 +++++++++++++++++++++++++++ 4 files changed, 69 insertions(+) create mode 100644 .dockerignore create mode 100644 docker-compose.yml create mode 100755 docker_compose_setup.sh diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 000000000..a3b485ed7 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,5 @@ +Dockerfile +docker-compose.yml +*~ +/tmp +/app/data diff --git a/.gitignore b/.gitignore index 40e3bb926..3bf4afef2 100644 --- a/.gitignore +++ b/.gitignore @@ -91,3 +91,4 @@ staticfiles .env db.sqlite3 /app/data/ +/tmp diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000..0b2b006ee --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,36 @@ +services: + webapp: + build: . + ports: + - "8080:8080" + volumes: + - ./app:/code/app + - ./tmp:/code/app/data + links: + - db + - solr + environment: + SECRET_KEY: secret + DB_NAME: webapp + DB_USER: postgres + DB_PASSWORD: password + DB_HOST: db + DB_PORT: 5432 + SOLR_URL: http://solr:8983/solr/macports + SOLR_ADMIN_URL: http://solr:8983/solr/admin/cores + db: + image: postgres:13 + volumes: + - db:/var/lib/postgresql/data + environment: + POSTGRES_PASSWORD: password + POSTGRES_DB: webapp + PGDATA: /var/lib/postgresql/data/pgdata + solr: + image: solr:6 + volumes: + - ./tmp/solr:/opt/solr/server/solr/mycores + command: solr-precreate macports + +volumes: + db: {} diff --git a/docker_compose_setup.sh b/docker_compose_setup.sh new file mode 100755 index 000000000..7537d5e29 --- /dev/null +++ b/docker_compose_setup.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +set -euo pipefail + +# Add ports data +docker compose run -w /code/app webapp python3 manage.py update-portinfo --type=full + +# Add builders +docker compose exec db psql -U postgres -d webapp -c " + INSERT INTO builder (id, name, display_name, natural_name) VALUES + (1, '10.15_x86_64', '10.15', 'Catalina'), + (2, '11_x86_64', '11', 'Big Sur'), + (3, '12_x86_64', '12', 'Monterey') + ON CONFLICT DO NOTHING; +" + +# Fetch build history +docker compose run -d -w /code/app webapp python3 manage.py fetch-build-history + +# Run livecheck +docker compose run -d -w /code/app webapp sh -c 'port -d selfupdate && python3 manage.py run-full-livecheck' + +# Add a solr schema and generate index +docker compose exec -w /code/app webapp python3 manage.py build_solr_schema \ + --configure-directory=/code/app/data/solr/macports/conf +docker compose exec -w /code/app webapp python3 manage.py build_solr_schema -r RELOAD_CORE +docker compose run -d -w /code/app webapp python3 manage.py rebuild_index --noinput From 9c2908a309f828ded51753e134c155f296791e6f Mon Sep 17 00:00:00 2001 From: Aaron Madlon-Kay Date: Thu, 5 May 2022 22:33:04 +0900 Subject: [PATCH 2/3] Fix whitespace, proper noun orthography --- docs/DEPLOYMENT.md | 82 +++++++++++++++++++++++----------------------- 1 file changed, 41 insertions(+), 41 deletions(-) diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index e8dc89264..92ecefeb2 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -5,7 +5,7 @@ inside the `app` directory in the root of the project. ###### Project structure ``` -|-- app (main django app) +|-- app (main Django app) |-- config |-- docs |-- .gitignore @@ -17,7 +17,7 @@ inside the `app` directory in the root of the project. |-- README.md ``` -As mentioned, the `app` directory is a standalone django app. However, the app has been Dockerised and all the pain of +As mentioned, the `app` directory is a standalone Django app. However, the app has been Dockerised and all the pain of setting up a local environment can be avoided when working with Docker. But in case you do not want to work with Docker, we will also walk through the standard process of setting up the app. @@ -38,7 +38,7 @@ After this, connect as root: su - postgres ``` -Create your user(replace username and password with your preferred credentials): +Create your user (replace username and password with your preferred credentials): ```bash CREATE USER username PASSWORD 'password'; ``` @@ -51,7 +51,7 @@ CREATE DATABASE webapp; Grant permissions to your user on this database. -```bash +```bash GRANT ALL PRIVILEGES ON DATABASE webapp TO username; ``` @@ -59,11 +59,11 @@ That's it for PostgreSQL, our database for the app is ready. ### Solr -To get the required version of Solr working we need [openjdk8](https://openjdk.java.net/install/). On a mac, you may use -[MacPorts to install openjdk8](https://ports.macports.org/port/openjdk8/). New versions of openjdk may raise problems, +To get the required version of Solr working we need [openjdk8](https://openjdk.java.net/install/). On a Mac, you may use +[MacPorts to install openjdk8](https://ports.macports.org/port/openjdk8/). New versions of OpenJDK may raise problems; we recommend openjdk8. -Once opendjk8 has been installed, we can proceed to downloading and launching Solr. We would need +Once opendjk8 has been installed, we can proceed to downloading and launching Solr. We would need [Solr 6.6.6](https://archive.apache.org/dist/lucene/solr/6.6.6/). The below steps are self explanatory: ```bash @@ -76,11 +76,11 @@ cd solr ``` By default this will create a core with a managed schema. This setup is dynamic but not useful for the app, later we will -configure solr to use a static schema, which we will generate with the help of `django-haystack`. +configure Solr to use a static schema, which we will generate with the help of `django-haystack`. ### Memcached (optional) -Setting up memcached is straightforward. On Linux: +Setting up Memcached is straightforward. On Linux: ```bash wget http://memcached.org/latest @@ -89,9 +89,9 @@ cd memcached-1.x.x ./configure && make && make test && sudo make install ``` -Now you may start memcached by running `memcached -d` in the terminal. +Now you may start Memcached by running `memcached -d` in the terminal. -On a mac, you may use [MacPorts to install memcached](https://ports.macports.org/port/memcached/summary). +On a Mac, you may use [MacPorts to install Memcached](https://ports.macports.org/port/memcached/summary). --- @@ -101,7 +101,7 @@ We have the dependencies ready now and are ready to move forward to actually set There are two options available: -- Build the docker image from source. (long but useful for development) +- Build the Docker image from source. (long but useful for development) - Download an image from the Docker registry. (ready to go in one command) ### 1.a: Building the image from source @@ -111,12 +111,12 @@ There are two options available: git clone https://github.com/macports/macports-webapp.git cd macports-webapp ``` - - Build the docker image + - Build the Docker image ```bash docker build -t macports-webapp . ``` - Once build finishes, the image is ready to run. - + ### 1.b: Downloading image from Docker registry This is rather simple, but useless if your purpose is development as you won't be able to test your changes. @@ -128,7 +128,7 @@ docker pull arjunsalyan/macports-webapp ### 2. Starting the container After using any one of the methods described in (1.a) or (1.b), you have an image with you. We will start this image with -several environment variables that are needed to connect to PostgreSQL, Solr and memcached. A file `env.sample` has been +several environment variables that are needed to connect to PostgreSQL, Solr and Memcached. A file `env.sample` has been added in the repo for your convenience. ```bash @@ -146,19 +146,19 @@ EMAIL_HOST_USER= (smtp user, optional) EMAIL_HOST_PASSWORD= (smtp user password, optional) ``` -On a mac, `host.docker.internal` should be used instead of "localhost" or "127.0.0.1". +On a Mac, `host.docker.internal` should be used instead of "localhost" or "127.0.0.1". Set values for the variables and save this as a file, let's name it `env`. -Environment variables are ready, we will also mount some volumes to our Docker container. Get the path to your solr -directory, from the step where we installed Solr. Also, create a folder for the app to store some utility data -(portindex, portindex.json etc.). We will mount both these directories as volumes to the docker container, call them `path/to/solr` +Environment variables are ready, we will also mount some volumes to our Docker container. Get the path to your Solr +directory, from the step where we installed Solr. Also, create a folder for the app to store some utility data +(portindex, portindex.json etc.). We will mount both these directories as volumes to the Docker container, call them `path/to/solr` and `path/to/data`. Start the image: On Linux: -```bash +```bash docker run -d \ -v /path/to/solr:/solr \ -v /path/to/data:/code/app/data \ @@ -176,9 +176,9 @@ On Mac: sudo docker run -d \ -p 8080:8080 \ -v /path/to/solr:/solr \ - -v /path/to/data:/code/app/data \ + -v /path/to/data:/code/app/data \ --name=webapp-container \ - --env-file=env \ + --env-file=env \ macports-webapp ``` @@ -186,8 +186,8 @@ Open `127.0.0.1:8080` in the browser to test if the app has started, you won't h ## Setup without Docker -This is the traditional python-django route that we will follow. Make sure PostgreSQL, SOLR and Memcached(optional) have -been setup as described above. Python3 is recommended. +This is the traditional Python-Django route that we will follow. Make sure PostgreSQL, Solr and Memcached (optional) have +been setup as described above. Python 3 is recommended. Start by cloning the repository: ```bash @@ -213,7 +213,7 @@ pip install -r app/requirements.txt ``` Once all the dependencies are installed we can move to the most important step, that is connecting various services -to the django app. If you followed the setup with Docker, you already know that we do this by using environment variables. +to the Django app. If you followed the setup with Docker, you already know that we do this by using environment variables. To set the environment variables inside your virtual env, run the following commands: ```bash @@ -241,7 +241,7 @@ export DB_NAME=webapp Setting up env variables every time is a tedious task and you might want to reduce the friction by following some really good guides: https://help.pythonanywhere.com/pages/environment-variables-for-web-apps/ -Once the database (and other services) have been connected, run the migrations. Before that let's cd into the main src +Once the database (and other services) have been connected, run the migrations. Before that let's cd into the main src directory for the project, that is `app`. ```bash @@ -261,7 +261,7 @@ python3 manage.py runserver Now that you have the app running (using either of two routes above), it is time to add some data to the app. Some of these commands take a long time in their first run. -Before proceeding make sure you are in the main django directory, i.e. `app`. +Before proceeding make sure you are in the main Django directory, i.e. `app`. **For docker:** ```bash @@ -293,7 +293,7 @@ python3 manage.py createsuperuser After this enter, username and password according to your choice- remember the credentials. ##### Add a builder -- Login to django admin dashboard using your credentials: `/admin` +- Login to Django admin dashboard using your credentials: `/admin` - Go to `BUILDHISTORY -> Builder -> Add builder` - An example of a builder is ```bash @@ -301,9 +301,9 @@ After this enter, username and password according to your choice- remember the c Simplified builder name: 10.XX: = 10.15 Name of the macOS version, e.g. Catalina: = Catalina ``` - + You may add more builders if you wish - + ##### Fetch some build history ```bash python3 manage.py fetch-build-history & @@ -311,7 +311,7 @@ python3 manage.py fetch-build-history & **NOTE**: This command should be sent to background using `&` as the fetching can proceed in the background without any issues. - + ##### Run livecheck ```bash port selfupdate && python3 manage.py run-full-livecheck & @@ -320,9 +320,9 @@ port selfupdate && python3 manage.py run-full-livecheck & **NOTE**: This is the most time consuming command. Livecheck can take 3-4 hours to finish for all ports and hence the command should always be ran in background. -##### Add a solr schema and generate index +##### Add a Solr schema and generate index - Generate Solr schema: - + **Docker**: ```bash python3 manage.py build_solr_schema --configure-directory=/solr/server/solr/tester/conf @@ -331,10 +331,10 @@ should always be ran in background. ```bash pyton3 manage.py build_solr_schema --configure-directory=path/to/solr/server/solr/tester/conf ``` - + This is because for Docker, we already the know path to Solr, but without Docker you should add path to Solr as per your machine. - + - Reload Solr core: ```bash @@ -345,14 +345,14 @@ should always be ran in background. ```bash python3 manage.py rebuild_index --noinput ``` - + All needed data has been added. ## Keeping the data up-to-date To update port information: -```bash +```bash python3 manage.py update-portinfo ``` @@ -360,7 +360,7 @@ To fetch new builds: ```bash python3 manage.py fetch-build-history ``` -**NOTE**: For all added builder, this will fetch all builds that have finished on the buildbot after the most recent +**NOTE**: For all added builder, this will fetch all builds that have finished on the buildbot after the most recent build in your database. To run livecheck again: @@ -390,15 +390,15 @@ Two crontabs can keep the up-to-date in a production environment. 1. Run the following every 10 minutes: - + ```bash python3 manage.py update-portinfo python3 manage.py fetch-build-history python3 manage.py update_index --age=5 ``` - + The 5-hour window for updating the Solr index is to make sure no builds are missed, as builds might take some time to finish. - + 2. Run the following every two days ```bash python3 manage.py run-full-livecheck From 514e01891c94b98c590a7a5a95afa65d351389bb Mon Sep 17 00:00:00 2001 From: Aaron Madlon-Kay Date: Thu, 5 May 2022 22:34:01 +0900 Subject: [PATCH 3/3] Document usage of Docker Compose dev setup --- docs/DEPLOYMENT.md | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index 92ecefeb2..c113fed63 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -21,9 +21,28 @@ As mentioned, the `app` directory is a standalone Django app. However, the app h setting up a local environment can be avoided when working with Docker. But in case you do not want to work with Docker, we will also walk through the standard process of setting up the app. -## Getting the dependencies ready +## The full-Docker way -The external dependencies of the app are: PostgreSQL, Solr and Memcached. The search page(/search/) is completely derived from +If you would like to work with the Dockerised app and dependencies, all you need to do is: + +1. Run `docker compose up` from the project root +2. Run `./docker_compose_setup.sh` from the project root +3. Access the site at http://localhost:8080 + +This will take care of all dependencies and initialization. Note, however, that in particular step 2 takes a *long* time +(several hours). Until it completes, some data will not be available in the app (ports, search, build info, livecheck +results). + +Data is stored in two places: + +- `tmp` in the repo root: webapp data and Solr data +- `db` Docker volume: PostgreSQL data + +To reset your local environment completely, be sure to delete both. + +## The other way: Getting the dependencies ready + +The external dependencies of the app are: PostgreSQL, Solr and Memcached. The search page (/search/) is completely derived from Solr. Memcached is optional, and can be avoided if you don't want to cache the results of stats and trac tickets. If you are new to these technologies, you may use the instructions provided below, otherwise you are free to install and setup these @@ -104,6 +123,9 @@ There are two options available: - Build the Docker image from source. (long but useful for development) - Download an image from the Docker registry. (ready to go in one command) +(Note that this is only using Docker for the webapp itself; if you would like to use Docker for everything, see [The +full-Docker way](#the-full-docker-way) above.) + ### 1.a: Building the image from source - Clone the repository **(for development, it is recommended that you clone a fork of the repository)**.