[DEV-14608] Change to local environment for Spark upgrade

.env.template

@@ -21,21 +21,24 @@ PROJECT_LOG_DIR=./usaspending_api/logs
 
 # ==== [Local Development] ====
 IS_LOCAL=True
-RUN_LOCAL_DOWNLOAD_IN_PROCESS=False
+RUN_LOCAL_DOWNLOAD_IN_PROCESS=True
 
 # ==== [Django] ====
 # Use env var to toggle Django's DEBUG mode for local docker envs
 DJANGO_DEBUG=False
+DJANGO_SETTINGS_MODULE=usaspending_api.settings
 
 # ==== [Postgres] ====
 # All values of USASPENDING_DB_* must match what is in DATABASE_URL if BOTH are given
 DATABASE_URL=postgres://usaspending:usaspender@usaspending-db:5432/data_store_api
+DOWNLOAD_DATABASE_URL=postgres://usaspending:usaspender@usaspending-db:5432/data_store_api
 # Change USASPENDING_DB_HOST to host.docker.internal if you are running a local Postgres server on the host machine
 # Otherwise leave as-is, so other docker containerized services will use the Postgres created by Compose.
 USASPENDING_DB_HOST=usaspending-db
 USASPENDING_DB_PORT=5432
 USASPENDING_DB_USER=usaspending
 USASPENDING_DB_PASSWORD=usaspender
+USASPENDING_DB_NAME=data_store_api
 
 # The Broker configuration below supports tests creating a Broker DB on the usaspending-db
 # container as part of standing up the test suite.
@@ -47,6 +50,15 @@ BROKER_DB_HOST=usaspending-db
 BROKER_DB_PORT=5432
 BROKER_DB_USER=usaspending
 BROKER_DB_PASSWORD=usaspender
+BROKER_DB_NAME=data_broker
+
+# If using the Broker DB container created via https://github.com/fedspendingtransparency/data-act-broker-backend
+# then the following values should be used instead
+# BROKER_DB=postgres://admin:root@host.docker.internal:5435/data_broker
+# BROKER_DB_HOST=host.docker.internal
+# BROKER_DB_PORT=5435
+# BROKER_DB_USER=admin
+# BROKER_DB_PASSWORD=root
 
 # ==== [Elasticsearch] ====
 # Where to connect to elasticsearch.
@@ -57,18 +69,20 @@ ES_HOST=usaspending-es
 ES_PORT=9200
 
 # ==== [Spark] ====
+# Logging level defaults to "warn" to prevent overloading log output in test suite
+SPARK_LOGGING_LEVEL=WARN
+SPARK_MASTER_HOST=spark-master
 SPARK_MASTER_PORT=7077
 SPARK_MASTER_WEBUI_PORT=4040
 SPARK_HISTORY_SERVER_PORT=18080
-# Should point to a path where data can be persisted beyond docker restarts, outside of the git source repository
-# The specified directory needs to exist before Docker can mount it
-SPARK_CLUSTER_DATA_DIR=${HOME}/Development/data/usaspending/docker/usaspending-spark
-# Optionally uncomment these sql warehouse and hive metastore env vars to use the explicit locations specified below
-# If NOT uncommented, config should fall back to matching values in
-#   1) These vars' values in local.py LocalConfig directed at a spark-warehouse dir under the project root
-#   2) docker-compose.yml not finding these env vars, would ALSO fall back to a spark-warehouse dir under the project root
-#SPARK_SQL_WAREHOUSE_DIR=${SPARK_CLUSTER_DATA_DIR}/spark-warehouse
-#HIVE_METASTORE_DERBY_DB_DIR=${SPARK_SQL_WAREHOUSE_DIR}/metastore_db
+SPARK_WORKER_WEBUI_PORT=4041
+# Should point to paths where data can be persisted beyond docker restarts
+# NOTE: The specified directory needs to exist before Docker can mount it
+SOURCE_SPARK_SQL_WAREHOUSE_DIR=./spark-warehouse
+SOURCE_HIVE_METASTORE_DERBY_DB_DIR=./spark-warehouse/metastore_db
+# Location in the docker containers to create and manage the Derby DB powered Hive metastore
+TARGET_SPARK_SQL_WAREHOUSE_DIR=/spark-warehouse
+TARGET_HIVE_METASTORE_DERBY_DB_DIR=/spark-warehouse/metastore_db
 # Uncomment if on MacOS. See note here: https://github.com/fedspendingtransparency/usaspending-api#test-setup for below
 #OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES
 
@@ -85,6 +99,8 @@ SPARK_CLUSTER_DATA_DIR=${HOME}/Development/data/usaspending/docker/usaspending-s
 # ==== [MinIO] ====
 MINIO_HOST=minio
 MINIO_PORT=10001
+MINIO_ACCESS_KEY=usaspending
+MINIO_SECRET_KEY=usaspender
 MINIO_CONSOLE_PORT=10002
 # Should point to a path where data can be persisted beyond docker restarts, outside of the git source repository
 # The specified directory needs to exist before Docker can mount it

.github/actions/init-python-environment/action.yaml

@@ -21,4 +21,4 @@ runs:
     - name: Install dev dependencies
       working-directory: ${{ inputs.working-directory }}
       shell: bash
-      run: pip install ".[dev,spark]"
+      run: pip install ".[dev]"

.github/actions/init-test-environment/action.yaml

@@ -14,26 +14,17 @@ inputs:
 runs:
   using: composite
   steps:
-    - name: Set combined ENV
+    - name: Create .env for docker compose
+      working-directory: ${{ inputs.working-directory }}
       shell: bash
-      run: |
-        echo "BROKER_DB=postgres://$BROKER_DB_USER:$BROKER_DB_PASSWORD@$BROKER_DB_HOST:$BROKER_DB_PORT/$BROKER_DB_NAME" >> $GITHUB_ENV
-        echo "DATABASE_URL=postgres://$USASPENDING_DB_USER:$USASPENDING_DB_PASSWORD@$USASPENDING_DB_HOST:$USASPENDING_DB_PORT/$USASPENDING_DB_NAME" >> $GITHUB_ENV
-        echo "DOWNLOAD_DATABASE_URL=postgres://$USASPENDING_DB_USER:$USASPENDING_DB_PASSWORD@$USASPENDING_DB_HOST:$USASPENDING_DB_PORT/$USASPENDING_DB_NAME" >> $GITHUB_ENV
-        echo "ES_HOSTNAME=$ES_SCHEME://$ES_HOST:$ES_PORT" >> $GITHUB_ENV
-        echo "MINIO_DATA_DIR=$HOME/Development/data/usaspending/docker/usaspending-s3" >> $GITHUB_ENV
+      run: cp .env.template .env
 
     # The conditionals below this all check against a string because inputs for actions are actually
     # strings, not booleans: https://github.com/actions/runner/issues/1483
     - name: Create directories needed for Minio
       if: ${{ inputs.is-integration-test == 'true' }}
       shell: bash
-      run: mkdir -p "$MINIO_DATA_DIR"
-
-    - name: Create directories needed for Spark
-      if: ${{ inputs.is-spark-test == 'true' }}
-      shell: bash
-      run: mkdir -p "$HOME/.ivy2"
+      run: mkdir -p "$HOME/Development/data/usaspending/docker/usaspending-s3"  # same value as .env.template
 
     - name: Build docker containers for DB, ES, and Minio
       working-directory: ${{ inputs.working-directory }}
@@ -53,58 +44,32 @@ runs:
       if: ${{ inputs.is-integration-test == 'true' }}
       run: docker load --input /tmp/dataact-broker-backend.tar
 
+    - name: Build the usaspending-development image
+      working-directory: ${{ inputs.working-directory }}
+      shell: bash
+      run: docker compose build usaspending-test
+
     - name: Wait on DB and ES containers to be available
       if: ${{ inputs.is-integration-test == 'true' }}
       shell: bash
       run: |
-        ttl=30; echo "Try DB conn from container for $ttl seconds"; until [ $ttl -le 0 ] || psql $DATABASE_URL -c 'select 1 where 1=1'; do echo $ttl; ((ttl--)); sleep 1; done; [ $ttl -gt 0 ]
-        ttl=30; echo "Try ES conn from container for $ttl seconds"; until [ $ttl -le 0 ] || curl --silent -XGET --fail $ES_HOSTNAME; do echo $ttl; ((ttl--)); sleep 1; done; [ $ttl -gt 0 ]
+        ttl=30; container_name="usaspending-db"; echo "Wait for $container_name to be healthy for $ttl seconds"; until [ $ttl -le 0 ] || [ "$(docker inspect $container_name --format='{{.State.Health.Status}}')" = "healthy" ]; do echo $ttl; ((ttl--)); sleep 1; done; [ $ttl -gt 0 ]
+        ttl=30; container_name="usaspending-es"; echo "Wait for $container_name to be healthy for $ttl seconds"; until [ $ttl -le 0 ] || [ "$(docker inspect $container_name --format='{{.State.Health.Status}}')" = "healthy" ]; do echo $ttl; ((ttl--)); sleep 1; done; [ $ttl -gt 0 ]
 
     - name: Add DB users and set search_path
       if: ${{ inputs.is-integration-test == 'true' }}
       shell: bash
       run: |
-        psql $DATABASE_URL -c "ALTER USER $USASPENDING_DB_USER SET search_path TO public,raw,int,temp,rpt"
-        psql $DATABASE_URL -c "CREATE USER $BROKER_DB_USER PASSWORD '$BROKER_DB_PASSWORD' SUPERUSER"
-        psql $DATABASE_URL -c "CREATE ROLE readonly"
-        psql $DATABASE_URL -c "SELECT 'GRANT USAGE ON SCHEMA ' || nspname || ' TO readonly' FROM pg_namespace WHERE nspname IN ('raw','int','rpt','temp','public')"
-        psql $DATABASE_URL -c "SELECT 'GRANT SELECT ON ALL TABLES IN SCHEMA ' || nspname || ' TO readonly' FROM pg_namespace WHERE nspname IN ('raw','int','rpt','temp','public')"
-        psql $DATABASE_URL -c "SELECT 'ALTER DEFAULT PRIVILEGES IN SCHEMA ' || nspname || ' GRANT SELECT ON TABLES TO readonly' FROM pg_namespace WHERE nspname IN ('raw','int','rpt','temp','public')"
-
-    - name: Capture SPARK_HOME environment variable
-      if: ${{ inputs.is-spark-test == 'true' }}
-      shell: bash
-      run: echo "SPARK_HOME=$(python -c "import site;print(f'{site.getsitepackages()[0]}/pyspark')")" >> $GITHUB_ENV
-
-    - name: Setting spark log levels to warning
-      if: ${{ inputs.is-spark-test == 'true' }}
-      shell: bash
-      run: |
-        mkdir $SPARK_HOME/conf
-        touch $SPARK_HOME/conf/log4j2.properties
-        echo "
-        appender.console.type = Console
-        appender.console.name = CONSOLE
-        appender.console.layout.type = PatternLayout
-        appender.console.layout.pattern = [%d{yyyy-MM-dd HH:mm:ss.SSS}][%p] - %m%n
-
-        rootLogger.level = WARN
-        rootLogger.appenderRef.0.ref = CONSOLE
-        rootLogger.appenderRef.0.level = WARN
-        " >> $SPARK_HOME/conf/log4j2.properties
+        docker exec usaspending-db psql -U usaspending -d data_store_api -c "ALTER USER usaspending SET search_path TO public,raw,int,temp,rpt"
+        docker exec usaspending-db psql -U usaspending -d data_store_api -c "CREATE ROLE readonly"
+        docker exec usaspending-db psql -U usaspending -d data_store_api -c "SELECT 'GRANT USAGE ON SCHEMA ' || nspname || ' TO readonly' FROM pg_namespace WHERE nspname IN ('public','raw','int','rpt','temp')"
+        docker exec usaspending-db psql -U usaspending -d data_store_api -c "SELECT 'GRANT SELECT ON ALL TABLES IN SCHEMA ' || nspname || ' TO readonly' FROM pg_namespace WHERE nspname IN ('public','raw','int','rpt','temp')"
+        docker exec usaspending-db psql -U usaspending -d data_store_api -c "SELECT 'ALTER DEFAULT PRIVILEGES IN SCHEMA ' || nspname || ' GRANT SELECT ON TABLES TO readonly' FROM pg_namespace WHERE nspname IN ('public','raw','int','rpt','temp')"
 
     - name: Run initial test to trigger DB creation
       if: ${{ inputs.is-integration-test == 'true' }}
       working-directory: ${{ inputs.working-directory }}
       shell: bash
       run: >
-        pytest --create-db --numprocesses logical --no-cov --disable-warnings -r=fEs --verbosity=3 --capture=no --log-cli-level=WARNING --show-capture=log
+        docker compose run --rm usaspending-test pytest --create-db --numprocesses auto --no-cov --disable-warnings -r=fEs --verbosity=3 --capture=no --log-cli-level=WARNING --show-capture=log
         2> /dev/null "usaspending_api/tests/integration/test_setup_of_test_dbs.py::test_trigger_test_db_setup"
-
-    - name: Run initial test to trigger JAR download
-      if: ${{ inputs.is-spark-test == 'true' }}
-      working-directory: ${{ inputs.working-directory }}
-      shell: bash
-      run: >
-        pytest --reuse-db --numprocesses logical --no-cov --disable-warnings -r=fEs --verbosity=3
-        "usaspending_api/tests/integration/test_setup_of_spark_dependencies.py::test_preload_spark_jars"

.github/actions/run-pytest/action.yaml

@@ -74,7 +74,7 @@ runs:
     - name: Run tests
       working-directory: ${{ inputs.working-directory }}
       shell: bash
-      run: ${{ steps.build-pytest-command.outputs.pytest-command }}
+      run: docker compose run --rm usaspending-test ${{ steps.build-pytest-command.outputs.pytest-command }}
 
     - name: Upload Artifact
       uses: actions/upload-artifact@v4

.github/workflows/build-broker-docker-image-for-test.yaml

@@ -2,24 +2,22 @@ name: Build Broker Docker Image for Test
 
 on:
   workflow_call:
+    inputs:
+      broker-branch:
+        required: true
+        type: string
 
 jobs:
-  # This job is a workaround that allows defining a dynamic variable that is provided as input
-  # to another job. See here: https://github.com/orgs/community/discussions/26671#discussioncomment-3931817
-  Setup-Broker-Branch:
-    uses: ./.github/workflows/determine-broker-branch.yaml
-
   Run:
     name: Run
-    needs: Setup-Broker-Branch
     runs-on: ${{ vars.RUNNER_VERSION }}
     steps:
       - name: Checkout Broker Backend
         uses: actions/checkout@v4
         with:
           repository: fedspendingtransparency/data-act-broker-backend
           path: data-act-broker-backend
-          ref: ${{ needs.Setup-Broker-Branch.outputs.branch }}
+          ref: ${{ inputs.broker-branch }}
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3

.github/workflows/code-style-checks.yaml

@@ -1,21 +1,22 @@
 name: Code Style Checks
 
 env:
-  BROKER_DB_HOST: localhost
+  BROKER_DB_HOST: usaspending-db
   BROKER_DB_PORT: 5432
   BROKER_DB_USER: admin
   BROKER_DB_PASSWORD: root
   BROKER_DB_NAME: data_broker
   DJANGO_SETTINGS_MODULE: usaspending_api.settings
   ES_SCHEME: http
-  ES_HOST: localhost
+  ES_HOST: usaspending-es
   ES_PORT: 9200
-  MINIO_HOST: localhost
-  USASPENDING_DB_HOST: localhost
+  MINIO_HOST: minio
+  USASPENDING_DB_HOST: usaspending-db
   USASPENDING_DB_PORT: 5432
   USASPENDING_DB_USER: usaspending
   USASPENDING_DB_PASSWORD: usaspender
   USASPENDING_DB_NAME: data_store_api
+  IS_LOCAL: true
 
 on:
   workflow_call:
@@ -71,6 +72,12 @@ jobs:
         if: steps.changed-python-files.outputs.all_changed_files != ''
         run: ruff check ${{ steps.changed-python-files.outputs.all_changed_files }}
 
+      - name: Run Ruff Formatter
+        if: steps.changed-python-files.outputs.all_changed_files != ''
+        run: ruff format --check ${{ steps.changed-python-files.outputs.all_changed_files }}
+
+      # TODO: Should rethink placement of this check. This runs as a Django management command which requires similar
+      #       setup steps as the rest of the test suite. Removing would reduce setup time and overall CI/CD time.
       - name: Run Check For Endpoint Documentation
         run: python manage.py check_for_endpoint_documentation
 

.github/workflows/pull-request-checks.yaml

@@ -33,66 +33,95 @@ jobs:
       needs.Cancel-If-Draft-Pull-Request.result == 'skipped'
     uses: ./.github/workflows/code-style-checks.yaml
 
+  # Determine the Broker branch to use for tests and add it as output to the workflow for consumption later
+  Setup-Broker-Branch:
+    name: Setup Broker Branch
+    needs: Cancel-If-Draft-Pull-Request
+    if: |
+      always() &&
+      needs.Cancel-If-Draft-Pull-Request.result == 'skipped'
+    uses: ./.github/workflows/determine-broker-branch.yaml
+
   # Many of the integration tests below rely on testing against Broker which requires building the
   # Broker backend image. To avoid having to run this multiple times we instead build it once
   # and store it as an artifact on the workflow.
   Build-Broker-Docker-Image:
     name: Build Broker Docker Image
-    needs: Cancel-If-Draft-Pull-Request
+    needs: Setup-Broker-Branch
     if: |
       always() &&
       needs.Cancel-If-Draft-Pull-Request.result == 'skipped'
     uses: ./.github/workflows/build-broker-docker-image-for-test.yaml
+    with:
+      broker-branch: ${{ needs.Setup-Broker-Branch.outputs.branch }}
 
   # The test suite is broken into multiple groups based on a combination of what is under test
   # and the time that it takes to run tests for that group.
   Run-Spark-Integration-Load-Transaction-FABS-and-FPDS-Tests:
     name: Run Spark Integration Load Transactions FABS and FPDS Tests
     needs:
       - Run-Code-Style-Checks
+      - Setup-Broker-Branch
       - Build-Broker-Docker-Image
     if: ${{ !failure() && !cancelled() }}
     uses: ./.github/workflows/test-spark-integration-load-transactions-fabs-fpds.yaml
+    with:
+      broker-branch: ${{ needs.Setup-Broker-Branch.outputs.branch }}
 
   Run-Spark-Integration-Load-Transactions-Lookup-Tests:
     name: Run Spark Integration Load Transactions Lookup Tests
     needs:
       - Run-Code-Style-Checks
+      - Setup-Broker-Branch
       - Build-Broker-Docker-Image
     if: ${{ !failure() && !cancelled() }}
     uses: ./.github/workflows/test-spark-integration-load-transactions-lookup.yaml
+    with:
+      broker-branch: ${{ needs.Setup-Broker-Branch.outputs.branch }}
 
   Run-Spark-Integration-Load-To-From-Delta-Tests:
     name: Run Spark Integration Load To From Delta Tests
     needs:
       - Run-Code-Style-Checks
+      - Setup-Broker-Branch
       - Build-Broker-Docker-Image
     if: ${{ !failure() && !cancelled() }}
     uses: ./.github/workflows/test-spark-integration-load-to-from-delta.yaml
+    with:
+      broker-branch: ${{ needs.Setup-Broker-Branch.outputs.branch }}
 
   Run-Spark-Integration-Other-Tests:
     name: Run Spark Integration Other Tests
     needs:
       - Run-Code-Style-Checks
+      - Setup-Broker-Branch
       - Build-Broker-Docker-Image
     if: ${{ !failure() && !cancelled() }}
     uses: ./.github/workflows/test-spark-integration-other.yaml
+    with:
+      broker-branch: ${{ needs.Setup-Broker-Branch.outputs.branch }}
 
   Run-Non-Spark-Integration-Tests:
     name: Run Non-Spark Integration Tests
     needs:
       - Run-Code-Style-Checks
+      - Setup-Broker-Branch
       - Build-Broker-Docker-Image
     if: ${{ !failure() && !cancelled() }}
     uses: ./.github/workflows/test-non-spark-integration.yaml
+    with:
+      broker-branch: ${{ needs.Setup-Broker-Branch.outputs.branch }}
 
   Run-Non-Spark-Integration-Tests-With-Signal-Handling:
     name: Run Non-Spark Integration Tests With Signal Handling
     needs:
       - Run-Code-Style-Checks
+      - Setup-Broker-Branch
       - Build-Broker-Docker-Image
     if: ${{ !failure() && !cancelled() }}
     uses: ./.github/workflows/test-non-spark-integration-signal-handling.yaml
+    with:
+      broker-branch: ${{ needs.Setup-Broker-Branch.outputs.branch }}
 
   Run-Unit-Tests:
     name: Run Unit Tests