diff --git a/.env.template b/.env.template
index a8a01996cd..6c65e8cb23 100644
--- a/.env.template
+++ b/.env.template
@@ -21,21 +21,24 @@ PROJECT_LOG_DIR=./usaspending_api/logs
 
 # ==== [Local Development] ====
 IS_LOCAL=True
-RUN_LOCAL_DOWNLOAD_IN_PROCESS=False
+RUN_LOCAL_DOWNLOAD_IN_PROCESS=True
 
 # ==== [Django] ====
 # Use env var to toggle Django's DEBUG mode for local docker envs
 DJANGO_DEBUG=False
+DJANGO_SETTINGS_MODULE=usaspending_api.settings
 
 # ==== [Postgres] ====
 # All values of USASPENDING_DB_* must match what is in DATABASE_URL if BOTH are given
 DATABASE_URL=postgres://usaspending:usaspender@usaspending-db:5432/data_store_api
+DOWNLOAD_DATABASE_URL=postgres://usaspending:usaspender@usaspending-db:5432/data_store_api
 # Change USASPENDING_DB_HOST to host.docker.internal if you are running a local Postgres server on the host machine
 # Otherwise leave as-is, so other docker containerized services will use the Postgres created by Compose.
 USASPENDING_DB_HOST=usaspending-db
 USASPENDING_DB_PORT=5432
 USASPENDING_DB_USER=usaspending
 USASPENDING_DB_PASSWORD=usaspender
+USASPENDING_DB_NAME=data_store_api
 
 # The Broker configuration below supports tests creating a Broker DB on the usaspending-db
 # container as part of standing up the test suite.
@@ -47,6 +50,15 @@ BROKER_DB_HOST=usaspending-db
 BROKER_DB_PORT=5432
 BROKER_DB_USER=usaspending
 BROKER_DB_PASSWORD=usaspender
+BROKER_DB_NAME=data_broker
+
+# If using the Broker DB container created via https://github.com/fedspendingtransparency/data-act-broker-backend
+# then the following values should be used instead
+# BROKER_DB=postgres://admin:root@host.docker.internal:5435/data_broker
+# BROKER_DB_HOST=host.docker.internal
+# BROKER_DB_PORT=5435
+# BROKER_DB_USER=admin
+# BROKER_DB_PASSWORD=root
 
 # ==== [Elasticsearch] ====
 # Where to connect to elasticsearch.
@@ -57,18 +69,20 @@ ES_HOST=usaspending-es
 ES_PORT=9200
 
 # ==== [Spark] ====
+# Logging level defaults to "warn" to prevent overloading log output in test suite
+SPARK_LOGGING_LEVEL=WARN
+SPARK_MASTER_HOST=spark-master
 SPARK_MASTER_PORT=7077
 SPARK_MASTER_WEBUI_PORT=4040
 SPARK_HISTORY_SERVER_PORT=18080
-# Should point to a path where data can be persisted beyond docker restarts, outside of the git source repository
-# The specified directory needs to exist before Docker can mount it
-SPARK_CLUSTER_DATA_DIR=${HOME}/Development/data/usaspending/docker/usaspending-spark
-# Optionally uncomment these sql warehouse and hive metastore env vars to use the explicit locations specified below
-# If NOT uncommented, config should fall back to matching values in
-#   1) These vars' values in local.py LocalConfig directed at a spark-warehouse dir under the project root
-#   2) docker-compose.yml not finding these env vars, would ALSO fall back to a spark-warehouse dir under the project root
-#SPARK_SQL_WAREHOUSE_DIR=${SPARK_CLUSTER_DATA_DIR}/spark-warehouse
-#HIVE_METASTORE_DERBY_DB_DIR=${SPARK_SQL_WAREHOUSE_DIR}/metastore_db
+SPARK_WORKER_WEBUI_PORT=4041
+# Should point to paths where data can be persisted beyond docker restarts
+# NOTE: The specified directory needs to exist before Docker can mount it
+SOURCE_SPARK_SQL_WAREHOUSE_DIR=./spark-warehouse
+SOURCE_HIVE_METASTORE_DERBY_DB_DIR=./spark-warehouse/metastore_db
+# Location in the docker containers to create and manage the Derby DB powered Hive metastore
+TARGET_SPARK_SQL_WAREHOUSE_DIR=/spark-warehouse
+TARGET_HIVE_METASTORE_DERBY_DB_DIR=/spark-warehouse/metastore_db
 # Uncomment if on MacOS. See note here: https://github.com/fedspendingtransparency/usaspending-api#test-setup for below
 #OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES
 
@@ -85,6 +99,8 @@ SPARK_CLUSTER_DATA_DIR=${HOME}/Development/data/usaspending/docker/usaspending-s
 # ==== [MinIO] ====
 MINIO_HOST=minio
 MINIO_PORT=10001
+MINIO_ACCESS_KEY=usaspending
+MINIO_SECRET_KEY=usaspender
 MINIO_CONSOLE_PORT=10002
 # Should point to a path where data can be persisted beyond docker restarts, outside of the git source repository
 # The specified directory needs to exist before Docker can mount it
diff --git a/.github/actions/init-python-environment/action.yaml b/.github/actions/init-python-environment/action.yaml
index 9e77fe4618..503dcfe68f 100644
--- a/.github/actions/init-python-environment/action.yaml
+++ b/.github/actions/init-python-environment/action.yaml
@@ -21,4 +21,4 @@ runs:
     - name: Install dev dependencies
       working-directory: ${{ inputs.working-directory }}
       shell: bash
-      run: pip install ".[dev,spark]"
+      run: pip install ".[dev]"
diff --git a/.github/actions/init-test-environment/action.yaml b/.github/actions/init-test-environment/action.yaml
index f5b6f2d85d..3c97dc2d67 100644
--- a/.github/actions/init-test-environment/action.yaml
+++ b/.github/actions/init-test-environment/action.yaml
@@ -14,26 +14,17 @@ inputs:
 runs:
   using: composite
   steps:
-    - name: Set combined ENV
+    - name: Create .env for docker compose
+      working-directory: ${{ inputs.working-directory }}
       shell: bash
-      run: |
-        echo "BROKER_DB=postgres://$BROKER_DB_USER:$BROKER_DB_PASSWORD@$BROKER_DB_HOST:$BROKER_DB_PORT/$BROKER_DB_NAME" >> $GITHUB_ENV
-        echo "DATABASE_URL=postgres://$USASPENDING_DB_USER:$USASPENDING_DB_PASSWORD@$USASPENDING_DB_HOST:$USASPENDING_DB_PORT/$USASPENDING_DB_NAME" >> $GITHUB_ENV
-        echo "DOWNLOAD_DATABASE_URL=postgres://$USASPENDING_DB_USER:$USASPENDING_DB_PASSWORD@$USASPENDING_DB_HOST:$USASPENDING_DB_PORT/$USASPENDING_DB_NAME" >> $GITHUB_ENV
-        echo "ES_HOSTNAME=$ES_SCHEME://$ES_HOST:$ES_PORT" >> $GITHUB_ENV
-        echo "MINIO_DATA_DIR=$HOME/Development/data/usaspending/docker/usaspending-s3" >> $GITHUB_ENV
+      run: cp .env.template .env
 
     # The conditionals below this all check against a string because inputs for actions are actually
     # strings, not booleans: https://github.com/actions/runner/issues/1483
     - name: Create directories needed for Minio
       if: ${{ inputs.is-integration-test == 'true' }}
       shell: bash
-      run: mkdir -p "$MINIO_DATA_DIR"
-
-    - name: Create directories needed for Spark
-      if: ${{ inputs.is-spark-test == 'true' }}
-      shell: bash
-      run: mkdir -p "$HOME/.ivy2"
+      run: mkdir -p "$HOME/Development/data/usaspending/docker/usaspending-s3"  # same value as .env.template
 
     - name: Build docker containers for DB, ES, and Minio
       working-directory: ${{ inputs.working-directory }}
@@ -53,58 +44,32 @@ runs:
       if: ${{ inputs.is-integration-test == 'true' }}
       run: docker load --input /tmp/dataact-broker-backend.tar
 
+    - name: Build the usaspending-development image
+      working-directory: ${{ inputs.working-directory }}
+      shell: bash
+      run: docker compose build usaspending-test
+
     - name: Wait on DB and ES containers to be available
       if: ${{ inputs.is-integration-test == 'true' }}
       shell: bash
       run: |
-        ttl=30; echo "Try DB conn from container for $ttl seconds"; until [ $ttl -le 0 ] || psql $DATABASE_URL -c 'select 1 where 1=1'; do echo $ttl; ((ttl--)); sleep 1; done; [ $ttl -gt 0 ]
-        ttl=30; echo "Try ES conn from container for $ttl seconds"; until [ $ttl -le 0 ] || curl --silent -XGET --fail $ES_HOSTNAME; do echo $ttl; ((ttl--)); sleep 1; done; [ $ttl -gt 0 ]
+        ttl=30; container_name="usaspending-db"; echo "Wait for $container_name to be healthy for $ttl seconds"; until [ $ttl -le 0 ] || [ "$(docker inspect $container_name --format='{{.State.Health.Status}}')" = "healthy" ]; do echo $ttl; ((ttl--)); sleep 1; done; [ $ttl -gt 0 ]
+        ttl=30; container_name="usaspending-es"; echo "Wait for $container_name to be healthy for $ttl seconds"; until [ $ttl -le 0 ] || [ "$(docker inspect $container_name --format='{{.State.Health.Status}}')" = "healthy" ]; do echo $ttl; ((ttl--)); sleep 1; done; [ $ttl -gt 0 ]
 
     - name: Add DB users and set search_path
       if: ${{ inputs.is-integration-test == 'true' }}
       shell: bash
       run: |
-        psql $DATABASE_URL -c "ALTER USER $USASPENDING_DB_USER SET search_path TO public,raw,int,temp,rpt"
-        psql $DATABASE_URL -c "CREATE USER $BROKER_DB_USER PASSWORD '$BROKER_DB_PASSWORD' SUPERUSER"
-        psql $DATABASE_URL -c "CREATE ROLE readonly"
-        psql $DATABASE_URL -c "SELECT 'GRANT USAGE ON SCHEMA ' || nspname || ' TO readonly' FROM pg_namespace WHERE nspname IN ('raw','int','rpt','temp','public')"
-        psql $DATABASE_URL -c "SELECT 'GRANT SELECT ON ALL TABLES IN SCHEMA ' || nspname || ' TO readonly' FROM pg_namespace WHERE nspname IN ('raw','int','rpt','temp','public')"
-        psql $DATABASE_URL -c "SELECT 'ALTER DEFAULT PRIVILEGES IN SCHEMA ' || nspname || ' GRANT SELECT ON TABLES TO readonly' FROM pg_namespace WHERE nspname IN ('raw','int','rpt','temp','public')"
-
-    - name: Capture SPARK_HOME environment variable
-      if: ${{ inputs.is-spark-test == 'true' }}
-      shell: bash
-      run: echo "SPARK_HOME=$(python -c "import site;print(f'{site.getsitepackages()[0]}/pyspark')")" >> $GITHUB_ENV
-
-    - name: Setting spark log levels to warning
-      if: ${{ inputs.is-spark-test == 'true' }}
-      shell: bash
-      run: |
-        mkdir $SPARK_HOME/conf
-        touch $SPARK_HOME/conf/log4j2.properties
-        echo "
-        appender.console.type = Console
-        appender.console.name = CONSOLE
-        appender.console.layout.type = PatternLayout
-        appender.console.layout.pattern = [%d{yyyy-MM-dd HH:mm:ss.SSS}][%p] - %m%n
-
-        rootLogger.level = WARN
-        rootLogger.appenderRef.0.ref = CONSOLE
-        rootLogger.appenderRef.0.level = WARN
-        " >> $SPARK_HOME/conf/log4j2.properties
+        docker exec usaspending-db psql -U usaspending -d data_store_api -c "ALTER USER usaspending SET search_path TO public,raw,int,temp,rpt"
+        docker exec usaspending-db psql -U usaspending -d data_store_api -c "CREATE ROLE readonly"
+        docker exec usaspending-db psql -U usaspending -d data_store_api -c "SELECT 'GRANT USAGE ON SCHEMA ' || nspname || ' TO readonly' FROM pg_namespace WHERE nspname IN ('public','raw','int','rpt','temp')"
+        docker exec usaspending-db psql -U usaspending -d data_store_api -c "SELECT 'GRANT SELECT ON ALL TABLES IN SCHEMA ' || nspname || ' TO readonly' FROM pg_namespace WHERE nspname IN ('public','raw','int','rpt','temp')"
+        docker exec usaspending-db psql -U usaspending -d data_store_api -c "SELECT 'ALTER DEFAULT PRIVILEGES IN SCHEMA ' || nspname || ' GRANT SELECT ON TABLES TO readonly' FROM pg_namespace WHERE nspname IN ('public','raw','int','rpt','temp')"
 
     - name: Run initial test to trigger DB creation
       if: ${{ inputs.is-integration-test == 'true' }}
       working-directory: ${{ inputs.working-directory }}
       shell: bash
       run: >
-        pytest --create-db --numprocesses logical --no-cov --disable-warnings -r=fEs --verbosity=3 --capture=no --log-cli-level=WARNING --show-capture=log
+        docker compose run --rm usaspending-test pytest --create-db --numprocesses auto --no-cov --disable-warnings -r=fEs --verbosity=3 --capture=no --log-cli-level=WARNING --show-capture=log
         2> /dev/null "usaspending_api/tests/integration/test_setup_of_test_dbs.py::test_trigger_test_db_setup"
-
-    - name: Run initial test to trigger JAR download
-      if: ${{ inputs.is-spark-test == 'true' }}
-      working-directory: ${{ inputs.working-directory }}
-      shell: bash
-      run: >
-        pytest --reuse-db --numprocesses logical --no-cov --disable-warnings -r=fEs --verbosity=3
-        "usaspending_api/tests/integration/test_setup_of_spark_dependencies.py::test_preload_spark_jars"
diff --git a/.github/actions/run-pytest/action.yaml b/.github/actions/run-pytest/action.yaml
index 84d720fc3f..f09613c181 100644
--- a/.github/actions/run-pytest/action.yaml
+++ b/.github/actions/run-pytest/action.yaml
@@ -74,7 +74,7 @@ runs:
     - name: Run tests
       working-directory: ${{ inputs.working-directory }}
       shell: bash
-      run: ${{ steps.build-pytest-command.outputs.pytest-command }}
+      run: docker compose run --rm usaspending-test ${{ steps.build-pytest-command.outputs.pytest-command }}
 
     - name: Upload Artifact
       uses: actions/upload-artifact@v4
diff --git a/.github/workflows/build-broker-docker-image-for-test.yaml b/.github/workflows/build-broker-docker-image-for-test.yaml
index a089e97546..88f356301d 100644
--- a/.github/workflows/build-broker-docker-image-for-test.yaml
+++ b/.github/workflows/build-broker-docker-image-for-test.yaml
@@ -2,16 +2,14 @@ name: Build Broker Docker Image for Test
 
 on:
   workflow_call:
+    inputs:
+      broker-branch:
+        required: true
+        type: string
 
 jobs:
-  # This job is a workaround that allows defining a dynamic variable that is provided as input
-  # to another job. See here: https://github.com/orgs/community/discussions/26671#discussioncomment-3931817
-  Setup-Broker-Branch:
-    uses: ./.github/workflows/determine-broker-branch.yaml
-
   Run:
     name: Run
-    needs: Setup-Broker-Branch
     runs-on: ${{ vars.RUNNER_VERSION }}
     steps:
       - name: Checkout Broker Backend
@@ -19,7 +17,7 @@ jobs:
         with:
           repository: fedspendingtransparency/data-act-broker-backend
           path: data-act-broker-backend
-          ref: ${{ needs.Setup-Broker-Branch.outputs.branch }}
+          ref: ${{ inputs.broker-branch }}
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
diff --git a/.github/workflows/code-style-checks.yaml b/.github/workflows/code-style-checks.yaml
index fabb217a47..c8ed9603ca 100644
--- a/.github/workflows/code-style-checks.yaml
+++ b/.github/workflows/code-style-checks.yaml
@@ -1,21 +1,22 @@
 name: Code Style Checks
 
 env:
-  BROKER_DB_HOST: localhost
+  BROKER_DB_HOST: usaspending-db
   BROKER_DB_PORT: 5432
   BROKER_DB_USER: admin
   BROKER_DB_PASSWORD: root
   BROKER_DB_NAME: data_broker
   DJANGO_SETTINGS_MODULE: usaspending_api.settings
   ES_SCHEME: http
-  ES_HOST: localhost
+  ES_HOST: usaspending-es
   ES_PORT: 9200
-  MINIO_HOST: localhost
-  USASPENDING_DB_HOST: localhost
+  MINIO_HOST: minio
+  USASPENDING_DB_HOST: usaspending-db
   USASPENDING_DB_PORT: 5432
   USASPENDING_DB_USER: usaspending
   USASPENDING_DB_PASSWORD: usaspender
   USASPENDING_DB_NAME: data_store_api
+  IS_LOCAL: true
 
 on:
   workflow_call:
@@ -71,6 +72,12 @@ jobs:
         if: steps.changed-python-files.outputs.all_changed_files != ''
         run: ruff check ${{ steps.changed-python-files.outputs.all_changed_files }}
 
+      - name: Run Ruff Formatter
+        if: steps.changed-python-files.outputs.all_changed_files != ''
+        run: ruff format --check ${{ steps.changed-python-files.outputs.all_changed_files }}
+
+      # TODO: Should rethink placement of this check. This runs as a Django management command which requires similar
+      #       setup steps as the rest of the test suite. Removing would reduce setup time and overall CI/CD time.
       - name: Run Check For Endpoint Documentation
         run: python manage.py check_for_endpoint_documentation
 
diff --git a/.github/workflows/pull-request-checks.yaml b/.github/workflows/pull-request-checks.yaml
index 04c83ecf09..81e360fbf1 100644
--- a/.github/workflows/pull-request-checks.yaml
+++ b/.github/workflows/pull-request-checks.yaml
@@ -33,16 +33,27 @@ jobs:
       needs.Cancel-If-Draft-Pull-Request.result == 'skipped'
     uses: ./.github/workflows/code-style-checks.yaml
 
+  # Determine the Broker branch to use for tests and add it as output to the workflow for consumption later
+  Setup-Broker-Branch:
+    name: Setup Broker Branch
+    needs: Cancel-If-Draft-Pull-Request
+    if: |
+      always() &&
+      needs.Cancel-If-Draft-Pull-Request.result == 'skipped'
+    uses: ./.github/workflows/determine-broker-branch.yaml
+
   # Many of the integration tests below rely on testing against Broker which requires building the
   # Broker backend image. To avoid having to run this multiple times we instead build it once
   # and store it as an artifact on the workflow.
   Build-Broker-Docker-Image:
     name: Build Broker Docker Image
-    needs: Cancel-If-Draft-Pull-Request
+    needs: Setup-Broker-Branch
     if: |
       always() &&
       needs.Cancel-If-Draft-Pull-Request.result == 'skipped'
     uses: ./.github/workflows/build-broker-docker-image-for-test.yaml
+    with:
+      broker-branch: ${{ needs.Setup-Broker-Branch.outputs.branch }}
 
   # The test suite is broken into multiple groups based on a combination of what is under test
   # and the time that it takes to run tests for that group.
@@ -50,49 +61,67 @@ jobs:
     name: Run Spark Integration Load Transactions FABS and FPDS Tests
     needs:
       - Run-Code-Style-Checks
+      - Setup-Broker-Branch
       - Build-Broker-Docker-Image
     if: ${{ !failure() && !cancelled() }}
     uses: ./.github/workflows/test-spark-integration-load-transactions-fabs-fpds.yaml
+    with:
+      broker-branch: ${{ needs.Setup-Broker-Branch.outputs.branch }}
 
   Run-Spark-Integration-Load-Transactions-Lookup-Tests:
     name: Run Spark Integration Load Transactions Lookup Tests
     needs:
       - Run-Code-Style-Checks
+      - Setup-Broker-Branch
       - Build-Broker-Docker-Image
     if: ${{ !failure() && !cancelled() }}
     uses: ./.github/workflows/test-spark-integration-load-transactions-lookup.yaml
+    with:
+      broker-branch: ${{ needs.Setup-Broker-Branch.outputs.branch }}
 
   Run-Spark-Integration-Load-To-From-Delta-Tests:
     name: Run Spark Integration Load To From Delta Tests
     needs:
       - Run-Code-Style-Checks
+      - Setup-Broker-Branch
       - Build-Broker-Docker-Image
     if: ${{ !failure() && !cancelled() }}
     uses: ./.github/workflows/test-spark-integration-load-to-from-delta.yaml
+    with:
+      broker-branch: ${{ needs.Setup-Broker-Branch.outputs.branch }}
 
   Run-Spark-Integration-Other-Tests:
     name: Run Spark Integration Other Tests
     needs:
       - Run-Code-Style-Checks
+      - Setup-Broker-Branch
       - Build-Broker-Docker-Image
     if: ${{ !failure() && !cancelled() }}
     uses: ./.github/workflows/test-spark-integration-other.yaml
+    with:
+      broker-branch: ${{ needs.Setup-Broker-Branch.outputs.branch }}
 
   Run-Non-Spark-Integration-Tests:
     name: Run Non-Spark Integration Tests
     needs:
       - Run-Code-Style-Checks
+      - Setup-Broker-Branch
       - Build-Broker-Docker-Image
     if: ${{ !failure() && !cancelled() }}
     uses: ./.github/workflows/test-non-spark-integration.yaml
+    with:
+      broker-branch: ${{ needs.Setup-Broker-Branch.outputs.branch }}
 
   Run-Non-Spark-Integration-Tests-With-Signal-Handling:
     name: Run Non-Spark Integration Tests With Signal Handling
     needs:
       - Run-Code-Style-Checks
+      - Setup-Broker-Branch
       - Build-Broker-Docker-Image
     if: ${{ !failure() && !cancelled() }}
     uses: ./.github/workflows/test-non-spark-integration-signal-handling.yaml
+    with:
+      broker-branch: ${{ needs.Setup-Broker-Branch.outputs.branch }}
 
   Run-Unit-Tests:
     name: Run Unit Tests
diff --git a/.github/workflows/test-non-spark-integration-signal-handling.yaml b/.github/workflows/test-non-spark-integration-signal-handling.yaml
index 6b69ac35ac..3ef463d7ab 100644
--- a/.github/workflows/test-non-spark-integration-signal-handling.yaml
+++ b/.github/workflows/test-non-spark-integration-signal-handling.yaml
@@ -1,37 +1,15 @@
 name: Non-Spark Integration Tests with Signal Handling
 
-env:
-  BROKER_DB_HOST: localhost
-  BROKER_DB_PORT: 5432
-  BROKER_DB_USER: admin
-  BROKER_DB_PASSWORD: root
-  BROKER_DB_NAME: data_broker
-  DJANGO_SETTINGS_MODULE: usaspending_api.settings
-  ES_SCHEME: http
-  ES_HOST: localhost
-  ES_PORT: 9200
-  MINIO_HOST: localhost
-  USASPENDING_DB_HOST: localhost
-  USASPENDING_DB_PORT: 5432
-  USASPENDING_DB_USER: usaspending
-  USASPENDING_DB_PASSWORD: usaspender
-  USASPENDING_DB_NAME: data_store_api
-  IS_LOCAL: true
-
 on:
   workflow_call:
-
-defaults:
-  run:
-    working-directory: usaspending-api
+    inputs:
+      broker-branch:
+        required: true
+        type: string
 
 jobs:
-  Setup-Broker-Branch:
-    uses: ./.github/workflows/determine-broker-branch.yaml
-
   Run:
     name: Run
-    needs: Setup-Broker-Branch
     runs-on: ${{ vars.RUNNER_VERSION }}
     steps:
       - name: Checkout Source Repository
@@ -44,12 +22,7 @@ jobs:
         with:
           repository: fedspendingtransparency/data-act-broker-backend
           path: data-act-broker-backend
-          ref: ${{ needs.Setup-Broker-Branch.outputs.branch }}
-
-      - name: Init Python Environment
-        uses: ./usaspending-api/.github/actions/init-python-environment
-        with:
-          working-directory: ./usaspending-api
+          ref: ${{ inputs.broker-branch }}
 
       - name: Init Test Environment
         uses: ./usaspending-api/.github/actions/init-test-environment
diff --git a/.github/workflows/test-non-spark-integration.yaml b/.github/workflows/test-non-spark-integration.yaml
index 52a40a221f..9fa61d0c74 100644
--- a/.github/workflows/test-non-spark-integration.yaml
+++ b/.github/workflows/test-non-spark-integration.yaml
@@ -1,34 +1,13 @@
 name: Non-Spark Integration Tests
 
-env:
-  BROKER_DB_HOST: localhost
-  BROKER_DB_PORT: 5432
-  BROKER_DB_USER: admin
-  BROKER_DB_PASSWORD: root
-  BROKER_DB_NAME: data_broker
-  DJANGO_SETTINGS_MODULE: usaspending_api.settings
-  ES_SCHEME: http
-  ES_HOST: localhost
-  ES_PORT: 9200
-  MINIO_HOST: localhost
-  USASPENDING_DB_HOST: localhost
-  USASPENDING_DB_PORT: 5432
-  USASPENDING_DB_USER: usaspending
-  USASPENDING_DB_PASSWORD: usaspender
-  USASPENDING_DB_NAME: data_store_api
-  IS_LOCAL: true
-
 on:
   workflow_call:
-
-defaults:
-  run:
-    working-directory: usaspending-api
+    inputs:
+      broker-branch:
+        required: true
+        type: string
 
 jobs:
-  Setup-Broker-Branch:
-    uses: ./.github/workflows/determine-broker-branch.yaml
-
   Run:
     name: Run
     runs-on: ${{ vars.RUNNER_VERSION }}
@@ -43,12 +22,7 @@ jobs:
         with:
           repository: fedspendingtransparency/data-act-broker-backend
           path: data-act-broker-backend
-          ref: ${{ needs.Setup-Broker-Branch.outputs.branch }}
-
-      - name: Init Python Environment
-        uses: ./usaspending-api/.github/actions/init-python-environment
-        with:
-          working-directory: ./usaspending-api
+          ref: ${{ inputs.broker-branch }}
 
       - name: Init Test Environment
         uses: ./usaspending-api/.github/actions/init-test-environment
diff --git a/.github/workflows/test-spark-integration-load-to-from-delta.yaml b/.github/workflows/test-spark-integration-load-to-from-delta.yaml
index cf9ab7c1cf..ba3ea489a5 100644
--- a/.github/workflows/test-spark-integration-load-to-from-delta.yaml
+++ b/.github/workflows/test-spark-integration-load-to-from-delta.yaml
@@ -1,34 +1,13 @@
 name: Spark Integration Tests - test_load_to_from_delta.py
 
-env:
-  BROKER_DB_HOST: localhost
-  BROKER_DB_PORT: 5432
-  BROKER_DB_USER: admin
-  BROKER_DB_PASSWORD: root
-  BROKER_DB_NAME: data_broker
-  DJANGO_SETTINGS_MODULE: usaspending_api.settings
-  ES_SCHEME: http
-  ES_HOST: localhost
-  ES_PORT: 9200
-  MINIO_HOST: localhost
-  USASPENDING_DB_HOST: localhost
-  USASPENDING_DB_PORT: 5432
-  USASPENDING_DB_USER: usaspending
-  USASPENDING_DB_PASSWORD: usaspender
-  USASPENDING_DB_NAME: data_store_api
-  IS_LOCAL: true
-
 on:
   workflow_call:
-
-defaults:
-  run:
-    working-directory: ./usaspending-api
+    inputs:
+      broker-branch:
+        required: true
+        type: string
 
 jobs:
-  Setup-Broker-Branch:
-    uses: ./.github/workflows/determine-broker-branch.yaml
-
   Run:
     name: Run
     runs-on: ${{ vars.RUNNER_VERSION }}
@@ -43,12 +22,7 @@ jobs:
         with:
           repository: fedspendingtransparency/data-act-broker-backend
           path: data-act-broker-backend
-          ref: ${{ needs.Setup-Broker-Branch.outputs.branch }}
-
-      - name: Init Python Environment
-        uses: ./usaspending-api/.github/actions/init-python-environment
-        with:
-          working-directory: ./usaspending-api
+          ref: ${{ inputs.broker-branch }}
 
       - name: Init Test Environment
         uses: ./usaspending-api/.github/actions/init-test-environment
diff --git a/.github/workflows/test-spark-integration-load-transactions-fabs-fpds.yaml b/.github/workflows/test-spark-integration-load-transactions-fabs-fpds.yaml
index cfef1cb856..48f20fdec3 100644
--- a/.github/workflows/test-spark-integration-load-transactions-fabs-fpds.yaml
+++ b/.github/workflows/test-spark-integration-load-transactions-fabs-fpds.yaml
@@ -1,34 +1,13 @@
 name: Spark Integration Tests - test_load_transactions_in_delta_fabs_fpds.py
 
-env:
-  BROKER_DB_HOST: localhost
-  BROKER_DB_PORT: 5432
-  BROKER_DB_USER: admin
-  BROKER_DB_PASSWORD: root
-  BROKER_DB_NAME: data_broker
-  DJANGO_SETTINGS_MODULE: usaspending_api.settings
-  ES_SCHEME: http
-  ES_HOST: localhost
-  ES_PORT: 9200
-  MINIO_HOST: localhost
-  USASPENDING_DB_HOST: localhost
-  USASPENDING_DB_PORT: 5432
-  USASPENDING_DB_USER: usaspending
-  USASPENDING_DB_PASSWORD: usaspender
-  USASPENDING_DB_NAME: data_store_api
-  IS_LOCAL: true
-
 on:
   workflow_call:
-
-defaults:
-  run:
-    working-directory: ./usaspending-api
+    inputs:
+      broker-branch:
+        required: true
+        type: string
 
 jobs:
-  Setup-Broker-Branch:
-    uses: ./.github/workflows/determine-broker-branch.yaml
-
   Run:
     name: Run
     runs-on: ${{ vars.RUNNER_VERSION }}
@@ -43,12 +22,7 @@ jobs:
         with:
           repository: fedspendingtransparency/data-act-broker-backend
           path: data-act-broker-backend
-          ref: ${{ needs.Setup-Broker-Branch.outputs.branch }}
-
-      - name: Init Python Environment
-        uses: ./usaspending-api/.github/actions/init-python-environment
-        with:
-          working-directory: ./usaspending-api
+          ref: ${{ inputs.broker-branch }}
 
       - name: Init Test Environment
         uses: ./usaspending-api/.github/actions/init-test-environment
diff --git a/.github/workflows/test-spark-integration-load-transactions-lookup.yaml b/.github/workflows/test-spark-integration-load-transactions-lookup.yaml
index 329c37d612..1f6b870d65 100644
--- a/.github/workflows/test-spark-integration-load-transactions-lookup.yaml
+++ b/.github/workflows/test-spark-integration-load-transactions-lookup.yaml
@@ -1,34 +1,13 @@
 name: Spark Integration Tests - test_load_transactions_in_delta_lookups.py
 
-env:
-  BROKER_DB_HOST: localhost
-  BROKER_DB_PORT: 5432
-  BROKER_DB_USER: admin
-  BROKER_DB_PASSWORD: root
-  BROKER_DB_NAME: data_broker
-  DJANGO_SETTINGS_MODULE: usaspending_api.settings
-  ES_SCHEME: http
-  ES_HOST: localhost
-  ES_PORT: 9200
-  MINIO_HOST: localhost
-  USASPENDING_DB_HOST: localhost
-  USASPENDING_DB_PORT: 5432
-  USASPENDING_DB_USER: usaspending
-  USASPENDING_DB_PASSWORD: usaspender
-  USASPENDING_DB_NAME: data_store_api
-  IS_LOCAL: true
-
 on:
   workflow_call:
-
-defaults:
-  run:
-    working-directory: ./usaspending-api
+    inputs:
+      broker-branch:
+        required: true
+        type: string
 
 jobs:
-  Setup-Broker-Branch:
-    uses: ./.github/workflows/determine-broker-branch.yaml
-
   Run:
     name: Run
     runs-on: ${{ vars.RUNNER_VERSION }}
@@ -43,12 +22,7 @@ jobs:
         with:
           repository: fedspendingtransparency/data-act-broker-backend
           path: data-act-broker-backend
-          ref: ${{ needs.Setup-Broker-Branch.outputs.branch }}
-
-      - name: Init Python Environment
-        uses: ./usaspending-api/.github/actions/init-python-environment
-        with:
-          working-directory: ./usaspending-api
+          ref: ${{ inputs.broker-branch }}
 
       - name: Init Test Environment
         uses: ./usaspending-api/.github/actions/init-test-environment
diff --git a/.github/workflows/test-spark-integration-other.yaml b/.github/workflows/test-spark-integration-other.yaml
index 9545878495..08ca247009 100644
--- a/.github/workflows/test-spark-integration-other.yaml
+++ b/.github/workflows/test-spark-integration-other.yaml
@@ -1,34 +1,13 @@
 name: Spark Integration Tests - Other
 
-env:
-  BROKER_DB_HOST: localhost
-  BROKER_DB_PORT: 5432
-  BROKER_DB_USER: admin
-  BROKER_DB_PASSWORD: root
-  BROKER_DB_NAME: data_broker
-  DJANGO_SETTINGS_MODULE: usaspending_api.settings
-  ES_SCHEME: http
-  ES_HOST: localhost
-  ES_PORT: 9200
-  MINIO_HOST: localhost
-  USASPENDING_DB_HOST: localhost
-  USASPENDING_DB_PORT: 5432
-  USASPENDING_DB_USER: usaspending
-  USASPENDING_DB_PASSWORD: usaspender
-  USASPENDING_DB_NAME: data_store_api
-  IS_LOCAL: true
-
 on:
   workflow_call:
-
-defaults:
-  run:
-    working-directory: ./usaspending-api
+    inputs:
+      broker-branch:
+        required: true
+        type: string
 
 jobs:
-  Setup-Broker-Branch:
-    uses: ./.github/workflows/determine-broker-branch.yaml
-
   Run:
     name: Run
     runs-on: ${{ vars.RUNNER_VERSION }}
@@ -43,12 +22,7 @@ jobs:
         with:
           repository: fedspendingtransparency/data-act-broker-backend
           path: data-act-broker-backend
-          ref: ${{ needs.Setup-Broker-Branch.outputs.branch }}
-
-      - name: Init Python Environment
-        uses: ./usaspending-api/.github/actions/init-python-environment
-        with:
-          working-directory: ./usaspending-api
+          ref: ${{ inputs.broker-branch }}
 
       - name: Init Test Environment
         uses: ./usaspending-api/.github/actions/init-test-environment
diff --git a/.github/workflows/test-unit-signal-handling.yaml b/.github/workflows/test-unit-signal-handling.yaml
index 858799e417..59acf3b974 100644
--- a/.github/workflows/test-unit-signal-handling.yaml
+++ b/.github/workflows/test-unit-signal-handling.yaml
@@ -1,23 +1,5 @@
 name: Unit Tests with Signal Handling
 
-env:
-  BROKER_DB_HOST: localhost
-  BROKER_DB_PORT: 5432
-  BROKER_DB_USER: admin
-  BROKER_DB_PASSWORD: root
-  BROKER_DB_NAME: data_broker
-  DJANGO_SETTINGS_MODULE: usaspending_api.settings
-  ES_SCHEME: http
-  ES_HOST: localhost
-  ES_PORT: 9200
-  MINIO_HOST: localhost
-  USASPENDING_DB_HOST: localhost
-  USASPENDING_DB_PORT: 5432
-  USASPENDING_DB_USER: usaspending
-  USASPENDING_DB_PASSWORD: usaspender
-  USASPENDING_DB_NAME: data_store_api
-  IS_LOCAL: true
-
 on:
   workflow_call:
 
@@ -31,11 +13,6 @@ jobs:
         with:
           path: usaspending-api
 
-      - name: Init Python Environment
-        uses: ./usaspending-api/.github/actions/init-python-environment
-        with:
-          working-directory: ./usaspending-api
-
       - name: Init Test Environment
         uses: ./usaspending-api/.github/actions/init-test-environment
         with:
diff --git a/.github/workflows/test-unit.yaml b/.github/workflows/test-unit.yaml
index bccbd056b7..7901864eca 100644
--- a/.github/workflows/test-unit.yaml
+++ b/.github/workflows/test-unit.yaml
@@ -1,23 +1,5 @@
 name: Unit Tests
 
-env:
-  BROKER_DB_HOST: localhost
-  BROKER_DB_PORT: 5432
-  BROKER_DB_USER: admin
-  BROKER_DB_PASSWORD: root
-  BROKER_DB_NAME: data_broker
-  DJANGO_SETTINGS_MODULE: usaspending_api.settings
-  ES_SCHEME: http
-  ES_HOST: localhost
-  ES_PORT: 9200
-  MINIO_HOST: localhost
-  USASPENDING_DB_HOST: localhost
-  USASPENDING_DB_PORT: 5432
-  USASPENDING_DB_USER: usaspending
-  USASPENDING_DB_PASSWORD: usaspender
-  USASPENDING_DB_NAME: data_store_api
-  IS_LOCAL: true
-
 on:
   workflow_call:
 
@@ -31,11 +13,6 @@ jobs:
         with:
           path: usaspending-api
 
-      - name: Init Python Environment
-        uses: ./usaspending-api/.github/actions/init-python-environment
-        with:
-          working-directory: ./usaspending-api
-
       - name: Init Test Environment
         uses: ./usaspending-api/.github/actions/init-test-environment
         with:
diff --git a/.gitignore b/.gitignore
index 73c050cd42..3631c65eb0 100644
--- a/.gitignore
+++ b/.gitignore
@@ -12,6 +12,7 @@ chunked_matviews/**
 csv_downloads/**
 data/**
 derby.log
+dredd-results.txt
 log/**
 logs/**
 matviews/**
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 850453ca2c..3dd3819f2c 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -8,4 +8,6 @@ repos:
     rev: v0.14.14
     hooks:
         - id: ruff-check
+          args: [ --fix ]
           types_or: [ python, pyi ]  # avoid linting other python file types like Jupyter notebooks
+        - id: ruff-format
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 2d8c7902e8..6ff1325c14 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -30,14 +30,10 @@ In general, follow the form `<prefix>/[dev|gh]-###-short-description`
 
 ## Code Style
 Use of these tools locally creates an excellent chance of passing our automated CI syntax checks.
-* [flake8](https://flake8.pycqa.org/en/latest/)
-    * Will be installed when following the steps in the [readme](README.md)
-    * From repo root dir, run: `flake8`
-    * If using a plugin in an IDE or code editor, ensure the configuration matches the `[flake8]` section in [setup.cfg](setup.cfg)
-* [black](https://black.readthedocs.io/en/stable/)
-    * Will be installed when following the steps in the [readme](README.md)
-    * From repo root dir, run: `black .`
-    * If using a plugin in an IDE or code editor, ensure the Black tool configuration matches the `[tool.black]` section in [pyproject.toml](pyproject.toml)
+* [ruff](https://docs.astral.sh/ruff/)
+    * Installed as part of the dev dependencies
+    * Can be run locally via `uv run ruff check ...` or `uv run ruff format ...`
+    * Optionally, can use the `docker compose --profile ci run --rm usaspending-style-checks` to run all relevant style checks
 
 ### Pre-commit Hooks
 To assist developers it is possible to leverage pre-commit hooks to run several checks before git creates a commit. The Python dependency is included in the requirements files. To get started, run
@@ -119,6 +115,6 @@ All contributions to this project will be released under the [CC0 dedication](LI
 - [USAspending Release Notes](https://github.com/fedspendingtransparency/usaspending-website/wiki)
 - [About Federal Spending Transparency](http://fedspendingtransparency.github.io/)
 - [Our Amazing Frontend Repo](https://github.com/fedspendingtransparency/usaspending-website)
-- [Full Dataset as PostgreSQL Dump](https://files.usaspending.gov/database_download/)
+- [Full Dataset as PostgreSQL Dump](https://onevoicecrm.my.site.com/usaspending/s/database-download)
 - [General GitHub Documentation](https://help.github.com/)
 - [GitHub Pull Request Documentation](https://help.github.com/articles/creating-a-pull-request/)
diff --git a/Dockerfile b/Dockerfile
index 24726f1954..4f3da6ede1 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -10,7 +10,7 @@ FROM python:3.10.12-slim-bullseye
 
 COPY --from=ghcr.io/astral-sh/uv:0.7.19 /uv /uvx /bin/
 
-WORKDIR /dockermount
+WORKDIR /usaspending-api
 
 RUN apt update && apt install -y \
     ca-certificates \
@@ -41,7 +41,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
     uv sync --extra server --extra ansible --extra awscli --extra spark --locked --no-install-project --no-dev
 
 # Copy the project into the image
-COPY . /dockermount
+COPY . /usaspending-api
 
 # Sync the project
 RUN --mount=type=cache,target=/root/.cache/uv \
diff --git a/Makefile b/Makefile
index e868a1d733..44304f8196 100644
--- a/Makefile
+++ b/Makefile
@@ -28,7 +28,8 @@ ENV_CODE ?= lcl
 PYTHON_VERSION ?= 3.10.12
 venv_name := usaspending-api
 docker_compose_file := docker-compose.yml
-dockerfile_for_spark := Dockerfile.spark
+dockerfile_for_backend := Dockerfile
+dockerfile_for_development := development.Dockerfile
 # Root directories under which python (namespace) packages start, for all python code in this project
 src_root_paths = "."
 
@@ -55,11 +56,8 @@ printvars: ## Print the Environment variables present in calls to make, plus var
 uv-sync:
 	uv sync --no-cache --extra dev --extra spark --locked
 
-.ivy2: ## Ensure user has a ~/.ivy2 dir, which will be bound to in a docker container volume to save on dependency downloads
-	@mkdir -p ~/.ivy2
-
 .PHONY: local-dev-setup
-local-dev-setup: uv-sync check-dependencies .ivy2 ## Setup virtual environment and pip dependencies, then check version info
+local-dev-setup: uv-sync check-dependencies ## Setup virtual environment and pip dependencies, then check version info
 
 .PHONY: check-dependencies
 check-dependencies: ## Prints out the versions of dependencies in use
@@ -68,12 +66,6 @@ check-dependencies: ## Prints out the versions of dependencies in use
 	@uv run python3 --version
 	@printf "\n==== [PIP PACKAGE VERSIONS] ====\n\n"
 	@uv pip list
-	@printf "\n==== [SPARK VERSION] ====\n\n"
-	@uv run --no-project pyspark --version
-	@printf "\n==== [HADOOP VERSION] ====\n\n"
-	@uv run python3 -c "from pyspark.sql import SparkSession; \
-spark = spark = SparkSession.builder.getOrCreate(); \
-print('Hadoop ' + spark.sparkContext._gateway.jvm.org.apache.hadoop.util.VersionInfo.getVersion());"
 
 .PHONY: env-code
 env-code:  ## Print the value of ENV_CODE environment variable
@@ -82,19 +74,15 @@ env-code:  ## Print the value of ENV_CODE environment variable
 .PHONY: test-dbs
 createdb :=  #unset it
 test-dbs:  ## Trigger the setup of multiple test DBs that can be reused with pytest --numprocesses. Add createdb=true to force (re-)creation of Test DBs rather than reuse.
-	uv run pytest ${if ${createdb},--create-db,} --reuse-db --numprocesses=auto --no-cov --disable-warnings -rP -vvv --capture=no --log-cli-level=WARNING --show-capture=log 2> /dev/null 'usaspending_api/tests/integration/test_setup_of_test_dbs.py::test_trigger_test_db_setup'
-
-.PHONY: test-spark-deps
-test-spark-deps:  ## Trigger a singular test in one pytest session that does nothing but cause Maven dependencies to be downloaded and cached through Ivy; reduces contention when parallel spark builds need the depdencies
-	uv run pytest --no-cov --disable-warnings -r=fEs --verbosity=3 'usaspending_api/tests/integration/test_setup_of_spark_dependencies.py::test_preload_spark_jars'
+	docker compose run --rm usaspending-test pytest ${if ${createdb},--create-db,} --reuse-db --numprocesses=auto --no-cov --disable-warnings -rP -vvv --capture=no --log-cli-level=WARNING --show-capture=log 2> /dev/null 'usaspending_api/tests/integration/test_setup_of_test_dbs.py::test_trigger_test_db_setup'
 
 .PHONY: tests
 tests: local-dev-setup test-dbs test-spark-deps ## Run automated unit/integration tests. Configured for useful logging. add args="..." to append additional pytest args
-	uv run pytest --failed-first --reuse-db --numprocesses=auto --dist=worksteal -rP -vv --capture=no --show-capture=log 2> /dev/null ${args} 'usaspending_api/'
+	docker compose run --rm usaspending-test pytest --failed-first --reuse-db --numprocesses=auto --dist=worksteal -rP -vv --capture=no --show-capture=log 2> /dev/null ${args} 'usaspending_api/'
 
 .PHONY: tests-failed
 tests-failed: local-dev-setup test-dbs test-spark-deps ## Re-run only automated unit/integration tests that failed on the previous run. Configured for verbose logging to get more detail on failures. logging. add args="..." to append additional pytest args
-	uv run pytest --last-failed --reuse-db --numprocesses=auto --dist=worksteal -rP -vvv ${args}
+	docker compose run --rm usaspending-test pytest --last-failed --reuse-db --numprocesses=auto --dist=worksteal -rP -vvv ${args}
 
 .PHONY: confirm-clean-all
 no-prompt := 'false'
@@ -159,11 +147,10 @@ docker-compose-down: ## Run docker compose down to bring down services listed in
 	# NOTE: [See NOTE in docker compose rule about .env file]
 	docker compose --project-directory . --file ${docker_compose_file} down ${args}
 
-.PHONY: docker-build-spark
-docker-build-spark: ## Run docker build to build a base container image for spark, hadoop, and python installed
+.PHONY: docker-build-development
+docker-build-development: ## Run docker build to build a base container image for development
 	# NOTE: [See NOTE in above docker compose rule about .env file]
-	echo "docker build --tag spark-base --build-arg PROJECT_LOG_DIR=${PROJECT_LOG_DIR} ${args} --file ${dockerfile_for_spark} $$(dirname ${dockerfile_for_spark})"
-	docker build --tag spark-base --build-arg PROJECT_LOG_DIR=${PROJECT_LOG_DIR} ${args} --file ${dockerfile_for_spark} $$(dirname ${dockerfile_for_spark})
+	docker build --tag usaspending-development ${args} --file ${dockerfile_for_development} $$(dirname ${dockerfile_for_development})
 
 .PHONY: docker-compose-build
 docker-compose-build: ## Ensure ALL services in the docker-compose.yaml file have an image built for them according to their build: key
@@ -176,51 +163,35 @@ docker-compose-build: ## Ensure ALL services in the docker-compose.yaml file hav
 	#       where project defaults to the directory name you're in. Therefore you MUST always run this command (or the manual version of it)
 	#       anytime you want services run with Docker Compose to accommodate recent changes in the image (e.g. python package dependency changes)
 	# NOTE: [See NOTE in above docker compose rule about .env file]
-	echo "docker compose --profile usaspending --project-directory . --file ${docker_compose_file} build --build-arg PROJECT_LOG_DIR=${PROJECT_LOG_DIR} ${args}"
-	docker compose --profile usaspending --project-directory . --file ${docker_compose_file} build --build-arg PROJECT_LOG_DIR=${PROJECT_LOG_DIR} ${args}
+	echo "docker compose --profile usaspending --project-directory . --file ${docker_compose_file} build ${args}"
+	docker compose --profile usaspending --project-directory . --file ${docker_compose_file} build ${args}
 
-.PHONY: docker-compose-build-spark
-docker-compose-build-spark: ## See: docker-compose-build rule. This builds just the subset of spark services.
+.PHONY: docker-compose-build-development
+docker-compose-build-development: ## See: docker-compose-build rule. This builds just the subset of spark services.
 	# NOTE: [See NOTE in above docker compose rule about .env file]=
-	echo "docker compose --profile spark --project-directory . --file ${docker_compose_file} build --build-arg PROJECT_LOG_DIR=${PROJECT_LOG_DIR} ${args}"
-	docker compose --profile spark --project-directory . --file ${docker_compose_file} build --build-arg PROJECT_LOG_DIR=${PROJECT_LOG_DIR} ${args}
+	echo "docker compose --profile spark --project-directory . --file ${docker_compose_file} build ${args}"
+	docker compose --profile spark --project-directory . --file ${docker_compose_file} build ${args}
 
-.PHONY: docker-compose-spark-submit
-docker-compose-spark-submit: ## Run spark-submit from within local docker containerized infrastructure (which must be running first). Set params with django_command="..."
-	docker compose --profile=spark --project-directory . --file ${docker_compose_file} run \
-		-e MINIO_HOST=minio \
-		-e COMPONENT_NAME='${django_command}${python_script}' \
-		-e DATABASE_URL=${DATABASE_URL} \
-	spark-submit \
-	--driver-memory "2g" \
-	--packages org.postgresql:postgresql:42.2.23,io.delta:delta-spark_2.12:3.1.0,org.apache.hadoop:hadoop-aws:3.3.4,org.apache.spark:spark-hive_2.12:3.5.0 \
-	${if ${python_script}, \
-		${python_script}, \
-		/project/manage.py ${django_command} \
-	}
+.PHONY: style-checks
+style-checks:
+	docker compose --profile=ci --project-directory . --file ${docker_compose_file} run --rm usaspending-style-checks
 
-.PHONY: localhost-spark-submit
-localhost-spark-submit: ## Run spark-submit from with localhost as the driver and worker (single node). Set params with django_command="..."
-	SPARK_LOCAL_IP=127.0.0.1 \
+.PHONY: spark-submit
+spark-submit: ## Run spark-submit from within local docker containerized infrastructure (which must be running first). Set params with django_command="..."
+	docker compose --profile=spark --project-directory . --file ${docker_compose_file} run --rm \
+		-e COMPONENT_NAME='${django_command}${python_script}' \
 	spark-submit \
 	--driver-memory "2g" \
-	--packages org.postgresql:postgresql:42.2.23,io.delta:delta-spark_2.12:3.1.0,org.apache.hadoop:hadoop-aws:3.3.4,org.apache.spark:spark-hive_2.12:3.5.0 \
 	${if ${python_script}, \
 		${python_script}, \
-		manage.py ${django_command} \
+		/usaspending-api/manage.py ${django_command} \
 	}
 
 .PHONY: pyspark-shell
 pyspark-shell: ## Launch a local pyspark REPL shell with all of the packages and spark config pre-set
-	SPARK_LOCAL_IP=127.0.0.1 pyspark \
-	--packages org.postgresql:postgresql:42.2.23,io.delta:delta-spark_2.12:3.1.0,org.apache.hadoop:hadoop-aws:3.3.4 \
-	--conf spark.sql.extensions=io.delta.sql.DeltaSparkSessionExtension \
-	--conf spark.sql.catalog.spark_catalog=org.apache.spark.sql.delta.catalog.DeltaCatalog \
-	--conf spark.hadoop.fs.s3a.endpoint=localhost:${MINIO_PORT} \
-	--conf spark.hadoop.fs.s3a.access.key=usaspending \
-	--conf spark.hadoop.fs.s3a.secret.key=usaspender \
-	--conf spark.hadoop.fs.s3a.connection.ssl.enabled=false \
-	--conf spark.hadoop.fs.s3a.path.style.access=true \
-	--conf spark.sql.catalogImplementation=hive \
-	--conf spark.sql.warehouse.dir='$(PWD)/spark-warehouse' \
-	--conf spark.hadoop.javax.jdo.option.ConnectionURL='jdbc:derby:;databaseName=$(PWD)/spark-warehouse/metastore_db;create=true'
+	docker compose --profile=spark --project-directory . --file ${docker_compose_file} run --rm spark-shell
+
+.PHONY: django-manage
+django-manage:  ## Run Django management command with the usaspending-manage container
+	docker compose --profile=usaspending --project-directory . --file ${docker_compose_file} run --rm \
+	usaspending-manage ${args}
diff --git a/README.md b/README.md
index 41d082875d..0b4e03961d 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,7 @@
 # <p align="center"><img src="https://www.usaspending.gov/img/logo@2x.png" alt="USAspending API"></p>
 
-[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/python/black) [![Pull Request Checks](https://github.com/fedspendingtransparency/usaspending-api/actions/workflows/pull-request-checks.yaml/badge.svg?branch=staging)](https://github.com/fedspendingtransparency/usaspending-api/actions/workflows/pull-request-checks.yaml) [![Test Coverage](https://codeclimate.com/github/fedspendingtransparency/usaspending-api/badges/coverage.svg)](https://codeclimate.com/github/fedspendingtransparency/usaspending-api/coverage) [![Code Climate](https://codeclimate.com/github/fedspendingtransparency/usaspending-api/badges/gpa.svg)](https://codeclimate.com/github/fedspendingtransparency/usaspending-api)
+[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
+[![Pull Request Checks](https://github.com/fedspendingtransparency/usaspending-api/actions/workflows/pull-request-checks.yaml/badge.svg?branch=staging)](https://github.com/fedspendingtransparency/usaspending-api/actions/workflows/pull-request-checks.yaml)
 
 _This API is utilized by USAspending.gov to obtain all federal spending data which is open source and provided to the public as part of the DATA Act._
 
@@ -25,8 +26,8 @@ _**If not using Docker, you'll need to install app components on your machine:**
     - Windows' WSL bash uses `apt`
     - MacOS users can use [`Homebrew`](https://brew.sh/)
     - Linux users already know their package manager (`yum`, `apt`, `pacman`, etc.)
-- [`PostgreSQL`](https://www.postgresql.org/download/) version 13.x (with a dedicated `data_store_api` database)
-- [`Elasticsearch`](https://www.elastic.co/downloads/elasticsearch) version 7.1
+- [`PostgreSQL`](https://www.postgresql.org/download/) version 16.x (with a dedicated `data_store_api` database)
+- [`OpenSearch`](https://www.elastic.co/downloads/elasticsearch) version 2.9
 - `Python` version 3.10 environment
   - Highly recommended to use a virtual environment. There are various tools and associated instructions depending on preferences
   - See [Required Python Libraries](#required-python-libraries) for an example using `uv`
@@ -73,14 +74,25 @@ _Alternatively, you could skip using `direnv` and just export these variables in
 
 **Just make sure your env vars declared in the shell and in `.env` match for a consistent experience inside and outside of Docker**
 
-### Build `usaspending-backend` Docker Image
-_This image is used as the basis for running application components and running containerized setup services._
+### Build the Docker Images
+
+_:bangbang: Re-run these commands if any python package dependencies change (in `pyproject.toml`/`uv.lock`), since they are baked into the docker image at build-time._
+
+#### usaspending-backend | usaspending-download
+
+_These images are used as the basis for running the API and download containerized setup services._
 
 ```shell
 docker compose --profile usaspending build
 ```
 
-_:bangbang: Re-run this command if any python package dependencies change (in `pyproject.toml`/`uv.lock`), since they are baked into the docker image at build-time._
+#### usaspending-development
+
+_This image is used as the basis for local development including but not limited to: tests, spark jobs, linting, etc._
+
+```shell
+docker compose --profile test build
+```
 
 ### Database Setup
 A postgres database is required to run the app. You can run it in a `postgres` docker container (preferred), or run a PostgreSQL server on your local machine. In either case, it will be empty until data is loaded.
@@ -132,7 +144,7 @@ _To just get essential reference data, you can run:_
     docker compose run --rm usaspending-manage python3 -u manage.py load_reference_data
     ```
 
-_Alternatively, to download a fully populuated production snapshot of the database (full or a subset) and restore it into PostgreSQL, use the `pg_restore` tool as described here: [USAspending Database Download](https://onevoicecrm.my.site.com/usaspending/s/database-download)_
+_Alternatively, to download a fully populated production snapshot of the database (full or a subset) and restore it into PostgreSQL, use the `pg_restore` tool as described here: [USAspending Database Download](https://onevoicecrm.my.site.com/usaspending/s/database-download)_
 
 - Recreate matviews with the command documented in the previous section if this is done
 
@@ -141,22 +153,22 @@ _**Executing individual data-loaders** to load in data is also possible, but req
 - For details on loading reference data, Data Accountability Broker Submissions, and current USAspending data into the API, see [loading_data.md](loading_data.md).
 - For details on how our data loaders modify incoming data, see [data_reformatting.md](data_reformatting.md).
 
-### Elasticsearch Setup
-Some API endpoints reach into Elasticsearch for data.
+### OpenSearch Setup
+Some API endpoints reach into OpenSearch for data.
 
 ```shell
 docker compose --profile usaspending up -d usaspending-es
 ```
-... will create and start a single-node Elasticsearch cluster as a docker container with data persisted to a docker volume.
+... will create and start a single-node OpenSearch cluster as a docker container with data persisted to a docker volume.
 
 - The cluster should be reachable via at http://localhost:9200 ("You Know, for Search").
 
 - Optionally, to see log output, use `docker compose logs usaspending-es` (these logs are stored by docker even if you don't use this).
 
-While not required, it is highly recommended to also create the Kibana docker container for querying the Elasticsearch cluster.
+While not required, it is highly recommended to also create the OpenSearch Dashboards docker container for querying the OpenSearch cluster.
 
 ```shell
-docker compose --profile usaspending up usaspending-kibana-es
+docker compose --profile usaspending up -d usaspending-kibana-es
 ```
 
 #### Generate Elasticsearch Indexes
@@ -164,10 +176,10 @@ The following will generate the indexes:
 
 ```shell
 CURR_DATE=$(date '+%Y-%m-%d-%H-%M-%S')
-docker compose run --rm usaspending-manage python3 -u manage.py elasticsearch_indexer --create-new-index --index-name "$CURR_DATE-transactions" --load-type transaction
-docker compose run --rm usaspending-manage python3 -u manage.py elasticsearch_indexer --create-new-index --index-name "$CURR_DATE-awards" --load-type award
-docker compose run --rm usaspending-manage python3 -u manage.py elasticsearch_indexer --create-new-index --index-name "$CURR_DATE-recipients" --load-type recipient
-docker compose run --rm usaspending-manage python3 -u manage.py elasticsearch_indexer --create-new-index --index-name "$CURR_DATE-subaward" --load-type subaward
+docker compose run --rm usaspending-manage elasticsearch_indexer --create-new-index --index-name "$CURR_DATE-transactions" --load-type transaction
+docker compose run --rm usaspending-manage elasticsearch_indexer --create-new-index --index-name "$CURR_DATE-awards" --load-type award
+docker compose run --rm usaspending-manage elasticsearch_indexer --create-new-index --index-name "$CURR_DATE-recipients" --load-type recipient
+docker compose run --rm usaspending-manage elasticsearch_indexer --create-new-index --index-name "$CURR_DATE-subaward" --load-type subaward
 ```
 
 ### Observability/Tracing Setup
@@ -179,6 +191,23 @@ docker compose --profile tracing up -d
 ```
 When using these containers the `TOGGLE_OTEL_CONSOLE_LOGGING` must be set to `False` and `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` must be set to `http://otel-collector:4318/v1/traces`
 
+### Install python version with uv
+Install the currently supported python version with uv.
+
+This is useful for access to a python shell outside a container. However, it is recommended to use the docker compose containerized services for interaction with the API, spark, and tests.
+
+
+```shell
+uv python install 3.10.12
+```
+
+#### Sync uv project with lockfile
+Setup uv project and check version info:
+
+```shell
+make local-dev-setup
+```
+
 ## Running the API
 
 Run the following to bring up the Django app for the RESTful API:
@@ -207,10 +236,10 @@ Deployed production API endpoints and docs are found by following links here: `h
 
 ### Test Setup
 
-1. Build the base `usaspending-backend` Docker image (the test container is based on this Docker image). In the parent **usaspending-api** directory run:
+1. Build the base `usaspending-development` Docker image (the test container is based on this Docker image). In the parent **usaspending-api** directory run:
 
     ```shell
-    docker build -t usaspending-backend .
+    docker build -t usaspending-development -f development.Dockerfile .
     ```
 
 2. Start the Spark containers for the Spark related tests
@@ -228,40 +257,13 @@ _**NOTE**: If an env var named `BROKER_DB` is set, Broker Integration tests will
 To run tests locally and not in the docker services, you need:
 
 1. **Postgres**: A running PostgreSQL database server _(See [Database Setup above](#database-setup))_
-1. **Elasticsearch**: A running Elasticsearch cluster _(See [Elasticsearch Setup above](#elasticsearch-setup))_
-1. **Environment Variables**: Tell python where to connect to the various data stores _(See [Environmnet Variables](#environment-variables))_
+1. **OpenSearch**: A running OpenSearch cluster _(See [OpenSearch Setup above](#opensearch-setup))_
+1. **Environment Variables**: Tell python where to connect to the various data stores _(See [Environment Variables](#environment-variables))_
 1. **Install python version with uv**: python is installed with uv  _(See below)_
 1. **Sync uv project**: uv project is synced with lockfile  _(See below)_
 
 _**NOTE**: Running test locally might require you to run `export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES`. As discussed [here](https://github.com/rails/rails/issues/38560), there is an issue than can cause some of the Spark tests to fail without this environment variable set._
 
-#### Install python version with uv
-Install the currently supported python version with uv
-
-```shell
-uv python install 3.10.12
-```
-
-#### Sync uv project with lockfile
-Setup uv project and check version info:
-
-```shell
-make local-dev-setup
-```
-
-
-Once these are satisfied, run:
-
-```shell
-make tests
-```
-
-or, alternatively to skip using `make`
-
-```shell
-uv run pytest
-```
-
 ### Including Broker Integration Tests
 Some automated integration tests run against a [Broker](https://github.com/fedspendingtransparency/data-act-broker-backend) database. If certain dependencies to run such integration tests are not satisfied, those tests will bail out and be marked as _Skipped_.
 (You can see messages about those skipped tests by adding the `-rs` flag to pytest, like: `pytest -rs`)
diff --git a/development.Dockerfile b/development.Dockerfile
new file mode 100644
index 0000000000..a620d9a84c
--- /dev/null
+++ b/development.Dockerfile
@@ -0,0 +1,83 @@
+# Dockerfile to support local developement and tests
+FROM public.ecr.aws/emr-serverless/spark/emr-7.12.0:latest
+
+USER root
+
+# Copy UV from uv image
+COPY --from=ghcr.io/astral-sh/uv:0.7.19 /uv /uvx /bin/
+
+# Build ARGs
+ARG PROJECT_LOG_DIR=/logs
+ARG PYTHON_VERSION=3.10.12
+ARG SPARK_LOGGING_LEVEL=INFO
+
+# Install dependencies
+RUN dnf update \
+    && dnf install -y \
+        git \
+        libffi-devel \
+        libpq-devel \
+        nodejs \
+        npm \
+        postgresql16 \
+        sqlite-devel \
+        wget \
+        zlib-devel \
+    && dnf clean all
+
+# Download the Postgres JAR
+RUN wget -P /usr/lib/spark/jars/ https://jdbc.postgresql.org/download/postgresql-42.7.5.jar
+
+# Install Dredd
+RUN npm install --global dredd@13.1.2
+
+WORKDIR /usaspending-api
+
+# Allow git operations to be run inside of the container
+RUN git config --global --add safe.directory /usaspending-api
+
+##### The following ENV vars are optimizations from https://github.com/astral-sh/uv-docker-example/blob/main/Dockerfile
+##### and https://docs.astral.sh/uv/guides/integration/docker/#optimizations
+# Specify .venv location outside of the project to avoid conflict
+ENV UV_PROJECT_ENVIRONMENT=/usr/local/.venv
+
+# Enable bytecode compilation
+ENV UV_COMPILE_BYTECODE=1
+
+# Copy from the cache instead of linking since it's a mounted volume
+ENV UV_LINK_MODE=copy
+
+# Install Python and dev dependencies
+RUN --mount=type=cache,target=/root/.cache/uv \
+    --mount=type=bind,source=.,target=/usaspending-api \
+    uv sync --extra dev --extra spark --locked --python ${PYTHON_VERSION}
+
+# Make sure uv environment is active
+ENV PATH="/usr/local/.venv/bin:$PATH"
+
+# Set Python 3.10.12 as the default Python for PySpark
+ENV PYSPARK_PYTHON=/usr/local/.venv/bin/python3
+ENV PYSPARK_DRIVER_PYTHON=/usr/local/.venv/bin/python3
+
+# Configure console logging for Spark
+RUN cat <<EOF | sed 's/^[[:space:]]*//' >> $SPARK_HOME/conf/log4j2.properties
+	appender.console.type = Console
+    appender.console.name = console
+    appender.console.layout.type = PatternLayout
+    appender.console.layout.pattern = [%d{yyyy-MM-dd HH:mm:ss.SSS}][%p] - %m%n
+
+    rootLogger.level = ${SPARK_LOGGING_LEVEL}
+    rootLogger.appenderRefs = stdout
+    rootLogger.appenderRef.stdout.ref = console
+EOF
+
+# Update logging for Spark
+RUN sed -i "s|^spark.eventLog.dir.*$|spark.eventLog.dir               file:///usaspending-api/$PROJECT_LOG_DIR/spark-events|g" $SPARK_HOME/conf/spark-defaults.conf \
+    && sed -i "/spark.eventLog.enabled/s/^# //g" $SPARK_HOME/conf/spark-defaults.conf \
+    && sed -i "s|spark.history.fs.logDirectory.*$|spark.history.fs.logDirectory    file:///usaspending-api/$PROJECT_LOG_DIR/spark-events|" $SPARK_HOME/conf/spark-defaults.conf
+
+# Disable authentication for local development
+RUN sed -i "s|spark.authenticate \{1,\}true|spark.authenticate false|" $SPARK_HOME/conf/spark-defaults.conf
+
+# Ensure Python STDOUT gets sent to container logs
+ENV PYTHONUNBUFFERED=1
diff --git a/docker-compose.yml b/docker-compose.yml
index a12a78ea98..f72af94504 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -5,16 +5,63 @@ volumes:
   local_es_data:
     driver: local
 
+# There are populated by .env when using docker compose
+x-application-variables: &application-vars
+  # Primary data stores for the API
+  DATABASE_URL: ${DATABASE_URL}
+  DOWNLOAD_DATABASE_URL: ${DOWNLOAD_DATABASE_URL}
+  ES_HOSTNAME: ${ES_HOSTNAME}
+  # For connection to Broker
+  BROKER_DB: ${BROKER_DB}
+  # For local development
+  DJANGO_DEBUG: ${DJANGO_DEBUG}
+  DJANGO_SETTINGS_MODULE: ${DJANGO_SETTINGS_MODULE}
+  IS_LOCAL: ${IS_LOCAL}
+  # MinIO
+  MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY}
+  MINIO_SECRET_KEY: ${MINIO_SECRET_KEY}
+  MINIO_HOST: ${MINIO_HOST}
+  MINIO_PORT: ${MINIO_PORT}
+  # Defined for tracing
+  OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: ${OTEL_EXPORTER_OTLP_TRACES_ENDPOINT}
+  TOGGLE_OTEL_CONSOLE_LOGGING: ${TOGGLE_OTEL_CONSOLE_LOGGING}
+
+x-spark-submit-variables: &spark-submit-vars
+  <<: *application-vars
+  AWS_ACCESS_KEY_ID: ${MINIO_ACCESS_KEY}
+  AWS_ENDPOINT_URL: http://${MINIO_HOST}:${MINIO_PORT}
+  AWS_SECRET_ACCESS_KEY: ${MINIO_SECRET_KEY}
+  # Needed for the Spark containers to interact
+  SPARK_MASTER_HOST: ${SPARK_MASTER_HOST}
+  SPARK_MASTER_PORT: ${SPARK_MASTER_PORT}
+  SPARK_WORKER_WEBUI_PORT: ${SPARK_WORKER_WEBUI_PORT}
+  # This env var needs to be picked up as the config for the spark.sql.warehouse.dir spark conf setting when
+  # SparkSessions are created inside a spark-submitted job
+  # (i.e., target where host warehouse dir is bound in volume config)
+  SPARK_SQL_WAREHOUSE_DIR: ${TARGET_SPARK_SQL_WAREHOUSE_DIR}
+  # This env var needs to be picked up as the path part of the config for the
+  # spark.hadoop.javax.jdo.option.ConnectionURL spark conf setting when SparkSessions are created inside a
+  # spark-submitted job
+  # (i.e., a metastore_db sub dir of the target where host warehouse dir is bound in volume config.)
+  HIVE_METASTORE_DERBY_DB_DIR: ${TARGET_HIVE_METASTORE_DERBY_DB_DIR}
+
+
 services:
 
   usaspending-db:
     command: postgres -c 'max_connections=500'
-    profiles:
-      - usaspending  # must pass --profile usaspending to docker compose for this to come up, or run a service with one of these other profiles
+    profiles:  # must pass --profile usaspending to docker compose for this to come up, or run a service with one of these other profiles
+      - ci
       - manage
+      - spark
       - test
-      - ci
+      - usaspending
     image: postgres:16-alpine
+    healthcheck:
+      test: [ "CMD-SHELL", "pg_isready -U ${USASPENDING_DB_USER} -d ${USASPENDING_DB_NAME}" ]
+      interval: 5s
+      timeout: 5s
+      retries: 5
     shm_size: 1g
     container_name: usaspending-db
     volumes:
@@ -22,40 +69,47 @@ services:
         source: local_pg_data
         target: /var/lib/postgresql/data
     ports:
-      - ${USASPENDING_DB_PORT:-5432}:5432
+      - ${USASPENDING_DB_PORT}:5432
     environment:
-      POSTGRES_USER: ${USASPENDING_DB_USER:-usaspending}
-      POSTGRES_PASSWORD: ${USASPENDING_DB_PASSWORD:-usaspender}
-      POSTGRES_DB: ${USASPENDING_DB_NAME:-data_store_api}
+      POSTGRES_USER: ${USASPENDING_DB_USER}
+      POSTGRES_PASSWORD: ${USASPENDING_DB_PASSWORD}
+      POSTGRES_DB: ${USASPENDING_DB_NAME}
 
   usaspending-manage:
     profiles:
       - manage  # must pass --profile manage to docker compose for this to come up, or use docker compose run
-    image: usaspending-backend  # when an image by this name is not found in the local repo, and it is forced to build, it will use this as the tag
+    image: usaspending-development  # when an image by this name is not found in the local repo, and it is forced to build, it will use this as the tag
     build: .
     container_name: usaspending-manage
     volumes:
-      - .:/dockermount
+      - type: bind
+        source: .
+        target: /usaspending-api
+        read_only: false
     depends_on:
       - usaspending-db
     # For an interactive shell, override this command with: docker compose run --rm usaspending-manage python3 -u manage.py shell
-    command: python3 -u manage.py help
+    entrypoint: ["uv", "run", "manage.py"]
     environment:
-      DJANGO_DEBUG: ${DJANGO_DEBUG}
-      DATABASE_URL: postgres://${USASPENDING_DB_USER}:${USASPENDING_DB_PASSWORD}@${USASPENDING_DB_HOST}:${USASPENDING_DB_PORT}/data_store_api
-      ES_HOSTNAME: ${ES_HOSTNAME}
-      BROKER_DB: postgresql://${BROKER_DB_USER}:${BROKER_DB_PASSWORD}@${BROKER_DB_HOST}:${BROKER_DB_PORT}/data_broker
+      <<: *application-vars
 
   usaspending-test:
     profiles:
       - test  # must pass --profile test to docker compose for this to come up, or use docker compose run
-    image: usaspending-testing  # when an image by this name is not found in the local repo, and it is forced to build, it will use this as the tag
+    image: usaspending-development  # when an image by this name is not found in the local repo, and it is forced to build, it will use this as the tag
     build:
       context: .
-      dockerfile: testing.Dockerfile
+      dockerfile: development.Dockerfile
+      args:
+        PROJECT_LOG_DIR: ${PROJECT_LOG_DIR}
+        SPARK_LOGGING_LEVEL: ${SPARK_LOGGING_LEVEL}
+    init: true  # needed for the reaping of processes created for tests
     container_name: usaspending-test
     volumes:
-      - ${PROJECT_DIRECTORY:-.}:/usaspending-api
+      - type: bind
+        source: .
+        target: /usaspending-api
+        read_only: false
       # Required to interact with host's docker daemon from within this running container,
       # to spin up the data-act-broker-init-test-db container used for broker integration tests (see: conftest.broker_db_setup)
       - /var/run/docker.sock:/var/run/docker.sock
@@ -63,62 +117,107 @@ services:
       - minio
       - usaspending-db
       - usaspending-es
-    command: make tests
+    entrypoint: []  # override entrypoint
     environment:
-      DATABASE_URL: postgres://${USASPENDING_DB_USER}:${USASPENDING_DB_PASSWORD}@${USASPENDING_DB_HOST}:${USASPENDING_DB_PORT}/data_store_api
-      ES_HOST: ${ES_HOST}
-      ES_HOSTNAME: ${ES_HOSTNAME}
-      BROKER_DB: postgresql://${BROKER_DB_USER}:${BROKER_DB_PASSWORD}@${BROKER_DB_HOST}:${BROKER_DB_PORT}/data_broker
-      MINIO_HOST: ${MINIO_HOST}
-      DOWNLOAD_DATABASE_URL: postgres://${USASPENDING_DB_USER}:${USASPENDING_DB_PASSWORD}@${USASPENDING_DB_HOST}:${USASPENDING_DB_PORT}/data_store_api
+      <<: *spark-submit-vars
       # Location in host machine where broker src code root can be found
       DATA_BROKER_SRC_PATH: "${PWD}/../data-act-broker-backend"
       # Tests will retrieve the root directory from the path and use it to assume the docker network name
       PROJECT_SRC_PATH: "${PWD}"
 
+  usaspending-style-checks:
+    profiles:
+      - ci  # must pass --profile ci to docker compose for this to come up, or use docker compose run
+    image: usaspending-development
+    build:
+      context: .
+      dockerfile: development.Dockerfile
+    container_name: usaspending-style-checks
+    volumes:
+      - type: bind
+        source: .
+        target: /usaspending-api
+        read_only: false
+    entrypoint: []  # override entrypoint
+    command:
+      - sh
+      - -c
+      - |
+        printf -- "==============\nFormatting code:\n"
+        git diff --name-only --diff-filter=ACMR | grep '\.py$' | xargs -r uv run ruff format
+        printf -- "==============\nChecking code syntax and quality:\n"
+        git diff --name-only --diff-filter=ACMR | grep '\.py$' | xargs -r uv run ruff check --fix || exit 1
+        printf -- "==============\nChecking API documentation files:\n"
+        python3 manage.py check_for_endpoint_documentation
+        printf -- "==============\nChecking API contracts:\n"
+        dredd > dredd-results.txt && echo '! grep -E "^[warn:|error:]" dredd-results.txt' | bash
+    environment:
+      <<: *application-vars
+
   usaspending-ci:
     profiles:
       - ci  # must pass --profile ci to docker compose for this to come up, or use docker compose run
-    image: usaspending-backend  # when an image by this name is not found in the local repo, and it is forced to build, it will use this as the tag
-    build: .
+    image: usaspending-development  # when an image by this name is not found in the local repo, and it is forced to build, it will use this as the tag
+    build:
+      context: .
+      dockerfile: development.Dockerfile
+    init: true  # needed for the reaping of processes created for tests
     container_name: usaspending-ci
     volumes:
-      - .:/dockermount
+      - type: bind
+        source: .
+        target: /usaspending-api
+        read_only: false
       # Required to interact with host's docker daemon from within this running container,
       # to spin up the data-act-broker-init-test-db container used for broker integration tests (see: conftest.broker_db_setup)
       - /var/run/docker.sock:/var/run/docker.sock
     depends_on:
-      - usaspending-db
-      - usaspending-es
+      usaspending-db:
+        condition: service_healthy
+      usaspending-es:
+        condition: service_healthy
+      usaspending-style-checks:
+        condition: service_completed_successfully
+    entrypoint: []  # override entrypoint
     command:
       - sh
       - -c
       - |
-        printf "==============\nChecking code format:\n"
-        black --check --diff .
-        printf -- "-------\nChecking code syntax:\n"
-        flake8 && echo "Successfully passed"
-        printf -- "-------\nChecking API documentation files:\n"
-        python3 manage.py check_for_endpoint_documentation
+        printf -- "-------\nRunning non-spark integration tests:\n"
+        pytest --override-ini=python_files="**/tests/integration/*" -m "(not signal_handling and not spark)" --no-cov --disable-warnings --reuse-db --numprocesses auto
+        printf -- "-------\nRunning non-spark integration tests with signal handling:\n"
+        pytest --override-ini=python_files="**/tests/integration/*" -m "(signal_handling and not spark)" --no-cov --disable-warnings --reuse-db --numprocesses 0
+        printf -- "-------\nRunning spark integration tests -- load to / from delta:\n"
+        pytest --override-ini=python_files="test_*.py *_test.py" -m "(spark)" -k "test_load_to_from_delta.py" --no-cov --disable-warnings --reuse-db --numprocesses 2
+        printf -- "-------\nRunning spark integration tests -- load transaction fabs / fpds:\n"
+        pytest --override-ini=python_files="test_*.py *_test.py" -m "(spark)" -k "test_load_transactions_in_delta_fabs_fpds.py" --no-cov --disable-warnings --reuse-db --numprocesses 0
+        printf -- "-------\nRunning spark integration tests -- load transaction lookup:\n"
+        pytest --override-ini=python_files="test_*.py *_test.py" -m "(spark)" -k "test_load_transactions_in_delta_lookups.py" --no-cov --disable-warnings --reuse-db --numprocesses 0
+        printf -- "-------\nRunning spark integration tests -- other:\n"
+        pytest --override-ini=python_files="test_*.py *_test.py" -m "(spark)" -k "(not test_load_to_from_delta.py and not test_load_transactions_in_delta_lookups.py and not test_load_transactions_in_delta_fabs_fpds.py)" --no-cov --disable-warnings --reuse-db --numprocesses auto
         printf -- "-------\nRunning unit tests:\n"
-        pytest --durations 50 --ignore-glob='**/tests/integration/*' --cov=usaspending_api --cov-report= --reuse-db -rsx
-        printf -- "-------\nRunning integration tests:\n"
-        pytest --durations 50 --override-ini=python_files='**/tests/integration/*' --cov=usaspending_api --cov-append --cov-report term --cov-report xml:coverage.xml --reuse-db -rsx
+        pytest --override-ini=python_files="test_*.py *_test.py" -m "(not signal_handling)" --ignore-glob="**/tests/integration/*" --no-cov --disable-warnings --reuse-db --numprocesses auto
+        printf -- "-------\nRunning unit tests with signal handling:\n"
+        pytest --override-ini=python_files="test_*.py *_test.py" -m "(signal_handling)" --ignore-glob="**/tests/integration/*" --no-cov --disable-warnings --reuse-db --numprocesses 0
     environment:
-      DATABASE_URL: postgres://${USASPENDING_DB_USER}:${USASPENDING_DB_PASSWORD}@${USASPENDING_DB_HOST}:${USASPENDING_DB_PORT}/data_store_api
-      ES_HOSTNAME: ${ES_HOSTNAME}
-      BROKER_DB: postgresql://${BROKER_DB_USER}:${BROKER_DB_PASSWORD}@${BROKER_DB_HOST}:${BROKER_DB_PORT}/data_broker
+      <<: *spark-submit-vars
       # Location in host machine where broker src code root can be found
       DATA_BROKER_SRC_PATH: "${PWD}/../data-act-broker-backend"
+      # Tests will retrieve the root directory from the path and use it to assume the docker network name
+      PROJECT_SRC_PATH: "${PWD}"
 
-  usaspending-api:
+  usaspending-api:  # builds using the default Dockerfile to mirror deployed environments for local testing
     profiles:
       - usaspending  # must pass --profile usaspending to docker compose for this to come up
     image: usaspending-backend  # when an image by this name is not found in the local repo, and it is forced to build, it will use this as the tag
     build: .
     container_name: usaspending-api
+    init: true  # needed for the reaping of processes; mostly related to downloads run in a process
     volumes:
-      - .:/dockermount
+      - type: bind
+        source: .
+        target: /usaspending-api
+        read_only: false
     ports:
       - 8000:8000
     depends_on:
@@ -128,15 +227,7 @@ services:
     # Must wait on postgres db to be up (~9s)
     command: /bin/sh -c "sleep 9s; python3 -u manage.py runserver --verbosity 2 0.0.0.0:8000"
     environment:
-      IS_LOCAL: ${IS_LOCAL}
-      DJANGO_DEBUG: ${DJANGO_DEBUG}
-      RUN_LOCAL_DOWNLOAD_IN_PROCESS: "False"
-      DB_SOURCE: postgres://${USASPENDING_DB_USER}:${USASPENDING_DB_PASSWORD}@${USASPENDING_DB_HOST}:${USASPENDING_DB_PORT}/data_store_api
-      DB_R1: postgres://${USASPENDING_DB_USER}:${USASPENDING_DB_PASSWORD}@${USASPENDING_DB_HOST}:${USASPENDING_DB_PORT}/data_store_api
-      DOWNLOAD_DATABASE_URL: postgres://${USASPENDING_DB_USER}:${USASPENDING_DB_PASSWORD}@${USASPENDING_DB_HOST}:${USASPENDING_DB_PORT}/data_store_api
-      ES_HOSTNAME: ${ES_HOSTNAME}
-      TOGGLE_OTEL_CONSOLE_LOGGING: ${TOGGLE_OTEL_CONSOLE_LOGGING}
-      OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: ${OTEL_EXPORTER_OTLP_TRACES_ENDPOINT}
+      <<: *application-vars
 
   usaspending-bulk-download:
     profiles:
@@ -145,22 +236,20 @@ services:
     build:
       context: .
       dockerfile: download.Dockerfile
+    init: true  # needed for the reaping of processes
     container_name: usaspending-bulk-download
     restart: on-failure:5 # 5 max attempt, and then it will stop restarting. NOTE: bulk download errors will cause one failure+restart iterations
     volumes:
-      - .:/dockermount
+      - type: bind
+        source: .
+        target: /usaspending-api
+        read_only: false
       # Required to interact with host's docker daemon from within this running container,
       # to submit a spark job via the spark-submit container (see download_sqs_worker._run_spark_download)
       - /var/run/docker.sock:/var/run/docker.sock
     command: python3 manage.py download_sqs_worker
     environment:
-      IS_LOCAL: ${IS_LOCAL}
-      DJANGO_DEBUG: ${DJANGO_DEBUG}
-      DATABASE_URL: postgres://${USASPENDING_DB_USER}:${USASPENDING_DB_PASSWORD}@${USASPENDING_DB_HOST}:${USASPENDING_DB_PORT}/data_store_api
-      DOWNLOAD_DATABASE_URL: postgres://${USASPENDING_DB_USER}:${USASPENDING_DB_PASSWORD}@${USASPENDING_DB_HOST}:${USASPENDING_DB_PORT}/data_store_api
-      ES_HOSTNAME: ${ES_HOSTNAME}
-      TOGGLE_OTEL_CONSOLE_LOGGING: ${TOGGLE_OTEL_CONSOLE_LOGGING}
-      OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: ${OTEL_EXPORTER_OTLP_TRACES_ENDPOINT}
+      <<: *application-vars
 
   usaspending-es:
     profiles:
@@ -168,6 +257,12 @@ services:
       - test
       - ci
     image: opensearchproject/opensearch:2.9.0
+    healthcheck:
+      test: [ "CMD-SHELL", "curl -f -k http://localhost:9200/_cluster/health || exit 1" ]
+      interval: 10s
+      timeout: 5s
+      retries: 10
+      start_period: 60s
     container_name: usaspending-es
     environment:
       - node.name=usaspending-es
@@ -195,13 +290,15 @@ services:
       source: local_es_data
       target: /usr/share/opensearch/data
     ports:
-      - 9200:9200
+      - ${ES_PORT}:9200
 
   usaspending-kibana-es:
     profiles:
       - usaspending  # must pass --profile usaspending to docker compose for this to come up
     image: opensearchproject/opensearch-dashboards:2.9.0
     container_name: usaspending-kibana-es
+    depends_on:
+      - usaspending-es
     # ELASTICSEARCH_HOSTS should match the port for "usaspending-es"; value will need to be updated if using Windows
     environment:
       - OPENSEARCH_HOSTS="http://usaspending-es:9200"
@@ -218,16 +315,19 @@ services:
     image: minio/minio:RELEASE.2022-04-12T06-55-35Z
     container_name: minio
     volumes:
-      - .:/dockermount
       - type: bind
-        source: ${MINIO_DATA_DIR:-../data/s3}
+        source: .
+        target: /usaspending-api
+        read_only: false
+      - type: bind
+        source: ${MINIO_DATA_DIR}
         target: /data
     ports:
-      - ${MINIO_PORT:-10001}:10001
-      - ${MINIO_CONSOLE_PORT:-10002}:10002
+      - ${MINIO_PORT}:10001
+      - ${MINIO_CONSOLE_PORT}:10002
     environment:
-      MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY:-usaspending}
-      MINIO_SECRET_KEY: ${MINIO_SECRET_KEY:-usaspender}
+      MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY}
+      MINIO_SECRET_KEY: ${MINIO_SECRET_KEY}
     entrypoint: >
       /bin/sh -c "
         # Create the bucket within MinIO and file path for the data dict
@@ -239,12 +339,12 @@ services:
         # Create the bucket for MinIO used for Spark
         mkdir -p data/data/files
         # Populate initial files in buckets
-        cp dockermount/usaspending_api/data/Data_Dictionary_Crosswalk.xlsx data/dti-da-public-files-nonprod/user_reference_docs/Data_Dictionary_Crosswalk.xlsx
-        cp dockermount/usaspending_api/data/COVID-19_download_readme.txt data/data/files/COVID-19_download_readme.txt
+        cp usaspending-api/usaspending_api/data/Data_Dictionary_Crosswalk.xlsx data/dti-da-public-files-nonprod/user_reference_docs/Data_Dictionary_Crosswalk.xlsx
+        cp usaspending-api/usaspending_api/data/COVID-19_download_readme.txt data/data/files/COVID-19_download_readme.txt
         minio server --address ":10001" --console-address ":10002" /data
       "
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://${MINIO_HOST:-localhost}:${MINIO_PORT:-10001}/minio/health/live"]
+      test: ["CMD", "curl", "-f", "http://${MINIO_HOST}:${MINIO_PORT}/minio/health/live"]
       interval: 30s
       timeout: 20s
       retries: 3
@@ -253,83 +353,84 @@ services:
     profiles:
       - spark  # must pass --profile spark to docker compose for this to come up
       - test
-    image: spark-base  # when an image by this name is not found in the local repo, and it is forced to build, it will use this as the tag
+    image: usaspending-development
     # build context path needs to be relative to project root, from where docker compose will be run
     build:
       context: .
-      dockerfile: spark.Dockerfile
+      dockerfile: development.Dockerfile
       args:
         PROJECT_LOG_DIR: ${PROJECT_LOG_DIR}
+        SPARK_LOGGING_LEVEL: ${SPARK_LOGGING_LEVEL}
     container_name: spark-master
     environment:
-      SPARK_MASTER_HOST: ${SPARK_MASTER_HOST:-spark-master}
-      SPARK_MASTER_PORT: ${SPARK_MASTER_PORT:-7077}
-      SPARK_MASTER_WEBUI_PORT: ${SPARK_MASTER_WEBUI_PORT:-4040}
+      <<: *spark-submit-vars
+    entrypoint: []  # override the entrypoint
     command: >
       /bin/sh -c "
         $${SPARK_HOME}/bin/spark-class org.apache.spark.deploy.master.Master \
         --port $${SPARK_MASTER_PORT} \
         --webui-port $${SPARK_MASTER_WEBUI_PORT}"
     ports:
-      - ${SPARK_MASTER_PORT:-7077}:7077
-      - ${SPARK_MASTER_WEBUI_PORT:-4040}:4040
+      - ${SPARK_MASTER_PORT}
+      - ${SPARK_MASTER_WEBUI_PORT}
     volumes:
       - type: bind
         source: .
-        target: /project
+        target: /usaspending-api
         read_only: false
-
+      # Required to interact with host's docker daemon from within this running container and access the Broker DB
+      - /var/run/docker.sock:/var/run/docker.sock
   spark-worker:
     profiles:
       - spark  # must pass --profile spark to docker compose for this to come up
       - test
-    image: spark-base  # when an image by this name is not found in the local repo, and it is forced to build, it will use this as the tag
+    image: usaspending-development
     # build context path needs to be relative to project root, from where docker compose will be run
     build:
       context: .
-      dockerfile: spark.Dockerfile
+      dockerfile: development.Dockerfile
       args:
         PROJECT_LOG_DIR: ${PROJECT_LOG_DIR}
+        SPARK_LOGGING_LEVEL: ${SPARK_LOGGING_LEVEL}
     container_name: spark-worker
     depends_on:
       - spark-master
     environment:
-      SPARK_MASTER_HOST: ${SPARK_MASTER_HOST:-spark-master}
-      SPARK_MASTER_PORT: ${SPARK_MASTER_PORT:-7077}
-      SPARK_WORKER_WEBUI_PORT: ${SPARK_WORKER_WEBUI_PORT:-4041}
-      PYTHONPATH: /project
-      DJANGO_SETTINGS_MODULE: "usaspending_api.settings"
-      DATABASE_URL: postgres://${USASPENDING_DB_USER}:${USASPENDING_DB_PASSWORD}@${USASPENDING_DB_HOST}:${USASPENDING_DB_PORT}/data_store_api
+      <<: *spark-submit-vars
+    entrypoint: []  # override the entrypoint
     command: /bin/sh -c "$${SPARK_HOME}/bin/spark-class org.apache.spark.deploy.worker.Worker --webui-port $${SPARK_WORKER_WEBUI_PORT} spark://$${SPARK_MASTER_HOST}:$${SPARK_MASTER_PORT}"
     ports:
-      - ${SPARK_WORKER_WEBUI_PORT:-4041}:4041
+      - ${SPARK_WORKER_WEBUI_PORT}:4041
     volumes:
       - type: bind
         source: .
-        target: /project
+        target: /usaspending-api
         read_only: false
-
+      # Required to interact with host's docker daemon from within this running container and access the Broker DB
+      - /var/run/docker.sock:/var/run/docker.sock
   spark-history-server:
     profiles:
       - spark  # must pass --profile spark to docker compose for this to come up
       - test
-    image: spark-base  # when an image by this name is not found in the local repo, and it is forced to build, it will use this as the tag
+    image: usaspending-development
     # build context path needs to be relative to project root, from where docker compose will be run
     build:
       context: .
-      dockerfile: spark.Dockerfile
+      dockerfile: development.Dockerfile
       args:
         PROJECT_LOG_DIR: ${PROJECT_LOG_DIR}
+        SPARK_LOGGING_LEVEL: ${SPARK_LOGGING_LEVEL}
     container_name: spark-history-server
     environment:
-      SPARK_HISTORY_SERVER_PORT: ${SPARK_HISTORY_SERVER_PORT:-18080}
-    command: /bin/sh -c "$${SPARK_HOME}/bin/spark-class org.apache.spark.deploy.history.HistoryServer"
+      SPARK_HISTORY_SERVER_PORT: ${SPARK_HISTORY_SERVER_PORT}
+    entrypoint: []  # override the entrypoint
+    command: /usr/bin/sh -c "$${SPARK_HOME}/bin/spark-class org.apache.spark.deploy.history.HistoryServer"
     ports:
-      - ${SPARK_HISTORY_SERVER_PORT:-18080}:18080
+      - ${SPARK_HISTORY_SERVER_PORT}:18080
     volumes:
       - type: bind
         source: .
-        target: /project
+        target: /usaspending-api
         read_only: false
 
   # Example of running spark-submit container:
@@ -352,63 +453,94 @@ services:
   #        - the Docker image at this time only installs spark and hadoop standalone, which does not seem to include all the needed Hive jars
   #
   #    make docker-compose-run profiles="--profile spark" args="--rm -e MINIO_HOST=minio -e JDBC_URL -e COMPONENT_NAME='My Spark Prototype Script' spark-submit \
-  #      --packages org.postgresql:postgresql:42.2.23,io.delta:delta-core_2.12:1.2.1,org.apache.hadoop:hadoop-aws:3.3.1,org.apache.spark:spark-hive_2.12:3.2.1 \
-  #      /project/usaspending_api/etl/tests/path_to_your_spark_prototype_script.py"
+  #      --packages <SEE MAKEFILE> \
+  #      /usaspending-api/usaspending_api/etl/tests/path_to_your_spark_prototype_script.py"
 
   spark-submit:
     profiles:
       - spark  # must pass --profile spark to docker compose for this to come up
       - test
-    image: spark-base  # when an image by this name is not found in the local repo, and it is forced to build, it will use this as the tag
+    image: usaspending-development
     # build context path needs to be relative to project root, from where docker compose will be run
     build:
       context: .
-      dockerfile: spark.Dockerfile
+      dockerfile: development.Dockerfile
       args:
         PROJECT_LOG_DIR: ${PROJECT_LOG_DIR}
+        SPARK_LOGGING_LEVEL: ${SPARK_LOGGING_LEVEL}
     container_name: spark-submit
     depends_on:
+      - minio
+      - spark-history-server
       - spark-master
       - spark-worker
-      - spark-history-server
-      - minio
+      - usaspending-db
     environment:
-      IS_LOCAL: ${IS_LOCAL}
-      DATABASE_URL: postgres://${USASPENDING_DB_USER}:${USASPENDING_DB_PASSWORD}@${USASPENDING_DB_HOST}:${USASPENDING_DB_PORT}/data_store_api
-      MINIO_HOST: ${MINIO_HOST}
-      SPARK_MASTER_HOST: ${SPARK_MASTER_HOST:-spark-master}
-      SPARK_MASTER_PORT: ${SPARK_MASTER_PORT:-7077}
-      # i.e. target where host warehouse dir is bound in below volume config.
-      #   This env var needs to be picked up as the config for the spark.sql.warehouse.dir spark conf setting when SparkSessions are created inside of a spark-submitted job
-      SPARK_SQL_WAREHOUSE_DIR: /spark-warehouse
-      # i.e. a metastore_db sub dir of the target where host warehouse dir is bound in below volume config.
-      #   This env var needs to be picked up as the path part of the config for the spark.hadoop.javax.jdo.option.ConnectionURL spark conf setting when SparkSessions are created inside of a spark-submitted job
-      HIVE_METASTORE_DERBY_DB_DIR: /spark-warehouse/metastore_db
-      PYTHONPATH: "/project"
-      TOGGLE_OTEL_CONSOLE_LOGGING: ${TOGGLE_OTEL_CONSOLE_LOGGING}
-      OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: ${OTEL_EXPORTER_OTLP_TRACES_ENDPOINT}
+      <<: *spark-submit-vars
     # NOTE: entrypoint CANNOT interpolate env vars when processed. They are passed through literally.
     # So in using 1 $ rather than 2 $$, the var is evaluated based on the current SHELL ENV when docker compose is run,
     # and interpolated before accessed as the entrypoint.
     # While this service has values for these interpolated vars in the environment: element, those are not used here,
     # but merely passed into the container. KEEP the two references to these vars and their defaults consistent!
     # To see what it will be, you can run docker compose config (i.e. make docker-compose-config in this project's Makefile)
-    entrypoint: ./bin/spark-submit --master spark://${SPARK_MASTER_HOST:-spark-master}:${SPARK_MASTER_PORT:-7077}
+    entrypoint: /usr/bin/spark-submit --master spark://${SPARK_MASTER_HOST}:${SPARK_MASTER_PORT}
     command: --help
     volumes:
       - type: bind
-        source: ${PROJECT_DIRECTORY:-.}
-        target: /project
+        source: .
+        target: /usaspending-api
         read_only: false
       # NOTE: The hive metastore_db Derby database folder is expected to be configured to show up as a subfolder of the spark-warehouse dir
       - type: bind
-        source: ${SPARK_SQL_WAREHOUSE_DIR:-./spark-warehouse}
-        target: /spark-warehouse
-      # Mount the JAR dependencies local repo on host into container to take advantage of caching/reuse
-      # i.e., to download the dependencies only once and reuse on subsequent docker compose run calls
+        source: ${SOURCE_SPARK_SQL_WAREHOUSE_DIR}
+        target: ${TARGET_SPARK_SQL_WAREHOUSE_DIR}
+      # Required to interact with host's docker daemon from within this running container and access the Broker DB
+      - /var/run/docker.sock:/var/run/docker.sock
+  spark-shell:
+    profiles:
+      - spark  # must pass --profile spark to docker compose for this to come up
+      - test
+    #image: public.ecr.aws/emr-serverless/spark/emr-7.12.0:latest
+    image: usaspending-development
+    # build context path needs to be relative to project root, from where docker compose will be run
+    build:
+      context: .
+      dockerfile: development.Dockerfile
+      args:
+        PROJECT_LOG_DIR: ${PROJECT_LOG_DIR}
+        SPARK_LOGGING_LEVEL: ${SPARK_LOGGING_LEVEL}
+    container_name: spark-shell
+    depends_on:
+      - minio
+    environment:
+      <<: *spark-submit-vars
+    # NOTE: entrypoint CANNOT interpolate env vars when processed. They are passed through literally.
+    # So in using 1 $ rather than 2 $$, the var is evaluated based on the current SHELL ENV when docker compose is run,
+    # and interpolated before accessed as the entrypoint.
+    # While this service has values for these interpolated vars in the environment: element, those are not used here,
+    # but merely passed into the container. KEEP the two references to these vars and their defaults consistent!
+    # To see what it will be, you can run docker compose config (i.e. make docker-compose-config in this project's Makefile)
+    entrypoint: >
+      /usr/bin/pyspark --master local[*]
+        --conf spark.sql.extensions=io.delta.sql.DeltaSparkSessionExtension
+        --conf spark.sql.catalog.spark_catalog=org.apache.spark.sql.delta.catalog.DeltaCatalog
+        --conf spark.hadoop.fs.s3a.endpoint=${MINIO_HOST}:${MINIO_PORT}
+        --conf spark.hadoop.fs.s3a.access.key=${MINIO_ACCESS_KEY}
+        --conf spark.hadoop.fs.s3a.secret.key=${MINIO_SECRET_KEY}
+        --conf spark.hadoop.fs.s3a.connection.ssl.enabled=false
+        --conf spark.hadoop.fs.s3a.path.style.access=true
+        --conf spark.sql.catalogImplementation=hive
+        --conf spark.sql.warehouse.dir='${TARGET_SPARK_SQL_WAREHOUSE_DIR}'
+        --conf spark.hadoop.javax.jdo.option.ConnectionURL='jdbc:derby:;databaseName=${TARGET_HIVE_METASTORE_DERBY_DB_DIR};create=true'
+    volumes:
+      - type: bind
+        source: .
+        target: /usaspending-api
+        read_only: false
+      # NOTE: The hive metastore_db Derby database folder is expected to be configured to show up as a subfolder of the spark-warehouse dir
       - type: bind
-        source: ${HOME}/.ivy2
-        target: /root/.ivy2
+        source: ${SOURCE_SPARK_SQL_WAREHOUSE_DIR}
+        target: ${TARGET_SPARK_SQL_WAREHOUSE_DIR}
         read_only: false
 
   otel-collector:
@@ -422,6 +554,8 @@ services:
   # Tempo runs as user 10001, and docker compose creates the volume as root.
   # As such, we need to chown the volume in order for Tempo to start correctly.
   tempo-init:
+    profiles:
+      - tracing
     image: &tempoImage grafana/tempo:2.8.2
     user: root
     entrypoint:
diff --git a/download.Dockerfile b/download.Dockerfile
index 0cf278072f..bacaa898f1 100644
--- a/download.Dockerfile
+++ b/download.Dockerfile
@@ -1,6 +1,6 @@
 FROM usaspending-backend:latest
 
-WORKDIR /dockermount
+WORKDIR /usaspending-api
 
 RUN apt update && apt install -y \
     build-essential \
diff --git a/Dockerfile.duckdb b/duckdb.Dockerfile
similarity index 100%
rename from Dockerfile.duckdb
rename to duckdb.Dockerfile
diff --git a/pyproject.toml b/pyproject.toml
index f04239b640..1be1e8536a 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -117,8 +117,8 @@ dev = [
     "ruff==0.14.14",
 ]
 spark = [
-    "delta-spark==3.2.*",
-    "pyspark==3.5.0",
+    "delta-spark==3.3.2",
+    "pyspark==3.5.6",
 ]
 server = [
     "django-redis==5.4.*",
diff --git a/spark.Dockerfile b/spark.Dockerfile
deleted file mode 100644
index 3aef9d3ec3..0000000000
--- a/spark.Dockerfile
+++ /dev/null
@@ -1,80 +0,0 @@
-FROM python:3.10.12-slim-bullseye
-
-COPY --from=ghcr.io/astral-sh/uv:0.7.19 /uv /uvx /bin/
-
-# Build ARGs
-ARG HADOOP_VERSION=3.3.4
-ARG SPARK_VERSION=3.5.0
-ARG PROJECT_LOG_DIR=/logs
-
-# Install dependencies
-RUN apt update && apt install -y \
-    build-essential \
-    coreutils \
-    curl \
-    gcc \
-    libbz2-dev \
-    libpq-dev \
-    libssl-dev \
-    openssl \
-    procps \
-    wget
-
-# Install Amazon Corretto 8 (a Long-Term Supported (LTS) distribution of OpenJDK 8)
-RUN wget -qO - https://apt.corretto.aws/corretto.key | gpg --dearmor -o /usr/share/keyrings/corretto-keyring.gpg && \
-    echo "deb [signed-by=/usr/share/keyrings/corretto-keyring.gpg] https://apt.corretto.aws stable main" | tee /etc/apt/sources.list.d/corretto.list
-RUN apt update && \
-    apt install -y java-1.8.0-amazon-corretto-jdk
-ENV JAVA_HOME=/usr/lib/jvm/java-1.8.0-amazon-corretto
-
-# Install Hadoop and Spark
-WORKDIR /usr/local
-
-RUN wget --quiet https://archive.apache.org/dist/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz \
-    && tar xzf hadoop-${HADOOP_VERSION}.tar.gz \
-    && ln -sfn /usr/local/hadoop-${HADOOP_VERSION} /usr/local/hadoop \
-    && wget --quiet https://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-without-hadoop.tgz \
-    && tar xzf spark-${SPARK_VERSION}-bin-without-hadoop.tgz \
-    && ln -sfn /usr/local/spark-${SPARK_VERSION}-bin-without-hadoop /usr/local/spark \
-    && echo "Installed $(/usr/local/hadoop/bin/hadoop version)"
-ENV HADOOP_HOME=/usr/local/hadoop
-ENV SPARK_HOME=/usr/local/spark
-# Cannot set ENV var = command-result, [i.e. doing: ENV SPARK_DIST_CLASSPATH=$(${HADOOP_HOME}/bin/hadoop classpath)], so interpolating the hadoop classpath the long way
-ENV SPARK_DIST_CLASSPATH="$HADOOP_HOME/etc/hadoop/*:$HADOOP_HOME/share/hadoop/common/lib/*:$HADOOP_HOME/share/hadoop/common/*:$HADOOP_HOME/share/hadoop/hdfs/*:$HADOOP_HOME/share/hadoop/hdfs/lib/*:$HADOOP_HOME/share/hadoop/hdfs/*:$HADOOP_HOME/share/hadoop/yarn/lib/*:$HADOOP_HOME/share/hadoop/yarn/*:$HADOOP_HOME/share/hadoop/mapreduce/lib/*:$HADOOP_HOME/share/hadoop/mapreduce/*:$HADOOP_HOME/share/hadoop/tools/lib/*"
-ENV PATH=${SPARK_HOME}/bin:${HADOOP_HOME}/bin:${JAVA_HOME}/bin:${PATH}
-RUN echo "Installed Spark" && echo "$(${SPARK_HOME}/bin/pyspark --version)"
-
-# Config for starting up the Spark History Server
-# This allows a web entrypoint to see past job runs, and bring up their Spark UI
-# It allows job-run event logs to be monitored (e.g. at /tmp/spark-events)
-# NOTE: The specified eventlog dir and its path does not exist. It is assumed that a host dir will be bind-mounted at /project
-#       so that all containers that do the same can share the events.
-RUN cp $SPARK_HOME/conf/spark-defaults.conf.template $SPARK_HOME/conf/spark-defaults.conf \
-    && sed -i "s|^# spark.eventLog.dir.*$|spark.eventLog.dir                 file:///project/$PROJECT_LOG_DIR/spark-events|g" $SPARK_HOME/conf/spark-defaults.conf \
-    && sed -i '/spark.eventLog.enabled/s/^# //g' $SPARK_HOME/conf/spark-defaults.conf \
-    && echo "spark.history.fs.logDirectory      file:///project/$PROJECT_LOG_DIR/spark-events" >> $SPARK_HOME/conf/spark-defaults.conf
-
-# Bake project dependencies into any spark-base -based container that will run Project python code (and require these dependencies)
-# NOTE: We must exclude any dependencies on the host machine installed in a virtual environment (assumed to be stored under ./.venv)
-WORKDIR /project
-
-##### The following ENV vars are optimizations from https://github.com/astral-sh/uv-docker-example/blob/main/Dockerfile
-##### and https://docs.astral.sh/uv/guides/integration/docker/#optimizations
-# Enable bytecode compilation
-ENV UV_COMPILE_BYTECODE=1
-
-# Copy from the cache instead of linking since it's a mounted volume
-ENV UV_LINK_MODE=copy
-
-# Use the system Python environment since the container is already isolated
-ENV UV_PROJECT_ENVIRONMENT=/usr/local
-ENV UV_SYSTEM_PYTHON=1
-
-# Install dependencies
-RUN --mount=type=cache,target=/root/.cache/uv \
-    --mount=type=bind,source=uv.lock,target=uv.lock \
-    --mount=type=bind,source=pyproject.toml,target=pyproject.toml \
-    uv sync --locked --no-install-project --no-dev
-
-# Allow for entrypoints and commands to infer that they are running relative to this directory
-WORKDIR ${SPARK_HOME}
diff --git a/testing.Dockerfile b/testing.Dockerfile
deleted file mode 100644
index c126e93147..0000000000
--- a/testing.Dockerfile
+++ /dev/null
@@ -1,32 +0,0 @@
-# Dockerfile for usaspending-test image
-
-# If any changes are made to the usaspending-backend image (Dockerfile) then this image needs to be rebuilt to stay in sync.
-
-# If you do not already have the `usaspending-backend` image locally you can build it by running:
-#       docker compose --profile usaspending build
-# in the parent usaspending-api directory.
-
-FROM usaspending-backend:latest
-
-# Install dependencies
-RUN apt update && apt install -y \
-    build-essential \
-    coreutils \
-    procps \
-    wget
-
-# Install Amazon Corretto 8 (a Long-Term Supported (LTS) distribution of OpenJDK 8)
-RUN wget -qO - https://apt.corretto.aws/corretto.key | gpg --dearmor -o /usr/share/keyrings/corretto-keyring.gpg && \
-    echo "deb [signed-by=/usr/share/keyrings/corretto-keyring.gpg] https://apt.corretto.aws stable main" | tee /etc/apt/sources.list.d/corretto.list
-RUN apt update && \
-    apt install -y java-1.8.0-amazon-corretto-jdk
-RUN export JAVA_HOME=/usr/lib/jvm/java-1.8.0-amazon-corretto
-RUN export PATH=${JAVA_HOME}/bin:$PATH
-
-# Install dev dependencies
-RUN --mount=type=cache,target=/root/.cache/uv \
-    --mount=type=bind,source=uv.lock,target=uv.lock \
-    --mount=type=bind,source=pyproject.toml,target=pyproject.toml \
-    uv sync --extra dev --extra spark --locked
-
-WORKDIR /dockermount
\ No newline at end of file
diff --git a/usaspending_api/common/helpers/spark_helpers.py b/usaspending_api/common/helpers/spark_helpers.py
index 51df833e45..068b27db3c 100644
--- a/usaspending_api/common/helpers/spark_helpers.py
+++ b/usaspending_api/common/helpers/spark_helpers.py
@@ -69,13 +69,7 @@ def is_spark_context_stopped() -> bool:
         # Check the Singleton instance populated if there's an active SparkContext
         if SparkContext._active_spark_context is not None:
             sc = SparkContext._active_spark_context
-            is_stopped = not (
-                sc._jvm
-                and not sc._jvm.SparkSession.getDefaultSession()
-                .get()
-                .sparkContext()
-                .isStopped()
-            )
+            is_stopped = not (sc._jvm and not sc._jvm.SparkSession.getDefaultSession().get().sparkContext().isStopped())
     return is_stopped
 
 
@@ -89,10 +83,7 @@ def stop_spark_context() -> bool:
                 sc._jvm
                 and hasattr(sc._jvm, "SparkSession")
                 and sc._jvm.SparkSession
-                and not sc._jvm.SparkSession.getDefaultSession()
-                .get()
-                .sparkContext()
-                .isStopped()
+                and not sc._jvm.SparkSession.getDefaultSession().get().sparkContext().isStopped()
             ):
                 try:
                     sc.stop()
@@ -157,15 +148,9 @@ def configure_spark_session(  # noqa: C901,PLR0912,PLR0913,PLR0915
             property, otherwise an error will be thrown.
     """
     if spark_context and (
-        not spark_context._jvm
-        or spark_context._jvm.SparkSession.getDefaultSession()
-        .get()
-        .sparkContext()
-        .isStopped()
+        not spark_context._jvm or spark_context._jvm.SparkSession.getDefaultSession().get().sparkContext().isStopped()
     ):
-        raise ValueError(
-            "The provided spark_context arg is a stopped SparkContext. It must be active."
-        )
+        raise ValueError("The provided spark_context arg is a stopped SparkContext. It must be active.")
     if spark_context and java_gateway:
         raise Exception(
             "Cannot provide BOTH spark_context and java_gateway args. The active spark_context supplies its own gateway"
@@ -187,6 +172,9 @@ def configure_spark_session(  # noqa: C901,PLR0912,PLR0913,PLR0915
     # Assume that random errors are rare, and jobs have long runtimes, so fail fast, fix and retry manually.
     conf.set("spark.yarn.maxAppAttempts", "1")
     conf.set("spark.hadoop.fs.s3a.endpoint", CONFIG.AWS_S3_ENDPOINT)
+    # We work with a lot of larger tables and Delta wants to try and materialize the "source" data in a merge statement
+    # into memory which can spill onto disk and cause problems.
+    conf.set("spark.databricks.delta.merge.materializeSource", "none")
 
     if not CONFIG.USE_AWS:  # i.e. running in a "local" [development] environment
         # Set configs to allow the S3AFileSystem to work against a local MinIO object storage proxy
@@ -271,9 +259,7 @@ def configure_spark_session(  # noqa: C901,PLR0912,PLR0913,PLR0915
     if spark_context:
         built_conf = spark.conf
         provided_conf_keys = [item[0] for item in conf.getAll()]
-        non_modifiable_conf = [
-            k for k in provided_conf_keys if not built_conf.isModifiable(k)
-        ]
+        non_modifiable_conf = [k for k in provided_conf_keys if not built_conf.isModifiable(k)]
         if non_modifiable_conf:
             raise ValueError(
                 "An active SparkContext was given along with NEW spark config values. The following "
@@ -298,7 +284,8 @@ def configure_spark_session(  # noqa: C901,PLR0912,PLR0913,PLR0915
 @       Found SPARK_HOME: {_find_spark_home()}
 @       Python Version: {sys.version}
 @       Spark Version: {spark.version}
-@       Hadoop Version: {spark.sparkContext._gateway.jvm.org.apache.hadoop.util.VersionInfo.getVersion()}
+@       Hadoop Version: {spark.sparkContext._jvm.org.apache.hadoop.util.VersionInfo.getVersion()}
+@       Hive Version: {spark.sparkContext._jvm.org.apache.hive.common.util.HiveVersionInfo.getVersion()}
     """
     )
     es_config = get_es_config()
@@ -388,9 +375,7 @@ def attach_java_gateway(
     return gateway
 
 
-def get_jdbc_connection_properties(
-    fix_strings: bool = True, truncate: bool = False
-) -> dict:
+def get_jdbc_connection_properties(fix_strings: bool = True, truncate: bool = False) -> dict:
     jdbc_props = {
         "driver": "org.postgresql.Driver",
         "fetchsize": str(CONFIG.SPARK_PARTITION_ROWS),
@@ -410,9 +395,7 @@ def get_jdbc_url_from_pg_uri(pg_uri: str) -> str:
     """Converts the passed-in Postgres DB connection URI to a JDBC-compliant Postgres DB connection string"""
     url_parts, user, password = parse_pg_uri(pg_uri)
     if user is None or password is None:
-        raise ValueError(
-            "pg_uri provided must have username and password with host or in query string"
-        )
+        raise ValueError("pg_uri provided must have username and password with host or in query string")
     # JDBC URLs only support postgresql://
     pg_uri = f"postgresql://{url_parts.hostname}:{url_parts.port}{url_parts.path}?user={user}&password={password}"
 
@@ -472,9 +455,7 @@ def get_es_config() -> dict[str, Any]:  # pragma: no cover -- will be used event
         "es.net.ssl": str(ssl).lower(),  # default false
         "es.net.ssl.cert.allow.self.signed": "true",  # default false
         "es.batch.size.entries": str(CONFIG.ES_BATCH_ENTRIES),  # default 1000
-        "es.batch.size.bytes": str(
-            CONFIG.ES_MAX_BATCH_BYTES
-        ),  # default 1024*1024 (1mb)
+        "es.batch.size.bytes": str(CONFIG.ES_MAX_BATCH_BYTES),  # default 1024*1024 (1mb)
         "es.batch.write.refresh": "false",  # default true, to refresh after configured batch size completes
     }
 
@@ -527,12 +508,8 @@ def configure_s3_credentials(
             "org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider",
         )
         conf.set("spark.hadoop.fs.s3a.session.token", aws_creds.token)
-        conf.set(
-            "spark.hadoop.fs.s3a.assumed.role.sts.endpoint", CONFIG.AWS_STS_ENDPOINT
-        )
-        conf.set(
-            "spark.hadoop.fs.s3a.assumed.role.sts.endpoint.region", CONFIG.AWS_REGION
-        )
+        conf.set("spark.hadoop.fs.s3a.assumed.role.sts.endpoint", CONFIG.AWS_STS_ENDPOINT)
+        conf.set("spark.hadoop.fs.s3a.assumed.role.sts.endpoint.region", CONFIG.AWS_REGION)
 
 
 def log_spark_config(spark: SparkSession, config_key_contains: str = "") -> None:
@@ -551,9 +528,7 @@ def log_hadoop_config(spark: SparkSession, config_key_contains: str = "") -> Non
     conf = spark.sparkContext._jsc.hadoopConfiguration()
     [
         logger.info(f"{k}={v}")
-        for (k, v) in {
-            str(_).split("=")[0]: str(_).split("=")[1] for _ in conf.iterator()
-        }.items()
+        for (k, v) in {str(_).split("=")[0]: str(_).split("=")[1] for _ in conf.iterator()}.items()
         if config_key_contains in k
     ]
 
@@ -582,16 +557,10 @@ def load_dict_to_delta_table(  # noqa: PLR0912
     table_to_col_names_dict = {}
     table_to_col_names_dict["transaction_fabs"] = TRANSACTION_FABS_COLUMNS
     table_to_col_names_dict["transaction_fpds"] = TRANSACTION_FPDS_COLUMNS
-    table_to_col_names_dict["transaction_normalized"] = list(
-        TRANSACTION_NORMALIZED_COLUMNS
-    )
+    table_to_col_names_dict["transaction_normalized"] = list(TRANSACTION_NORMALIZED_COLUMNS)
     table_to_col_names_dict["awards"] = list(AWARDS_COLUMNS)
-    table_to_col_names_dict["financial_accounts_by_awards"] = list(
-        FINANCIAL_ACCOUNTS_BY_AWARDS_COLUMNS
-    )
-    table_to_col_names_dict["detached_award_procurement"] = list(
-        DETACHED_AWARD_PROCUREMENT_DELTA_COLUMNS
-    )
+    table_to_col_names_dict["financial_accounts_by_awards"] = list(FINANCIAL_ACCOUNTS_BY_AWARDS_COLUMNS)
+    table_to_col_names_dict["detached_award_procurement"] = list(DETACHED_AWARD_PROCUREMENT_DELTA_COLUMNS)
     table_to_col_names_dict["published_fabs"] = list(PUBLISHED_FABS_COLUMNS)
 
     table_to_col_info_dict = {}
@@ -686,9 +655,7 @@ def clean_postgres_sql_for_spark_sql(
     # Treat these type casts as string in Spark SQL
     # NOTE: If replacing a ::JSON cast, be sure that the string data coming from delta is treated as needed (e.g. as
     # JSON or converted to JSON or a dict) on the receiving side, and not just left as a string
-    spark_sql = re.sub(
-        r"::text|::json", r"::string", spark_sql, flags=re.IGNORECASE | re.MULTILINE
-    )
+    spark_sql = re.sub(r"::text|::json", r"::string", spark_sql, flags=re.IGNORECASE | re.MULTILINE)
 
     if global_temp_view_proxies:
         for vw in global_temp_view_proxies:
diff --git a/usaspending_api/common/spark/configs.py b/usaspending_api/common/spark/configs.py
index 4113a4afb3..7faefaff92 100644
--- a/usaspending_api/common/spark/configs.py
+++ b/usaspending_api/common/spark/configs.py
@@ -1,25 +1,5 @@
 from usaspending_api.config import CONFIG
 
-# The versions below are determined by the current version of Databricks in use
-_SCALA_VERSION = "2.12"
-_HADOOP_VERSION = "3.3.4"
-_SPARK_VERSION = "3.5.0"
-_DELTA_VERSION = "3.1.0"
-
-# List of Maven coordinates for required JAR files used by running code, which can be added to the driver and
-# executor class paths
-SPARK_SESSION_JARS = [
-    # "com.amazonaws:aws-java-sdk:1.12.31",
-    # hadoop-aws is an add-on to hadoop with Classes that allow hadoop to interface with an S3A (AWS S3) FileSystem
-    # NOTE That in order to work, the version number should be the same as the Hadoop version used by your Spark runtime
-    # It SHOULD pull in (via Ivy package manager from maven repo) the version of com.amazonaws:aws-java-sdk that is
-    # COMPATIBLE with it (so that should not  be set as a dependent package by us)
-    f"org.apache.hadoop:hadoop-aws:{_HADOOP_VERSION}",
-    "org.postgresql:postgresql:42.2.23",
-    f"io.delta:delta-spark_{_SCALA_VERSION}:{_DELTA_VERSION}",
-]
-OPTIONAL_SPARK_HIVE_JAR = f"org.apache.spark:spark-hive_{_SCALA_VERSION}:{_SPARK_VERSION}"
-
 # TODO: This should be used more widely across our different commands
 DEFAULT_EXTRA_CONF = {
     # Config for Delta Lake tables and SQL. Need these to keep Dela table metadata in the metastore
@@ -45,7 +25,6 @@
     "spark.driver.memory": "1g",
     "spark.executor.memory": "1g",
     "spark.ui.enabled": "false",  # Does the same as setting SPARK_TESTING=true env var
-    "spark.jars.packages": ",".join(SPARK_SESSION_JARS),
 }
 
 
diff --git a/usaspending_api/common/spark/jobs.py b/usaspending_api/common/spark/jobs.py
index 5e3f95e23e..1f62fdd0ab 100644
--- a/usaspending_api/common/spark/jobs.py
+++ b/usaspending_api/common/spark/jobs.py
@@ -13,9 +13,9 @@
 from django.conf import settings
 from django.core.management import call_command
 from duckdb.experimental.spark.sql import SparkSession as DuckDBSparkSession
-from usaspending_api.config import CONFIG
 
-from usaspending_api.common.spark.configs import LOCAL_EXTENDED_EXTRA_CONF, OPTIONAL_SPARK_HIVE_JAR, SPARK_SESSION_JARS
+from usaspending_api.common.spark.configs import LOCAL_EXTENDED_EXTRA_CONF
+from usaspending_api.config import CONFIG
 
 if TYPE_CHECKING:
     from pyspark.sql import SparkSession
@@ -153,9 +153,9 @@ def handle_start(self, job_name: str, command_name: str, command_options: list[s
         execution_role_arn = kwargs.get("execution_role_arn")
 
         if not execution_role_arn:
-            raise ValueError(f"Execution role ARN is required to start an EMR Serverless job")
+            raise ValueError("Execution role ARN is required to start an EMR Serverless job")
         elif not application_name and not application_id:
-            raise ValueError(f"Application Name or ID is required to start an EMR Serverless job")
+            raise ValueError("Application Name or ID is required to start an EMR Serverless job")
         elif application_name and not application_id:
             application_id = self._get_application_id(application_name)
 
@@ -230,15 +230,12 @@ def _run_in_container(job_name: str, command_name: str, command_options: list[st
         network = template_container.attrs["HostConfig"]["NetworkMode"]
         options_as_string = " ".join(command_options)
         container_name = f"spark-submit_{job_name}_{datetime.now().strftime('%Y-%m-%d-%H-%M-%S')}"
-        required_jars = ",".join([*SPARK_SESSION_JARS, OPTIONAL_SPARK_HIVE_JAR])
         client.containers.run(
             image_name,
             name=container_name,
             network=network,
             command=(
-                'spark-submit --driver-memory "2g"'
-                f" --packages {required_jars}"
-                f" /project/manage.py {command_name} {options_as_string}"
+                f'spark-submit --driver-memory "2g" /usaspending-api/manage.py {command_name} {options_as_string}'
             ),
             environment=[
                 f"COMPONENT_NAME={command_name} {options_as_string}",
diff --git a/usaspending_api/config/envs/local.py b/usaspending_api/config/envs/local.py
index 430a6d8984..2533d620de 100644
--- a/usaspending_api/config/envs/local.py
+++ b/usaspending_api/config/envs/local.py
@@ -51,18 +51,20 @@ class LocalConfig(DefaultConfig):
     USASPENDING_DB_PORT: str = "5432"
 
     # ==== [Postgres Broker] ====
-    BROKER_DB_USER: str | None = "admin"
-    BROKER_DB_PASSWORD: SecretStr | None = "root"
+    # These values match the USAspending values above because the test suite will create a new Broker DB on the same
+    # container as the USAspending DB
+    BROKER_DB_USER: str | None = "usaspending"
+    BROKER_DB_PASSWORD: SecretStr | None = "usaspender"
 
     # Change to host.docker.internal if you are running a local Postgres. Otherwise leave as-is, so
     # Docker will use the Postgres created by Compose.
-    BROKER_DB_HOST: str | None = "dataact-broker-db"
+    BROKER_DB_HOST: str | None = "usaspending-db"
     BROKER_DB_PORT: str | None = "5432"
 
     # ==== [Elasticsearch] ====
     # Where to connect to elasticsearch.
     ES_SCHEME: str = "http"
-    ES_HOST: str = "localhost"
+    ES_HOST: str = "usaspending-es"
     ES_PORT: str = "9200"
 
     # ==== [Spark] ====
@@ -71,7 +73,7 @@ class LocalConfig(DefaultConfig):
     HIVE_METASTORE_DERBY_DB_DIR: str = str(_PROJECT_ROOT_DIR / "spark-warehouse" / "metastore_db")
 
     # ==== [MinIO] ====
-    MINIO_HOST: str = "localhost"
+    MINIO_HOST: str = "minio"
     # Changing MinIO ports from defaults. Known to have port conflicts with proxies on developer laptops
     MINIO_PORT: str = "10001"
     MINIO_CONSOLE_PORT: str = "10002"
diff --git a/usaspending_api/etl/tests/integration/test_execute_spark_sql.py b/usaspending_api/etl/tests/integration/test_execute_spark_sql.py
index 200ca77252..aeb4164052 100644
--- a/usaspending_api/etl/tests/integration/test_execute_spark_sql.py
+++ b/usaspending_api/etl/tests/integration/test_execute_spark_sql.py
@@ -1,10 +1,8 @@
-from pytest import raises
-
-from unittest.mock import patch, MagicMock
+from unittest.mock import MagicMock, patch
 
 from django.core.management import call_command
 from django.core.management.base import CommandError
-
+from pytest import raises
 
 # Argument Validation
 
@@ -31,25 +29,44 @@ def test_no_sql_and_file_raises_error():
 def test_sql_functionality():
     """Tests a variety of cases in a single test for Spark performance"""
 
+    mock_spark = MagicMock()
+
     # Test --sql approach
-    with patch("usaspending_api.etl.management.commands.execute_spark_sql.logger") as mock_logger:
+    with (
+        patch(
+            "usaspending_api.etl.management.commands.execute_spark_sql.get_active_spark_session",
+            return_value=mock_spark,
+        ),
+        patch("usaspending_api.etl.management.commands.execute_spark_sql.logger") as mock_logger,
+    ):
         call_command("execute_spark_sql", "--sql", "SHOW DATABASES; SELECT 1;")
         mock_logger.info.assert_any_call("Found 2 SQL statement(s)")
 
     # Test --file approach with two SQL statements and a blank line
     test_file_path = "usaspending_api/etl/tests/data/test_execute_spark_files/two_queries_with_blank_line.sql"
-    with patch("usaspending_api.etl.management.commands.execute_spark_sql.logger") as mock_logger:
+    with (
+        patch(
+            "usaspending_api.etl.management.commands.execute_spark_sql.get_active_spark_session",
+            return_value=mock_spark,
+        ),
+        patch("usaspending_api.etl.management.commands.execute_spark_sql.logger") as mock_logger,
+    ):
         call_command("execute_spark_sql", "--file", test_file_path)
         mock_logger.info.assert_any_call("Found 2 SQL statement(s)")
 
     # Test --file approach with an empty file
     test_file_path = "usaspending_api/etl/tests/data/test_execute_spark_files/empty_file.sql"
-    with patch("usaspending_api.etl.management.commands.execute_spark_sql.logger") as mock_logger:
+    with (
+        patch(
+            "usaspending_api.etl.management.commands.execute_spark_sql.get_active_spark_session",
+            return_value=mock_spark,
+        ),
+        patch("usaspending_api.etl.management.commands.execute_spark_sql.logger") as mock_logger,
+    ):
         call_command("execute_spark_sql", "--file", test_file_path)
         mock_logger.info.assert_any_call("Found 0 SQL statement(s)")
 
     # Test SQL executes
-    mock_spark = MagicMock()
     with patch(
         "usaspending_api.etl.management.commands.execute_spark_sql.get_active_spark_session", return_value=mock_spark
     ):
diff --git a/usaspending_api/etl/tests/integration/test_load_to_from_delta.py b/usaspending_api/etl/tests/integration/test_load_to_from_delta.py
index 56428ab0be..e8f02299b8 100644
--- a/usaspending_api/etl/tests/integration/test_load_to_from_delta.py
+++ b/usaspending_api/etl/tests/integration/test_load_to_from_delta.py
@@ -987,19 +987,21 @@ def test_incremental_load_table_to_delta_for_award_search(
         .table("award_search")
         .select(["award_id", "_change_type", "_commit_version"])
         .toPandas()
+        .sort_values(by=["award_id", "_change_type", "_commit_version"])
+        .reset_index(drop=True)
     )
     expected = pd.DataFrame(
         {
-            "award_id": [4, 4, 1, 3, 2, 4],
+            "award_id": [1, 2, 3, 4, 4, 4],
             "_change_type": [
-                "delete",
                 "insert",
                 "insert",
                 "insert",
+                "delete",
                 "insert",
                 "insert",
             ],
-            "_commit_version": [2, 3, 1, 1, 1, 1],
+            "_commit_version": [1, 1, 1, 2, 1, 3],
         }
     )
     pd.testing.assert_frame_equal(result, expected)
@@ -1085,21 +1087,23 @@ def test_incremental_load_table_to_delta_for_transaction_search(
         .table("transaction_search")
         .select(["transaction_id", "_change_type", "_commit_version"])
         .toPandas()
+        .sort_values(by=["transaction_id", "_change_type", "_commit_version"])
+        .reset_index(drop=True)
     )
     expected = pd.DataFrame(
         {
-            "transaction_id": [4, 4, 1, 2, 434, 3, 4, 5],
+            "transaction_id": [1, 2, 3, 4, 4, 4, 5, 434],
             "_change_type": [
-                "delete",
                 "insert",
                 "insert",
                 "insert",
+                "delete",
                 "insert",
                 "insert",
                 "insert",
                 "insert",
             ],
-            "_commit_version": [2, 3, 1, 1, 1, 1, 1, 1],
+            "_commit_version": [1, 1, 1, 2, 1, 3, 1, 1],
         }
     )
     pd.testing.assert_frame_equal(result, expected)
diff --git a/usaspending_api/search/delta_models/dataframes/abstract_search.py b/usaspending_api/search/delta_models/dataframes/abstract_search.py
index 1cbb91064f..a152a0911d 100644
--- a/usaspending_api/search/delta_models/dataframes/abstract_search.py
+++ b/usaspending_api/search/delta_models/dataframes/abstract_search.py
@@ -19,7 +19,6 @@ def extract_numbers_as_string(col: Column, length: int = 2, pad: str = "0") -> C
 
 
 class AbstractSearch(ABC):
-
     def __init__(self, spark: SparkSession):
         # Base Tables
         self.transaction_normalized = spark.table("int.transaction_normalized")
@@ -50,6 +49,9 @@ def __init__(self, spark: SparkSession):
             .withColumn("awarding_subtier_agency_name", sf.col("name"))
             .withColumn("awarding_subtier_agency_code", sf.col("subtier_code"))
         )
+        # TODO: This logic should be updated to match self.funding_agency_id below to account for
+        #       special case agencies such as FEMA that don't have a record with toptier_flag=true,
+        #       but are a toptier agency to many subtier agencies.
         self.awarding_agency_id = spark.table("global_temp.agency").withColumn(
             "awarding_toptier_agency_id", sf.col("id")
         )
diff --git a/usaspending_api/search/delta_models/dataframes/award_search.py b/usaspending_api/search/delta_models/dataframes/award_search.py
index 7d00392247..50207c7612 100644
--- a/usaspending_api/search/delta_models/dataframes/award_search.py
+++ b/usaspending_api/search/delta_models/dataframes/award_search.py
@@ -283,7 +283,10 @@ def dataframe(self) -> DataFrame:
                 "leftouter",
             )
             .join(
-                self.funding_agency_id,
+                # Without marking this dataframe as a broadcast join it will shuffle the partitions such that agencies
+                # are grouped together. This creates heavily skewed data across tasks which run into resource
+                # constraints.
+                sf.broadcast(self.funding_agency_id),
                 (self.funding_agency_id.toptier_agency_id == self.funding_toptier_agency.funding_toptier_agency_id)
                 & (self.funding_agency_id.row_num == 1),
                 "leftouter",
diff --git a/usaspending_api/search/delta_models/dataframes/transaction_search.py b/usaspending_api/search/delta_models/dataframes/transaction_search.py
index e334764c06..ed13e7c97c 100644
--- a/usaspending_api/search/delta_models/dataframes/transaction_search.py
+++ b/usaspending_api/search/delta_models/dataframes/transaction_search.py
@@ -23,11 +23,7 @@ class TransactionSearch(AbstractSearch):
     def recipient_hash_and_levels(self) -> DataFrame:
         return (
             self.recipient_profile.groupBy("recipient_hash", "uei")
-            .agg(
-                sf.sort_array(sf.collect_set("recipient_level")).alias(
-                    "recipient_levels"
-                )
-            )
+            .agg(sf.sort_array(sf.collect_set("recipient_level")).alias("recipient_levels"))
             .select(
                 sf.col("recipient_hash").alias("recipient_level_hash"),
                 sf.col("recipient_levels"),
@@ -39,20 +35,17 @@ def fed_and_tres_acct(self) -> DataFrame:
         return (
             self.faba.join(
                 self.treasury_appropriation_account,
-                self.treasury_appropriation_account.treasury_account_identifier
-                == self.faba.treasury_account_id,
+                self.treasury_appropriation_account.treasury_account_identifier == self.faba.treasury_account_id,
                 "inner",
             )
             .join(
                 self.federal_account,
-                self.federal_account.id
-                == self.treasury_appropriation_account.federal_account_id,
+                self.federal_account.id == self.treasury_appropriation_account.federal_account_id,
                 "inner",
             )
             .join(
                 self.awarding_toptier_agency,
-                self.federal_account.parent_toptier_agency_id
-                == self.awarding_toptier_agency.toptier_agency_id,
+                self.federal_account.parent_toptier_agency_id == self.awarding_toptier_agency.toptier_agency_id,
                 "inner",
             )
             .join(
@@ -62,8 +55,7 @@ def fed_and_tres_acct(self) -> DataFrame:
             )
             .join(
                 self.program_activity_park,
-                self.faba.program_activity_reporting_key
-                == self.program_activity_park.code,
+                self.faba.program_activity_reporting_key == self.program_activity_park.code,
                 "left",
             )
             .filter(self.faba["award_id"].isNotNull())
@@ -86,15 +78,11 @@ def key_cols(self) -> list[Column]:
     def date_cols(self) -> list[Column]:
         return [
             sf.to_date(self.transaction_normalized.action_date).alias("action_date"),
-            sf.add_months(sf.to_date(self.transaction_normalized.action_date), 3).alias(
-                "fiscal_action_date"
-            ),
+            sf.add_months(sf.to_date(self.transaction_normalized.action_date), 3).alias("fiscal_action_date"),
             self.transaction_normalized.last_modified_date,
             self.transaction_normalized.fiscal_year,
             self.awards.certified_date.alias("award_certified_date"),
-            sf.year(sf.add_months(sf.to_date(self.awards.certified_date), 3)).alias(
-                "award_fiscal_year"
-            ),
+            sf.year(sf.add_months(sf.to_date(self.awards.certified_date), 3)).alias("award_fiscal_year"),
             self.transaction_normalized.create_date.cast(TimestampType()),
             self.transaction_normalized.update_date.cast(TimestampType()),
             self.awards.update_date.cast(TimestampType()).alias("award_update_date"),
@@ -103,12 +91,12 @@ def date_cols(self) -> list[Column]:
                 sf.to_timestamp(self.transaction_normalized.update_date),
                 self.awards.update_date,
             ).alias("etl_update_date"),
-            sf.to_date(
-                self.transaction_normalized.period_of_performance_start_date
-            ).alias("period_of_performance_start_date"),
-            sf.to_date(
-                self.transaction_normalized.period_of_performance_current_end_date
-            ).alias("period_of_performance_current_end_date"),
+            sf.to_date(self.transaction_normalized.period_of_performance_start_date).alias(
+                "period_of_performance_start_date"
+            ),
+            sf.to_date(self.transaction_normalized.period_of_performance_current_end_date).alias(
+                "period_of_performance_current_end_date"
+            ),
             sf.coalesce(
                 self.transaction_fabs.created_at,
                 sf.to_timestamp(self.transaction_fpds.initial_report_date),
@@ -230,24 +218,16 @@ def amounts_cols(self) -> list[Column]:
             )
             .cast(DecimalType(23, 2))
             .alias("generated_pragmatic_obligation"),
-            sf.coalesce(
-                self.transaction_normalized.federal_action_obligation, sf.lit(0)
-            )
+            sf.coalesce(self.transaction_normalized.federal_action_obligation, sf.lit(0))
             .cast(DecimalType(23, 2))
             .alias("federal_action_obligation"),
-            sf.coalesce(
-                self.transaction_normalized.original_loan_subsidy_cost, sf.lit(0)
-            )
+            sf.coalesce(self.transaction_normalized.original_loan_subsidy_cost, sf.lit(0))
             .cast(DecimalType(23, 2))
             .alias("original_loan_subsidy_cost"),
-            sf.coalesce(
-                self.transaction_normalized.face_value_loan_guarantee, sf.lit(0)
-            )
+            sf.coalesce(self.transaction_normalized.face_value_loan_guarantee, sf.lit(0))
             .cast(DecimalType(23, 2))
             .alias("face_value_loan_guarantee"),
-            self.transaction_normalized.indirect_federal_sharing.cast(
-                DecimalType(23, 2)
-            ),
+            self.transaction_normalized.indirect_federal_sharing.cast(DecimalType(23, 2)),
             self.transaction_normalized.funding_amount,
             sf.coalesce(self.transaction_fabs.total_funding_amount, sf.lit("0"))
             .cast(DecimalType(23, 2))
@@ -369,9 +349,7 @@ def recipient_cols(self) -> list[Column]:
                 self.transaction_fpds.ultimate_parent_legal_enti,
                 self.transaction_fabs.ultimate_parent_legal_enti,
             ).alias("parent_recipient_name_raw"),
-            sf.upper(self.parent_recipient.parent_recipient_name).alias(
-                "parent_recipient_name"
-            ),
+            sf.upper(self.parent_recipient.parent_recipient_name).alias("parent_recipient_name"),
             sf.coalesce(
                 self.transaction_fpds.ultimate_parent_unique_ide,
                 self.transaction_fabs.ultimate_parent_unique_ide,
@@ -766,9 +744,7 @@ def fpds_cols(self) -> list[Column]:
             self.transaction_fpds.price_evaluation_adjustmen,
             self.transaction_fpds.private_university_or_coll,
             self.transaction_fpds.product_or_service_code,
-            self.transaction_fpds.product_or_service_co_desc.alias(
-                "product_or_service_description"
-            ),
+            self.transaction_fpds.product_or_service_co_desc.alias("product_or_service_description"),
             self.transaction_fpds.program_acronym,
             self.transaction_fpds.program_system_or_equ_desc,
             self.transaction_fpds.program_system_or_equipmen,
@@ -857,8 +833,7 @@ def dataframe(self) -> DataFrame:
             )
             .join(
                 self.references_cfda,
-                self.transaction_fabs.cfda_number
-                == self.references_cfda.program_number,
+                self.transaction_fabs.cfda_number == self.references_cfda.program_number,
                 "leftouter",
             )
             .join(
@@ -873,29 +848,23 @@ def dataframe(self) -> DataFrame:
             )
             .join(
                 self.awarding_agency,
-                self.transaction_normalized.awarding_agency_id
-                == self.awarding_agency.id,
+                self.transaction_normalized.awarding_agency_id == self.awarding_agency.id,
                 "leftouter",
             )
             .join(
                 self.awarding_toptier_agency,
-                self.awarding_agency.toptier_agency_id
-                == self.awarding_toptier_agency.toptier_agency_id,
+                self.awarding_agency.toptier_agency_id == self.awarding_toptier_agency.toptier_agency_id,
                 "leftouter",
             )
             .join(
                 self.awarding_subtier_agency,
-                self.awarding_agency.subtier_agency_id
-                == self.awarding_subtier_agency.subtier_agency_id,
+                self.awarding_agency.subtier_agency_id == self.awarding_subtier_agency.subtier_agency_id,
                 "leftouter",
             )
             .join(
                 self.awarding_agency_id,
                 (
-                    (
-                        self.awarding_agency_id.toptier_agency_id
-                        == self.awarding_toptier_agency.toptier_agency_id
-                    )
+                    (self.awarding_agency_id.toptier_agency_id == self.awarding_toptier_agency.toptier_agency_id)
                     & self.awarding_agency_id.toptier_flag
                 ),
                 "leftouter",
@@ -907,29 +876,26 @@ def dataframe(self) -> DataFrame:
             )
             .join(
                 self.funding_toptier_agency,
-                self.funding_agency.funding_toptier_agency_id
-                == self.funding_toptier_agency.toptier_agency_id,
+                self.funding_agency.funding_toptier_agency_id == self.funding_toptier_agency.toptier_agency_id,
                 "leftouter",
             )
             .join(
                 self.funding_subtier_agency,
-                self.funding_agency.funding_subtier_agency_id
-                == self.funding_subtier_agency.subtier_agency_id,
+                self.funding_agency.funding_subtier_agency_id == self.funding_subtier_agency.subtier_agency_id,
                 "leftouter",
             )
             .join(
-                self.funding_agency_id,
-                (
-                    self.funding_agency_id.toptier_agency_id
-                    == self.funding_toptier_agency.funding_toptier_agency_id
-                )
+                # Without marking this dataframe as a broadcast join it will shuffle the partitions such that agencies
+                # are grouped together. This creates heavily skewed data across tasks which run into resource
+                # constraints.
+                sf.broadcast(self.funding_agency_id),
+                (self.funding_agency_id.toptier_agency_id == self.funding_toptier_agency.funding_toptier_agency_id)
                 & (self.funding_agency_id.row_num == 1),
                 "leftouter",
             )
             .join(
                 self.parent_recipient,
-                self.parent_recipient.parent_recipient_hash
-                == self.generated_parent_recipient_hash,
+                self.parent_recipient.parent_recipient_hash == self.generated_parent_recipient_hash,
                 "leftouter",
             )
             .join(
@@ -987,9 +953,7 @@ def dataframe(self) -> DataFrame:
         )
 
 
-def load_transaction_search(
-    spark: SparkSession, destination_database: str, destination_table_name: str
-) -> None:
+def load_transaction_search(spark: SparkSession, destination_database: str, destination_table_name: str) -> None:
     df = TransactionSearch(spark).dataframe
     df.write.saveAsTable(
         f"{destination_database}.{destination_table_name}",
@@ -1001,9 +965,7 @@ def load_transaction_search(
 def load_transaction_search_incremental(
     spark: SparkSession, destination_database: str, destination_table_name: str
 ) -> None:
-    target = DeltaTable.forName(
-        spark, f"{destination_database}.{destination_table_name}"
-    ).alias("t")
+    target = DeltaTable.forName(spark, f"{destination_database}.{destination_table_name}").alias("t")
     source = TransactionSearch(spark).dataframe.alias("s")
     (
         target.merge(
diff --git a/usaspending_api/tests/integration/test_setup_of_spark_dependencies.py b/usaspending_api/tests/integration/test_setup_of_spark_dependencies.py
deleted file mode 100644
index 5b886e2c77..0000000000
--- a/usaspending_api/tests/integration/test_setup_of_spark_dependencies.py
+++ /dev/null
@@ -1,27 +0,0 @@
-"""
-This module should contain only the one test.
-It is to be called by name in a pytest session that is used to prime or prepare a second pytest session where the
-bulk of tests are performed. The intention of this test is to trigger the loading of Spark JAR dependencies to the
-Ivy dependency management cache directory, so subsequent tests do not need to re-download them. This works around a
-collision that happens using pytest-xdist where multiple pytest sessions from multiple workers try to download
-dependencies simultaneously and fail because of race conditions.
-
-NOTE: It should be run in a single-process pytest session (no xdist parallel sessions)!
-"""
-
-from pyspark.sql import SparkSession
-
-
-def test_preload_spark_jars(spark: SparkSession):
-    """See module docstring above.
-
-    Run this test alone, like:
-
-        pytest --no-cov --disable-warnings -r=fEs -vv 'usaspending_api/tests/integration/test_setup_of_spark_dependencies.py::test_preload_spark_jars'
-
-    If running in a Continuous Integration pipeline (or if you want to nuke and pave your own local test DBs,
-    add --create-db to ensure that the test DBs are (re)created from scratch
-    """
-    # The prep work here is done by the spark fixture loading dependencies
-    assert spark.sparkContext._jsc, "Could not get a reference to the Java SparkContext from PySpark SparkSession"
-    assert spark.sparkContext._jsc.sc().listJars().size() > 0
diff --git a/uv.lock b/uv.lock
index 05da9a2881..d34a6f8344 100644
--- a/uv.lock
+++ b/uv.lock
@@ -299,15 +299,15 @@ wheels = [
 
 [[package]]
 name = "delta-spark"
-version = "3.2.0"
+version = "3.3.2"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "importlib-metadata" },
     { name = "pyspark" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/38/06/a64cc4e17fe959cf60dc126bf3283fc9f22fc91f000b7f3f5e465338022d/delta-spark-3.2.0.tar.gz", hash = "sha256:641967828e47c64805f8c746513da80bea24b5f19b069cdcf64561cd3692e11d", size = 22147, upload-time = "2024-05-09T17:26:10.754Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/8e/c9/a4d34fee1dafcad33d4e810ff2ce14a9f6e8e7af9797a9de79ebc3334987/delta_spark-3.3.2.tar.gz", hash = "sha256:f3e89ec4a15a541988f9d6496016a67cf619475e8eeca71da5e073f6ba0da845", size = 23066, upload-time = "2025-05-29T22:25:49.412Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/3c/8e/bb778f5049aa371bf80a7a781b237eb10ef104f8736fe00d25fcfee80c2b/delta_spark-3.2.0-py3-none-any.whl", hash = "sha256:c4ff3fa7218e58a702cb71eb64384b0005c4d6f0bbdd0fe0b38a53564d946e09", size = 21186, upload-time = "2024-05-09T17:26:09.544Z" },
+    { url = "https://files.pythonhosted.org/packages/4b/b7/f484bca84628f9f7a9a16ecfd8b010edbda9fa3f5e3918e967ddd69bf6af/delta_spark-3.3.2-py3-none-any.whl", hash = "sha256:0a34c720b4368f1655e2348e380b5b3baaca0f2b3d519a3198208f00f65e4062", size = 22024, upload-time = "2025-05-29T22:25:48.008Z" },
 ]
 
 [[package]]
@@ -1558,12 +1558,12 @@ wheels = [
 
 [[package]]
 name = "pyspark"
-version = "3.5.0"
+version = "3.5.6"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "py4j" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/92/11/5c0f24537fa68624cb6271d2a8a8982e81bd05f5cb51e9d2d2a89baa0e1a/pyspark-3.5.0.tar.gz", hash = "sha256:d41a9b76bd2aca370a6100d075c029e22ba44c5940927877e9435a3a9c566558", size = 316890381, upload-time = "2023-09-26T20:30:58.621Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/2e/62/36e50d38e5fe158e97cddec983b44f9417b1e205b02320e3c463b5f802fa/pyspark-3.5.6.tar.gz", hash = "sha256:f8b1c4360e41ab398c64904fae08740503bcb6bd389457d659fa6d9f2952cc48", size = 317359167, upload-time = "2025-05-27T08:24:20.82Z" }
 
 [[package]]
 name = "pytest"
@@ -2026,7 +2026,7 @@ requires-dist = [
     { name = "click", marker = "extra == 'dev'", specifier = "==8.1.7" },
     { name = "databricks-sdk", specifier = "==0.44.1" },
     { name = "dataclasses-json", specifier = "==0.6.*" },
-    { name = "delta-spark", marker = "extra == 'spark'", specifier = "==3.2.*" },
+    { name = "delta-spark", marker = "extra == 'spark'", specifier = "==3.3.2" },
     { name = "dj-database-url", specifier = "==2.1.0" },
     { name = "django", specifier = "==5.2.*" },
     { name = "django-cors-headers", specifier = "==4.3.*" },
@@ -2098,7 +2098,7 @@ requires-dist = [
     { name = "pydantic", specifier = "==2.12" },
     { name = "pydantic-settings", specifier = ">=2.13.1" },
     { name = "pyparsing", marker = "extra == 'server'", specifier = "==3.2.1" },
-    { name = "pyspark", marker = "extra == 'spark'", specifier = "==3.5.0" },
+    { name = "pyspark", marker = "extra == 'spark'", specifier = "==3.5.6" },
     { name = "pytest", marker = "extra == 'dev'", specifier = "==8.0.*" },
     { name = "pytest-cov", marker = "extra == 'dev'", specifier = "==4.1.*" },
     { name = "pytest-django", marker = "extra == 'dev'", specifier = "==4.8.*" },