From eeba01faecba4afd6177eceb69be80a2c5060fda Mon Sep 17 00:00:00 2001
From: Seth Stoudenmier <stoudenmier_seth@bah.com>
Date: Fri, 8 May 2026 00:36:49 -0400
Subject: [PATCH 01/48] [DEV-14608] Initial progress using EMR image for
 development

---
 .env.template                                 |   1 +
 .../actions/init-test-environment/action.yaml |  37 +--
 .github/actions/run-pytest/action.yaml        |   7 +-
 ...non-spark-integration-signal-handling.yaml |  12 +-
 .../workflows/test-non-spark-integration.yaml |  13 +-
 ...-spark-integration-load-to-from-delta.yaml |  13 +-
 ...tegration-load-transactions-fabs-fpds.yaml |  13 +-
 ...-integration-load-transactions-lookup.yaml |  13 +-
 .../test-spark-integration-other.yaml         |  13 +-
 .../workflows/test-unit-signal-handling.yaml  |  13 +-
 .github/workflows/test-unit.yaml              |  13 +-
 .gitignore                                    |   1 +
 Makefile                                      |  70 ++---
 development.Dockerfile                        |  92 ++++++
 docker-compose.yml                            | 270 ++++++++++++------
 Dockerfile.duckdb => duckdb.Dockerfile        |   0
 pyproject.toml                                |   4 +-
 spark.Dockerfile                              |  80 ------
 testing.Dockerfile                            |  32 ---
 .../common/helpers/spark_helpers.py           |  70 ++---
 usaspending_api/common/spark/configs.py       |  21 --
 usaspending_api/common/spark/jobs.py          |  15 +-
 uv.lock                                       |  14 +-
 23 files changed, 357 insertions(+), 460 deletions(-)
 create mode 100644 development.Dockerfile
 rename Dockerfile.duckdb => duckdb.Dockerfile (100%)
 delete mode 100644 spark.Dockerfile
 delete mode 100644 testing.Dockerfile

diff --git a/.env.template b/.env.template
index 8438dd955f..fc1bb7103c 100644
--- a/.env.template
+++ b/.env.template
@@ -30,6 +30,7 @@ DJANGO_DEBUG=False
 # ==== [Postgres] ====
 # All values of USASPENDING_DB_* must match what is in DATABASE_URL if BOTH are given
 DATABASE_URL=postgres://usaspending:usaspender@usaspending-db:5432/data_store_api
+DOWNLOAD_DATABASE_URL=postgres://usaspending:usaspender@usaspending-db:5432/data_store_api
 # Change USASPENDING_DB_HOST to host.docker.internal if you are running a local Postgres server on the host machine
 # Otherwise leave as-is, so other docker containerized services will use the Postgres created by Compose.
 USASPENDING_DB_HOST=usaspending-db
diff --git a/.github/actions/init-test-environment/action.yaml b/.github/actions/init-test-environment/action.yaml
index f5b6f2d85d..38b26b454b 100644
--- a/.github/actions/init-test-environment/action.yaml
+++ b/.github/actions/init-test-environment/action.yaml
@@ -30,11 +30,6 @@ runs:
       shell: bash
       run: mkdir -p "$MINIO_DATA_DIR"
 
-    - name: Create directories needed for Spark
-      if: ${{ inputs.is-spark-test == 'true' }}
-      shell: bash
-      run: mkdir -p "$HOME/.ivy2"
-
     - name: Build docker containers for DB, ES, and Minio
       working-directory: ${{ inputs.working-directory }}
       if: ${{ inputs.is-integration-test == 'true' }}
@@ -71,40 +66,10 @@ runs:
         psql $DATABASE_URL -c "SELECT 'GRANT SELECT ON ALL TABLES IN SCHEMA ' || nspname || ' TO readonly' FROM pg_namespace WHERE nspname IN ('raw','int','rpt','temp','public')"
         psql $DATABASE_URL -c "SELECT 'ALTER DEFAULT PRIVILEGES IN SCHEMA ' || nspname || ' GRANT SELECT ON TABLES TO readonly' FROM pg_namespace WHERE nspname IN ('raw','int','rpt','temp','public')"
 
-    - name: Capture SPARK_HOME environment variable
-      if: ${{ inputs.is-spark-test == 'true' }}
-      shell: bash
-      run: echo "SPARK_HOME=$(python -c "import site;print(f'{site.getsitepackages()[0]}/pyspark')")" >> $GITHUB_ENV
-
-    - name: Setting spark log levels to warning
-      if: ${{ inputs.is-spark-test == 'true' }}
-      shell: bash
-      run: |
-        mkdir $SPARK_HOME/conf
-        touch $SPARK_HOME/conf/log4j2.properties
-        echo "
-        appender.console.type = Console
-        appender.console.name = CONSOLE
-        appender.console.layout.type = PatternLayout
-        appender.console.layout.pattern = [%d{yyyy-MM-dd HH:mm:ss.SSS}][%p] - %m%n
-
-        rootLogger.level = WARN
-        rootLogger.appenderRef.0.ref = CONSOLE
-        rootLogger.appenderRef.0.level = WARN
-        " >> $SPARK_HOME/conf/log4j2.properties
-
     - name: Run initial test to trigger DB creation
       if: ${{ inputs.is-integration-test == 'true' }}
       working-directory: ${{ inputs.working-directory }}
       shell: bash
       run: >
-        pytest --create-db --numprocesses logical --no-cov --disable-warnings -r=fEs --verbosity=3 --capture=no --log-cli-level=WARNING --show-capture=log
+        docker compose run --rm usaspending-test --create-db --numprocesses logical --no-cov --disable-warnings -r=fEs --verbosity=3 --capture=no --log-cli-level=WARNING --show-capture=log
         2> /dev/null "usaspending_api/tests/integration/test_setup_of_test_dbs.py::test_trigger_test_db_setup"
-
-    - name: Run initial test to trigger JAR download
-      if: ${{ inputs.is-spark-test == 'true' }}
-      working-directory: ${{ inputs.working-directory }}
-      shell: bash
-      run: >
-        pytest --reuse-db --numprocesses logical --no-cov --disable-warnings -r=fEs --verbosity=3
-        "usaspending_api/tests/integration/test_setup_of_spark_dependencies.py::test_preload_spark_jars"
diff --git a/.github/actions/run-pytest/action.yaml b/.github/actions/run-pytest/action.yaml
index 84d720fc3f..c4f65c0346 100644
--- a/.github/actions/run-pytest/action.yaml
+++ b/.github/actions/run-pytest/action.yaml
@@ -40,14 +40,13 @@ runs:
         echo "MARKER=${{ inputs.marker }}" >> $GITHUB_ENV
         echo "NUM_PROCESSES=${{ inputs.num-processes }}" >> $GITHUB_ENV
 
-    - name: Build pytest command
+    - name: Build pytest args
       working-directory: ${{ inputs.working-directory }}
       id: build-pytest-command
       shell: bash
       run: >
         echo pytest-command=$(python -c 'import os;print(
-        "pytest
-        -r=fEs
+        "-r=fEs
         -x
         --cov=usaspending_api
         --cov-report term
@@ -74,7 +73,7 @@ runs:
     - name: Run tests
       working-directory: ${{ inputs.working-directory }}
       shell: bash
-      run: ${{ steps.build-pytest-command.outputs.pytest-command }}
+      run: docker compose run --rm ${{ steps.build-pytest-command.outputs.pytest-command }}
 
     - name: Upload Artifact
       uses: actions/upload-artifact@v4
diff --git a/.github/workflows/test-non-spark-integration-signal-handling.yaml b/.github/workflows/test-non-spark-integration-signal-handling.yaml
index 6b69ac35ac..fc0f56f54d 100644
--- a/.github/workflows/test-non-spark-integration-signal-handling.yaml
+++ b/.github/workflows/test-non-spark-integration-signal-handling.yaml
@@ -1,17 +1,17 @@
 name: Non-Spark Integration Tests with Signal Handling
 
 env:
-  BROKER_DB_HOST: localhost
+  BROKER_DB_HOST: usaspending-db
   BROKER_DB_PORT: 5432
   BROKER_DB_USER: admin
   BROKER_DB_PASSWORD: root
   BROKER_DB_NAME: data_broker
   DJANGO_SETTINGS_MODULE: usaspending_api.settings
   ES_SCHEME: http
-  ES_HOST: localhost
+  ES_HOST: usaspending-es
   ES_PORT: 9200
-  MINIO_HOST: localhost
-  USASPENDING_DB_HOST: localhost
+  MINIO_HOST: minio
+  USASPENDING_DB_HOST: usaspending-db
   USASPENDING_DB_PORT: 5432
   USASPENDING_DB_USER: usaspending
   USASPENDING_DB_PASSWORD: usaspender
@@ -46,10 +46,6 @@ jobs:
           path: data-act-broker-backend
           ref: ${{ needs.Setup-Broker-Branch.outputs.branch }}
 
-      - name: Init Python Environment
-        uses: ./usaspending-api/.github/actions/init-python-environment
-        with:
-          working-directory: ./usaspending-api
 
       - name: Init Test Environment
         uses: ./usaspending-api/.github/actions/init-test-environment
diff --git a/.github/workflows/test-non-spark-integration.yaml b/.github/workflows/test-non-spark-integration.yaml
index 52a40a221f..0e2517cee5 100644
--- a/.github/workflows/test-non-spark-integration.yaml
+++ b/.github/workflows/test-non-spark-integration.yaml
@@ -1,17 +1,17 @@
 name: Non-Spark Integration Tests
 
 env:
-  BROKER_DB_HOST: localhost
+  BROKER_DB_HOST: usaspending-db
   BROKER_DB_PORT: 5432
   BROKER_DB_USER: admin
   BROKER_DB_PASSWORD: root
   BROKER_DB_NAME: data_broker
   DJANGO_SETTINGS_MODULE: usaspending_api.settings
   ES_SCHEME: http
-  ES_HOST: localhost
+  ES_HOST: usaspending-es
   ES_PORT: 9200
-  MINIO_HOST: localhost
-  USASPENDING_DB_HOST: localhost
+  MINIO_HOST: minio
+  USASPENDING_DB_HOST: usaspending-db
   USASPENDING_DB_PORT: 5432
   USASPENDING_DB_USER: usaspending
   USASPENDING_DB_PASSWORD: usaspender
@@ -45,11 +45,6 @@ jobs:
           path: data-act-broker-backend
           ref: ${{ needs.Setup-Broker-Branch.outputs.branch }}
 
-      - name: Init Python Environment
-        uses: ./usaspending-api/.github/actions/init-python-environment
-        with:
-          working-directory: ./usaspending-api
-
       - name: Init Test Environment
         uses: ./usaspending-api/.github/actions/init-test-environment
         with:
diff --git a/.github/workflows/test-spark-integration-load-to-from-delta.yaml b/.github/workflows/test-spark-integration-load-to-from-delta.yaml
index cf9ab7c1cf..44967060d9 100644
--- a/.github/workflows/test-spark-integration-load-to-from-delta.yaml
+++ b/.github/workflows/test-spark-integration-load-to-from-delta.yaml
@@ -1,17 +1,17 @@
 name: Spark Integration Tests - test_load_to_from_delta.py
 
 env:
-  BROKER_DB_HOST: localhost
+  BROKER_DB_HOST: usaspending-db
   BROKER_DB_PORT: 5432
   BROKER_DB_USER: admin
   BROKER_DB_PASSWORD: root
   BROKER_DB_NAME: data_broker
   DJANGO_SETTINGS_MODULE: usaspending_api.settings
   ES_SCHEME: http
-  ES_HOST: localhost
+  ES_HOST: usaspending-es
   ES_PORT: 9200
-  MINIO_HOST: localhost
-  USASPENDING_DB_HOST: localhost
+  MINIO_HOST: minio
+  USASPENDING_DB_HOST: usaspending-db
   USASPENDING_DB_PORT: 5432
   USASPENDING_DB_USER: usaspending
   USASPENDING_DB_PASSWORD: usaspender
@@ -45,11 +45,6 @@ jobs:
           path: data-act-broker-backend
           ref: ${{ needs.Setup-Broker-Branch.outputs.branch }}
 
-      - name: Init Python Environment
-        uses: ./usaspending-api/.github/actions/init-python-environment
-        with:
-          working-directory: ./usaspending-api
-
       - name: Init Test Environment
         uses: ./usaspending-api/.github/actions/init-test-environment
         with:
diff --git a/.github/workflows/test-spark-integration-load-transactions-fabs-fpds.yaml b/.github/workflows/test-spark-integration-load-transactions-fabs-fpds.yaml
index cfef1cb856..fe33671fd4 100644
--- a/.github/workflows/test-spark-integration-load-transactions-fabs-fpds.yaml
+++ b/.github/workflows/test-spark-integration-load-transactions-fabs-fpds.yaml
@@ -1,17 +1,17 @@
 name: Spark Integration Tests - test_load_transactions_in_delta_fabs_fpds.py
 
 env:
-  BROKER_DB_HOST: localhost
+  BROKER_DB_HOST: usaspending-db
   BROKER_DB_PORT: 5432
   BROKER_DB_USER: admin
   BROKER_DB_PASSWORD: root
   BROKER_DB_NAME: data_broker
   DJANGO_SETTINGS_MODULE: usaspending_api.settings
   ES_SCHEME: http
-  ES_HOST: localhost
+  ES_HOST: usaspending-es
   ES_PORT: 9200
-  MINIO_HOST: localhost
-  USASPENDING_DB_HOST: localhost
+  MINIO_HOST: minio
+  USASPENDING_DB_HOST: usaspending-db
   USASPENDING_DB_PORT: 5432
   USASPENDING_DB_USER: usaspending
   USASPENDING_DB_PASSWORD: usaspender
@@ -45,11 +45,6 @@ jobs:
           path: data-act-broker-backend
           ref: ${{ needs.Setup-Broker-Branch.outputs.branch }}
 
-      - name: Init Python Environment
-        uses: ./usaspending-api/.github/actions/init-python-environment
-        with:
-          working-directory: ./usaspending-api
-
       - name: Init Test Environment
         uses: ./usaspending-api/.github/actions/init-test-environment
         with:
diff --git a/.github/workflows/test-spark-integration-load-transactions-lookup.yaml b/.github/workflows/test-spark-integration-load-transactions-lookup.yaml
index 329c37d612..71c6359840 100644
--- a/.github/workflows/test-spark-integration-load-transactions-lookup.yaml
+++ b/.github/workflows/test-spark-integration-load-transactions-lookup.yaml
@@ -1,17 +1,17 @@
 name: Spark Integration Tests - test_load_transactions_in_delta_lookups.py
 
 env:
-  BROKER_DB_HOST: localhost
+  BROKER_DB_HOST: usaspending-db
   BROKER_DB_PORT: 5432
   BROKER_DB_USER: admin
   BROKER_DB_PASSWORD: root
   BROKER_DB_NAME: data_broker
   DJANGO_SETTINGS_MODULE: usaspending_api.settings
   ES_SCHEME: http
-  ES_HOST: localhost
+  ES_HOST: usaspending-es
   ES_PORT: 9200
-  MINIO_HOST: localhost
-  USASPENDING_DB_HOST: localhost
+  MINIO_HOST: minio
+  USASPENDING_DB_HOST: usaspending-db
   USASPENDING_DB_PORT: 5432
   USASPENDING_DB_USER: usaspending
   USASPENDING_DB_PASSWORD: usaspender
@@ -45,11 +45,6 @@ jobs:
           path: data-act-broker-backend
           ref: ${{ needs.Setup-Broker-Branch.outputs.branch }}
 
-      - name: Init Python Environment
-        uses: ./usaspending-api/.github/actions/init-python-environment
-        with:
-          working-directory: ./usaspending-api
-
       - name: Init Test Environment
         uses: ./usaspending-api/.github/actions/init-test-environment
         with:
diff --git a/.github/workflows/test-spark-integration-other.yaml b/.github/workflows/test-spark-integration-other.yaml
index 9545878495..c780d93a77 100644
--- a/.github/workflows/test-spark-integration-other.yaml
+++ b/.github/workflows/test-spark-integration-other.yaml
@@ -1,17 +1,17 @@
 name: Spark Integration Tests - Other
 
 env:
-  BROKER_DB_HOST: localhost
+  BROKER_DB_HOST: usaspending-db
   BROKER_DB_PORT: 5432
   BROKER_DB_USER: admin
   BROKER_DB_PASSWORD: root
   BROKER_DB_NAME: data_broker
   DJANGO_SETTINGS_MODULE: usaspending_api.settings
   ES_SCHEME: http
-  ES_HOST: localhost
+  ES_HOST: usaspending-es
   ES_PORT: 9200
-  MINIO_HOST: localhost
-  USASPENDING_DB_HOST: localhost
+  MINIO_HOST: minio
+  USASPENDING_DB_HOST: usaspending-db
   USASPENDING_DB_PORT: 5432
   USASPENDING_DB_USER: usaspending
   USASPENDING_DB_PASSWORD: usaspender
@@ -45,11 +45,6 @@ jobs:
           path: data-act-broker-backend
           ref: ${{ needs.Setup-Broker-Branch.outputs.branch }}
 
-      - name: Init Python Environment
-        uses: ./usaspending-api/.github/actions/init-python-environment
-        with:
-          working-directory: ./usaspending-api
-
       - name: Init Test Environment
         uses: ./usaspending-api/.github/actions/init-test-environment
         with:
diff --git a/.github/workflows/test-unit-signal-handling.yaml b/.github/workflows/test-unit-signal-handling.yaml
index 858799e417..ed9722f235 100644
--- a/.github/workflows/test-unit-signal-handling.yaml
+++ b/.github/workflows/test-unit-signal-handling.yaml
@@ -1,17 +1,17 @@
 name: Unit Tests with Signal Handling
 
 env:
-  BROKER_DB_HOST: localhost
+  BROKER_DB_HOST: usaspending-db
   BROKER_DB_PORT: 5432
   BROKER_DB_USER: admin
   BROKER_DB_PASSWORD: root
   BROKER_DB_NAME: data_broker
   DJANGO_SETTINGS_MODULE: usaspending_api.settings
   ES_SCHEME: http
-  ES_HOST: localhost
+  ES_HOST: usaspending-es
   ES_PORT: 9200
-  MINIO_HOST: localhost
-  USASPENDING_DB_HOST: localhost
+  MINIO_HOST: minio
+  USASPENDING_DB_HOST: usaspending-db
   USASPENDING_DB_PORT: 5432
   USASPENDING_DB_USER: usaspending
   USASPENDING_DB_PASSWORD: usaspender
@@ -31,11 +31,6 @@ jobs:
         with:
           path: usaspending-api
 
-      - name: Init Python Environment
-        uses: ./usaspending-api/.github/actions/init-python-environment
-        with:
-          working-directory: ./usaspending-api
-
       - name: Init Test Environment
         uses: ./usaspending-api/.github/actions/init-test-environment
         with:
diff --git a/.github/workflows/test-unit.yaml b/.github/workflows/test-unit.yaml
index bccbd056b7..6718033489 100644
--- a/.github/workflows/test-unit.yaml
+++ b/.github/workflows/test-unit.yaml
@@ -1,17 +1,17 @@
 name: Unit Tests
 
 env:
-  BROKER_DB_HOST: localhost
+  BROKER_DB_HOST: usaspending-db
   BROKER_DB_PORT: 5432
   BROKER_DB_USER: admin
   BROKER_DB_PASSWORD: root
   BROKER_DB_NAME: data_broker
   DJANGO_SETTINGS_MODULE: usaspending_api.settings
   ES_SCHEME: http
-  ES_HOST: localhost
+  ES_HOST: usaspending-es
   ES_PORT: 9200
-  MINIO_HOST: localhost
-  USASPENDING_DB_HOST: localhost
+  MINIO_HOST: minio
+  USASPENDING_DB_HOST: usaspending-db
   USASPENDING_DB_PORT: 5432
   USASPENDING_DB_USER: usaspending
   USASPENDING_DB_PASSWORD: usaspender
@@ -31,11 +31,6 @@ jobs:
         with:
           path: usaspending-api
 
-      - name: Init Python Environment
-        uses: ./usaspending-api/.github/actions/init-python-environment
-        with:
-          working-directory: ./usaspending-api
-
       - name: Init Test Environment
         uses: ./usaspending-api/.github/actions/init-test-environment
         with:
diff --git a/.gitignore b/.gitignore
index 73c050cd42..3631c65eb0 100644
--- a/.gitignore
+++ b/.gitignore
@@ -12,6 +12,7 @@ chunked_matviews/**
 csv_downloads/**
 data/**
 derby.log
+dredd-results.txt
 log/**
 logs/**
 matviews/**
diff --git a/Makefile b/Makefile
index 29f0141ed1..229ff8d574 100644
--- a/Makefile
+++ b/Makefile
@@ -28,7 +28,8 @@ ENV_CODE ?= lcl
 PYTHON_VERSION ?= 3.10.12
 venv_name := usaspending-api
 docker_compose_file := docker-compose.yml
-dockerfile_for_spark := Dockerfile.spark
+dockerfile_for_backend := Dockerfile
+dockerfile_for_development := development.Dockerfile
 # Root directories under which python (namespace) packages start, for all python code in this project
 src_root_paths = "."
 
@@ -53,13 +54,10 @@ printvars: ## Print the Environment variables present in calls to make, plus var
 	@printenv
 
 uv-sync:
-	uv sync --no-cache --extra dev --locked
-
-.ivy2: ## Ensure user has a ~/.ivy2 dir, which will be bound to in a docker container volume to save on dependency downloads
-	@mkdir -p ~/.ivy2
+	uv sync --no-cache --extra dev --extra spark --locked
 
 .PHONY: local-dev-setup
-local-dev-setup: uv-sync check-dependencies .ivy2 ## Setup virtual environment and pip dependencies, then check version info
+local-dev-setup: uv-sync check-dependencies ## Setup virtual environment and pip dependencies, then check version info
 
 .PHONY: check-dependencies
 check-dependencies: ## Prints out the versions of dependencies in use
@@ -68,12 +66,6 @@ check-dependencies: ## Prints out the versions of dependencies in use
 	@uv run python3 --version
 	@printf "\n==== [PIP PACKAGE VERSIONS] ====\n\n"
 	@uv pip list
-	@printf "\n==== [SPARK VERSION] ====\n\n"
-	@uv run --no-project pyspark --version
-	@printf "\n==== [HADOOP VERSION] ====\n\n"
-	@uv run python3 -c "from pyspark.sql import SparkSession; \
-spark = spark = SparkSession.builder.getOrCreate(); \
-print('Hadoop ' + spark.sparkContext._gateway.jvm.org.apache.hadoop.util.VersionInfo.getVersion());"
 
 .PHONY: env-code
 env-code:  ## Print the value of ENV_CODE environment variable
@@ -82,19 +74,15 @@ env-code:  ## Print the value of ENV_CODE environment variable
 .PHONY: test-dbs
 createdb :=  #unset it
 test-dbs:  ## Trigger the setup of multiple test DBs that can be reused with pytest --numprocesses. Add createdb=true to force (re-)creation of Test DBs rather than reuse.
-	uv run pytest ${if ${createdb},--create-db,} --reuse-db --numprocesses=auto --no-cov --disable-warnings -rP -vvv --capture=no --log-cli-level=WARNING --show-capture=log 2> /dev/null 'usaspending_api/tests/integration/test_setup_of_test_dbs.py::test_trigger_test_db_setup'
-
-.PHONY: test-spark-deps
-test-spark-deps:  ## Trigger a singular test in one pytest session that does nothing but cause Maven dependencies to be downloaded and cached through Ivy; reduces contention when parallel spark builds need the depdencies
-	uv run pytest --no-cov --disable-warnings -r=fEs --verbosity=3 'usaspending_api/tests/integration/test_setup_of_spark_dependencies.py::test_preload_spark_jars'
+	docker compose run --rm usaspending-test ${if ${createdb},--create-db,} --reuse-db --numprocesses=auto --no-cov --disable-warnings -rP -vvv --capture=no --log-cli-level=WARNING --show-capture=log 2> /dev/null 'usaspending_api/tests/integration/test_setup_of_test_dbs.py::test_trigger_test_db_setup'
 
 .PHONY: tests
 tests: local-dev-setup test-dbs test-spark-deps ## Run automated unit/integration tests. Configured for useful logging. add args="..." to append additional pytest args
-	uv run pytest --failed-first --reuse-db --numprocesses=auto --dist=worksteal -rP -vv --capture=no --show-capture=log 2> /dev/null ${args} 'usaspending_api/'
+	docker compose run --rm usaspending-test --failed-first --reuse-db --numprocesses=auto --dist=worksteal -rP -vv --capture=no --show-capture=log 2> /dev/null ${args} 'usaspending_api/'
 
 .PHONY: tests-failed
 tests-failed: local-dev-setup test-dbs test-spark-deps ## Re-run only automated unit/integration tests that failed on the previous run. Configured for verbose logging to get more detail on failures. logging. add args="..." to append additional pytest args
-	uv run pytest --last-failed --reuse-db --numprocesses=auto --dist=worksteal -rP -vvv ${args}
+	docker compose run --rm usaspending-test --last-failed --reuse-db --numprocesses=auto --dist=worksteal -rP -vvv ${args}
 
 .PHONY: confirm-clean-all
 no-prompt := 'false'
@@ -159,11 +147,10 @@ docker-compose-down: ## Run docker compose down to bring down services listed in
 	# NOTE: [See NOTE in docker compose rule about .env file]
 	docker compose --project-directory . --file ${docker_compose_file} down ${args}
 
-.PHONY: docker-build-spark
-docker-build-spark: ## Run docker build to build a base container image for spark, hadoop, and python installed
+.PHONY: docker-build-development
+docker-build-development: ## Run docker build to build a base container image for development
 	# NOTE: [See NOTE in above docker compose rule about .env file]
-	echo "docker build --tag spark-base --build-arg PROJECT_LOG_DIR=${PROJECT_LOG_DIR} ${args} --file ${dockerfile_for_spark} $$(dirname ${dockerfile_for_spark})"
-	docker build --tag spark-base --build-arg PROJECT_LOG_DIR=${PROJECT_LOG_DIR} ${args} --file ${dockerfile_for_spark} $$(dirname ${dockerfile_for_spark})
+	docker build --tag usaspending-development --build-arg PROJECT_LOG_DIR=${PROJECT_LOG_DIR} ${args} --file ${dockerfile_for_development} $$(dirname ${dockerfile_for_development})
 
 .PHONY: docker-compose-build
 docker-compose-build: ## Ensure ALL services in the docker-compose.yaml file have an image built for them according to their build: key
@@ -179,48 +166,23 @@ docker-compose-build: ## Ensure ALL services in the docker-compose.yaml file hav
 	echo "docker compose --profile usaspending --project-directory . --file ${docker_compose_file} build --build-arg PROJECT_LOG_DIR=${PROJECT_LOG_DIR} ${args}"
 	docker compose --profile usaspending --project-directory . --file ${docker_compose_file} build --build-arg PROJECT_LOG_DIR=${PROJECT_LOG_DIR} ${args}
 
-.PHONY: docker-compose-build-spark
-docker-compose-build-spark: ## See: docker-compose-build rule. This builds just the subset of spark services.
+.PHONY: docker-compose-build-development
+docker-compose-build-development: ## See: docker-compose-build rule. This builds just the subset of spark services.
 	# NOTE: [See NOTE in above docker compose rule about .env file]=
 	echo "docker compose --profile spark --project-directory . --file ${docker_compose_file} build --build-arg PROJECT_LOG_DIR=${PROJECT_LOG_DIR} ${args}"
 	docker compose --profile spark --project-directory . --file ${docker_compose_file} build --build-arg PROJECT_LOG_DIR=${PROJECT_LOG_DIR} ${args}
 
 .PHONY: docker-compose-spark-submit
-docker-compose-spark-submit: ## Run spark-submit from within local docker containerized infrastructure (which must be running first). Set params with django_command="..."
-	docker compose --profile=spark --project-directory . --file ${docker_compose_file} run \
-		-e MINIO_HOST=minio \
+spark-submit: ## Run spark-submit from within local docker containerized infrastructure (which must be running first). Set params with django_command="..."
+	docker compose --profile=spark --project-directory . --file ${docker_compose_file} run --rm \
 		-e COMPONENT_NAME='${django_command}${python_script}' \
-		-e DATABASE_URL=${DATABASE_URL} \
-	spark-submit \
-	--driver-memory "2g" \
-	--packages org.postgresql:postgresql:42.2.23,io.delta:delta-spark_2.12:3.1.0,org.apache.hadoop:hadoop-aws:3.3.4,org.apache.spark:spark-hive_2.12:3.5.0 \
-	${if ${python_script}, \
-		${python_script}, \
-		/project/manage.py ${django_command} \
-	}
-
-.PHONY: localhost-spark-submit
-localhost-spark-submit: ## Run spark-submit from with localhost as the driver and worker (single node). Set params with django_command="..."
-	SPARK_LOCAL_IP=127.0.0.1 \
 	spark-submit \
 	--driver-memory "2g" \
-	--packages org.postgresql:postgresql:42.2.23,io.delta:delta-spark_2.12:3.1.0,org.apache.hadoop:hadoop-aws:3.3.4,org.apache.spark:spark-hive_2.12:3.5.0 \
 	${if ${python_script}, \
 		${python_script}, \
-		manage.py ${django_command} \
+		/dockermount/manage.py ${django_command} \
 	}
 
 .PHONY: pyspark-shell
 pyspark-shell: ## Launch a local pyspark REPL shell with all of the packages and spark config pre-set
-	SPARK_LOCAL_IP=127.0.0.1 pyspark \
-	--packages org.postgresql:postgresql:42.2.23,io.delta:delta-spark_2.12:3.1.0,org.apache.hadoop:hadoop-aws:3.3.4 \
-	--conf spark.sql.extensions=io.delta.sql.DeltaSparkSessionExtension \
-	--conf spark.sql.catalog.spark_catalog=org.apache.spark.sql.delta.catalog.DeltaCatalog \
-	--conf spark.hadoop.fs.s3a.endpoint=localhost:${MINIO_PORT} \
-	--conf spark.hadoop.fs.s3a.access.key=usaspending \
-	--conf spark.hadoop.fs.s3a.secret.key=usaspender \
-	--conf spark.hadoop.fs.s3a.connection.ssl.enabled=false \
-	--conf spark.hadoop.fs.s3a.path.style.access=true \
-	--conf spark.sql.catalogImplementation=hive \
-	--conf spark.sql.warehouse.dir='$(PWD)/spark-warehouse' \
-	--conf spark.hadoop.javax.jdo.option.ConnectionURL='jdbc:derby:;databaseName=$(PWD)/spark-warehouse/metastore_db;create=true'
+	docker compose --profile=spark --project-directory . --file ${docker_compose_file} run --rm spark-shell
diff --git a/development.Dockerfile b/development.Dockerfile
new file mode 100644
index 0000000000..cc49044f28
--- /dev/null
+++ b/development.Dockerfile
@@ -0,0 +1,92 @@
+# Dockerfile to support local developement and tests
+
+FROM public.ecr.aws/emr-serverless/spark/emr-7.12.0:latest
+
+USER root
+
+# Copy UV from uv image
+COPY --from=ghcr.io/astral-sh/uv:0.7.19 /uv /uvx /bin/
+
+# Install dependencies
+RUN dnf update \
+    && dnf install -y \
+        git \
+        libffi-devel \
+        libpq-devel \
+        nodejs \
+        npm \
+        sqlite-devel \
+        wget \
+        zlib-devel \
+    && dnf clean all
+
+# Install python
+ARG PYTHON_VERSION=3.10.12
+RUN wget -q https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz && \
+    tar -xf Python-${PYTHON_VERSION}.tgz && \
+    cd Python-${PYTHON_VERSION} && \
+    ./configure \
+        --prefix=/usr/local \
+        --enable-optimizations \
+        --with-lto \
+        --enable-shared \
+        LDFLAGS="-Wl,-rpath /usr/local/lib" && \
+    make -j"$(nproc)" && \
+    make altinstall && \
+    cd / && rm -rf Python-${PYTHON_VERSION} Python-${PYTHON_VERSION}.tgz
+
+# Symlinks so `python3` and `pip3` resolve to correct version
+RUN ln -sf /usr/local/bin/python3.10 /usr/local/bin/python3  \
+    && ln -sf /usr/local/bin/pip3.10    /usr/local/bin/pip3 \
+    && ln -sf /usr/local/bin/python3.10 /usr/local/bin/python  \
+    && ln -sf /usr/local/bin/pip3.10    /usr/local/bin/pip
+
+# Download the Postgres JAR
+RUN wget -P /usr/lib/spark/jars/ https://jdbc.postgresql.org/download/postgresql-42.7.5.jar
+
+# Install Dredd
+RUN npm install --global dredd@13.1.2
+
+WORKDIR /dockermount
+
+##### The following ENV vars are optimizations from https://github.com/astral-sh/uv-docker-example/blob/main/Dockerfile
+##### and https://docs.astral.sh/uv/guides/integration/docker/#optimizations
+# Enable bytecode compilation
+ENV UV_COMPILE_BYTECODE=1
+
+# Copy from the cache instead of linking since it's a mounted volume
+ENV UV_LINK_MODE=copy
+
+# Use the system Python environment since the container is already isolated
+ENV UV_PROJECT_ENVIRONMENT=/usr/local
+ENV UV_SYSTEM_PYTHON=1
+
+# Copy the project into the image
+COPY . /dockermount
+
+# Install dev dependencies
+RUN --mount=type=cache,target=/root/.cache/uv \
+    --mount=type=bind,source=uv.lock,target=uv.lock \
+    --mount=type=bind,source=pyproject.toml,target=pyproject.toml \
+    uv sync --extra dev --extra spark --locked --no-install-project
+
+# Set Python 3.10.12 as the default Python for PySpark
+ENV PYSPARK_PYTHON=/usr/local/bin/python3.10
+ENV PYSPARK_DRIVER_PYTHON=/usr/local/bin/python3.10
+
+# Limit Spark logging to WARN level
+RUN cat <<'EOF' | sed 's/^[[:space:]]*//' >> $SPARK_HOME/conf/log4j2.properties
+	appender.console.type = Console
+    appender.console.name = CONSOLE
+    appender.console.layout.type = PatternLayout
+    appender.console.layout.pattern = [%d{yyyy-MM-dd HH:mm:ss.SSS}][%p] - %m%n
+    rootLogger.level = WARN
+    rootLogger.appenderRef.0.ref = CONSOLE
+    rootLogger.appenderRef.0.level = WARN
+EOF
+
+# Set default values for Spark with local development
+RUN echo "spark.authenticate false" >> $SPARK_HOME/conf/spark-defaults.conf
+
+# Ensure Python STDOUT gets sent to container logs
+ENV PYTHONUNBUFFERED=1
diff --git a/docker-compose.yml b/docker-compose.yml
index 2a8959ad53..48eb39b033 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -5,6 +5,43 @@ volumes:
   local_es_data:
     driver: local
 
+# There are populated by .env when using docker compose
+x-application-variables: &application-vars
+  BROKER_DB: ${BROKER_DB}
+  DB_SOURCE: ${DATABASE_URL}
+  DB_R1: ${DATABASE_URL}
+  DJANGO_DEBUG: ${DJANGO_DEBUG}
+  DOWNLOAD_DATABASE_URL: ${DOWNLOAD_DATABASE_URL}
+  ES_HOSTNAME: ${ES_HOSTNAME}
+  IS_LOCAL: ${IS_LOCAL:-true}
+  MINIO_HOST: ${MINIO_HOST}
+  MINIO_PORT: ${MINIO_PORT}
+  OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: ${OTEL_EXPORTER_OTLP_TRACES_ENDPOINT}
+  TOGGLE_OTEL_CONSOLE_LOGGING: ${TOGGLE_OTEL_CONSOLE_LOGGING}
+
+x-spark-submit-variables: &spark-submit-vars
+  <<: *application-vars
+  AWS_ACCESS_KEY_ID: ${MINIO_ACCESS_KEY}
+  AWS_ENDPOINT_URL: ${MINIO_HOSTNAME}
+  AWS_SECRET_ACCESS_KEY: ${MINIO_SECRET_KEY}
+  PYTHONPATH: "/dockermount"
+  #SPARK_HIVE_METASTORE_DB_TYPE: derby
+  #SPARK_HIVE_METASTORE_DB_PATH: /spark-warehouse
+  # Needed for the Spark containers to interact
+  SPARK_MASTER_HOST: ${SPARK_MASTER_HOST:-spark-master}
+  SPARK_MASTER_PORT: ${SPARK_MASTER_PORT:-7077}
+  SPARK_WORKER_WEBUI_PORT: ${SPARK_WORKER_WEBUI_PORT:-4041}
+  # This env var needs to be picked up as the config for the spark.sql.warehouse.dir spark conf setting when
+  # SparkSessions are created inside a spark-submitted job
+  # (i.e., target where host warehouse dir is bound in volume config)
+  SPARK_SQL_WAREHOUSE_DIR: /spark-warehouse
+  # This env var needs to be picked up as the path part of the config for the
+  # spark.hadoop.javax.jdo.option.ConnectionURL spark conf setting when SparkSessions are created inside a
+  # spark-submitted job
+  # (i.e., a metastore_db sub dir of the target where host warehouse dir is bound in volume config.)
+  HIVE_METASTORE_DERBY_DB_DIR: /spark-warehouse/metastore_db
+
+
 services:
 
   usaspending-db:
@@ -15,6 +52,11 @@ services:
       - test
       - ci
     image: postgres:13.8-alpine
+    healthcheck:
+      test: [ "CMD-SHELL", "pg_isready -U postgres" ]
+      interval: 5s
+      timeout: 5s
+      retries: 5
     shm_size: 1g
     container_name: usaspending-db
     volumes:
@@ -31,7 +73,7 @@ services:
   usaspending-manage:
     profiles:
       - manage  # must pass --profile manage to docker compose for this to come up, or use docker compose run
-    image: usaspending-backend  # when an image by this name is not found in the local repo, and it is forced to build, it will use this as the tag
+    image: usaspending-development  # when an image by this name is not found in the local repo, and it is forced to build, it will use this as the tag
     build: .
     container_name: usaspending-manage
     volumes:
@@ -39,23 +81,20 @@ services:
     depends_on:
       - usaspending-db
     # For an interactive shell, override this command with: docker compose run --rm usaspending-manage python3 -u manage.py shell
-    command: python3 -u manage.py help
+    entrypoint: ["uv", "run", "manage.py"]
     environment:
-      DJANGO_DEBUG: ${DJANGO_DEBUG}
-      DATABASE_URL: postgres://${USASPENDING_DB_USER}:${USASPENDING_DB_PASSWORD}@${USASPENDING_DB_HOST}:${USASPENDING_DB_PORT}/data_store_api
-      ES_HOSTNAME: ${ES_HOSTNAME}
-      BROKER_DB: postgresql://${BROKER_DB_USER}:${BROKER_DB_PASSWORD}@${BROKER_DB_HOST}:${BROKER_DB_PORT}/data_broker
+      <<: *application-vars
 
   usaspending-test:
     profiles:
       - test  # must pass --profile test to docker compose for this to come up, or use docker compose run
-    image: usaspending-testing  # when an image by this name is not found in the local repo, and it is forced to build, it will use this as the tag
+    image: usaspending-development  # when an image by this name is not found in the local repo, and it is forced to build, it will use this as the tag
     build:
       context: .
-      dockerfile: testing.Dockerfile
+      dockerfile: development.Dockerfile
     container_name: usaspending-test
     volumes:
-      - ${PROJECT_DIRECTORY:-.}:/usaspending-api
+      - ${PROJECT_DIRECTORY:-.}:/dockermount
       # Required to interact with host's docker daemon from within this running container,
       # to spin up the data-act-broker-init-test-db container used for broker integration tests (see: conftest.broker_db_setup)
       - /var/run/docker.sock:/var/run/docker.sock
@@ -63,24 +102,50 @@ services:
       - minio
       - usaspending-db
       - usaspending-es
+    # , "--reuse-db", "--no-cov", "--disable-warnings", "--reuse-db", "--numprocesses", "auto"
+    entrypoint: ["uv", "run", "pytest"]
     command: make tests
     environment:
-      DATABASE_URL: postgres://${USASPENDING_DB_USER}:${USASPENDING_DB_PASSWORD}@${USASPENDING_DB_HOST}:${USASPENDING_DB_PORT}/data_store_api
-      ES_HOST: ${ES_HOST}
-      ES_HOSTNAME: ${ES_HOSTNAME}
-      BROKER_DB: postgresql://${BROKER_DB_USER}:${BROKER_DB_PASSWORD}@${BROKER_DB_HOST}:${BROKER_DB_PORT}/data_broker
-      MINIO_HOST: ${MINIO_HOST}
-      DOWNLOAD_DATABASE_URL: postgres://${USASPENDING_DB_USER}:${USASPENDING_DB_PASSWORD}@${USASPENDING_DB_HOST}:${USASPENDING_DB_PORT}/data_store_api
+      <<: *spark-submit-vars
       # Location in host machine where broker src code root can be found
       DATA_BROKER_SRC_PATH: "${PWD}/../data-act-broker-backend"
       # Tests will retrieve the root directory from the path and use it to assume the docker network name
       PROJECT_SRC_PATH: "${PWD}"
 
+  usaspending-style-checks:
+    profiles:
+      - ci  # must pass --profile ci to docker compose for this to come up, or use docker compose run
+    image: usaspending-development
+    build:
+      context: .
+      dockerfile: development.Dockerfile
+    container_name: usaspending-style-checks
+    volumes:
+      - .:/dockermount
+    entrypoint: []  # override entrypoint
+    command:
+      - sh
+      - -c
+      - |
+        git config --global --add safe.directory /dockermount
+        printf -- "==============\nFormatting code:\n"
+        git diff --name-only --diff-filter=ACMR | grep '\.py$' | xargs uv run ruff format
+        printf -- "==============\nChecking code syntax and quality:\n"
+        git diff --name-only --diff-filter=ACMR | grep '\.py$' | xargs uv run ruff check --fix || exit 1
+        printf -- "==============\nChecking API documentation files:\n"
+        python3 manage.py check_for_endpoint_documentation
+        printf -- "==============\nChecking API contracts:\n"
+        dredd > dredd-results.txt && echo '! grep -E "^[warn:|error:]" dredd-results.txt' | bash
+    environment:
+      <<: *application-vars
+
   usaspending-ci:
     profiles:
       - ci  # must pass --profile ci to docker compose for this to come up, or use docker compose run
-    image: usaspending-backend  # when an image by this name is not found in the local repo, and it is forced to build, it will use this as the tag
-    build: .
+    image: usaspending-development  # when an image by this name is not found in the local repo, and it is forced to build, it will use this as the tag
+    build:
+      context: .
+      dockerfile: development.Dockerfile
     container_name: usaspending-ci
     volumes:
       - .:/dockermount
@@ -88,26 +153,34 @@ services:
       # to spin up the data-act-broker-init-test-db container used for broker integration tests (see: conftest.broker_db_setup)
       - /var/run/docker.sock:/var/run/docker.sock
     depends_on:
-      - usaspending-db
-      - usaspending-es
+      usaspending-db:
+        condition: service_healthy
+      usaspending-es:
+        condition: service_healthy
+      usaspending-style-checks:
+        condition: service_completed_successfully
     command:
       - sh
       - -c
       - |
-        printf "==============\nChecking code format:\n"
-        black --check --diff .
-        printf -- "-------\nChecking code syntax:\n"
-        flake8 && echo "Successfully passed"
-        printf -- "-------\nChecking API documentation files:\n"
-        python3 manage.py check_for_endpoint_documentation
+        printf -- "-------\nRunning non-spark integration tests:\n"
+        pytest --override-ini=python_files="**/tests/integration/*" -m "(not signal_handling and not spark)" --no-cov --disable-warnings --reuse-db --numprocesses logical
+        printf -- "-------\nRunning non-spark integration tests with signal handling:\n"
+        pytest --override-ini=python_files="**/tests/integration/*" -m "(signal_handling and not spark)" --no-cov --disable-warnings --reuse-db --numprocesses 0
+        printf -- "-------\nRunning spark integration tests -- load to / from delta:\n"
+        pytest --override-ini=python_files="test_*.py *_test.py" -m "(spark)" -k "test_load_to_from_delta.py" --no-cov --disable-warnings --reuse-db --numprocesses 2
+        printf -- "-------\nRunning spark integration tests -- load transaction fabs / fpds:\n"
+        pytest --override-ini=python_files="test_*.py *_test.py" -m "(spark)" -k "test_load_transactions_in_delta_fabs_fpds.py" --no-cov --disable-warnings --reuse-db --numprocesses 0
+        printf -- "-------\nRunning spark integration tests -- load transaction lookup:\n"
+        pytest --override-ini=python_files="test_*.py *_test.py" -m "(spark)" -k "test_load_transactions_in_delta_lookups.py" --no-cov --disable-warnings --reuse-db --numprocesses 0
+        printf -- "-------\nRunning spark integration tests -- other:\n"
+        pytest --override-ini=python_files="test_*.py *_test.py" -m "(spark)" -k "(not test_load_to_from_delta.py and not test_load_transactions_in_delta_lookups.py and not test_load_transactions_in_delta_fabs_fpds.py)" --no-cov --disable-warnings --reuse-db --numprocesses logical
         printf -- "-------\nRunning unit tests:\n"
-        pytest --durations 50 --ignore-glob='**/tests/integration/*' --cov=usaspending_api --cov-report= --reuse-db -rsx
-        printf -- "-------\nRunning integration tests:\n"
-        pytest --durations 50 --override-ini=python_files='**/tests/integration/*' --cov=usaspending_api --cov-append --cov-report term --cov-report xml:coverage.xml --reuse-db -rsx
+        pytest --override-ini=python_files="test_*.py *_test.py" -m "(not signal_handling)" --ignore-glob="**/tests/integration/*" --no-cov --disable-warnings --reuse-db --numprocesses logical
+        printf -- "-------\nRunning unit tests with signal handling:\n"
+        pytest --override-ini=python_files="test_*.py *_test.py" -m "(signal_handling)" --ignore-glob="**/tests/integration/*" --no-cov --disable-warnings --reuse-db --numprocesses 0
     environment:
-      DATABASE_URL: postgres://${USASPENDING_DB_USER}:${USASPENDING_DB_PASSWORD}@${USASPENDING_DB_HOST}:${USASPENDING_DB_PORT}/data_store_api
-      ES_HOSTNAME: ${ES_HOSTNAME}
-      BROKER_DB: postgresql://${BROKER_DB_USER}:${BROKER_DB_PASSWORD}@${BROKER_DB_HOST}:${BROKER_DB_PORT}/data_broker
+      <<: *application-vars
       # Location in host machine where broker src code root can be found
       DATA_BROKER_SRC_PATH: "${PWD}/../data-act-broker-backend"
 
@@ -128,15 +201,7 @@ services:
     # Must wait on postgres db to be up (~9s)
     command: /bin/sh -c "sleep 9s; python3 -u manage.py runserver --verbosity 2 0.0.0.0:8000"
     environment:
-      IS_LOCAL: ${IS_LOCAL}
-      DJANGO_DEBUG: ${DJANGO_DEBUG}
-      RUN_LOCAL_DOWNLOAD_IN_PROCESS: "False"
-      DB_SOURCE: postgres://${USASPENDING_DB_USER}:${USASPENDING_DB_PASSWORD}@${USASPENDING_DB_HOST}:${USASPENDING_DB_PORT}/data_store_api
-      DB_R1: postgres://${USASPENDING_DB_USER}:${USASPENDING_DB_PASSWORD}@${USASPENDING_DB_HOST}:${USASPENDING_DB_PORT}/data_store_api
-      DOWNLOAD_DATABASE_URL: postgres://${USASPENDING_DB_USER}:${USASPENDING_DB_PASSWORD}@${USASPENDING_DB_HOST}:${USASPENDING_DB_PORT}/data_store_api
-      ES_HOSTNAME: ${ES_HOSTNAME}
-      TOGGLE_OTEL_CONSOLE_LOGGING: ${TOGGLE_OTEL_CONSOLE_LOGGING}
-      OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: ${OTEL_EXPORTER_OTLP_TRACES_ENDPOINT}
+      <<: *application-vars
 
   usaspending-bulk-download:
     profiles:
@@ -154,13 +219,7 @@ services:
       - /var/run/docker.sock:/var/run/docker.sock
     command: python3 manage.py download_sqs_worker
     environment:
-      IS_LOCAL: ${IS_LOCAL}
-      DJANGO_DEBUG: ${DJANGO_DEBUG}
-      DATABASE_URL: postgres://${USASPENDING_DB_USER}:${USASPENDING_DB_PASSWORD}@${USASPENDING_DB_HOST}:${USASPENDING_DB_PORT}/data_store_api
-      DOWNLOAD_DATABASE_URL: postgres://${USASPENDING_DB_USER}:${USASPENDING_DB_PASSWORD}@${USASPENDING_DB_HOST}:${USASPENDING_DB_PORT}/data_store_api
-      ES_HOSTNAME: ${ES_HOSTNAME}
-      TOGGLE_OTEL_CONSOLE_LOGGING: ${TOGGLE_OTEL_CONSOLE_LOGGING}
-      OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: ${OTEL_EXPORTER_OTLP_TRACES_ENDPOINT}
+      <<: *application-vars
 
   usaspending-es:
     profiles:
@@ -168,6 +227,12 @@ services:
       - test
       - ci
     image: opensearchproject/opensearch:2.9.0
+    healthcheck:
+      test: [ "CMD-SHELL", "curl -f -k http://localhost:9200/_cluster/health || exit 1" ]
+      interval: 10s
+      timeout: 5s
+      retries: 10
+      start_period: 60s
     container_name: usaspending-es
     environment:
       - node.name=usaspending-es
@@ -202,6 +267,8 @@ services:
       - usaspending  # must pass --profile usaspending to docker compose for this to come up
     image: opensearchproject/opensearch-dashboards:2.9.0
     container_name: usaspending-kibana-es
+    depends_on:
+      - usaspending-es
     # ELASTICSEARCH_HOSTS should match the port for "usaspending-es"; value will need to be updated if using Windows
     environment:
       - OPENSEARCH_HOSTS="http://usaspending-es:9200"
@@ -253,18 +320,17 @@ services:
     profiles:
       - spark  # must pass --profile spark to docker compose for this to come up
       - test
-    image: spark-base  # when an image by this name is not found in the local repo, and it is forced to build, it will use this as the tag
+    image: usaspending-development
     # build context path needs to be relative to project root, from where docker compose will be run
     build:
       context: .
-      dockerfile: spark.Dockerfile
+      dockerfile: development.Dockerfile
       args:
         PROJECT_LOG_DIR: ${PROJECT_LOG_DIR}
     container_name: spark-master
     environment:
-      SPARK_MASTER_HOST: ${SPARK_MASTER_HOST:-spark-master}
-      SPARK_MASTER_PORT: ${SPARK_MASTER_PORT:-7077}
-      SPARK_MASTER_WEBUI_PORT: ${SPARK_MASTER_WEBUI_PORT:-4040}
+      <<: *spark-submit-vars
+    entrypoint: []  # override the entrypoint
     command: >
       /bin/sh -c "
         $${SPARK_HOME}/bin/spark-class org.apache.spark.deploy.master.Master \
@@ -276,60 +342,57 @@ services:
     volumes:
       - type: bind
         source: .
-        target: /project
+        target: /dockermount
         read_only: false
 
   spark-worker:
     profiles:
       - spark  # must pass --profile spark to docker compose for this to come up
       - test
-    image: spark-base  # when an image by this name is not found in the local repo, and it is forced to build, it will use this as the tag
+    image: usaspending-development
     # build context path needs to be relative to project root, from where docker compose will be run
     build:
       context: .
-      dockerfile: spark.Dockerfile
+      dockerfile: development.Dockerfile
       args:
         PROJECT_LOG_DIR: ${PROJECT_LOG_DIR}
     container_name: spark-worker
     depends_on:
       - spark-master
     environment:
-      SPARK_MASTER_HOST: ${SPARK_MASTER_HOST:-spark-master}
-      SPARK_MASTER_PORT: ${SPARK_MASTER_PORT:-7077}
-      SPARK_WORKER_WEBUI_PORT: ${SPARK_WORKER_WEBUI_PORT:-4041}
-      PYTHONPATH: /project
-      DJANGO_SETTINGS_MODULE: "usaspending_api.settings"
-      DATABASE_URL: postgres://${USASPENDING_DB_USER}:${USASPENDING_DB_PASSWORD}@${USASPENDING_DB_HOST}:${USASPENDING_DB_PORT}/data_store_api
+      <<: *spark-submit-vars
+    entrypoint: []  # override the entrypoint
     command: /bin/sh -c "$${SPARK_HOME}/bin/spark-class org.apache.spark.deploy.worker.Worker --webui-port $${SPARK_WORKER_WEBUI_PORT} spark://$${SPARK_MASTER_HOST}:$${SPARK_MASTER_PORT}"
     ports:
       - ${SPARK_WORKER_WEBUI_PORT:-4041}:4041
     volumes:
       - type: bind
         source: .
-        target: /project
+        target: /dockermount
         read_only: false
 
   spark-history-server:
     profiles:
       - spark  # must pass --profile spark to docker compose for this to come up
       - test
-    image: spark-base  # when an image by this name is not found in the local repo, and it is forced to build, it will use this as the tag
+    image: usaspending-development
     # build context path needs to be relative to project root, from where docker compose will be run
     build:
       context: .
-      dockerfile: spark.Dockerfile
+      dockerfile: development.Dockerfile
       args:
         PROJECT_LOG_DIR: ${PROJECT_LOG_DIR}
     container_name: spark-history-server
     environment:
       SPARK_HISTORY_SERVER_PORT: ${SPARK_HISTORY_SERVER_PORT:-18080}
-    command: /bin/sh -c "$${SPARK_HOME}/bin/spark-class org.apache.spark.deploy.history.HistoryServer"
+    entrypoint: []  # override the entrypoint
+    command: /usr/bin/sh -c "$${SPARK_HOME}/bin/spark-class org.apache.spark.deploy.history.HistoryServer"
     ports:
       - ${SPARK_HISTORY_SERVER_PORT:-18080}:18080
     volumes:
       - type: bind
         source: .
-        target: /project
+        target: /dockermount
         read_only: false
 
   # Example of running spark-submit container:
@@ -352,18 +415,18 @@ services:
   #        - the Docker image at this time only installs spark and hadoop standalone, which does not seem to include all the needed Hive jars
   #
   #    make docker-compose-run profiles="--profile spark" args="--rm -e MINIO_HOST=minio -e JDBC_URL -e COMPONENT_NAME='My Spark Prototype Script' spark-submit \
-  #      --packages org.postgresql:postgresql:42.2.23,io.delta:delta-core_2.12:1.2.1,org.apache.hadoop:hadoop-aws:3.3.1,org.apache.spark:spark-hive_2.12:3.2.1 \
-  #      /project/usaspending_api/etl/tests/path_to_your_spark_prototype_script.py"
+  #      --packages <SEE MAKEFILE> \
+  #      /dockermount/usaspending_api/etl/tests/path_to_your_spark_prototype_script.py"
 
   spark-submit:
     profiles:
       - spark  # must pass --profile spark to docker compose for this to come up
       - test
-    image: spark-base  # when an image by this name is not found in the local repo, and it is forced to build, it will use this as the tag
+    image: usaspending-development
     # build context path needs to be relative to project root, from where docker compose will be run
     build:
       context: .
-      dockerfile: spark.Dockerfile
+      dockerfile: development.Dockerfile
       args:
         PROJECT_LOG_DIR: ${PROJECT_LOG_DIR}
     container_name: spark-submit
@@ -373,42 +436,69 @@ services:
       - spark-history-server
       - minio
     environment:
-      IS_LOCAL: ${IS_LOCAL}
-      DATABASE_URL: postgres://${USASPENDING_DB_USER}:${USASPENDING_DB_PASSWORD}@${USASPENDING_DB_HOST}:${USASPENDING_DB_PORT}/data_store_api
-      MINIO_HOST: ${MINIO_HOST}
-      SPARK_MASTER_HOST: ${SPARK_MASTER_HOST:-spark-master}
-      SPARK_MASTER_PORT: ${SPARK_MASTER_PORT:-7077}
-      # i.e. target where host warehouse dir is bound in below volume config.
-      #   This env var needs to be picked up as the config for the spark.sql.warehouse.dir spark conf setting when SparkSessions are created inside of a spark-submitted job
-      SPARK_SQL_WAREHOUSE_DIR: /spark-warehouse
-      # i.e. a metastore_db sub dir of the target where host warehouse dir is bound in below volume config.
-      #   This env var needs to be picked up as the path part of the config for the spark.hadoop.javax.jdo.option.ConnectionURL spark conf setting when SparkSessions are created inside of a spark-submitted job
-      HIVE_METASTORE_DERBY_DB_DIR: /spark-warehouse/metastore_db
-      PYTHONPATH: "/project"
-      TOGGLE_OTEL_CONSOLE_LOGGING: ${TOGGLE_OTEL_CONSOLE_LOGGING}
-      OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: ${OTEL_EXPORTER_OTLP_TRACES_ENDPOINT}
+      <<: *spark-submit-vars
     # NOTE: entrypoint CANNOT interpolate env vars when processed. They are passed through literally.
     # So in using 1 $ rather than 2 $$, the var is evaluated based on the current SHELL ENV when docker compose is run,
     # and interpolated before accessed as the entrypoint.
     # While this service has values for these interpolated vars in the environment: element, those are not used here,
     # but merely passed into the container. KEEP the two references to these vars and their defaults consistent!
     # To see what it will be, you can run docker compose config (i.e. make docker-compose-config in this project's Makefile)
-    entrypoint: ./bin/spark-submit --master spark://${SPARK_MASTER_HOST:-spark-master}:${SPARK_MASTER_PORT:-7077}
+    entrypoint: /usr/bin/spark-submit --master spark://${SPARK_MASTER_HOST:-spark-master}:${SPARK_MASTER_PORT:-7077}
     command: --help
     volumes:
       - type: bind
         source: ${PROJECT_DIRECTORY:-.}
-        target: /project
+        target: /dockermount
         read_only: false
       # NOTE: The hive metastore_db Derby database folder is expected to be configured to show up as a subfolder of the spark-warehouse dir
       - type: bind
         source: ${SPARK_SQL_WAREHOUSE_DIR:-./spark-warehouse}
         target: /spark-warehouse
-      # Mount the JAR dependencies local repo on host into container to take advantage of caching/reuse
-      # i.e., to download the dependencies only once and reuse on subsequent docker compose run calls
+
+  spark-shell:
+    profiles:
+      - spark  # must pass --profile spark to docker compose for this to come up
+      - test
+    #image: public.ecr.aws/emr-serverless/spark/emr-7.12.0:latest
+    image: usaspending-development
+    # build context path needs to be relative to project root, from where docker compose will be run
+    build:
+      context: .
+      dockerfile: development.Dockerfile
+      args:
+        PROJECT_LOG_DIR: ${PROJECT_LOG_DIR}
+    container_name: spark-shell
+    depends_on:
+      - minio
+    environment:
+      <<: *spark-submit-vars
+    # NOTE: entrypoint CANNOT interpolate env vars when processed. They are passed through literally.
+    # So in using 1 $ rather than 2 $$, the var is evaluated based on the current SHELL ENV when docker compose is run,
+    # and interpolated before accessed as the entrypoint.
+    # While this service has values for these interpolated vars in the environment: element, those are not used here,
+    # but merely passed into the container. KEEP the two references to these vars and their defaults consistent!
+    # To see what it will be, you can run docker compose config (i.e. make docker-compose-config in this project's Makefile)
+    entrypoint: >
+      /usr/bin/pyspark --master local[*]
+        --conf spark.sql.extensions=io.delta.sql.DeltaSparkSessionExtension
+        --conf spark.sql.catalog.spark_catalog=org.apache.spark.sql.delta.catalog.DeltaCatalog
+        --conf spark.hadoop.fs.s3a.endpoint=${MINIO_HOST}:${MINIO_PORT}
+        --conf spark.hadoop.fs.s3a.access.key=usaspending
+        --conf spark.hadoop.fs.s3a.secret.key=usaspender
+        --conf spark.hadoop.fs.s3a.connection.ssl.enabled=false
+        --conf spark.hadoop.fs.s3a.path.style.access=true
+        --conf spark.sql.catalogImplementation=hive
+        --conf spark.sql.warehouse.dir='/spark-warehouse'
+        --conf spark.hadoop.javax.jdo.option.ConnectionURL='jdbc:derby:;databaseName=/spark-warehouse/metastore_db;create=true'
+    volumes:
       - type: bind
-        source: ${HOME}/.ivy2
-        target: /root/.ivy2
+        source: ${PROJECT_DIRECTORY:-.}
+        target: /dockermount
+        read_only: false
+      # NOTE: The hive metastore_db Derby database folder is expected to be configured to show up as a subfolder of the spark-warehouse dir
+      - type: bind
+        source: ${SPARK_SQL_WAREHOUSE_DIR:-./spark-warehouse}
+        target: /spark-warehouse
         read_only: false
 
   otel-collector:
diff --git a/Dockerfile.duckdb b/duckdb.Dockerfile
similarity index 100%
rename from Dockerfile.duckdb
rename to duckdb.Dockerfile
diff --git a/pyproject.toml b/pyproject.toml
index 282e9228a1..ba2307603c 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -114,8 +114,8 @@ dev = [
     "ruff==0.14.14",
 ]
 spark = [
-    "delta-spark==3.2.*",
-    "pyspark==3.5.0",
+    "delta-spark==3.3.2",
+    "pyspark==3.5.6",
 ]
 server = [
     "django-redis==5.4.*",
diff --git a/spark.Dockerfile b/spark.Dockerfile
deleted file mode 100644
index 3aef9d3ec3..0000000000
--- a/spark.Dockerfile
+++ /dev/null
@@ -1,80 +0,0 @@
-FROM python:3.10.12-slim-bullseye
-
-COPY --from=ghcr.io/astral-sh/uv:0.7.19 /uv /uvx /bin/
-
-# Build ARGs
-ARG HADOOP_VERSION=3.3.4
-ARG SPARK_VERSION=3.5.0
-ARG PROJECT_LOG_DIR=/logs
-
-# Install dependencies
-RUN apt update && apt install -y \
-    build-essential \
-    coreutils \
-    curl \
-    gcc \
-    libbz2-dev \
-    libpq-dev \
-    libssl-dev \
-    openssl \
-    procps \
-    wget
-
-# Install Amazon Corretto 8 (a Long-Term Supported (LTS) distribution of OpenJDK 8)
-RUN wget -qO - https://apt.corretto.aws/corretto.key | gpg --dearmor -o /usr/share/keyrings/corretto-keyring.gpg && \
-    echo "deb [signed-by=/usr/share/keyrings/corretto-keyring.gpg] https://apt.corretto.aws stable main" | tee /etc/apt/sources.list.d/corretto.list
-RUN apt update && \
-    apt install -y java-1.8.0-amazon-corretto-jdk
-ENV JAVA_HOME=/usr/lib/jvm/java-1.8.0-amazon-corretto
-
-# Install Hadoop and Spark
-WORKDIR /usr/local
-
-RUN wget --quiet https://archive.apache.org/dist/hadoop/common/hadoop-${HADOOP_VERSION}/hadoop-${HADOOP_VERSION}.tar.gz \
-    && tar xzf hadoop-${HADOOP_VERSION}.tar.gz \
-    && ln -sfn /usr/local/hadoop-${HADOOP_VERSION} /usr/local/hadoop \
-    && wget --quiet https://archive.apache.org/dist/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-without-hadoop.tgz \
-    && tar xzf spark-${SPARK_VERSION}-bin-without-hadoop.tgz \
-    && ln -sfn /usr/local/spark-${SPARK_VERSION}-bin-without-hadoop /usr/local/spark \
-    && echo "Installed $(/usr/local/hadoop/bin/hadoop version)"
-ENV HADOOP_HOME=/usr/local/hadoop
-ENV SPARK_HOME=/usr/local/spark
-# Cannot set ENV var = command-result, [i.e. doing: ENV SPARK_DIST_CLASSPATH=$(${HADOOP_HOME}/bin/hadoop classpath)], so interpolating the hadoop classpath the long way
-ENV SPARK_DIST_CLASSPATH="$HADOOP_HOME/etc/hadoop/*:$HADOOP_HOME/share/hadoop/common/lib/*:$HADOOP_HOME/share/hadoop/common/*:$HADOOP_HOME/share/hadoop/hdfs/*:$HADOOP_HOME/share/hadoop/hdfs/lib/*:$HADOOP_HOME/share/hadoop/hdfs/*:$HADOOP_HOME/share/hadoop/yarn/lib/*:$HADOOP_HOME/share/hadoop/yarn/*:$HADOOP_HOME/share/hadoop/mapreduce/lib/*:$HADOOP_HOME/share/hadoop/mapreduce/*:$HADOOP_HOME/share/hadoop/tools/lib/*"
-ENV PATH=${SPARK_HOME}/bin:${HADOOP_HOME}/bin:${JAVA_HOME}/bin:${PATH}
-RUN echo "Installed Spark" && echo "$(${SPARK_HOME}/bin/pyspark --version)"
-
-# Config for starting up the Spark History Server
-# This allows a web entrypoint to see past job runs, and bring up their Spark UI
-# It allows job-run event logs to be monitored (e.g. at /tmp/spark-events)
-# NOTE: The specified eventlog dir and its path does not exist. It is assumed that a host dir will be bind-mounted at /project
-#       so that all containers that do the same can share the events.
-RUN cp $SPARK_HOME/conf/spark-defaults.conf.template $SPARK_HOME/conf/spark-defaults.conf \
-    && sed -i "s|^# spark.eventLog.dir.*$|spark.eventLog.dir                 file:///project/$PROJECT_LOG_DIR/spark-events|g" $SPARK_HOME/conf/spark-defaults.conf \
-    && sed -i '/spark.eventLog.enabled/s/^# //g' $SPARK_HOME/conf/spark-defaults.conf \
-    && echo "spark.history.fs.logDirectory      file:///project/$PROJECT_LOG_DIR/spark-events" >> $SPARK_HOME/conf/spark-defaults.conf
-
-# Bake project dependencies into any spark-base -based container that will run Project python code (and require these dependencies)
-# NOTE: We must exclude any dependencies on the host machine installed in a virtual environment (assumed to be stored under ./.venv)
-WORKDIR /project
-
-##### The following ENV vars are optimizations from https://github.com/astral-sh/uv-docker-example/blob/main/Dockerfile
-##### and https://docs.astral.sh/uv/guides/integration/docker/#optimizations
-# Enable bytecode compilation
-ENV UV_COMPILE_BYTECODE=1
-
-# Copy from the cache instead of linking since it's a mounted volume
-ENV UV_LINK_MODE=copy
-
-# Use the system Python environment since the container is already isolated
-ENV UV_PROJECT_ENVIRONMENT=/usr/local
-ENV UV_SYSTEM_PYTHON=1
-
-# Install dependencies
-RUN --mount=type=cache,target=/root/.cache/uv \
-    --mount=type=bind,source=uv.lock,target=uv.lock \
-    --mount=type=bind,source=pyproject.toml,target=pyproject.toml \
-    uv sync --locked --no-install-project --no-dev
-
-# Allow for entrypoints and commands to infer that they are running relative to this directory
-WORKDIR ${SPARK_HOME}
diff --git a/testing.Dockerfile b/testing.Dockerfile
deleted file mode 100644
index c126e93147..0000000000
--- a/testing.Dockerfile
+++ /dev/null
@@ -1,32 +0,0 @@
-# Dockerfile for usaspending-test image
-
-# If any changes are made to the usaspending-backend image (Dockerfile) then this image needs to be rebuilt to stay in sync.
-
-# If you do not already have the `usaspending-backend` image locally you can build it by running:
-#       docker compose --profile usaspending build
-# in the parent usaspending-api directory.
-
-FROM usaspending-backend:latest
-
-# Install dependencies
-RUN apt update && apt install -y \
-    build-essential \
-    coreutils \
-    procps \
-    wget
-
-# Install Amazon Corretto 8 (a Long-Term Supported (LTS) distribution of OpenJDK 8)
-RUN wget -qO - https://apt.corretto.aws/corretto.key | gpg --dearmor -o /usr/share/keyrings/corretto-keyring.gpg && \
-    echo "deb [signed-by=/usr/share/keyrings/corretto-keyring.gpg] https://apt.corretto.aws stable main" | tee /etc/apt/sources.list.d/corretto.list
-RUN apt update && \
-    apt install -y java-1.8.0-amazon-corretto-jdk
-RUN export JAVA_HOME=/usr/lib/jvm/java-1.8.0-amazon-corretto
-RUN export PATH=${JAVA_HOME}/bin:$PATH
-
-# Install dev dependencies
-RUN --mount=type=cache,target=/root/.cache/uv \
-    --mount=type=bind,source=uv.lock,target=uv.lock \
-    --mount=type=bind,source=pyproject.toml,target=pyproject.toml \
-    uv sync --extra dev --extra spark --locked
-
-WORKDIR /dockermount
\ No newline at end of file
diff --git a/usaspending_api/common/helpers/spark_helpers.py b/usaspending_api/common/helpers/spark_helpers.py
index 51df833e45..29a941a8a0 100644
--- a/usaspending_api/common/helpers/spark_helpers.py
+++ b/usaspending_api/common/helpers/spark_helpers.py
@@ -69,13 +69,7 @@ def is_spark_context_stopped() -> bool:
         # Check the Singleton instance populated if there's an active SparkContext
         if SparkContext._active_spark_context is not None:
             sc = SparkContext._active_spark_context
-            is_stopped = not (
-                sc._jvm
-                and not sc._jvm.SparkSession.getDefaultSession()
-                .get()
-                .sparkContext()
-                .isStopped()
-            )
+            is_stopped = not (sc._jvm and not sc._jvm.SparkSession.getDefaultSession().get().sparkContext().isStopped())
     return is_stopped
 
 
@@ -89,10 +83,7 @@ def stop_spark_context() -> bool:
                 sc._jvm
                 and hasattr(sc._jvm, "SparkSession")
                 and sc._jvm.SparkSession
-                and not sc._jvm.SparkSession.getDefaultSession()
-                .get()
-                .sparkContext()
-                .isStopped()
+                and not sc._jvm.SparkSession.getDefaultSession().get().sparkContext().isStopped()
             ):
                 try:
                     sc.stop()
@@ -157,15 +148,9 @@ def configure_spark_session(  # noqa: C901,PLR0912,PLR0913,PLR0915
             property, otherwise an error will be thrown.
     """
     if spark_context and (
-        not spark_context._jvm
-        or spark_context._jvm.SparkSession.getDefaultSession()
-        .get()
-        .sparkContext()
-        .isStopped()
+        not spark_context._jvm or spark_context._jvm.SparkSession.getDefaultSession().get().sparkContext().isStopped()
     ):
-        raise ValueError(
-            "The provided spark_context arg is a stopped SparkContext. It must be active."
-        )
+        raise ValueError("The provided spark_context arg is a stopped SparkContext. It must be active.")
     if spark_context and java_gateway:
         raise Exception(
             "Cannot provide BOTH spark_context and java_gateway args. The active spark_context supplies its own gateway"
@@ -271,9 +256,7 @@ def configure_spark_session(  # noqa: C901,PLR0912,PLR0913,PLR0915
     if spark_context:
         built_conf = spark.conf
         provided_conf_keys = [item[0] for item in conf.getAll()]
-        non_modifiable_conf = [
-            k for k in provided_conf_keys if not built_conf.isModifiable(k)
-        ]
+        non_modifiable_conf = [k for k in provided_conf_keys if not built_conf.isModifiable(k)]
         if non_modifiable_conf:
             raise ValueError(
                 "An active SparkContext was given along with NEW spark config values. The following "
@@ -298,7 +281,8 @@ def configure_spark_session(  # noqa: C901,PLR0912,PLR0913,PLR0915
 @       Found SPARK_HOME: {_find_spark_home()}
 @       Python Version: {sys.version}
 @       Spark Version: {spark.version}
-@       Hadoop Version: {spark.sparkContext._gateway.jvm.org.apache.hadoop.util.VersionInfo.getVersion()}
+@       Hadoop Version: {spark.sparkContext._jvm.org.apache.hadoop.util.VersionInfo.getVersion()}
+@       Hive Version: {spark.sparkContext._jvm.org.apache.hive.common.util.HiveVersionInfo.getVersion()}
     """
     )
     es_config = get_es_config()
@@ -388,9 +372,7 @@ def attach_java_gateway(
     return gateway
 
 
-def get_jdbc_connection_properties(
-    fix_strings: bool = True, truncate: bool = False
-) -> dict:
+def get_jdbc_connection_properties(fix_strings: bool = True, truncate: bool = False) -> dict:
     jdbc_props = {
         "driver": "org.postgresql.Driver",
         "fetchsize": str(CONFIG.SPARK_PARTITION_ROWS),
@@ -410,9 +392,7 @@ def get_jdbc_url_from_pg_uri(pg_uri: str) -> str:
     """Converts the passed-in Postgres DB connection URI to a JDBC-compliant Postgres DB connection string"""
     url_parts, user, password = parse_pg_uri(pg_uri)
     if user is None or password is None:
-        raise ValueError(
-            "pg_uri provided must have username and password with host or in query string"
-        )
+        raise ValueError("pg_uri provided must have username and password with host or in query string")
     # JDBC URLs only support postgresql://
     pg_uri = f"postgresql://{url_parts.hostname}:{url_parts.port}{url_parts.path}?user={user}&password={password}"
 
@@ -472,9 +452,7 @@ def get_es_config() -> dict[str, Any]:  # pragma: no cover -- will be used event
         "es.net.ssl": str(ssl).lower(),  # default false
         "es.net.ssl.cert.allow.self.signed": "true",  # default false
         "es.batch.size.entries": str(CONFIG.ES_BATCH_ENTRIES),  # default 1000
-        "es.batch.size.bytes": str(
-            CONFIG.ES_MAX_BATCH_BYTES
-        ),  # default 1024*1024 (1mb)
+        "es.batch.size.bytes": str(CONFIG.ES_MAX_BATCH_BYTES),  # default 1024*1024 (1mb)
         "es.batch.write.refresh": "false",  # default true, to refresh after configured batch size completes
     }
 
@@ -527,12 +505,8 @@ def configure_s3_credentials(
             "org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider",
         )
         conf.set("spark.hadoop.fs.s3a.session.token", aws_creds.token)
-        conf.set(
-            "spark.hadoop.fs.s3a.assumed.role.sts.endpoint", CONFIG.AWS_STS_ENDPOINT
-        )
-        conf.set(
-            "spark.hadoop.fs.s3a.assumed.role.sts.endpoint.region", CONFIG.AWS_REGION
-        )
+        conf.set("spark.hadoop.fs.s3a.assumed.role.sts.endpoint", CONFIG.AWS_STS_ENDPOINT)
+        conf.set("spark.hadoop.fs.s3a.assumed.role.sts.endpoint.region", CONFIG.AWS_REGION)
 
 
 def log_spark_config(spark: SparkSession, config_key_contains: str = "") -> None:
@@ -551,9 +525,7 @@ def log_hadoop_config(spark: SparkSession, config_key_contains: str = "") -> Non
     conf = spark.sparkContext._jsc.hadoopConfiguration()
     [
         logger.info(f"{k}={v}")
-        for (k, v) in {
-            str(_).split("=")[0]: str(_).split("=")[1] for _ in conf.iterator()
-        }.items()
+        for (k, v) in {str(_).split("=")[0]: str(_).split("=")[1] for _ in conf.iterator()}.items()
         if config_key_contains in k
     ]
 
@@ -582,16 +554,10 @@ def load_dict_to_delta_table(  # noqa: PLR0912
     table_to_col_names_dict = {}
     table_to_col_names_dict["transaction_fabs"] = TRANSACTION_FABS_COLUMNS
     table_to_col_names_dict["transaction_fpds"] = TRANSACTION_FPDS_COLUMNS
-    table_to_col_names_dict["transaction_normalized"] = list(
-        TRANSACTION_NORMALIZED_COLUMNS
-    )
+    table_to_col_names_dict["transaction_normalized"] = list(TRANSACTION_NORMALIZED_COLUMNS)
     table_to_col_names_dict["awards"] = list(AWARDS_COLUMNS)
-    table_to_col_names_dict["financial_accounts_by_awards"] = list(
-        FINANCIAL_ACCOUNTS_BY_AWARDS_COLUMNS
-    )
-    table_to_col_names_dict["detached_award_procurement"] = list(
-        DETACHED_AWARD_PROCUREMENT_DELTA_COLUMNS
-    )
+    table_to_col_names_dict["financial_accounts_by_awards"] = list(FINANCIAL_ACCOUNTS_BY_AWARDS_COLUMNS)
+    table_to_col_names_dict["detached_award_procurement"] = list(DETACHED_AWARD_PROCUREMENT_DELTA_COLUMNS)
     table_to_col_names_dict["published_fabs"] = list(PUBLISHED_FABS_COLUMNS)
 
     table_to_col_info_dict = {}
@@ -686,9 +652,7 @@ def clean_postgres_sql_for_spark_sql(
     # Treat these type casts as string in Spark SQL
     # NOTE: If replacing a ::JSON cast, be sure that the string data coming from delta is treated as needed (e.g. as
     # JSON or converted to JSON or a dict) on the receiving side, and not just left as a string
-    spark_sql = re.sub(
-        r"::text|::json", r"::string", spark_sql, flags=re.IGNORECASE | re.MULTILINE
-    )
+    spark_sql = re.sub(r"::text|::json", r"::string", spark_sql, flags=re.IGNORECASE | re.MULTILINE)
 
     if global_temp_view_proxies:
         for vw in global_temp_view_proxies:
diff --git a/usaspending_api/common/spark/configs.py b/usaspending_api/common/spark/configs.py
index 4113a4afb3..7faefaff92 100644
--- a/usaspending_api/common/spark/configs.py
+++ b/usaspending_api/common/spark/configs.py
@@ -1,25 +1,5 @@
 from usaspending_api.config import CONFIG
 
-# The versions below are determined by the current version of Databricks in use
-_SCALA_VERSION = "2.12"
-_HADOOP_VERSION = "3.3.4"
-_SPARK_VERSION = "3.5.0"
-_DELTA_VERSION = "3.1.0"
-
-# List of Maven coordinates for required JAR files used by running code, which can be added to the driver and
-# executor class paths
-SPARK_SESSION_JARS = [
-    # "com.amazonaws:aws-java-sdk:1.12.31",
-    # hadoop-aws is an add-on to hadoop with Classes that allow hadoop to interface with an S3A (AWS S3) FileSystem
-    # NOTE That in order to work, the version number should be the same as the Hadoop version used by your Spark runtime
-    # It SHOULD pull in (via Ivy package manager from maven repo) the version of com.amazonaws:aws-java-sdk that is
-    # COMPATIBLE with it (so that should not  be set as a dependent package by us)
-    f"org.apache.hadoop:hadoop-aws:{_HADOOP_VERSION}",
-    "org.postgresql:postgresql:42.2.23",
-    f"io.delta:delta-spark_{_SCALA_VERSION}:{_DELTA_VERSION}",
-]
-OPTIONAL_SPARK_HIVE_JAR = f"org.apache.spark:spark-hive_{_SCALA_VERSION}:{_SPARK_VERSION}"
-
 # TODO: This should be used more widely across our different commands
 DEFAULT_EXTRA_CONF = {
     # Config for Delta Lake tables and SQL. Need these to keep Dela table metadata in the metastore
@@ -45,7 +25,6 @@
     "spark.driver.memory": "1g",
     "spark.executor.memory": "1g",
     "spark.ui.enabled": "false",  # Does the same as setting SPARK_TESTING=true env var
-    "spark.jars.packages": ",".join(SPARK_SESSION_JARS),
 }
 
 
diff --git a/usaspending_api/common/spark/jobs.py b/usaspending_api/common/spark/jobs.py
index 5e3f95e23e..16ae69054c 100644
--- a/usaspending_api/common/spark/jobs.py
+++ b/usaspending_api/common/spark/jobs.py
@@ -13,9 +13,9 @@
 from django.conf import settings
 from django.core.management import call_command
 from duckdb.experimental.spark.sql import SparkSession as DuckDBSparkSession
-from usaspending_api.config import CONFIG
 
-from usaspending_api.common.spark.configs import LOCAL_EXTENDED_EXTRA_CONF, OPTIONAL_SPARK_HIVE_JAR, SPARK_SESSION_JARS
+from usaspending_api.common.spark.configs import LOCAL_EXTENDED_EXTRA_CONF
+from usaspending_api.config import CONFIG
 
 if TYPE_CHECKING:
     from pyspark.sql import SparkSession
@@ -153,9 +153,9 @@ def handle_start(self, job_name: str, command_name: str, command_options: list[s
         execution_role_arn = kwargs.get("execution_role_arn")
 
         if not execution_role_arn:
-            raise ValueError(f"Execution role ARN is required to start an EMR Serverless job")
+            raise ValueError("Execution role ARN is required to start an EMR Serverless job")
         elif not application_name and not application_id:
-            raise ValueError(f"Application Name or ID is required to start an EMR Serverless job")
+            raise ValueError("Application Name or ID is required to start an EMR Serverless job")
         elif application_name and not application_id:
             application_id = self._get_application_id(application_name)
 
@@ -230,16 +230,11 @@ def _run_in_container(job_name: str, command_name: str, command_options: list[st
         network = template_container.attrs["HostConfig"]["NetworkMode"]
         options_as_string = " ".join(command_options)
         container_name = f"spark-submit_{job_name}_{datetime.now().strftime('%Y-%m-%d-%H-%M-%S')}"
-        required_jars = ",".join([*SPARK_SESSION_JARS, OPTIONAL_SPARK_HIVE_JAR])
         client.containers.run(
             image_name,
             name=container_name,
             network=network,
-            command=(
-                'spark-submit --driver-memory "2g"'
-                f" --packages {required_jars}"
-                f" /project/manage.py {command_name} {options_as_string}"
-            ),
+            command=(f'spark-submit --driver-memory "2g" /dockermount/manage.py {command_name} {options_as_string}'),
             environment=[
                 f"COMPONENT_NAME={command_name} {options_as_string}",
                 *environment_variables,
diff --git a/uv.lock b/uv.lock
index 8022025477..d15520710f 100644
--- a/uv.lock
+++ b/uv.lock
@@ -560,15 +560,15 @@ wheels = [
 
 [[package]]
 name = "delta-spark"
-version = "3.2.0"
+version = "3.3.2"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "importlib-metadata" },
     { name = "pyspark" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/38/06/a64cc4e17fe959cf60dc126bf3283fc9f22fc91f000b7f3f5e465338022d/delta-spark-3.2.0.tar.gz", hash = "sha256:641967828e47c64805f8c746513da80bea24b5f19b069cdcf64561cd3692e11d", size = 22147, upload-time = "2024-05-09T17:26:10.754Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/8e/c9/a4d34fee1dafcad33d4e810ff2ce14a9f6e8e7af9797a9de79ebc3334987/delta_spark-3.3.2.tar.gz", hash = "sha256:f3e89ec4a15a541988f9d6496016a67cf619475e8eeca71da5e073f6ba0da845", size = 23066, upload-time = "2025-05-29T22:25:49.412Z" }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/3c/8e/bb778f5049aa371bf80a7a781b237eb10ef104f8736fe00d25fcfee80c2b/delta_spark-3.2.0-py3-none-any.whl", hash = "sha256:c4ff3fa7218e58a702cb71eb64384b0005c4d6f0bbdd0fe0b38a53564d946e09", size = 21186, upload-time = "2024-05-09T17:26:09.544Z" },
+    { url = "https://files.pythonhosted.org/packages/4b/b7/f484bca84628f9f7a9a16ecfd8b010edbda9fa3f5e3918e967ddd69bf6af/delta_spark-3.3.2-py3-none-any.whl", hash = "sha256:0a34c720b4368f1655e2348e380b5b3baaca0f2b3d519a3198208f00f65e4062", size = 22024, upload-time = "2025-05-29T22:25:48.008Z" },
 ]
 
 [[package]]
@@ -2038,12 +2038,12 @@ wheels = [
 
 [[package]]
 name = "pyspark"
-version = "3.5.0"
+version = "3.5.6"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "py4j" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/92/11/5c0f24537fa68624cb6271d2a8a8982e81bd05f5cb51e9d2d2a89baa0e1a/pyspark-3.5.0.tar.gz", hash = "sha256:d41a9b76bd2aca370a6100d075c029e22ba44c5940927877e9435a3a9c566558", size = 316890381, upload-time = "2023-09-26T20:30:58.621Z" }
+sdist = { url = "https://files.pythonhosted.org/packages/2e/62/36e50d38e5fe158e97cddec983b44f9417b1e205b02320e3c463b5f802fa/pyspark-3.5.6.tar.gz", hash = "sha256:f8b1c4360e41ab398c64904fae08740503bcb6bd389457d659fa6d9f2952cc48", size = 317359167, upload-time = "2025-05-27T08:24:20.82Z" }
 
 [[package]]
 name = "pytest"
@@ -2607,7 +2607,7 @@ requires-dist = [
     { name = "click", marker = "extra == 'dev'", specifier = "==8.1.7" },
     { name = "databricks-sdk", specifier = "==0.44.1" },
     { name = "dataclasses-json", specifier = "==0.6.*" },
-    { name = "delta-spark", marker = "extra == 'spark'", specifier = "==3.2.*" },
+    { name = "delta-spark", marker = "extra == 'spark'", specifier = "==3.3.2" },
     { name = "dj-database-url", specifier = "==2.1.0" },
     { name = "django", specifier = "==4.2.*" },
     { name = "django-cors-headers", specifier = "==4.3.*" },
@@ -2677,7 +2677,7 @@ requires-dist = [
     { name = "pydantic", specifier = "==2.12" },
     { name = "pydantic-settings", specifier = ">=2.13.1" },
     { name = "pyparsing", marker = "extra == 'server'", specifier = "==3.2.1" },
-    { name = "pyspark", marker = "extra == 'spark'", specifier = "==3.5.0" },
+    { name = "pyspark", marker = "extra == 'spark'", specifier = "==3.5.6" },
     { name = "pytest", marker = "extra == 'dev'", specifier = "==8.0.*" },
     { name = "pytest-cov", marker = "extra == 'dev'", specifier = "==4.1.*" },
     { name = "pytest-django", marker = "extra == 'dev'", specifier = "==4.8.*" },

From 9235d5d46389393d8c2eecf127c59be5a798c471 Mon Sep 17 00:00:00 2001
From: Seth Stoudenmier <stoudenmier_seth@bah.com>
Date: Fri, 8 May 2026 09:43:15 -0400
Subject: [PATCH 02/48] [DEV-14608] fix typo

---
 .github/actions/run-pytest/action.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/actions/run-pytest/action.yaml b/.github/actions/run-pytest/action.yaml
index c4f65c0346..3689a220f2 100644
--- a/.github/actions/run-pytest/action.yaml
+++ b/.github/actions/run-pytest/action.yaml
@@ -73,7 +73,7 @@ runs:
     - name: Run tests
       working-directory: ${{ inputs.working-directory }}
       shell: bash
-      run: docker compose run --rm ${{ steps.build-pytest-command.outputs.pytest-command }}
+      run: docker compose run --rm usaspending-test ${{ steps.build-pytest-command.outputs.pytest-command }}
 
     - name: Upload Artifact
       uses: actions/upload-artifact@v4

From 2c848ec12757707c5584746fb7d98508b21a1c0f Mon Sep 17 00:00:00 2001
From: Seth Stoudenmier <stoudenmier_seth@bah.com>
Date: Fri, 8 May 2026 10:10:45 -0400
Subject: [PATCH 03/48] [DEV-14608] additional updates for running test suite

---
 .github/actions/init-test-environment/action.yaml | 14 ++++++--------
 docker-compose.yml                                |  6 +++---
 usaspending_api/config/envs/local.py              |  4 ++--
 3 files changed, 11 insertions(+), 13 deletions(-)

diff --git a/.github/actions/init-test-environment/action.yaml b/.github/actions/init-test-environment/action.yaml
index 38b26b454b..815271f49d 100644
--- a/.github/actions/init-test-environment/action.yaml
+++ b/.github/actions/init-test-environment/action.yaml
@@ -48,13 +48,6 @@ runs:
       if: ${{ inputs.is-integration-test == 'true' }}
       run: docker load --input /tmp/dataact-broker-backend.tar
 
-    - name: Wait on DB and ES containers to be available
-      if: ${{ inputs.is-integration-test == 'true' }}
-      shell: bash
-      run: |
-        ttl=30; echo "Try DB conn from container for $ttl seconds"; until [ $ttl -le 0 ] || psql $DATABASE_URL -c 'select 1 where 1=1'; do echo $ttl; ((ttl--)); sleep 1; done; [ $ttl -gt 0 ]
-        ttl=30; echo "Try ES conn from container for $ttl seconds"; until [ $ttl -le 0 ] || curl --silent -XGET --fail $ES_HOSTNAME; do echo $ttl; ((ttl--)); sleep 1; done; [ $ttl -gt 0 ]
-
     - name: Add DB users and set search_path
       if: ${{ inputs.is-integration-test == 'true' }}
       shell: bash
@@ -66,10 +59,15 @@ runs:
         psql $DATABASE_URL -c "SELECT 'GRANT SELECT ON ALL TABLES IN SCHEMA ' || nspname || ' TO readonly' FROM pg_namespace WHERE nspname IN ('raw','int','rpt','temp','public')"
         psql $DATABASE_URL -c "SELECT 'ALTER DEFAULT PRIVILEGES IN SCHEMA ' || nspname || ' GRANT SELECT ON TABLES TO readonly' FROM pg_namespace WHERE nspname IN ('raw','int','rpt','temp','public')"
 
+    - name: Build the usaspending-development image
+      working-directory: ${{ inputs.working-directory }}
+      shell: bash
+      run: docker compose build usaspending-test
+
     - name: Run initial test to trigger DB creation
       if: ${{ inputs.is-integration-test == 'true' }}
       working-directory: ${{ inputs.working-directory }}
       shell: bash
       run: >
-        docker compose run --rm usaspending-test --create-db --numprocesses logical --no-cov --disable-warnings -r=fEs --verbosity=3 --capture=no --log-cli-level=WARNING --show-capture=log
+        docker compose run --rm usaspending-test --create-db --numprocesses auto --no-cov --disable-warnings -r=fEs --verbosity=3 --capture=no --log-cli-level=WARNING --show-capture=log
         2> /dev/null "usaspending_api/tests/integration/test_setup_of_test_dbs.py::test_trigger_test_db_setup"
diff --git a/docker-compose.yml b/docker-compose.yml
index 8df644ab51..f4893dfe8c 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -164,7 +164,7 @@ services:
       - -c
       - |
         printf -- "-------\nRunning non-spark integration tests:\n"
-        pytest --override-ini=python_files="**/tests/integration/*" -m "(not signal_handling and not spark)" --no-cov --disable-warnings --reuse-db --numprocesses logical
+        pytest --override-ini=python_files="**/tests/integration/*" -m "(not signal_handling and not spark)" --no-cov --disable-warnings --reuse-db --numprocesses auto
         printf -- "-------\nRunning non-spark integration tests with signal handling:\n"
         pytest --override-ini=python_files="**/tests/integration/*" -m "(signal_handling and not spark)" --no-cov --disable-warnings --reuse-db --numprocesses 0
         printf -- "-------\nRunning spark integration tests -- load to / from delta:\n"
@@ -174,9 +174,9 @@ services:
         printf -- "-------\nRunning spark integration tests -- load transaction lookup:\n"
         pytest --override-ini=python_files="test_*.py *_test.py" -m "(spark)" -k "test_load_transactions_in_delta_lookups.py" --no-cov --disable-warnings --reuse-db --numprocesses 0
         printf -- "-------\nRunning spark integration tests -- other:\n"
-        pytest --override-ini=python_files="test_*.py *_test.py" -m "(spark)" -k "(not test_load_to_from_delta.py and not test_load_transactions_in_delta_lookups.py and not test_load_transactions_in_delta_fabs_fpds.py)" --no-cov --disable-warnings --reuse-db --numprocesses logical
+        pytest --override-ini=python_files="test_*.py *_test.py" -m "(spark)" -k "(not test_load_to_from_delta.py and not test_load_transactions_in_delta_lookups.py and not test_load_transactions_in_delta_fabs_fpds.py)" --no-cov --disable-warnings --reuse-db --numprocesses auto
         printf -- "-------\nRunning unit tests:\n"
-        pytest --override-ini=python_files="test_*.py *_test.py" -m "(not signal_handling)" --ignore-glob="**/tests/integration/*" --no-cov --disable-warnings --reuse-db --numprocesses logical
+        pytest --override-ini=python_files="test_*.py *_test.py" -m "(not signal_handling)" --ignore-glob="**/tests/integration/*" --no-cov --disable-warnings --reuse-db --numprocesses auto
         printf -- "-------\nRunning unit tests with signal handling:\n"
         pytest --override-ini=python_files="test_*.py *_test.py" -m "(signal_handling)" --ignore-glob="**/tests/integration/*" --no-cov --disable-warnings --reuse-db --numprocesses 0
     environment:
diff --git a/usaspending_api/config/envs/local.py b/usaspending_api/config/envs/local.py
index 430a6d8984..40fb72e459 100644
--- a/usaspending_api/config/envs/local.py
+++ b/usaspending_api/config/envs/local.py
@@ -62,7 +62,7 @@ class LocalConfig(DefaultConfig):
     # ==== [Elasticsearch] ====
     # Where to connect to elasticsearch.
     ES_SCHEME: str = "http"
-    ES_HOST: str = "localhost"
+    ES_HOST: str = "usaspending-es"
     ES_PORT: str = "9200"
 
     # ==== [Spark] ====
@@ -71,7 +71,7 @@ class LocalConfig(DefaultConfig):
     HIVE_METASTORE_DERBY_DB_DIR: str = str(_PROJECT_ROOT_DIR / "spark-warehouse" / "metastore_db")
 
     # ==== [MinIO] ====
-    MINIO_HOST: str = "localhost"
+    MINIO_HOST: str = "minio"
     # Changing MinIO ports from defaults. Known to have port conflicts with proxies on developer laptops
     MINIO_PORT: str = "10001"
     MINIO_CONSOLE_PORT: str = "10002"

From 9cc06f4795b26a59206db16d0fd35aed71cf66c5 Mon Sep 17 00:00:00 2001
From: Seth Stoudenmier <stoudenmier_seth@bah.com>
Date: Fri, 8 May 2026 10:38:59 -0400
Subject: [PATCH 04/48] [DEV-14608] additional updates for running test suite

---
 .github/actions/init-test-environment/action.yaml | 7 +++++++
 .github/workflows/code-style-checks.yaml          | 9 +++++----
 usaspending_api/config/envs/local.py              | 2 +-
 3 files changed, 13 insertions(+), 5 deletions(-)

diff --git a/.github/actions/init-test-environment/action.yaml b/.github/actions/init-test-environment/action.yaml
index 815271f49d..4e212dce4a 100644
--- a/.github/actions/init-test-environment/action.yaml
+++ b/.github/actions/init-test-environment/action.yaml
@@ -48,6 +48,13 @@ runs:
       if: ${{ inputs.is-integration-test == 'true' }}
       run: docker load --input /tmp/dataact-broker-backend.tar
 
+    - name: Wait on DB and ES containers to be available
+      if: ${{ inputs.is-integration-test == 'true' }}
+      shell: bash
+      run: |
+        ttl=30; container_name="usaspending-db"; echo "Wait for $container_name to be healthy for $ttl seconds"; until [ $ttl -le 0 ] || [ "$(docker inspect $container_name --format='{{.State.Health.Status}}')" = "healthy" ]; do echo $ttl; ((ttl--)); sleep 1; done; [ $ttl -gt 0 ]
+        ttl=30; container_name="usaspending-es"; echo "Wait for $container_name to be healthy for $ttl seconds"; until [ $ttl -le 0 ] || [ "$(docker inspect $container_name --format='{{.State.Health.Status}}')" = "healthy" ]; do echo $ttl; ((ttl--)); sleep 1; done; [ $ttl -gt 0 ]
+
     - name: Add DB users and set search_path
       if: ${{ inputs.is-integration-test == 'true' }}
       shell: bash
diff --git a/.github/workflows/code-style-checks.yaml b/.github/workflows/code-style-checks.yaml
index 41963673e1..38d833b60f 100644
--- a/.github/workflows/code-style-checks.yaml
+++ b/.github/workflows/code-style-checks.yaml
@@ -1,21 +1,22 @@
 name: Code Style Checks
 
 env:
-  BROKER_DB_HOST: localhost
+  BROKER_DB_HOST: usaspending-db
   BROKER_DB_PORT: 5432
   BROKER_DB_USER: admin
   BROKER_DB_PASSWORD: root
   BROKER_DB_NAME: data_broker
   DJANGO_SETTINGS_MODULE: usaspending_api.settings
   ES_SCHEME: http
-  ES_HOST: localhost
+  ES_HOST: usaspending-es
   ES_PORT: 9200
-  MINIO_HOST: localhost
-  USASPENDING_DB_HOST: localhost
+  MINIO_HOST: minio
+  USASPENDING_DB_HOST: usaspending-db
   USASPENDING_DB_PORT: 5432
   USASPENDING_DB_USER: usaspending
   USASPENDING_DB_PASSWORD: usaspender
   USASPENDING_DB_NAME: data_store_api
+  IS_LOCAL: true
 
 on:
   workflow_call:
diff --git a/usaspending_api/config/envs/local.py b/usaspending_api/config/envs/local.py
index 40fb72e459..2efb03829b 100644
--- a/usaspending_api/config/envs/local.py
+++ b/usaspending_api/config/envs/local.py
@@ -56,7 +56,7 @@ class LocalConfig(DefaultConfig):
 
     # Change to host.docker.internal if you are running a local Postgres. Otherwise leave as-is, so
     # Docker will use the Postgres created by Compose.
-    BROKER_DB_HOST: str | None = "dataact-broker-db"
+    BROKER_DB_HOST: str | None = "usaspending-db"
     BROKER_DB_PORT: str | None = "5432"
 
     # ==== [Elasticsearch] ====

From 5997fe58bc71d4e120d06c04b50109e7ce4340f7 Mon Sep 17 00:00:00 2001
From: Seth Stoudenmier <stoudenmier_seth@bah.com>
Date: Fri, 8 May 2026 11:52:14 -0400
Subject: [PATCH 05/48] [DEV-14608] updates for test suite

---
 .github/actions/init-test-environment/action.yaml     | 11 +++++------
 .../test-non-spark-integration-signal-handling.yaml   |  3 ++-
 .github/workflows/test-non-spark-integration.yaml     |  3 ++-
 .../test-spark-integration-load-to-from-delta.yaml    |  3 ++-
 ...spark-integration-load-transactions-fabs-fpds.yaml |  3 ++-
 ...st-spark-integration-load-transactions-lookup.yaml |  3 ++-
 .github/workflows/test-spark-integration-other.yaml   |  3 ++-
 .github/workflows/test-unit-signal-handling.yaml      |  3 ++-
 .github/workflows/test-unit.yaml                      |  3 ++-
 development.Dockerfile                                |  2 +-
 docker-compose.yml                                    |  2 +-
 11 files changed, 23 insertions(+), 16 deletions(-)

diff --git a/.github/actions/init-test-environment/action.yaml b/.github/actions/init-test-environment/action.yaml
index 4e212dce4a..31b4ea4aa7 100644
--- a/.github/actions/init-test-environment/action.yaml
+++ b/.github/actions/init-test-environment/action.yaml
@@ -59,12 +59,11 @@ runs:
       if: ${{ inputs.is-integration-test == 'true' }}
       shell: bash
       run: |
-        psql $DATABASE_URL -c "ALTER USER $USASPENDING_DB_USER SET search_path TO public,raw,int,temp,rpt"
-        psql $DATABASE_URL -c "CREATE USER $BROKER_DB_USER PASSWORD '$BROKER_DB_PASSWORD' SUPERUSER"
-        psql $DATABASE_URL -c "CREATE ROLE readonly"
-        psql $DATABASE_URL -c "SELECT 'GRANT USAGE ON SCHEMA ' || nspname || ' TO readonly' FROM pg_namespace WHERE nspname IN ('raw','int','rpt','temp','public')"
-        psql $DATABASE_URL -c "SELECT 'GRANT SELECT ON ALL TABLES IN SCHEMA ' || nspname || ' TO readonly' FROM pg_namespace WHERE nspname IN ('raw','int','rpt','temp','public')"
-        psql $DATABASE_URL -c "SELECT 'ALTER DEFAULT PRIVILEGES IN SCHEMA ' || nspname || ' GRANT SELECT ON TABLES TO readonly' FROM pg_namespace WHERE nspname IN ('raw','int','rpt','temp','public')"
+        docker exec usaspending-db psql -U usaspending -d data_store_api -c "CREATE USER $BROKER_DB_USER PASSWORD '$BROKER_DB_PASSWORD' SUPERUSER"
+        docker exec usaspending-db psql -U usaspending -d data_store_api -c "CREATE ROLE readonly"
+        docker exec usaspending-db psql -U usaspending -d data_store_api -c "SELECT 'GRANT USAGE ON SCHEMA ' || nspname || ' TO readonly' FROM pg_namespace WHERE nspname IN ('raw','int','rpt','temp','public')"
+        docker exec usaspending-db psql -U usaspending -d data_store_api -c "SELECT 'GRANT SELECT ON ALL TABLES IN SCHEMA ' || nspname || ' TO readonly' FROM pg_namespace WHERE nspname IN ('raw','int','rpt','temp','public')"
+        docker exec usaspending-db psql -U usaspending -d data_store_api -c "SELECT 'ALTER DEFAULT PRIVILEGES IN SCHEMA ' || nspname || ' GRANT SELECT ON TABLES TO readonly' FROM pg_namespace WHERE nspname IN ('raw','int','rpt','temp','public')"
 
     - name: Build the usaspending-development image
       working-directory: ${{ inputs.working-directory }}
diff --git a/.github/workflows/test-non-spark-integration-signal-handling.yaml b/.github/workflows/test-non-spark-integration-signal-handling.yaml
index fc0f56f54d..3e65ec8a61 100644
--- a/.github/workflows/test-non-spark-integration-signal-handling.yaml
+++ b/.github/workflows/test-non-spark-integration-signal-handling.yaml
@@ -10,13 +10,14 @@ env:
   ES_SCHEME: http
   ES_HOST: usaspending-es
   ES_PORT: 9200
+  IS_LOCAL: true
   MINIO_HOST: minio
+  PROJECT_DIRECTORY: usaspending-api
   USASPENDING_DB_HOST: usaspending-db
   USASPENDING_DB_PORT: 5432
   USASPENDING_DB_USER: usaspending
   USASPENDING_DB_PASSWORD: usaspender
   USASPENDING_DB_NAME: data_store_api
-  IS_LOCAL: true
 
 on:
   workflow_call:
diff --git a/.github/workflows/test-non-spark-integration.yaml b/.github/workflows/test-non-spark-integration.yaml
index 0e2517cee5..5373fe84e2 100644
--- a/.github/workflows/test-non-spark-integration.yaml
+++ b/.github/workflows/test-non-spark-integration.yaml
@@ -10,13 +10,14 @@ env:
   ES_SCHEME: http
   ES_HOST: usaspending-es
   ES_PORT: 9200
+  IS_LOCAL: true
   MINIO_HOST: minio
+  PROJECT_DIRECTORY: usaspending-api
   USASPENDING_DB_HOST: usaspending-db
   USASPENDING_DB_PORT: 5432
   USASPENDING_DB_USER: usaspending
   USASPENDING_DB_PASSWORD: usaspender
   USASPENDING_DB_NAME: data_store_api
-  IS_LOCAL: true
 
 on:
   workflow_call:
diff --git a/.github/workflows/test-spark-integration-load-to-from-delta.yaml b/.github/workflows/test-spark-integration-load-to-from-delta.yaml
index 44967060d9..e140e082d3 100644
--- a/.github/workflows/test-spark-integration-load-to-from-delta.yaml
+++ b/.github/workflows/test-spark-integration-load-to-from-delta.yaml
@@ -10,13 +10,14 @@ env:
   ES_SCHEME: http
   ES_HOST: usaspending-es
   ES_PORT: 9200
+  IS_LOCAL: true
   MINIO_HOST: minio
+  PROJECT_DIRECTORY: usaspending-api
   USASPENDING_DB_HOST: usaspending-db
   USASPENDING_DB_PORT: 5432
   USASPENDING_DB_USER: usaspending
   USASPENDING_DB_PASSWORD: usaspender
   USASPENDING_DB_NAME: data_store_api
-  IS_LOCAL: true
 
 on:
   workflow_call:
diff --git a/.github/workflows/test-spark-integration-load-transactions-fabs-fpds.yaml b/.github/workflows/test-spark-integration-load-transactions-fabs-fpds.yaml
index fe33671fd4..0232c70e8b 100644
--- a/.github/workflows/test-spark-integration-load-transactions-fabs-fpds.yaml
+++ b/.github/workflows/test-spark-integration-load-transactions-fabs-fpds.yaml
@@ -10,13 +10,14 @@ env:
   ES_SCHEME: http
   ES_HOST: usaspending-es
   ES_PORT: 9200
+  IS_LOCAL: true
   MINIO_HOST: minio
+  PROJECT_DIRECTORY: usaspending-api
   USASPENDING_DB_HOST: usaspending-db
   USASPENDING_DB_PORT: 5432
   USASPENDING_DB_USER: usaspending
   USASPENDING_DB_PASSWORD: usaspender
   USASPENDING_DB_NAME: data_store_api
-  IS_LOCAL: true
 
 on:
   workflow_call:
diff --git a/.github/workflows/test-spark-integration-load-transactions-lookup.yaml b/.github/workflows/test-spark-integration-load-transactions-lookup.yaml
index 71c6359840..29d6f4505f 100644
--- a/.github/workflows/test-spark-integration-load-transactions-lookup.yaml
+++ b/.github/workflows/test-spark-integration-load-transactions-lookup.yaml
@@ -10,13 +10,14 @@ env:
   ES_SCHEME: http
   ES_HOST: usaspending-es
   ES_PORT: 9200
+  IS_LOCAL: true
   MINIO_HOST: minio
+  PROJECT_DIRECTORY: usaspending-api
   USASPENDING_DB_HOST: usaspending-db
   USASPENDING_DB_PORT: 5432
   USASPENDING_DB_USER: usaspending
   USASPENDING_DB_PASSWORD: usaspender
   USASPENDING_DB_NAME: data_store_api
-  IS_LOCAL: true
 
 on:
   workflow_call:
diff --git a/.github/workflows/test-spark-integration-other.yaml b/.github/workflows/test-spark-integration-other.yaml
index c780d93a77..e040e08712 100644
--- a/.github/workflows/test-spark-integration-other.yaml
+++ b/.github/workflows/test-spark-integration-other.yaml
@@ -10,13 +10,14 @@ env:
   ES_SCHEME: http
   ES_HOST: usaspending-es
   ES_PORT: 9200
+  IS_LOCAL: true
   MINIO_HOST: minio
+  PROJECT_DIRECTORY: usaspending-api
   USASPENDING_DB_HOST: usaspending-db
   USASPENDING_DB_PORT: 5432
   USASPENDING_DB_USER: usaspending
   USASPENDING_DB_PASSWORD: usaspender
   USASPENDING_DB_NAME: data_store_api
-  IS_LOCAL: true
 
 on:
   workflow_call:
diff --git a/.github/workflows/test-unit-signal-handling.yaml b/.github/workflows/test-unit-signal-handling.yaml
index ed9722f235..5a46a76e82 100644
--- a/.github/workflows/test-unit-signal-handling.yaml
+++ b/.github/workflows/test-unit-signal-handling.yaml
@@ -10,13 +10,14 @@ env:
   ES_SCHEME: http
   ES_HOST: usaspending-es
   ES_PORT: 9200
+  IS_LOCAL: true
   MINIO_HOST: minio
+  PROJECT_DIRECTORY: usaspending-api
   USASPENDING_DB_HOST: usaspending-db
   USASPENDING_DB_PORT: 5432
   USASPENDING_DB_USER: usaspending
   USASPENDING_DB_PASSWORD: usaspender
   USASPENDING_DB_NAME: data_store_api
-  IS_LOCAL: true
 
 on:
   workflow_call:
diff --git a/.github/workflows/test-unit.yaml b/.github/workflows/test-unit.yaml
index 6718033489..9b2cc21b35 100644
--- a/.github/workflows/test-unit.yaml
+++ b/.github/workflows/test-unit.yaml
@@ -10,13 +10,14 @@ env:
   ES_SCHEME: http
   ES_HOST: usaspending-es
   ES_PORT: 9200
+  IS_LOCAL: true
   MINIO_HOST: minio
+  PROJECT_DIRECTORY: usaspending-api
   USASPENDING_DB_HOST: usaspending-db
   USASPENDING_DB_PORT: 5432
   USASPENDING_DB_USER: usaspending
   USASPENDING_DB_PASSWORD: usaspender
   USASPENDING_DB_NAME: data_store_api
-  IS_LOCAL: true
 
 on:
   workflow_call:
diff --git a/development.Dockerfile b/development.Dockerfile
index cc49044f28..763283af69 100644
--- a/development.Dockerfile
+++ b/development.Dockerfile
@@ -62,7 +62,7 @@ ENV UV_PROJECT_ENVIRONMENT=/usr/local
 ENV UV_SYSTEM_PYTHON=1
 
 # Copy the project into the image
-COPY . /dockermount
+COPY pyproject.toml uv.lock /dockermount/
 
 # Install dev dependencies
 RUN --mount=type=cache,target=/root/.cache/uv \
diff --git a/docker-compose.yml b/docker-compose.yml
index f4893dfe8c..2cd16f313a 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -53,7 +53,7 @@ services:
       - ci
     image: postgres:16-alpine
     healthcheck:
-      test: [ "CMD-SHELL", "pg_isready -U postgres" ]
+      test: [ "CMD-SHELL", "pg_isready -U ${USASPENDING_DB_USER:-usaspending} -d ${USASPENDING_DB_NAME:-data_store_api}" ]
       interval: 5s
       timeout: 5s
       retries: 5

From 41b7364f21d822e8f6f9dd0041a69d479549ee09 Mon Sep 17 00:00:00 2001
From: Seth Stoudenmier <stoudenmier_seth@bah.com>
Date: Fri, 8 May 2026 12:39:00 -0400
Subject: [PATCH 06/48] [DEV-14608] troubleshoot github actions

---
 ...est-non-spark-integration-signal-handling.yaml |  1 -
 .github/workflows/test-non-spark-integration.yaml |  1 -
 ...test-spark-integration-load-to-from-delta.yaml |  1 -
 ...k-integration-load-transactions-fabs-fpds.yaml |  1 -
 ...park-integration-load-transactions-lookup.yaml |  1 -
 .../workflows/test-spark-integration-other.yaml   |  1 -
 .github/workflows/test-unit-signal-handling.yaml  |  1 -
 .github/workflows/test-unit.yaml                  | 15 ++++++++++++++-
 8 files changed, 14 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/test-non-spark-integration-signal-handling.yaml b/.github/workflows/test-non-spark-integration-signal-handling.yaml
index 3e65ec8a61..67d17cf251 100644
--- a/.github/workflows/test-non-spark-integration-signal-handling.yaml
+++ b/.github/workflows/test-non-spark-integration-signal-handling.yaml
@@ -12,7 +12,6 @@ env:
   ES_PORT: 9200
   IS_LOCAL: true
   MINIO_HOST: minio
-  PROJECT_DIRECTORY: usaspending-api
   USASPENDING_DB_HOST: usaspending-db
   USASPENDING_DB_PORT: 5432
   USASPENDING_DB_USER: usaspending
diff --git a/.github/workflows/test-non-spark-integration.yaml b/.github/workflows/test-non-spark-integration.yaml
index 5373fe84e2..0a861f30c7 100644
--- a/.github/workflows/test-non-spark-integration.yaml
+++ b/.github/workflows/test-non-spark-integration.yaml
@@ -12,7 +12,6 @@ env:
   ES_PORT: 9200
   IS_LOCAL: true
   MINIO_HOST: minio
-  PROJECT_DIRECTORY: usaspending-api
   USASPENDING_DB_HOST: usaspending-db
   USASPENDING_DB_PORT: 5432
   USASPENDING_DB_USER: usaspending
diff --git a/.github/workflows/test-spark-integration-load-to-from-delta.yaml b/.github/workflows/test-spark-integration-load-to-from-delta.yaml
index e140e082d3..c06730d334 100644
--- a/.github/workflows/test-spark-integration-load-to-from-delta.yaml
+++ b/.github/workflows/test-spark-integration-load-to-from-delta.yaml
@@ -12,7 +12,6 @@ env:
   ES_PORT: 9200
   IS_LOCAL: true
   MINIO_HOST: minio
-  PROJECT_DIRECTORY: usaspending-api
   USASPENDING_DB_HOST: usaspending-db
   USASPENDING_DB_PORT: 5432
   USASPENDING_DB_USER: usaspending
diff --git a/.github/workflows/test-spark-integration-load-transactions-fabs-fpds.yaml b/.github/workflows/test-spark-integration-load-transactions-fabs-fpds.yaml
index 0232c70e8b..4a4cc289c9 100644
--- a/.github/workflows/test-spark-integration-load-transactions-fabs-fpds.yaml
+++ b/.github/workflows/test-spark-integration-load-transactions-fabs-fpds.yaml
@@ -12,7 +12,6 @@ env:
   ES_PORT: 9200
   IS_LOCAL: true
   MINIO_HOST: minio
-  PROJECT_DIRECTORY: usaspending-api
   USASPENDING_DB_HOST: usaspending-db
   USASPENDING_DB_PORT: 5432
   USASPENDING_DB_USER: usaspending
diff --git a/.github/workflows/test-spark-integration-load-transactions-lookup.yaml b/.github/workflows/test-spark-integration-load-transactions-lookup.yaml
index 29d6f4505f..da361879be 100644
--- a/.github/workflows/test-spark-integration-load-transactions-lookup.yaml
+++ b/.github/workflows/test-spark-integration-load-transactions-lookup.yaml
@@ -12,7 +12,6 @@ env:
   ES_PORT: 9200
   IS_LOCAL: true
   MINIO_HOST: minio
-  PROJECT_DIRECTORY: usaspending-api
   USASPENDING_DB_HOST: usaspending-db
   USASPENDING_DB_PORT: 5432
   USASPENDING_DB_USER: usaspending
diff --git a/.github/workflows/test-spark-integration-other.yaml b/.github/workflows/test-spark-integration-other.yaml
index e040e08712..e48a22f96c 100644
--- a/.github/workflows/test-spark-integration-other.yaml
+++ b/.github/workflows/test-spark-integration-other.yaml
@@ -12,7 +12,6 @@ env:
   ES_PORT: 9200
   IS_LOCAL: true
   MINIO_HOST: minio
-  PROJECT_DIRECTORY: usaspending-api
   USASPENDING_DB_HOST: usaspending-db
   USASPENDING_DB_PORT: 5432
   USASPENDING_DB_USER: usaspending
diff --git a/.github/workflows/test-unit-signal-handling.yaml b/.github/workflows/test-unit-signal-handling.yaml
index 5a46a76e82..087cdeea51 100644
--- a/.github/workflows/test-unit-signal-handling.yaml
+++ b/.github/workflows/test-unit-signal-handling.yaml
@@ -12,7 +12,6 @@ env:
   ES_PORT: 9200
   IS_LOCAL: true
   MINIO_HOST: minio
-  PROJECT_DIRECTORY: usaspending-api
   USASPENDING_DB_HOST: usaspending-db
   USASPENDING_DB_PORT: 5432
   USASPENDING_DB_USER: usaspending
diff --git a/.github/workflows/test-unit.yaml b/.github/workflows/test-unit.yaml
index 9b2cc21b35..3322eb5e20 100644
--- a/.github/workflows/test-unit.yaml
+++ b/.github/workflows/test-unit.yaml
@@ -12,7 +12,6 @@ env:
   ES_PORT: 9200
   IS_LOCAL: true
   MINIO_HOST: minio
-  PROJECT_DIRECTORY: usaspending-api
   USASPENDING_DB_HOST: usaspending-db
   USASPENDING_DB_PORT: 5432
   USASPENDING_DB_USER: usaspending
@@ -32,6 +31,13 @@ jobs:
         with:
           path: usaspending-api
 
+      - name: Validate file structure
+        working-directory: ./usaspending-api
+        shell: bash
+        run: |
+          pwd
+          ls
+
       - name: Init Test Environment
         uses: ./usaspending-api/.github/actions/init-test-environment
         with:
@@ -39,6 +45,13 @@ jobs:
           is-spark-test: false
           working-directory: ./usaspending-api
 
+      - name: Validate file structure
+        working-directory: ./usaspending-api
+        shell: bash
+        run: |
+          pwd
+          ls
+
       - name: Run Test Cases
         id: run-test-cases
         uses: ./usaspending-api/.github/actions/run-pytest

From 21d06f3a255e2925987a47170571524b807349cd Mon Sep 17 00:00:00 2001
From: Seth Stoudenmier <stoudenmier_seth@bah.com>
Date: Fri, 8 May 2026 13:00:35 -0400
Subject: [PATCH 07/48] [DEV-14608] breakout creation of broker branch

---
 .../build-broker-docker-image-for-test.yaml   | 11 +++----
 .github/workflows/pull-request-checks.yaml    | 29 ++++++++++++++++++-
 ...non-spark-integration-signal-handling.yaml | 11 ++++---
 .../workflows/test-non-spark-integration.yaml | 13 ++++++---
 ...-spark-integration-load-to-from-delta.yaml |  9 +++---
 ...tegration-load-transactions-fabs-fpds.yaml |  9 +++---
 ...-integration-load-transactions-lookup.yaml |  9 +++---
 .../test-spark-integration-other.yaml         |  9 +++---
 8 files changed, 66 insertions(+), 34 deletions(-)

diff --git a/.github/workflows/build-broker-docker-image-for-test.yaml b/.github/workflows/build-broker-docker-image-for-test.yaml
index a089e97546..16ebe86960 100644
--- a/.github/workflows/build-broker-docker-image-for-test.yaml
+++ b/.github/workflows/build-broker-docker-image-for-test.yaml
@@ -2,16 +2,13 @@ name: Build Broker Docker Image for Test
 
 on:
   workflow_call:
+    broker-branch:
+      required: true
+      type: string
 
 jobs:
-  # This job is a workaround that allows defining a dynamic variable that is provided as input
-  # to another job. See here: https://github.com/orgs/community/discussions/26671#discussioncomment-3931817
-  Setup-Broker-Branch:
-    uses: ./.github/workflows/determine-broker-branch.yaml
-
   Run:
     name: Run
-    needs: Setup-Broker-Branch
     runs-on: ${{ vars.RUNNER_VERSION }}
     steps:
       - name: Checkout Broker Backend
@@ -19,7 +16,7 @@ jobs:
         with:
           repository: fedspendingtransparency/data-act-broker-backend
           path: data-act-broker-backend
-          ref: ${{ needs.Setup-Broker-Branch.outputs.branch }}
+          ref: ${{ inputs.broker-branch }}
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
diff --git a/.github/workflows/pull-request-checks.yaml b/.github/workflows/pull-request-checks.yaml
index 04c83ecf09..f7681937ab 100644
--- a/.github/workflows/pull-request-checks.yaml
+++ b/.github/workflows/pull-request-checks.yaml
@@ -33,12 +33,21 @@ jobs:
       needs.Cancel-If-Draft-Pull-Request.result == 'skipped'
     uses: ./.github/workflows/code-style-checks.yaml
 
+  # Determine the Broker branch to use for tests and add it as output to the workflow for consumption later
+  Setup-Broker-Branch:
+    name: Setup Broker Branch
+    needs: Cancel-If-Draft-Pull-Request
+    if: |
+      always() &&
+      needs.Cancel-If-Draft-Pull-Request.result == 'skipped'
+    uses: ./.github/workflows/determine-broker-branch.yaml
+
   # Many of the integration tests below rely on testing against Broker which requires building the
   # Broker backend image. To avoid having to run this multiple times we instead build it once
   # and store it as an artifact on the workflow.
   Build-Broker-Docker-Image:
     name: Build Broker Docker Image
-    needs: Cancel-If-Draft-Pull-Request
+    needs: Setup-Broker-Branch
     if: |
       always() &&
       needs.Cancel-If-Draft-Pull-Request.result == 'skipped'
@@ -50,49 +59,67 @@ jobs:
     name: Run Spark Integration Load Transactions FABS and FPDS Tests
     needs:
       - Run-Code-Style-Checks
+      - Setup-Broker-Branch
       - Build-Broker-Docker-Image
     if: ${{ !failure() && !cancelled() }}
     uses: ./.github/workflows/test-spark-integration-load-transactions-fabs-fpds.yaml
+    with:
+      broker-branch: ${{ needs.Setup-Broker-Branch.outputs.branch }}
 
   Run-Spark-Integration-Load-Transactions-Lookup-Tests:
     name: Run Spark Integration Load Transactions Lookup Tests
     needs:
       - Run-Code-Style-Checks
+      - Setup-Broker-Branch
       - Build-Broker-Docker-Image
     if: ${{ !failure() && !cancelled() }}
     uses: ./.github/workflows/test-spark-integration-load-transactions-lookup.yaml
+    with:
+      broker-branch: ${{ needs.Setup-Broker-Branch.outputs.branch }}
 
   Run-Spark-Integration-Load-To-From-Delta-Tests:
     name: Run Spark Integration Load To From Delta Tests
     needs:
       - Run-Code-Style-Checks
+      - Setup-Broker-Branch
       - Build-Broker-Docker-Image
     if: ${{ !failure() && !cancelled() }}
     uses: ./.github/workflows/test-spark-integration-load-to-from-delta.yaml
+    with:
+      broker-branch: ${{ needs.Setup-Broker-Branch.outputs.branch }}
 
   Run-Spark-Integration-Other-Tests:
     name: Run Spark Integration Other Tests
     needs:
       - Run-Code-Style-Checks
+      - Setup-Broker-Branch
       - Build-Broker-Docker-Image
     if: ${{ !failure() && !cancelled() }}
     uses: ./.github/workflows/test-spark-integration-other.yaml
+    with:
+      broker-branch: ${{ needs.Setup-Broker-Branch.outputs.branch }}
 
   Run-Non-Spark-Integration-Tests:
     name: Run Non-Spark Integration Tests
     needs:
       - Run-Code-Style-Checks
+      - Setup-Broker-Branch
       - Build-Broker-Docker-Image
     if: ${{ !failure() && !cancelled() }}
     uses: ./.github/workflows/test-non-spark-integration.yaml
+    with:
+      broker-branch: ${{ needs.Setup-Broker-Branch.outputs.branch }}
 
   Run-Non-Spark-Integration-Tests-With-Signal-Handling:
     name: Run Non-Spark Integration Tests With Signal Handling
     needs:
       - Run-Code-Style-Checks
+      - Setup-Broker-Branch
       - Build-Broker-Docker-Image
     if: ${{ !failure() && !cancelled() }}
     uses: ./.github/workflows/test-non-spark-integration-signal-handling.yaml
+    with:
+      broker-branch: ${{ needs.Setup-Broker-Branch.outputs.branch }}
 
   Run-Unit-Tests:
     name: Run Unit Tests
diff --git a/.github/workflows/test-non-spark-integration-signal-handling.yaml b/.github/workflows/test-non-spark-integration-signal-handling.yaml
index 67d17cf251..fd20a07377 100644
--- a/.github/workflows/test-non-spark-integration-signal-handling.yaml
+++ b/.github/workflows/test-non-spark-integration-signal-handling.yaml
@@ -20,18 +20,18 @@ env:
 
 on:
   workflow_call:
+    inputs:
+      broker-branch:
+        required: true
+        type: string
 
 defaults:
   run:
     working-directory: usaspending-api
 
 jobs:
-  Setup-Broker-Branch:
-    uses: ./.github/workflows/determine-broker-branch.yaml
-
   Run:
     name: Run
-    needs: Setup-Broker-Branch
     runs-on: ${{ vars.RUNNER_VERSION }}
     steps:
       - name: Checkout Source Repository
@@ -44,8 +44,7 @@ jobs:
         with:
           repository: fedspendingtransparency/data-act-broker-backend
           path: data-act-broker-backend
-          ref: ${{ needs.Setup-Broker-Branch.outputs.branch }}
-
+          ref: ${{ inputs.broker-branch }}
 
       - name: Init Test Environment
         uses: ./usaspending-api/.github/actions/init-test-environment
diff --git a/.github/workflows/test-non-spark-integration.yaml b/.github/workflows/test-non-spark-integration.yaml
index 0a861f30c7..cd30116025 100644
--- a/.github/workflows/test-non-spark-integration.yaml
+++ b/.github/workflows/test-non-spark-integration.yaml
@@ -20,19 +20,24 @@ env:
 
 on:
   workflow_call:
+    inputs:
+      broker-branch:
+        required: true
+        type: string
 
 defaults:
   run:
     working-directory: usaspending-api
 
 jobs:
-  Setup-Broker-Branch:
-    uses: ./.github/workflows/determine-broker-branch.yaml
-
   Run:
     name: Run
     runs-on: ${{ vars.RUNNER_VERSION }}
     steps:
+      - name: Placeholder test broke branch
+        shell: bash
+        run: echo ${{ inputs.broker-branch }}
+
       - name: Checkout Source Repository
         uses: actions/checkout@v4
         with:
@@ -43,7 +48,7 @@ jobs:
         with:
           repository: fedspendingtransparency/data-act-broker-backend
           path: data-act-broker-backend
-          ref: ${{ needs.Setup-Broker-Branch.outputs.branch }}
+          ref: ${{ inputs.broker-branch }}
 
       - name: Init Test Environment
         uses: ./usaspending-api/.github/actions/init-test-environment
diff --git a/.github/workflows/test-spark-integration-load-to-from-delta.yaml b/.github/workflows/test-spark-integration-load-to-from-delta.yaml
index c06730d334..2426fe1f03 100644
--- a/.github/workflows/test-spark-integration-load-to-from-delta.yaml
+++ b/.github/workflows/test-spark-integration-load-to-from-delta.yaml
@@ -20,15 +20,16 @@ env:
 
 on:
   workflow_call:
+    inputs:
+      broker-branch:
+        required: true
+        type: string
 
 defaults:
   run:
     working-directory: ./usaspending-api
 
 jobs:
-  Setup-Broker-Branch:
-    uses: ./.github/workflows/determine-broker-branch.yaml
-
   Run:
     name: Run
     runs-on: ${{ vars.RUNNER_VERSION }}
@@ -43,7 +44,7 @@ jobs:
         with:
           repository: fedspendingtransparency/data-act-broker-backend
           path: data-act-broker-backend
-          ref: ${{ needs.Setup-Broker-Branch.outputs.branch }}
+          ref: ${{ inputs.broker-branch }}
 
       - name: Init Test Environment
         uses: ./usaspending-api/.github/actions/init-test-environment
diff --git a/.github/workflows/test-spark-integration-load-transactions-fabs-fpds.yaml b/.github/workflows/test-spark-integration-load-transactions-fabs-fpds.yaml
index 4a4cc289c9..10febb2f57 100644
--- a/.github/workflows/test-spark-integration-load-transactions-fabs-fpds.yaml
+++ b/.github/workflows/test-spark-integration-load-transactions-fabs-fpds.yaml
@@ -20,15 +20,16 @@ env:
 
 on:
   workflow_call:
+    inputs:
+      broker-branch:
+        required: true
+        type: string
 
 defaults:
   run:
     working-directory: ./usaspending-api
 
 jobs:
-  Setup-Broker-Branch:
-    uses: ./.github/workflows/determine-broker-branch.yaml
-
   Run:
     name: Run
     runs-on: ${{ vars.RUNNER_VERSION }}
@@ -43,7 +44,7 @@ jobs:
         with:
           repository: fedspendingtransparency/data-act-broker-backend
           path: data-act-broker-backend
-          ref: ${{ needs.Setup-Broker-Branch.outputs.branch }}
+          ref: ${{ inputs.broker-branch }}
 
       - name: Init Test Environment
         uses: ./usaspending-api/.github/actions/init-test-environment
diff --git a/.github/workflows/test-spark-integration-load-transactions-lookup.yaml b/.github/workflows/test-spark-integration-load-transactions-lookup.yaml
index da361879be..2c0d061fc4 100644
--- a/.github/workflows/test-spark-integration-load-transactions-lookup.yaml
+++ b/.github/workflows/test-spark-integration-load-transactions-lookup.yaml
@@ -20,15 +20,16 @@ env:
 
 on:
   workflow_call:
+    inputs:
+      broker-branch:
+        required: true
+        type: string
 
 defaults:
   run:
     working-directory: ./usaspending-api
 
 jobs:
-  Setup-Broker-Branch:
-    uses: ./.github/workflows/determine-broker-branch.yaml
-
   Run:
     name: Run
     runs-on: ${{ vars.RUNNER_VERSION }}
@@ -43,7 +44,7 @@ jobs:
         with:
           repository: fedspendingtransparency/data-act-broker-backend
           path: data-act-broker-backend
-          ref: ${{ needs.Setup-Broker-Branch.outputs.branch }}
+          ref: ${{ inputs.broker-branch }}
 
       - name: Init Test Environment
         uses: ./usaspending-api/.github/actions/init-test-environment
diff --git a/.github/workflows/test-spark-integration-other.yaml b/.github/workflows/test-spark-integration-other.yaml
index e48a22f96c..bcd3960e89 100644
--- a/.github/workflows/test-spark-integration-other.yaml
+++ b/.github/workflows/test-spark-integration-other.yaml
@@ -20,15 +20,16 @@ env:
 
 on:
   workflow_call:
+    inputs:
+      broker-branch:
+        required: true
+        type: string
 
 defaults:
   run:
     working-directory: ./usaspending-api
 
 jobs:
-  Setup-Broker-Branch:
-    uses: ./.github/workflows/determine-broker-branch.yaml
-
   Run:
     name: Run
     runs-on: ${{ vars.RUNNER_VERSION }}
@@ -43,7 +44,7 @@ jobs:
         with:
           repository: fedspendingtransparency/data-act-broker-backend
           path: data-act-broker-backend
-          ref: ${{ needs.Setup-Broker-Branch.outputs.branch }}
+          ref: ${{ inputs.broker-branch }}
 
       - name: Init Test Environment
         uses: ./usaspending-api/.github/actions/init-test-environment

From 3d2ec0214df61a536414c6806176cb92a96f27eb Mon Sep 17 00:00:00 2001
From: Seth Stoudenmier <stoudenmier_seth@bah.com>
Date: Fri, 8 May 2026 13:02:21 -0400
Subject: [PATCH 08/48] [DEV-14608] cleanup

---
 .github/workflows/build-broker-docker-image-for-test.yaml | 7 ++++---
 .github/workflows/pull-request-checks.yaml                | 2 ++
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/build-broker-docker-image-for-test.yaml b/.github/workflows/build-broker-docker-image-for-test.yaml
index 16ebe86960..88f356301d 100644
--- a/.github/workflows/build-broker-docker-image-for-test.yaml
+++ b/.github/workflows/build-broker-docker-image-for-test.yaml
@@ -2,9 +2,10 @@ name: Build Broker Docker Image for Test
 
 on:
   workflow_call:
-    broker-branch:
-      required: true
-      type: string
+    inputs:
+      broker-branch:
+        required: true
+        type: string
 
 jobs:
   Run:
diff --git a/.github/workflows/pull-request-checks.yaml b/.github/workflows/pull-request-checks.yaml
index f7681937ab..81e360fbf1 100644
--- a/.github/workflows/pull-request-checks.yaml
+++ b/.github/workflows/pull-request-checks.yaml
@@ -52,6 +52,8 @@ jobs:
       always() &&
       needs.Cancel-If-Draft-Pull-Request.result == 'skipped'
     uses: ./.github/workflows/build-broker-docker-image-for-test.yaml
+    with:
+      broker-branch: ${{ needs.Setup-Broker-Branch.outputs.branch }}
 
   # The test suite is broken into multiple groups based on a combination of what is under test
   # and the time that it takes to run tests for that group.

From 3479f355c74f04e657e77c3a9db88bf05313b732 Mon Sep 17 00:00:00 2001
From: Seth Stoudenmier <stoudenmier_seth@bah.com>
Date: Fri, 8 May 2026 13:28:32 -0400
Subject: [PATCH 09/48] [DEV-14608] troubleshoot github actions

---
 .../test-non-spark-integration-signal-handling.yaml      | 3 +++
 .github/workflows/test-non-spark-integration.yaml        | 7 +++----
 .../test-spark-integration-load-to-from-delta.yaml       | 3 +++
 ...st-spark-integration-load-transactions-fabs-fpds.yaml | 3 +++
 .../test-spark-integration-load-transactions-lookup.yaml | 3 +++
 .github/workflows/test-spark-integration-other.yaml      | 3 +++
 .github/workflows/test-unit-signal-handling.yaml         | 3 +++
 .github/workflows/test-unit.yaml                         | 9 ++++++++-
 docker-compose.yml                                       | 2 +-
 9 files changed, 30 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/test-non-spark-integration-signal-handling.yaml b/.github/workflows/test-non-spark-integration-signal-handling.yaml
index fd20a07377..1ddec1e053 100644
--- a/.github/workflows/test-non-spark-integration-signal-handling.yaml
+++ b/.github/workflows/test-non-spark-integration-signal-handling.yaml
@@ -12,6 +12,9 @@ env:
   ES_PORT: 9200
   IS_LOCAL: true
   MINIO_HOST: minio
+  MINIO_PORT: 10001
+  MINIO_ACCESS_KEY: usaspending
+  MINIO_SECRET_KEY: usaspender
   USASPENDING_DB_HOST: usaspending-db
   USASPENDING_DB_PORT: 5432
   USASPENDING_DB_USER: usaspending
diff --git a/.github/workflows/test-non-spark-integration.yaml b/.github/workflows/test-non-spark-integration.yaml
index cd30116025..b9b145d295 100644
--- a/.github/workflows/test-non-spark-integration.yaml
+++ b/.github/workflows/test-non-spark-integration.yaml
@@ -12,6 +12,9 @@ env:
   ES_PORT: 9200
   IS_LOCAL: true
   MINIO_HOST: minio
+  MINIO_PORT: 10001
+  MINIO_ACCESS_KEY: usaspending
+  MINIO_SECRET_KEY: usaspender
   USASPENDING_DB_HOST: usaspending-db
   USASPENDING_DB_PORT: 5432
   USASPENDING_DB_USER: usaspending
@@ -34,10 +37,6 @@ jobs:
     name: Run
     runs-on: ${{ vars.RUNNER_VERSION }}
     steps:
-      - name: Placeholder test broke branch
-        shell: bash
-        run: echo ${{ inputs.broker-branch }}
-
       - name: Checkout Source Repository
         uses: actions/checkout@v4
         with:
diff --git a/.github/workflows/test-spark-integration-load-to-from-delta.yaml b/.github/workflows/test-spark-integration-load-to-from-delta.yaml
index 2426fe1f03..d4be24c943 100644
--- a/.github/workflows/test-spark-integration-load-to-from-delta.yaml
+++ b/.github/workflows/test-spark-integration-load-to-from-delta.yaml
@@ -12,6 +12,9 @@ env:
   ES_PORT: 9200
   IS_LOCAL: true
   MINIO_HOST: minio
+  MINIO_PORT: 10001
+  MINIO_ACCESS_KEY: usaspending
+  MINIO_SECRET_KEY: usaspender
   USASPENDING_DB_HOST: usaspending-db
   USASPENDING_DB_PORT: 5432
   USASPENDING_DB_USER: usaspending
diff --git a/.github/workflows/test-spark-integration-load-transactions-fabs-fpds.yaml b/.github/workflows/test-spark-integration-load-transactions-fabs-fpds.yaml
index 10febb2f57..1ac26e3765 100644
--- a/.github/workflows/test-spark-integration-load-transactions-fabs-fpds.yaml
+++ b/.github/workflows/test-spark-integration-load-transactions-fabs-fpds.yaml
@@ -12,6 +12,9 @@ env:
   ES_PORT: 9200
   IS_LOCAL: true
   MINIO_HOST: minio
+  MINIO_PORT: 10001
+  MINIO_ACCESS_KEY: usaspending
+  MINIO_SECRET_KEY: usaspender
   USASPENDING_DB_HOST: usaspending-db
   USASPENDING_DB_PORT: 5432
   USASPENDING_DB_USER: usaspending
diff --git a/.github/workflows/test-spark-integration-load-transactions-lookup.yaml b/.github/workflows/test-spark-integration-load-transactions-lookup.yaml
index 2c0d061fc4..bc3af98b3f 100644
--- a/.github/workflows/test-spark-integration-load-transactions-lookup.yaml
+++ b/.github/workflows/test-spark-integration-load-transactions-lookup.yaml
@@ -12,6 +12,9 @@ env:
   ES_PORT: 9200
   IS_LOCAL: true
   MINIO_HOST: minio
+  MINIO_PORT: 10001
+  MINIO_ACCESS_KEY: usaspending
+  MINIO_SECRET_KEY: usaspender
   USASPENDING_DB_HOST: usaspending-db
   USASPENDING_DB_PORT: 5432
   USASPENDING_DB_USER: usaspending
diff --git a/.github/workflows/test-spark-integration-other.yaml b/.github/workflows/test-spark-integration-other.yaml
index bcd3960e89..a6a9cb95c2 100644
--- a/.github/workflows/test-spark-integration-other.yaml
+++ b/.github/workflows/test-spark-integration-other.yaml
@@ -12,6 +12,9 @@ env:
   ES_PORT: 9200
   IS_LOCAL: true
   MINIO_HOST: minio
+  MINIO_PORT: 10001
+  MINIO_ACCESS_KEY: usaspending
+  MINIO_SECRET_KEY: usaspender
   USASPENDING_DB_HOST: usaspending-db
   USASPENDING_DB_PORT: 5432
   USASPENDING_DB_USER: usaspending
diff --git a/.github/workflows/test-unit-signal-handling.yaml b/.github/workflows/test-unit-signal-handling.yaml
index 087cdeea51..e06a828e31 100644
--- a/.github/workflows/test-unit-signal-handling.yaml
+++ b/.github/workflows/test-unit-signal-handling.yaml
@@ -12,6 +12,9 @@ env:
   ES_PORT: 9200
   IS_LOCAL: true
   MINIO_HOST: minio
+  MINIO_PORT: 10001
+  MINIO_ACCESS_KEY: usaspending
+  MINIO_SECRET_KEY: usaspender
   USASPENDING_DB_HOST: usaspending-db
   USASPENDING_DB_PORT: 5432
   USASPENDING_DB_USER: usaspending
diff --git a/.github/workflows/test-unit.yaml b/.github/workflows/test-unit.yaml
index 3322eb5e20..236a7029be 100644
--- a/.github/workflows/test-unit.yaml
+++ b/.github/workflows/test-unit.yaml
@@ -12,6 +12,9 @@ env:
   ES_PORT: 9200
   IS_LOCAL: true
   MINIO_HOST: minio
+  MINIO_PORT: 10001
+  MINIO_ACCESS_KEY: usaspending
+  MINIO_SECRET_KEY: usaspender
   USASPENDING_DB_HOST: usaspending-db
   USASPENDING_DB_PORT: 5432
   USASPENDING_DB_USER: usaspending
@@ -45,12 +48,16 @@ jobs:
           is-spark-test: false
           working-directory: ./usaspending-api
 
-      - name: Validate file structure
+      - name: Troubleshoot
         working-directory: ./usaspending-api
         shell: bash
         run: |
           pwd
           ls
+          docker compose run --rm --entrypoint "python -c 'import sys; print(sys.path)'" usaspending-test
+          docker compose run --rm --entrypoint "pwd" usaspending-test
+          docker compose run --rm --entrypoint "ls" usaspending-test
+          docker compose run --rm --entrypoint "env" usaspending-test
 
       - name: Run Test Cases
         id: run-test-cases
diff --git a/docker-compose.yml b/docker-compose.yml
index 2cd16f313a..bfe9ba1729 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -22,7 +22,7 @@ x-application-variables: &application-vars
 x-spark-submit-variables: &spark-submit-vars
   <<: *application-vars
   AWS_ACCESS_KEY_ID: ${MINIO_ACCESS_KEY}
-  AWS_ENDPOINT_URL: ${MINIO_HOSTNAME}
+  AWS_ENDPOINT_URL: http://${MINIO_HOST}:{MINIO_PORT}
   AWS_SECRET_ACCESS_KEY: ${MINIO_SECRET_KEY}
   PYTHONPATH: "/dockermount"
   #SPARK_HIVE_METASTORE_DB_TYPE: derby

From c7ac20df5b6b2b8172fc69be922b071ef89a0dd3 Mon Sep 17 00:00:00 2001
From: Seth Stoudenmier <stoudenmier_seth@bah.com>
Date: Fri, 8 May 2026 14:45:26 -0400
Subject: [PATCH 10/48] [DEV-14608] troubleshoot github actions

---
 .github/actions/init-test-environment/action.yaml | 4 ++--
 .github/actions/run-pytest/action.yaml            | 6 +++---
 .github/workflows/test-unit.yaml                  | 1 -
 3 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/.github/actions/init-test-environment/action.yaml b/.github/actions/init-test-environment/action.yaml
index 31b4ea4aa7..c8fe163665 100644
--- a/.github/actions/init-test-environment/action.yaml
+++ b/.github/actions/init-test-environment/action.yaml
@@ -34,7 +34,7 @@ runs:
       working-directory: ${{ inputs.working-directory }}
       if: ${{ inputs.is-integration-test == 'true' }}
       shell: bash
-      run: docker compose up -d usaspending-db usaspending-es minio
+      run: docker compose --project-directory ${{ inputs.working-directory }} up -d usaspending-db usaspending-es minio
 
     - name: Download broker docker image
       if: ${{ inputs.is-integration-test == 'true' }}
@@ -68,7 +68,7 @@ runs:
     - name: Build the usaspending-development image
       working-directory: ${{ inputs.working-directory }}
       shell: bash
-      run: docker compose build usaspending-test
+      run: docker compose --project-directory ${{ inputs.working-directory }} build usaspending-test
 
     - name: Run initial test to trigger DB creation
       if: ${{ inputs.is-integration-test == 'true' }}
diff --git a/.github/actions/run-pytest/action.yaml b/.github/actions/run-pytest/action.yaml
index 3689a220f2..dd387ee5c2 100644
--- a/.github/actions/run-pytest/action.yaml
+++ b/.github/actions/run-pytest/action.yaml
@@ -42,10 +42,10 @@ runs:
 
     - name: Build pytest args
       working-directory: ${{ inputs.working-directory }}
-      id: build-pytest-command
+      id: build-pytest-args
       shell: bash
       run: >
-        echo pytest-command=$(python -c 'import os;print(
+        echo pytest-args=$(python -c 'import os;print(
         "-r=fEs
         -x
         --cov=usaspending_api
@@ -73,7 +73,7 @@ runs:
     - name: Run tests
       working-directory: ${{ inputs.working-directory }}
       shell: bash
-      run: docker compose run --rm usaspending-test ${{ steps.build-pytest-command.outputs.pytest-command }}
+      run: docker compose --project-directory ${{ inputs.working-directory }} run --rm usaspending-test ${{ steps.build-pytest-args.outputs.pytest-args }}
 
     - name: Upload Artifact
       uses: actions/upload-artifact@v4
diff --git a/.github/workflows/test-unit.yaml b/.github/workflows/test-unit.yaml
index 236a7029be..eac446f93b 100644
--- a/.github/workflows/test-unit.yaml
+++ b/.github/workflows/test-unit.yaml
@@ -54,7 +54,6 @@ jobs:
         run: |
           pwd
           ls
-          docker compose run --rm --entrypoint "python -c 'import sys; print(sys.path)'" usaspending-test
           docker compose run --rm --entrypoint "pwd" usaspending-test
           docker compose run --rm --entrypoint "ls" usaspending-test
           docker compose run --rm --entrypoint "env" usaspending-test

From d6caa91dd7b717a9b70912140dae3fde0f0aad9b Mon Sep 17 00:00:00 2001
From: Seth Stoudenmier <stoudenmier_seth@bah.com>
Date: Fri, 8 May 2026 15:11:24 -0400
Subject: [PATCH 11/48] [DEV-14608] troubleshoot github actions

---
 .github/actions/init-test-environment/action.yaml | 4 ++--
 .github/actions/run-pytest/action.yaml            | 2 +-
 .github/workflows/test-unit.yaml                  | 7 +++++--
 3 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/.github/actions/init-test-environment/action.yaml b/.github/actions/init-test-environment/action.yaml
index c8fe163665..31b4ea4aa7 100644
--- a/.github/actions/init-test-environment/action.yaml
+++ b/.github/actions/init-test-environment/action.yaml
@@ -34,7 +34,7 @@ runs:
       working-directory: ${{ inputs.working-directory }}
       if: ${{ inputs.is-integration-test == 'true' }}
       shell: bash
-      run: docker compose --project-directory ${{ inputs.working-directory }} up -d usaspending-db usaspending-es minio
+      run: docker compose up -d usaspending-db usaspending-es minio
 
     - name: Download broker docker image
       if: ${{ inputs.is-integration-test == 'true' }}
@@ -68,7 +68,7 @@ runs:
     - name: Build the usaspending-development image
       working-directory: ${{ inputs.working-directory }}
       shell: bash
-      run: docker compose --project-directory ${{ inputs.working-directory }} build usaspending-test
+      run: docker compose build usaspending-test
 
     - name: Run initial test to trigger DB creation
       if: ${{ inputs.is-integration-test == 'true' }}
diff --git a/.github/actions/run-pytest/action.yaml b/.github/actions/run-pytest/action.yaml
index dd387ee5c2..46388b7b8b 100644
--- a/.github/actions/run-pytest/action.yaml
+++ b/.github/actions/run-pytest/action.yaml
@@ -73,7 +73,7 @@ runs:
     - name: Run tests
       working-directory: ${{ inputs.working-directory }}
       shell: bash
-      run: docker compose --project-directory ${{ inputs.working-directory }} run --rm usaspending-test ${{ steps.build-pytest-args.outputs.pytest-args }}
+      run: docker compose run --rm usaspending-test ${{ steps.build-pytest-args.outputs.pytest-args }}
 
     - name: Upload Artifact
       uses: actions/upload-artifact@v4
diff --git a/.github/workflows/test-unit.yaml b/.github/workflows/test-unit.yaml
index eac446f93b..66f62f9721 100644
--- a/.github/workflows/test-unit.yaml
+++ b/.github/workflows/test-unit.yaml
@@ -15,6 +15,7 @@ env:
   MINIO_PORT: 10001
   MINIO_ACCESS_KEY: usaspending
   MINIO_SECRET_KEY: usaspender
+  PROJECT_LOG_DIR: ./usaspending_api/logs
   USASPENDING_DB_HOST: usaspending-db
   USASPENDING_DB_PORT: 5432
   USASPENDING_DB_USER: usaspending
@@ -24,6 +25,10 @@ env:
 on:
   workflow_call:
 
+defaults:
+  run:
+    working-directory: usaspending-api
+
 jobs:
   Run:
     name: Run
@@ -35,7 +40,6 @@ jobs:
           path: usaspending-api
 
       - name: Validate file structure
-        working-directory: ./usaspending-api
         shell: bash
         run: |
           pwd
@@ -49,7 +53,6 @@ jobs:
           working-directory: ./usaspending-api
 
       - name: Troubleshoot
-        working-directory: ./usaspending-api
         shell: bash
         run: |
           pwd

From 6b9fd455f9d9142b665f772b6c9ca0186f454ae5 Mon Sep 17 00:00:00 2001
From: Seth Stoudenmier <stoudenmier_seth@bah.com>
Date: Mon, 11 May 2026 15:53:42 -0400
Subject: [PATCH 12/48] [DEV-14608] troubleshoot failing tests and github
 actions

---
 .env.template                                 |  5 ++-
 .../init-python-environment/action.yaml       | 24 ----------
 .../actions/init-test-environment/action.yaml | 17 +++----
 ...non-spark-integration-signal-handling.yaml | 21 ---------
 .../workflows/test-non-spark-integration.yaml | 21 ---------
 ...-spark-integration-load-to-from-delta.yaml | 21 ---------
 ...tegration-load-transactions-fabs-fpds.yaml | 21 ---------
 ...-integration-load-transactions-lookup.yaml | 21 ---------
 .../test-spark-integration-other.yaml         | 21 ---------
 .../workflows/test-unit-signal-handling.yaml  | 21 ---------
 .github/workflows/test-unit.yaml              | 22 ----------
 development.Dockerfile                        | 44 ++++++-------------
 docker-compose.yml                            | 43 +++++++++++++++---
 13 files changed, 61 insertions(+), 241 deletions(-)
 delete mode 100644 .github/actions/init-python-environment/action.yaml

diff --git a/.env.template b/.env.template
index fc1bb7103c..ec9a53c7c4 100644
--- a/.env.template
+++ b/.env.template
@@ -21,11 +21,12 @@ PROJECT_LOG_DIR=./usaspending_api/logs
 
 # ==== [Local Development] ====
 IS_LOCAL=True
-RUN_LOCAL_DOWNLOAD_IN_PROCESS=False
+RUN_LOCAL_DOWNLOAD_IN_PROCESS=True
 
 # ==== [Django] ====
 # Use env var to toggle Django's DEBUG mode for local docker envs
 DJANGO_DEBUG=False
+DJANGO_SETTINGS_MODULE=usaspending_api.settings
 
 # ==== [Postgres] ====
 # All values of USASPENDING_DB_* must match what is in DATABASE_URL if BOTH are given
@@ -37,6 +38,7 @@ USASPENDING_DB_HOST=usaspending-db
 USASPENDING_DB_PORT=5432
 USASPENDING_DB_USER=usaspending
 USASPENDING_DB_PASSWORD=usaspender
+USASPENDING_DB_NAME=data_store_api
 
 # The Broker configuration below supports tests creating a Broker DB on the usaspending-db
 # container as part of standing up the test suite.
@@ -48,6 +50,7 @@ BROKER_DB_HOST=usaspending-db
 BROKER_DB_PORT=5432
 BROKER_DB_USER=usaspending
 BROKER_DB_PASSWORD=usaspender
+BROKER_DB_NAME=data_broker
 
 # ==== [Elasticsearch] ====
 # Where to connect to elasticsearch.
diff --git a/.github/actions/init-python-environment/action.yaml b/.github/actions/init-python-environment/action.yaml
deleted file mode 100644
index 9e77fe4618..0000000000
--- a/.github/actions/init-python-environment/action.yaml
+++ /dev/null
@@ -1,24 +0,0 @@
-name: Initialize Python Environment
-
-inputs:
-  python-version:
-    description: Version of Python to use in the environment and install dependencies with
-    default: "3.10.12"
-  working-directory:
-    description: Directory where the requirements can be found; used when multiple repos are checked out
-    default: ""
-
-runs:
-  using: composite
-  steps:
-    - name: Install Python
-      uses: actions/setup-python@v5
-      with:
-        python-version: ${{ inputs.python-version }}
-        cache: "pip"
-        cache-dependency-path: setup.py
-
-    - name: Install dev dependencies
-      working-directory: ${{ inputs.working-directory }}
-      shell: bash
-      run: pip install ".[dev,spark]"
diff --git a/.github/actions/init-test-environment/action.yaml b/.github/actions/init-test-environment/action.yaml
index 31b4ea4aa7..cf80a5deaa 100644
--- a/.github/actions/init-test-environment/action.yaml
+++ b/.github/actions/init-test-environment/action.yaml
@@ -14,14 +14,10 @@ inputs:
 runs:
   using: composite
   steps:
-    - name: Set combined ENV
+    - name: Create .env for docker compose
+      working-directory: ${{ inputs.working-directory }}
       shell: bash
-      run: |
-        echo "BROKER_DB=postgres://$BROKER_DB_USER:$BROKER_DB_PASSWORD@$BROKER_DB_HOST:$BROKER_DB_PORT/$BROKER_DB_NAME" >> $GITHUB_ENV
-        echo "DATABASE_URL=postgres://$USASPENDING_DB_USER:$USASPENDING_DB_PASSWORD@$USASPENDING_DB_HOST:$USASPENDING_DB_PORT/$USASPENDING_DB_NAME" >> $GITHUB_ENV
-        echo "DOWNLOAD_DATABASE_URL=postgres://$USASPENDING_DB_USER:$USASPENDING_DB_PASSWORD@$USASPENDING_DB_HOST:$USASPENDING_DB_PORT/$USASPENDING_DB_NAME" >> $GITHUB_ENV
-        echo "ES_HOSTNAME=$ES_SCHEME://$ES_HOST:$ES_PORT" >> $GITHUB_ENV
-        echo "MINIO_DATA_DIR=$HOME/Development/data/usaspending/docker/usaspending-s3" >> $GITHUB_ENV
+      run: cp .env.template .env
 
     # The conditionals below this all check against a string because inputs for actions are actually
     # strings, not booleans: https://github.com/actions/runner/issues/1483
@@ -59,11 +55,10 @@ runs:
       if: ${{ inputs.is-integration-test == 'true' }}
       shell: bash
       run: |
-        docker exec usaspending-db psql -U usaspending -d data_store_api -c "CREATE USER $BROKER_DB_USER PASSWORD '$BROKER_DB_PASSWORD' SUPERUSER"
         docker exec usaspending-db psql -U usaspending -d data_store_api -c "CREATE ROLE readonly"
-        docker exec usaspending-db psql -U usaspending -d data_store_api -c "SELECT 'GRANT USAGE ON SCHEMA ' || nspname || ' TO readonly' FROM pg_namespace WHERE nspname IN ('raw','int','rpt','temp','public')"
-        docker exec usaspending-db psql -U usaspending -d data_store_api -c "SELECT 'GRANT SELECT ON ALL TABLES IN SCHEMA ' || nspname || ' TO readonly' FROM pg_namespace WHERE nspname IN ('raw','int','rpt','temp','public')"
-        docker exec usaspending-db psql -U usaspending -d data_store_api -c "SELECT 'ALTER DEFAULT PRIVILEGES IN SCHEMA ' || nspname || ' GRANT SELECT ON TABLES TO readonly' FROM pg_namespace WHERE nspname IN ('raw','int','rpt','temp','public')"
+        docker exec usaspending-db psql -U usaspending -d data_store_api -c "SELECT 'GRANT USAGE ON SCHEMA ' || nspname || ' TO readonly' FROM pg_namespace WHERE nspname IN ('public','raw','int','rpt','temp')"
+        docker exec usaspending-db psql -U usaspending -d data_store_api -c "SELECT 'GRANT SELECT ON ALL TABLES IN SCHEMA ' || nspname || ' TO readonly' FROM pg_namespace WHERE nspname IN ('public','raw','int','rpt','temp')"
+        docker exec usaspending-db psql -U usaspending -d data_store_api -c "SELECT 'ALTER DEFAULT PRIVILEGES IN SCHEMA ' || nspname || ' GRANT SELECT ON TABLES TO readonly' FROM pg_namespace WHERE nspname IN ('public','raw','int','rpt','temp')"
 
     - name: Build the usaspending-development image
       working-directory: ${{ inputs.working-directory }}
diff --git a/.github/workflows/test-non-spark-integration-signal-handling.yaml b/.github/workflows/test-non-spark-integration-signal-handling.yaml
index 1ddec1e053..2e2c42b504 100644
--- a/.github/workflows/test-non-spark-integration-signal-handling.yaml
+++ b/.github/workflows/test-non-spark-integration-signal-handling.yaml
@@ -1,26 +1,5 @@
 name: Non-Spark Integration Tests with Signal Handling
 
-env:
-  BROKER_DB_HOST: usaspending-db
-  BROKER_DB_PORT: 5432
-  BROKER_DB_USER: admin
-  BROKER_DB_PASSWORD: root
-  BROKER_DB_NAME: data_broker
-  DJANGO_SETTINGS_MODULE: usaspending_api.settings
-  ES_SCHEME: http
-  ES_HOST: usaspending-es
-  ES_PORT: 9200
-  IS_LOCAL: true
-  MINIO_HOST: minio
-  MINIO_PORT: 10001
-  MINIO_ACCESS_KEY: usaspending
-  MINIO_SECRET_KEY: usaspender
-  USASPENDING_DB_HOST: usaspending-db
-  USASPENDING_DB_PORT: 5432
-  USASPENDING_DB_USER: usaspending
-  USASPENDING_DB_PASSWORD: usaspender
-  USASPENDING_DB_NAME: data_store_api
-
 on:
   workflow_call:
     inputs:
diff --git a/.github/workflows/test-non-spark-integration.yaml b/.github/workflows/test-non-spark-integration.yaml
index b9b145d295..7484a04d47 100644
--- a/.github/workflows/test-non-spark-integration.yaml
+++ b/.github/workflows/test-non-spark-integration.yaml
@@ -1,26 +1,5 @@
 name: Non-Spark Integration Tests
 
-env:
-  BROKER_DB_HOST: usaspending-db
-  BROKER_DB_PORT: 5432
-  BROKER_DB_USER: admin
-  BROKER_DB_PASSWORD: root
-  BROKER_DB_NAME: data_broker
-  DJANGO_SETTINGS_MODULE: usaspending_api.settings
-  ES_SCHEME: http
-  ES_HOST: usaspending-es
-  ES_PORT: 9200
-  IS_LOCAL: true
-  MINIO_HOST: minio
-  MINIO_PORT: 10001
-  MINIO_ACCESS_KEY: usaspending
-  MINIO_SECRET_KEY: usaspender
-  USASPENDING_DB_HOST: usaspending-db
-  USASPENDING_DB_PORT: 5432
-  USASPENDING_DB_USER: usaspending
-  USASPENDING_DB_PASSWORD: usaspender
-  USASPENDING_DB_NAME: data_store_api
-
 on:
   workflow_call:
     inputs:
diff --git a/.github/workflows/test-spark-integration-load-to-from-delta.yaml b/.github/workflows/test-spark-integration-load-to-from-delta.yaml
index d4be24c943..296f315430 100644
--- a/.github/workflows/test-spark-integration-load-to-from-delta.yaml
+++ b/.github/workflows/test-spark-integration-load-to-from-delta.yaml
@@ -1,26 +1,5 @@
 name: Spark Integration Tests - test_load_to_from_delta.py
 
-env:
-  BROKER_DB_HOST: usaspending-db
-  BROKER_DB_PORT: 5432
-  BROKER_DB_USER: admin
-  BROKER_DB_PASSWORD: root
-  BROKER_DB_NAME: data_broker
-  DJANGO_SETTINGS_MODULE: usaspending_api.settings
-  ES_SCHEME: http
-  ES_HOST: usaspending-es
-  ES_PORT: 9200
-  IS_LOCAL: true
-  MINIO_HOST: minio
-  MINIO_PORT: 10001
-  MINIO_ACCESS_KEY: usaspending
-  MINIO_SECRET_KEY: usaspender
-  USASPENDING_DB_HOST: usaspending-db
-  USASPENDING_DB_PORT: 5432
-  USASPENDING_DB_USER: usaspending
-  USASPENDING_DB_PASSWORD: usaspender
-  USASPENDING_DB_NAME: data_store_api
-
 on:
   workflow_call:
     inputs:
diff --git a/.github/workflows/test-spark-integration-load-transactions-fabs-fpds.yaml b/.github/workflows/test-spark-integration-load-transactions-fabs-fpds.yaml
index 1ac26e3765..52da316493 100644
--- a/.github/workflows/test-spark-integration-load-transactions-fabs-fpds.yaml
+++ b/.github/workflows/test-spark-integration-load-transactions-fabs-fpds.yaml
@@ -1,26 +1,5 @@
 name: Spark Integration Tests - test_load_transactions_in_delta_fabs_fpds.py
 
-env:
-  BROKER_DB_HOST: usaspending-db
-  BROKER_DB_PORT: 5432
-  BROKER_DB_USER: admin
-  BROKER_DB_PASSWORD: root
-  BROKER_DB_NAME: data_broker
-  DJANGO_SETTINGS_MODULE: usaspending_api.settings
-  ES_SCHEME: http
-  ES_HOST: usaspending-es
-  ES_PORT: 9200
-  IS_LOCAL: true
-  MINIO_HOST: minio
-  MINIO_PORT: 10001
-  MINIO_ACCESS_KEY: usaspending
-  MINIO_SECRET_KEY: usaspender
-  USASPENDING_DB_HOST: usaspending-db
-  USASPENDING_DB_PORT: 5432
-  USASPENDING_DB_USER: usaspending
-  USASPENDING_DB_PASSWORD: usaspender
-  USASPENDING_DB_NAME: data_store_api
-
 on:
   workflow_call:
     inputs:
diff --git a/.github/workflows/test-spark-integration-load-transactions-lookup.yaml b/.github/workflows/test-spark-integration-load-transactions-lookup.yaml
index bc3af98b3f..ed36927e74 100644
--- a/.github/workflows/test-spark-integration-load-transactions-lookup.yaml
+++ b/.github/workflows/test-spark-integration-load-transactions-lookup.yaml
@@ -1,26 +1,5 @@
 name: Spark Integration Tests - test_load_transactions_in_delta_lookups.py
 
-env:
-  BROKER_DB_HOST: usaspending-db
-  BROKER_DB_PORT: 5432
-  BROKER_DB_USER: admin
-  BROKER_DB_PASSWORD: root
-  BROKER_DB_NAME: data_broker
-  DJANGO_SETTINGS_MODULE: usaspending_api.settings
-  ES_SCHEME: http
-  ES_HOST: usaspending-es
-  ES_PORT: 9200
-  IS_LOCAL: true
-  MINIO_HOST: minio
-  MINIO_PORT: 10001
-  MINIO_ACCESS_KEY: usaspending
-  MINIO_SECRET_KEY: usaspender
-  USASPENDING_DB_HOST: usaspending-db
-  USASPENDING_DB_PORT: 5432
-  USASPENDING_DB_USER: usaspending
-  USASPENDING_DB_PASSWORD: usaspender
-  USASPENDING_DB_NAME: data_store_api
-
 on:
   workflow_call:
     inputs:
diff --git a/.github/workflows/test-spark-integration-other.yaml b/.github/workflows/test-spark-integration-other.yaml
index a6a9cb95c2..2f1526f655 100644
--- a/.github/workflows/test-spark-integration-other.yaml
+++ b/.github/workflows/test-spark-integration-other.yaml
@@ -1,26 +1,5 @@
 name: Spark Integration Tests - Other
 
-env:
-  BROKER_DB_HOST: usaspending-db
-  BROKER_DB_PORT: 5432
-  BROKER_DB_USER: admin
-  BROKER_DB_PASSWORD: root
-  BROKER_DB_NAME: data_broker
-  DJANGO_SETTINGS_MODULE: usaspending_api.settings
-  ES_SCHEME: http
-  ES_HOST: usaspending-es
-  ES_PORT: 9200
-  IS_LOCAL: true
-  MINIO_HOST: minio
-  MINIO_PORT: 10001
-  MINIO_ACCESS_KEY: usaspending
-  MINIO_SECRET_KEY: usaspender
-  USASPENDING_DB_HOST: usaspending-db
-  USASPENDING_DB_PORT: 5432
-  USASPENDING_DB_USER: usaspending
-  USASPENDING_DB_PASSWORD: usaspender
-  USASPENDING_DB_NAME: data_store_api
-
 on:
   workflow_call:
     inputs:
diff --git a/.github/workflows/test-unit-signal-handling.yaml b/.github/workflows/test-unit-signal-handling.yaml
index e06a828e31..59acf3b974 100644
--- a/.github/workflows/test-unit-signal-handling.yaml
+++ b/.github/workflows/test-unit-signal-handling.yaml
@@ -1,26 +1,5 @@
 name: Unit Tests with Signal Handling
 
-env:
-  BROKER_DB_HOST: usaspending-db
-  BROKER_DB_PORT: 5432
-  BROKER_DB_USER: admin
-  BROKER_DB_PASSWORD: root
-  BROKER_DB_NAME: data_broker
-  DJANGO_SETTINGS_MODULE: usaspending_api.settings
-  ES_SCHEME: http
-  ES_HOST: usaspending-es
-  ES_PORT: 9200
-  IS_LOCAL: true
-  MINIO_HOST: minio
-  MINIO_PORT: 10001
-  MINIO_ACCESS_KEY: usaspending
-  MINIO_SECRET_KEY: usaspender
-  USASPENDING_DB_HOST: usaspending-db
-  USASPENDING_DB_PORT: 5432
-  USASPENDING_DB_USER: usaspending
-  USASPENDING_DB_PASSWORD: usaspender
-  USASPENDING_DB_NAME: data_store_api
-
 on:
   workflow_call:
 
diff --git a/.github/workflows/test-unit.yaml b/.github/workflows/test-unit.yaml
index 66f62f9721..449601798b 100644
--- a/.github/workflows/test-unit.yaml
+++ b/.github/workflows/test-unit.yaml
@@ -1,27 +1,5 @@
 name: Unit Tests
 
-env:
-  BROKER_DB_HOST: usaspending-db
-  BROKER_DB_PORT: 5432
-  BROKER_DB_USER: admin
-  BROKER_DB_PASSWORD: root
-  BROKER_DB_NAME: data_broker
-  DJANGO_SETTINGS_MODULE: usaspending_api.settings
-  ES_SCHEME: http
-  ES_HOST: usaspending-es
-  ES_PORT: 9200
-  IS_LOCAL: true
-  MINIO_HOST: minio
-  MINIO_PORT: 10001
-  MINIO_ACCESS_KEY: usaspending
-  MINIO_SECRET_KEY: usaspender
-  PROJECT_LOG_DIR: ./usaspending_api/logs
-  USASPENDING_DB_HOST: usaspending-db
-  USASPENDING_DB_PORT: 5432
-  USASPENDING_DB_USER: usaspending
-  USASPENDING_DB_PASSWORD: usaspender
-  USASPENDING_DB_NAME: data_store_api
-
 on:
   workflow_call:
 
diff --git a/development.Dockerfile b/development.Dockerfile
index 763283af69..3368173b0a 100644
--- a/development.Dockerfile
+++ b/development.Dockerfile
@@ -1,5 +1,4 @@
 # Dockerfile to support local developement and tests
-
 FROM public.ecr.aws/emr-serverless/spark/emr-7.12.0:latest
 
 USER root
@@ -20,27 +19,6 @@ RUN dnf update \
         zlib-devel \
     && dnf clean all
 
-# Install python
-ARG PYTHON_VERSION=3.10.12
-RUN wget -q https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz && \
-    tar -xf Python-${PYTHON_VERSION}.tgz && \
-    cd Python-${PYTHON_VERSION} && \
-    ./configure \
-        --prefix=/usr/local \
-        --enable-optimizations \
-        --with-lto \
-        --enable-shared \
-        LDFLAGS="-Wl,-rpath /usr/local/lib" && \
-    make -j"$(nproc)" && \
-    make altinstall && \
-    cd / && rm -rf Python-${PYTHON_VERSION} Python-${PYTHON_VERSION}.tgz
-
-# Symlinks so `python3` and `pip3` resolve to correct version
-RUN ln -sf /usr/local/bin/python3.10 /usr/local/bin/python3  \
-    && ln -sf /usr/local/bin/pip3.10    /usr/local/bin/pip3 \
-    && ln -sf /usr/local/bin/python3.10 /usr/local/bin/python  \
-    && ln -sf /usr/local/bin/pip3.10    /usr/local/bin/pip
-
 # Download the Postgres JAR
 RUN wget -P /usr/lib/spark/jars/ https://jdbc.postgresql.org/download/postgresql-42.7.5.jar
 
@@ -57,22 +35,26 @@ ENV UV_COMPILE_BYTECODE=1
 # Copy from the cache instead of linking since it's a mounted volume
 ENV UV_LINK_MODE=copy
 
-# Use the system Python environment since the container is already isolated
-ENV UV_PROJECT_ENVIRONMENT=/usr/local
-ENV UV_SYSTEM_PYTHON=1
-
 # Copy the project into the image
-COPY pyproject.toml uv.lock /dockermount/
+COPY . /dockermount
 
-# Install dev dependencies
+# Install Python and dev dependencies
+ARG PYTHON_VERSION=3.10.12
 RUN --mount=type=cache,target=/root/.cache/uv \
     --mount=type=bind,source=uv.lock,target=uv.lock \
     --mount=type=bind,source=pyproject.toml,target=pyproject.toml \
-    uv sync --extra dev --extra spark --locked --no-install-project
+    uv sync --extra dev --extra spark --locked --no-install-project --python ${PYTHON_VERSION}
+
+# Make sure uv environment is active
+ENV PATH="/dockermount/.venv/bin:$PATH"
+
+# Create a symlink for the python commands
+RUN ln -sf .venv/bin/python3 /usr/local/bin/python3 & \
+    ln -sf .venv/bin/python /usr/local/bin/python
 
 # Set Python 3.10.12 as the default Python for PySpark
-ENV PYSPARK_PYTHON=/usr/local/bin/python3.10
-ENV PYSPARK_DRIVER_PYTHON=/usr/local/bin/python3.10
+ENV PYSPARK_PYTHON=/dockermount/.venv/bin/python3
+ENV PYSPARK_DRIVER_PYTHON=/dockermount/.venv/bin/python3
 
 # Limit Spark logging to WARN level
 RUN cat <<'EOF' | sed 's/^[[:space:]]*//' >> $SPARK_HOME/conf/log4j2.properties
diff --git a/docker-compose.yml b/docker-compose.yml
index bfe9ba1729..69fa63595a 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -4,13 +4,15 @@ volumes:
     driver: local
   local_es_data:
     driver: local
+  venv:
+    driver: local
 
 # There are populated by .env when using docker compose
 x-application-variables: &application-vars
   BROKER_DB: ${BROKER_DB}
-  DB_SOURCE: ${DATABASE_URL}
-  DB_R1: ${DATABASE_URL}
+  DATABASE_URL: ${DATABASE_URL}
   DJANGO_DEBUG: ${DJANGO_DEBUG}
+  DJANGO_SETTINGS_MODULE : ${DJANGO_SETTINGS_MODULE}
   DOWNLOAD_DATABASE_URL: ${DOWNLOAD_DATABASE_URL}
   ES_HOSTNAME: ${ES_HOSTNAME}
   IS_LOCAL: ${IS_LOCAL:-true}
@@ -78,6 +80,8 @@ services:
     container_name: usaspending-manage
     volumes:
       - .:/dockermount
+      # Python is installed and managed in the image using uv; avoid conflicts with local python .venv
+      - venv:/dockermount/.venv
     depends_on:
       - usaspending-db
     # For an interactive shell, override this command with: docker compose run --rm usaspending-manage python3 -u manage.py shell
@@ -98,13 +102,14 @@ services:
       # Required to interact with host's docker daemon from within this running container,
       # to spin up the data-act-broker-init-test-db container used for broker integration tests (see: conftest.broker_db_setup)
       - /var/run/docker.sock:/var/run/docker.sock
+      # Python is installed and managed in the image using uv; avoid conflicts with local python .venv
+      - venv:/dockermount/.venv
     depends_on:
       - minio
       - usaspending-db
       - usaspending-es
     # , "--reuse-db", "--no-cov", "--disable-warnings", "--reuse-db", "--numprocesses", "auto"
-    entrypoint: ["uv", "run", "pytest"]
-    command: make tests
+    entrypoint: []  # override entrypoint
     environment:
       <<: *spark-submit-vars
       # Location in host machine where broker src code root can be found
@@ -122,6 +127,8 @@ services:
     container_name: usaspending-style-checks
     volumes:
       - .:/dockermount
+      # Python is installed and managed in the image using uv; avoid conflicts with local python .venv
+      - venv:/dockermount/.venv
     entrypoint: []  # override entrypoint
     command:
       - sh
@@ -152,6 +159,8 @@ services:
       # Required to interact with host's docker daemon from within this running container,
       # to spin up the data-act-broker-init-test-db container used for broker integration tests (see: conftest.broker_db_setup)
       - /var/run/docker.sock:/var/run/docker.sock
+      # Python is installed and managed in the image using uv; avoid conflicts with local python .venv
+      - venv:/dockermount/.venv
     depends_on:
       usaspending-db:
         condition: service_healthy
@@ -184,7 +193,7 @@ services:
       # Location in host machine where broker src code root can be found
       DATA_BROKER_SRC_PATH: "${PWD}/../data-act-broker-backend"
 
-  usaspending-api:
+  usaspending-api:  # builds using the default Dockerfile to mirror deployed environments for local testing
     profiles:
       - usaspending  # must pass --profile usaspending to docker compose for this to come up
     image: usaspending-backend  # when an image by this name is not found in the local repo, and it is forced to build, it will use this as the tag
@@ -259,6 +268,10 @@ services:
     - type: volume
       source: local_es_data
       target: /usr/share/opensearch/data
+      # Python is installed and managed in the image, the volume prevents needing to reinstall dependencies
+    - type: volume
+      source: venv
+      target: /dockermount/.venv
     ports:
       - 9200:9200
 
@@ -344,6 +357,10 @@ services:
         source: .
         target: /dockermount
         read_only: false
+      # Python is installed and managed in the image, the volume prevents needing to reinstall dependencies
+      - type: volume
+        source: venv
+        target: /dockermount/.venv
 
   spark-worker:
     profiles:
@@ -370,6 +387,10 @@ services:
         source: .
         target: /dockermount
         read_only: false
+      # Python is installed and managed in the image, the volume prevents needing to reinstall dependencies
+      - type: volume
+        source: venv
+        target: /dockermount/.venv
 
   spark-history-server:
     profiles:
@@ -394,6 +415,10 @@ services:
         source: .
         target: /dockermount
         read_only: false
+      # Python is installed and managed in the image, the volume prevents needing to reinstall dependencies
+      - type: volume
+        source: venv
+        target: /dockermount/.venv
 
   # Example of running spark-submit container:
   #  NOTE: double check package dependency versions here with those used in unit tests (conftest_spark.py), as these docs could have gotten stale
@@ -454,6 +479,10 @@ services:
       - type: bind
         source: ${SPARK_SQL_WAREHOUSE_DIR:-./spark-warehouse}
         target: /spark-warehouse
+      # Python is installed and managed in the image, the volume prevents needing to reinstall dependencies
+      - type: volume
+        source: venv
+        target: /dockermount/.venv
 
   spark-shell:
     profiles:
@@ -500,6 +529,10 @@ services:
         source: ${SPARK_SQL_WAREHOUSE_DIR:-./spark-warehouse}
         target: /spark-warehouse
         read_only: false
+      # Python is installed and managed in the image, the volume prevents needing to reinstall dependencies
+      - type: volume
+        source: venv
+        target: /dockermount/.venv
 
   otel-collector:
     profiles:

From 488307f25ea7bcd01bcdcebac96d1a6c6bc028fc Mon Sep 17 00:00:00 2001
From: Seth Stoudenmier <stoudenmier_seth@bah.com>
Date: Mon, 11 May 2026 16:06:10 -0400
Subject: [PATCH 13/48] [DEV-14608] troubleshoot failing tests and github
 actions

---
 .../init-python-environment/action.yaml       | 24 +++++++++++++++++++
 .github/workflows/code-style-checks.yaml      |  2 ++
 2 files changed, 26 insertions(+)
 create mode 100644 .github/actions/init-python-environment/action.yaml

diff --git a/.github/actions/init-python-environment/action.yaml b/.github/actions/init-python-environment/action.yaml
new file mode 100644
index 0000000000..503dcfe68f
--- /dev/null
+++ b/.github/actions/init-python-environment/action.yaml
@@ -0,0 +1,24 @@
+name: Initialize Python Environment
+
+inputs:
+  python-version:
+    description: Version of Python to use in the environment and install dependencies with
+    default: "3.10.12"
+  working-directory:
+    description: Directory where the requirements can be found; used when multiple repos are checked out
+    default: ""
+
+runs:
+  using: composite
+  steps:
+    - name: Install Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ inputs.python-version }}
+        cache: "pip"
+        cache-dependency-path: setup.py
+
+    - name: Install dev dependencies
+      working-directory: ${{ inputs.working-directory }}
+      shell: bash
+      run: pip install ".[dev]"
diff --git a/.github/workflows/code-style-checks.yaml b/.github/workflows/code-style-checks.yaml
index 38d833b60f..879fdfca07 100644
--- a/.github/workflows/code-style-checks.yaml
+++ b/.github/workflows/code-style-checks.yaml
@@ -70,6 +70,8 @@ jobs:
         if: steps.changed-python-files.outputs.all_changed_files != ''
         run: ruff check ${{ steps.changed-python-files.outputs.all_changed_files }}
 
+      # TODO: Should rethink placement of this check. This runs as a Django management command which requires similar
+      #       setup steps as the rest of the test suite. Removing would reduce setup time and overall CI/CD time.
       - name: Run Check For Endpoint Documentation
         run: python manage.py check_for_endpoint_documentation
 

From 301e0698755c47bf2489ae6966908525d17c40ee Mon Sep 17 00:00:00 2001
From: Seth Stoudenmier <stoudenmier_seth@bah.com>
Date: Mon, 11 May 2026 16:18:07 -0400
Subject: [PATCH 14/48] [DEV-14608] troubleshoot github actions

---
 .github/actions/init-test-environment/action.yaml |  4 ++--
 .github/actions/run-pytest/action.yaml            | 11 ++++++-----
 Makefile                                          |  6 +++---
 3 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/.github/actions/init-test-environment/action.yaml b/.github/actions/init-test-environment/action.yaml
index cf80a5deaa..d39233e501 100644
--- a/.github/actions/init-test-environment/action.yaml
+++ b/.github/actions/init-test-environment/action.yaml
@@ -24,7 +24,7 @@ runs:
     - name: Create directories needed for Minio
       if: ${{ inputs.is-integration-test == 'true' }}
       shell: bash
-      run: mkdir -p "$MINIO_DATA_DIR"
+      run: mkdir -p "$HOME/Development/data/usaspending/docker/usaspending-s3"  # same value as .env.template
 
     - name: Build docker containers for DB, ES, and Minio
       working-directory: ${{ inputs.working-directory }}
@@ -70,5 +70,5 @@ runs:
       working-directory: ${{ inputs.working-directory }}
       shell: bash
       run: >
-        docker compose run --rm usaspending-test --create-db --numprocesses auto --no-cov --disable-warnings -r=fEs --verbosity=3 --capture=no --log-cli-level=WARNING --show-capture=log
+        docker compose run --rm usaspending-test pytest --create-db --numprocesses auto --no-cov --disable-warnings -r=fEs --verbosity=3 --capture=no --log-cli-level=WARNING --show-capture=log
         2> /dev/null "usaspending_api/tests/integration/test_setup_of_test_dbs.py::test_trigger_test_db_setup"
diff --git a/.github/actions/run-pytest/action.yaml b/.github/actions/run-pytest/action.yaml
index 46388b7b8b..f09613c181 100644
--- a/.github/actions/run-pytest/action.yaml
+++ b/.github/actions/run-pytest/action.yaml
@@ -40,13 +40,14 @@ runs:
         echo "MARKER=${{ inputs.marker }}" >> $GITHUB_ENV
         echo "NUM_PROCESSES=${{ inputs.num-processes }}" >> $GITHUB_ENV
 
-    - name: Build pytest args
+    - name: Build pytest command
       working-directory: ${{ inputs.working-directory }}
-      id: build-pytest-args
+      id: build-pytest-command
       shell: bash
       run: >
-        echo pytest-args=$(python -c 'import os;print(
-        "-r=fEs
+        echo pytest-command=$(python -c 'import os;print(
+        "pytest
+        -r=fEs
         -x
         --cov=usaspending_api
         --cov-report term
@@ -73,7 +74,7 @@ runs:
     - name: Run tests
       working-directory: ${{ inputs.working-directory }}
       shell: bash
-      run: docker compose run --rm usaspending-test ${{ steps.build-pytest-args.outputs.pytest-args }}
+      run: docker compose run --rm usaspending-test ${{ steps.build-pytest-command.outputs.pytest-command }}
 
     - name: Upload Artifact
       uses: actions/upload-artifact@v4
diff --git a/Makefile b/Makefile
index 229ff8d574..f751ee3750 100644
--- a/Makefile
+++ b/Makefile
@@ -74,15 +74,15 @@ env-code:  ## Print the value of ENV_CODE environment variable
 .PHONY: test-dbs
 createdb :=  #unset it
 test-dbs:  ## Trigger the setup of multiple test DBs that can be reused with pytest --numprocesses. Add createdb=true to force (re-)creation of Test DBs rather than reuse.
-	docker compose run --rm usaspending-test ${if ${createdb},--create-db,} --reuse-db --numprocesses=auto --no-cov --disable-warnings -rP -vvv --capture=no --log-cli-level=WARNING --show-capture=log 2> /dev/null 'usaspending_api/tests/integration/test_setup_of_test_dbs.py::test_trigger_test_db_setup'
+	docker compose run --rm usaspending-test pytest ${if ${createdb},--create-db,} --reuse-db --numprocesses=auto --no-cov --disable-warnings -rP -vvv --capture=no --log-cli-level=WARNING --show-capture=log 2> /dev/null 'usaspending_api/tests/integration/test_setup_of_test_dbs.py::test_trigger_test_db_setup'
 
 .PHONY: tests
 tests: local-dev-setup test-dbs test-spark-deps ## Run automated unit/integration tests. Configured for useful logging. add args="..." to append additional pytest args
-	docker compose run --rm usaspending-test --failed-first --reuse-db --numprocesses=auto --dist=worksteal -rP -vv --capture=no --show-capture=log 2> /dev/null ${args} 'usaspending_api/'
+	docker compose run --rm usaspending-test pytest --failed-first --reuse-db --numprocesses=auto --dist=worksteal -rP -vv --capture=no --show-capture=log 2> /dev/null ${args} 'usaspending_api/'
 
 .PHONY: tests-failed
 tests-failed: local-dev-setup test-dbs test-spark-deps ## Re-run only automated unit/integration tests that failed on the previous run. Configured for verbose logging to get more detail on failures. logging. add args="..." to append additional pytest args
-	docker compose run --rm usaspending-test --last-failed --reuse-db --numprocesses=auto --dist=worksteal -rP -vvv ${args}
+	docker compose run --rm usaspending-test pytest --last-failed --reuse-db --numprocesses=auto --dist=worksteal -rP -vvv ${args}
 
 .PHONY: confirm-clean-all
 no-prompt := 'false'

From 2e63f9939dc185f0707256a12317b47816f6b173 Mon Sep 17 00:00:00 2001
From: Seth Stoudenmier <stoudenmier_seth@bah.com>
Date: Wed, 13 May 2026 15:26:43 -0400
Subject: [PATCH 15/48] Use .env for all docker compose env

---
 .env.template                    |  19 ++--
 .github/workflows/test-unit.yaml |   9 --
 Makefile                         |   4 +
 development.Dockerfile           |  18 ++--
 docker-compose.yml               | 145 ++++++++++++++++++-------------
 5 files changed, 106 insertions(+), 89 deletions(-)

diff --git a/.env.template b/.env.template
index ec9a53c7c4..aa15e5db2a 100644
--- a/.env.template
+++ b/.env.template
@@ -61,18 +61,17 @@ ES_HOST=usaspending-es
 ES_PORT=9200
 
 # ==== [Spark] ====
+SPARK_MASTER_HOST=spark-master
 SPARK_MASTER_PORT=7077
 SPARK_MASTER_WEBUI_PORT=4040
 SPARK_HISTORY_SERVER_PORT=18080
-# Should point to a path where data can be persisted beyond docker restarts, outside of the git source repository
-# The specified directory needs to exist before Docker can mount it
-SPARK_CLUSTER_DATA_DIR=${HOME}/Development/data/usaspending/docker/usaspending-spark
-# Optionally uncomment these sql warehouse and hive metastore env vars to use the explicit locations specified below
-# If NOT uncommented, config should fall back to matching values in
-#   1) These vars' values in local.py LocalConfig directed at a spark-warehouse dir under the project root
-#   2) docker-compose.yml not finding these env vars, would ALSO fall back to a spark-warehouse dir under the project root
-#SPARK_SQL_WAREHOUSE_DIR=${SPARK_CLUSTER_DATA_DIR}/spark-warehouse
-#HIVE_METASTORE_DERBY_DB_DIR=${SPARK_SQL_WAREHOUSE_DIR}/metastore_db
+# Should point to paths where data can be persisted beyond docker restarts
+# NOTE: The specified directory needs to exist before Docker can mount it
+SOURCE_SPARK_SQL_WAREHOUSE_DIR=./spark-warehouse
+SOURCE_HIVE_METASTORE_DERBY_DB_DIR=./spark-warehouse/metastore_db
+# Location in the docker containers to create and manage the Derby DB powered Hive metastore
+TARGET_SPARK_SQL_WAREHOUSE_DIR=/spark-warehouse
+TARGET_HIVE_METASTORE_DERBY_DB_DIR=/spark-warehouse/metastore_db
 # Uncomment if on MacOS. See note here: https://github.com/fedspendingtransparency/usaspending-api#test-setup for below
 #OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES
 
@@ -84,6 +83,8 @@ SPARK_CLUSTER_DATA_DIR=${HOME}/Development/data/usaspending/docker/usaspending-s
 # ==== [MinIO] ====
 MINIO_HOST=minio
 MINIO_PORT=10001
+MINIO_ACCESS_KEY=usaspending
+MINIO_SECRET_KEY=usaspender
 MINIO_CONSOLE_PORT=10002
 # Should point to a path where data can be persisted beyond docker restarts, outside of the git source repository
 # The specified directory needs to exist before Docker can mount it
diff --git a/.github/workflows/test-unit.yaml b/.github/workflows/test-unit.yaml
index 449601798b..00dac71a99 100644
--- a/.github/workflows/test-unit.yaml
+++ b/.github/workflows/test-unit.yaml
@@ -30,15 +30,6 @@ jobs:
           is-spark-test: false
           working-directory: ./usaspending-api
 
-      - name: Troubleshoot
-        shell: bash
-        run: |
-          pwd
-          ls
-          docker compose run --rm --entrypoint "pwd" usaspending-test
-          docker compose run --rm --entrypoint "ls" usaspending-test
-          docker compose run --rm --entrypoint "env" usaspending-test
-
       - name: Run Test Cases
         id: run-test-cases
         uses: ./usaspending-api/.github/actions/run-pytest
diff --git a/Makefile b/Makefile
index f751ee3750..2d8d45d3dc 100644
--- a/Makefile
+++ b/Makefile
@@ -172,6 +172,10 @@ docker-compose-build-development: ## See: docker-compose-build rule. This builds
 	echo "docker compose --profile spark --project-directory . --file ${docker_compose_file} build --build-arg PROJECT_LOG_DIR=${PROJECT_LOG_DIR} ${args}"
 	docker compose --profile spark --project-directory . --file ${docker_compose_file} build --build-arg PROJECT_LOG_DIR=${PROJECT_LOG_DIR} ${args}
 
+.PHONY: style-checks
+style-checks:
+	docker compose --profile=ci --project-directory . --file ${docker_compose_file} run --rm usaspending-style-checks
+
 .PHONY: docker-compose-spark-submit
 spark-submit: ## Run spark-submit from within local docker containerized infrastructure (which must be running first). Set params with django_command="..."
 	docker compose --profile=spark --project-directory . --file ${docker_compose_file} run --rm \
diff --git a/development.Dockerfile b/development.Dockerfile
index 3368173b0a..c0f62cc917 100644
--- a/development.Dockerfile
+++ b/development.Dockerfile
@@ -27,6 +27,9 @@ RUN npm install --global dredd@13.1.2
 
 WORKDIR /dockermount
 
+# Allow git operations to be run inside of the container
+RUN git config --global --add safe.directory /dockermount
+
 ##### The following ENV vars are optimizations from https://github.com/astral-sh/uv-docker-example/blob/main/Dockerfile
 ##### and https://docs.astral.sh/uv/guides/integration/docker/#optimizations
 # Enable bytecode compilation
@@ -56,16 +59,11 @@ RUN ln -sf .venv/bin/python3 /usr/local/bin/python3 & \
 ENV PYSPARK_PYTHON=/dockermount/.venv/bin/python3
 ENV PYSPARK_DRIVER_PYTHON=/dockermount/.venv/bin/python3
 
-# Limit Spark logging to WARN level
-RUN cat <<'EOF' | sed 's/^[[:space:]]*//' >> $SPARK_HOME/conf/log4j2.properties
-	appender.console.type = Console
-    appender.console.name = CONSOLE
-    appender.console.layout.type = PatternLayout
-    appender.console.layout.pattern = [%d{yyyy-MM-dd HH:mm:ss.SSS}][%p] - %m%n
-    rootLogger.level = WARN
-    rootLogger.appenderRef.0.ref = CONSOLE
-    rootLogger.appenderRef.0.level = WARN
-EOF
+# Update logging
+ARG PROJECT_LOG_DIR
+RUN sed -i "s|^# spark.eventLog.dir.*$|spark.eventLog.dir                 file:///dockermount/$PROJECT_LOG_DIR/spark-events|g" $SPARK_HOME/conf/spark-defaults.conf \
+    && sed -i '/spark.eventLog.enabled/s/^# //g' $SPARK_HOME/conf/spark-defaults.conf \
+    && echo "spark.history.fs.logDirectory      file:///project/$PROJECT_LOG_DIR/spark-events" >> $SPARK_HOME/conf/spark-defaults.conf
 
 # Set default values for Spark with local development
 RUN echo "spark.authenticate false" >> $SPARK_HOME/conf/spark-defaults.conf
diff --git a/docker-compose.yml b/docker-compose.yml
index 69fa63595a..3853384254 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -4,44 +4,54 @@ volumes:
     driver: local
   local_es_data:
     driver: local
-  venv:
+  local_venv:
     driver: local
 
 # There are populated by .env when using docker compose
 x-application-variables: &application-vars
-  BROKER_DB: ${BROKER_DB}
+  # Primary data stores for the API
   DATABASE_URL: ${DATABASE_URL}
-  DJANGO_DEBUG: ${DJANGO_DEBUG}
-  DJANGO_SETTINGS_MODULE : ${DJANGO_SETTINGS_MODULE}
   DOWNLOAD_DATABASE_URL: ${DOWNLOAD_DATABASE_URL}
   ES_HOSTNAME: ${ES_HOSTNAME}
-  IS_LOCAL: ${IS_LOCAL:-true}
+  # For connection to Broker
+  BROKER_DB: ${BROKER_DB}
+  # For local development
+  DJANGO_DEBUG: ${DJANGO_DEBUG}
+  DJANGO_SETTINGS_MODULE: ${DJANGO_SETTINGS_MODULE}
+  IS_LOCAL: ${IS_LOCAL}
+  # MinIO
+  MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY}
+  MINIO_SECRET_KEY: ${MINIO_SECRET_KEY}
   MINIO_HOST: ${MINIO_HOST}
   MINIO_PORT: ${MINIO_PORT}
+  # Add network proxy to the containers
+#  HTTP_PROXY: ${HTTP_PROXY}
+#  HTTPS_PROXY: ${HTTPS_PROXY}
+#  NO_PROXY: 127.0.0.1,localhost,minio,${NO_PROXY}
+  # Defined for tracing
   OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: ${OTEL_EXPORTER_OTLP_TRACES_ENDPOINT}
   TOGGLE_OTEL_CONSOLE_LOGGING: ${TOGGLE_OTEL_CONSOLE_LOGGING}
 
 x-spark-submit-variables: &spark-submit-vars
   <<: *application-vars
   AWS_ACCESS_KEY_ID: ${MINIO_ACCESS_KEY}
-  AWS_ENDPOINT_URL: http://${MINIO_HOST}:{MINIO_PORT}
+  AWS_ENDPOINT_URL: http://${MINIO_HOST}:${MINIO_PORT}
   AWS_SECRET_ACCESS_KEY: ${MINIO_SECRET_KEY}
-  PYTHONPATH: "/dockermount"
   #SPARK_HIVE_METASTORE_DB_TYPE: derby
   #SPARK_HIVE_METASTORE_DB_PATH: /spark-warehouse
   # Needed for the Spark containers to interact
-  SPARK_MASTER_HOST: ${SPARK_MASTER_HOST:-spark-master}
-  SPARK_MASTER_PORT: ${SPARK_MASTER_PORT:-7077}
-  SPARK_WORKER_WEBUI_PORT: ${SPARK_WORKER_WEBUI_PORT:-4041}
+  SPARK_MASTER_HOST: ${SPARK_MASTER_HOST}
+  SPARK_MASTER_PORT: ${SPARK_MASTER_PORT}
+  SPARK_WORKER_WEBUI_PORT: ${SPARK_WORKER_WEBUI_PORT}
   # This env var needs to be picked up as the config for the spark.sql.warehouse.dir spark conf setting when
   # SparkSessions are created inside a spark-submitted job
   # (i.e., target where host warehouse dir is bound in volume config)
-  SPARK_SQL_WAREHOUSE_DIR: /spark-warehouse
+  SPARK_SQL_WAREHOUSE_DIR: ${TARGET_SPARK_SQL_WAREHOUSE_DIR}
   # This env var needs to be picked up as the path part of the config for the
   # spark.hadoop.javax.jdo.option.ConnectionURL spark conf setting when SparkSessions are created inside a
   # spark-submitted job
   # (i.e., a metastore_db sub dir of the target where host warehouse dir is bound in volume config.)
-  HIVE_METASTORE_DERBY_DB_DIR: /spark-warehouse/metastore_db
+  HIVE_METASTORE_DERBY_DB_DIR: ${TARGET_HIVE_METASTORE_DERBY_DB_DIR}
 
 
 services:
@@ -55,7 +65,7 @@ services:
       - ci
     image: postgres:16-alpine
     healthcheck:
-      test: [ "CMD-SHELL", "pg_isready -U ${USASPENDING_DB_USER:-usaspending} -d ${USASPENDING_DB_NAME:-data_store_api}" ]
+      test: [ "CMD-SHELL", "pg_isready -U ${USASPENDING_DB_USER} -d ${USASPENDING_DB_NAME}" ]
       interval: 5s
       timeout: 5s
       retries: 5
@@ -66,11 +76,11 @@ services:
         source: local_pg_data
         target: /var/lib/postgresql/data
     ports:
-      - ${USASPENDING_DB_PORT:-5432}:5432
+      - ${USASPENDING_DB_PORT}:5432
     environment:
-      POSTGRES_USER: ${USASPENDING_DB_USER:-usaspending}
-      POSTGRES_PASSWORD: ${USASPENDING_DB_PASSWORD:-usaspender}
-      POSTGRES_DB: ${USASPENDING_DB_NAME:-data_store_api}
+      POSTGRES_USER: ${USASPENDING_DB_USER}
+      POSTGRES_PASSWORD: ${USASPENDING_DB_PASSWORD}
+      POSTGRES_DB: ${USASPENDING_DB_NAME}
 
   usaspending-manage:
     profiles:
@@ -81,7 +91,9 @@ services:
     volumes:
       - .:/dockermount
       # Python is installed and managed in the image using uv; avoid conflicts with local python .venv
-      - venv:/dockermount/.venv
+      - type: volume
+        source: local_venv
+        target: /dockermount/.venv
     depends_on:
       - usaspending-db
     # For an interactive shell, override this command with: docker compose run --rm usaspending-manage python3 -u manage.py shell
@@ -96,14 +108,18 @@ services:
     build:
       context: .
       dockerfile: development.Dockerfile
+      args:
+        PROJECT_LOG_DIR: ${PROJECT_LOG_DIR}
     container_name: usaspending-test
     volumes:
-      - ${PROJECT_DIRECTORY:-.}:/dockermount
+      - .:/dockermount
+      # Python is installed and managed in the image using uv; avoid conflicts with local python .venv
+      - type: volume
+        source: local_venv
+        target: /dockermount/.venv
       # Required to interact with host's docker daemon from within this running container,
       # to spin up the data-act-broker-init-test-db container used for broker integration tests (see: conftest.broker_db_setup)
       - /var/run/docker.sock:/var/run/docker.sock
-      # Python is installed and managed in the image using uv; avoid conflicts with local python .venv
-      - venv:/dockermount/.venv
     depends_on:
       - minio
       - usaspending-db
@@ -128,13 +144,14 @@ services:
     volumes:
       - .:/dockermount
       # Python is installed and managed in the image using uv; avoid conflicts with local python .venv
-      - venv:/dockermount/.venv
+      - type: volume
+        source: local_venv
+        target: /dockermount/.venv
     entrypoint: []  # override entrypoint
     command:
       - sh
       - -c
       - |
-        git config --global --add safe.directory /dockermount
         printf -- "==============\nFormatting code:\n"
         git diff --name-only --diff-filter=ACMR | grep '\.py$' | xargs uv run ruff format
         printf -- "==============\nChecking code syntax and quality:\n"
@@ -160,7 +177,9 @@ services:
       # to spin up the data-act-broker-init-test-db container used for broker integration tests (see: conftest.broker_db_setup)
       - /var/run/docker.sock:/var/run/docker.sock
       # Python is installed and managed in the image using uv; avoid conflicts with local python .venv
-      - venv:/dockermount/.venv
+      - type: volume
+        source: local_venv
+        target: /dockermount/.venv
     depends_on:
       usaspending-db:
         condition: service_healthy
@@ -201,6 +220,10 @@ services:
     container_name: usaspending-api
     volumes:
       - .:/dockermount
+      # Python is installed and managed in the image; avoid conflicts with local python .venv
+      - type: volume
+        source: local_venv
+        target: /dockermount/.venv
     ports:
       - 8000:8000
     depends_on:
@@ -226,6 +249,10 @@ services:
       # Required to interact with host's docker daemon from within this running container,
       # to submit a spark job via the spark-submit container (see download_sqs_worker._run_spark_download)
       - /var/run/docker.sock:/var/run/docker.sock
+      # Python is installed and managed in the image; avoid conflicts with local python .venv
+      - type: volume
+        source: local_venv
+        target: /dockermount/.venv
     command: python3 manage.py download_sqs_worker
     environment:
       <<: *application-vars
@@ -268,12 +295,8 @@ services:
     - type: volume
       source: local_es_data
       target: /usr/share/opensearch/data
-      # Python is installed and managed in the image, the volume prevents needing to reinstall dependencies
-    - type: volume
-      source: venv
-      target: /dockermount/.venv
     ports:
-      - 9200:9200
+      - ${ES_PORT}:9200
 
   usaspending-kibana-es:
     profiles:
@@ -300,14 +323,14 @@ services:
     volumes:
       - .:/dockermount
       - type: bind
-        source: ${MINIO_DATA_DIR:-../data/s3}
+        source: ${MINIO_DATA_DIR}
         target: /data
     ports:
-      - ${MINIO_PORT:-10001}:10001
-      - ${MINIO_CONSOLE_PORT:-10002}:10002
+      - ${MINIO_PORT}:10001
+      - ${MINIO_CONSOLE_PORT}:10002
     environment:
-      MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY:-usaspending}
-      MINIO_SECRET_KEY: ${MINIO_SECRET_KEY:-usaspender}
+      MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY}
+      MINIO_SECRET_KEY: ${MINIO_SECRET_KEY}
     entrypoint: >
       /bin/sh -c "
         # Create the bucket within MinIO and file path for the data dict
@@ -324,7 +347,7 @@ services:
         minio server --address ":10001" --console-address ":10002" /data
       "
     healthcheck:
-      test: ["CMD", "curl", "-f", "http://${MINIO_HOST:-localhost}:${MINIO_PORT:-10001}/minio/health/live"]
+      test: ["CMD", "curl", "-f", "http://${MINIO_HOST}:${MINIO_PORT}/minio/health/live"]
       interval: 30s
       timeout: 20s
       retries: 3
@@ -350,16 +373,16 @@ services:
         --port $${SPARK_MASTER_PORT} \
         --webui-port $${SPARK_MASTER_WEBUI_PORT}"
     ports:
-      - ${SPARK_MASTER_PORT:-7077}:7077
-      - ${SPARK_MASTER_WEBUI_PORT:-4040}:4040
+      - ${SPARK_MASTER_PORT}
+      - ${SPARK_MASTER_WEBUI_PORT}
     volumes:
       - type: bind
         source: .
         target: /dockermount
         read_only: false
-      # Python is installed and managed in the image, the volume prevents needing to reinstall dependencies
+      # Python is installed and managed in the image using uv; avoid conflicts with local python .venv
       - type: volume
-        source: venv
+        source: local_venv
         target: /dockermount/.venv
 
   spark-worker:
@@ -381,15 +404,15 @@ services:
     entrypoint: []  # override the entrypoint
     command: /bin/sh -c "$${SPARK_HOME}/bin/spark-class org.apache.spark.deploy.worker.Worker --webui-port $${SPARK_WORKER_WEBUI_PORT} spark://$${SPARK_MASTER_HOST}:$${SPARK_MASTER_PORT}"
     ports:
-      - ${SPARK_WORKER_WEBUI_PORT:-4041}:4041
+      - ${SPARK_WORKER_WEBUI_PORT}:4041
     volumes:
       - type: bind
         source: .
         target: /dockermount
         read_only: false
-      # Python is installed and managed in the image, the volume prevents needing to reinstall dependencies
+      # Python is installed and managed in the image using uv; avoid conflicts with local python .venv
       - type: volume
-        source: venv
+        source: local_venv
         target: /dockermount/.venv
 
   spark-history-server:
@@ -405,19 +428,19 @@ services:
         PROJECT_LOG_DIR: ${PROJECT_LOG_DIR}
     container_name: spark-history-server
     environment:
-      SPARK_HISTORY_SERVER_PORT: ${SPARK_HISTORY_SERVER_PORT:-18080}
+      SPARK_HISTORY_SERVER_PORT: ${SPARK_HISTORY_SERVER_PORT}
     entrypoint: []  # override the entrypoint
     command: /usr/bin/sh -c "$${SPARK_HOME}/bin/spark-class org.apache.spark.deploy.history.HistoryServer"
     ports:
-      - ${SPARK_HISTORY_SERVER_PORT:-18080}:18080
+      - ${SPARK_HISTORY_SERVER_PORT}:18080
     volumes:
       - type: bind
         source: .
         target: /dockermount
         read_only: false
-      # Python is installed and managed in the image, the volume prevents needing to reinstall dependencies
+      # Python is installed and managed in the image using uv; avoid conflicts with local python .venv
       - type: volume
-        source: venv
+        source: local_venv
         target: /dockermount/.venv
 
   # Example of running spark-submit container:
@@ -468,20 +491,20 @@ services:
     # While this service has values for these interpolated vars in the environment: element, those are not used here,
     # but merely passed into the container. KEEP the two references to these vars and their defaults consistent!
     # To see what it will be, you can run docker compose config (i.e. make docker-compose-config in this project's Makefile)
-    entrypoint: /usr/bin/spark-submit --master spark://${SPARK_MASTER_HOST:-spark-master}:${SPARK_MASTER_PORT:-7077}
+    entrypoint: /usr/bin/spark-submit --master spark://${SPARK_MASTER_HOST}:${SPARK_MASTER_PORT}
     command: --help
     volumes:
       - type: bind
-        source: ${PROJECT_DIRECTORY:-.}
+        source: .
         target: /dockermount
         read_only: false
       # NOTE: The hive metastore_db Derby database folder is expected to be configured to show up as a subfolder of the spark-warehouse dir
       - type: bind
-        source: ${SPARK_SQL_WAREHOUSE_DIR:-./spark-warehouse}
-        target: /spark-warehouse
-      # Python is installed and managed in the image, the volume prevents needing to reinstall dependencies
+        source: ${SOURCE_SPARK_SQL_WAREHOUSE_DIR}
+        target: ${TARGET_SPARK_SQL_WAREHOUSE_DIR}
+      # Python is installed and managed in the image using uv; avoid conflicts with local python .venv
       - type: volume
-        source: venv
+        source: local_venv
         target: /dockermount/.venv
 
   spark-shell:
@@ -512,26 +535,26 @@ services:
         --conf spark.sql.extensions=io.delta.sql.DeltaSparkSessionExtension
         --conf spark.sql.catalog.spark_catalog=org.apache.spark.sql.delta.catalog.DeltaCatalog
         --conf spark.hadoop.fs.s3a.endpoint=${MINIO_HOST}:${MINIO_PORT}
-        --conf spark.hadoop.fs.s3a.access.key=usaspending
-        --conf spark.hadoop.fs.s3a.secret.key=usaspender
+        --conf spark.hadoop.fs.s3a.access.key=${MINIO_ACCESS_KEY}
+        --conf spark.hadoop.fs.s3a.secret.key=${MINIO_SECRET_KEY}
         --conf spark.hadoop.fs.s3a.connection.ssl.enabled=false
         --conf spark.hadoop.fs.s3a.path.style.access=true
         --conf spark.sql.catalogImplementation=hive
-        --conf spark.sql.warehouse.dir='/spark-warehouse'
-        --conf spark.hadoop.javax.jdo.option.ConnectionURL='jdbc:derby:;databaseName=/spark-warehouse/metastore_db;create=true'
+        --conf spark.sql.warehouse.dir='${TARGET_SPARK_SQL_WAREHOUSE_DIR}'
+        --conf spark.hadoop.javax.jdo.option.ConnectionURL='jdbc:derby:;databaseName=${TARGET_HIVE_METASTORE_DERBY_DB_DIR};create=true'
     volumes:
       - type: bind
-        source: ${PROJECT_DIRECTORY:-.}
+        source: .
         target: /dockermount
         read_only: false
       # NOTE: The hive metastore_db Derby database folder is expected to be configured to show up as a subfolder of the spark-warehouse dir
       - type: bind
-        source: ${SPARK_SQL_WAREHOUSE_DIR:-./spark-warehouse}
-        target: /spark-warehouse
+        source: ${SOURCE_SPARK_SQL_WAREHOUSE_DIR}
+        target: ${TARGET_SPARK_SQL_WAREHOUSE_DIR}
         read_only: false
-      # Python is installed and managed in the image, the volume prevents needing to reinstall dependencies
+      # Python is installed and managed in the image using uv; avoid conflicts with local python .venv
       - type: volume
-        source: venv
+        source: local_venv
         target: /dockermount/.venv
 
   otel-collector:

From 9f45537c19a77d2b812083d178af51a7dbc22930 Mon Sep 17 00:00:00 2001
From: Seth Stoudenmier <stoudenmier_seth@bah.com>
Date: Mon, 18 May 2026 17:03:05 -0400
Subject: [PATCH 16/48] [DEV-14608] troubleshoot github actions

---
 development.Dockerfile | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/development.Dockerfile b/development.Dockerfile
index c0f62cc917..11bd30f3b8 100644
--- a/development.Dockerfile
+++ b/development.Dockerfile
@@ -38,8 +38,8 @@ ENV UV_COMPILE_BYTECODE=1
 # Copy from the cache instead of linking since it's a mounted volume
 ENV UV_LINK_MODE=copy
 
-# Copy the project into the image
-COPY . /dockermount
+# Copy relevant parts of the project for package management
+COPY pyproject.toml uv.lock /dockermount/
 
 # Install Python and dev dependencies
 ARG PYTHON_VERSION=3.10.12
@@ -48,6 +48,8 @@ RUN --mount=type=cache,target=/root/.cache/uv \
     --mount=type=bind,source=pyproject.toml,target=pyproject.toml \
     uv sync --extra dev --extra spark --locked --no-install-project --python ${PYTHON_VERSION}
 
+ENV PYTHONPATH=/dockermount
+
 # Make sure uv environment is active
 ENV PATH="/dockermount/.venv/bin:$PATH"
 

From fdaa4ecc80c5887806723fef597fd19495f2e61c Mon Sep 17 00:00:00 2001
From: Seth Stoudenmier <stoudenmier_seth@bah.com>
Date: Tue, 19 May 2026 09:37:39 -0400
Subject: [PATCH 17/48] [DEV-14608] troubleshoot github actions

---
 .github/workflows/test-unit.yaml | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/test-unit.yaml b/.github/workflows/test-unit.yaml
index 00dac71a99..c60850c22e 100644
--- a/.github/workflows/test-unit.yaml
+++ b/.github/workflows/test-unit.yaml
@@ -14,8 +14,6 @@ jobs:
     steps:
       - name: Checkout Source Repository
         uses: actions/checkout@v4
-        with:
-          path: usaspending-api
 
       - name: Validate file structure
         shell: bash
@@ -28,14 +26,14 @@ jobs:
         with:
           is-integration-test: false
           is-spark-test: false
-          working-directory: ./usaspending-api
+          working-directory: .
 
       - name: Run Test Cases
         id: run-test-cases
-        uses: ./usaspending-api/.github/actions/run-pytest
+        uses: .github/actions/run-pytest
         with:
           cov-report-name: 'unit-tests'
           ignore-glob: '**/tests/integration/*'
           include-glob: 'test_*.py *_test.py'
           marker: '(not signal_handling)'
-          working-directory: ./usaspending-api
+          working-directory: .

From 6fb26862320daf66ba6851f3ce7b4b3ef968aa57 Mon Sep 17 00:00:00 2001
From: Seth Stoudenmier <stoudenmier_seth@bah.com>
Date: Tue, 19 May 2026 09:41:35 -0400
Subject: [PATCH 18/48] [DEV-14608] troubleshoot github actions

---
 .github/workflows/test-unit.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/test-unit.yaml b/.github/workflows/test-unit.yaml
index c60850c22e..677de27512 100644
--- a/.github/workflows/test-unit.yaml
+++ b/.github/workflows/test-unit.yaml
@@ -22,7 +22,7 @@ jobs:
           ls
 
       - name: Init Test Environment
-        uses: ./usaspending-api/.github/actions/init-test-environment
+        uses: ./.github/actions/init-test-environment
         with:
           is-integration-test: false
           is-spark-test: false
@@ -30,7 +30,7 @@ jobs:
 
       - name: Run Test Cases
         id: run-test-cases
-        uses: .github/actions/run-pytest
+        uses: ./.github/actions/run-pytest
         with:
           cov-report-name: 'unit-tests'
           ignore-glob: '**/tests/integration/*'

From f6bce38c7cf73582fae4292d91714f9b7dceec70 Mon Sep 17 00:00:00 2001
From: Seth Stoudenmier <stoudenmier_seth@bah.com>
Date: Tue, 19 May 2026 09:48:17 -0400
Subject: [PATCH 19/48] [DEV-14608] troubleshoot github actions

---
 .github/actions/init-test-environment/action.yaml | 7 -------
 .github/workflows/test-unit.yaml                  | 6 ------
 2 files changed, 13 deletions(-)

diff --git a/.github/actions/init-test-environment/action.yaml b/.github/actions/init-test-environment/action.yaml
index d39233e501..3275242b4e 100644
--- a/.github/actions/init-test-environment/action.yaml
+++ b/.github/actions/init-test-environment/action.yaml
@@ -7,15 +7,11 @@ inputs:
   is-spark-test:
     description: Triggers build steps needed to support tests that utilize Spark
     required: true
-  working-directory:
-    description: Directory where the requirements can be found; used when multiple repos are checked out
-    default: ""
 
 runs:
   using: composite
   steps:
     - name: Create .env for docker compose
-      working-directory: ${{ inputs.working-directory }}
       shell: bash
       run: cp .env.template .env
 
@@ -27,7 +23,6 @@ runs:
       run: mkdir -p "$HOME/Development/data/usaspending/docker/usaspending-s3"  # same value as .env.template
 
     - name: Build docker containers for DB, ES, and Minio
-      working-directory: ${{ inputs.working-directory }}
       if: ${{ inputs.is-integration-test == 'true' }}
       shell: bash
       run: docker compose up -d usaspending-db usaspending-es minio
@@ -61,13 +56,11 @@ runs:
         docker exec usaspending-db psql -U usaspending -d data_store_api -c "SELECT 'ALTER DEFAULT PRIVILEGES IN SCHEMA ' || nspname || ' GRANT SELECT ON TABLES TO readonly' FROM pg_namespace WHERE nspname IN ('public','raw','int','rpt','temp')"
 
     - name: Build the usaspending-development image
-      working-directory: ${{ inputs.working-directory }}
       shell: bash
       run: docker compose build usaspending-test
 
     - name: Run initial test to trigger DB creation
       if: ${{ inputs.is-integration-test == 'true' }}
-      working-directory: ${{ inputs.working-directory }}
       shell: bash
       run: >
         docker compose run --rm usaspending-test pytest --create-db --numprocesses auto --no-cov --disable-warnings -r=fEs --verbosity=3 --capture=no --log-cli-level=WARNING --show-capture=log
diff --git a/.github/workflows/test-unit.yaml b/.github/workflows/test-unit.yaml
index 677de27512..e9311f7574 100644
--- a/.github/workflows/test-unit.yaml
+++ b/.github/workflows/test-unit.yaml
@@ -3,10 +3,6 @@ name: Unit Tests
 on:
   workflow_call:
 
-defaults:
-  run:
-    working-directory: usaspending-api
-
 jobs:
   Run:
     name: Run
@@ -26,7 +22,6 @@ jobs:
         with:
           is-integration-test: false
           is-spark-test: false
-          working-directory: .
 
       - name: Run Test Cases
         id: run-test-cases
@@ -36,4 +31,3 @@ jobs:
           ignore-glob: '**/tests/integration/*'
           include-glob: 'test_*.py *_test.py'
           marker: '(not signal_handling)'
-          working-directory: .

From a41fda662ce8ad61dd248a12bdb5e1a613e7e26f Mon Sep 17 00:00:00 2001
From: Seth Stoudenmier <stoudenmier_seth@bah.com>
Date: Tue, 19 May 2026 16:12:18 -0400
Subject: [PATCH 20/48] [DEV-14608] troubleshoot github actions

---
 .../workflows/test-non-spark-integration.yaml |  14 +--
 Dockerfile                                    |   4 +-
 Makefile                                      |   2 +-
 development.Dockerfile                        |  33 +++---
 docker-compose.yml                            | 100 +++++++-----------
 download.Dockerfile                           |   2 +-
 6 files changed, 59 insertions(+), 96 deletions(-)

diff --git a/.github/workflows/test-non-spark-integration.yaml b/.github/workflows/test-non-spark-integration.yaml
index 7484a04d47..31656ecc02 100644
--- a/.github/workflows/test-non-spark-integration.yaml
+++ b/.github/workflows/test-non-spark-integration.yaml
@@ -7,10 +7,6 @@ on:
         required: true
         type: string
 
-defaults:
-  run:
-    working-directory: usaspending-api
-
 jobs:
   Run:
     name: Run
@@ -18,27 +14,23 @@ jobs:
     steps:
       - name: Checkout Source Repository
         uses: actions/checkout@v4
-        with:
-          path: usaspending-api
 
       - name: Checkout Broker Backend Repository
         uses: actions/checkout@v4
         with:
           repository: fedspendingtransparency/data-act-broker-backend
-          path: data-act-broker-backend
+          path: ../data-act-broker-backend
           ref: ${{ inputs.broker-branch }}
 
       - name: Init Test Environment
-        uses: ./usaspending-api/.github/actions/init-test-environment
+        uses: ./.github/actions/init-test-environment
         with:
           is-integration-test: true
           is-spark-test: false
-          working-directory: ./usaspending-api
 
       - name: Run Test Cases
-        uses: ./usaspending-api/.github/actions/run-pytest
+        uses: ./.github/actions/run-pytest
         with:
           cov-report-name: 'non-spark-integration-tests'
           include-glob: '**/tests/integration/*'
           marker: '(not signal_handling and not spark)'
-          working-directory: ./usaspending-api
diff --git a/Dockerfile b/Dockerfile
index 24726f1954..4f3da6ede1 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -10,7 +10,7 @@ FROM python:3.10.12-slim-bullseye
 
 COPY --from=ghcr.io/astral-sh/uv:0.7.19 /uv /uvx /bin/
 
-WORKDIR /dockermount
+WORKDIR /usaspending-api
 
 RUN apt update && apt install -y \
     ca-certificates \
@@ -41,7 +41,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
     uv sync --extra server --extra ansible --extra awscli --extra spark --locked --no-install-project --no-dev
 
 # Copy the project into the image
-COPY . /dockermount
+COPY . /usaspending-api
 
 # Sync the project
 RUN --mount=type=cache,target=/root/.cache/uv \
diff --git a/Makefile b/Makefile
index 2d8d45d3dc..21c8da679c 100644
--- a/Makefile
+++ b/Makefile
@@ -184,7 +184,7 @@ spark-submit: ## Run spark-submit from within local docker containerized infrast
 	--driver-memory "2g" \
 	${if ${python_script}, \
 		${python_script}, \
-		/dockermount/manage.py ${django_command} \
+		/usaspending-api/manage.py ${django_command} \
 	}
 
 .PHONY: pyspark-shell
diff --git a/development.Dockerfile b/development.Dockerfile
index 11bd30f3b8..b1fd50b3bf 100644
--- a/development.Dockerfile
+++ b/development.Dockerfile
@@ -6,6 +6,10 @@ USER root
 # Copy UV from uv image
 COPY --from=ghcr.io/astral-sh/uv:0.7.19 /uv /uvx /bin/
 
+# Build ARGs
+ARG PROJECT_LOG_DIR=/logs
+ARG PYTHON_VERSION=3.10.12
+
 # Install dependencies
 RUN dnf update \
     && dnf install -y \
@@ -25,13 +29,16 @@ RUN wget -P /usr/lib/spark/jars/ https://jdbc.postgresql.org/download/postgresql
 # Install Dredd
 RUN npm install --global dredd@13.1.2
 
-WORKDIR /dockermount
+WORKDIR /usaspending-api
 
 # Allow git operations to be run inside of the container
-RUN git config --global --add safe.directory /dockermount
+# RUN git config --global --add safe.directory /usaspending-api
 
 ##### The following ENV vars are optimizations from https://github.com/astral-sh/uv-docker-example/blob/main/Dockerfile
 ##### and https://docs.astral.sh/uv/guides/integration/docker/#optimizations
+# Specify .venv location outside of the project to avoid conflict
+ENV UV_PROJECT_ENVIRONMENT=/usr/local/.venv
+
 # Enable bytecode compilation
 ENV UV_COMPILE_BYTECODE=1
 
@@ -39,33 +46,25 @@ ENV UV_COMPILE_BYTECODE=1
 ENV UV_LINK_MODE=copy
 
 # Copy relevant parts of the project for package management
-COPY pyproject.toml uv.lock /dockermount/
+COPY pyproject.toml uv.lock /usaspending-api/
 
 # Install Python and dev dependencies
-ARG PYTHON_VERSION=3.10.12
 RUN --mount=type=cache,target=/root/.cache/uv \
     --mount=type=bind,source=uv.lock,target=uv.lock \
     --mount=type=bind,source=pyproject.toml,target=pyproject.toml \
     uv sync --extra dev --extra spark --locked --no-install-project --python ${PYTHON_VERSION}
 
-ENV PYTHONPATH=/dockermount
-
 # Make sure uv environment is active
-ENV PATH="/dockermount/.venv/bin:$PATH"
-
-# Create a symlink for the python commands
-RUN ln -sf .venv/bin/python3 /usr/local/bin/python3 & \
-    ln -sf .venv/bin/python /usr/local/bin/python
+ENV PATH="/usr/local/.venv/bin:$PATH"
 
 # Set Python 3.10.12 as the default Python for PySpark
-ENV PYSPARK_PYTHON=/dockermount/.venv/bin/python3
-ENV PYSPARK_DRIVER_PYTHON=/dockermount/.venv/bin/python3
+ENV PYSPARK_PYTHON=/usr/local/.venv/bin/python3
+ENV PYSPARK_DRIVER_PYTHON=/usr/local/.venv/bin/python3
 
-# Update logging
-ARG PROJECT_LOG_DIR
-RUN sed -i "s|^# spark.eventLog.dir.*$|spark.eventLog.dir                 file:///dockermount/$PROJECT_LOG_DIR/spark-events|g" $SPARK_HOME/conf/spark-defaults.conf \
+# Update logging for Spark
+RUN sed -i "s|^# spark.eventLog.dir.*$|spark.eventLog.dir                 file:///usaspending-api/$PROJECT_LOG_DIR/spark-events|g" $SPARK_HOME/conf/spark-defaults.conf \
     && sed -i '/spark.eventLog.enabled/s/^# //g' $SPARK_HOME/conf/spark-defaults.conf \
-    && echo "spark.history.fs.logDirectory      file:///project/$PROJECT_LOG_DIR/spark-events" >> $SPARK_HOME/conf/spark-defaults.conf
+    && echo "spark.history.fs.logDirectory      file:///usaspending-api/$PROJECT_LOG_DIR/spark-events" >> $SPARK_HOME/conf/spark-defaults.conf
 
 # Set default values for Spark with local development
 RUN echo "spark.authenticate false" >> $SPARK_HOME/conf/spark-defaults.conf
diff --git a/docker-compose.yml b/docker-compose.yml
index 3853384254..f2e70a63c0 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -4,8 +4,6 @@ volumes:
     driver: local
   local_es_data:
     driver: local
-  local_venv:
-    driver: local
 
 # There are populated by .env when using docker compose
 x-application-variables: &application-vars
@@ -89,11 +87,10 @@ services:
     build: .
     container_name: usaspending-manage
     volumes:
-      - .:/dockermount
-      # Python is installed and managed in the image using uv; avoid conflicts with local python .venv
-      - type: volume
-        source: local_venv
-        target: /dockermount/.venv
+      - type: bind
+        source: .
+        target: /usaspending-api
+        read_only: false
     depends_on:
       - usaspending-db
     # For an interactive shell, override this command with: docker compose run --rm usaspending-manage python3 -u manage.py shell
@@ -112,11 +109,10 @@ services:
         PROJECT_LOG_DIR: ${PROJECT_LOG_DIR}
     container_name: usaspending-test
     volumes:
-      - .:/dockermount
-      # Python is installed and managed in the image using uv; avoid conflicts with local python .venv
-      - type: volume
-        source: local_venv
-        target: /dockermount/.venv
+      - type: bind
+        source: .
+        target: /usaspending-api
+        read_only: false
       # Required to interact with host's docker daemon from within this running container,
       # to spin up the data-act-broker-init-test-db container used for broker integration tests (see: conftest.broker_db_setup)
       - /var/run/docker.sock:/var/run/docker.sock
@@ -142,11 +138,10 @@ services:
       dockerfile: development.Dockerfile
     container_name: usaspending-style-checks
     volumes:
-      - .:/dockermount
-      # Python is installed and managed in the image using uv; avoid conflicts with local python .venv
-      - type: volume
-        source: local_venv
-        target: /dockermount/.venv
+      - type: bind
+        source: .
+        target: /usaspending-api
+        read_only: false
     entrypoint: []  # override entrypoint
     command:
       - sh
@@ -172,14 +167,13 @@ services:
       dockerfile: development.Dockerfile
     container_name: usaspending-ci
     volumes:
-      - .:/dockermount
+      - type: bind
+        source: .
+        target: /usaspending-api
+        read_only: false
       # Required to interact with host's docker daemon from within this running container,
       # to spin up the data-act-broker-init-test-db container used for broker integration tests (see: conftest.broker_db_setup)
       - /var/run/docker.sock:/var/run/docker.sock
-      # Python is installed and managed in the image using uv; avoid conflicts with local python .venv
-      - type: volume
-        source: local_venv
-        target: /dockermount/.venv
     depends_on:
       usaspending-db:
         condition: service_healthy
@@ -219,11 +213,10 @@ services:
     build: .
     container_name: usaspending-api
     volumes:
-      - .:/dockermount
-      # Python is installed and managed in the image; avoid conflicts with local python .venv
-      - type: volume
-        source: local_venv
-        target: /dockermount/.venv
+      - type: bind
+        source: .
+        target: /usaspending-api
+        read_only: false
     ports:
       - 8000:8000
     depends_on:
@@ -245,14 +238,13 @@ services:
     container_name: usaspending-bulk-download
     restart: on-failure:5 # 5 max attempt, and then it will stop restarting. NOTE: bulk download errors will cause one failure+restart iterations
     volumes:
-      - .:/dockermount
+      - type: bind
+        source: .
+        target: /usaspending-api
+        read_only: false
       # Required to interact with host's docker daemon from within this running container,
       # to submit a spark job via the spark-submit container (see download_sqs_worker._run_spark_download)
       - /var/run/docker.sock:/var/run/docker.sock
-      # Python is installed and managed in the image; avoid conflicts with local python .venv
-      - type: volume
-        source: local_venv
-        target: /dockermount/.venv
     command: python3 manage.py download_sqs_worker
     environment:
       <<: *application-vars
@@ -321,7 +313,10 @@ services:
     image: minio/minio:RELEASE.2022-04-12T06-55-35Z
     container_name: minio
     volumes:
-      - .:/dockermount
+      - type: bind
+        source: .
+        target: /usaspending-api
+        read_only: false
       - type: bind
         source: ${MINIO_DATA_DIR}
         target: /data
@@ -342,8 +337,8 @@ services:
         # Create the bucket for MinIO used for Spark
         mkdir -p data/data/files
         # Populate initial files in buckets
-        cp dockermount/usaspending_api/data/Data_Dictionary_Crosswalk.xlsx data/dti-da-public-files-nonprod/user_reference_docs/Data_Dictionary_Crosswalk.xlsx
-        cp dockermount/usaspending_api/data/COVID-19_download_readme.txt data/data/files/COVID-19_download_readme.txt
+        cp usaspending-api/usaspending_api/data/Data_Dictionary_Crosswalk.xlsx data/dti-da-public-files-nonprod/user_reference_docs/Data_Dictionary_Crosswalk.xlsx
+        cp usaspending-api/usaspending_api/data/COVID-19_download_readme.txt data/data/files/COVID-19_download_readme.txt
         minio server --address ":10001" --console-address ":10002" /data
       "
     healthcheck:
@@ -378,13 +373,8 @@ services:
     volumes:
       - type: bind
         source: .
-        target: /dockermount
+        target: /usaspending-api
         read_only: false
-      # Python is installed and managed in the image using uv; avoid conflicts with local python .venv
-      - type: volume
-        source: local_venv
-        target: /dockermount/.venv
-
   spark-worker:
     profiles:
       - spark  # must pass --profile spark to docker compose for this to come up
@@ -408,13 +398,8 @@ services:
     volumes:
       - type: bind
         source: .
-        target: /dockermount
+        target: /usaspending-api
         read_only: false
-      # Python is installed and managed in the image using uv; avoid conflicts with local python .venv
-      - type: volume
-        source: local_venv
-        target: /dockermount/.venv
-
   spark-history-server:
     profiles:
       - spark  # must pass --profile spark to docker compose for this to come up
@@ -436,12 +421,8 @@ services:
     volumes:
       - type: bind
         source: .
-        target: /dockermount
+        target: /usaspending-api
         read_only: false
-      # Python is installed and managed in the image using uv; avoid conflicts with local python .venv
-      - type: volume
-        source: local_venv
-        target: /dockermount/.venv
 
   # Example of running spark-submit container:
   #  NOTE: double check package dependency versions here with those used in unit tests (conftest_spark.py), as these docs could have gotten stale
@@ -464,7 +445,7 @@ services:
   #
   #    make docker-compose-run profiles="--profile spark" args="--rm -e MINIO_HOST=minio -e JDBC_URL -e COMPONENT_NAME='My Spark Prototype Script' spark-submit \
   #      --packages <SEE MAKEFILE> \
-  #      /dockermount/usaspending_api/etl/tests/path_to_your_spark_prototype_script.py"
+  #      /usaspending-api/usaspending_api/etl/tests/path_to_your_spark_prototype_script.py"
 
   spark-submit:
     profiles:
@@ -496,17 +477,12 @@ services:
     volumes:
       - type: bind
         source: .
-        target: /dockermount
+        target: /usaspending-api
         read_only: false
       # NOTE: The hive metastore_db Derby database folder is expected to be configured to show up as a subfolder of the spark-warehouse dir
       - type: bind
         source: ${SOURCE_SPARK_SQL_WAREHOUSE_DIR}
         target: ${TARGET_SPARK_SQL_WAREHOUSE_DIR}
-      # Python is installed and managed in the image using uv; avoid conflicts with local python .venv
-      - type: volume
-        source: local_venv
-        target: /dockermount/.venv
-
   spark-shell:
     profiles:
       - spark  # must pass --profile spark to docker compose for this to come up
@@ -545,17 +521,13 @@ services:
     volumes:
       - type: bind
         source: .
-        target: /dockermount
+        target: /usaspending-api
         read_only: false
       # NOTE: The hive metastore_db Derby database folder is expected to be configured to show up as a subfolder of the spark-warehouse dir
       - type: bind
         source: ${SOURCE_SPARK_SQL_WAREHOUSE_DIR}
         target: ${TARGET_SPARK_SQL_WAREHOUSE_DIR}
         read_only: false
-      # Python is installed and managed in the image using uv; avoid conflicts with local python .venv
-      - type: volume
-        source: local_venv
-        target: /dockermount/.venv
 
   otel-collector:
     profiles:
diff --git a/download.Dockerfile b/download.Dockerfile
index 0cf278072f..bacaa898f1 100644
--- a/download.Dockerfile
+++ b/download.Dockerfile
@@ -1,6 +1,6 @@
 FROM usaspending-backend:latest
 
-WORKDIR /dockermount
+WORKDIR /usaspending-api
 
 RUN apt update && apt install -y \
     build-essential \

From 7322c62b7fae0db72bc815ae08f9f885dc034949 Mon Sep 17 00:00:00 2001
From: Seth Stoudenmier <stoudenmier_seth@bah.com>
Date: Tue, 19 May 2026 16:34:17 -0400
Subject: [PATCH 21/48] [DEV-14608] troubleshoot github actions

---
 .github/actions/init-test-environment/action.yaml      |  6 ++++++
 .../test-non-spark-integration-signal-handling.yaml    |  4 ----
 .github/workflows/test-non-spark-integration.yaml      |  6 +++++-
 .../test-spark-integration-load-to-from-delta.yaml     |  4 ----
 ...-spark-integration-load-transactions-fabs-fpds.yaml |  4 ----
 ...est-spark-integration-load-transactions-lookup.yaml |  4 ----
 .github/workflows/test-spark-integration-other.yaml    |  4 ----
 .github/workflows/test-unit.yaml                       | 10 ++++------
 development.Dockerfile                                 |  2 +-
 9 files changed, 16 insertions(+), 28 deletions(-)

diff --git a/.github/actions/init-test-environment/action.yaml b/.github/actions/init-test-environment/action.yaml
index 3275242b4e..98280428b4 100644
--- a/.github/actions/init-test-environment/action.yaml
+++ b/.github/actions/init-test-environment/action.yaml
@@ -7,6 +7,9 @@ inputs:
   is-spark-test:
     description: Triggers build steps needed to support tests that utilize Spark
     required: true
+  working-directory:
+    description: Directory where the requirements can be found; used when multiple repos are checked out
+    default: ""
 
 runs:
   using: composite
@@ -23,6 +26,7 @@ runs:
       run: mkdir -p "$HOME/Development/data/usaspending/docker/usaspending-s3"  # same value as .env.template
 
     - name: Build docker containers for DB, ES, and Minio
+      working-directory: ${{ inputs.working-directory }}
       if: ${{ inputs.is-integration-test == 'true' }}
       shell: bash
       run: docker compose up -d usaspending-db usaspending-es minio
@@ -56,11 +60,13 @@ runs:
         docker exec usaspending-db psql -U usaspending -d data_store_api -c "SELECT 'ALTER DEFAULT PRIVILEGES IN SCHEMA ' || nspname || ' GRANT SELECT ON TABLES TO readonly' FROM pg_namespace WHERE nspname IN ('public','raw','int','rpt','temp')"
 
     - name: Build the usaspending-development image
+      working-directory: ${{ inputs.working-directory }}
       shell: bash
       run: docker compose build usaspending-test
 
     - name: Run initial test to trigger DB creation
       if: ${{ inputs.is-integration-test == 'true' }}
+      working-directory: ${{ inputs.working-directory }}
       shell: bash
       run: >
         docker compose run --rm usaspending-test pytest --create-db --numprocesses auto --no-cov --disable-warnings -r=fEs --verbosity=3 --capture=no --log-cli-level=WARNING --show-capture=log
diff --git a/.github/workflows/test-non-spark-integration-signal-handling.yaml b/.github/workflows/test-non-spark-integration-signal-handling.yaml
index 2e2c42b504..3ef463d7ab 100644
--- a/.github/workflows/test-non-spark-integration-signal-handling.yaml
+++ b/.github/workflows/test-non-spark-integration-signal-handling.yaml
@@ -7,10 +7,6 @@ on:
         required: true
         type: string
 
-defaults:
-  run:
-    working-directory: usaspending-api
-
 jobs:
   Run:
     name: Run
diff --git a/.github/workflows/test-non-spark-integration.yaml b/.github/workflows/test-non-spark-integration.yaml
index 31656ecc02..87ad812a90 100644
--- a/.github/workflows/test-non-spark-integration.yaml
+++ b/.github/workflows/test-non-spark-integration.yaml
@@ -14,12 +14,14 @@ jobs:
     steps:
       - name: Checkout Source Repository
         uses: actions/checkout@v4
+        with:
+          path: usaspending-api
 
       - name: Checkout Broker Backend Repository
         uses: actions/checkout@v4
         with:
           repository: fedspendingtransparency/data-act-broker-backend
-          path: ../data-act-broker-backend
+          path: data-act-broker-backend
           ref: ${{ inputs.broker-branch }}
 
       - name: Init Test Environment
@@ -27,6 +29,7 @@ jobs:
         with:
           is-integration-test: true
           is-spark-test: false
+          working-directory: ./usaspending-api
 
       - name: Run Test Cases
         uses: ./.github/actions/run-pytest
@@ -34,3 +37,4 @@ jobs:
           cov-report-name: 'non-spark-integration-tests'
           include-glob: '**/tests/integration/*'
           marker: '(not signal_handling and not spark)'
+          working-directory: ./usaspending-api
diff --git a/.github/workflows/test-spark-integration-load-to-from-delta.yaml b/.github/workflows/test-spark-integration-load-to-from-delta.yaml
index 296f315430..ba3ea489a5 100644
--- a/.github/workflows/test-spark-integration-load-to-from-delta.yaml
+++ b/.github/workflows/test-spark-integration-load-to-from-delta.yaml
@@ -7,10 +7,6 @@ on:
         required: true
         type: string
 
-defaults:
-  run:
-    working-directory: ./usaspending-api
-
 jobs:
   Run:
     name: Run
diff --git a/.github/workflows/test-spark-integration-load-transactions-fabs-fpds.yaml b/.github/workflows/test-spark-integration-load-transactions-fabs-fpds.yaml
index 52da316493..48f20fdec3 100644
--- a/.github/workflows/test-spark-integration-load-transactions-fabs-fpds.yaml
+++ b/.github/workflows/test-spark-integration-load-transactions-fabs-fpds.yaml
@@ -7,10 +7,6 @@ on:
         required: true
         type: string
 
-defaults:
-  run:
-    working-directory: ./usaspending-api
-
 jobs:
   Run:
     name: Run
diff --git a/.github/workflows/test-spark-integration-load-transactions-lookup.yaml b/.github/workflows/test-spark-integration-load-transactions-lookup.yaml
index ed36927e74..1f6b870d65 100644
--- a/.github/workflows/test-spark-integration-load-transactions-lookup.yaml
+++ b/.github/workflows/test-spark-integration-load-transactions-lookup.yaml
@@ -7,10 +7,6 @@ on:
         required: true
         type: string
 
-defaults:
-  run:
-    working-directory: ./usaspending-api
-
 jobs:
   Run:
     name: Run
diff --git a/.github/workflows/test-spark-integration-other.yaml b/.github/workflows/test-spark-integration-other.yaml
index 2f1526f655..08ca247009 100644
--- a/.github/workflows/test-spark-integration-other.yaml
+++ b/.github/workflows/test-spark-integration-other.yaml
@@ -7,10 +7,6 @@ on:
         required: true
         type: string
 
-defaults:
-  run:
-    working-directory: ./usaspending-api
-
 jobs:
   Run:
     name: Run
diff --git a/.github/workflows/test-unit.yaml b/.github/workflows/test-unit.yaml
index e9311f7574..a81ee42666 100644
--- a/.github/workflows/test-unit.yaml
+++ b/.github/workflows/test-unit.yaml
@@ -10,18 +10,15 @@ jobs:
     steps:
       - name: Checkout Source Repository
         uses: actions/checkout@v4
-
-      - name: Validate file structure
-        shell: bash
-        run: |
-          pwd
-          ls
+        with:
+          path: usaspending-api
 
       - name: Init Test Environment
         uses: ./.github/actions/init-test-environment
         with:
           is-integration-test: false
           is-spark-test: false
+          working-directory: ./usaspending-api
 
       - name: Run Test Cases
         id: run-test-cases
@@ -31,3 +28,4 @@ jobs:
           ignore-glob: '**/tests/integration/*'
           include-glob: 'test_*.py *_test.py'
           marker: '(not signal_handling)'
+          working-directory: ./usaspending-api
diff --git a/development.Dockerfile b/development.Dockerfile
index b1fd50b3bf..973938aee3 100644
--- a/development.Dockerfile
+++ b/development.Dockerfile
@@ -32,7 +32,7 @@ RUN npm install --global dredd@13.1.2
 WORKDIR /usaspending-api
 
 # Allow git operations to be run inside of the container
-# RUN git config --global --add safe.directory /usaspending-api
+RUN git config --global --add safe.directory /usaspending-api
 
 ##### The following ENV vars are optimizations from https://github.com/astral-sh/uv-docker-example/blob/main/Dockerfile
 ##### and https://docs.astral.sh/uv/guides/integration/docker/#optimizations

From c669f0cf643535c9f20a0375fc3a980cbee45c50 Mon Sep 17 00:00:00 2001
From: Seth Stoudenmier <stoudenmier_seth@bah.com>
Date: Tue, 19 May 2026 16:39:36 -0400
Subject: [PATCH 22/48] [DEV-14608] troubleshoot github actions

---
 .github/actions/init-test-environment/action.yaml | 1 +
 .github/workflows/test-non-spark-integration.yaml | 4 ++--
 .github/workflows/test-unit.yaml                  | 4 ++--
 3 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/.github/actions/init-test-environment/action.yaml b/.github/actions/init-test-environment/action.yaml
index 98280428b4..d39233e501 100644
--- a/.github/actions/init-test-environment/action.yaml
+++ b/.github/actions/init-test-environment/action.yaml
@@ -15,6 +15,7 @@ runs:
   using: composite
   steps:
     - name: Create .env for docker compose
+      working-directory: ${{ inputs.working-directory }}
       shell: bash
       run: cp .env.template .env
 
diff --git a/.github/workflows/test-non-spark-integration.yaml b/.github/workflows/test-non-spark-integration.yaml
index 87ad812a90..9fa61d0c74 100644
--- a/.github/workflows/test-non-spark-integration.yaml
+++ b/.github/workflows/test-non-spark-integration.yaml
@@ -25,14 +25,14 @@ jobs:
           ref: ${{ inputs.broker-branch }}
 
       - name: Init Test Environment
-        uses: ./.github/actions/init-test-environment
+        uses: ./usaspending-api/.github/actions/init-test-environment
         with:
           is-integration-test: true
           is-spark-test: false
           working-directory: ./usaspending-api
 
       - name: Run Test Cases
-        uses: ./.github/actions/run-pytest
+        uses: ./usaspending-api/.github/actions/run-pytest
         with:
           cov-report-name: 'non-spark-integration-tests'
           include-glob: '**/tests/integration/*'
diff --git a/.github/workflows/test-unit.yaml b/.github/workflows/test-unit.yaml
index a81ee42666..7901864eca 100644
--- a/.github/workflows/test-unit.yaml
+++ b/.github/workflows/test-unit.yaml
@@ -14,7 +14,7 @@ jobs:
           path: usaspending-api
 
       - name: Init Test Environment
-        uses: ./.github/actions/init-test-environment
+        uses: ./usaspending-api/.github/actions/init-test-environment
         with:
           is-integration-test: false
           is-spark-test: false
@@ -22,7 +22,7 @@ jobs:
 
       - name: Run Test Cases
         id: run-test-cases
-        uses: ./.github/actions/run-pytest
+        uses: ./usaspending-api/.github/actions/run-pytest
         with:
           cov-report-name: 'unit-tests'
           ignore-glob: '**/tests/integration/*'

From 1a55247bd9fe1a6b5d2297f46fd3d260e10b9976 Mon Sep 17 00:00:00 2001
From: Seth Stoudenmier <stoudenmier_seth@bah.com>
Date: Wed, 20 May 2026 11:45:31 -0400
Subject: [PATCH 23/48] [DEV-14608] adjust test cases

---
 .env.template                        | 1 +
 usaspending_api/config/envs/local.py | 6 ++++--
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/.env.template b/.env.template
index 653b698170..e91b650c3d 100644
--- a/.env.template
+++ b/.env.template
@@ -65,6 +65,7 @@ SPARK_MASTER_HOST=spark-master
 SPARK_MASTER_PORT=7077
 SPARK_MASTER_WEBUI_PORT=4040
 SPARK_HISTORY_SERVER_PORT=18080
+SPARK_WORKER_WEBUI_PORT=4041
 # Should point to paths where data can be persisted beyond docker restarts
 # NOTE: The specified directory needs to exist before Docker can mount it
 SOURCE_SPARK_SQL_WAREHOUSE_DIR=./spark-warehouse
diff --git a/usaspending_api/config/envs/local.py b/usaspending_api/config/envs/local.py
index 2efb03829b..2533d620de 100644
--- a/usaspending_api/config/envs/local.py
+++ b/usaspending_api/config/envs/local.py
@@ -51,8 +51,10 @@ class LocalConfig(DefaultConfig):
     USASPENDING_DB_PORT: str = "5432"
 
     # ==== [Postgres Broker] ====
-    BROKER_DB_USER: str | None = "admin"
-    BROKER_DB_PASSWORD: SecretStr | None = "root"
+    # These values match the USAspending values above because the test suite will create a new Broker DB on the same
+    # container as the USAspending DB
+    BROKER_DB_USER: str | None = "usaspending"
+    BROKER_DB_PASSWORD: SecretStr | None = "usaspender"
 
     # Change to host.docker.internal if you are running a local Postgres. Otherwise leave as-is, so
     # Docker will use the Postgres created by Compose.

From fdf5b104025bf1129acbe22aa15a8f0253ace21d Mon Sep 17 00:00:00 2001
From: Seth Stoudenmier <stoudenmier_seth@bah.com>
Date: Wed, 20 May 2026 15:25:38 -0400
Subject: [PATCH 24/48] [DEV-14608] make sure processes are cleaned up on
 containers

---
 .github/actions/init-test-environment/action.yaml | 10 +++++-----
 docker-compose.yml                                |  3 +++
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/.github/actions/init-test-environment/action.yaml b/.github/actions/init-test-environment/action.yaml
index d39233e501..addcd027e8 100644
--- a/.github/actions/init-test-environment/action.yaml
+++ b/.github/actions/init-test-environment/action.yaml
@@ -44,6 +44,11 @@ runs:
       if: ${{ inputs.is-integration-test == 'true' }}
       run: docker load --input /tmp/dataact-broker-backend.tar
 
+    - name: Build the usaspending-development image
+      working-directory: ${{ inputs.working-directory }}
+      shell: bash
+      run: docker compose build usaspending-test
+
     - name: Wait on DB and ES containers to be available
       if: ${{ inputs.is-integration-test == 'true' }}
       shell: bash
@@ -60,11 +65,6 @@ runs:
         docker exec usaspending-db psql -U usaspending -d data_store_api -c "SELECT 'GRANT SELECT ON ALL TABLES IN SCHEMA ' || nspname || ' TO readonly' FROM pg_namespace WHERE nspname IN ('public','raw','int','rpt','temp')"
         docker exec usaspending-db psql -U usaspending -d data_store_api -c "SELECT 'ALTER DEFAULT PRIVILEGES IN SCHEMA ' || nspname || ' GRANT SELECT ON TABLES TO readonly' FROM pg_namespace WHERE nspname IN ('public','raw','int','rpt','temp')"
 
-    - name: Build the usaspending-development image
-      working-directory: ${{ inputs.working-directory }}
-      shell: bash
-      run: docker compose build usaspending-test
-
     - name: Run initial test to trigger DB creation
       if: ${{ inputs.is-integration-test == 'true' }}
       working-directory: ${{ inputs.working-directory }}
diff --git a/docker-compose.yml b/docker-compose.yml
index f2e70a63c0..760d9abaf3 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -107,6 +107,7 @@ services:
       dockerfile: development.Dockerfile
       args:
         PROJECT_LOG_DIR: ${PROJECT_LOG_DIR}
+    init: true  # needed for the reaping of processes created for tests
     container_name: usaspending-test
     volumes:
       - type: bind
@@ -212,6 +213,7 @@ services:
     image: usaspending-backend  # when an image by this name is not found in the local repo, and it is forced to build, it will use this as the tag
     build: .
     container_name: usaspending-api
+    init: true  # needed for the reaping of processes; mostly related to downloads run in a process
     volumes:
       - type: bind
         source: .
@@ -235,6 +237,7 @@ services:
     build:
       context: .
       dockerfile: download.Dockerfile
+    init: true  # needed for the reaping of processes
     container_name: usaspending-bulk-download
     restart: on-failure:5 # 5 max attempt, and then it will stop restarting. NOTE: bulk download errors will cause one failure+restart iterations
     volumes:

From be85dc6e11d981276fd80d18f3b599a2eab06a8d Mon Sep 17 00:00:00 2001
From: Seth Stoudenmier <stoudenmier_seth@bah.com>
Date: Wed, 20 May 2026 16:17:17 -0400
Subject: [PATCH 25/48] [DEV-14608] undo removing search_path in github action

---
 .github/actions/init-test-environment/action.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/actions/init-test-environment/action.yaml b/.github/actions/init-test-environment/action.yaml
index addcd027e8..3c97dc2d67 100644
--- a/.github/actions/init-test-environment/action.yaml
+++ b/.github/actions/init-test-environment/action.yaml
@@ -60,6 +60,7 @@ runs:
       if: ${{ inputs.is-integration-test == 'true' }}
       shell: bash
       run: |
+        docker exec usaspending-db psql -U usaspending -d data_store_api -c "ALTER USER usaspending SET search_path TO public,raw,int,temp,rpt"
         docker exec usaspending-db psql -U usaspending -d data_store_api -c "CREATE ROLE readonly"
         docker exec usaspending-db psql -U usaspending -d data_store_api -c "SELECT 'GRANT USAGE ON SCHEMA ' || nspname || ' TO readonly' FROM pg_namespace WHERE nspname IN ('public','raw','int','rpt','temp')"
         docker exec usaspending-db psql -U usaspending -d data_store_api -c "SELECT 'GRANT SELECT ON ALL TABLES IN SCHEMA ' || nspname || ' TO readonly' FROM pg_namespace WHERE nspname IN ('public','raw','int','rpt','temp')"

From bccb9c1be82635081af6a7d8aba2e72ce62c1cc5 Mon Sep 17 00:00:00 2001
From: Seth Stoudenmier <stoudenmier_seth@bah.com>
Date: Thu, 21 May 2026 10:21:43 -0400
Subject: [PATCH 26/48] [DEV-14608] add postgres package to dockerfile for psql

---
 .env.template                                 |  2 ++
 development.Dockerfile                        | 13 +++++++++
 docker-compose.yml                            |  8 ++++--
 .../test_setup_of_spark_dependencies.py       | 27 -------------------
 4 files changed, 21 insertions(+), 29 deletions(-)
 delete mode 100644 usaspending_api/tests/integration/test_setup_of_spark_dependencies.py

diff --git a/.env.template b/.env.template
index e91b650c3d..c5ddd28978 100644
--- a/.env.template
+++ b/.env.template
@@ -61,6 +61,8 @@ ES_HOST=usaspending-es
 ES_PORT=9200
 
 # ==== [Spark] ====
+# Logging level defaults to "warn" to prevent overloading log output in test suite
+SPARK_LOGGING_LEVEL=WARN
 SPARK_MASTER_HOST=spark-master
 SPARK_MASTER_PORT=7077
 SPARK_MASTER_WEBUI_PORT=4040
diff --git a/development.Dockerfile b/development.Dockerfile
index 973938aee3..c0c676bdfe 100644
--- a/development.Dockerfile
+++ b/development.Dockerfile
@@ -9,6 +9,7 @@ COPY --from=ghcr.io/astral-sh/uv:0.7.19 /uv /uvx /bin/
 # Build ARGs
 ARG PROJECT_LOG_DIR=/logs
 ARG PYTHON_VERSION=3.10.12
+ARG SPARK_LOGGING_LEVEL=INFO
 
 # Install dependencies
 RUN dnf update \
@@ -18,6 +19,7 @@ RUN dnf update \
         libpq-devel \
         nodejs \
         npm \
+        postgresql16 \
         sqlite-devel \
         wget \
         zlib-devel \
@@ -61,6 +63,17 @@ ENV PATH="/usr/local/.venv/bin:$PATH"
 ENV PYSPARK_PYTHON=/usr/local/.venv/bin/python3
 ENV PYSPARK_DRIVER_PYTHON=/usr/local/.venv/bin/python3
 
+# Configure console logging for Spark
+RUN cat <<EOF | sed 's/^[[:space:]]*//' >> $SPARK_HOME/conf/log4j2.properties
+	appender.console.type = Console
+    appender.console.name = CONSOLE
+    appender.console.layout.type = PatternLayout
+    appender.console.layout.pattern = [%d{yyyy-MM-dd HH:mm:ss.SSS}][%p] - %m%n
+    rootLogger.level = ${SPARK_LOGGING_LEVEL}
+    rootLogger.appenderRef.0.ref = CONSOLE
+    rootLogger.appenderRef.0.level = ${SPARK_LOGGING_LEVEL}
+EOF
+
 # Update logging for Spark
 RUN sed -i "s|^# spark.eventLog.dir.*$|spark.eventLog.dir                 file:///usaspending-api/$PROJECT_LOG_DIR/spark-events|g" $SPARK_HOME/conf/spark-defaults.conf \
     && sed -i '/spark.eventLog.enabled/s/^# //g' $SPARK_HOME/conf/spark-defaults.conf \
diff --git a/docker-compose.yml b/docker-compose.yml
index 760d9abaf3..54db92fca6 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -35,8 +35,6 @@ x-spark-submit-variables: &spark-submit-vars
   AWS_ACCESS_KEY_ID: ${MINIO_ACCESS_KEY}
   AWS_ENDPOINT_URL: http://${MINIO_HOST}:${MINIO_PORT}
   AWS_SECRET_ACCESS_KEY: ${MINIO_SECRET_KEY}
-  #SPARK_HIVE_METASTORE_DB_TYPE: derby
-  #SPARK_HIVE_METASTORE_DB_PATH: /spark-warehouse
   # Needed for the Spark containers to interact
   SPARK_MASTER_HOST: ${SPARK_MASTER_HOST}
   SPARK_MASTER_PORT: ${SPARK_MASTER_PORT}
@@ -107,6 +105,7 @@ services:
       dockerfile: development.Dockerfile
       args:
         PROJECT_LOG_DIR: ${PROJECT_LOG_DIR}
+        SPARK_LOGGING_LEVEL: ${SPARK_LOGGING_LEVEL}
     init: true  # needed for the reaping of processes created for tests
     container_name: usaspending-test
     volumes:
@@ -361,6 +360,7 @@ services:
       dockerfile: development.Dockerfile
       args:
         PROJECT_LOG_DIR: ${PROJECT_LOG_DIR}
+        SPARK_LOGGING_LEVEL: ${SPARK_LOGGING_LEVEL}
     container_name: spark-master
     environment:
       <<: *spark-submit-vars
@@ -389,6 +389,7 @@ services:
       dockerfile: development.Dockerfile
       args:
         PROJECT_LOG_DIR: ${PROJECT_LOG_DIR}
+        SPARK_LOGGING_LEVEL: ${SPARK_LOGGING_LEVEL}
     container_name: spark-worker
     depends_on:
       - spark-master
@@ -414,6 +415,7 @@ services:
       dockerfile: development.Dockerfile
       args:
         PROJECT_LOG_DIR: ${PROJECT_LOG_DIR}
+        SPARK_LOGGING_LEVEL: ${SPARK_LOGGING_LEVEL}
     container_name: spark-history-server
     environment:
       SPARK_HISTORY_SERVER_PORT: ${SPARK_HISTORY_SERVER_PORT}
@@ -461,6 +463,7 @@ services:
       dockerfile: development.Dockerfile
       args:
         PROJECT_LOG_DIR: ${PROJECT_LOG_DIR}
+        SPARK_LOGGING_LEVEL: ${SPARK_LOGGING_LEVEL}
     container_name: spark-submit
     depends_on:
       - spark-master
@@ -498,6 +501,7 @@ services:
       dockerfile: development.Dockerfile
       args:
         PROJECT_LOG_DIR: ${PROJECT_LOG_DIR}
+        SPARK_LOGGING_LEVEL: ${SPARK_LOGGING_LEVEL}
     container_name: spark-shell
     depends_on:
       - minio
diff --git a/usaspending_api/tests/integration/test_setup_of_spark_dependencies.py b/usaspending_api/tests/integration/test_setup_of_spark_dependencies.py
deleted file mode 100644
index 5b886e2c77..0000000000
--- a/usaspending_api/tests/integration/test_setup_of_spark_dependencies.py
+++ /dev/null
@@ -1,27 +0,0 @@
-"""
-This module should contain only the one test.
-It is to be called by name in a pytest session that is used to prime or prepare a second pytest session where the
-bulk of tests are performed. The intention of this test is to trigger the loading of Spark JAR dependencies to the
-Ivy dependency management cache directory, so subsequent tests do not need to re-download them. This works around a
-collision that happens using pytest-xdist where multiple pytest sessions from multiple workers try to download
-dependencies simultaneously and fail because of race conditions.
-
-NOTE: It should be run in a single-process pytest session (no xdist parallel sessions)!
-"""
-
-from pyspark.sql import SparkSession
-
-
-def test_preload_spark_jars(spark: SparkSession):
-    """See module docstring above.
-
-    Run this test alone, like:
-
-        pytest --no-cov --disable-warnings -r=fEs -vv 'usaspending_api/tests/integration/test_setup_of_spark_dependencies.py::test_preload_spark_jars'
-
-    If running in a Continuous Integration pipeline (or if you want to nuke and pave your own local test DBs,
-    add --create-db to ensure that the test DBs are (re)created from scratch
-    """
-    # The prep work here is done by the spark fixture loading dependencies
-    assert spark.sparkContext._jsc, "Could not get a reference to the Java SparkContext from PySpark SparkSession"
-    assert spark.sparkContext._jsc.sc().listJars().size() > 0

From 70946380b91d14f2ca1ca9b5ef83f6ae2a7456e9 Mon Sep 17 00:00:00 2001
From: Seth Stoudenmier <stoudenmier_seth@bah.com>
Date: Thu, 21 May 2026 14:42:47 -0400
Subject: [PATCH 27/48] [DEV-14608] fix tests

---
 usaspending_api/common/spark/jobs.py             |  4 +++-
 .../tests/integration/test_load_to_from_delta.py | 16 ++++++++++------
 2 files changed, 13 insertions(+), 7 deletions(-)

diff --git a/usaspending_api/common/spark/jobs.py b/usaspending_api/common/spark/jobs.py
index 16ae69054c..1f62fdd0ab 100644
--- a/usaspending_api/common/spark/jobs.py
+++ b/usaspending_api/common/spark/jobs.py
@@ -234,7 +234,9 @@ def _run_in_container(job_name: str, command_name: str, command_options: list[st
             image_name,
             name=container_name,
             network=network,
-            command=(f'spark-submit --driver-memory "2g" /dockermount/manage.py {command_name} {options_as_string}'),
+            command=(
+                f'spark-submit --driver-memory "2g" /usaspending-api/manage.py {command_name} {options_as_string}'
+            ),
             environment=[
                 f"COMPONENT_NAME={command_name} {options_as_string}",
                 *environment_variables,
diff --git a/usaspending_api/etl/tests/integration/test_load_to_from_delta.py b/usaspending_api/etl/tests/integration/test_load_to_from_delta.py
index 57c8522092..cef52f21d7 100644
--- a/usaspending_api/etl/tests/integration/test_load_to_from_delta.py
+++ b/usaspending_api/etl/tests/integration/test_load_to_from_delta.py
@@ -987,19 +987,21 @@ def test_incremental_load_table_to_delta_for_award_search(
         .table("award_search")
         .select(["award_id", "_change_type", "_commit_version"])
         .toPandas()
+        .sort_values(by=["award_id", "_change_type", "_commit_version"])
+        .reset_index(drop=True)
     )
     expected = pd.DataFrame(
         {
-            "award_id": [4, 4, 1, 3, 2, 4],
+            "award_id": [1, 2, 3, 4, 4, 4],
             "_change_type": [
-                "delete",
                 "insert",
                 "insert",
                 "insert",
+                "delete",
                 "insert",
                 "insert",
             ],
-            "_commit_version": [2, 3, 1, 1, 1, 1],
+            "_commit_version": [1, 1, 1, 2, 1, 3],
         }
     )
     pd.testing.assert_frame_equal(result, expected)
@@ -1085,21 +1087,23 @@ def test_incremental_load_table_to_delta_for_transaction_search(
         .table("transaction_search")
         .select(["transaction_id", "_change_type", "_commit_version"])
         .toPandas()
+        .sort_values(by=["transaction_id", "_change_type", "_commit_version"])
+        .reset_index(drop=True)
     )
     expected = pd.DataFrame(
         {
-            "transaction_id": [4, 4, 1, 2, 434, 3, 4, 5],
+            "transaction_id": [1, 2, 3, 4, 4, 4, 5, 434],
             "_change_type": [
-                "delete",
                 "insert",
                 "insert",
                 "insert",
+                "delete",
                 "insert",
                 "insert",
                 "insert",
                 "insert",
             ],
-            "_commit_version": [2, 3, 1, 1, 1, 1, 1, 1],
+            "_commit_version": [1, 1, 1, 2, 1, 3, 1, 1],
         }
     )
     pd.testing.assert_frame_equal(result, expected)

From cb197b2c52bfd45f076687e8e94bedb0f4d5185e Mon Sep 17 00:00:00 2001
From: Seth Stoudenmier <stoudenmier_seth@bah.com>
Date: Thu, 21 May 2026 15:32:29 -0400
Subject: [PATCH 28/48] [DEV-14608] fix test

---
 .../integration/test_execute_spark_sql.py     | 33 ++++++++++++++-----
 1 file changed, 25 insertions(+), 8 deletions(-)

diff --git a/usaspending_api/etl/tests/integration/test_execute_spark_sql.py b/usaspending_api/etl/tests/integration/test_execute_spark_sql.py
index 200ca77252..aeb4164052 100644
--- a/usaspending_api/etl/tests/integration/test_execute_spark_sql.py
+++ b/usaspending_api/etl/tests/integration/test_execute_spark_sql.py
@@ -1,10 +1,8 @@
-from pytest import raises
-
-from unittest.mock import patch, MagicMock
+from unittest.mock import MagicMock, patch
 
 from django.core.management import call_command
 from django.core.management.base import CommandError
-
+from pytest import raises
 
 # Argument Validation
 
@@ -31,25 +29,44 @@ def test_no_sql_and_file_raises_error():
 def test_sql_functionality():
     """Tests a variety of cases in a single test for Spark performance"""
 
+    mock_spark = MagicMock()
+
     # Test --sql approach
-    with patch("usaspending_api.etl.management.commands.execute_spark_sql.logger") as mock_logger:
+    with (
+        patch(
+            "usaspending_api.etl.management.commands.execute_spark_sql.get_active_spark_session",
+            return_value=mock_spark,
+        ),
+        patch("usaspending_api.etl.management.commands.execute_spark_sql.logger") as mock_logger,
+    ):
         call_command("execute_spark_sql", "--sql", "SHOW DATABASES; SELECT 1;")
         mock_logger.info.assert_any_call("Found 2 SQL statement(s)")
 
     # Test --file approach with two SQL statements and a blank line
     test_file_path = "usaspending_api/etl/tests/data/test_execute_spark_files/two_queries_with_blank_line.sql"
-    with patch("usaspending_api.etl.management.commands.execute_spark_sql.logger") as mock_logger:
+    with (
+        patch(
+            "usaspending_api.etl.management.commands.execute_spark_sql.get_active_spark_session",
+            return_value=mock_spark,
+        ),
+        patch("usaspending_api.etl.management.commands.execute_spark_sql.logger") as mock_logger,
+    ):
         call_command("execute_spark_sql", "--file", test_file_path)
         mock_logger.info.assert_any_call("Found 2 SQL statement(s)")
 
     # Test --file approach with an empty file
     test_file_path = "usaspending_api/etl/tests/data/test_execute_spark_files/empty_file.sql"
-    with patch("usaspending_api.etl.management.commands.execute_spark_sql.logger") as mock_logger:
+    with (
+        patch(
+            "usaspending_api.etl.management.commands.execute_spark_sql.get_active_spark_session",
+            return_value=mock_spark,
+        ),
+        patch("usaspending_api.etl.management.commands.execute_spark_sql.logger") as mock_logger,
+    ):
         call_command("execute_spark_sql", "--file", test_file_path)
         mock_logger.info.assert_any_call("Found 0 SQL statement(s)")
 
     # Test SQL executes
-    mock_spark = MagicMock()
     with patch(
         "usaspending_api.etl.management.commands.execute_spark_sql.get_active_spark_session", return_value=mock_spark
     ):

From ffc80e3ab4a245221c4472d61c61e0d32e4ab77e Mon Sep 17 00:00:00 2001
From: Seth Stoudenmier <stoudenmier_seth@bah.com>
Date: Tue, 26 May 2026 17:04:50 -0400
Subject: [PATCH 29/48] [DEV-14608] troubleshoot slow merge statement

---
 development.Dockerfile                          | 17 +++++++++--------
 usaspending_api/common/helpers/spark_helpers.py |  3 +++
 2 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/development.Dockerfile b/development.Dockerfile
index c0c676bdfe..130999eb7b 100644
--- a/development.Dockerfile
+++ b/development.Dockerfile
@@ -66,21 +66,22 @@ ENV PYSPARK_DRIVER_PYTHON=/usr/local/.venv/bin/python3
 # Configure console logging for Spark
 RUN cat <<EOF | sed 's/^[[:space:]]*//' >> $SPARK_HOME/conf/log4j2.properties
 	appender.console.type = Console
-    appender.console.name = CONSOLE
+    appender.console.name = console
     appender.console.layout.type = PatternLayout
     appender.console.layout.pattern = [%d{yyyy-MM-dd HH:mm:ss.SSS}][%p] - %m%n
+
     rootLogger.level = ${SPARK_LOGGING_LEVEL}
-    rootLogger.appenderRef.0.ref = CONSOLE
-    rootLogger.appenderRef.0.level = ${SPARK_LOGGING_LEVEL}
+    rootLogger.appenderRefs = stdout
+    rootLogger.appenderRef.stdout.ref = console
 EOF
 
 # Update logging for Spark
-RUN sed -i "s|^# spark.eventLog.dir.*$|spark.eventLog.dir                 file:///usaspending-api/$PROJECT_LOG_DIR/spark-events|g" $SPARK_HOME/conf/spark-defaults.conf \
-    && sed -i '/spark.eventLog.enabled/s/^# //g' $SPARK_HOME/conf/spark-defaults.conf \
-    && echo "spark.history.fs.logDirectory      file:///usaspending-api/$PROJECT_LOG_DIR/spark-events" >> $SPARK_HOME/conf/spark-defaults.conf
+RUN sed -i "s|^spark.eventLog.dir.*$|spark.eventLog.dir               file:///usaspending-api/$PROJECT_LOG_DIR/spark-events|g" $SPARK_HOME/conf/spark-defaults.conf \
+    && sed -i "/spark.eventLog.enabled/s/^# //g" $SPARK_HOME/conf/spark-defaults.conf \
+    && sed -i "s|spark.history.fs.logDirectory.*$|spark.history.fs.logDirectory    file:///usaspending-api/$PROJECT_LOG_DIR/spark-events|" $SPARK_HOME/conf/spark-defaults.conf
 
-# Set default values for Spark with local development
-RUN echo "spark.authenticate false" >> $SPARK_HOME/conf/spark-defaults.conf
+# Disable authentication for local development
+RUN sed -i "s|spark.authenticate \{1,\}true|spark.authenticate false|" $SPARK_HOME/conf/spark-defaults.conf
 
 # Ensure Python STDOUT gets sent to container logs
 ENV PYTHONUNBUFFERED=1
diff --git a/usaspending_api/common/helpers/spark_helpers.py b/usaspending_api/common/helpers/spark_helpers.py
index 29a941a8a0..068b27db3c 100644
--- a/usaspending_api/common/helpers/spark_helpers.py
+++ b/usaspending_api/common/helpers/spark_helpers.py
@@ -172,6 +172,9 @@ def configure_spark_session(  # noqa: C901,PLR0912,PLR0913,PLR0915
     # Assume that random errors are rare, and jobs have long runtimes, so fail fast, fix and retry manually.
     conf.set("spark.yarn.maxAppAttempts", "1")
     conf.set("spark.hadoop.fs.s3a.endpoint", CONFIG.AWS_S3_ENDPOINT)
+    # We work with a lot of larger tables and Delta wants to try and materialize the "source" data in a merge statement
+    # into memory which can spill onto disk and cause problems.
+    conf.set("spark.databricks.delta.merge.materializeSource", "none")
 
     if not CONFIG.USE_AWS:  # i.e. running in a "local" [development] environment
         # Set configs to allow the S3AFileSystem to work against a local MinIO object storage proxy

From f72587d8239c7010a86f76c4744b9122a5034c14 Mon Sep 17 00:00:00 2001
From: Seth Stoudenmier <stoudenmier_seth@bah.com>
Date: Wed, 27 May 2026 11:45:47 -0400
Subject: [PATCH 30/48] [DEV-14608] further local config updates; test changes
 for transaction search incremental load

---
 .env.template                                 |  7 ++++
 docker-compose.yml                            | 12 ++++---
 .../dataframes/transaction_search.py          | 33 ++++++++++++++-----
 3 files changed, 39 insertions(+), 13 deletions(-)

diff --git a/.env.template b/.env.template
index c5ddd28978..8a1e8f8770 100644
--- a/.env.template
+++ b/.env.template
@@ -52,6 +52,13 @@ BROKER_DB_USER=usaspending
 BROKER_DB_PASSWORD=usaspender
 BROKER_DB_NAME=data_broker
 
+# If using the Broker DB container created via https://github.com/fedspendingtransparency/data-act-broker-backend
+# then the following values should be used instead
+# BROKER_DB=postgres://admin:root@dataact-broker-db:5432/data_broker
+# BROKER_DB_HOST=dataact-broker-db
+# BROKER_DB_USER=admin
+# BROKER_DB_PASSWORD=root
+
 # ==== [Elasticsearch] ====
 # Where to connect to elasticsearch.
 # All values of ES_* must match what is in ES_HOSTNAME if BOTH are given
diff --git a/docker-compose.yml b/docker-compose.yml
index 54db92fca6..f5b0640330 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -54,11 +54,12 @@ services:
 
   usaspending-db:
     command: postgres -c 'max_connections=500'
-    profiles:
-      - usaspending  # must pass --profile usaspending to docker compose for this to come up, or run a service with one of these other profiles
+    profiles:  # must pass --profile usaspending to docker compose for this to come up, or run a service with one of these other profiles
+      - ci
+      - usaspending
       - manage
+      - spark
       - test
-      - ci
     image: postgres:16-alpine
     healthcheck:
       test: [ "CMD-SHELL", "pg_isready -U ${USASPENDING_DB_USER} -d ${USASPENDING_DB_NAME}" ]
@@ -466,10 +467,11 @@ services:
         SPARK_LOGGING_LEVEL: ${SPARK_LOGGING_LEVEL}
     container_name: spark-submit
     depends_on:
+      - minio
+      - spark-history-server
       - spark-master
       - spark-worker
-      - spark-history-server
-      - minio
+      - usaspending-db
     environment:
       <<: *spark-submit-vars
     # NOTE: entrypoint CANNOT interpolate env vars when processed. They are passed through literally.
diff --git a/usaspending_api/search/delta_models/dataframes/transaction_search.py b/usaspending_api/search/delta_models/dataframes/transaction_search.py
index e334764c06..f6deb5e577 100644
--- a/usaspending_api/search/delta_models/dataframes/transaction_search.py
+++ b/usaspending_api/search/delta_models/dataframes/transaction_search.py
@@ -1001,16 +1001,33 @@ def load_transaction_search(
 def load_transaction_search_incremental(
     spark: SparkSession, destination_database: str, destination_table_name: str
 ) -> None:
-    target = DeltaTable.forName(
+    target_df = spark.table(f"{destination_database}.{destination_table_name}")
+    source_df = TransactionSearch(spark).dataframe
+    to_insert = source_df.join(
+        target_df,
+        on=["transaction_id", "merge_hash_key"],
+        how="left_anti"
+    ).withColumn("merge_action", sf.lit("INSERT"))
+    to_delete = target_df.join(
+        source_df,
+        on=["transaction_id", "merge_hash_key"],
+        how="left_anti"
+    ).withColumn("merge_action", sf.lit("DELETE"))
+    changes_df = to_insert.unionByName(to_delete, allowMissingColumns=True)
+    target_table = DeltaTable.forName(
         spark, f"{destination_database}.{destination_table_name}"
-    ).alias("t")
-    source = TransactionSearch(spark).dataframe.alias("s")
+    )
     (
-        target.merge(
-            source,
-            "s.transaction_id = t.transaction_id and s.merge_hash_key = t.merge_hash_key",
+        target_table.alias("t")
+        .merge(
+            changes_df.alias("c"),
+            "c.transaction_id = t.transaction_id and c.merge_hash_key = t.merge_hash_key",
+        )
+        .whenMatchedDelete(
+            condition="c.merge_action = 'DELETE'"
+        )
+        .whenNotMatchedInsertAll(
+            condition='c.merge_action = "INSERT"',
         )
-        .whenNotMatchedInsertAll()
-        .whenNotMatchedBySourceDelete()
         .execute()
     )

From 5b239cede0e37dbe5bbd0f72d45ebb7905171356 Mon Sep 17 00:00:00 2001
From: Seth Stoudenmier <stoudenmier_seth@bah.com>
Date: Wed, 27 May 2026 15:59:25 -0400
Subject: [PATCH 31/48] [DEV-14608] cleanup and validating docker compose
 further

---
 Makefile           |  7 ++++++-
 README.md          |  3 ++-
 docker-compose.yml | 17 ++++++++---------
 3 files changed, 16 insertions(+), 11 deletions(-)

diff --git a/Makefile b/Makefile
index 21c8da679c..c5651dc129 100644
--- a/Makefile
+++ b/Makefile
@@ -176,7 +176,7 @@ docker-compose-build-development: ## See: docker-compose-build rule. This builds
 style-checks:
 	docker compose --profile=ci --project-directory . --file ${docker_compose_file} run --rm usaspending-style-checks
 
-.PHONY: docker-compose-spark-submit
+.PHONY: spark-submit
 spark-submit: ## Run spark-submit from within local docker containerized infrastructure (which must be running first). Set params with django_command="..."
 	docker compose --profile=spark --project-directory . --file ${docker_compose_file} run --rm \
 		-e COMPONENT_NAME='${django_command}${python_script}' \
@@ -190,3 +190,8 @@ spark-submit: ## Run spark-submit from within local docker containerized infrast
 .PHONY: pyspark-shell
 pyspark-shell: ## Launch a local pyspark REPL shell with all of the packages and spark config pre-set
 	docker compose --profile=spark --project-directory . --file ${docker_compose_file} run --rm spark-shell
+
+.PHONY: django-manage
+django-manage:  ## Run Django management command with the usaspending-manage container
+	docker compose --profile=usaspending --project-directory . --file ${docker_compose_file} run --rm \
+	usaspending-manage ${args}
diff --git a/README.md b/README.md
index 41d082875d..68a0c13675 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,7 @@
 # <p align="center"><img src="https://www.usaspending.gov/img/logo@2x.png" alt="USAspending API"></p>
 
-[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/python/black) [![Pull Request Checks](https://github.com/fedspendingtransparency/usaspending-api/actions/workflows/pull-request-checks.yaml/badge.svg?branch=staging)](https://github.com/fedspendingtransparency/usaspending-api/actions/workflows/pull-request-checks.yaml) [![Test Coverage](https://codeclimate.com/github/fedspendingtransparency/usaspending-api/badges/coverage.svg)](https://codeclimate.com/github/fedspendingtransparency/usaspending-api/coverage) [![Code Climate](https://codeclimate.com/github/fedspendingtransparency/usaspending-api/badges/gpa.svg)](https://codeclimate.com/github/fedspendingtransparency/usaspending-api)
+[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
+[![Pull Request Checks](https://github.com/fedspendingtransparency/usaspending-api/actions/workflows/pull-request-checks.yaml/badge.svg?branch=staging)](https://github.com/fedspendingtransparency/usaspending-api/actions/workflows/pull-request-checks.yaml)
 
 _This API is utilized by USAspending.gov to obtain all federal spending data which is open source and provided to the public as part of the DATA Act._
 
diff --git a/docker-compose.yml b/docker-compose.yml
index f5b0640330..f8e8959f87 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -22,10 +22,6 @@ x-application-variables: &application-vars
   MINIO_SECRET_KEY: ${MINIO_SECRET_KEY}
   MINIO_HOST: ${MINIO_HOST}
   MINIO_PORT: ${MINIO_PORT}
-  # Add network proxy to the containers
-#  HTTP_PROXY: ${HTTP_PROXY}
-#  HTTPS_PROXY: ${HTTPS_PROXY}
-#  NO_PROXY: 127.0.0.1,localhost,minio,${NO_PROXY}
   # Defined for tracing
   OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: ${OTEL_EXPORTER_OTLP_TRACES_ENDPOINT}
   TOGGLE_OTEL_CONSOLE_LOGGING: ${TOGGLE_OTEL_CONSOLE_LOGGING}
@@ -56,10 +52,10 @@ services:
     command: postgres -c 'max_connections=500'
     profiles:  # must pass --profile usaspending to docker compose for this to come up, or run a service with one of these other profiles
       - ci
-      - usaspending
       - manage
       - spark
       - test
+      - usaspending
     image: postgres:16-alpine
     healthcheck:
       test: [ "CMD-SHELL", "pg_isready -U ${USASPENDING_DB_USER} -d ${USASPENDING_DB_NAME}" ]
@@ -121,7 +117,6 @@ services:
       - minio
       - usaspending-db
       - usaspending-es
-    # , "--reuse-db", "--no-cov", "--disable-warnings", "--reuse-db", "--numprocesses", "auto"
     entrypoint: []  # override entrypoint
     environment:
       <<: *spark-submit-vars
@@ -149,9 +144,9 @@ services:
       - -c
       - |
         printf -- "==============\nFormatting code:\n"
-        git diff --name-only --diff-filter=ACMR | grep '\.py$' | xargs uv run ruff format
+        git diff --name-only --diff-filter=ACMR | grep '\.py$' | xargs -r uv run ruff format
         printf -- "==============\nChecking code syntax and quality:\n"
-        git diff --name-only --diff-filter=ACMR | grep '\.py$' | xargs uv run ruff check --fix || exit 1
+        git diff --name-only --diff-filter=ACMR | grep '\.py$' | xargs -r uv run ruff check --fix || exit 1
         printf -- "==============\nChecking API documentation files:\n"
         python3 manage.py check_for_endpoint_documentation
         printf -- "==============\nChecking API contracts:\n"
@@ -166,6 +161,7 @@ services:
     build:
       context: .
       dockerfile: development.Dockerfile
+    init: true  # needed for the reaping of processes created for tests
     container_name: usaspending-ci
     volumes:
       - type: bind
@@ -182,6 +178,7 @@ services:
         condition: service_healthy
       usaspending-style-checks:
         condition: service_completed_successfully
+    entrypoint: []  # override entrypoint
     command:
       - sh
       - -c
@@ -203,9 +200,11 @@ services:
         printf -- "-------\nRunning unit tests with signal handling:\n"
         pytest --override-ini=python_files="test_*.py *_test.py" -m "(signal_handling)" --ignore-glob="**/tests/integration/*" --no-cov --disable-warnings --reuse-db --numprocesses 0
     environment:
-      <<: *application-vars
+      <<: *spark-submit-vars
       # Location in host machine where broker src code root can be found
       DATA_BROKER_SRC_PATH: "${PWD}/../data-act-broker-backend"
+      # Tests will retrieve the root directory from the path and use it to assume the docker network name
+      PROJECT_SRC_PATH: "${PWD}"
 
   usaspending-api:  # builds using the default Dockerfile to mirror deployed environments for local testing
     profiles:

From faf76baba794c642aff6d2051848fcfd4d29d8e1 Mon Sep 17 00:00:00 2001
From: Seth Stoudenmier <stoudenmier_seth@bah.com>
Date: Thu, 28 May 2026 14:29:58 -0400
Subject: [PATCH 32/48] [DEV-14608] cleanup documentation

---
 CONTRIBUTING.md | 16 ++++-----
 README.md       | 93 +++++++++++++++++++++++++------------------------
 2 files changed, 53 insertions(+), 56 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index eb1bd9b836..6ff1325c14 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -30,14 +30,10 @@ In general, follow the form `<prefix>/[dev|gh]-###-short-description`
 
 ## Code Style
 Use of these tools locally creates an excellent chance of passing our automated CI syntax checks.
-* [flake8](https://flake8.pycqa.org/en/latest/)
-    * Will be installed when following the steps in the [readme](README.md)
-    * From repo root dir, run: `flake8`
-    * If using a plugin in an IDE or code editor, ensure the configuration matches the `[flake8]` section in [setup.cfg](setup.cfg)
-* [black](https://black.readthedocs.io/en/stable/)
-    * Will be installed when following the steps in the [readme](README.md)
-    * From repo root dir, run: `black .`
-    * If using a plugin in an IDE or code editor, ensure the Black tool configuration matches the `[tool.black]` section in [pyproject.toml](pyproject.toml)
+* [ruff](https://docs.astral.sh/ruff/)
+    * Installed as part of the dev dependencies
+    * Can be run locally via `uv run ruff check ...` or `uv run ruff format ...`
+    * Optionally, can use the `docker compose --profile ci run --rm usaspending-style-checks` to run all relevant style checks
 
 ### Pre-commit Hooks
 To assist developers it is possible to leverage pre-commit hooks to run several checks before git creates a commit. The Python dependency is included in the requirements files. To get started, run
@@ -115,10 +111,10 @@ This project is in the public domain within the United States, and copyright and
 All contributions to this project will be released under the [CC0 dedication](LICENSE). By submitting a pull request, you are agreeing to comply with this waiver of copyright interest.
 
 ## Additional Resources
-- [USAspending.gov Community](https://usaspending-help.zendesk.com/hc/en-us/community/topics)
+- [USAspending.gov Community](https://onevoicecrm.my.site.com/usaspending/s/)
 - [USAspending Release Notes](https://github.com/fedspendingtransparency/usaspending-website/wiki)
 - [About Federal Spending Transparency](http://fedspendingtransparency.github.io/)
 - [Our Amazing Frontend Repo](https://github.com/fedspendingtransparency/usaspending-website)
-- [Full Dataset as PostgreSQL Dump](https://files.usaspending.gov/database_download/)
+- [Full Dataset as PostgreSQL Dump](https://onevoicecrm.my.site.com/usaspending/s/database-download)
 - [General GitHub Documentation](https://help.github.com/)
 - [GitHub Pull Request Documentation](https://help.github.com/articles/creating-a-pull-request/)
diff --git a/README.md b/README.md
index 68a0c13675..0b4e03961d 100644
--- a/README.md
+++ b/README.md
@@ -26,8 +26,8 @@ _**If not using Docker, you'll need to install app components on your machine:**
     - Windows' WSL bash uses `apt`
     - MacOS users can use [`Homebrew`](https://brew.sh/)
     - Linux users already know their package manager (`yum`, `apt`, `pacman`, etc.)
-- [`PostgreSQL`](https://www.postgresql.org/download/) version 13.x (with a dedicated `data_store_api` database)
-- [`Elasticsearch`](https://www.elastic.co/downloads/elasticsearch) version 7.1
+- [`PostgreSQL`](https://www.postgresql.org/download/) version 16.x (with a dedicated `data_store_api` database)
+- [`OpenSearch`](https://www.elastic.co/downloads/elasticsearch) version 2.9
 - `Python` version 3.10 environment
   - Highly recommended to use a virtual environment. There are various tools and associated instructions depending on preferences
   - See [Required Python Libraries](#required-python-libraries) for an example using `uv`
@@ -74,14 +74,25 @@ _Alternatively, you could skip using `direnv` and just export these variables in
 
 **Just make sure your env vars declared in the shell and in `.env` match for a consistent experience inside and outside of Docker**
 
-### Build `usaspending-backend` Docker Image
-_This image is used as the basis for running application components and running containerized setup services._
+### Build the Docker Images
+
+_:bangbang: Re-run these commands if any python package dependencies change (in `pyproject.toml`/`uv.lock`), since they are baked into the docker image at build-time._
+
+#### usaspending-backend | usaspending-download
+
+_These images are used as the basis for running the API and download containerized setup services._
 
 ```shell
 docker compose --profile usaspending build
 ```
 
-_:bangbang: Re-run this command if any python package dependencies change (in `pyproject.toml`/`uv.lock`), since they are baked into the docker image at build-time._
+#### usaspending-development
+
+_This image is used as the basis for local development including but not limited to: tests, spark jobs, linting, etc._
+
+```shell
+docker compose --profile test build
+```
 
 ### Database Setup
 A postgres database is required to run the app. You can run it in a `postgres` docker container (preferred), or run a PostgreSQL server on your local machine. In either case, it will be empty until data is loaded.
@@ -133,7 +144,7 @@ _To just get essential reference data, you can run:_
     docker compose run --rm usaspending-manage python3 -u manage.py load_reference_data
     ```
 
-_Alternatively, to download a fully populuated production snapshot of the database (full or a subset) and restore it into PostgreSQL, use the `pg_restore` tool as described here: [USAspending Database Download](https://onevoicecrm.my.site.com/usaspending/s/database-download)_
+_Alternatively, to download a fully populated production snapshot of the database (full or a subset) and restore it into PostgreSQL, use the `pg_restore` tool as described here: [USAspending Database Download](https://onevoicecrm.my.site.com/usaspending/s/database-download)_
 
 - Recreate matviews with the command documented in the previous section if this is done
 
@@ -142,22 +153,22 @@ _**Executing individual data-loaders** to load in data is also possible, but req
 - For details on loading reference data, Data Accountability Broker Submissions, and current USAspending data into the API, see [loading_data.md](loading_data.md).
 - For details on how our data loaders modify incoming data, see [data_reformatting.md](data_reformatting.md).
 
-### Elasticsearch Setup
-Some API endpoints reach into Elasticsearch for data.
+### OpenSearch Setup
+Some API endpoints reach into OpenSearch for data.
 
 ```shell
 docker compose --profile usaspending up -d usaspending-es
 ```
-... will create and start a single-node Elasticsearch cluster as a docker container with data persisted to a docker volume.
+... will create and start a single-node OpenSearch cluster as a docker container with data persisted to a docker volume.
 
 - The cluster should be reachable via at http://localhost:9200 ("You Know, for Search").
 
 - Optionally, to see log output, use `docker compose logs usaspending-es` (these logs are stored by docker even if you don't use this).
 
-While not required, it is highly recommended to also create the Kibana docker container for querying the Elasticsearch cluster.
+While not required, it is highly recommended to also create the OpenSearch Dashboards docker container for querying the OpenSearch cluster.
 
 ```shell
-docker compose --profile usaspending up usaspending-kibana-es
+docker compose --profile usaspending up -d usaspending-kibana-es
 ```
 
 #### Generate Elasticsearch Indexes
@@ -165,10 +176,10 @@ The following will generate the indexes:
 
 ```shell
 CURR_DATE=$(date '+%Y-%m-%d-%H-%M-%S')
-docker compose run --rm usaspending-manage python3 -u manage.py elasticsearch_indexer --create-new-index --index-name "$CURR_DATE-transactions" --load-type transaction
-docker compose run --rm usaspending-manage python3 -u manage.py elasticsearch_indexer --create-new-index --index-name "$CURR_DATE-awards" --load-type award
-docker compose run --rm usaspending-manage python3 -u manage.py elasticsearch_indexer --create-new-index --index-name "$CURR_DATE-recipients" --load-type recipient
-docker compose run --rm usaspending-manage python3 -u manage.py elasticsearch_indexer --create-new-index --index-name "$CURR_DATE-subaward" --load-type subaward
+docker compose run --rm usaspending-manage elasticsearch_indexer --create-new-index --index-name "$CURR_DATE-transactions" --load-type transaction
+docker compose run --rm usaspending-manage elasticsearch_indexer --create-new-index --index-name "$CURR_DATE-awards" --load-type award
+docker compose run --rm usaspending-manage elasticsearch_indexer --create-new-index --index-name "$CURR_DATE-recipients" --load-type recipient
+docker compose run --rm usaspending-manage elasticsearch_indexer --create-new-index --index-name "$CURR_DATE-subaward" --load-type subaward
 ```
 
 ### Observability/Tracing Setup
@@ -180,6 +191,23 @@ docker compose --profile tracing up -d
 ```
 When using these containers the `TOGGLE_OTEL_CONSOLE_LOGGING` must be set to `False` and `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` must be set to `http://otel-collector:4318/v1/traces`
 
+### Install python version with uv
+Install the currently supported python version with uv.
+
+This is useful for access to a python shell outside a container. However, it is recommended to use the docker compose containerized services for interaction with the API, spark, and tests.
+
+
+```shell
+uv python install 3.10.12
+```
+
+#### Sync uv project with lockfile
+Setup uv project and check version info:
+
+```shell
+make local-dev-setup
+```
+
 ## Running the API
 
 Run the following to bring up the Django app for the RESTful API:
@@ -208,10 +236,10 @@ Deployed production API endpoints and docs are found by following links here: `h
 
 ### Test Setup
 
-1. Build the base `usaspending-backend` Docker image (the test container is based on this Docker image). In the parent **usaspending-api** directory run:
+1. Build the base `usaspending-development` Docker image (the test container is based on this Docker image). In the parent **usaspending-api** directory run:
 
     ```shell
-    docker build -t usaspending-backend .
+    docker build -t usaspending-development -f development.Dockerfile .
     ```
 
 2. Start the Spark containers for the Spark related tests
@@ -229,40 +257,13 @@ _**NOTE**: If an env var named `BROKER_DB` is set, Broker Integration tests will
 To run tests locally and not in the docker services, you need:
 
 1. **Postgres**: A running PostgreSQL database server _(See [Database Setup above](#database-setup))_
-1. **Elasticsearch**: A running Elasticsearch cluster _(See [Elasticsearch Setup above](#elasticsearch-setup))_
-1. **Environment Variables**: Tell python where to connect to the various data stores _(See [Environmnet Variables](#environment-variables))_
+1. **OpenSearch**: A running OpenSearch cluster _(See [OpenSearch Setup above](#opensearch-setup))_
+1. **Environment Variables**: Tell python where to connect to the various data stores _(See [Environment Variables](#environment-variables))_
 1. **Install python version with uv**: python is installed with uv  _(See below)_
 1. **Sync uv project**: uv project is synced with lockfile  _(See below)_
 
 _**NOTE**: Running test locally might require you to run `export OBJC_DISABLE_INITIALIZE_FORK_SAFETY=YES`. As discussed [here](https://github.com/rails/rails/issues/38560), there is an issue than can cause some of the Spark tests to fail without this environment variable set._
 
-#### Install python version with uv
-Install the currently supported python version with uv
-
-```shell
-uv python install 3.10.12
-```
-
-#### Sync uv project with lockfile
-Setup uv project and check version info:
-
-```shell
-make local-dev-setup
-```
-
-
-Once these are satisfied, run:
-
-```shell
-make tests
-```
-
-or, alternatively to skip using `make`
-
-```shell
-uv run pytest
-```
-
 ### Including Broker Integration Tests
 Some automated integration tests run against a [Broker](https://github.com/fedspendingtransparency/data-act-broker-backend) database. If certain dependencies to run such integration tests are not satisfied, those tests will bail out and be marked as _Skipped_.
 (You can see messages about those skipped tests by adding the `-rs` flag to pytest, like: `pytest -rs`)

From 07ed1b4b97434b5767eebcafb080a3376abd164c Mon Sep 17 00:00:00 2001
From: Seth Stoudenmier <stoudenmier_seth@bah.com>
Date: Fri, 29 May 2026 14:34:43 -0400
Subject: [PATCH 33/48] [DEV-14608] cleanup makefile

---
 Makefile | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/Makefile b/Makefile
index c5651dc129..44304f8196 100644
--- a/Makefile
+++ b/Makefile
@@ -150,7 +150,7 @@ docker-compose-down: ## Run docker compose down to bring down services listed in
 .PHONY: docker-build-development
 docker-build-development: ## Run docker build to build a base container image for development
 	# NOTE: [See NOTE in above docker compose rule about .env file]
-	docker build --tag usaspending-development --build-arg PROJECT_LOG_DIR=${PROJECT_LOG_DIR} ${args} --file ${dockerfile_for_development} $$(dirname ${dockerfile_for_development})
+	docker build --tag usaspending-development ${args} --file ${dockerfile_for_development} $$(dirname ${dockerfile_for_development})
 
 .PHONY: docker-compose-build
 docker-compose-build: ## Ensure ALL services in the docker-compose.yaml file have an image built for them according to their build: key
@@ -163,14 +163,14 @@ docker-compose-build: ## Ensure ALL services in the docker-compose.yaml file hav
 	#       where project defaults to the directory name you're in. Therefore you MUST always run this command (or the manual version of it)
 	#       anytime you want services run with Docker Compose to accommodate recent changes in the image (e.g. python package dependency changes)
 	# NOTE: [See NOTE in above docker compose rule about .env file]
-	echo "docker compose --profile usaspending --project-directory . --file ${docker_compose_file} build --build-arg PROJECT_LOG_DIR=${PROJECT_LOG_DIR} ${args}"
-	docker compose --profile usaspending --project-directory . --file ${docker_compose_file} build --build-arg PROJECT_LOG_DIR=${PROJECT_LOG_DIR} ${args}
+	echo "docker compose --profile usaspending --project-directory . --file ${docker_compose_file} build ${args}"
+	docker compose --profile usaspending --project-directory . --file ${docker_compose_file} build ${args}
 
 .PHONY: docker-compose-build-development
 docker-compose-build-development: ## See: docker-compose-build rule. This builds just the subset of spark services.
 	# NOTE: [See NOTE in above docker compose rule about .env file]=
-	echo "docker compose --profile spark --project-directory . --file ${docker_compose_file} build --build-arg PROJECT_LOG_DIR=${PROJECT_LOG_DIR} ${args}"
-	docker compose --profile spark --project-directory . --file ${docker_compose_file} build --build-arg PROJECT_LOG_DIR=${PROJECT_LOG_DIR} ${args}
+	echo "docker compose --profile spark --project-directory . --file ${docker_compose_file} build ${args}"
+	docker compose --profile spark --project-directory . --file ${docker_compose_file} build ${args}
 
 .PHONY: style-checks
 style-checks:

From 1042e9a67f40ea273956a31a30aa8666648e6aff Mon Sep 17 00:00:00 2001
From: Seth Stoudenmier <stoudenmier_seth@bah.com>
Date: Fri, 29 May 2026 16:23:05 -0400
Subject: [PATCH 34/48] [DEV-14608] repartition transaction_search dataframe

---
 .../search/delta_models/dataframes/transaction_search.py     | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/usaspending_api/search/delta_models/dataframes/transaction_search.py b/usaspending_api/search/delta_models/dataframes/transaction_search.py
index f6deb5e577..326ce62151 100644
--- a/usaspending_api/search/delta_models/dataframes/transaction_search.py
+++ b/usaspending_api/search/delta_models/dataframes/transaction_search.py
@@ -940,7 +940,7 @@ def dataframe(self) -> DataFrame:
             )
         )
         df_with_location = self.join_location_data(df)
-        return (
+        final_df = (
             df_with_location.join(
                 self.current_cd,
                 self.transaction_normalized.id == self.current_cd.transaction_id,
@@ -985,6 +985,9 @@ def dataframe(self) -> DataFrame:
             )
             .withColumn("merge_hash_key", sf.xxhash64("*"))
         )
+        # Repartitioning the dataframe to match shuffle partitions which should be the number of cores * 2
+        num_partitions = self.spark.sparkContext.defaultParallelism * 2
+        return final_df.repartition(num_partitions)
 
 
 def load_transaction_search(

From ccc4e7b7a04f0886fbb8dd6830c21b1ead361303 Mon Sep 17 00:00:00 2001
From: Seth Stoudenmier <stoudenmier_seth@bah.com>
Date: Mon, 1 Jun 2026 11:59:43 -0400
Subject: [PATCH 35/48] [DEV-14608] repartition search dataframes

---
 .../delta_models/dataframes/abstract_search.py   |  4 +++-
 .../delta_models/dataframes/award_search.py      |  5 ++++-
 .../dataframes/transaction_search.py             | 16 ++++++++++++++++
 3 files changed, 23 insertions(+), 2 deletions(-)

diff --git a/usaspending_api/search/delta_models/dataframes/abstract_search.py b/usaspending_api/search/delta_models/dataframes/abstract_search.py
index 42e8bdd452..18e6912a5b 100644
--- a/usaspending_api/search/delta_models/dataframes/abstract_search.py
+++ b/usaspending_api/search/delta_models/dataframes/abstract_search.py
@@ -1,6 +1,7 @@
 from abc import ABC, abstractmethod
 
-from pyspark.sql import DataFrame, SparkSession, functions as sf, Column, Window
+from pyspark.sql import Column, DataFrame, SparkSession, Window
+from pyspark.sql import functions as sf
 from pyspark.sql.types import (
     ShortType,
     StringType,
@@ -20,6 +21,7 @@ def extract_numbers_as_string(col: Column, length: int = 2, pad: str = "0") -> C
 class AbstractSearch(ABC):
 
     def __init__(self, spark: SparkSession):
+        self.spark = spark
         # Base Tables
         self.transaction_normalized = spark.table("int.transaction_normalized")
         self.transaction_fpds = spark.table("int.transaction_fpds")
diff --git a/usaspending_api/search/delta_models/dataframes/award_search.py b/usaspending_api/search/delta_models/dataframes/award_search.py
index 1551eb2031..5e3b924982 100644
--- a/usaspending_api/search/delta_models/dataframes/award_search.py
+++ b/usaspending_api/search/delta_models/dataframes/award_search.py
@@ -270,7 +270,7 @@ def dataframe(self) -> DataFrame:
         )
         df_with_location = self.join_location_data(df)
 
-        return (
+        final_df = (
             df_with_location.join(
                 self.current_cd, (self.awards.latest_transaction_id == self.current_cd.transaction_id), "leftouter"
             )
@@ -633,6 +633,9 @@ def dataframe(self) -> DataFrame:
             )
             .withColumn("merge_hash_key", sf.xxhash64("*"))
         )
+        # Repartitioning the dataframe to match shuffle partitions which should be the number of cores * 2
+        num_partitions = self.spark.sparkContext.defaultParallelism * 2
+        return final_df.repartition(num_partitions)
 
 
 def load_award_search(spark: SparkSession, destination_database: str, destination_table_name: str) -> None:
diff --git a/usaspending_api/search/delta_models/dataframes/transaction_search.py b/usaspending_api/search/delta_models/dataframes/transaction_search.py
index 326ce62151..bbc57183b0 100644
--- a/usaspending_api/search/delta_models/dataframes/transaction_search.py
+++ b/usaspending_api/search/delta_models/dataframes/transaction_search.py
@@ -1004,6 +1004,22 @@ def load_transaction_search(
 def load_transaction_search_incremental(
     spark: SparkSession, destination_database: str, destination_table_name: str
 ) -> None:
+    # CURRENT APPROACH
+    # target = DeltaTable.forName(
+    #     spark, f"{destination_database}.{destination_table_name}"
+    # ).alias("t")
+    # source = TransactionSearch(spark).dataframe.alias("s")
+    # (
+    #     target.merge(
+    #         source,
+    #         "s.transaction_id = t.transaction_id and s.merge_hash_key = t.merge_hash_key",
+    #     )
+    #     .whenNotMatchedInsertAll()
+    #     .whenNotMatchedBySourceDelete()
+    #     .execute()
+    # )
+
+    # TEST APPROACH
     target_df = spark.table(f"{destination_database}.{destination_table_name}")
     source_df = TransactionSearch(spark).dataframe
     to_insert = source_df.join(

From 03c813df60fa30e8edb24ba2fd3df309ab97cfdd Mon Sep 17 00:00:00 2001
From: Seth Stoudenmier <stoudenmier_seth@bah.com>
Date: Mon, 1 Jun 2026 19:08:21 -0400
Subject: [PATCH 36/48] [DEV-14608] add skew hint

---
 .../search/delta_models/dataframes/transaction_search.py         | 1 +
 1 file changed, 1 insertion(+)

diff --git a/usaspending_api/search/delta_models/dataframes/transaction_search.py b/usaspending_api/search/delta_models/dataframes/transaction_search.py
index bbc57183b0..5534f25959 100644
--- a/usaspending_api/search/delta_models/dataframes/transaction_search.py
+++ b/usaspending_api/search/delta_models/dataframes/transaction_search.py
@@ -987,6 +987,7 @@ def dataframe(self) -> DataFrame:
         )
         # Repartitioning the dataframe to match shuffle partitions which should be the number of cores * 2
         num_partitions = self.spark.sparkContext.defaultParallelism * 2
+        final_df = final_df.hint("skew", "transaction_id")
         return final_df.repartition(num_partitions)
 
 

From 11a05fbdb633fabfbb9e222ed112216d219998e0 Mon Sep 17 00:00:00 2001
From: Seth Stoudenmier <stoudenmier_seth@bah.com>
Date: Tue, 2 Jun 2026 14:26:20 -0400
Subject: [PATCH 37/48] [DEV-14608] capture explain plan

---
 .../search/delta_models/dataframes/transaction_search.py  | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/usaspending_api/search/delta_models/dataframes/transaction_search.py b/usaspending_api/search/delta_models/dataframes/transaction_search.py
index 5534f25959..6b10e6efe6 100644
--- a/usaspending_api/search/delta_models/dataframes/transaction_search.py
+++ b/usaspending_api/search/delta_models/dataframes/transaction_search.py
@@ -986,15 +986,17 @@ def dataframe(self) -> DataFrame:
             .withColumn("merge_hash_key", sf.xxhash64("*"))
         )
         # Repartitioning the dataframe to match shuffle partitions which should be the number of cores * 2
-        num_partitions = self.spark.sparkContext.defaultParallelism * 2
-        final_df = final_df.hint("skew", "transaction_id")
-        return final_df.repartition(num_partitions)
+        # num_partitions = self.spark.sparkContext.defaultParallelism * 2
+        # final_df = final_df.hint("skew", "transaction_id")
+        # return final_df.repartition(num_partitions)
+        return final_df
 
 
 def load_transaction_search(
     spark: SparkSession, destination_database: str, destination_table_name: str
 ) -> None:
     df = TransactionSearch(spark).dataframe
+    df.explain(mode="formatted")
     df.write.saveAsTable(
         f"{destination_database}.{destination_table_name}",
         mode="overwrite",

From 941546b67ee66bd755e9167dc89001e313ecef14 Mon Sep 17 00:00:00 2001
From: Seth Stoudenmier <stoudenmier_seth@bah.com>
Date: Wed, 3 Jun 2026 12:19:33 -0400
Subject: [PATCH 38/48] [DEV-14608] test broadcast join

---
 .../search/delta_models/dataframes/transaction_search.py      | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/usaspending_api/search/delta_models/dataframes/transaction_search.py b/usaspending_api/search/delta_models/dataframes/transaction_search.py
index 6b10e6efe6..abefb02d48 100644
--- a/usaspending_api/search/delta_models/dataframes/transaction_search.py
+++ b/usaspending_api/search/delta_models/dataframes/transaction_search.py
@@ -906,7 +906,8 @@ def dataframe(self) -> DataFrame:
                 "leftouter",
             )
             .join(
-                self.funding_toptier_agency,
+                # self.funding_toptier_agency,
+                sf.broadcast(self.funding_toptier_agency),
                 self.funding_agency.funding_toptier_agency_id
                 == self.funding_toptier_agency.toptier_agency_id,
                 "leftouter",
@@ -996,7 +997,6 @@ def load_transaction_search(
     spark: SparkSession, destination_database: str, destination_table_name: str
 ) -> None:
     df = TransactionSearch(spark).dataframe
-    df.explain(mode="formatted")
     df.write.saveAsTable(
         f"{destination_database}.{destination_table_name}",
         mode="overwrite",

From 8e16d4318876f6c80315ee8a8125a3534c6d9239 Mon Sep 17 00:00:00 2001
From: Seth Stoudenmier <stoudenmier_seth@bah.com>
Date: Wed, 3 Jun 2026 13:35:47 -0400
Subject: [PATCH 39/48] [DEV-14608] test broadcast join

---
 .../search/delta_models/dataframes/transaction_search.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/usaspending_api/search/delta_models/dataframes/transaction_search.py b/usaspending_api/search/delta_models/dataframes/transaction_search.py
index abefb02d48..990a270121 100644
--- a/usaspending_api/search/delta_models/dataframes/transaction_search.py
+++ b/usaspending_api/search/delta_models/dataframes/transaction_search.py
@@ -862,7 +862,7 @@ def dataframe(self) -> DataFrame:
                 "leftouter",
             )
             .join(
-                self.recipient_lookup,
+                sf.broadcast(self.recipient_lookup),
                 self.recipient_lookup.recipient_hash == self.generated_recipient_hash,
                 "leftouter",
             )
@@ -906,8 +906,7 @@ def dataframe(self) -> DataFrame:
                 "leftouter",
             )
             .join(
-                # self.funding_toptier_agency,
-                sf.broadcast(self.funding_toptier_agency),
+                self.funding_toptier_agency,
                 self.funding_agency.funding_toptier_agency_id
                 == self.funding_toptier_agency.toptier_agency_id,
                 "leftouter",
@@ -928,13 +927,13 @@ def dataframe(self) -> DataFrame:
                 "leftouter",
             )
             .join(
-                self.parent_recipient,
+                sf.broadcast(self.parent_recipient),
                 self.parent_recipient.parent_recipient_hash
                 == self.generated_parent_recipient_hash,
                 "leftouter",
             )
             .join(
-                self.recipient_hash_and_levels,
+                sf.broadcast(self.recipient_hash_and_levels),
                 (sf.col("recipient_hash") == sf.col("recipient_level_hash"))
                 & ~(sf.col("legal_business_name").isin(SPECIAL_CASES)),
                 "leftouter",

From 83cb20612df0eb7c3370c1735b753ec9ea34496b Mon Sep 17 00:00:00 2001
From: Seth Stoudenmier <stoudenmier_seth@bah.com>
Date: Wed, 3 Jun 2026 14:31:37 -0400
Subject: [PATCH 40/48] [DEV-14608] troubleshoot

---
 .../search/delta_models/dataframes/transaction_search.py    | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/usaspending_api/search/delta_models/dataframes/transaction_search.py b/usaspending_api/search/delta_models/dataframes/transaction_search.py
index 990a270121..dec5625778 100644
--- a/usaspending_api/search/delta_models/dataframes/transaction_search.py
+++ b/usaspending_api/search/delta_models/dataframes/transaction_search.py
@@ -862,7 +862,7 @@ def dataframe(self) -> DataFrame:
                 "leftouter",
             )
             .join(
-                sf.broadcast(self.recipient_lookup),
+                self.recipient_lookup,
                 self.recipient_lookup.recipient_hash == self.generated_recipient_hash,
                 "leftouter",
             )
@@ -927,13 +927,13 @@ def dataframe(self) -> DataFrame:
                 "leftouter",
             )
             .join(
-                sf.broadcast(self.parent_recipient),
+                self.parent_recipient,
                 self.parent_recipient.parent_recipient_hash
                 == self.generated_parent_recipient_hash,
                 "leftouter",
             )
             .join(
-                sf.broadcast(self.recipient_hash_and_levels),
+                self.recipient_hash_and_levels,
                 (sf.col("recipient_hash") == sf.col("recipient_level_hash"))
                 & ~(sf.col("legal_business_name").isin(SPECIAL_CASES)),
                 "leftouter",

From 2ec0e832cbb2be811088a27d09389ebcb99101c9 Mon Sep 17 00:00:00 2001
From: Seth Stoudenmier <stoudenmier_seth@bah.com>
Date: Wed, 3 Jun 2026 15:33:49 -0400
Subject: [PATCH 41/48] [DEV-14608] troubleshoot

---
 .../search/delta_models/dataframes/transaction_search.py    | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/usaspending_api/search/delta_models/dataframes/transaction_search.py b/usaspending_api/search/delta_models/dataframes/transaction_search.py
index dec5625778..d2da4cd48f 100644
--- a/usaspending_api/search/delta_models/dataframes/transaction_search.py
+++ b/usaspending_api/search/delta_models/dataframes/transaction_search.py
@@ -986,10 +986,10 @@ def dataframe(self) -> DataFrame:
             .withColumn("merge_hash_key", sf.xxhash64("*"))
         )
         # Repartitioning the dataframe to match shuffle partitions which should be the number of cores * 2
-        # num_partitions = self.spark.sparkContext.defaultParallelism * 2
+        num_partitions = self.spark.sparkContext.defaultParallelism * 2
         # final_df = final_df.hint("skew", "transaction_id")
-        # return final_df.repartition(num_partitions)
-        return final_df
+        return final_df.repartition(num_partitions)
+        # return final_df
 
 
 def load_transaction_search(

From 6b7f2caf00aa49b8dfbc14c399737a8511e98bf6 Mon Sep 17 00:00:00 2001
From: Seth Stoudenmier <stoudenmier_seth@bah.com>
Date: Thu, 4 Jun 2026 23:21:42 -0400
Subject: [PATCH 42/48] [DEV-14608] undo troubleshooting

---
 .../dataframes/abstract_search.py             |  1 -
 .../delta_models/dataframes/award_search.py   |  5 +-
 .../dataframes/transaction_search.py          | 56 +++----------------
 3 files changed, 10 insertions(+), 52 deletions(-)

diff --git a/usaspending_api/search/delta_models/dataframes/abstract_search.py b/usaspending_api/search/delta_models/dataframes/abstract_search.py
index 18e6912a5b..6a62ce526c 100644
--- a/usaspending_api/search/delta_models/dataframes/abstract_search.py
+++ b/usaspending_api/search/delta_models/dataframes/abstract_search.py
@@ -21,7 +21,6 @@ def extract_numbers_as_string(col: Column, length: int = 2, pad: str = "0") -> C
 class AbstractSearch(ABC):
 
     def __init__(self, spark: SparkSession):
-        self.spark = spark
         # Base Tables
         self.transaction_normalized = spark.table("int.transaction_normalized")
         self.transaction_fpds = spark.table("int.transaction_fpds")
diff --git a/usaspending_api/search/delta_models/dataframes/award_search.py b/usaspending_api/search/delta_models/dataframes/award_search.py
index 5e3b924982..1551eb2031 100644
--- a/usaspending_api/search/delta_models/dataframes/award_search.py
+++ b/usaspending_api/search/delta_models/dataframes/award_search.py
@@ -270,7 +270,7 @@ def dataframe(self) -> DataFrame:
         )
         df_with_location = self.join_location_data(df)
 
-        final_df = (
+        return (
             df_with_location.join(
                 self.current_cd, (self.awards.latest_transaction_id == self.current_cd.transaction_id), "leftouter"
             )
@@ -633,9 +633,6 @@ def dataframe(self) -> DataFrame:
             )
             .withColumn("merge_hash_key", sf.xxhash64("*"))
         )
-        # Repartitioning the dataframe to match shuffle partitions which should be the number of cores * 2
-        num_partitions = self.spark.sparkContext.defaultParallelism * 2
-        return final_df.repartition(num_partitions)
 
 
 def load_award_search(spark: SparkSession, destination_database: str, destination_table_name: str) -> None:
diff --git a/usaspending_api/search/delta_models/dataframes/transaction_search.py b/usaspending_api/search/delta_models/dataframes/transaction_search.py
index d2da4cd48f..e334764c06 100644
--- a/usaspending_api/search/delta_models/dataframes/transaction_search.py
+++ b/usaspending_api/search/delta_models/dataframes/transaction_search.py
@@ -940,7 +940,7 @@ def dataframe(self) -> DataFrame:
             )
         )
         df_with_location = self.join_location_data(df)
-        final_df = (
+        return (
             df_with_location.join(
                 self.current_cd,
                 self.transaction_normalized.id == self.current_cd.transaction_id,
@@ -985,11 +985,6 @@ def dataframe(self) -> DataFrame:
             )
             .withColumn("merge_hash_key", sf.xxhash64("*"))
         )
-        # Repartitioning the dataframe to match shuffle partitions which should be the number of cores * 2
-        num_partitions = self.spark.sparkContext.defaultParallelism * 2
-        # final_df = final_df.hint("skew", "transaction_id")
-        return final_df.repartition(num_partitions)
-        # return final_df
 
 
 def load_transaction_search(
@@ -1006,49 +1001,16 @@ def load_transaction_search(
 def load_transaction_search_incremental(
     spark: SparkSession, destination_database: str, destination_table_name: str
 ) -> None:
-    # CURRENT APPROACH
-    # target = DeltaTable.forName(
-    #     spark, f"{destination_database}.{destination_table_name}"
-    # ).alias("t")
-    # source = TransactionSearch(spark).dataframe.alias("s")
-    # (
-    #     target.merge(
-    #         source,
-    #         "s.transaction_id = t.transaction_id and s.merge_hash_key = t.merge_hash_key",
-    #     )
-    #     .whenNotMatchedInsertAll()
-    #     .whenNotMatchedBySourceDelete()
-    #     .execute()
-    # )
-
-    # TEST APPROACH
-    target_df = spark.table(f"{destination_database}.{destination_table_name}")
-    source_df = TransactionSearch(spark).dataframe
-    to_insert = source_df.join(
-        target_df,
-        on=["transaction_id", "merge_hash_key"],
-        how="left_anti"
-    ).withColumn("merge_action", sf.lit("INSERT"))
-    to_delete = target_df.join(
-        source_df,
-        on=["transaction_id", "merge_hash_key"],
-        how="left_anti"
-    ).withColumn("merge_action", sf.lit("DELETE"))
-    changes_df = to_insert.unionByName(to_delete, allowMissingColumns=True)
-    target_table = DeltaTable.forName(
+    target = DeltaTable.forName(
         spark, f"{destination_database}.{destination_table_name}"
-    )
+    ).alias("t")
+    source = TransactionSearch(spark).dataframe.alias("s")
     (
-        target_table.alias("t")
-        .merge(
-            changes_df.alias("c"),
-            "c.transaction_id = t.transaction_id and c.merge_hash_key = t.merge_hash_key",
-        )
-        .whenMatchedDelete(
-            condition="c.merge_action = 'DELETE'"
-        )
-        .whenNotMatchedInsertAll(
-            condition='c.merge_action = "INSERT"',
+        target.merge(
+            source,
+            "s.transaction_id = t.transaction_id and s.merge_hash_key = t.merge_hash_key",
         )
+        .whenNotMatchedInsertAll()
+        .whenNotMatchedBySourceDelete()
         .execute()
     )

From 0818e684ea650d446a4ef9702f3a26b4b2a4d3e5 Mon Sep 17 00:00:00 2001
From: Seth Stoudenmier <stoudenmier_seth@bah.com>
Date: Fri, 5 Jun 2026 12:07:27 -0400
Subject: [PATCH 43/48] [DEV-14608] test checkpoints

---
 .../common/helpers/spark_helpers.py           |  1 +
 .../dataframes/transaction_search.py          | 50 ++++++++++++-------
 2 files changed, 33 insertions(+), 18 deletions(-)

diff --git a/usaspending_api/common/helpers/spark_helpers.py b/usaspending_api/common/helpers/spark_helpers.py
index 068b27db3c..28b1825bfb 100644
--- a/usaspending_api/common/helpers/spark_helpers.py
+++ b/usaspending_api/common/helpers/spark_helpers.py
@@ -253,6 +253,7 @@ def configure_spark_session(  # noqa: C901,PLR0912,PLR0913,PLR0915
     if enable_hive_support:
         builder = builder.enableHiveSupport()
     spark = builder.config(conf=conf).getOrCreate()
+    spark.sparkContext.setCheckpointDir(f"s3a://{CONFIG.SPARK_S3_BUCKET}/tmp/spark-checkpoints")
 
     # Now that the SparkSession was created, check whether certain provided config values were ignored if given a
     # pre-existing SparkContext, and error-out if so
diff --git a/usaspending_api/search/delta_models/dataframes/transaction_search.py b/usaspending_api/search/delta_models/dataframes/transaction_search.py
index e334764c06..0aaad7718d 100644
--- a/usaspending_api/search/delta_models/dataframes/transaction_search.py
+++ b/usaspending_api/search/delta_models/dataframes/transaction_search.py
@@ -842,6 +842,7 @@ def fpds_cols(self) -> list[Column]:
 
     @property
     def dataframe(self) -> DataFrame:
+        # Create and checkpoint the initial dataframe with Transactions and Awards
         df = (
             self.transaction_normalized.join(
                 self.transaction_fabs,
@@ -856,19 +857,41 @@ def dataframe(self) -> DataFrame:
                 "leftouter",
             )
             .join(
-                self.references_cfda,
-                self.transaction_fabs.cfda_number
-                == self.references_cfda.program_number,
+                self.awards,
+                self.transaction_normalized.award_id == self.awards.id,
                 "leftouter",
             )
+            .checkpoint()
+        )
+        # Create a new checkpoint that adds in the recipient data
+        df = (
+            df
             .join(
                 self.recipient_lookup,
                 self.recipient_lookup.recipient_hash == self.generated_recipient_hash,
                 "leftouter",
             )
             .join(
-                self.awards,
-                self.transaction_normalized.award_id == self.awards.id,
+                self.parent_recipient,
+                self.parent_recipient.parent_recipient_hash
+                == self.generated_parent_recipient_hash,
+                "leftouter",
+            )
+            .join(
+                self.recipient_hash_and_levels,
+                (sf.col("recipient_hash") == sf.col("recipient_level_hash"))
+                & ~(sf.col("legal_business_name").isin(SPECIAL_CASES)),
+                "leftouter",
+            )
+            .checkpoint()
+        )
+        # Add in the agency reference data
+        df = (
+            df
+            .join(
+                self.references_cfda,
+                self.transaction_fabs.cfda_number
+                == self.references_cfda.program_number,
                 "leftouter",
             )
             .join(
@@ -926,20 +949,11 @@ def dataframe(self) -> DataFrame:
                 & (self.funding_agency_id.row_num == 1),
                 "leftouter",
             )
-            .join(
-                self.parent_recipient,
-                self.parent_recipient.parent_recipient_hash
-                == self.generated_parent_recipient_hash,
-                "leftouter",
-            )
-            .join(
-                self.recipient_hash_and_levels,
-                (sf.col("recipient_hash") == sf.col("recipient_level_hash"))
-                & ~(sf.col("legal_business_name").isin(SPECIAL_CASES)),
-                "leftouter",
-            )
+            .checkpoint()
         )
-        df_with_location = self.join_location_data(df)
+        # Checkpoint that includes the location data
+        df_with_location = self.join_location_data(df).checkpoint()
+        # Add remaining data
         return (
             df_with_location.join(
                 self.current_cd,

From ef59250bf5fd433b63acbbe64793c8621496a9e0 Mon Sep 17 00:00:00 2001
From: Seth Stoudenmier <stoudenmier_seth@bah.com>
Date: Fri, 5 Jun 2026 12:44:48 -0400
Subject: [PATCH 44/48] [DEV-14608] remove checkpoints

---
 .../common/helpers/spark_helpers.py           |  1 -
 .../dataframes/transaction_search.py          | 50 +++++++------------
 2 files changed, 18 insertions(+), 33 deletions(-)

diff --git a/usaspending_api/common/helpers/spark_helpers.py b/usaspending_api/common/helpers/spark_helpers.py
index 28b1825bfb..068b27db3c 100644
--- a/usaspending_api/common/helpers/spark_helpers.py
+++ b/usaspending_api/common/helpers/spark_helpers.py
@@ -253,7 +253,6 @@ def configure_spark_session(  # noqa: C901,PLR0912,PLR0913,PLR0915
     if enable_hive_support:
         builder = builder.enableHiveSupport()
     spark = builder.config(conf=conf).getOrCreate()
-    spark.sparkContext.setCheckpointDir(f"s3a://{CONFIG.SPARK_S3_BUCKET}/tmp/spark-checkpoints")
 
     # Now that the SparkSession was created, check whether certain provided config values were ignored if given a
     # pre-existing SparkContext, and error-out if so
diff --git a/usaspending_api/search/delta_models/dataframes/transaction_search.py b/usaspending_api/search/delta_models/dataframes/transaction_search.py
index 0aaad7718d..e334764c06 100644
--- a/usaspending_api/search/delta_models/dataframes/transaction_search.py
+++ b/usaspending_api/search/delta_models/dataframes/transaction_search.py
@@ -842,7 +842,6 @@ def fpds_cols(self) -> list[Column]:
 
     @property
     def dataframe(self) -> DataFrame:
-        # Create and checkpoint the initial dataframe with Transactions and Awards
         df = (
             self.transaction_normalized.join(
                 self.transaction_fabs,
@@ -857,41 +856,19 @@ def dataframe(self) -> DataFrame:
                 "leftouter",
             )
             .join(
-                self.awards,
-                self.transaction_normalized.award_id == self.awards.id,
+                self.references_cfda,
+                self.transaction_fabs.cfda_number
+                == self.references_cfda.program_number,
                 "leftouter",
             )
-            .checkpoint()
-        )
-        # Create a new checkpoint that adds in the recipient data
-        df = (
-            df
             .join(
                 self.recipient_lookup,
                 self.recipient_lookup.recipient_hash == self.generated_recipient_hash,
                 "leftouter",
             )
             .join(
-                self.parent_recipient,
-                self.parent_recipient.parent_recipient_hash
-                == self.generated_parent_recipient_hash,
-                "leftouter",
-            )
-            .join(
-                self.recipient_hash_and_levels,
-                (sf.col("recipient_hash") == sf.col("recipient_level_hash"))
-                & ~(sf.col("legal_business_name").isin(SPECIAL_CASES)),
-                "leftouter",
-            )
-            .checkpoint()
-        )
-        # Add in the agency reference data
-        df = (
-            df
-            .join(
-                self.references_cfda,
-                self.transaction_fabs.cfda_number
-                == self.references_cfda.program_number,
+                self.awards,
+                self.transaction_normalized.award_id == self.awards.id,
                 "leftouter",
             )
             .join(
@@ -949,11 +926,20 @@ def dataframe(self) -> DataFrame:
                 & (self.funding_agency_id.row_num == 1),
                 "leftouter",
             )
-            .checkpoint()
+            .join(
+                self.parent_recipient,
+                self.parent_recipient.parent_recipient_hash
+                == self.generated_parent_recipient_hash,
+                "leftouter",
+            )
+            .join(
+                self.recipient_hash_and_levels,
+                (sf.col("recipient_hash") == sf.col("recipient_level_hash"))
+                & ~(sf.col("legal_business_name").isin(SPECIAL_CASES)),
+                "leftouter",
+            )
         )
-        # Checkpoint that includes the location data
-        df_with_location = self.join_location_data(df).checkpoint()
-        # Add remaining data
+        df_with_location = self.join_location_data(df)
         return (
             df_with_location.join(
                 self.current_cd,

From 08b5624f6e541cfa3b11a49ed40ffbcbdf6971a6 Mon Sep 17 00:00:00 2001
From: Seth Stoudenmier <stoudenmier_seth@bah.com>
Date: Mon, 3 Aug 2026 12:41:07 -0400
Subject: [PATCH 45/48] [DEV-14608] add ruff formatter to style checks

---
 .github/workflows/code-style-checks.yaml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.github/workflows/code-style-checks.yaml b/.github/workflows/code-style-checks.yaml
index 181abe7646..c8ed9603ca 100644
--- a/.github/workflows/code-style-checks.yaml
+++ b/.github/workflows/code-style-checks.yaml
@@ -72,6 +72,10 @@ jobs:
         if: steps.changed-python-files.outputs.all_changed_files != ''
         run: ruff check ${{ steps.changed-python-files.outputs.all_changed_files }}
 
+      - name: Run Ruff Formatter
+        if: steps.changed-python-files.outputs.all_changed_files != ''
+        run: ruff format --check ${{ steps.changed-python-files.outputs.all_changed_files }}
+
       # TODO: Should rethink placement of this check. This runs as a Django management command which requires similar
       #       setup steps as the rest of the test suite. Removing would reduce setup time and overall CI/CD time.
       - name: Run Check For Endpoint Documentation

From 0487e797f769f8fb96e1788ad2f123a65d370e11 Mon Sep 17 00:00:00 2001
From: Seth Stoudenmier <stoudenmier_seth@bah.com>
Date: Wed, 5 Aug 2026 21:44:37 -0400
Subject: [PATCH 46/48] [DEV-14608] force broadcast join and some cleanup

---
 .env.template                                             | 5 +++--
 development.Dockerfile                                    | 8 ++------
 docker-compose.yml                                        | 6 ++++++
 .../search/delta_models/dataframes/abstract_search.py     | 3 +++
 .../search/delta_models/dataframes/award_search.py        | 5 ++++-
 .../search/delta_models/dataframes/transaction_search.py  | 5 ++++-
 6 files changed, 22 insertions(+), 10 deletions(-)

diff --git a/.env.template b/.env.template
index 8a1e8f8770..6c65e8cb23 100644
--- a/.env.template
+++ b/.env.template
@@ -54,8 +54,9 @@ BROKER_DB_NAME=data_broker
 
 # If using the Broker DB container created via https://github.com/fedspendingtransparency/data-act-broker-backend
 # then the following values should be used instead
-# BROKER_DB=postgres://admin:root@dataact-broker-db:5432/data_broker
-# BROKER_DB_HOST=dataact-broker-db
+# BROKER_DB=postgres://admin:root@host.docker.internal:5435/data_broker
+# BROKER_DB_HOST=host.docker.internal
+# BROKER_DB_PORT=5435
 # BROKER_DB_USER=admin
 # BROKER_DB_PASSWORD=root
 
diff --git a/development.Dockerfile b/development.Dockerfile
index 130999eb7b..a620d9a84c 100644
--- a/development.Dockerfile
+++ b/development.Dockerfile
@@ -47,14 +47,10 @@ ENV UV_COMPILE_BYTECODE=1
 # Copy from the cache instead of linking since it's a mounted volume
 ENV UV_LINK_MODE=copy
 
-# Copy relevant parts of the project for package management
-COPY pyproject.toml uv.lock /usaspending-api/
-
 # Install Python and dev dependencies
 RUN --mount=type=cache,target=/root/.cache/uv \
-    --mount=type=bind,source=uv.lock,target=uv.lock \
-    --mount=type=bind,source=pyproject.toml,target=pyproject.toml \
-    uv sync --extra dev --extra spark --locked --no-install-project --python ${PYTHON_VERSION}
+    --mount=type=bind,source=.,target=/usaspending-api \
+    uv sync --extra dev --extra spark --locked --python ${PYTHON_VERSION}
 
 # Make sure uv environment is active
 ENV PATH="/usr/local/.venv/bin:$PATH"
diff --git a/docker-compose.yml b/docker-compose.yml
index f8e8959f87..de800d2fe2 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -378,6 +378,8 @@ services:
         source: .
         target: /usaspending-api
         read_only: false
+      # Required to interact with host's docker daemon from within this running container and access the Broker DB
+      - /var/run/docker.sock:/var/run/docker.sock
   spark-worker:
     profiles:
       - spark  # must pass --profile spark to docker compose for this to come up
@@ -404,6 +406,8 @@ services:
         source: .
         target: /usaspending-api
         read_only: false
+      # Required to interact with host's docker daemon from within this running container and access the Broker DB
+      - /var/run/docker.sock:/var/run/docker.sock
   spark-history-server:
     profiles:
       - spark  # must pass --profile spark to docker compose for this to come up
@@ -490,6 +494,8 @@ services:
       - type: bind
         source: ${SOURCE_SPARK_SQL_WAREHOUSE_DIR}
         target: ${TARGET_SPARK_SQL_WAREHOUSE_DIR}
+      # Required to interact with host's docker daemon from within this running container and access the Broker DB
+      - /var/run/docker.sock:/var/run/docker.sock
   spark-shell:
     profiles:
       - spark  # must pass --profile spark to docker compose for this to come up
diff --git a/usaspending_api/search/delta_models/dataframes/abstract_search.py b/usaspending_api/search/delta_models/dataframes/abstract_search.py
index 1cbb91064f..fbeb0e9e81 100644
--- a/usaspending_api/search/delta_models/dataframes/abstract_search.py
+++ b/usaspending_api/search/delta_models/dataframes/abstract_search.py
@@ -50,6 +50,9 @@ def __init__(self, spark: SparkSession):
             .withColumn("awarding_subtier_agency_name", sf.col("name"))
             .withColumn("awarding_subtier_agency_code", sf.col("subtier_code"))
         )
+        # TODO: This logic should be updated to match self.funding_agency_id below to account for
+        #       special case agencies such as FEMA that don't have a record with toptier_flag=true,
+        #       but are a toptier agency to many subtier agencies.
         self.awarding_agency_id = spark.table("global_temp.agency").withColumn(
             "awarding_toptier_agency_id", sf.col("id")
         )
diff --git a/usaspending_api/search/delta_models/dataframes/award_search.py b/usaspending_api/search/delta_models/dataframes/award_search.py
index 7d00392247..50207c7612 100644
--- a/usaspending_api/search/delta_models/dataframes/award_search.py
+++ b/usaspending_api/search/delta_models/dataframes/award_search.py
@@ -283,7 +283,10 @@ def dataframe(self) -> DataFrame:
                 "leftouter",
             )
             .join(
-                self.funding_agency_id,
+                # Without marking this dataframe as a broadcast join it will shuffle the partitions such that agencies
+                # are grouped together. This creates heavily skewed data across tasks which run into resource
+                # constraints.
+                sf.broadcast(self.funding_agency_id),
                 (self.funding_agency_id.toptier_agency_id == self.funding_toptier_agency.funding_toptier_agency_id)
                 & (self.funding_agency_id.row_num == 1),
                 "leftouter",
diff --git a/usaspending_api/search/delta_models/dataframes/transaction_search.py b/usaspending_api/search/delta_models/dataframes/transaction_search.py
index e334764c06..0485c8b1e3 100644
--- a/usaspending_api/search/delta_models/dataframes/transaction_search.py
+++ b/usaspending_api/search/delta_models/dataframes/transaction_search.py
@@ -918,7 +918,10 @@ def dataframe(self) -> DataFrame:
                 "leftouter",
             )
             .join(
-                self.funding_agency_id,
+                # Without marking this dataframe as a broadcast join it will shuffle the partitions such that agencies
+                # are grouped together. This creates heavily skewed data across tasks which run into resource
+                # constraints.
+                sf.broadcast(self.funding_agency_id),
                 (
                     self.funding_agency_id.toptier_agency_id
                     == self.funding_toptier_agency.funding_toptier_agency_id

From 99aeab5b81c46a733237d0f45be6413dd1d41f1e Mon Sep 17 00:00:00 2001
From: Seth Stoudenmier <stoudenmier_seth@bah.com>
Date: Wed, 5 Aug 2026 22:24:49 -0400
Subject: [PATCH 47/48] [DEV-14608] cleanup and update pre-commit

---
 .pre-commit-config.yaml                       |   2 +
 .../dataframes/abstract_search.py             |   1 -
 .../dataframes/transaction_search.py          | 101 ++++++------------
 3 files changed, 32 insertions(+), 72 deletions(-)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 850453ca2c..3dd3819f2c 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -8,4 +8,6 @@ repos:
     rev: v0.14.14
     hooks:
         - id: ruff-check
+          args: [ --fix ]
           types_or: [ python, pyi ]  # avoid linting other python file types like Jupyter notebooks
+        - id: ruff-format
diff --git a/usaspending_api/search/delta_models/dataframes/abstract_search.py b/usaspending_api/search/delta_models/dataframes/abstract_search.py
index fbeb0e9e81..a152a0911d 100644
--- a/usaspending_api/search/delta_models/dataframes/abstract_search.py
+++ b/usaspending_api/search/delta_models/dataframes/abstract_search.py
@@ -19,7 +19,6 @@ def extract_numbers_as_string(col: Column, length: int = 2, pad: str = "0") -> C
 
 
 class AbstractSearch(ABC):
-
     def __init__(self, spark: SparkSession):
         # Base Tables
         self.transaction_normalized = spark.table("int.transaction_normalized")
diff --git a/usaspending_api/search/delta_models/dataframes/transaction_search.py b/usaspending_api/search/delta_models/dataframes/transaction_search.py
index 0485c8b1e3..ed13e7c97c 100644
--- a/usaspending_api/search/delta_models/dataframes/transaction_search.py
+++ b/usaspending_api/search/delta_models/dataframes/transaction_search.py
@@ -23,11 +23,7 @@ class TransactionSearch(AbstractSearch):
     def recipient_hash_and_levels(self) -> DataFrame:
         return (
             self.recipient_profile.groupBy("recipient_hash", "uei")
-            .agg(
-                sf.sort_array(sf.collect_set("recipient_level")).alias(
-                    "recipient_levels"
-                )
-            )
+            .agg(sf.sort_array(sf.collect_set("recipient_level")).alias("recipient_levels"))
             .select(
                 sf.col("recipient_hash").alias("recipient_level_hash"),
                 sf.col("recipient_levels"),
@@ -39,20 +35,17 @@ def fed_and_tres_acct(self) -> DataFrame:
         return (
             self.faba.join(
                 self.treasury_appropriation_account,
-                self.treasury_appropriation_account.treasury_account_identifier
-                == self.faba.treasury_account_id,
+                self.treasury_appropriation_account.treasury_account_identifier == self.faba.treasury_account_id,
                 "inner",
             )
             .join(
                 self.federal_account,
-                self.federal_account.id
-                == self.treasury_appropriation_account.federal_account_id,
+                self.federal_account.id == self.treasury_appropriation_account.federal_account_id,
                 "inner",
             )
             .join(
                 self.awarding_toptier_agency,
-                self.federal_account.parent_toptier_agency_id
-                == self.awarding_toptier_agency.toptier_agency_id,
+                self.federal_account.parent_toptier_agency_id == self.awarding_toptier_agency.toptier_agency_id,
                 "inner",
             )
             .join(
@@ -62,8 +55,7 @@ def fed_and_tres_acct(self) -> DataFrame:
             )
             .join(
                 self.program_activity_park,
-                self.faba.program_activity_reporting_key
-                == self.program_activity_park.code,
+                self.faba.program_activity_reporting_key == self.program_activity_park.code,
                 "left",
             )
             .filter(self.faba["award_id"].isNotNull())
@@ -86,15 +78,11 @@ def key_cols(self) -> list[Column]:
     def date_cols(self) -> list[Column]:
         return [
             sf.to_date(self.transaction_normalized.action_date).alias("action_date"),
-            sf.add_months(sf.to_date(self.transaction_normalized.action_date), 3).alias(
-                "fiscal_action_date"
-            ),
+            sf.add_months(sf.to_date(self.transaction_normalized.action_date), 3).alias("fiscal_action_date"),
             self.transaction_normalized.last_modified_date,
             self.transaction_normalized.fiscal_year,
             self.awards.certified_date.alias("award_certified_date"),
-            sf.year(sf.add_months(sf.to_date(self.awards.certified_date), 3)).alias(
-                "award_fiscal_year"
-            ),
+            sf.year(sf.add_months(sf.to_date(self.awards.certified_date), 3)).alias("award_fiscal_year"),
             self.transaction_normalized.create_date.cast(TimestampType()),
             self.transaction_normalized.update_date.cast(TimestampType()),
             self.awards.update_date.cast(TimestampType()).alias("award_update_date"),
@@ -103,12 +91,12 @@ def date_cols(self) -> list[Column]:
                 sf.to_timestamp(self.transaction_normalized.update_date),
                 self.awards.update_date,
             ).alias("etl_update_date"),
-            sf.to_date(
-                self.transaction_normalized.period_of_performance_start_date
-            ).alias("period_of_performance_start_date"),
-            sf.to_date(
-                self.transaction_normalized.period_of_performance_current_end_date
-            ).alias("period_of_performance_current_end_date"),
+            sf.to_date(self.transaction_normalized.period_of_performance_start_date).alias(
+                "period_of_performance_start_date"
+            ),
+            sf.to_date(self.transaction_normalized.period_of_performance_current_end_date).alias(
+                "period_of_performance_current_end_date"
+            ),
             sf.coalesce(
                 self.transaction_fabs.created_at,
                 sf.to_timestamp(self.transaction_fpds.initial_report_date),
@@ -230,24 +218,16 @@ def amounts_cols(self) -> list[Column]:
             )
             .cast(DecimalType(23, 2))
             .alias("generated_pragmatic_obligation"),
-            sf.coalesce(
-                self.transaction_normalized.federal_action_obligation, sf.lit(0)
-            )
+            sf.coalesce(self.transaction_normalized.federal_action_obligation, sf.lit(0))
             .cast(DecimalType(23, 2))
             .alias("federal_action_obligation"),
-            sf.coalesce(
-                self.transaction_normalized.original_loan_subsidy_cost, sf.lit(0)
-            )
+            sf.coalesce(self.transaction_normalized.original_loan_subsidy_cost, sf.lit(0))
             .cast(DecimalType(23, 2))
             .alias("original_loan_subsidy_cost"),
-            sf.coalesce(
-                self.transaction_normalized.face_value_loan_guarantee, sf.lit(0)
-            )
+            sf.coalesce(self.transaction_normalized.face_value_loan_guarantee, sf.lit(0))
             .cast(DecimalType(23, 2))
             .alias("face_value_loan_guarantee"),
-            self.transaction_normalized.indirect_federal_sharing.cast(
-                DecimalType(23, 2)
-            ),
+            self.transaction_normalized.indirect_federal_sharing.cast(DecimalType(23, 2)),
             self.transaction_normalized.funding_amount,
             sf.coalesce(self.transaction_fabs.total_funding_amount, sf.lit("0"))
             .cast(DecimalType(23, 2))
@@ -369,9 +349,7 @@ def recipient_cols(self) -> list[Column]:
                 self.transaction_fpds.ultimate_parent_legal_enti,
                 self.transaction_fabs.ultimate_parent_legal_enti,
             ).alias("parent_recipient_name_raw"),
-            sf.upper(self.parent_recipient.parent_recipient_name).alias(
-                "parent_recipient_name"
-            ),
+            sf.upper(self.parent_recipient.parent_recipient_name).alias("parent_recipient_name"),
             sf.coalesce(
                 self.transaction_fpds.ultimate_parent_unique_ide,
                 self.transaction_fabs.ultimate_parent_unique_ide,
@@ -766,9 +744,7 @@ def fpds_cols(self) -> list[Column]:
             self.transaction_fpds.price_evaluation_adjustmen,
             self.transaction_fpds.private_university_or_coll,
             self.transaction_fpds.product_or_service_code,
-            self.transaction_fpds.product_or_service_co_desc.alias(
-                "product_or_service_description"
-            ),
+            self.transaction_fpds.product_or_service_co_desc.alias("product_or_service_description"),
             self.transaction_fpds.program_acronym,
             self.transaction_fpds.program_system_or_equ_desc,
             self.transaction_fpds.program_system_or_equipmen,
@@ -857,8 +833,7 @@ def dataframe(self) -> DataFrame:
             )
             .join(
                 self.references_cfda,
-                self.transaction_fabs.cfda_number
-                == self.references_cfda.program_number,
+                self.transaction_fabs.cfda_number == self.references_cfda.program_number,
                 "leftouter",
             )
             .join(
@@ -873,29 +848,23 @@ def dataframe(self) -> DataFrame:
             )
             .join(
                 self.awarding_agency,
-                self.transaction_normalized.awarding_agency_id
-                == self.awarding_agency.id,
+                self.transaction_normalized.awarding_agency_id == self.awarding_agency.id,
                 "leftouter",
             )
             .join(
                 self.awarding_toptier_agency,
-                self.awarding_agency.toptier_agency_id
-                == self.awarding_toptier_agency.toptier_agency_id,
+                self.awarding_agency.toptier_agency_id == self.awarding_toptier_agency.toptier_agency_id,
                 "leftouter",
             )
             .join(
                 self.awarding_subtier_agency,
-                self.awarding_agency.subtier_agency_id
-                == self.awarding_subtier_agency.subtier_agency_id,
+                self.awarding_agency.subtier_agency_id == self.awarding_subtier_agency.subtier_agency_id,
                 "leftouter",
             )
             .join(
                 self.awarding_agency_id,
                 (
-                    (
-                        self.awarding_agency_id.toptier_agency_id
-                        == self.awarding_toptier_agency.toptier_agency_id
-                    )
+                    (self.awarding_agency_id.toptier_agency_id == self.awarding_toptier_agency.toptier_agency_id)
                     & self.awarding_agency_id.toptier_flag
                 ),
                 "leftouter",
@@ -907,14 +876,12 @@ def dataframe(self) -> DataFrame:
             )
             .join(
                 self.funding_toptier_agency,
-                self.funding_agency.funding_toptier_agency_id
-                == self.funding_toptier_agency.toptier_agency_id,
+                self.funding_agency.funding_toptier_agency_id == self.funding_toptier_agency.toptier_agency_id,
                 "leftouter",
             )
             .join(
                 self.funding_subtier_agency,
-                self.funding_agency.funding_subtier_agency_id
-                == self.funding_subtier_agency.subtier_agency_id,
+                self.funding_agency.funding_subtier_agency_id == self.funding_subtier_agency.subtier_agency_id,
                 "leftouter",
             )
             .join(
@@ -922,17 +889,13 @@ def dataframe(self) -> DataFrame:
                 # are grouped together. This creates heavily skewed data across tasks which run into resource
                 # constraints.
                 sf.broadcast(self.funding_agency_id),
-                (
-                    self.funding_agency_id.toptier_agency_id
-                    == self.funding_toptier_agency.funding_toptier_agency_id
-                )
+                (self.funding_agency_id.toptier_agency_id == self.funding_toptier_agency.funding_toptier_agency_id)
                 & (self.funding_agency_id.row_num == 1),
                 "leftouter",
             )
             .join(
                 self.parent_recipient,
-                self.parent_recipient.parent_recipient_hash
-                == self.generated_parent_recipient_hash,
+                self.parent_recipient.parent_recipient_hash == self.generated_parent_recipient_hash,
                 "leftouter",
             )
             .join(
@@ -990,9 +953,7 @@ def dataframe(self) -> DataFrame:
         )
 
 
-def load_transaction_search(
-    spark: SparkSession, destination_database: str, destination_table_name: str
-) -> None:
+def load_transaction_search(spark: SparkSession, destination_database: str, destination_table_name: str) -> None:
     df = TransactionSearch(spark).dataframe
     df.write.saveAsTable(
         f"{destination_database}.{destination_table_name}",
@@ -1004,9 +965,7 @@ def load_transaction_search(
 def load_transaction_search_incremental(
     spark: SparkSession, destination_database: str, destination_table_name: str
 ) -> None:
-    target = DeltaTable.forName(
-        spark, f"{destination_database}.{destination_table_name}"
-    ).alias("t")
+    target = DeltaTable.forName(spark, f"{destination_database}.{destination_table_name}").alias("t")
     source = TransactionSearch(spark).dataframe.alias("s")
     (
         target.merge(

From 32301c295020b6983cf27cc2fb437a7f5a2960da Mon Sep 17 00:00:00 2001
From: aguest-kc <andrew.guest@kc.frb.org>
Date: Mon, 10 Aug 2026 08:42:06 -0500
Subject: [PATCH 48/48] [DEV-14608] Add tempo-init service to the tracing
 profile

---
 docker-compose.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docker-compose.yml b/docker-compose.yml
index de800d2fe2..f72af94504 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -554,6 +554,8 @@ services:
   # Tempo runs as user 10001, and docker compose creates the volume as root.
   # As such, we need to chown the volume in order for Tempo to start correctly.
   tempo-init:
+    profiles:
+      - tracing
     image: &tempoImage grafana/tempo:2.8.2
     user: root
     entrypoint: