Bump org.junit.jupiter:junit-jupiter-engine from 5.8.1 to 6.1.3 in /spark

.github/dependabot.yml

@@ -1,11 +1,31 @@
-# To get started with Dependabot version updates, you'll need to specify which
-# package ecosystems to update and where the package manifests are located.
-# Please see the documentation for all configuration options:
-# https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
-
+# ---------------------------------------------------------------------------
+# AUTO-GENERATED by scripts/gen-dependabot.py from the poms' dependency scopes.
+# Do not edit by hand — run the script and commit the result.
+#   allow  (include) = `compile`-scope deps we actually ship  -> watch these
+#   ignore (exclude) = `provided`-scope deps supplied by the runtime -> noise
+# A dep that is `provided` in any module is treated as provided (ignore wins).
+# ---------------------------------------------------------------------------
 version: 2
 updates:
-  - package-ecosystem: "*" # See documentation for possible values
-    directory: "/" # Location of package manifests
+  - package-ecosystem: "maven"
+    directories:
+      - "/**" # root + all Maven submodules
     schedule:
       interval: "weekly"
+    open-pull-requests-limit: 10
+    allow:
+      - dependency-name: "com.fasterxml.jackson.core:jackson-annotations"
+      - dependency-name: "com.fasterxml.jackson.module:jackson-module-scala_2.12"
+      - dependency-name: "com.wcohen.ss:secondstring"
+      - dependency-name: "commons-logging:commons-logging"
+      - dependency-name: "io.graphframes:graphframes-spark3_2.12"
+      - dependency-name: "javax.mail:mail"
+      - dependency-name: "org.apache.httpcomponents:httpclient"
+      - dependency-name: "org.freemarker:freemarker"
+    ignore:
+      - dependency-name: "com.fasterxml.jackson.core:jackson-core"
+      - dependency-name: "com.fasterxml.jackson.core:jackson-databind"
+      - dependency-name: "org.apache.spark:spark-core_*"
+      - dependency-name: "org.apache.spark:spark-graphx_*"
+      - dependency-name: "org.apache.spark:spark-mllib_*"
+      - dependency-name: "org.scala-lang:scala-library"

.github/workflows/build.yml

@@ -15,9 +15,9 @@ jobs:
     strategy:
       fail-fast: false
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up JDK 11
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           java-version: '11'
           distribution: 'adopt'

.github/workflows/dependabot-config.yml

@@ -0,0 +1,29 @@
+name: dependabot-config
+
+# Ensures .github/dependabot.yml stays in sync with the poms' dependency scopes.
+# The file is generated by scripts/gen-dependabot.py; this fails the build if a
+# pom changed but the config wasn't regenerated.
+on:
+  pull_request:
+    paths:
+      - '**/pom.xml'
+      - '.github/dependabot.yml'
+      - 'scripts/gen-dependabot.py'
+      - '.github/workflows/dependabot-config.yml'
+  push:
+    branches: [ main ]
+    paths:
+      - '**/pom.xml'
+      - '.github/dependabot.yml'
+      - 'scripts/gen-dependabot.py'
+
+jobs:
+  check:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.x'
+      - name: Verify dependabot.yml matches the poms
+        run: python3 scripts/gen-dependabot.py --check

.github/workflows/perfTest-febrl120K.yml

@@ -1,80 +1,14 @@
 name: performance-test-febrl120K
 
 on:
-  workflow_dispatch:
-  schedule:
-    - cron: "0 0 */3 * *" # runs every 3 days at 00 AM UTC
+  workflow_dispatch:
+  schedule:
+    - cron: "0 0 */3 * *" # runs every 3 days at 00 AM UTC
+
 jobs:
-  perf-test:
-    name: performance-test-febrl120K
-    runs-on: ubuntu-latest
-    env:
-      SPARK_MASTER: local[*]
-      ZINGG_HOME: assembly/target
-      ZINGG_USER: zingg_user
-      INPUT: "zingg_community_performance_reports/perf_test/perfTestInput_febrl120K.json"
-    steps:
-      - name: checkout repo content
-        uses: actions/checkout@v3 # checkout the repository content to github runner.
-      - name: Load Spark config
-        id: cfg
-        run: |
-          set -a  #Automatically marks all subsequently defined variables for export
-          source spark.env
-          echo "spark_version=$SPARK_VERSION" >> "$GITHUB_OUTPUT"
-          echo "hadoop_version=$HADOOP_VERSION" >> "$GITHUB_OUTPUT"
-          echo "spark_url=$SPARK_URL" >> "$GITHUB_OUTPUT"
-      - name: setup python
-        uses: actions/setup-python@v4
-        with:
-          python-version: 3.12.7 #install the python needed
-      - uses: actions/setup-java@v4
-        with:
-          java-version: '11'
-          distribution: temurin
-      - name: setup spark
-        uses: vemonet/setup-spark@v1
-        with:
-          spark-version: ${{ steps.cfg.outputs.spark_version }}
-          hadoop-version: ${{ steps.cfg.outputs.hadoop_version }}
-          spark-url: ${{ steps.cfg.outputs.spark_url }}
-      - name: check spark
-        run: spark-submit --version
-      - name: mvn clean
-        run: mvn clean
-      - name: Build with Maven
-        run: mvn clean compile package -DskipTests=true
-      - name: Clone performance reports repo
-        run: |
-          set -e #Exit immediately if a command exits with a non-zero status.
-          echo "Cloning performance reports repo..."
-          git clone https://x-access-token:${{ secrets.GH_PERF_COMMUNITY_ACTION }}@github.com/zinggAI/zingg_community_performance_reports.git
-          echo "Listing inside cloned repo:"
-          ls -al zingg_community_performance_reports || true
-      - name: download and execute py script
-        run: |
-          wget -O perfTestRunner.py https://raw.githubusercontent.com/zinggAI/zingg_performance/main/perfTestRunner.py
-          echo "Running script..."
-          python perfTestRunner.py
-      - name: Commit test results
-        if: always()
-        run: |
-          cd zingg_community_performance_reports
-          git remote set-url origin https://x-access-token:${{ secrets.GH_PERF_COMMUNITY_ACTION }}@github.com/zinggAI/zingg_community_performance_reports.git
-          git config user.name sonalgoyal
-          git config user.email sonalgoyal4@gmail.com
-          git add perf_test/perf_test_report/*.json
-          if git diff --cached --quiet; then
-            echo "No changes to commit for perfTest-febrl120K."
-          else
-            echo "Committing changes..."
-            git commit -m "report generated"
-
-            echo "Fetching remote..."
-            git fetch origin main
-
-            echo "Rebasing onto origin/main..."
-            git rebase origin/main || (echo "Rebase failed; resetting hard to origin/main" && git rebase --abort && git reset --hard origin/main)
-            echo "Attempting push..."
-            git push origin HEAD:main
-          fi
+  perf-test:
+    uses: ./.github/workflows/perfTest-reusable.yml
+    with:
+      test_name: performance-test-febrl120K
+      input_file: "zingg_community_performance_reports/perf_test/perfTestInput_febrl120K.json"
+    secrets: inherit

.github/workflows/perfTest-ncVoters5M.yml

@@ -1,80 +1,14 @@
 name: performance-test-ncVoters5M
 
 on:
-  workflow_dispatch:
-  schedule:
-    - cron: "0 3 */3 * *" # runs every 3 days at 3 AM UTC
+  workflow_dispatch:
+  schedule:
+    - cron: "0 3 */3 * *" # runs every 3 days at 3 AM UTC
+
 jobs:
-  perf-test:
-    name: performance-test-ncVoters5M
-    runs-on: ubuntu-latest
-    env:
-      SPARK_MASTER: local[*]
-      ZINGG_HOME: assembly/target
-      ZINGG_USER: zingg_user
-      INPUT: "zingg_community_performance_reports/perf_test/perfTestInput_ncVoters5M.json"
-    steps:
-      - name: checkout repo content
-        uses: actions/checkout@v3 # checkout the repository content to github runner.
-      - name: Load Spark config
-        id: cfg
-        run: |
-          set -a  #Automatically marks all subsequently defined variables for export
-          source spark.env
-          echo "spark_version=$SPARK_VERSION" >> "$GITHUB_OUTPUT"
-          echo "hadoop_version=$HADOOP_VERSION" >> "$GITHUB_OUTPUT"
-          echo "spark_url=$SPARK_URL" >> "$GITHUB_OUTPUT"
-      - name: setup python
-        uses: actions/setup-python@v4
-        with:
-          python-version: 3.12.7 #install the python needed
-      - uses: actions/setup-java@v4
-        with:
-          java-version: '11'
-          distribution: temurin
-      - name: setup spark
-        uses: vemonet/setup-spark@v1
-        with:
-          spark-version: ${{ steps.cfg.outputs.spark_version }}
-          hadoop-version: ${{ steps.cfg.outputs.hadoop_version }}
-          spark-url: ${{ steps.cfg.outputs.spark_url }}
-      - name: check spark
-        run: spark-submit --version
-      - name: mvn clean
-        run: mvn clean
-      - name: Build with Maven
-        run: mvn clean compile package -DskipTests=true
-      - name: Clone performance reports repo
-        run: |
-          set -e #Exit immediately if a command exits with a non-zero status.
-          echo "Cloning performance reports repo..."
-          git clone https://x-access-token:${{ secrets.GH_PERF_COMMUNITY_ACTION }}@github.com/zinggAI/zingg_community_performance_reports.git
-          echo "Listing inside cloned repo:"
-          ls -al zingg_community_performance_reports || true
-      - name: download and execute py script
-        run: |
-          wget -O perfTestRunner.py https://raw.githubusercontent.com/zinggAI/zingg_performance/main/perfTestRunner.py
-          echo "Running script..."
-          python perfTestRunner.py
-      - name: Commit test results
-        if: always()
-        run: |
-          cd zingg_community_performance_reports
-          git remote set-url origin https://x-access-token:${{ secrets.GH_PERF_COMMUNITY_ACTION }}@github.com/zinggAI/zingg_community_performance_reports.git
-          git config user.name sonalgoyal
-          git config user.email sonalgoyal4@gmail.com
-          git add perf_test/perf_test_report/*.json
-          if git diff --cached --quiet; then
-            echo "No changes to commit for perfTest-febrl120K."
-          else
-            echo "Committing changes..."
-            git commit -m "report generated"
-
-            echo "Fetching remote..."
-            git fetch origin main
-
-            echo "Rebasing onto origin/main..."
-            git rebase origin/main || (echo "Rebase failed; resetting hard to origin/main" && git rebase --abort && git reset --hard origin/main)
-            echo "Attempting push..."
-            git push origin HEAD:main
-          fi
+  perf-test:
+    uses: ./.github/workflows/perfTest-reusable.yml
+    with:
+      test_name: performance-test-ncVoters5M
+      input_file: "zingg_community_performance_reports/perf_test/perfTestInput_ncVoters5M.json"
+    secrets: inherit

.github/workflows/perfTest-reusable.yml

@@ -0,0 +1,94 @@
+name: performance-test-reusable
+
+on:
+  workflow_call:
+    inputs:
+      test_name:
+        description: "Name of the performance test (used for the job name and logs)"
+        required: true
+        type: string
+      input_file:
+        description: "Path to the perf test input JSON (passed to the runner as INPUT)"
+        required: true
+        type: string
+
+jobs:
+  perf-test:
+    name: ${{ inputs.test_name }}
+    runs-on: ubuntu-latest
+    env:
+      SPARK_MASTER: local[*]
+      ZINGG_HOME: assembly/target
+      ZINGG_USER: zingg_user
+      INPUT: ${{ inputs.input_file }}
+    steps:
+      - name: checkout repo content
+        uses: actions/checkout@v4 # checkout the repository content to github runner.
+      - name: Load Spark config
+        id: cfg
+        run: |
+          set -a  #Automatically marks all subsequently defined variables for export
+          source spark.env
+          echo "spark_version=$SPARK_VERSION" >> "$GITHUB_OUTPUT"
+          echo "hadoop_version=$HADOOP_VERSION" >> "$GITHUB_OUTPUT"
+          echo "spark_url=$SPARK_URL" >> "$GITHUB_OUTPUT"
+      - name: setup python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.12.7 #install the python needed
+      - uses: actions/setup-java@v4
+        with:
+          java-version: '11'
+          distribution: temurin
+      - name: setup spark
+        uses: vemonet/setup-spark@v1
+        with:
+          spark-version: ${{ steps.cfg.outputs.spark_version }}
+          hadoop-version: ${{ steps.cfg.outputs.hadoop_version }}
+          spark-url: ${{ steps.cfg.outputs.spark_url }}
+      - name: check spark
+        run: spark-submit --version
+      - name: mvn clean
+        run: mvn clean
+      - name: Build with Maven
+        run: mvn clean compile package -DskipTests=true
+      - name: Clone performance reports repo
+        run: |
+          set -e #Exit immediately if a command exits with a non-zero status.
+          echo "Cloning performance reports repo..."
+          git clone https://x-access-token:${{ secrets.GH_PERF_COMMUNITY_ACTION }}@github.com/zinggAI/zingg_community_performance_reports.git
+          echo "Listing inside cloned repo:"
+          ls -al zingg_community_performance_reports || true
+      - name: Install plotting dependencies
+        run: |
+          pip install pandas matplotlib
+      - name: download and execute py script
+        run: |
+          wget -O perfTestRunner.py https://raw.githubusercontent.com/zinggAI/zingg_performance/main/perfTestRunner.py
+          wget -O plot_result.py https://raw.githubusercontent.com/zinggAI/zingg_performance/main/plot_result.py
+          echo "Running script..."
+          python perfTestRunner.py
+          echo "Plotting results..."
+          python plot_result.py
+      - name: Commit test results
+        if: always()
+        run: |
+          cd zingg_community_performance_reports
+          git remote set-url origin https://x-access-token:${{ secrets.GH_PERF_COMMUNITY_ACTION }}@github.com/zinggAI/zingg_community_performance_reports.git
+          git config user.name sonalgoyal
+          git config user.email sonalgoyal4@gmail.com
+          git add perf_test/perf_test_report/
+          if git diff --cached --quiet; then
+            echo "No changes to commit for ${{ inputs.test_name }}."
+          else
+            echo "Committing changes..."
+            git commit -m "report generated"
+
+            echo "Fetching remote..."
+            git fetch origin main
+
+            echo "Rebasing onto origin/main..."
+            git rebase origin/main || (echo "Rebase failed; resetting hard to origin/main" && git rebase --abort && git reset --hard origin/main)
+            echo "Attempting push..."
+            git push origin HEAD:main
+          fi

.github/workflows/pmd.yml

@@ -16,8 +16,8 @@ jobs:
       contents: read
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-java@v3
+      - uses: actions/checkout@v4
+      - uses: actions/setup-java@v4
         with:
           distribution: 'temurin'
           java-version: '11'

.pre-commit-config.yaml

@@ -0,0 +1,18 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+      - id: trailing-whitespace      # remove spaces at the end of the line
+      - id: end-of-file-fixer        # file ends with exactly one new line
+      - id: check-yaml
+      - id: check-merge-conflict
+
+  - repo: local
+    hooks:
+      - id: dependabot-config
+        name: dependabot.yml is generated from the poms
+        entry: python3 scripts/gen-dependabot.py --check
+        language: system
+        pass_filenames: false
+        # Re-check whenever a pom or the generated config changes.
+        files: '(pom\.xml|\.github/dependabot\.yml|scripts/gen-dependabot\.py)$'

Dockerfile

@@ -1,5 +1,5 @@
 #FROM docker.io/bitnami/spark:3.1.2
-FROM apache/spark:3.5.0-python3
+FROM apache/spark:3.5.5-python3
 USER 0
 RUN apt-get update && \
 	apt install -y curl vim