crawl.yml

name: Crawl Maven Central for Java Modules

on:
  schedule:
    - cron: '7 */12 * * *'
  workflow_dispatch:
    inputs:
      budget_minutes:
        description: Wall-clock budget in minutes
        required: false
        default: '180'
      concurrency:
        description: Concurrent artifact fetches
        required: false
        default: '64'
      push_every:
        description: Number of checkpoints between git pushes
        required: false
        default: '10'
      resume:
        description: false re-streams the Maven Central index from scratch (already-scanned coordinates are still skipped)
        required: false
        default: 'true'
        type: choice
        options:
          - 'true'
          - 'false'

permissions:
  contents: write
  actions: read

# Manual dispatches share the `crawl` group so a double-click on "Run workflow"
# queues rather than overlaps. Scheduled triggers use a unique per-run group so
# they start immediately and let the guard job below decide whether to skip --
# this is what prevents a long manual crawl from being followed by a queued
# scheduled run that fires the moment the manual one ends.
concurrency:
  group: ${{ github.event_name == 'schedule' && format('crawl-schedule-{0}', github.run_id) || 'crawl' }}
  cancel-in-progress: false

jobs:
  guard:
    runs-on: ubuntu-latest
    outputs:
      skip: ${{ steps.check.outputs.skip }}
    steps:
      - name: Skip scheduled trigger if another crawl is in flight
        id: check
        env:
          GH_TOKEN: ${{ github.token }}
        run: |
          set -euo pipefail
          if [ "${{ github.event_name }}" != "schedule" ]; then
            echo "Manual trigger; proceeding regardless of other runs."
            echo "skip=false" >> "$GITHUB_OUTPUT"
            exit 0
          fi
          OTHERS=$(gh api \
            "/repos/${{ github.repository }}/actions/workflows/crawl.yml/runs?status=in_progress&per_page=100" \
            -q "[.workflow_runs[] | select(.id != ${{ github.run_id }})] | length")
          if [ "$OTHERS" -gt 0 ]; then
            echo "::notice::Skipping scheduled crawl: ${OTHERS} other crawl run(s) already in flight."
            echo "skip=true" >> "$GITHUB_OUTPUT"
          else
            echo "No other crawl in flight; proceeding."
            echo "skip=false" >> "$GITHUB_OUTPUT"
          fi

  crawl:
    needs: guard
    if: needs.guard.outputs.skip != 'true'
    runs-on: ubuntu-latest
    timeout-minutes: 200

    steps:
      - name: Check out repository
        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
        with:
          fetch-depth: 50
          persist-credentials: true
          submodules: recursive

      - name: Set up JDK 26
        uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0
        with:
          distribution: temurin
          java-version: '26'

      - name: Configure git
        run: |
          git config user.name "github-actions[bot]"
          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
          git config pull.rebase true
          mkdir -p data

      - name: Run crawler with incremental commits
        env:
          ARTIFACT_BASE: ${{ vars.ARTIFACT_BASE || 'https://maven-central.storage-download.googleapis.com/maven2/' }}
          INDEX_BASE: ${{ vars.INDEX_BASE || 'https://repo.maven.apache.org/maven2/.index/' }}
          # HEAD target for the canonical Last-Modified when ARTIFACT_BASE is a mirror that
          # rewrites mtimes (the GCS mirror serves pre-2019 imports with bucket-landing
          # times rather than publish times). Falls back to ARTIFACT_BASE when unset, which
          # disables the upgrade pass for forks that point everything at a single source.
          CANONICAL_TIMESTAMP_BASE: ${{ vars.CANONICAL_TIMESTAMP_BASE || 'https://repo.maven.apache.org/maven2/' }}
          BUDGET_MINUTES: ${{ github.event.inputs.budget_minutes || vars.BUDGET_MINUTES || '180' }}
          CONCURRENCY: ${{ github.event.inputs.concurrency || vars.CONCURRENCY || '64' }}
          TAIL_SIZE: ${{ vars.TAIL_SIZE }}
          SMALL_JAR_THRESHOLD: ${{ vars.SMALL_JAR_THRESHOLD }}
          CHECKPOINT_EVERY: ${{ vars.CHECKPOINT_EVERY }}
          RESUME: ${{ github.event.inputs.resume || 'true' }}
          PUSH_EVERY: ${{ github.event.inputs.push_every || vars.GIT_PUSH_EVERY || '10' }}
        run: |
          set -euo pipefail
          props=(
            "-Djenesis.crawler.data=$GITHUB_WORKSPACE/data"
            "-Djenesis.crawler.budget=$BUDGET_MINUTES"
            "-Djenesis.crawler.concurrency=$CONCURRENCY"
            "-Djenesis.crawler.resume=$RESUME"
            "-Djenesis.crawler.git.publish=true"
            "-Djenesis.crawler.git.work.dir=$GITHUB_WORKSPACE"
            "-Djenesis.crawler.git.push.every=$PUSH_EVERY"
            "-Djenesis.crawler.canonical.timestamp.uri=$CANONICAL_TIMESTAMP_BASE"
          )
          [ -n "$TAIL_SIZE" ]            && props+=("-Djenesis.crawler.tail.size=$TAIL_SIZE")
          [ -n "$SMALL_JAR_THRESHOLD" ]  && props+=("-Djenesis.crawler.small.jar.threshold=$SMALL_JAR_THRESHOLD")
          [ -n "$CHECKPOINT_EVERY" ]    && props+=("-Djenesis.crawler.checkpoint.every=$CHECKPOINT_EVERY")
          # Heap left at the JVM ergonomic default (~25% of runner RAM, i.e. ~4 GB
          # on ubuntu-latest). The earlier 512 MB cap was a debugging measure -
          # the heap-dump analysis confirmed the "leak" was actually one legitimate
          # ~175 MB allocation by the software.amazon.awssdk group (470 K scanned
          # entries) tripping the cap. The ScannedStore now emits a [scanned-store]
          # WARNING on first load of any group above LARGE_GROUP_WARN_THRESHOLD so
          # the next OOM-prone group becomes visible in the logs without needing a
          # heap dump.
          java -XX:+ExitOnOutOfMemoryError \
               -XX:+HeapDumpOnOutOfMemoryError \
               -XX:HeapDumpPath="$GITHUB_WORKSPACE/crawl.hprof" \
               "${props[@]}" sources/build/jenesis/crawler/Crawl.java \
               "$ARTIFACT_BASE" "$INDEX_BASE"

      - name: Upload heap dump on failure
        if: ${{ failure() }}
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: crawl-heap-dump-${{ github.run_id }}
          path: crawl.hprof
          if-no-files-found: ignore
          retention-days: 7
          compression-level: 9

      - name: Commit any uncommitted progress
        if: ${{ !cancelled() }}
        run: |
          set -euo pipefail
          git add data
          if git diff --cached --quiet; then
            echo "Nothing left to commit; the crawler pushed everything inline."
            exit 0
          fi
          stamp="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
          git commit -m "crawl: tail flush $stamp"
          for attempt in 1 2 3; do
            if git push; then
              exit 0
            fi
            git pull --rebase || true
          done
          echo "Final push failed after 3 attempts" >&2
          exit 1