fix(ci/full-run): Use sqlite WAL and busy timeout

Signed-off-by: Marcel Klehr <mklehr@gmx.net>

Author: marcelklehr

.github/workflows/cluster-faces-test.yml

@@ -637,7 +637,7 @@ jobs:
         if: steps.db-cache.outputs.cache-hit != 'true'
         run: |
           ./occ app_api:daemon:register --net host manual_install "Manual Install" manual-install http localhost http://localhost:8080
-          ./occ app_api:app:register recognize_backend manual_install --json-info "{\"appid\":\"recognize_backend\",\"name\":\"Recognize Backend\",\"daemon_config_name\":\"manual_install\",\"version\":\"${{ steps.backendinfo.outputs.result }}\",\"secret\":\"12345\",\"port\":9031,\"scopes\":[\"TASK_PROCESSING\",\"FILES\"],\"system_app\":0}" --force-scopes --wait-finish
+          ./occ app_api:app:register recognize_backend manual_install --json-info "{\"appid\":\"recognize_backend\",\"name\":\"Recognize Backend\",\"daemon_config_name\":\"manual_install\",\"version\":\"${{ steps.backendinfo.outputs.result }}\",\"secret\":\"12345\",\"port\":9031,\"scopes\":[\"TASK_PROCESSING\",\"FILES\"]}" --force-scopes --wait-finish
 
       - name: install sqlite3
         if: steps.db-cache.outputs.cache-hit != 'true' && env.ACT # Skip this on normal GitHub Actions

.github/workflows/full-run-test.yml

@@ -323,6 +323,15 @@ jobs:
           # 4 workers by default
           composer run serve &
 
+      - name: Enable SQLite WAL mode
+        if: ${{ matrix.databases == 'sqlite' }}
+        run: |
+          # WAL lets the ExApp's readers and cron's writers proceed concurrently
+          # instead of serializing on SQLite's single-writer lock, which otherwise
+          # stalls task scheduling for many minutes per cron run. The mode is
+          # persisted in the database header, so it survives across connections.
+          sqlite3 data/nextcloud.db "PRAGMA journal_mode=WAL;"
+
       - name: Checkout app_api
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
@@ -379,7 +388,7 @@ jobs:
         run: |
           ./occ app_api:daemon:register --net host manual_install "Manual Install" manual-install http localhost http://localhost:8080
           ./occ app_api:app:register ${{ env.APP_ID }} manual_install --json-info \
-            "{\"appid\":\"${{ env.APP_ID }}\",\"name\":\"Recognize Backend\",\"daemon_config_name\":\"manual_install\",\"version\":\"${{ env.APP_VERSION }}\",\"secret\":\"${{ env.APP_SECRET }}\",\"port\":${{ env.APP_PORT }},\"scopes\":[\"TASK_PROCESSING\",\"FILES\"],\"system_app\":1}" \
+            "{\"appid\":\"${{ env.APP_ID }}\",\"name\":\"Recognize Backend\",\"daemon_config_name\":\"manual_install\",\"version\":\"${{ env.APP_VERSION }}\",\"secret\":\"${{ env.APP_SECRET }}\",\"port\":${{ env.APP_PORT }},\"scopes\":[\"TASK_PROCESSING\",\"FILES\"]}" \
             --force-scopes --wait-finish
 
       - name: Upload photos
@@ -394,36 +403,71 @@ jobs:
           ./occ config:app:set --value ${{ matrix.musicnn-enabled }} recognize musicnn.enabled
           ./occ config:app:set --value ${{ matrix.movinet-enabled }} recognize movinet.enabled
 
-      - name: Schedule classification tasks
-        env:
-          GITHUB_REF: ${{ github.ref }}
+      - name: Run classification via TaskProcessing
+        id: classify
         run: |
+          # Probe available disk so we can tell whether runs die on ENOSPC.
+          disk_free() { df -h / | awk 'NR==2 {print $4" free ("$5" used)"}'; }
+          # Query the DB with a busy timeout so a transient WAL lock (the ExApp and
+          # cron write concurrently) makes the poll wait instead of failing with
+          # "database is locked" (exit 5), which would otherwise kill the step.
+          sq() { sqlite3 -cmd ".timeout 60000" data/nextcloud.db "$1"; }
+          echo "disk before classification: $(disk_free)"
+          ./occ upgrade # in case server master has new migrations in the meantime
           ./occ files:scan admin
-          # recognize:classify does not work in taskprocessing mode; instead let the
-          # background jobs crawl the storages and schedule TaskProcessing tasks.
+          # Kick off a full classification run: SchedulerJob -> StorageCrawlJob
+          # fills the faces queue and ClassifyFacesJob hands each batch to the
+          # recognize_backend ExApp as a TaskProcessing task. Results are written
+          # back asynchronously by the TaskResultListener when the ExApp reports.
           ./occ recognize:recrawl
-          # Run cron a few times so SchedulerJob -> StorageCrawlJob -> Classify*Job run
-          # in sequence and schedule the TaskProcessing tasks for the uploaded files.
-          for i in $(seq 1 12); do
-            php cron.php -v
-            sleep 30
+          # Drive the background jobs by running cron in a loop until the faces
+          # queue is drained and no crawl/scheduler jobs remain.
+          for i in $(seq 1 180); do
+            # Clustering is done explicitly in the "Run clustering" step below, so
+            # drop these jobs before each cron run to keep classification cron fast.
+            sq "delete from oc_jobs where class like '%ClusterFacesJob';" || true
+            php cron.php || true
+            QUEUE=$(sq "select count(*) from oc_recognize_queue_faces;")
+            CRAWL=$(sq "select count(*) from oc_jobs where class like '%StorageCrawlJob' or class like '%SchedulerJob';")
+            echo "round $i: faces queue=$QUEUE, pending crawl/scheduler jobs=$CRAWL, disk=$(disk_free)"
+            if [ "$QUEUE" -eq 0 ] && [ "$CRAWL" -eq 0 ] && [ "$i" -gt 3 ]; then break; fi
+            sleep 10
           done
-
-      - name: Wait for tasks to be processed by recognize_backend
-        run: |
-          set -x
-          # The backend downloads the models on the first run and processes the tasks;
-          # TaskResultListener applies the results as each task succeeds.
-          NEXT_WAIT_TIME=0
-          DETECTIONS=0
-          until [ $NEXT_WAIT_TIME -eq 60 ] || [ "$DETECTIONS" -gt 0 ]; do
-            php cron.php -v
-            DETECTIONS=$(sqlite3 data/nextcloud.db "select count(*) from oc_recognize_face_detections;" 2>/dev/null || echo 0)
-            echo "face detections so far: $DETECTIONS (iteration $NEXT_WAIT_TIME)"
+          # Wait for the ExApp to finish processing all scheduled TaskProcessing
+          # tasks (status 0=unknown, 1=scheduled, 2=running are still pending).
+          for i in $(seq 1 240); do
+            PENDING=$(sq "select count(*) from oc_taskprocessing_tasks where app_id = 'recognize' and status in (0, 1, 2);")
+            echo "wait $i: pending recognize taskprocessing tasks=$PENDING, disk=$(disk_free)"
+            if [ "$PENDING" -eq 0 ]; then break; fi
             sleep 30
-            NEXT_WAIT_TIME=$((NEXT_WAIT_TIME + 1))
           done
-          # Fail if the backend never produced any results
+          echo "disk after classification: $(disk_free)"
+
+          # Whether the run actually finished. The faces queue emptying is not
+          # enough: files leave the queue when their TaskProcessing task is
+          # *scheduled*, not when its results are written back, so a drained queue
+          # with tasks still pending means those photos have no detections at all.
+          # Detection counts cannot be used here - a photo with no face in it
+          # legitimately produces no rows - so completeness is queue + task state.
+          QUEUE=$(sq "select count(*) from oc_recognize_queue_faces;")
+          PENDING=$(sq "select count(*) from oc_taskprocessing_tasks where app_id = 'recognize' and status in (0, 1, 2);")
+          FAILED=$(sq "select count(*) from oc_taskprocessing_tasks where app_id = 'recognize' and status = 4;")
+          TOTAL=$(sq "select count(*) from oc_taskprocessing_tasks where app_id = 'recognize';")
+          echo "final: faces queue=$QUEUE, pending tasks=$PENDING, failed tasks=$FAILED, total tasks=$TOTAL"
+          echo "photos on disk: $(find data/admin/files -type f | wc -l)"
+          DETECTIONS=$(sq "select count(*) from oc_recognize_face_detections;")
+          echo "photos with detections: $(sq "select count(distinct file_id) from oc_recognize_face_detections where user_id = 'admin';")"
+          echo "face detections: $DETECTIONS"
+
+          if [ "$QUEUE" -eq 0 ] && [ "$PENDING" -eq 0 ] && [ "$FAILED" -eq 0 ]; then
+            echo "complete=true" >> "$GITHUB_OUTPUT"
+          else
+            echo "complete=false" >> "$GITHUB_OUTPUT"
+            echo "::error title=Incomplete classification run::faces queue=$QUEUE, pending tasks=$PENDING, failed tasks=$FAILED of $TOTAL. Each pending task covers up to 500 photos, whose detections were never written. The detection DB will NOT be cached; the metrics below score only the photos that made it through."
+          fi
+
+          # Hard signal for this job: the TaskProcessing pipeline must actually
+          # persist face detections.
           [ "$DETECTIONS" -gt 0 ]
 
       - name: Save recognize_backend models cache