Merge pull request #1561 from nextcloud/fix/upgrade-node #406
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Cluster-faces command test | |
| on: | |
| pull_request: | |
| paths: | |
| - 'lib/**' | |
| - 'src/**' | |
| push: | |
| branches: | |
| - main | |
| - stable* | |
| paths: | |
| - 'lib/**' | |
| - 'src/**' | |
| workflow_dispatch: | |
| env: | |
| APP_NAME: recognize | |
| # Ground truth annotations are matched to detections by IoU rather than by | |
| # top-left-corner distance: a backend that reports tighter or differently | |
| # anchored boxes (RetinaFace vs. the built-in detector) would otherwise look | |
| # like it had lost recall. | |
| MIN_IOU: '0.3' | |
| # Purity floor. Both backends have historically scored >= 0.95 here. | |
| MIN_CLUSTER_TARGET_ACCURACY: '0.85' | |
| # Combined score floor, per job: each "Analyse face assignments" step overrides | |
| # this with a value baselined against its own backend. The two are ~2x apart | |
| # (0.38 built-in vs 0.80 via the ExApp), so a single shared threshold would have | |
| # to sit under the lower one and could not catch the better backend regressing. | |
| # 0 here is only the fallback for a job that sets no floor of its own. | |
| MIN_COMBINED_SCORE: '0' | |
| # Coverage floor. Every purity metric in the report gets *better* as coverage | |
| # drops - a run that only ever detected faces in a third of the identity | |
| # directories scores near-perfect cluster accuracy on that third. Without this | |
| # gate a truncated run (ExApp tasks that never came back, a crawl that stalled) | |
| # reads as an improvement. Both backends detect at least one annotated face for | |
| # essentially every identity when they process the whole dataset. | |
| MIN_IDENTITY_DETECTION_RATE: '0.9' | |
| # Keep every Nth identity directory of the IMDb-Face set so the whole run fits | |
| # in the job time limit. The full zip is ~702 directories / ~23650 photos, i.e. | |
| # ~34 photos per identity. | |
| # | |
| # The TaskProcessing job is the constraint. The ExApp needs ~1.3s per image on a | |
| # single CPU worker, i.e. ~11 min per 500-photo TaskProcessing task, and the | |
| # classification step has a budget of roughly 3h (the cron loop plus the pending | |
| # task wait). The previous keep-every-2 set was ~11800 photos = ~24 tasks = | |
| # ~4.5h of ExApp work, which is why runs timed out with two thirds of the | |
| # dataset never processed. Observed throughput was lower still, about 8 tasks | |
| # per run, because in-flight photos get re-queued and scheduled twice. | |
| # | |
| # keep-every-8 is ~88 directories / ~2950 photos = ~6 tasks = ~1.1h, so roughly | |
| # 2x margin against what a run actually achieved. It can go back down towards 2 | |
| # once the classifier applies backpressure and stops dropping files from the | |
| # queue at schedule time, since that removes the duplicate work and makes the | |
| # loop's exit condition mean what it says. | |
| # | |
| # Both jobs use the same value so their metrics stay directly comparable, and | |
| # each keep-every-N set is a strict subset of the smaller N. Note that changing | |
| # this re-baselines both jobs: metrics are means over the identities on disk, so | |
| # they are not comparable across different values of N. | |
| DATASET_KEEP_EVERY: '10' | |
| jobs: | |
| php: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| # do not stop on another job's failure | |
| fail-fast: false | |
| matrix: | |
| php-versions: ['8.3'] | |
| databases: ['sqlite'] | |
| server-versions: ['master'] | |
| pure-js-mode: ['false'] | |
| name: Test cluster-faces command on ${{ matrix.server-versions }} wasm:${{ matrix.pure-js-mode }} | |
| env: | |
| MYSQL_PORT: 4444 | |
| PGSQL_PORT: 4445 | |
| services: | |
| mysql: | |
| image: mariadb:10.5 | |
| ports: | |
| - 4444:3306/tcp | |
| env: | |
| MYSQL_ROOT_PASSWORD: rootpassword | |
| options: --health-cmd="mysqladmin ping" --health-interval 5s --health-timeout 2s --health-retries 5 | |
| postgres: | |
| image: postgres | |
| ports: | |
| - 4445:5432/tcp | |
| env: | |
| POSTGRES_USER: root | |
| POSTGRES_PASSWORD: rootpassword | |
| POSTGRES_DB: nextcloud | |
| options: --health-cmd pg_isready --health-interval 5s --health-timeout 2s --health-retries 5 | |
| steps: | |
| - name: Checkout server | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| with: | |
| repository: nextcloud/server | |
| ref: ${{ matrix.server-versions }} | |
| - name: Checkout submodules | |
| shell: bash | |
| run: | | |
| auth_header="$(git config --local --get http.https://github.com/.extraheader)" | |
| git submodule sync --recursive | |
| git -c "http.extraheader=$auth_header" -c protocol.version=2 submodule update --init --force --recursive --depth=1 | |
| - name: install ssl-cert | |
| if: env.ACT # Skip this on normal GitHub Actions | |
| run: sudo apt update && sudo apt install -y ssl-cert | |
| - name: Set up php ${{ matrix.php-versions }} | |
| uses: shivammathur/setup-php@cf4cade2721270509d5b1c766ab3549210a39a2a # v2.33.0 | |
| with: | |
| php-version: ${{ matrix.php-versions }} | |
| tools: phpunit | |
| extensions: mbstring, iconv, fileinfo, intl, sqlite, pdo_mysql, pdo_sqlite, pgsql, pdo_pgsql, gd, zip | |
| - name: Checkout app | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| with: | |
| path: apps/${{ env.APP_NAME }} | |
| - name: Read package.json node and npm engines version | |
| uses: skjnldsv/read-package-engines-version-actions@06d6baf7d8f41934ab630e97d9e6c0bc9c9ac5e4 # v3 | |
| id: versions | |
| with: | |
| path: apps/${{ env.APP_NAME }} | |
| fallbackNode: '^12' | |
| fallbackNpm: '^6' | |
| - name: Set up node ${{ steps.versions.outputs.nodeVersion }} | |
| uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 | |
| with: | |
| node-version: ${{ steps.versions.outputs.nodeVersion }} | |
| - name: Set up npm ${{ steps.versions.outputs.npmVersion }} | |
| run: npm i -g npm@"${{ steps.versions.outputs.npmVersion }}" | |
| - name: install make wget unzip | |
| if: env.ACT # Skip this on normal GitHub Actions | |
| run: sudo apt update && sudo apt install -y make wget unzip | |
| - name: Install app | |
| working-directory: apps/${{ env.APP_NAME }} | |
| run: | | |
| composer install --no-dev | |
| make all | |
| make remove-binaries | |
| make remove-devdeps | |
| - name: Set up Nextcloud and install app | |
| if: ${{ matrix.databases != 'pgsql'}} | |
| run: | | |
| sleep 25 | |
| mkdir data | |
| ./occ maintenance:install --verbose --database=${{ matrix.databases }} --database-name=nextcloud --database-host=127.0.0.1 --database-port=$MYSQL_PORT --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass password | |
| ./occ app:enable -vvv -f ${{ env.APP_NAME }} | |
| php -S localhost:8080 & | |
| - name: Set up Nextcloud and install app | |
| if: ${{ matrix.databases == 'pgsql'}} | |
| run: | | |
| sleep 25 | |
| mkdir data | |
| ./occ maintenance:install --verbose --database=${{ matrix.databases }} --database-name=nextcloud --database-host=127.0.0.1 --database-port=$PGSQL_PORT --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass password | |
| ./occ app:enable -vvv -f ${{ env.APP_NAME }} | |
| php -S localhost:8080 & | |
| - name: Checkout app | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| with: | |
| repository: nextcloud/viewer | |
| path: apps/viewer | |
| - name: Install viewer | |
| run: | | |
| ./occ app:enable -vvv viewer | |
| - name: Remove unnecessary models to make space | |
| run: | | |
| rm -rf apps/recognize/models | |
| - uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 | |
| id: photos-cache | |
| with: | |
| path: data/admin/files/ | |
| key: https://cloud.nextcloud.com/public.php/dav/files/wfDk23DBsXYrd4S/IMDb-Face.zip-every${{ env.DATASET_KEEP_EVERY }} | |
| - name: Upload photos | |
| if: steps.photos-cache.outputs.cache-hit != 'true' | |
| run: | | |
| mkdir -p data/admin/files/ | |
| cd data/admin/files | |
| wget https://cloud.nextcloud.com/public.php/dav/files/wfDk23DBsXYrd4S/IMDb-Face.zip | |
| unzip IMDb-Face.zip | |
| rm IMDb-Face.zip | |
| # Keep only every DATASET_KEEP_EVERY'th identity directory so the | |
| # classification run stays within the job time limit. Deterministic | |
| # (sorted glob) so both jobs cache an identical set under the same key. | |
| cd IMDb-Face | |
| i=0 | |
| for d in */; do | |
| if [ $((i % ${{ env.DATASET_KEEP_EVERY }})) -ne 0 ]; then rm -rf "$d"; fi | |
| i=$((i + 1)) | |
| done | |
| echo "Kept $(ls -d */ | wc -l) identity directories" | |
| - uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 | |
| with: | |
| path: data/admin/files/ | |
| key: https://cloud.nextcloud.com/public.php/dav/files/wfDk23DBsXYrd4S/IMDb-Face.zip-every${{ env.DATASET_KEEP_EVERY }} | |
| - name: Set config | |
| run: | | |
| ./occ config:app:set --lazy --value ${{ matrix.pure-js-mode }} recognize tensorflow.purejs | |
| ./occ config:app:set --value true recognize faces.enabled | |
| # Don't force API key usage to allow tests to run | |
| ./occ config:app:set --value false recognize require_api_key | |
| # only use one core. GH actions has 2 | |
| ./occ config:app:set --value 1 recognize tensorflow.cores | |
| - uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 | |
| id: db-cache | |
| with: | |
| path: data/nextcloud.db | |
| key: ${{ runner.os }}-${{ matrix.server-versions }}-${{ hashFiles('data/admin/files/**', 'apps/recognize/src/classifier_faces.js', 'apps/recognize/lib/Classifiers/Classifier.php', 'apps/recognize/lib/Classifiers/Images/ClusteringFaceClassifier.php') }}-${{ matrix.pure-js-mode }} | |
| - name: Run classifier | |
| if: steps.db-cache.outputs.cache-hit != 'true' | |
| env: | |
| GITHUB_REF: ${{ github.ref }} | |
| run: | | |
| disk_free() { df -h / | awk 'NR==2 {print $4" free ("$5" used)"}'; } | |
| echo "disk before classification: $(disk_free)" | |
| ./occ files:scan admin | |
| ./occ recognize:classify | |
| echo "disk after classification: $(disk_free)" | |
| - uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 | |
| with: | |
| path: data/nextcloud.db | |
| key: ${{ steps.db-cache.outputs.cache-primary-key }} | |
| # The photos on disk are the run-independent denominator for every rate the | |
| # analysis step reports, so take the inventory before they are truncated. | |
| - name: Create photo inventory | |
| run: | | |
| find data/admin/files/IMDb-Face -type f -printf '%P\n' | sort > photos.txt | |
| wc -l photos.txt | |
| - name: Reduce space | |
| run: | | |
| for dirname in data/admin/files/IMDb-Face/*; do truncate -s 0 "${dirname}"/*; done | |
| - name: install sqlite3 | |
| if: env.ACT # Skip this on normal GitHub Actions | |
| run: sudo apt update && sudo apt install -y sqlite3 | |
| - name: Create detection summary | |
| run: | | |
| sqlite3 data/nextcloud.db "select x, y, width, height, path from oc_recognize_face_detections d LEFT JOIN oc_filecache c ON c.fileid = d.file_id where user_id = 'admin' ORDER BY path;" > out.txt | |
| - uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 | |
| id: clustering-cache | |
| with: | |
| path: out.json | |
| key: ${{ runner.os }}-${{ hashFiles('out.txt', 'apps/recognize/src/classifier_faces.js', 'apps/recognize/lib/Classifiers/Classifier.php', 'apps/recognize/lib/Classifiers/Images/ClusteringFaceClassifier.php', 'apps/recognize/lib/Clustering/**', 'apps/recognize/lib/Dav/**', 'apps/recognize/lib/Service/FaceClusterAnalyzer.php', 'apps/recognize/lib/Service/SettingsService.php', 'apps/recognize/lib/Command/ClusterFaces.php') }}-${{ matrix.pure-js-mode }} | |
| - name: Run clustering | |
| if: steps.clustering-cache.outputs.cache-hit != 'true' | |
| run: | | |
| ./occ upgrade # in case server master has new migrations in the meantime | |
| ./occ recognize:cluster-faces -b 10000 | |
| ./occ recognize:cluster-faces -b 10000 | |
| ./occ recognize:cluster-faces -b 10000 | |
| ./occ recognize:cluster-faces -b 10000 | |
| ./occ recognize:cluster-faces -b 10000 | |
| ./occ recognize:cluster-faces -b 10000 | |
| - name: install python3 python3-pip jq curl | |
| if: steps.clustering-cache.outputs.cache-hit != 'true' && env.ACT # Skip this on normal GitHub Actions | |
| run: sudo apt update && sudo apt install -y python3 python3-pip jq curl | |
| - name: Install xq | |
| if: steps.clustering-cache.outputs.cache-hit != 'true' | |
| run: | | |
| pip install yq --break-system-packages | |
| - name: Download face assignments | |
| if: steps.clustering-cache.outputs.cache-hit != 'true' | |
| run: | | |
| curl -u 'admin:password' --request PROPFIND 'http://localhost:8080/remote.php/dav/recognize/admin/faces/' --header 'Depth: 2' --data '<?xml version="1.0"?> | |
| <d:propfind xmlns:d="DAV:" | |
| xmlns:oc="http://owncloud.org/ns" | |
| xmlns:nc="http://nextcloud.org/ns" | |
| xmlns:ocs="http://open-collaboration-services.org/ns"> | |
| <d:prop> | |
| <d:getcontentlength /> | |
| <d:getcontenttype /> | |
| <d:getetag /> | |
| <d:getlastmodified /> | |
| <d:resourcetype /> | |
| <nc:face-detections /> | |
| <nc:file-metadata-size /> | |
| <nc:has-preview /> | |
| <nc:realpath /> | |
| <oc:favorite /> | |
| <oc:fileid /> | |
| <oc:permissions /> | |
| <nc:nbItems /> | |
| </d:prop> | |
| </d:propfind>' > out.xml | |
| cat out.xml | |
| - name: Parse face assignments | |
| if: steps.clustering-cache.outputs.cache-hit != 'true' | |
| run: | | |
| PATH=$PATH:/home/runner/.local/bin | |
| cat out.xml | xq '.["d:multistatus"]["d:response"] | map(select(.["d:href"] | test("faces/.+?/.+?"))) | map({"href": .["d:href"], "realpath": .["d:propstat"][0]["d:prop"]["nc:realpath"], "face-detections": .["d:propstat"][0]["d:prop"]["nc:face-detections"] | fromjson | map({userId, x, y, height, width, clusterId}) })' > out.json | |
| cat out.json | |
| - uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 | |
| with: | |
| path: out.json | |
| key: ${{ steps.clustering-cache.outputs.cache-primary-key }} | |
| - name: Download IMDb-Face.csv | |
| working-directory: apps/${{ env.APP_NAME }}/tests/res | |
| run: | | |
| wget https://cloud.nextcloud.com/public.php/dav/files/wfDk23DBsXYrd4S/IMDb-Face-csv.zip | |
| unzip IMDb-Face-csv.zip | |
| rm IMDb-Face-csv.zip | |
| - name: Analyse face assignments | |
| env: | |
| LABEL: ${{ github.job }} | |
| CSV_PATH: apps/${{ env.APP_NAME }}/tests/res/IMDb-Face.csv | |
| PHOTOS_TXT: photos.txt | |
| DETECTIONS_TXT: out.txt | |
| CLUSTERS_JSON: out.json | |
| METRICS_JSON_OUT: metrics.json | |
| # Baselined at 0.3813 on the first run where both jobs processed the | |
| # whole dataset (keep-every-10, 71 identities). The built-in detector | |
| # only finds about half of the annotated faces at all - detectionRecall | |
| # 0.4559, medianBestIou 0 - which caps everything downstream of it. | |
| MIN_COMBINED_SCORE: '0.35' | |
| run: | | |
| DETECTED_FACES_TOTAL=$(sqlite3 data/nextcloud.db "select count(*) from oc_recognize_face_detections where user_id = 'admin';") \ | |
| node apps/${{ env.APP_NAME }}/tests/analyse-face-assignments.js | |
| php-taskprocessing: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| # do not stop on another job's failure | |
| fail-fast: false | |
| matrix: | |
| php-versions: ['8.3'] | |
| databases: ['sqlite'] | |
| server-versions: ['master'] | |
| name: Test cluster-faces command via TaskProcessing on ${{ matrix.server-versions }} | |
| env: | |
| MYSQL_PORT: 4444 | |
| PGSQL_PORT: 4445 | |
| # recognize_backend ExApp (manual-install deploy daemon) | |
| PYTHONUNBUFFERED: 1 | |
| APP_HOST: 0.0.0.0 | |
| APP_ID: recognize_backend | |
| APP_PORT: 9031 | |
| APP_SECRET: 12345 | |
| NEXTCLOUD_URL: http://localhost:8080 | |
| services: | |
| mysql: | |
| image: mariadb:10.5 | |
| ports: | |
| - 4444:3306/tcp | |
| env: | |
| MYSQL_ROOT_PASSWORD: rootpassword | |
| options: --health-cmd="mysqladmin ping" --health-interval 5s --health-timeout 2s --health-retries 5 | |
| postgres: | |
| image: postgres | |
| ports: | |
| - 4445:5432/tcp | |
| env: | |
| POSTGRES_USER: root | |
| POSTGRES_PASSWORD: rootpassword | |
| POSTGRES_DB: nextcloud | |
| options: --health-cmd pg_isready --health-interval 5s --health-timeout 2s --health-retries 5 | |
| steps: | |
| - name: Checkout server | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| with: | |
| repository: nextcloud/server | |
| ref: ${{ matrix.server-versions }} | |
| - name: Checkout submodules | |
| shell: bash | |
| run: | | |
| auth_header="$(git config --local --get http.https://github.com/.extraheader)" | |
| git submodule sync --recursive | |
| git -c "http.extraheader=$auth_header" -c protocol.version=2 submodule update --init --force --recursive --depth=1 | |
| - name: install ssl-cert | |
| if: env.ACT # Skip this on normal GitHub Actions | |
| run: sudo apt update && sudo apt install -y ssl-cert | |
| - name: Set up php ${{ matrix.php-versions }} | |
| uses: shivammathur/setup-php@cf4cade2721270509d5b1c766ab3549210a39a2a # v2.33.0 | |
| with: | |
| php-version: ${{ matrix.php-versions }} | |
| tools: phpunit | |
| extensions: mbstring, iconv, fileinfo, intl, sqlite, pdo_mysql, pdo_sqlite, pgsql, pdo_pgsql, gd, zip | |
| - name: Checkout app | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| with: | |
| path: apps/${{ env.APP_NAME }} | |
| - name: Read package.json node and npm engines version | |
| uses: skjnldsv/read-package-engines-version-actions@06d6baf7d8f41934ab630e97d9e6c0bc9c9ac5e4 # v3 | |
| id: versions | |
| with: | |
| path: apps/${{ env.APP_NAME }} | |
| fallbackNode: '^12' | |
| fallbackNpm: '^6' | |
| - name: Set up node ${{ steps.versions.outputs.nodeVersion }} | |
| uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 | |
| with: | |
| node-version: ${{ steps.versions.outputs.nodeVersion }} | |
| - name: Set up npm ${{ steps.versions.outputs.npmVersion }} | |
| run: npm i -g npm@"${{ steps.versions.outputs.npmVersion }}" | |
| - name: install make wget unzip | |
| if: env.ACT # Skip this on normal GitHub Actions | |
| run: sudo apt update && sudo apt install -y make wget unzip | |
| - name: Install app | |
| working-directory: apps/${{ env.APP_NAME }} | |
| run: | | |
| composer install --no-dev | |
| make all | |
| make remove-binaries | |
| make remove-devdeps | |
| - name: Set up Nextcloud and install app | |
| if: ${{ matrix.databases != 'pgsql'}} | |
| run: | | |
| sleep 25 | |
| mkdir data | |
| ./occ maintenance:install --verbose --database=${{ matrix.databases }} --database-name=nextcloud --database-host=127.0.0.1 --database-port=$MYSQL_PORT --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass password | |
| ./occ app:enable -vvv -f ${{ env.APP_NAME }} | |
| # 4 workers by default | |
| composer run serve & | |
| - name: Set up Nextcloud and install app | |
| if: ${{ matrix.databases == 'pgsql'}} | |
| run: | | |
| sleep 25 | |
| mkdir data | |
| ./occ maintenance:install --verbose --database=${{ matrix.databases }} --database-name=nextcloud --database-host=127.0.0.1 --database-port=$PGSQL_PORT --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass password | |
| ./occ app:enable -vvv -f ${{ env.APP_NAME }} | |
| # 4 workers by default | |
| composer run serve & | |
| - name: Enable SQLite WAL mode | |
| if: ${{ matrix.databases == 'sqlite' }} | |
| run: | | |
| # WAL lets the ExApp's readers and cron's writers proceed concurrently | |
| # instead of serializing on SQLite's single-writer lock, which otherwise | |
| # stalls task scheduling for many minutes per cron run. The mode is | |
| # persisted in the database header, so it survives across connections. | |
| sqlite3 data/nextcloud.db "PRAGMA journal_mode=WAL;" | |
| - name: Free up disk space | |
| run: | | |
| # This job installs the Python ML stack (torch, onnxruntime, insightface | |
| # models) on top of the dataset, which the pure-PHP job does not. On a | |
| # tight runner that pushes the disk to ~97% during classification, which | |
| # aborts the crawl, stalls the ExApp task and locks the (WAL) SQLite DB. | |
| # Drop preinstalled toolchains we don't use to reclaim ~20-30 GB up front. | |
| echo "disk before cleanup: $(df -h / | awk 'NR==2 {print $4" free ("$5" used)"}')" | |
| sudo rm -rf /usr/local/lib/android /usr/share/dotnet /opt/ghc /usr/local/.ghcup /opt/hostedtoolcache/CodeQL /usr/local/share/powershell /usr/local/share/chromium || true | |
| sudo docker image prune --all --force || true | |
| echo "disk after cleanup: $(df -h / | awk 'NR==2 {print $4" free ("$5" used)"}')" | |
| - name: Enable app_api | |
| run: ./occ app:enable -vvv -f app_api | |
| - name: Checkout app | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| with: | |
| repository: nextcloud/viewer | |
| path: apps/viewer | |
| - name: Install viewer | |
| run: | | |
| ./occ app:enable -vvv viewer | |
| - name: Remove unnecessary models to make space | |
| run: | | |
| rm -rf apps/recognize/models | |
| - uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 | |
| id: photos-cache | |
| with: | |
| path: data/admin/files/ | |
| key: https://cloud.nextcloud.com/public.php/dav/files/wfDk23DBsXYrd4S/IMDb-Face.zip-every${{ env.DATASET_KEEP_EVERY }} | |
| - name: Upload photos | |
| if: steps.photos-cache.outputs.cache-hit != 'true' | |
| run: | | |
| mkdir -p data/admin/files/ | |
| cd data/admin/files | |
| wget https://cloud.nextcloud.com/public.php/dav/files/wfDk23DBsXYrd4S/IMDb-Face.zip | |
| unzip IMDb-Face.zip | |
| rm IMDb-Face.zip | |
| # Keep only every DATASET_KEEP_EVERY'th identity directory so the | |
| # classification run stays within the job time limit. Deterministic | |
| # (sorted glob) so both jobs cache an identical set under the same key. | |
| cd IMDb-Face | |
| i=0 | |
| for d in */; do | |
| if [ $((i % ${{ env.DATASET_KEEP_EVERY }})) -ne 0 ]; then rm -rf "$d"; fi | |
| i=$((i + 1)) | |
| done | |
| echo "Kept $(ls -d */ | wc -l) identity directories" | |
| - uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 | |
| with: | |
| path: data/admin/files/ | |
| key: https://cloud.nextcloud.com/public.php/dav/files/wfDk23DBsXYrd4S/IMDb-Face.zip-every${{ env.DATASET_KEEP_EVERY }} | |
| - name: Set config | |
| run: | | |
| ./occ config:app:set --value true recognize faces.enabled | |
| # Don't force API key usage to allow tests to run | |
| ./occ config:app:set --value false recognize require_api_key | |
| # Hand files off to the recognize_backend ExApp via TaskProcessing | |
| # instead of running TensorFlow locally. | |
| ./occ config:app:set --value true recognize taskprocessing.enabled | |
| # Resolve the recognize_backend HEAD commit so it can be part of the | |
| # detection cache key: the ExApp does the actual face detection, so a | |
| # change there must invalidate the cached detections. The checkout below | |
| # happens after the cache restore, so resolve the SHA up front here. | |
| - name: Resolve recognize_backend commit | |
| id: backend-sha | |
| run: echo "sha=$(git ls-remote https://github.com/nextcloud/recognize_backend HEAD | cut -f1)" >> "$GITHUB_OUTPUT" | |
| # The cache stores a VACUUM INTO snapshot (nextcloud-cache.db), not the | |
| # live data/nextcloud.db: the running dev server / ExApp mutate the live DB | |
| # while actions/cache tars it, which failed the save with "tar exit code 1". | |
| - uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 | |
| id: db-cache | |
| with: | |
| path: nextcloud-cache.db | |
| key: ${{ runner.os }}-${{ matrix.server-versions }}-taskprocessing-${{ steps.backend-sha.outputs.sha }}-${{ hashFiles('data/admin/files/**', 'apps/recognize/lib/Classifiers/AbstractTaskProcessingClassifier.php', 'apps/recognize/lib/Classifiers/TaskProcessing/**', 'apps/recognize/lib/TaskProcessing/**', 'apps/recognize/lib/Db/FaceDetectionMapper.php') }} | |
| # On a cache hit, replace the freshly-installed DB with the cached detection | |
| # snapshot so clustering and the detection summary run against it. Drop any | |
| # stale WAL sidecars from the fresh install so SQLite reads the snapshot. | |
| - name: Restore detection DB from snapshot | |
| if: steps.db-cache.outputs.cache-hit == 'true' | |
| run: | | |
| rm -f data/nextcloud.db-wal data/nextcloud.db-shm | |
| cp nextcloud-cache.db data/nextcloud.db | |
| # The remaining classification steps only run on a cache miss. When the | |
| # detection database is restored from cache we can skip deploying the | |
| # ExApp entirely and go straight to clustering (pure PHP). | |
| - name: Checkout recognize_backend ExApp | |
| if: steps.db-cache.outputs.cache-hit != 'true' | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| with: | |
| repository: nextcloud/recognize_backend | |
| path: recognize_backend | |
| - name: Set up python 3.11 | |
| if: steps.db-cache.outputs.cache-hit != 'true' | |
| uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 | |
| with: | |
| python-version: '3.11' | |
| cache: 'pip' | |
| cache-dependency-path: recognize_backend/requirements.txt | |
| - name: Read recognize_backend version | |
| if: steps.db-cache.outputs.cache-hit != 'true' | |
| id: backendinfo | |
| uses: skjnldsv/xpath-action@7e6a7c379d0e9abc8acaef43df403ab4fc4f770c # master | |
| with: | |
| filename: recognize_backend/appinfo/info.xml | |
| expression: "/info/version/text()" | |
| - name: Install ExApp system dependencies | |
| if: steps.db-cache.outputs.cache-hit != 'true' | |
| run: sudo apt update && sudo apt install -y ffmpeg libsndfile1 libgl1 libglib2.0-0 | |
| - name: Install ExApp requirements | |
| if: steps.db-cache.outputs.cache-hit != 'true' | |
| working-directory: recognize_backend | |
| run: | | |
| # onnxruntime-gpu pulls in several GB of NVIDIA CUDA wheels that fill | |
| # up the runner disk; swap in the CPU build like the full-run test does. | |
| sed -i 's/^onnxruntime-gpu.*/onnxruntime/' requirements.txt | |
| python3 -m pip install --upgrade pip | |
| # The default PyPI torch wheels bundle ~5GB of NVIDIA CUDA libraries and | |
| # blow up the runner disk. Install the CPU-only build first so the torch | |
| # constraints in requirements.txt are already satisfied. | |
| python3 -m pip install --index-url https://download.pytorch.org/whl/cpu torch torchvision torchaudio | |
| python3 -m pip install -r requirements.txt | |
| - name: Run ExApp | |
| if: steps.db-cache.outputs.cache-hit != 'true' | |
| working-directory: recognize_backend/lib | |
| env: | |
| APP_VERSION: ${{ steps.backendinfo.outputs.result }} | |
| run: | | |
| python3 main.py > "$GITHUB_WORKSPACE/backend_logs" 2>&1 & | |
| - name: Register ExApp with app_api | |
| if: steps.db-cache.outputs.cache-hit != 'true' | |
| run: | | |
| ./occ app_api:daemon:register --net host manual_install "Manual Install" manual-install http localhost http://localhost:8080 | |
| ./occ app_api:app:register recognize_backend manual_install --json-info "{\"appid\":\"recognize_backend\",\"name\":\"Recognize Backend\",\"daemon_config_name\":\"manual_install\",\"version\":\"${{ steps.backendinfo.outputs.result }}\",\"secret\":\"12345\",\"port\":9031,\"scopes\":[\"TASK_PROCESSING\",\"FILES\"]}" --force-scopes --wait-finish | |
| - name: install sqlite3 | |
| if: steps.db-cache.outputs.cache-hit != 'true' && env.ACT # Skip this on normal GitHub Actions | |
| run: sudo apt update && sudo apt install -y sqlite3 | |
| - name: Run classification via TaskProcessing | |
| id: classify | |
| if: steps.db-cache.outputs.cache-hit != 'true' | |
| run: | | |
| # Probe available disk so we can tell whether runs die on ENOSPC. | |
| disk_free() { df -h / | awk 'NR==2 {print $4" free ("$5" used)"}'; } | |
| # Query the DB with a busy timeout so a transient WAL lock (the ExApp and | |
| # cron write concurrently) makes the poll wait instead of failing with | |
| # "database is locked" (exit 5), which would otherwise kill the step. | |
| sq() { sqlite3 -cmd ".timeout 60000" data/nextcloud.db "$1"; } | |
| echo "disk before classification: $(disk_free)" | |
| ./occ upgrade # in case server master has new migrations in the meantime | |
| ./occ files:scan admin | |
| # Kick off a full classification run: SchedulerJob -> StorageCrawlJob | |
| # fills the faces queue and ClassifyFacesJob hands each batch to the | |
| # recognize_backend ExApp as a TaskProcessing task. Results are written | |
| # back asynchronously by the TaskResultListener when the ExApp reports. | |
| ./occ recognize:recrawl | |
| # Drive the background jobs by running cron in a loop until the faces | |
| # queue is drained and no crawl/scheduler jobs remain. | |
| for i in $(seq 1 180); do | |
| # Clustering is done explicitly in the "Run clustering" step below, so | |
| # drop these jobs before each cron run to keep classification cron fast. | |
| sq "delete from oc_jobs where class like '%ClusterFacesJob';" || true | |
| php cron.php || true | |
| QUEUE=$(sq "select count(*) from oc_recognize_queue_faces;") | |
| CRAWL=$(sq "select count(*) from oc_jobs where class like '%StorageCrawlJob' or class like '%SchedulerJob';") | |
| echo "round $i: faces queue=$QUEUE, pending crawl/scheduler jobs=$CRAWL, disk=$(disk_free)" | |
| if [ "$QUEUE" -eq 0 ] && [ "$CRAWL" -eq 0 ] && [ "$i" -gt 3 ]; then break; fi | |
| sleep 10 | |
| done | |
| # Wait for the ExApp to finish processing all scheduled TaskProcessing | |
| # tasks (status 0=unknown, 1=scheduled, 2=running are still pending). | |
| for i in $(seq 1 240); do | |
| PENDING=$(sq "select count(*) from oc_taskprocessing_tasks where app_id = 'recognize' and status in (0, 1, 2);") | |
| echo "wait $i: pending recognize taskprocessing tasks=$PENDING, disk=$(disk_free)" | |
| if [ "$PENDING" -eq 0 ]; then break; fi | |
| sleep 30 | |
| done | |
| echo "disk after classification: $(disk_free)" | |
| # Whether the run actually finished. The faces queue emptying is not | |
| # enough: files leave the queue when their TaskProcessing task is | |
| # *scheduled*, not when its results are written back, so a drained queue | |
| # with tasks still pending means those photos have no detections at all. | |
| # Detection counts cannot be used here - a photo with no face in it | |
| # legitimately produces no rows - so completeness is queue + task state. | |
| QUEUE=$(sq "select count(*) from oc_recognize_queue_faces;") | |
| PENDING=$(sq "select count(*) from oc_taskprocessing_tasks where app_id = 'recognize' and status in (0, 1, 2);") | |
| FAILED=$(sq "select count(*) from oc_taskprocessing_tasks where app_id = 'recognize' and status = 4;") | |
| TOTAL=$(sq "select count(*) from oc_taskprocessing_tasks where app_id = 'recognize';") | |
| echo "final: faces queue=$QUEUE, pending tasks=$PENDING, failed tasks=$FAILED, total tasks=$TOTAL" | |
| echo "photos on disk: $(find data/admin/files/IMDb-Face -type f | wc -l)" | |
| echo "photos with detections: $(sq "select count(distinct file_id) from oc_recognize_face_detections where user_id = 'admin';")" | |
| if [ "$QUEUE" -eq 0 ] && [ "$PENDING" -eq 0 ] && [ "$FAILED" -eq 0 ]; then | |
| echo "complete=true" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "complete=false" >> "$GITHUB_OUTPUT" | |
| echo "::error title=Incomplete classification run::faces queue=$QUEUE, pending tasks=$PENDING, failed tasks=$FAILED of $TOTAL. Each pending task covers up to 500 photos, whose detections were never written. The detection DB will NOT be cached; the metrics below score only the photos that made it through." | |
| fi | |
| # VACUUM INTO writes a consistent point-in-time copy (WAL folded in) that | |
| # actions/cache can tar without the running dev server / ExApp mutating it | |
| # mid-read (which previously failed the save with "tar exit code 1"). | |
| # | |
| # Only ever snapshot a *complete* run. An incomplete one used to be cached | |
| # just the same, which pinned a partial detection DB under this key and made | |
| # every later run with the same backend SHA and file set replay the same | |
| # truncated numbers without touching the ExApp at all. A cache hit therefore | |
| # now implies a run that drained the queue with no pending or failed tasks. | |
| - name: Snapshot detection DB for caching | |
| if: steps.db-cache.outputs.cache-hit != 'true' && steps.classify.outputs.complete == 'true' | |
| run: | | |
| sqlite3 -cmd ".timeout 60000" data/nextcloud.db "VACUUM INTO 'nextcloud-cache.db';" | |
| - uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 | |
| if: steps.db-cache.outputs.cache-hit != 'true' && steps.classify.outputs.complete == 'true' | |
| with: | |
| path: nextcloud-cache.db | |
| key: ${{ steps.db-cache.outputs.cache-primary-key }} | |
| # The photos on disk are the run-independent denominator for every rate the | |
| # analysis step reports, so take the inventory before they are truncated. | |
| - name: Create photo inventory | |
| run: | | |
| find data/admin/files/IMDb-Face -type f -printf '%P\n' | sort > photos.txt | |
| wc -l photos.txt | |
| - name: Reduce space | |
| run: | | |
| for dirname in data/admin/files/IMDb-Face/*; do truncate -s 0 "${dirname}"/*; done | |
| - name: install sqlite3 | |
| if: env.ACT # Skip this on normal GitHub Actions | |
| run: sudo apt update && sudo apt install -y sqlite3 | |
| - name: Create detection summary | |
| run: | | |
| sqlite3 data/nextcloud.db "select x, y, width, height, path from oc_recognize_face_detections d LEFT JOIN oc_filecache c ON c.fileid = d.file_id where user_id = 'admin' ORDER BY path;" > out.txt | |
| - uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 | |
| id: clustering-cache | |
| with: | |
| path: out.json | |
| # lib/Classifiers/TaskProcessing/** has to be part of this key: in | |
| # taskprocessing mode FaceClusterAnalyzer reads MIN_CLUSTER_SEPARATION, | |
| # MAX_CLUSTER_EDGE_LENGTH and friends from ImageFaceRecognitionClassifier, | |
| # so without it, tuning those constants silently reuses a stale out.json. | |
| key: ${{ runner.os }}-taskprocessing-${{ hashFiles('out.txt', 'apps/recognize/lib/Clustering/**', 'apps/recognize/lib/Dav/**', 'apps/recognize/lib/Classifiers/TaskProcessing/**', 'apps/recognize/lib/Service/FaceClusterAnalyzer.php', 'apps/recognize/lib/Service/SettingsService.php', 'apps/recognize/lib/Command/ClusterFaces.php') }} | |
| - name: Run clustering | |
| if: steps.clustering-cache.outputs.cache-hit != 'true' | |
| run: | | |
| ./occ upgrade # in case server master has new migrations in the meantime | |
| ./occ recognize:cluster-faces -b 10000 | |
| ./occ recognize:cluster-faces -b 10000 | |
| ./occ recognize:cluster-faces -b 10000 | |
| ./occ recognize:cluster-faces -b 10000 | |
| ./occ recognize:cluster-faces -b 10000 | |
| ./occ recognize:cluster-faces -b 10000 | |
| - name: install python3 python3-pip jq curl | |
| if: steps.clustering-cache.outputs.cache-hit != 'true' && env.ACT # Skip this on normal GitHub Actions | |
| run: sudo apt update && sudo apt install -y python3 python3-pip jq curl | |
| - name: Install xq | |
| if: steps.clustering-cache.outputs.cache-hit != 'true' | |
| run: | | |
| pip install yq --break-system-packages | |
| - name: Download face assignments | |
| if: steps.clustering-cache.outputs.cache-hit != 'true' | |
| run: | | |
| curl -u 'admin:password' --request PROPFIND 'http://localhost:8080/remote.php/dav/recognize/admin/faces/' --header 'Depth: 2' --data '<?xml version="1.0"?> | |
| <d:propfind xmlns:d="DAV:" | |
| xmlns:oc="http://owncloud.org/ns" | |
| xmlns:nc="http://nextcloud.org/ns" | |
| xmlns:ocs="http://open-collaboration-services.org/ns"> | |
| <d:prop> | |
| <d:getcontentlength /> | |
| <d:getcontenttype /> | |
| <d:getetag /> | |
| <d:getlastmodified /> | |
| <d:resourcetype /> | |
| <nc:face-detections /> | |
| <nc:file-metadata-size /> | |
| <nc:has-preview /> | |
| <nc:realpath /> | |
| <oc:favorite /> | |
| <oc:fileid /> | |
| <oc:permissions /> | |
| <nc:nbItems /> | |
| </d:prop> | |
| </d:propfind>' > out.xml | |
| cat out.xml | |
| - name: Parse face assignments | |
| if: steps.clustering-cache.outputs.cache-hit != 'true' | |
| run: | | |
| PATH=$PATH:/home/runner/.local/bin | |
| cat out.xml | xq '.["d:multistatus"]["d:response"] | map(select(.["d:href"] | test("faces/.+?/.+?"))) | map({"href": .["d:href"], "realpath": .["d:propstat"][0]["d:prop"]["nc:realpath"], "face-detections": .["d:propstat"][0]["d:prop"]["nc:face-detections"] | fromjson | map({userId, x, y, height, width, clusterId}) })' > out.json | |
| cat out.json | |
| - uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 | |
| with: | |
| path: out.json | |
| key: ${{ steps.clustering-cache.outputs.cache-primary-key }} | |
| - name: Download IMDb-Face.csv | |
| working-directory: apps/${{ env.APP_NAME }}/tests/res | |
| run: | | |
| wget https://cloud.nextcloud.com/public.php/dav/files/wfDk23DBsXYrd4S/IMDb-Face-csv.zip | |
| unzip IMDb-Face-csv.zip | |
| rm IMDb-Face-csv.zip | |
| - name: Analyse face assignments | |
| env: | |
| LABEL: ${{ github.job }} | |
| CSV_PATH: apps/${{ env.APP_NAME }}/tests/res/IMDb-Face.csv | |
| PHOTOS_TXT: photos.txt | |
| DETECTIONS_TXT: out.txt | |
| CLUSTERS_JSON: out.json | |
| METRICS_JSON_OUT: metrics.json | |
| # Baselined at 0.8043 on the first run where both jobs processed the | |
| # whole dataset (keep-every-10, 71 identities). Deliberately much higher | |
| # than the built-in detector's floor: a shared threshold would have to | |
| # sit below 0.38 and would not notice this backend halving its score. | |
| MIN_COMBINED_SCORE: '0.75' | |
| run: | | |
| DETECTED_FACES_TOTAL=$(sqlite3 data/nextcloud.db "select count(*) from oc_recognize_face_detections where user_id = 'admin';") \ | |
| node apps/${{ env.APP_NAME }}/tests/analyse-face-assignments.js | |
| # Runs even when the analysis above failed its gates, so an incomplete run | |
| # is named as such instead of only showing up as a low score. Skipped on a | |
| # cache hit, where `complete` is unset - a cached DB is only saved for a run | |
| # that finished. | |
| - name: Fail on incomplete classification run | |
| if: always() && steps.classify.outputs.complete == 'false' | |
| run: | | |
| echo "The classification run did not finish: the ExApp had tasks still" | |
| echo "pending or failed when the wait loop expired, so an unknown share" | |
| echo "of the dataset has no detections and the metrics above are scored" | |
| echo "against whatever made it through. Nothing was cached." | |
| echo | |
| echo "Either raise the wait budget in 'Run classification via" | |
| echo "TaskProcessing', raise DATASET_KEEP_EVERY to shrink the dataset," | |
| echo "or give the ExApp more workers." | |
| exit 1 | |
| - name: Show ExApp logs | |
| if: always() | |
| run: | | |
| echo '---------------- nextcloud.log ----------------' | |
| tail -n 100 data/nextcloud.log || echo "No nextcloud.log" | |
| echo '---------------- recognize_backend logs ----------------' | |
| [ -f backend_logs ] && cat backend_logs || echo "No backend logs" |