Skip to content
Classify command test

Added a Setting to set the Model Download Folder #977

Sign in to view logs

Added a Setting to set the Model Download Folder

Added a Setting to set the Model Download Folder #977

Workflow file for this run

.github/workflows/full-run-test.yml at 796d972
name: Classify command test
on:
pull_request:
paths:
- 'lib/**'
- 'src/**'
- 'package.json'
- 'composer.lock'
push:
branches:
- main
- stable*
paths:
- 'lib/**'
- 'src/**'
- 'package.json'
- 'composer.lock'
env:
APP_NAME: recognize
jobs:
php:
runs-on: ubuntu-latest
strategy:
# do not stop on another job's failure
fail-fast: false
matrix:
php-versions: ['8.3']
databases: ['sqlite']
server-versions: ['master']
pure-js-mode: ['false']
imagenet-enabled: ['true']
faces-enabled: ['true']
landmarks-enabled: ['true']
musicnn-enabled: ['true']
movinet-enabled: ['true']
include:
# test pure-js once
- server-versions: master
databases: sqlite
php-versions: 8.3
pure-js-mode: true
imagenet-enabled: true
faces-enabled: true
landmarks-enabled: true
musicnn-enabled: true
movinet-enabled: false
name: Test classify command on ${{ matrix.databases }}-${{ matrix.server-versions }} wasm:${{ matrix.pure-js-mode }},imagenet:${{ matrix.imagenet-enabled }},faces:${{ matrix.faces-enabled }},landmarks:${{ matrix.landmarks-enabled }},movinet:${{ matrix.movinet-enabled }},musicnn:${{ matrix.musicnn-enabled }}
env:
MYSQL_PORT: 4444
PGSQL_PORT: 4445
timeout-minutes: 36 # max + 3*std over the last 212 successful runs
services:
mysql:
image: mariadb:10.5
ports:
- 4444:3306/tcp
env:
MYSQL_ROOT_PASSWORD: rootpassword
options: --health-cmd="mysqladmin ping" --health-interval 5s --health-timeout 2s --health-retries 5
postgres:
image: postgres
ports:
- 4445:5432/tcp
env:
POSTGRES_USER: root
POSTGRES_PASSWORD: rootpassword
POSTGRES_DB: nextcloud
options: --health-cmd pg_isready --health-interval 5s --health-timeout 2s --health-retries 5
steps:
- name: Checkout server
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
repository: nextcloud/server
ref: ${{ matrix.server-versions }}
- name: Checkout submodules
shell: bash
run: |
auth_header="$(git config --local --get http.https://github.com/.extraheader)"
git submodule sync --recursive
git -c "http.extraheader=$auth_header" -c protocol.version=2 submodule update --init --force --recursive --depth=1
- name: Set up php ${{ matrix.php-versions }}
uses: shivammathur/setup-php@cf4cade2721270509d5b1c766ab3549210a39a2a # v2.33.0
with:
php-version: ${{ matrix.php-versions }}
tools: phpunit
extensions: mbstring, iconv, fileinfo, intl, sqlite, pdo_mysql, pdo_sqlite, pgsql, pdo_pgsql, gd, zip
- name: Checkout app
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
path: apps/${{ env.APP_NAME }}
- name: Read package.json node and npm engines version
uses: skjnldsv/read-package-engines-version-actions@06d6baf7d8f41934ab630e97d9e6c0bc9c9ac5e4 # v3
id: versions
with:
path: apps/${{ env.APP_NAME }}
fallbackNode: '^12'
fallbackNpm: '^6'
- name: Set up node ${{ steps.versions.outputs.nodeVersion }}
uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
with:
node-version: ${{ steps.versions.outputs.nodeVersion }}
- name: Set up npm ${{ steps.versions.outputs.npmVersion }}
run: npm i -g npm@"${{ steps.versions.outputs.npmVersion }}"
- name: Install app
working-directory: apps/${{ env.APP_NAME }}
run: |
composer install --no-dev
make all
make remove-binaries
rm -rf models # Make it download from github
wget https://github.com/nextcloud/recognize/releases/download/v3.4.0/test-files.zip
unzip test-files.zip -d tests/res/
- name: Set up Nextcloud and install app
if: ${{ matrix.databases != 'pgsql'}}
run: |
sleep 25
mkdir data
./occ maintenance:install --verbose --database=${{ matrix.databases }} --database-name=nextcloud --database-host=127.0.0.1 --database-port=$MYSQL_PORT --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass password
./occ app:enable -vvv -f ${{ env.APP_NAME }}
php -S localhost:8080 &
- name: Set up Nextcloud and install app
if: ${{ matrix.databases == 'pgsql'}}
run: |
sleep 25
mkdir data
./occ maintenance:install --verbose --database=${{ matrix.databases }} --database-name=nextcloud --database-host=127.0.0.1 --database-port=$PGSQL_PORT --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass password
./occ app:enable -vvv -f ${{ env.APP_NAME }}
php -S localhost:8080 &
- name: Install
run: |
./occ app:enable -vvv ${{ env.APP_NAME }}
- name: Upload photos
run: |
find apps/${{ env.APP_NAME }}/tests/res/ -type f -exec curl -u 'admin:password' -T "{}" 'http://localhost:8080/remote.php/webdav/' \;
- name: Set pure-js mode
run: |
./occ config:app:set --lazy --value ${{ matrix.pure-js-mode }} recognize tensorflow.purejs
- name: Set imagenet.enabled
run: |
./occ config:app:set --value ${{ matrix.imagenet-enabled }} recognize imagenet.enabled
- name: Set faces.enabled
run: |
./occ config:app:set --value ${{ matrix.faces-enabled }} recognize faces.enabled
- name: Set landmarks.enabled
run: |
./occ config:app:set --value ${{ matrix.landmarks-enabled }} recognize landmarks.enabled
- name: Set musicnn.enabled
run: |
./occ config:app:set --value ${{ matrix.musicnn-enabled }} recognize musicnn.enabled
- name: Set movinet.enabled
run: |
./occ config:app:set --value ${{ matrix.movinet-enabled }} recognize movinet.enabled
- name: Run classifier
env:
GITHUB_REF: ${{ github.ref }}
run: |
./occ recognize:classify
./occ recognize:cluster-faces
- name: Install cadaver
if: ${{ matrix.faces-enabled }}
run: |
sudo apt -y install cadaver
- name: Test webdav access
if: ${{ matrix.faces-enabled }}
run: |
cat > ~/.netrc <<EOF
machine localhost
login admin
password password
EOF
cadaver http://localhost:8080/remote.php/dav/recognize/admin/ <<EOF
mv unassigned-faces/* faces/0/
mkdir faces/toot
mv faces/0/* faces/toot/
mv faces/toot/ faces/blah/
EOF
- name: Print logs
if: always()
run: |
cat data/nextcloud.log
taskprocessing:
runs-on: ubuntu-latest
strategy:
# do not stop on another job's failure
fail-fast: false
matrix:
php-versions: ['8.3']
databases: ['sqlite']
# recognize_backend requires Nextcloud >= 32, so only test against master here
server-versions: ['master']
imagenet-enabled: ['true']
faces-enabled: ['true']
musicnn-enabled: ['true']
movinet-enabled: ['true']
name: Test classify in taskprocessing mode on ${{ matrix.databases }}-${{ matrix.server-versions }} imagenet:${{ matrix.imagenet-enabled }},faces:${{ matrix.faces-enabled }},movinet:${{ matrix.movinet-enabled }},musicnn:${{ matrix.musicnn-enabled }}
env:
MYSQL_PORT: 4444
PGSQL_PORT: 4445
# recognize_backend ExApp (deployed via AppAPI's manual_install deploy daemon)
PYTHONUNBUFFERED: 1
APP_HOST: 0.0.0.0
APP_ID: recognize_backend
APP_PORT: 9031
APP_SECRET: 12345
APP_VERSION: 1.0.0
NEXTCLOUD_URL: http://localhost:8080
# No NVIDIA driver on the runner; the backend falls back to CPU automatically
COMPUTE_DEVICE: CPU
timeout-minutes: 90 # model download + CPU inference is slow on the first (uncached) run
services:
mysql:
image: mariadb:10.5
ports:
- 4444:3306/tcp
env:
MYSQL_ROOT_PASSWORD: rootpassword
options: --health-cmd="mysqladmin ping" --health-interval 5s --health-timeout 2s --health-retries 5
postgres:
image: postgres
ports:
- 4445:5432/tcp
env:
POSTGRES_USER: root
POSTGRES_PASSWORD: rootpassword
POSTGRES_DB: nextcloud
options: --health-cmd pg_isready --health-interval 5s --health-timeout 2s --health-retries 5
steps:
- name: Checkout server
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
repository: nextcloud/server
ref: ${{ matrix.server-versions }}
- name: Checkout submodules
shell: bash
run: |
auth_header="$(git config --local --get http.https://github.com/.extraheader)"
git submodule sync --recursive
git -c "http.extraheader=$auth_header" -c protocol.version=2 submodule update --init --force --recursive --depth=1
- name: Set up php ${{ matrix.php-versions }}
uses: shivammathur/setup-php@cf4cade2721270509d5b1c766ab3549210a39a2a # v2.33.0
with:
php-version: ${{ matrix.php-versions }}
tools: phpunit
extensions: mbstring, iconv, fileinfo, intl, sqlite, pdo_mysql, pdo_sqlite, pgsql, pdo_pgsql, gd, zip
- name: Checkout app
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
path: apps/${{ env.APP_NAME }}
- name: Read package.json node and npm engines version
uses: skjnldsv/read-package-engines-version-actions@06d6baf7d8f41934ab630e97d9e6c0bc9c9ac5e4 # v3
id: versions
with:
path: apps/${{ env.APP_NAME }}
fallbackNode: '^12'
fallbackNpm: '^6'
- name: Set up node ${{ steps.versions.outputs.nodeVersion }}
uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
with:
node-version: ${{ steps.versions.outputs.nodeVersion }}
- name: Set up npm ${{ steps.versions.outputs.npmVersion }}
run: npm i -g npm@"${{ steps.versions.outputs.npmVersion }}"
- name: Install app
working-directory: apps/${{ env.APP_NAME }}
run: |
composer install --no-dev
make all
make remove-binaries
rm -rf models # Make it download from github
wget https://github.com/nextcloud/recognize/releases/download/v3.4.0/test-files.zip
unzip test-files.zip -d tests/res/
- name: Set up Nextcloud and install app
run: |
sleep 25
mkdir data
./occ maintenance:install --verbose --database=${{ matrix.databases }} --database-name=nextcloud --database-host=127.0.0.1 --database-port=$MYSQL_PORT --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass password
./occ app:enable -vvv -f ${{ env.APP_NAME }}
# 4 workers by default
composer run serve &
- name: Enable SQLite WAL mode
if: ${{ matrix.databases == 'sqlite' }}
run: |
# WAL lets the ExApp's readers and cron's writers proceed concurrently
# instead of serializing on SQLite's single-writer lock, which otherwise
# stalls task scheduling for many minutes per cron run. The mode is
# persisted in the database header, so it survives across connections.
sqlite3 data/nextcloud.db "PRAGMA journal_mode=WAL;"
- name: Checkout app_api
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
path: apps/app_api
repository: nextcloud/app_api
ref: ${{ matrix.server-versions == 'master' && 'main' || matrix.server-versions }}
- name: Enable app_api
run: ./occ app:enable -vvv -f app_api
- name: Checkout recognize_backend
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
path: recognize_backend
repository: nextcloud/recognize_backend
ref: main
- name: Set up python 3.11
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
with:
python-version: '3.11'
cache: 'pip'
cache-dependency-path: recognize_backend/requirements.txt
- name: Install recognize_backend dependencies
working-directory: recognize_backend
run: |
# onnxruntime-gpu needs CUDA libs at import time; swap in the CPU build for the runner
sed -i 's/^onnxruntime-gpu.*/onnxruntime/' requirements.txt
# The default PyPI torch wheels bundle ~5GB of NVIDIA CUDA libraries and
# blow up the runner disk. Install the CPU-only build first so the torch
# constraints in requirements.txt are already satisfied.
pip install --index-url https://download.pytorch.org/whl/cpu torch torchvision torchaudio
pip install -r requirements.txt
- name: Cache recognize_backend models
uses: actions/cache/restore@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
id: models-cache-restore
with:
path: recognize_backend-persistent_storage/
key: ${{ runner.os }}-recognize-backend-models-v1
- name: Run recognize_backend
working-directory: recognize_backend/lib
env:
APP_PERSISTENT_STORAGE: ${{ github.workspace }}/recognize_backend-persistent_storage
HF_HOME: ${{ github.workspace }}/recognize_backend-persistent_storage/huggingface
INSIGHTFACE_HOME: ${{ github.workspace }}/recognize_backend-persistent_storage/insightface
run: |
mkdir -p "$APP_PERSISTENT_STORAGE"
python3 main.py > "$GITHUB_WORKSPACE/backend_logs" 2>&1 &
- name: Register recognize_backend with AppAPI
run: |
./occ app_api:daemon:register --net host manual_install "Manual Install" manual-install http localhost http://localhost:8080
./occ app_api:app:register ${{ env.APP_ID }} manual_install --json-info \
"{\"appid\":\"${{ env.APP_ID }}\",\"name\":\"Recognize Backend\",\"daemon_config_name\":\"manual_install\",\"version\":\"${{ env.APP_VERSION }}\",\"secret\":\"${{ env.APP_SECRET }}\",\"port\":${{ env.APP_PORT }},\"scopes\":[\"TASK_PROCESSING\",\"FILES\"]}" \
--force-scopes --wait-finish
- name: Upload photos
run: |
find apps/${{ env.APP_NAME }}/tests/res/ -type f -exec curl -u 'admin:password' -T "{}" 'http://localhost:8080/remote.php/webdav/' \;
- name: Enable taskprocessing mode
run: |
./occ config:app:set --value true recognize taskprocessing.enabled
./occ config:app:set --value ${{ matrix.imagenet-enabled }} recognize imagenet.enabled
./occ config:app:set --value ${{ matrix.faces-enabled }} recognize faces.enabled
./occ config:app:set --value ${{ matrix.musicnn-enabled }} recognize musicnn.enabled
./occ config:app:set --value ${{ matrix.movinet-enabled }} recognize movinet.enabled
- name: Run classification via TaskProcessing
id: classify
run: |
# Probe available disk so we can tell whether runs die on ENOSPC.
disk_free() { df -h / | awk 'NR==2 {print $4" free ("$5" used)"}'; }
# Query the DB with a busy timeout so a transient WAL lock (the ExApp and
# cron write concurrently) makes the poll wait instead of failing with
# "database is locked" (exit 5), which would otherwise kill the step.
sq() { sqlite3 -cmd ".timeout 60000" data/nextcloud.db "$1"; }
echo "disk before classification: $(disk_free)"
./occ upgrade # in case server master has new migrations in the meantime
./occ files:scan admin
# Kick off a full classification run: SchedulerJob -> StorageCrawlJob
# fills the faces queue and ClassifyFacesJob hands each batch to the
# recognize_backend ExApp as a TaskProcessing task. Results are written
# back asynchronously by the TaskResultListener when the ExApp reports.
./occ recognize:recrawl
# Drive the background jobs by running cron in a loop until the faces
# queue is drained and no crawl/scheduler jobs remain.
for i in $(seq 1 180); do
# Clustering is done explicitly in the "Run clustering" step below, so
# drop these jobs before each cron run to keep classification cron fast.
sq "delete from oc_jobs where class like '%ClusterFacesJob';" || true
php cron.php || true
QUEUE=$(sq "select count(*) from oc_recognize_queue_faces;")
CRAWL=$(sq "select count(*) from oc_jobs where class like '%StorageCrawlJob' or class like '%SchedulerJob';")
echo "round $i: faces queue=$QUEUE, pending crawl/scheduler jobs=$CRAWL, disk=$(disk_free)"
if [ "$QUEUE" -eq 0 ] && [ "$CRAWL" -eq 0 ] && [ "$i" -gt 3 ]; then break; fi
sleep 10
done
# Wait for the ExApp to finish processing all scheduled TaskProcessing
# tasks (status 0=unknown, 1=scheduled, 2=running are still pending).
for i in $(seq 1 240); do
PENDING=$(sq "select count(*) from oc_taskprocessing_tasks where app_id = 'recognize' and status in (0, 1, 2);")
echo "wait $i: pending recognize taskprocessing tasks=$PENDING, disk=$(disk_free)"
if [ "$PENDING" -eq 0 ]; then break; fi
sleep 30
done
echo "disk after classification: $(disk_free)"
# Whether the run actually finished. The faces queue emptying is not
# enough: files leave the queue when their TaskProcessing task is
# *scheduled*, not when its results are written back, so a drained queue
# with tasks still pending means those photos have no detections at all.
# Detection counts cannot be used here - a photo with no face in it
# legitimately produces no rows - so completeness is queue + task state.
QUEUE=$(sq "select count(*) from oc_recognize_queue_faces;")
PENDING=$(sq "select count(*) from oc_taskprocessing_tasks where app_id = 'recognize' and status in (0, 1, 2);")
FAILED=$(sq "select count(*) from oc_taskprocessing_tasks where app_id = 'recognize' and status = 4;")
TOTAL=$(sq "select count(*) from oc_taskprocessing_tasks where app_id = 'recognize';")
echo "final: faces queue=$QUEUE, pending tasks=$PENDING, failed tasks=$FAILED, total tasks=$TOTAL"
echo "photos on disk: $(find data/admin/files -type f | wc -l)"
DETECTIONS=$(sq "select count(*) from oc_recognize_face_detections;")
echo "photos with detections: $(sq "select count(distinct file_id) from oc_recognize_face_detections where user_id = 'admin';")"
echo "face detections: $DETECTIONS"
if [ "$QUEUE" -eq 0 ] && [ "$PENDING" -eq 0 ] && [ "$FAILED" -eq 0 ]; then
echo "complete=true" >> "$GITHUB_OUTPUT"
else
echo "complete=false" >> "$GITHUB_OUTPUT"
echo "::error title=Incomplete classification run::faces queue=$QUEUE, pending tasks=$PENDING, failed tasks=$FAILED of $TOTAL. Each pending task covers up to 500 photos, whose detections were never written. The detection DB will NOT be cached; the metrics below score only the photos that made it through."
fi
# Hard signal for this job: the TaskProcessing pipeline must actually
# persist face detections.
[ "$DETECTIONS" -gt 0 ]
- name: Save recognize_backend models cache
if: always() && steps.models-cache-restore.outputs.cache-hit != 'true'
uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
with:
path: recognize_backend-persistent_storage/
key: ${{ steps.models-cache-restore.outputs.cache-primary-key }}
- name: Cluster faces
run: |
./occ recognize:cluster-faces
- name: Install cadaver
if: ${{ matrix.faces-enabled }}
run: |
sudo apt -y install cadaver
- name: Test webdav access
if: ${{ matrix.faces-enabled }}
run: |
cat > ~/.netrc <<EOF
machine localhost
login admin
password password
EOF
cadaver http://localhost:8080/remote.php/dav/recognize/admin/ <<EOF
mv unassigned-faces/* faces/0/
mkdir faces/toot
mv faces/0/* faces/toot/
mv faces/toot/ faces/blah/
EOF
- name: Print logs
if: always()
run: |
cat data/nextcloud.log
echo '-------------------- recognize_backend logs --------------------'
cat "$GITHUB_WORKSPACE/backend_logs" || echo "No backend logs"