Fix handling of storage that has multiple shared mounts #123

Workflow file for this run

.github/workflows/integration-test-cron.yml at 546a21e

	# SPDX-FileCopyrightText: Nextcloud contributors
	# SPDX-License-Identifier: AGPL-3.0-or-later

	name: Integration test

	on:
	pull_request:
	branches:
	- main
	- stable*
	push:
	branches:
	- main
	- stable*

	env:
	APP_NAME: context_chat

	permissions:
	contents: read

	concurrency:
	group: integration-test-cron-${{ github.head_ref \|\| github.run_id }}
	cancel-in-progress: true


	jobs:
	changes:
	runs-on: ubuntu-latest-low
	permissions:
	contents: read
	pull-requests: read

	outputs:
	src: ${{ steps.changes.outputs.src}}

	steps:
	- uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1
	id: changes
	continue-on-error: true
	with:
	filters: \|
	src:
	- '.github/workflows/integration-test-cron.yml'
	- 'lib/**'
	- 'stubs/**'
	- 'templates/**'
	- 'appinfo/info.xml'
	- 'composer.json'
	- 'composer.lock'

	cron-test:
	runs-on: ubuntu-24.04

	strategy:
	# do not stop on another job's failure
	fail-fast: false
	matrix:
	php-versions: [ '8.2' ]
	databases: [ 'pgsql' ]
	server-versions: [ 'stable32', 'stable33', 'stable34' ]
	encryption: ['off']
	include:
	- server-versions: master
	databases: pgsql
	php-versions: 8.3
	encryption: 'on'
	- server-versions: stable32
	databases: pgsql
	php-versions: 8.1
	encryption: 'on'

	name: Integration test - Cron ☁️${{ matrix.server-versions }} 🐘${{ matrix.php-versions }} 🔐:${{ matrix.encryption }}

	env:
	MYSQL_PORT: 4444
	PGSQL_PORT: 4445
	# use the same db for ccb and nextcloud
	CCB_DB_URL: postgresql+psycopg://root:rootpassword@localhost:4445/nextcloud

	services:
	mysql:
	image: mariadb:10.5
	ports:
	- 4444:3306/tcp
	env:
	MYSQL_ROOT_PASSWORD: rootpassword
	options: --health-cmd="mysqladmin ping" --health-interval 5s --health-timeout 2s --health-retries 5
	postgres:
	image: pgvector/pgvector:pg17
	ports:
	- 4445:5432/tcp
	env:
	POSTGRES_USER: root
	POSTGRES_PASSWORD: rootpassword
	POSTGRES_DB: nextcloud
	options: --health-cmd pg_isready --health-interval 5s --health-timeout 2s --health-retries 5

	steps:
	- name: Checkout server
	uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
	with:
	repository: nextcloud/server
	ref: ${{ matrix.server-versions }}
	submodules: recursive
	persist-credentials: false

	- name: Set up php ${{ matrix.php-versions }}
	uses: shivammathur/setup-php@cf4cade2721270509d5b1c766ab3549210a39a2a # 2.33.0
	with:
	php-version: ${{ matrix.php-versions }}
	tools: phpunit
	extensions: mbstring, iconv, fileinfo, intl, sqlite, pdo_mysql, pdo_sqlite, pgsql, pdo_pgsql, gd, zip

	- name: Checkout app
	uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
	with:
	path: apps/${{ env.APP_NAME }}
	persist-credentials: false

	- name: Checkout backend
	uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
	with:
	repository: nextcloud/context_chat_backend
	path: context_chat_backend/
	persist-credentials: false

	- name: Checkout viewer
	uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
	with:
	repository: nextcloud/viewer
	path: apps/viewer
	persist-credentials: false
	ref: ${{ matrix.server-versions }}

	- name: Checkout app_api
	uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
	with:
	repository: nextcloud/app_api
	ref: ${{ matrix.server-versions == 'master' && 'main' \|\| matrix.server-versions }}
	path: apps/app_api
	persist-credentials: false

	- name: Get backend app version
	id: appinfo
	uses: skjnldsv/xpath-action@7e6a7c379d0e9abc8acaef43df403ab4fc4f770c # master
	with:
	filename: context_chat_backend/appinfo/info.xml
	expression: "/info/version/text()"

	- name: Read package.json node and npm engines version
	uses: skjnldsv/read-package-engines-version-actions@06d6baf7d8f41934ab630e97d9e6c0bc9c9ac5e4 # v3
	id: versions
	with:
	path: apps/${{ env.APP_NAME }}
	fallbackNode: '^20'
	fallbackNpm: '^10'

	- name: Set up node ${{ steps.versions.outputs.nodeVersion }}
	uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
	with:
	node-version: ${{ steps.versions.outputs.nodeVersion }}

	- name: Set up npm ${{ steps.versions.outputs.npmVersion }}
	run: npm i -g npm@"${{ steps.versions.outputs.npmVersion }}"

	- name: Install app
	working-directory: apps/${{ env.APP_NAME }}
	run: \|
	make all
	composer install --no-dev

	- name: Set up Nextcloud
	if: ${{ matrix.databases != 'pgsql'}}
	run: \|
	sleep 25
	mkdir data
	./occ maintenance:install --verbose --database=${{ matrix.databases }} --database-name=nextcloud --database-host=127.0.0.1 --database-port=$MYSQL_PORT --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass password
	composer run serve &

	- name: Set up Nextcloud
	if: ${{ matrix.databases == 'pgsql'}}
	run: \|
	sleep 25
	mkdir data
	./occ maintenance:install --verbose --database=${{ matrix.databases }} --database-name=nextcloud --database-host=127.0.0.1 --database-port=$PGSQL_PORT --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass password
	composer run serve &

	- name: Enable app_api, context_chat and testing
	run: ./occ app:enable -vvv -f app_api context_chat testing

	- name: Enable encryption
	if: ${{ matrix.encryption == 'on' }}
	run: \|
	./occ app:enable -vvv -f encryption
	./occ encryption:enable

	- name: Setup python 3.11
	uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
	with:
	python-version: '3.11'
	cache: 'pip'
	cache-dependency-path: context_chat_backend/requirements.txt

	- name: Install and init backend
	run: \|
	cd context_chat_backend
	pip install --upgrade pip setuptools wheel
	# use the cpu version of torch to not run out of space
	pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu
	sed -i '/torch(vision)?/d' requirements.txt
	pip install -r requirements.txt
	cp example.env .env
	echo "NEXTCLOUD_URL=http://localhost:8080" >> .env
	python3 -u ./main_em.py > em_backend_logs 2>&1 &
	python3 -u ./main.py > backend_logs 2>&1 &
	echo $! > ../pid.txt # Save the process ID (PID)
	sleep 60 # Wait for the backend to get ready

	- name: Register backend
	run: \|
	set -x
	./occ config:system:set debug --value true
	./occ config:system:set loglevel --value 1
	./occ app_api:daemon:register --net host manual_install "Manual Install" manual-install http localhost http://localhost:8080
	timeout 120 ./occ app_api:app:register context_chat_backend manual_install --json-info "{\"appid\":\"context_chat_backend\",\"name\":\"Context Chat Backend\",\"daemon_config_name\":\"manual_install\",\"version\":\"${{ fromJson(steps.appinfo.outputs.result).version }}\",\"secret\":\"12345\",\"port\":10034,\"scopes\":[],\"system_app\":0}" --force-scopes --wait-finish
	sleep 60 # Wait for the embedding server to get ready

	- name: Checkout documentation
	uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
	with:
	repository: nextcloud/documentation
	path: data/admin/files/documentation
	persist-credentials: false

	- name: Checkout pdfs
	uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
	with:
	repository: papers-we-love/papers-we-love
	path: data/admin/files/papers
	ref: ad039b1e29ddb6ab1e217d39f16a5051c18ccc5a # to be deterministic
	persist-credentials: false

	- name: Prepare docs
	run: \|
	cd data/admin/files
	mv documentation/admin_manual .
	cp -R documentation/developer_manual .
	cd developer_manual
	find . -type f -name "*.rst" -exec bash -c 'mv "$0" "${0%.rst}.md"' {} \;
	cd ..
	cp -R documentation/developer_manual ./developer_manual2
	cd developer_manual2
	find . -type f -name "*.rst" -exec bash -c 'mv "$0" "${0%.rst}.txt"' {} \;
	cd ..
	rm -rf documentation

	- name: Reduce number of files
	run: \|
	cd data/admin/files
	LIMIT=100

	# Find all files in the directory and its subdirectories
	FILES=($(find papers/ -type f))

	# Count the number of files
	FILE_COUNT=${#FILES[@]}

	# Check if the current number of files exceeds the limit
	if [ "$FILE_COUNT" -le "$LIMIT" ]; then
	echo "No files need to be deleted. Current count: $FILE_COUNT"
	exit 0
	fi

	# Calculate how many files to delete
	FILES_TO_DELETE=$((FILE_COUNT - LIMIT))

	# Sort files by modification time (oldest first) and delete the oldest ones
	for FILE in $(ls -t "${FILES[@]}" \| tail -n "$FILES_TO_DELETE"); do
	echo "Deleting: $FILE"
	rm "$FILE"
	done

	echo "Deleted $FILES_TO_DELETE files. Current count: $((FILE_COUNT - FILES_TO_DELETE))"

	- name: Run files scan
	run: \|
	./occ files:scan admin # Do the scan before enabling context chat

	- name: Run cron jobs
	run: \|
	# every 10 seconds indefinitely
	while true; do
	php cron.php
	sleep 10
	done &
	sleep 30
	# list all the bg jobs
	./occ background-job:list

	- name: Periodically check context_chat stats for 30 minutes to allow the backend to index the files
	run: \|
	success=0
	echo "::group::Checking stats periodically for 30 minutes to allow the backend to index the files"
	for i in {1..180}; do
	echo "Checking stats, attempt $i..."

	set +e # disable exit-on-error
	stats_err=$(mktemp)
	stats=$(timeout 10 ./occ context_chat:stats --json 2>"$stats_err")
	stats_exit=$?
	echo "Stats output:"
	echo "$stats"
	if [ -s "$stats_err" ]; then
	echo "Stderr:"
	cat "$stats_err"
	fi
	echo "---"
	rm -f "$stats_err"
	set -e # enable exit-on-error

	# Check for critical errors in output
	if [ $stats_exit -ne 0 ] \|\| echo "$stats" \| grep -q "Error during request"; then
	echo "Backend connection error detected (exit=$stats_exit), retrying..."
	sleep 10
	continue
	fi

	# Extract total eligible files
	total_eligible_files=$(echo "$stats" \| jq '.eligible_files_count' \|\| echo "")

	# Extract indexed documents count (files__default)
	indexed_count=$(echo "$stats" \| jq '.vectordb_document_counts.files__default' \|\| echo "")

	echo "Total eligible files: $total_eligible_files"
	echo "Indexed documents (files__default): $indexed_count"

	diff=$((total_eligible_files - indexed_count))
	threshold=$((total_eligible_files * 3 / 100))

	# Check if difference is within tolerance
	if [ $diff -le $threshold ]; then
	echo "Indexing within 3% tolerance (diff=$diff, threshold=$threshold)"
	success=1
	break
	else
	progress=$((diff * 100 / total_eligible_files))
	echo "Outside 3% tolerance: diff=$diff (${progress}%), threshold=$threshold"
	fi

	# Check if backend is still alive
	ccb_alive=$(ps -p $(cat pid.txt) -o cmd= \| grep -c "main.py" \|\| echo "0")
	if [ "$ccb_alive" -eq 0 ]; then
	echo "Error: Context Chat Backend process is not running. Exiting."
	exit 1
	fi

	sleep 10
	done

	echo "::endgroup::"

	if [ $success -ne 1 ]; then
	echo "Max attempts reached"
	exit 1
	fi

	- name: Run the prompts
	run: \|
	./occ background-job:worker -v 'OC\TaskProcessing\SynchronousBackgroundJob' &
	WORKER1_PID=$!
	./occ background-job:worker -v 'OC\TaskProcessing\SynchronousBackgroundJob' &
	WORKER2_PID=$!

	echo "::group::Check english prompt with admin user"
	OUT1=$(./occ context_chat:prompt admin "Which factors are taken into account for the Ethical AI Rating?")
	echo "$OUT1"
	echo "$OUT1" \| grep -q "If all of these points are met, we give a Green label." \|\| exit 1
	echo "::endgroup::"

	echo "::group::Check german prompt with admin user"
	OUT2=$(./occ context_chat:prompt admin "Welche Faktoren beeinflussen das Ethical AI Rating?")
	echo "$OUT2"
	echo "$OUT2" \| grep -q "If all of these points are met, we give a Green label." \|\| exit 1
	echo "::endgroup::"

	echo "::group::Check german search with admin user"
	OUT3=$(./occ context_chat:search admin "Welche Faktoren beeinflussen das Ethical AI Rating?")
	echo "$OUT3"
	echo "$OUT3" \| grep -q "overview.rst" \|\| exit 1
	echo "::endgroup::"

	kill -9 $WORKER1_PID
	kill -9 $WORKER2_PID

	- name: Show nextcloud logs
	if: always()
	run: \|
	cat data/nextcloud.log

	- name: Show context chat logs
	if: always()
	run: \|
	cat data/context_chat.log

	- name: Show CCB main app logs
	if: always()
	run: \|
	cat context_chat_backend/backend_logs \|\| echo "No main backend logs"

	- name: Show CCB main app JSON logs
	if: always()
	run: \|
	tail -v -n +1 context_chat_backend/persistent_storage/logs/ccb.log* \|\| echo "No logs in logs directory"

	- name: Show CCB embedding server logs
	if: always()
	run: \|
	cat context_chat_backend/em_backend_logs \|\| echo "No main backend logs"

	- name: Show CCB embedding server JSON logs
	if: always()
	run: \|
	tail -v -n +1 context_chat_backend/persistent_storage/logs/em_server.log* \|\| echo "No logs in logs directory"

	- name: Final stats log
	if: always()
	run: \|
	./occ context_chat:stats
	./occ context_chat:stats --json

	summary:
	permissions:
	contents: none
	runs-on: ubuntu-latest-low
	needs: [changes, cron-test]

	if: always()

	# This is the summary, we just avoid to rename it so that branch protection rules still match
	name: integration-test-cron

	steps:
	- name: Summary status
	run: if ${{ needs.changes.outputs.src != 'false' && needs.cron-test.result != 'success' }}; then exit 1; fi

Navigation Menu

Uh oh!

Fix handling of storage that has multiple shared mounts #123

Workflow file

Fix handling of storage that has multiple shared mounts #123

Uh oh!

Workflow file for this run