fix(l10n): Update translations from Transifex #143
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # SPDX-FileCopyrightText: Nextcloud contributors | |
| # SPDX-License-Identifier: AGPL-3.0-or-later | |
| name: Integration test | |
| on: | |
| pull_request: | |
| branches: | |
| - main | |
| - stable* | |
| push: | |
| branches: | |
| - main | |
| - stable* | |
| env: | |
| APP_NAME: context_chat | |
| permissions: | |
| contents: read | |
| concurrency: | |
| group: integration-test-cron-${{ github.head_ref || github.run_id }} | |
| cancel-in-progress: true | |
| jobs: | |
| changes: | |
| runs-on: ubuntu-latest-low | |
| permissions: | |
| contents: read | |
| pull-requests: read | |
| outputs: | |
| src: ${{ steps.changes.outputs.src}} | |
| steps: | |
| - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1 | |
| id: changes | |
| continue-on-error: true | |
| with: | |
| filters: | | |
| src: | |
| - '.github/workflows/integration-test-cron.yml' | |
| - 'lib/**' | |
| - 'stubs/**' | |
| - 'templates/**' | |
| - 'appinfo/info.xml' | |
| - 'composer.json' | |
| - 'composer.lock' | |
| cron-test: | |
| runs-on: ubuntu-24.04 | |
| strategy: | |
| # do not stop on another job's failure | |
| fail-fast: false | |
| matrix: | |
| php-versions: [ '8.2' ] | |
| databases: [ 'pgsql' ] | |
| server-versions: [ 'stable32', 'stable33', 'stable34' ] | |
| encryption: ['off'] | |
| include: | |
| - server-versions: master | |
| databases: pgsql | |
| php-versions: 8.3 | |
| encryption: 'on' | |
| - server-versions: stable32 | |
| databases: pgsql | |
| php-versions: 8.1 | |
| encryption: 'on' | |
| name: Integration test - Cron ☁️${{ matrix.server-versions }} 🐘${{ matrix.php-versions }} 🔐:${{ matrix.encryption }} | |
| env: | |
| MYSQL_PORT: 4444 | |
| PGSQL_PORT: 4445 | |
| # use the same db for ccb and nextcloud | |
| CCB_DB_URL: postgresql+psycopg://root:rootpassword@localhost:4445/nextcloud | |
| services: | |
| mysql: | |
| image: mariadb:10.5 | |
| ports: | |
| - 4444:3306/tcp | |
| env: | |
| MYSQL_ROOT_PASSWORD: rootpassword | |
| options: --health-cmd="mysqladmin ping" --health-interval 5s --health-timeout 2s --health-retries 5 | |
| postgres: | |
| image: pgvector/pgvector:pg17 | |
| ports: | |
| - 4445:5432/tcp | |
| env: | |
| POSTGRES_USER: root | |
| POSTGRES_PASSWORD: rootpassword | |
| POSTGRES_DB: nextcloud | |
| options: --health-cmd pg_isready --health-interval 5s --health-timeout 2s --health-retries 5 | |
| steps: | |
| - name: Checkout server | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| with: | |
| repository: nextcloud/server | |
| ref: ${{ matrix.server-versions }} | |
| submodules: recursive | |
| persist-credentials: false | |
| - name: Set up php ${{ matrix.php-versions }} | |
| uses: shivammathur/setup-php@cf4cade2721270509d5b1c766ab3549210a39a2a # 2.33.0 | |
| with: | |
| php-version: ${{ matrix.php-versions }} | |
| tools: phpunit | |
| extensions: mbstring, iconv, fileinfo, intl, sqlite, pdo_mysql, pdo_sqlite, pgsql, pdo_pgsql, gd, zip | |
| - name: Checkout app | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| with: | |
| path: apps/${{ env.APP_NAME }} | |
| persist-credentials: false | |
| - name: Checkout backend | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| with: | |
| repository: nextcloud/context_chat_backend | |
| path: context_chat_backend/ | |
| persist-credentials: false | |
| - name: Checkout viewer | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| with: | |
| repository: nextcloud/viewer | |
| path: apps/viewer | |
| persist-credentials: false | |
| ref: ${{ matrix.server-versions }} | |
| - name: Checkout app_api | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| with: | |
| repository: nextcloud/app_api | |
| ref: ${{ matrix.server-versions == 'master' && 'main' || matrix.server-versions }} | |
| path: apps/app_api | |
| persist-credentials: false | |
| - name: Get backend app version | |
| id: appinfo | |
| uses: skjnldsv/xpath-action@7e6a7c379d0e9abc8acaef43df403ab4fc4f770c # master | |
| with: | |
| filename: context_chat_backend/appinfo/info.xml | |
| expression: "/info/version/text()" | |
| - name: Read package.json node and npm engines version | |
| uses: skjnldsv/read-package-engines-version-actions@06d6baf7d8f41934ab630e97d9e6c0bc9c9ac5e4 # v3 | |
| id: versions | |
| with: | |
| path: apps/${{ env.APP_NAME }} | |
| fallbackNode: '^20' | |
| fallbackNpm: '^10' | |
| - name: Set up node ${{ steps.versions.outputs.nodeVersion }} | |
| uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 | |
| with: | |
| node-version: ${{ steps.versions.outputs.nodeVersion }} | |
| - name: Set up npm ${{ steps.versions.outputs.npmVersion }} | |
| run: npm i -g npm@"${{ steps.versions.outputs.npmVersion }}" | |
| - name: Install app | |
| working-directory: apps/${{ env.APP_NAME }} | |
| run: | | |
| make all | |
| composer install --no-dev | |
| - name: Set up Nextcloud | |
| if: ${{ matrix.databases != 'pgsql'}} | |
| run: | | |
| sleep 25 | |
| mkdir data | |
| ./occ maintenance:install --verbose --database=${{ matrix.databases }} --database-name=nextcloud --database-host=127.0.0.1 --database-port=$MYSQL_PORT --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass password | |
| composer run serve & | |
| - name: Set up Nextcloud | |
| if: ${{ matrix.databases == 'pgsql'}} | |
| run: | | |
| sleep 25 | |
| mkdir data | |
| ./occ maintenance:install --verbose --database=${{ matrix.databases }} --database-name=nextcloud --database-host=127.0.0.1 --database-port=$PGSQL_PORT --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass password | |
| composer run serve & | |
| - name: Enable app_api, context_chat and testing | |
| run: ./occ app:enable -vvv -f app_api context_chat testing | |
| - name: Enable encryption | |
| if: ${{ matrix.encryption == 'on' }} | |
| run: | | |
| ./occ app:enable -vvv -f encryption | |
| ./occ encryption:enable | |
| - name: Setup python 3.11 | |
| uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 | |
| with: | |
| python-version: '3.11' | |
| cache: 'pip' | |
| cache-dependency-path: context_chat_backend/requirements.txt | |
| - name: Install and init backend | |
| run: | | |
| cd context_chat_backend | |
| pip install --upgrade pip setuptools wheel | |
| # use the cpu version of torch to not run out of space | |
| pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu | |
| sed -i '/torch(vision)?/d' requirements.txt | |
| pip install -r requirements.txt | |
| cp example.env .env | |
| echo "NEXTCLOUD_URL=http://localhost:8080" >> .env | |
| python3 -u ./main_em.py > em_backend_logs 2>&1 & | |
| python3 -u ./main.py > backend_logs 2>&1 & | |
| echo $! > ../pid.txt # Save the process ID (PID) | |
| sleep 60 # Wait for the backend to get ready | |
| - name: Register backend | |
| run: | | |
| set -x | |
| ./occ config:system:set debug --value true | |
| ./occ config:system:set loglevel --value 1 | |
| ./occ app_api:daemon:register --net host manual_install "Manual Install" manual-install http localhost http://localhost:8080 | |
| timeout 120 ./occ app_api:app:register context_chat_backend manual_install --json-info "{\"appid\":\"context_chat_backend\",\"name\":\"Context Chat Backend\",\"daemon_config_name\":\"manual_install\",\"version\":\"${{ fromJson(steps.appinfo.outputs.result).version }}\",\"secret\":\"12345\",\"port\":10034,\"scopes\":[],\"system_app\":0}" --force-scopes --wait-finish | |
| sleep 60 # Wait for the embedding server to get ready | |
| - name: Checkout documentation | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| with: | |
| repository: nextcloud/documentation | |
| path: data/admin/files/documentation | |
| persist-credentials: false | |
| - name: Checkout pdfs | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| with: | |
| repository: papers-we-love/papers-we-love | |
| path: data/admin/files/papers | |
| ref: ad039b1e29ddb6ab1e217d39f16a5051c18ccc5a # to be deterministic | |
| persist-credentials: false | |
| - name: Prepare docs | |
| run: | | |
| cd data/admin/files | |
| mv documentation/admin_manual . | |
| cp -R documentation/developer_manual . | |
| cd developer_manual | |
| find . -type f -name "*.rst" -exec bash -c 'mv "$0" "${0%.rst}.md"' {} \; | |
| cd .. | |
| cp -R documentation/developer_manual ./developer_manual2 | |
| cd developer_manual2 | |
| find . -type f -name "*.rst" -exec bash -c 'mv "$0" "${0%.rst}.txt"' {} \; | |
| cd .. | |
| rm -rf documentation | |
| - name: Reduce number of files | |
| run: | | |
| cd data/admin/files | |
| LIMIT=100 | |
| # Find all files in the directory and its subdirectories | |
| FILES=($(find papers/ -type f)) | |
| # Count the number of files | |
| FILE_COUNT=${#FILES[@]} | |
| # Check if the current number of files exceeds the limit | |
| if [ "$FILE_COUNT" -le "$LIMIT" ]; then | |
| echo "No files need to be deleted. Current count: $FILE_COUNT" | |
| exit 0 | |
| fi | |
| # Calculate how many files to delete | |
| FILES_TO_DELETE=$((FILE_COUNT - LIMIT)) | |
| # Sort files by modification time (oldest first) and delete the oldest ones | |
| for FILE in $(ls -t "${FILES[@]}" | tail -n "$FILES_TO_DELETE"); do | |
| echo "Deleting: $FILE" | |
| rm "$FILE" | |
| done | |
| echo "Deleted $FILES_TO_DELETE files. Current count: $((FILE_COUNT - FILES_TO_DELETE))" | |
| - name: Run files scan | |
| run: | | |
| ./occ files:scan admin # Do the scan before enabling context chat | |
| - name: Run cron jobs | |
| run: | | |
| # every 10 seconds indefinitely | |
| while true; do | |
| php cron.php | |
| sleep 10 | |
| done & | |
| sleep 30 | |
| # list all the bg jobs | |
| ./occ background-job:list | |
| - name: Periodically check context_chat stats for 30 minutes to allow the backend to index the files | |
| run: | | |
| success=0 | |
| echo "::group::Checking stats periodically for 30 minutes to allow the backend to index the files" | |
| for i in {1..180}; do | |
| echo "Checking stats, attempt $i..." | |
| set +e # disable exit-on-error | |
| stats_err=$(mktemp) | |
| stats=$(timeout 10 ./occ context_chat:stats --json 2>"$stats_err") | |
| stats_exit=$? | |
| echo "Stats output:" | |
| echo "$stats" | |
| if [ -s "$stats_err" ]; then | |
| echo "Stderr:" | |
| cat "$stats_err" | |
| fi | |
| echo "---" | |
| rm -f "$stats_err" | |
| set -e # enable exit-on-error | |
| # Check for critical errors in output | |
| if [ $stats_exit -ne 0 ] || echo "$stats" | grep -q "Error during request"; then | |
| echo "Backend connection error detected (exit=$stats_exit), retrying..." | |
| sleep 10 | |
| continue | |
| fi | |
| # Extract total eligible files | |
| total_eligible_files=$(echo "$stats" | jq '.eligible_files_count' || echo "") | |
| # Extract indexed documents count (files__default) | |
| indexed_count=$(echo "$stats" | jq '.vectordb_document_counts.files__default' || echo "") | |
| echo "Total eligible files: $total_eligible_files" | |
| echo "Indexed documents (files__default): $indexed_count" | |
| diff=$((total_eligible_files - indexed_count)) | |
| threshold=$((total_eligible_files * 3 / 100)) | |
| # Check if difference is within tolerance | |
| if [ $diff -le $threshold ]; then | |
| echo "Indexing within 3% tolerance (diff=$diff, threshold=$threshold)" | |
| success=1 | |
| break | |
| else | |
| progress=$((diff * 100 / total_eligible_files)) | |
| echo "Outside 3% tolerance: diff=$diff (${progress}%), threshold=$threshold" | |
| fi | |
| # Check if backend is still alive | |
| ccb_alive=$(ps -p $(cat pid.txt) -o cmd= | grep -c "main.py" || echo "0") | |
| if [ "$ccb_alive" -eq 0 ]; then | |
| echo "Error: Context Chat Backend process is not running. Exiting." | |
| exit 1 | |
| fi | |
| sleep 10 | |
| done | |
| echo "::endgroup::" | |
| if [ $success -ne 1 ]; then | |
| echo "Max attempts reached" | |
| exit 1 | |
| fi | |
| - name: Run the prompts | |
| run: | | |
| ./occ background-job:worker -v 'OC\TaskProcessing\SynchronousBackgroundJob' & | |
| WORKER1_PID=$! | |
| ./occ background-job:worker -v 'OC\TaskProcessing\SynchronousBackgroundJob' & | |
| WORKER2_PID=$! | |
| echo "::group::Check english prompt with admin user" | |
| OUT1=$(./occ context_chat:prompt admin "Which factors are taken into account for the Ethical AI Rating?") | |
| echo "$OUT1" | |
| echo "$OUT1" | grep -q "If all of these points are met, we give a Green label." || exit 1 | |
| echo "::endgroup::" | |
| echo "::group::Check german prompt with admin user" | |
| OUT2=$(./occ context_chat:prompt admin "Welche Faktoren beeinflussen das Ethical AI Rating?") | |
| echo "$OUT2" | |
| echo "$OUT2" | grep -q "If all of these points are met, we give a Green label." || exit 1 | |
| echo "::endgroup::" | |
| echo "::group::Check german search with admin user" | |
| OUT3=$(./occ context_chat:search admin "Welche Faktoren beeinflussen das Ethical AI Rating?") | |
| echo "$OUT3" | |
| echo "$OUT3" | grep -q "overview.rst" || exit 1 | |
| echo "::endgroup::" | |
| kill -9 $WORKER1_PID | |
| kill -9 $WORKER2_PID | |
| - name: Show nextcloud logs | |
| if: always() | |
| run: | | |
| cat data/nextcloud.log | |
| - name: Show context chat logs | |
| if: always() | |
| run: | | |
| cat data/context_chat.log | |
| - name: Show CCB main app logs | |
| if: always() | |
| run: | | |
| cat context_chat_backend/backend_logs || echo "No main backend logs" | |
| - name: Show CCB main app JSON logs | |
| if: always() | |
| run: | | |
| tail -v -n +1 context_chat_backend/persistent_storage/logs/ccb.log* || echo "No logs in logs directory" | |
| - name: Show CCB embedding server logs | |
| if: always() | |
| run: | | |
| cat context_chat_backend/em_backend_logs || echo "No main backend logs" | |
| - name: Show CCB embedding server JSON logs | |
| if: always() | |
| run: | | |
| tail -v -n +1 context_chat_backend/persistent_storage/logs/em_server.log* || echo "No logs in logs directory" | |
| - name: Final stats log | |
| if: always() | |
| run: | | |
| ./occ context_chat:stats | |
| ./occ context_chat:stats --json | |
| summary: | |
| permissions: | |
| contents: none | |
| runs-on: ubuntu-latest-low | |
| needs: [changes, cron-test] | |
| if: always() | |
| # This is the summary, we just avoid to rename it so that branch protection rules still match | |
| name: integration-test-cron | |
| steps: | |
| - name: Summary status | |
| run: if ${{ needs.changes.outputs.src != 'false' && needs.cron-test.result != 'success' }}; then exit 1; fi |