Skip to content

Commit 0edd934

Browse files
marcelklehrmarcelklehr
authored
Merge pull request #226 from nextcloud/feat/reverse-content-flow
Feat: reverse content flow
2 parents 493c537 + 0e46a59 commit 0edd934

40 files changed

Lines changed: 1423 additions & 2152 deletions

‎.github/workflows/integration-test-cron.yml‎

Lines changed: 99 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,8 @@ jobs:
124124
repository: nextcloud/context_chat_backend
125125
path: context_chat_backend/
126126
persist-credentials: false
127+
# todo: remove later
128+
ref: feat/noid/rev-content-flow
127129

128130
- name: Checkout viewer
129131
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
@@ -178,8 +180,8 @@ jobs:
178180
./occ maintenance:install --verbose --database=${{ matrix.databases }} --database-name=nextcloud --database-host=127.0.0.1 --database-port=$PGSQL_PORT --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass password
179181
composer run serve &
180182
181-
- name: Enable app_api and testing
182-
run: ./occ app:enable -vvv -f app_api testing
183+
- name: Enable app_api, context_chat and testing
184+
run: ./occ app:enable -vvv -f app_api context_chat testing
183185

184186
- name: Enable encryption
185187
if: ${{ matrix.encryption == 'on' }}
@@ -279,48 +281,83 @@ jobs:
279281
run: |
280282
./occ files:scan admin # Do the scan before enabling context chat
281283
282-
- name: Enable context chat
283-
run: ./occ app:enable -vvv -f ${{ env.APP_NAME }}
284+
- name: Run cron jobs
285+
run: |
286+
# every 10 seconds indefinitely
287+
while true; do
288+
php cron.php
289+
sleep 10
290+
done &
291+
sleep 30
292+
# list all the bg jobs
293+
./occ background-job:list
284294
285-
- name: Run indexer cron
295+
- name: Periodically check context_chat stats for 30 minutes to allow the backend to index the files
286296
run: |
287-
# Run cron in speed mode: Set interval to 0 minutes
288-
./occ config:app:set --value 30 --type integer context_chat indexing_job_interval # 30 seconds
289-
./occ config:app:set --value 10 --type integer context_chat crawl_job_interval # 10 seconds
290-
./occ config:app:set --value '10' --type string context_chat action_job_interval # 10 seconds
291-
./occ config:app:set --value '10' --type string context_chat fs_listener_job_interval # 10 seconds
292-
for i in {1..100}; do
293-
php cron.php & # Starting with stable31 we can use -v here for better visibility
294-
wait
295-
./occ context_chat:stats
297+
success=0
298+
echo "::group::Checking stats periodically for 30 minutes to allow the backend to index the files"
299+
for i in {1..180}; do
300+
echo "Checking stats, attempt $i..."
301+
302+
set +e # disable exit-on-error
303+
stats_err=$(mktemp)
304+
stats=$(timeout 10 ./occ context_chat:stats --json 2>"$stats_err")
305+
stats_exit=$?
306+
echo "Stats output:"
307+
echo "$stats"
308+
if [ -s "$stats_err" ]; then
309+
echo "Stderr:"
310+
cat "$stats_err"
311+
fi
312+
echo "---"
313+
rm -f "$stats_err"
314+
set -e # enable exit-on-error
315+
316+
# Check for critical errors in output
317+
if [ $stats_exit -ne 0 ] || echo "$stats" | grep -q "Error during request"; then
318+
echo "Backend connection error detected (exit=$stats_exit), retrying..."
319+
sleep 10
320+
continue
321+
fi
322+
323+
# Extract total eligible files
324+
total_eligible_files=$(echo "$stats" | jq '.eligible_files_count' || echo "")
325+
326+
# Extract indexed documents count (files__default)
327+
indexed_count=$(echo "$stats" | jq '.vectordb_document_counts.files__default' || echo "")
328+
329+
echo "Total eligible files: $total_eligible_files"
330+
echo "Indexed documents (files__default): $indexed_count"
331+
332+
diff=$((total_eligible_files - indexed_count))
333+
threshold=$((total_eligible_files * 3 / 100))
334+
335+
# Check if difference is within tolerance
336+
if [ $diff -le $threshold ]; then
337+
echo "Indexing within 3% tolerance (diff=$diff, threshold=$threshold)"
338+
success=1
339+
break
340+
else
341+
progress=$((diff * 100 / total_eligible_files))
342+
echo "Outside 3% tolerance: diff=$diff (${progress}%), threshold=$threshold"
343+
fi
344+
345+
# Check if backend is still alive
346+
ccb_alive=$(ps -p $(cat pid.txt) -o cmd= | grep -c "main.py" || echo "0")
347+
if [ "$ccb_alive" -eq 0 ]; then
348+
echo "Error: Context Chat Backend process is not running. Exiting."
349+
exit 1
350+
fi
351+
352+
sleep 10
296353
done
297354
298-
- name: Check context chat state
299-
run: |
300-
./occ context_chat:stats
301-
./occ background-job:list
302-
./occ context_chat:stats | grep -q "Index complete time" || echo "Indexing did not complete"
303-
./occ context_chat:stats | awk '
304-
/Total eligible files/ {
305-
total = $NF + 0 # force conversion to number
306-
}
307-
/files__default/ {
308-
indexed = $NF;
309-
sub(/,$/, "", indexed) # remove trailing comma
310-
indexed = indexed + 0 # force conversion to number
311-
}
312-
END {
313-
low = total * 0.85;
314-
high = total * 1.15;
315-
if (indexed >= low && indexed <= high) {
316-
print "✅ Indexed files (" indexed ") are within 15% of eligible files (" total ").";
317-
exit 0;
318-
} else {
319-
print "❌ Indexed files (" indexed ") are OUTSIDE the 15% range of eligible files (" total ").";
320-
exit 1;
321-
}
322-
}
323-
'
355+
echo "::endgroup::"
356+
357+
if [ $success -ne 1 ]; then
358+
echo "Max attempts reached"
359+
exit 1
360+
fi
324361
325362
- name: Run the prompts
326363
run: |
@@ -360,13 +397,31 @@ jobs:
360397
run: |
361398
cat data/context_chat.log
362399
363-
- name: Show backend logs
400+
- name: Show CCB main app logs
364401
if: always()
365402
run: |
366403
cat context_chat_backend/backend_logs || echo "No main backend logs"
367-
cat context_chat_backend/em_backend_logs || echo "No embedding server logs"
368-
echo '--------------------------------------------------'
369-
tail -v -n +1 context_chat_backend/persistent_storage/logs/* || echo "No logs in logs directory"
404+
405+
- name: Show CCB main app JSON logs
406+
if: always()
407+
run: |
408+
tail -v -n +1 context_chat_backend/persistent_storage/logs/ccb.log* || echo "No logs in logs directory"
409+
410+
- name: Show CCB embedding server logs
411+
if: always()
412+
run: |
413+
cat context_chat_backend/em_backend_logs || echo "No main backend logs"
414+
415+
- name: Show CCB embedding server JSON logs
416+
if: always()
417+
run: |
418+
tail -v -n +1 context_chat_backend/persistent_storage/logs/em_server.log* || echo "No logs in logs directory"
419+
420+
- name: Final stats log
421+
if: always()
422+
run: |
423+
./occ context_chat:stats
424+
./occ context_chat:stats --json
370425
371426
summary:
372427
permissions:

‎.github/workflows/integration-test-file-listener.yml‎

Lines changed: 107 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,8 @@ jobs:
116116
repository: nextcloud/context_chat_backend
117117
path: context_chat_backend/
118118
persist-credentials: false
119+
# todo: remove later
120+
ref: feat/noid/rev-content-flow
119121

120122
- name: Checkout viewer
121123
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
@@ -170,8 +172,8 @@ jobs:
170172
./occ maintenance:install --verbose --database=${{ matrix.databases }} --database-name=nextcloud --database-host=127.0.0.1 --database-port=$PGSQL_PORT --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass password
171173
composer run serve &
172174
173-
- name: Enable app_api and testing
174-
run: ./occ app:enable -vvv -f app_api testing
175+
- name: Enable app_api, context_chat and testing
176+
run: ./occ app:enable -vvv -f app_api context_chat testing
175177

176178
- name: Enable encryption
177179
if: ${{ matrix.encryption == 'on' }}
@@ -210,21 +212,6 @@ jobs:
210212
timeout 120 ./occ app_api:app:register context_chat_backend manual_install --json-info "{\"appid\":\"context_chat_backend\",\"name\":\"Context Chat Backend\",\"daemon_config_name\":\"manual_install\",\"version\":\"${{ fromJson(steps.appinfo.outputs.result).version }}\",\"secret\":\"12345\",\"port\":10034,\"scopes\":[],\"system_app\":0}" --force-scopes --wait-finish
211213
sleep 60 # Wait for the embedding server to get ready
212214
213-
- name: Enable context chat
214-
run: |
215-
./occ app:enable -vvv -f ${{ env.APP_NAME }}
216-
# Run cron in speed mode
217-
./occ config:app:set --value 30 --type integer context_chat indexing_job_interval # 30 seconds
218-
./occ config:app:set --value 10 --type integer context_chat crawl_job_interval # 10 seconds
219-
./occ config:app:set --value '10' --type string context_chat action_job_interval # 10 seconds
220-
./occ config:app:set --value '10' --type string context_chat fs_listener_job_interval # 10 seconds
221-
# Run normal indexing jobs which will only pick up welcome.txt etc
222-
for i in {1..10}; do
223-
php cron.php & # Starting with stable31 we can use -v here for better visibility
224-
wait
225-
./occ context_chat:stats
226-
done
227-
228215
- name: Checkout documentation
229216
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
230217
with:
@@ -286,40 +273,86 @@ jobs:
286273
run: |
287274
./occ files:scan admin # We do the scan after enabling context chat here, so we can test the file listeners
288275
289-
- name: Run indexer cron
276+
- name: Run cron jobs
290277
run: |
291-
for i in {1..100}; do
292-
php cron.php & # Starting with stable31 we can use -v here for better visibility
293-
wait
294-
./occ context_chat:stats
295-
done
278+
./occ config:app:set --value 10 --type integer context_chat crawl_job_interval # 10 seconds
279+
./occ config:app:set --value '10' --type string context_chat action_job_interval # 10 seconds
280+
./occ config:app:set --value '10' --type string context_chat fs_listener_job_interval # 10 seconds
281+
# every 10 seconds indefinitely
282+
while true; do
283+
php cron.php
284+
sleep 10
285+
done &
286+
sleep 30
287+
# list all the bg jobs
288+
./occ background-job:list
296289
297-
- name: Check context chat state
290+
- name: Periodically check context_chat stats for 40 minutes to allow the backend to index the files
298291
run: |
299-
./occ context_chat:stats
300-
./occ background-job:list
301-
./occ context_chat:stats | grep -q "Index complete time" || echo "Indexing did not complete"
302-
./occ context_chat:stats | awk '
303-
/Total eligible files/ {
304-
total = $NF + 0 # force conversion to number
305-
}
306-
/files__default/ {
307-
indexed = $NF;
308-
sub(/,$/, "", indexed) # remove trailing comma
309-
indexed = indexed + 0 # force conversion to number
310-
}
311-
END {
312-
low = total * 0.85;
313-
high = total * 1.15;
314-
if (indexed >= low && indexed <= high) {
315-
print "✅ Indexed files (" indexed ") are within 15% of eligible files (" total ").";
316-
exit 0;
317-
} else {
318-
print "❌ Indexed files (" indexed ") are OUTSIDE the 15% range of eligible files (" total ").";
319-
exit 1;
320-
}
321-
}
322-
'
292+
success=0
293+
echo "::group::Checking stats periodically for 40 minutes to allow the backend to index the files"
294+
for i in {1..240}; do
295+
echo "Checking stats, attempt $i..."
296+
297+
set +e # disable exit-on-error
298+
stats_err=$(mktemp)
299+
stats=$(timeout 10 ./occ context_chat:stats --json 2>"$stats_err")
300+
stats_exit=$?
301+
echo "Stats output:"
302+
echo "$stats"
303+
if [ -s "$stats_err" ]; then
304+
echo "Stderr:"
305+
cat "$stats_err"
306+
fi
307+
echo "---"
308+
rm -f "$stats_err"
309+
set -e
310+
311+
# Check for critical errors in output
312+
if [ $stats_exit -ne 0 ] || echo "$stats" | grep -q "Error during request"; then
313+
echo "Backend connection error detected (exit=$stats_exit), retrying..."
314+
sleep 10
315+
continue
316+
fi
317+
318+
# Extract total eligible files
319+
total_eligible_files=$(echo "$stats" | jq '.eligible_files_count' || echo "")
320+
321+
# Extract indexed documents count (files__default)
322+
indexed_count=$(echo "$stats" | jq '.vectordb_document_counts.files__default' || echo "")
323+
324+
echo "Total eligible files: $total_eligible_files"
325+
echo "Indexed documents (files__default): $indexed_count"
326+
327+
diff=$((total_eligible_files - indexed_count))
328+
threshold=$((total_eligible_files * 3 / 100))
329+
330+
# Check if difference is within tolerance
331+
if [ $diff -le $threshold ]; then
332+
echo "Indexing within 3% tolerance (diff=$diff, threshold=$threshold)"
333+
success=1
334+
break
335+
else
336+
progress=$((diff * 100 / total_eligible_files))
337+
echo "Outside 3% tolerance: diff=$diff (${progress}%), threshold=$threshold"
338+
fi
339+
340+
# Check if backend is still alive
341+
ccb_alive=$(ps -p $(cat pid.txt) -o cmd= | grep -c "main.py" || echo "0")
342+
if [ "$ccb_alive" -eq 0 ]; then
343+
echo "Error: Context Chat Backend process is not running. Exiting."
344+
exit 1
345+
fi
346+
347+
sleep 10
348+
done
349+
350+
echo "::endgroup::"
351+
352+
if [ $success -ne 1 ]; then
353+
echo "Max attempts reached"
354+
exit 1
355+
fi
323356
324357
- name: Run the prompts
325358
run: |
@@ -409,12 +442,13 @@ jobs:
409442
# list files to trigger mounting of shares
410443
curl -X PROPFIND -H "Depth: 0" -u 'test:test' 'http://localhost:8080/remote.php/webdav/'
411444
412-
- name: Run indexer cron
445+
- name: Show context chat stats for actions changes
413446
run: |
414-
for i in {1..100}; do
415-
php cron.php & # Starting with stable31 we can use -v here for better visibility
447+
for i in {1..20}; do
448+
php cron.php -v
416449
wait
417450
./occ context_chat:stats
451+
echo '--------------------------------------'
418452
done
419453
420454
- name: Run the prompts again
@@ -494,13 +528,31 @@ jobs:
494528
run: |
495529
cat data/context_chat.log
496530
497-
- name: Show backend logs
531+
- name: Show CCB main app logs
498532
if: always()
499533
run: |
500534
cat context_chat_backend/backend_logs || echo "No main backend logs"
501-
cat context_chat_backend/em_backend_logs || echo "No embedding server logs"
502-
echo '--------------------------------------------------'
503-
tail -v -n +1 context_chat_backend/persistent_storage/logs/* || echo "No logs in logs directory"
535+
536+
- name: Show CCB main app JSON logs
537+
if: always()
538+
run: |
539+
tail -v -n +1 context_chat_backend/persistent_storage/logs/ccb.log* || echo "No logs in logs directory"
540+
541+
- name: Show CCB embedding server logs
542+
if: always()
543+
run: |
544+
cat context_chat_backend/em_backend_logs || echo "No main backend logs"
545+
546+
- name: Show CCB embedding server JSON logs
547+
if: always()
548+
run: |
549+
tail -v -n +1 context_chat_backend/persistent_storage/logs/em_server.log* || echo "No logs in logs directory"
550+
551+
- name: Final stats log
552+
if: always()
553+
run: |
554+
./occ context_chat:stats
555+
./occ context_chat:stats --json
504556
505557
summary:
506558
permissions:

0 commit comments

Comments
 (0)