From bd5fcdf741945a6f997f699e36e0ccb86e938e47 Mon Sep 17 00:00:00 2001
From: Marcel Klehr <mklehr@gmx.net>
Date: Thu, 24 Jul 2025 15:25:44 +0200
Subject: [PATCH 01/30] test(integration): test ContextChatSearch task

Signed-off-by: Marcel Klehr <mklehr@gmx.net>
---
 .github/workflows/integration-test.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml
index 0c8d6dd9..cacb5f0b 100644
--- a/.github/workflows/integration-test.yml
+++ b/.github/workflows/integration-test.yml
@@ -223,6 +223,7 @@ jobs:
           ./occ background-job:worker 'OC\TaskProcessing\SynchronousBackgroundJob' &
           ./occ context_chat:prompt admin "Which factors are taken into account for the Ethical AI Rating?"
           ./occ context_chat:prompt admin "Welche Faktoren beeinflussen das Ethical AI Rating?"
+          ./occ context_chat:search admin "Welche Faktoren beeinflussen das Ethical AI Rating?"
 
       - name: Check python memory usage
         run: |
@@ -493,6 +494,7 @@ jobs:
           ./occ background-job:worker 'OC\TaskProcessing\SynchronousBackgroundJob' &
           ./occ context_chat:prompt admin "Which factors are taken into account for the Ethical AI Rating?"
           ./occ context_chat:prompt admin "Welche Faktoren beeinflussen das Ethical AI Rating?"
+          ./occ context_chat:search admin "Welche Faktoren beeinflussen das Ethical AI Rating?"
 
       - name: Show nextcloud logs
         if: always()
@@ -739,6 +741,7 @@ jobs:
           ./occ background-job:worker 'OC\TaskProcessing\SynchronousBackgroundJob' &
           ./occ context_chat:prompt admin "Which factors are taken into account for the Ethical AI Rating?"
           ./occ context_chat:prompt admin "Welche Faktoren beeinflussen das Ethical AI Rating?"
+          ./occ context_chat:search admin "Welche Faktoren beeinflussen das Ethical AI Rating?"
 
       - name: Show nextcloud logs
         if: always()

From e54bbdc3166de5b22e70c1ede53c4183894394ec Mon Sep 17 00:00:00 2001
From: Marcel Klehr <mklehr@gmx.net>
Date: Thu, 24 Jul 2025 15:33:49 +0200
Subject: [PATCH 02/30] test(integration): test delete listeners

Signed-off-by: Marcel Klehr <mklehr@gmx.net>
---
 .github/workflows/integration-test.yml | 27 +++++++++++++++++++++++++-
 1 file changed, 26 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml
index cacb5f0b..0b354f23 100644
--- a/.github/workflows/integration-test.yml
+++ b/.github/workflows/integration-test.yml
@@ -698,7 +698,7 @@ jobs:
 
       - name: Run files scan
         run: |
-          ./occ files:scan admin # Do the scan before enabling context chat
+          ./occ files:scan admin # We do the scan after enabling context chat here, so we can test the file listeners
 
       - name: Run indexer cron
         run: |
@@ -743,6 +743,31 @@ jobs:
           ./occ context_chat:prompt admin "Welche Faktoren beeinflussen das Ethical AI Rating?"
           ./occ context_chat:search admin "Welche Faktoren beeinflussen das Ethical AI Rating?"
 
+      - name: Remove some files
+        run: |
+          cd data/admin/files
+          rm -rf ./admin_manual
+
+      - name: Run files scan
+        run: |
+          ./occ files:scan admin # We run the scan again to check if the files are successfully removed from the vectordb
+
+      - name: Run indexer cron
+        run: |
+          for i in {1..100}; do 
+            php cron.php & # Starting with stable31 we can use -v here for better visibility
+            wait
+            ./occ context_chat:stats
+          done
+
+      - name: Run the prompts again
+        run: |
+          ./occ background-job:worker 'OC\TaskProcessing\SynchronousBackgroundJob' &
+          ./occ background-job:worker 'OC\TaskProcessing\SynchronousBackgroundJob' &
+          ./occ context_chat:prompt admin "Which factors are taken into account for the Ethical AI Rating?"
+          ./occ context_chat:prompt admin "Welche Faktoren beeinflussen das Ethical AI Rating?"
+          ./occ context_chat:search admin "Welche Faktoren beeinflussen das Ethical AI Rating?"
+
       - name: Show nextcloud logs
         if: always()
         run: |

From 04767cbfbc6c850d39253e2409e80951f5827601 Mon Sep 17 00:00:00 2001
From: Marcel Klehr <mklehr@gmx.net>
Date: Fri, 25 Jul 2025 07:55:21 +0200
Subject: [PATCH 03/30] test(integration): Check results from prompts
 automatically

Signed-off-by: Marcel Klehr <mklehr@gmx.net>
---
 .github/workflows/integration-test.yml | 45 ++++++++++++++++++++------
 1 file changed, 36 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml
index 0b354f23..9f2d23a0 100644
--- a/.github/workflows/integration-test.yml
+++ b/.github/workflows/integration-test.yml
@@ -221,9 +221,18 @@ jobs:
         run: |
           ./occ background-job:worker 'OC\TaskProcessing\SynchronousBackgroundJob' &
           ./occ background-job:worker 'OC\TaskProcessing\SynchronousBackgroundJob' &
-          ./occ context_chat:prompt admin "Which factors are taken into account for the Ethical AI Rating?"
-          ./occ context_chat:prompt admin "Welche Faktoren beeinflussen das Ethical AI Rating?"
-          ./occ context_chat:search admin "Welche Faktoren beeinflussen das Ethical AI Rating?"
+          OUT1=$(./occ context_chat:prompt admin "Which factors are taken into account for the Ethical AI Rating?")
+          echo "$OUT1"
+          echo '--------------------------------------------------'
+          OUT2=$(./occ context_chat:prompt admin "Welche Faktoren beeinflussen das Ethical AI Rating?")
+          echo "$OUT2"
+          echo '--------------------------------------------------'
+
+          OUT3=$(./occ context_chat:search admin "Welche Faktoren beeinflussen das Ethical AI Rating?")
+          echo "$OUT3"
+          echo "$OUT1" | grep -q "If all of these points are met, we give a Green label." || exit 1
+          echo "$OUT2" | grep -q "If all of these points are met, we give a Green label." || exit 1
+          echo "$OUT3" | grep -q "overview.rst" || exit 1
 
       - name: Check python memory usage
         run: |
@@ -492,9 +501,18 @@ jobs:
         run: |
           ./occ background-job:worker 'OC\TaskProcessing\SynchronousBackgroundJob' &
           ./occ background-job:worker 'OC\TaskProcessing\SynchronousBackgroundJob' &
-          ./occ context_chat:prompt admin "Which factors are taken into account for the Ethical AI Rating?"
-          ./occ context_chat:prompt admin "Welche Faktoren beeinflussen das Ethical AI Rating?"
-          ./occ context_chat:search admin "Welche Faktoren beeinflussen das Ethical AI Rating?"
+          OUT1=$(./occ context_chat:prompt admin "Which factors are taken into account for the Ethical AI Rating?")
+          echo "$OUT1"
+          echo '--------------------------------------------------'
+          OUT2=$(./occ context_chat:prompt admin "Welche Faktoren beeinflussen das Ethical AI Rating?")
+          echo "$OUT2"
+          echo '--------------------------------------------------'
+
+          OUT3=$(./occ context_chat:search admin "Welche Faktoren beeinflussen das Ethical AI Rating?")
+          echo "$OUT3"
+          echo "$OUT1" | grep -q "If all of these points are met, we give a Green label." || exit 1
+          echo "$OUT2" | grep -q "If all of these points are met, we give a Green label." || exit 1
+          echo "$OUT3" | grep -q "overview.rst" || exit 1
 
       - name: Show nextcloud logs
         if: always()
@@ -739,9 +757,18 @@ jobs:
         run: |
           ./occ background-job:worker 'OC\TaskProcessing\SynchronousBackgroundJob' &
           ./occ background-job:worker 'OC\TaskProcessing\SynchronousBackgroundJob' &
-          ./occ context_chat:prompt admin "Which factors are taken into account for the Ethical AI Rating?"
-          ./occ context_chat:prompt admin "Welche Faktoren beeinflussen das Ethical AI Rating?"
-          ./occ context_chat:search admin "Welche Faktoren beeinflussen das Ethical AI Rating?"
+          OUT1=$(./occ context_chat:prompt admin "Which factors are taken into account for the Ethical AI Rating?")
+          echo "$OUT1"
+          echo '--------------------------------------------------'
+          OUT2=$(./occ context_chat:prompt admin "Welche Faktoren beeinflussen das Ethical AI Rating?")
+          echo "$OUT2"
+          echo '--------------------------------------------------'
+
+          OUT3=$(./occ context_chat:search admin "Welche Faktoren beeinflussen das Ethical AI Rating?")
+          echo "$OUT3"
+          echo "$OUT1" | grep -q "If all of these points are met, we give a Green label." || exit 1
+          echo "$OUT2" | grep -q "If all of these points are met, we give a Green label." || exit 1
+          echo "$OUT3" | grep -q "overview.rst" || exit 1
 
       - name: Remove some files
         run: |

From a2950b13f92599cf051445e27d93f16821cd1988 Mon Sep 17 00:00:00 2001
From: Marcel Klehr <mklehr@gmx.net>
Date: Fri, 25 Jul 2025 09:16:47 +0200
Subject: [PATCH 04/30] test(integration): Make ActionJob and
 FileSystemListenerJob intervals configurable

to streamline testing

Signed-off-by: Marcel Klehr <mklehr@gmx.net>
---
 .github/workflows/integration-test.yml       |  4 ++
 lib/BackgroundJobs/ActionJob.php             | 10 +++-
 lib/BackgroundJobs/FileSystemListenerJob.php | 50 ++++++++++++++------
 3 files changed, 47 insertions(+), 17 deletions(-)

diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml
index 9f2d23a0..913202d4 100644
--- a/.github/workflows/integration-test.yml
+++ b/.github/workflows/integration-test.yml
@@ -464,6 +464,8 @@ jobs:
           # Run cron in speed mode: Set interval to 0 minutes
           ./occ config:app:set --value 30 --type integer context_chat indexing_job_interval # 30 seconds
           ./occ config:app:set --value 10 --type integer context_chat crawl_job_interval # 10 seconds
+          ./occ config:app:set --value 10 --type integer context_chat action_job_interval # 10 seconds
+          ./occ config:app:set --value 10 --type integer context_chat fs_listener_job_interval # 10 seconds
           for i in {1..100}; do 
             php cron.php & # Starting with stable31 we can use -v here for better visibility
             wait
@@ -678,6 +680,8 @@ jobs:
           # Run cron in speed mode
           ./occ config:app:set --value 30 --type integer context_chat indexing_job_interval # 30 seconds
           ./occ config:app:set --value 10 --type integer context_chat crawl_job_interval # 10 seconds
+          ./occ config:app:set --value 10 --type integer context_chat action_job_interval # 10 seconds
+          ./occ config:app:set --value 10 --type integer context_chat fs_listener_job_interval # 10 seconds
           # Run normal indexing jobs which will only pick up welcome.txt etc
           for i in {1..10}; do
           php cron.php & # Starting with stable31 we can use -v here for better visibility
diff --git a/lib/BackgroundJobs/ActionJob.php b/lib/BackgroundJobs/ActionJob.php
index e3c2393d..c4f1cbe9 100644
--- a/lib/BackgroundJobs/ActionJob.php
+++ b/lib/BackgroundJobs/ActionJob.php
@@ -18,6 +18,7 @@
 use OCP\AppFramework\Utility\ITimeFactory;
 use OCP\BackgroundJob\IJobList;
 use OCP\BackgroundJob\QueuedJob;
+use OCP\IConfig;
 
 class ActionJob extends QueuedJob {
 	private const BATCH_SIZE = 1000;
@@ -30,10 +31,15 @@ public function __construct(
 		private Logger $logger,
 		private DiagnosticService $diagnosticService,
 		private IAppManager $appManager,
+		private IConfig $config,
 	) {
 		parent::__construct($timeFactory);
 	}
 
+	private function getJobInterval(): int {
+		return intval($this->config->getAppValue('context_chat', 'action_job_interval', (string)(5 * 60))); // 5 minutes
+	}
+
 	protected function run($argument): void {
 		if (!$this->appManager->isInstalled('app_api')) {
 			$this->logger->warning('ActionJob is skipped as app_api is disabled');
@@ -119,12 +125,12 @@ protected function run($argument): void {
 			}
 		} catch (\Throwable $e) {
 			// schedule in 5mins
-			$this->jobList->scheduleAfter(static::class, $this->time->getTime() + 5 * 60);
+			$this->jobList->scheduleAfter(static::class, $this->time->getTime() + $this->getJobInterval());
 			throw $e;
 		}
 
 		// schedule in 5mins
-		$this->jobList->scheduleAfter(static::class, $this->time->getTime() + 5 * 60);
+		$this->jobList->scheduleAfter(static::class, $this->time->getTime() + $this->getJobInterval());
 		$this->diagnosticService->sendJobEnd(static::class, $this->getId());
 	}
 }
diff --git a/lib/BackgroundJobs/FileSystemListenerJob.php b/lib/BackgroundJobs/FileSystemListenerJob.php
index f4ac31cf..87f7a42e 100644
--- a/lib/BackgroundJobs/FileSystemListenerJob.php
+++ b/lib/BackgroundJobs/FileSystemListenerJob.php
@@ -16,9 +16,10 @@
 use OCA\ContextChat\Type\FsEventType;
 use OCP\App\IAppManager;
 use OCP\AppFramework\Utility\ITimeFactory;
-use OCP\BackgroundJob\IJobList;
 use OCP\BackgroundJob\TimedJob;
+use OCP\DB\Exception;
 use OCP\Files\IRootFolder;
+use OCP\IConfig;
 
 class FileSystemListenerJob extends TimedJob {
 	private const BATCH_SIZE = 500;
@@ -26,16 +27,20 @@ class FileSystemListenerJob extends TimedJob {
 	public function __construct(
 		ITimeFactory $timeFactory,
 		private FsEventMapper $fsEventMapper,
-		private IJobList $jobList,
 		private Logger $logger,
 		private DiagnosticService $diagnosticService,
 		private IAppManager $appManager,
 		private FsEventService $fsEventService,
 		private IRootFolder $rootFolder,
+		private IConfig $config,
 	) {
 		parent::__construct($timeFactory);
 		$this->allowParallelRuns = false;
-		$this->setInterval(5 * 60); // 5 minutes
+		$this->setInterval($this->getJobInterval());
+	}
+
+	private function getJobInterval(): int {
+		return intval($this->config->getAppValue('context_chat', 'fs_listener_job_interval', (string)(5 * 60))); // 5 minutes
 	}
 
 	protected function run($argument): void {
@@ -46,9 +51,16 @@ protected function run($argument): void {
 
 		$this->diagnosticService->sendJobStart(static::class, $this->getId());
 		$this->diagnosticService->sendHeartbeat(static::class, $this->getId());
-		$fsEvents = $this->fsEventMapper->getFromQueue(static::BATCH_SIZE);
+		try {
+			$fsEvents = $this->fsEventMapper->getFromQueue(static::BATCH_SIZE);
+		} catch (Exception $e) {
+			$this->logger->warning('Error fetching fs events: ' . $e->getMessage(), ['exception' => $e]);
+			$this->diagnosticService->sendJobEnd(static::class, $this->getId());
+			return;
+		}
 
 		if (empty($fsEvents)) {
+			$this->diagnosticService->sendJobEnd(static::class, $this->getId());
 			return;
 		}
 
@@ -63,23 +75,31 @@ protected function run($argument): void {
 			}
 			if ($node === false) {
 				$this->logger->warning('Node with ID ' . $fsEvent->getNodeId() . ' not found for fs event "' . $fsEvent->getType() . '"');
-				$this->fsEventMapper->delete($fsEvent);
+				try {
+					$this->fsEventMapper->delete($fsEvent);
+				} catch (Exception $e) {
+					$this->logger->warning('Error deleting fs event "' . $fsEvent->getType() . '": ' . $e->getMessage(), ['exception' => $e]);
+				}
 				continue;
 			}
 
 			try {
-				switch ($fsEvent->getTypeObject()) {
-					case FsEventType::CREATE:
-						$this->fsEventService->onInsert($node);
-						break;
-					case FsEventType::ACCESS_UPDATE_DECL:
-						$this->fsEventService->onAccessUpdateDecl($node);
-						break;
+				try {
+					switch ($fsEvent->getTypeObject()) {
+						case FsEventType::CREATE:
+							$this->fsEventService->onInsert($node);
+							break;
+						case FsEventType::ACCESS_UPDATE_DECL:
+							$this->fsEventService->onAccessUpdateDecl($node);
+							break;
+					}
+					$this->diagnosticService->sendHeartbeat(static::class, $this->getId());
+				} catch (\RuntimeException $e) {
+					$this->logger->warning('Error handling fs event "' . $fsEvent->getType() . '": ' . $e->getMessage(), ['exception' => $e]);
 				}
-				$this->diagnosticService->sendHeartbeat(static::class, $this->getId());
 				$this->fsEventMapper->delete($fsEvent);
-			} catch (\RuntimeException $e) {
-				$this->logger->warning('Error handling fs event "' . $fsEvent->getType() . '": ' . $e->getMessage(), ['exception' => $e]);
+			} catch (Exception $e) {
+				$this->logger->warning('Error deleting fs event "' . $fsEvent->getType() . '": ' . $e->getMessage(), ['exception' => $e]);
 			}
 		}
 

From 461a9ffe5ffc3a45b956c3d9e665c5454ad5e0b2 Mon Sep 17 00:00:00 2001
From: Marcel Klehr <mklehr@gmx.net>
Date: Fri, 25 Jul 2025 10:08:46 +0200
Subject: [PATCH 05/30] fix(ActionJob): Improve error handling

Signed-off-by: Marcel Klehr <mklehr@gmx.net>
---
 lib/BackgroundJobs/ActionJob.php | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/lib/BackgroundJobs/ActionJob.php b/lib/BackgroundJobs/ActionJob.php
index c4f1cbe9..c12a849f 100644
--- a/lib/BackgroundJobs/ActionJob.php
+++ b/lib/BackgroundJobs/ActionJob.php
@@ -18,6 +18,7 @@
 use OCP\AppFramework\Utility\ITimeFactory;
 use OCP\BackgroundJob\IJobList;
 use OCP\BackgroundJob\QueuedJob;
+use OCP\DB\Exception;
 use OCP\IConfig;
 
 class ActionJob extends QueuedJob {
@@ -48,9 +49,16 @@ protected function run($argument): void {
 
 		$this->diagnosticService->sendJobStart(static::class, $this->getId());
 		$this->diagnosticService->sendHeartbeat(static::class, $this->getId());
-		$entities = $this->actionMapper->getFromQueue(static::BATCH_SIZE);
+		try {
+			$entities = $this->actionMapper->getFromQueue(static::BATCH_SIZE);
+		} catch (Exception $e) {
+			$this->logger->warning('Error fetching actions in action Job : ' . $e->getMessage(), ['exception' => $e]);
+			$this->diagnosticService->sendJobEnd(static::class, $this->getId());
+			return;
+		}
 
 		if (empty($entities)) {
+			$this->diagnosticService->sendJobEnd(static::class, $this->getId());
 			return;
 		}
 
@@ -124,6 +132,7 @@ protected function run($argument): void {
 				}
 			}
 		} catch (\Throwable $e) {
+			$this->logger->warning('Error in action Job : ' . $e->getMessage(), ['exception' => $e]);
 			// schedule in 5mins
 			$this->jobList->scheduleAfter(static::class, $this->time->getTime() + $this->getJobInterval());
 			throw $e;

From bdac2fb7f7dadaa09bd6791e8d1d45a2f3320f5d Mon Sep 17 00:00:00 2001
From: Marcel Klehr <mklehr@gmx.net>
Date: Fri, 25 Jul 2025 11:18:48 +0200
Subject: [PATCH 06/30] tests: Test delete listener using files:delete

Signed-off-by: Marcel Klehr <mklehr@gmx.net>
---
 .github/workflows/integration-test.yml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml
index 913202d4..d315f4d7 100644
--- a/.github/workflows/integration-test.yml
+++ b/.github/workflows/integration-test.yml
@@ -776,8 +776,10 @@ jobs:
 
       - name: Remove some files
         run: |
-          cd data/admin/files
-          rm -rf ./admin_manual
+          ./occ files:delete admin/files/admin_manual
+          # cd data/admin/files 
+          # rm -rf ./admin_manual # this + files:scan sadly doesn't work, currently see https://github.com/nextcloud/context_chat/issues/153
+          # ./occ files:scan admin # We run the scan again to check if the files are successfully removed from the vectordb
 
       - name: Run files scan
         run: |

From aabe47eb7da0b29d7e5b180ba22cfafd588b8366 Mon Sep 17 00:00:00 2001
From: Marcel Klehr <mklehr@gmx.net>
Date: Fri, 25 Jul 2025 11:18:57 +0200
Subject: [PATCH 07/30] tests: Test share listener

Signed-off-by: Marcel Klehr <mklehr@gmx.net>
---
 .github/workflows/integration-test.yml | 27 +++++++++++++++++++++-----
 1 file changed, 22 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml
index d315f4d7..cfe767ab 100644
--- a/.github/workflows/integration-test.yml
+++ b/.github/workflows/integration-test.yml
@@ -781,9 +781,11 @@ jobs:
           # rm -rf ./admin_manual # this + files:scan sadly doesn't work, currently see https://github.com/nextcloud/context_chat/issues/153
           # ./occ files:scan admin # We run the scan again to check if the files are successfully removed from the vectordb
 
-      - name: Run files scan
+      - name: Share some files
         run: |
-          ./occ files:scan admin # We run the scan again to check if the files are successfully removed from the vectordb
+          ./occ user:add --generate-password -- test
+          # share developer_manual to test user
+          curl -X POST http://localhost:8080/ocs/v2.php/apps/files_sharing/api/v1/shares?shareType=0&shareWith=test&path=admin/files/developer_manual
 
       - name: Run indexer cron
         run: |
@@ -797,9 +799,24 @@ jobs:
         run: |
           ./occ background-job:worker 'OC\TaskProcessing\SynchronousBackgroundJob' &
           ./occ background-job:worker 'OC\TaskProcessing\SynchronousBackgroundJob' &
-          ./occ context_chat:prompt admin "Which factors are taken into account for the Ethical AI Rating?"
-          ./occ context_chat:prompt admin "Welche Faktoren beeinflussen das Ethical AI Rating?"
-          ./occ context_chat:search admin "Welche Faktoren beeinflussen das Ethical AI Rating?"
+          OUT1=$(./occ context_chat:prompt admin "Which factors are taken into account for the Ethical AI Rating?")
+          OUT2=$(./occ context_chat:prompt admin "Welche Faktoren beeinflussen das Ethical AI Rating?")
+          OUT3=$(./occ context_chat:search admin "Welche Faktoren beeinflussen das Ethical AI Rating?")
+          echo "$OUT1" | grep -q -v "If all of these points are met, we give a Green label." && echo 'Admin does not see AI overview anymore' || exit 1
+          echo "$OUT2" | grep -q -v "If all of these points are met, we give a Green label." && echo 'Admin does not see AI overview anymore, after german question' || exit 1
+          echo "$OUT3" | grep -q -v "overview.rst" && echo 'Admin does not see AI overview anymore, when using search' || exit 1
+          
+          OUT1=$(./occ context_chat:prompt test "Which factors are taken into account for the Ethical AI Rating?")
+          OUT2=$(./occ context_chat:prompt test "Welche Faktoren beeinflussen das Ethical AI Rating?")
+          OUT3=$(./occ context_chat:search test "Welche Faktoren beeinflussen das Ethical AI Rating?")
+          echo "$OUT1" | grep -q -v "If all of these points are met, we give a Green label." && echo 'Test user does not see AI overview' || exit 1
+          echo "$OUT2" | grep -q -v "If all of these points are met, we give a Green label." && echo 'Test user does not see AI overview' || exit 1
+          echo "$OUT3" | grep -q -v "overview.rst" && echo 'Test user does not see AI overview' || exit 1
+          
+          OUT1=$(./occ context_chat:prompt test "How do I set my TimedJob to be time insensitive?")
+          OUT3=$(./occ context_chat:search test "How do I set my TimedJob to be time insensitive?")
+          echo "$OUT1" | grep -q "This allows the Nextcloud to delay the job until a given nightly time window" && echo 'Test user does see BackgroundJobs docs' || exit 1
+          echo "$OUT3" | grep -q "backgroundjobs.rst" && echo 'Test user does see BackgroundJobs docs' || exit 1
 
       - name: Show nextcloud logs
         if: always()

From 60b3ebdddbecbbc4119dcc2f7b3e54528ba59684 Mon Sep 17 00:00:00 2001
From: Marcel Klehr <mklehr@gmx.net>
Date: Fri, 25 Jul 2025 11:25:41 +0200
Subject: [PATCH 08/30] fix(FsEventService): Delete all files in a folder, not
 only the root ref

Signed-off-by: Marcel Klehr <mklehr@gmx.net>
---
 lib/Service/FsEventService.php | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/Service/FsEventService.php b/lib/Service/FsEventService.php
index 0863b514..d4c7653d 100644
--- a/lib/Service/FsEventService.php
+++ b/lib/Service/FsEventService.php
@@ -82,7 +82,7 @@ public function onDelete(Node $node, bool $recurse = true): void {
 			}
 
 			try {
-				$fileRef = ProviderConfigService::getSourceId($node->getId());
+				$fileRef = ProviderConfigService::getSourceId($file->getId());
 				$this->actionService->deleteSources($fileRef);
 			} catch (InvalidPathException|NotFoundException $e) {
 				$this->logger->warning($e->getMessage(), ['exception' => $e]);
@@ -107,7 +107,7 @@ public function onInsert(Node $node, bool $recurse = true, bool $update = false)
 			if (!$this->allowedMimeType($file)) {
 				continue;
 			}
-			if (!$this->allowedPath($node)) {
+			if (!$this->allowedPath($file)) {
 				continue;
 			}
 

From a51d804b33457e55622f598b90803ef578f46fbd Mon Sep 17 00:00:00 2001
From: Marcel Klehr <mklehr@gmx.net>
Date: Fri, 25 Jul 2025 11:54:53 +0200
Subject: [PATCH 09/30] tests: Test deleting files using both os and occ

Signed-off-by: Marcel Klehr <mklehr@gmx.net>
---
 .github/workflows/integration-test.yml | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml
index cfe767ab..669a8cbf 100644
--- a/.github/workflows/integration-test.yml
+++ b/.github/workflows/integration-test.yml
@@ -544,8 +544,9 @@ jobs:
         php-versions: [ '8.1' ]
         databases: [ 'pgsql' ]
         server-versions: [ 'stable30', 'stable31', 'master' ]
+        file-deletion-method: ['occ', 'os']
 
-    name: Integration test with file listener on ${{ matrix.server-versions }} php@${{ matrix.php-versions }}
+    name: Integration test with file listener on ${{ matrix.server-versions }} php@${{ matrix.php-versions }} (deleting files using ${{ matrix.file-deletion-method }})
 
     env:
       MYSQL_PORT: 4444
@@ -774,12 +775,17 @@ jobs:
           echo "$OUT2" | grep -q "If all of these points are met, we give a Green label." || exit 1
           echo "$OUT3" | grep -q "overview.rst" || exit 1
 
-      - name: Remove some files
+      - name: Remove some files using ${{ matrix.file-deletion-method }}
+        if: ${{ matrix.file-deletion-method == 'occ'}}
         run: |
           ./occ files:delete admin/files/admin_manual
-          # cd data/admin/files 
-          # rm -rf ./admin_manual # this + files:scan sadly doesn't work, currently see https://github.com/nextcloud/context_chat/issues/153
-          # ./occ files:scan admin # We run the scan again to check if the files are successfully removed from the vectordb
+
+      - name: Remove some files using ${{ matrix.file-deletion-method }}
+        if: ${{ matrix.file-deletion-method == 'os'}}
+        run: |
+          cd data/admin/files 
+          rm -rf ./admin_manual # this + files:scan sadly doesn't work, currently see https://github.com/nextcloud/context_chat/issues/153
+          ./occ files:scan admin # We run the scan again to check if the files are successfully removed from the vectordb
 
       - name: Share some files
         run: |

From c23459e689ae0c626c62cf1de1e629937e8ac447 Mon Sep 17 00:00:00 2001
From: Marcel Klehr <mklehr@gmx.net>
Date: Fri, 25 Jul 2025 11:59:16 +0200
Subject: [PATCH 10/30] fix(ActionJob): Make ActionJob a TimedJob

Signed-off-by: Marcel Klehr <mklehr@gmx.net>

# Conflicts:
#	lib/BackgroundJobs/ActionJob.php
---
 appinfo/info.xml                 |  1 +
 lib/BackgroundJobs/ActionJob.php | 11 ++++-------
 lib/Service/ActionScheduler.php  |  4 +---
 3 files changed, 6 insertions(+), 10 deletions(-)

diff --git a/appinfo/info.xml b/appinfo/info.xml
index 543cb85d..dc131af0 100644
--- a/appinfo/info.xml
+++ b/appinfo/info.xml
@@ -45,6 +45,7 @@ Refer to the [Context Chat Backend's readme](https://github.com/nextcloud/contex
     <background-jobs>
 		<job>OCA\ContextChat\BackgroundJobs\SchedulerJob</job>
 		<job>OCA\ContextChat\BackgroundJobs\FileSystemListenerJob</job>
+		<job>OCA\ContextChat\BackgroundJobs\ActionJob</job>
 		<job>OCA\ContextChat\BackgroundJobs\RotateLogsJob</job>
 	</background-jobs>
 	<commands>
diff --git a/lib/BackgroundJobs/ActionJob.php b/lib/BackgroundJobs/ActionJob.php
index c12a849f..d0f32261 100644
--- a/lib/BackgroundJobs/ActionJob.php
+++ b/lib/BackgroundJobs/ActionJob.php
@@ -17,11 +17,11 @@
 use OCP\App\IAppManager;
 use OCP\AppFramework\Utility\ITimeFactory;
 use OCP\BackgroundJob\IJobList;
-use OCP\BackgroundJob\QueuedJob;
+use OCP\BackgroundJob\TimedJob;
 use OCP\DB\Exception;
 use OCP\IConfig;
 
-class ActionJob extends QueuedJob {
+class ActionJob extends TimedJob {
 	private const BATCH_SIZE = 1000;
 
 	public function __construct(
@@ -35,6 +35,8 @@ public function __construct(
 		private IConfig $config,
 	) {
 		parent::__construct($timeFactory);
+		$this->setAllowParallelRuns(false);
+		$this->setInterval($this->getJobInterval());
 	}
 
 	private function getJobInterval(): int {
@@ -133,13 +135,8 @@ protected function run($argument): void {
 			}
 		} catch (\Throwable $e) {
 			$this->logger->warning('Error in action Job : ' . $e->getMessage(), ['exception' => $e]);
-			// schedule in 5mins
-			$this->jobList->scheduleAfter(static::class, $this->time->getTime() + $this->getJobInterval());
 			throw $e;
 		}
-
-		// schedule in 5mins
-		$this->jobList->scheduleAfter(static::class, $this->time->getTime() + $this->getJobInterval());
 		$this->diagnosticService->sendJobEnd(static::class, $this->getId());
 	}
 }
diff --git a/lib/Service/ActionScheduler.php b/lib/Service/ActionScheduler.php
index 308a7360..cc63f8eb 100644
--- a/lib/Service/ActionScheduler.php
+++ b/lib/Service/ActionScheduler.php
@@ -40,9 +40,7 @@ private function scheduleAction(string $type, string $payload): void {
 		// do not catch DB exceptions
 		$this->actionMapper->insertIntoQueue($item);
 
-		if (!$this->jobList->has(ActionJob::class, null)) {
-			$this->jobList->add(ActionJob::class, null);
-		}
+		// no need to schedule job, it's a timed job
 	}
 
 	/**

From 8dc70f24175b0f6ae795536ba7c55eeff463d64c Mon Sep 17 00:00:00 2001
From: Marcel Klehr <mklehr@gmx.net>
Date: Fri, 25 Jul 2025 12:31:52 +0200
Subject: [PATCH 11/30] fix(*Job): Add a try...finally to all jobs

Signed-off-by: Marcel Klehr <mklehr@gmx.net>
---
 lib/BackgroundJobs/ActionJob.php             |  27 +++--
 lib/BackgroundJobs/FileSystemListenerJob.php |  86 ++++++++--------
 lib/BackgroundJobs/IndexerJob.php            |  49 ++++-----
 lib/BackgroundJobs/StorageCrawlJob.php       |  67 ++++++------
 lib/BackgroundJobs/SubmitContentJob.php      | 101 ++++++++++---------
 lib/Service/ActionScheduler.php              |   1 -
 6 files changed, 172 insertions(+), 159 deletions(-)

diff --git a/lib/BackgroundJobs/ActionJob.php b/lib/BackgroundJobs/ActionJob.php
index d0f32261..06685e9e 100644
--- a/lib/BackgroundJobs/ActionJob.php
+++ b/lib/BackgroundJobs/ActionJob.php
@@ -49,22 +49,20 @@ protected function run($argument): void {
 			return;
 		}
 
-		$this->diagnosticService->sendJobStart(static::class, $this->getId());
-		$this->diagnosticService->sendHeartbeat(static::class, $this->getId());
 		try {
-			$entities = $this->actionMapper->getFromQueue(static::BATCH_SIZE);
-		} catch (Exception $e) {
-			$this->logger->warning('Error fetching actions in action Job : ' . $e->getMessage(), ['exception' => $e]);
-			$this->diagnosticService->sendJobEnd(static::class, $this->getId());
-			return;
-		}
+			$this->diagnosticService->sendJobStart(static::class, $this->getId());
+			$this->diagnosticService->sendHeartbeat(static::class, $this->getId());
+			try {
+				$entities = $this->actionMapper->getFromQueue(static::BATCH_SIZE);
+			} catch (Exception $e) {
+				$this->logger->warning('Error fetching actions in action Job : ' . $e->getMessage(), ['exception' => $e]);
+				return;
+			}
 
-		if (empty($entities)) {
-			$this->diagnosticService->sendJobEnd(static::class, $this->getId());
-			return;
-		}
+			if (empty($entities)) {
+				return;
+			}
 
-		try {
 			foreach ($entities as $entity) {
 				$this->diagnosticService->sendHeartbeat(static::class, $this->getId());
 
@@ -136,7 +134,8 @@ protected function run($argument): void {
 		} catch (\Throwable $e) {
 			$this->logger->warning('Error in action Job : ' . $e->getMessage(), ['exception' => $e]);
 			throw $e;
+		} finally {
+			$this->diagnosticService->sendJobEnd(static::class, $this->getId());
 		}
-		$this->diagnosticService->sendJobEnd(static::class, $this->getId());
 	}
 }
diff --git a/lib/BackgroundJobs/FileSystemListenerJob.php b/lib/BackgroundJobs/FileSystemListenerJob.php
index 87f7a42e..85631eb8 100644
--- a/lib/BackgroundJobs/FileSystemListenerJob.php
+++ b/lib/BackgroundJobs/FileSystemListenerJob.php
@@ -49,60 +49,60 @@ protected function run($argument): void {
 			return;
 		}
 
-		$this->diagnosticService->sendJobStart(static::class, $this->getId());
-		$this->diagnosticService->sendHeartbeat(static::class, $this->getId());
 		try {
-			$fsEvents = $this->fsEventMapper->getFromQueue(static::BATCH_SIZE);
-		} catch (Exception $e) {
-			$this->logger->warning('Error fetching fs events: ' . $e->getMessage(), ['exception' => $e]);
-			$this->diagnosticService->sendJobEnd(static::class, $this->getId());
-			return;
-		}
-
-		if (empty($fsEvents)) {
-			$this->diagnosticService->sendJobEnd(static::class, $this->getId());
-			return;
-		}
-
-		foreach ($fsEvents as $fsEvent) {
+			$this->diagnosticService->sendJobStart(static::class, $this->getId());
 			$this->diagnosticService->sendHeartbeat(static::class, $this->getId());
-
 			try {
-				$node = current($this->rootFolder->getUserFolder($fsEvent->getUserId())->getById($fsEvent->getNodeId()));
-			} catch (\Exception $e) {
-				$this->logger->warning('Error retrieving node for fs event "' . $fsEvent->getType() . '": ' . $e->getMessage(), ['exception' => $e]);
-				$node = false;
+				$fsEvents = $this->fsEventMapper->getFromQueue(static::BATCH_SIZE);
+			} catch (Exception $e) {
+				$this->logger->warning('Error fetching fs events: ' . $e->getMessage(), ['exception' => $e]);
+				return;
 			}
-			if ($node === false) {
-				$this->logger->warning('Node with ID ' . $fsEvent->getNodeId() . ' not found for fs event "' . $fsEvent->getType() . '"');
+
+			if (empty($fsEvents)) {
+				return;
+			}
+
+			foreach ($fsEvents as $fsEvent) {
+				$this->diagnosticService->sendHeartbeat(static::class, $this->getId());
+
 				try {
-					$this->fsEventMapper->delete($fsEvent);
-				} catch (Exception $e) {
-					$this->logger->warning('Error deleting fs event "' . $fsEvent->getType() . '": ' . $e->getMessage(), ['exception' => $e]);
+					$node = current($this->rootFolder->getUserFolder($fsEvent->getUserId())->getById($fsEvent->getNodeId()));
+				} catch (\Exception $e) {
+					$this->logger->warning('Error retrieving node for fs event "' . $fsEvent->getType() . '": ' . $e->getMessage(), ['exception' => $e]);
+					$node = false;
+				}
+				if ($node === false) {
+					$this->logger->warning('Node with ID ' . $fsEvent->getNodeId() . ' not found for fs event "' . $fsEvent->getType() . '"');
+					try {
+						$this->fsEventMapper->delete($fsEvent);
+					} catch (Exception $e) {
+						$this->logger->warning('Error deleting fs event "' . $fsEvent->getType() . '": ' . $e->getMessage(), ['exception' => $e]);
+					}
+					continue;
 				}
-				continue;
-			}
 
-			try {
 				try {
-					switch ($fsEvent->getTypeObject()) {
-						case FsEventType::CREATE:
-							$this->fsEventService->onInsert($node);
-							break;
-						case FsEventType::ACCESS_UPDATE_DECL:
-							$this->fsEventService->onAccessUpdateDecl($node);
-							break;
+					try {
+						switch ($fsEvent->getTypeObject()) {
+							case FsEventType::CREATE:
+								$this->fsEventService->onInsert($node);
+								break;
+							case FsEventType::ACCESS_UPDATE_DECL:
+								$this->fsEventService->onAccessUpdateDecl($node);
+								break;
+						}
+						$this->diagnosticService->sendHeartbeat(static::class, $this->getId());
+					} catch (\RuntimeException $e) {
+						$this->logger->warning('Error handling fs event "' . $fsEvent->getType() . '": ' . $e->getMessage(), ['exception' => $e]);
 					}
-					$this->diagnosticService->sendHeartbeat(static::class, $this->getId());
-				} catch (\RuntimeException $e) {
-					$this->logger->warning('Error handling fs event "' . $fsEvent->getType() . '": ' . $e->getMessage(), ['exception' => $e]);
+					$this->fsEventMapper->delete($fsEvent);
+				} catch (Exception $e) {
+					$this->logger->warning('Error deleting fs event "' . $fsEvent->getType() . '": ' . $e->getMessage(), ['exception' => $e]);
 				}
-				$this->fsEventMapper->delete($fsEvent);
-			} catch (Exception $e) {
-				$this->logger->warning('Error deleting fs event "' . $fsEvent->getType() . '": ' . $e->getMessage(), ['exception' => $e]);
 			}
+		} finally {
+			$this->diagnosticService->sendJobEnd(static::class, $this->getId());
 		}
-
-		$this->diagnosticService->sendJobEnd(static::class, $this->getId());
 	}
 }
diff --git a/lib/BackgroundJobs/IndexerJob.php b/lib/BackgroundJobs/IndexerJob.php
index d439ba1e..f93dd226 100644
--- a/lib/BackgroundJobs/IndexerJob.php
+++ b/lib/BackgroundJobs/IndexerJob.php
@@ -103,31 +103,31 @@ public function run($argument): void {
 			return;
 		}
 
-		$this->diagnosticService->sendJobStart(static::class, $this->getId());
-		$this->diagnosticService->sendHeartbeat(static::class, $this->getId());
+		try {
+			$this->diagnosticService->sendJobStart(static::class, $this->getId());
+			$this->diagnosticService->sendHeartbeat(static::class, $this->getId());
 
-		// Setup Filesystem for a users that can access this mount
-		$mounts = array_values(array_filter($this->userMountCache->getMountsForStorageId($this->storageId), function (ICachedMountInfo $mount) {
-			return $mount->getRootId() === $this->rootId;
-		}));
+			// Setup Filesystem for a users that can access this mount
+			$mounts = array_values(array_filter($this->userMountCache->getMountsForStorageId($this->storageId), function (ICachedMountInfo $mount) {
+				return $mount->getRootId() === $this->rootId;
+			}));
 
-		if (count($mounts) > 0) {
-			\OC_Util::setupFS($mounts[0]->getUser()->getUID());
-		}
+			if (count($mounts) > 0) {
+				\OC_Util::setupFS($mounts[0]->getUser()->getUID());
+			}
 
-		try {
-			$this->logger->debug('[IndexerJob] Running indexing', ['storageId' => $this->storageId, 'rootId' => $this->rootId]);
-			$this->index($files);
-		} catch (\RuntimeException $e) {
-			$this->logger->warning('[IndexerJob] Temporary problem with indexing', ['exception' => $e, 'storageId' => $this->storageId, 'rootId' => $this->rootId]);
-		} catch (\ErrorException $e) {
-			$this->logger->warning('[IndexerJob]  Problem with indexing', ['exception' => $e, 'storageId' => $this->storageId, 'rootId' => $this->rootId]);
-			$this->logger->info('[IndexerJob] Removing ' . static::class . ' with argument ' . var_export($argument, true) . 'from oc_jobs');
-			$this->jobList->remove(static::class, $argument);
-			throw $e;
-		}
+			try {
+				$this->logger->debug('[IndexerJob] Running indexing', ['storageId' => $this->storageId, 'rootId' => $this->rootId]);
+				$this->index($files);
+			} catch (\RuntimeException $e) {
+				$this->logger->warning('[IndexerJob] Temporary problem with indexing', ['exception' => $e, 'storageId' => $this->storageId, 'rootId' => $this->rootId]);
+			} catch (\ErrorException $e) {
+				$this->logger->warning('[IndexerJob]  Problem with indexing', ['exception' => $e, 'storageId' => $this->storageId, 'rootId' => $this->rootId]);
+				$this->logger->info('[IndexerJob] Removing ' . static::class . ' with argument ' . var_export($argument, true) . 'from oc_jobs');
+				$this->jobList->remove(static::class, $argument);
+				throw $e;
+			}
 
-		try {
 			// If there is at least one file left in the queue, reschedule this job
 			$files = $this->queue->getFromQueue($this->storageId, $this->rootId, 1);
 			$indexerJobCount = $this->getJobCount(IndexerJob::class);
@@ -138,13 +138,14 @@ public function run($argument): void {
 				$this->setInitialIndexCompletion();
 			} elseif (count($files) === 0) {
 				$this->logger->debug('[IndexerJob] No files left in queue, but we keep the job around to wait for potential StorageCrawlJob instances to finish');
-
 			}
 		} catch (Exception $e) {
 			$this->logger->error('[IndexerJob] Cannot retrieve items from queue', ['exception' => $e, 'storageId' => $this->storageId, 'rootId' => $this->rootId]);
-			return;
+		} catch (\Throwable $e) {
+			$this->logger->error('[IndexerJob] Failure during job run', ['exception' => $e, 'storageId' => $this->storageId, 'rootId' => $this->rootId]);
+		} finally {
+			$this->diagnosticService->sendJobEnd(static::class, $this->getId());
 		}
-		$this->diagnosticService->sendJobEnd(static::class, $this->getId());
 	}
 
 	protected function getBatchSize(): int {
diff --git a/lib/BackgroundJobs/StorageCrawlJob.php b/lib/BackgroundJobs/StorageCrawlJob.php
index 617c40c1..a10577a9 100644
--- a/lib/BackgroundJobs/StorageCrawlJob.php
+++ b/lib/BackgroundJobs/StorageCrawlJob.php
@@ -51,45 +51,48 @@ protected function run($argument): void {
 		// Remove current iteration
 		$this->jobList->remove(self::class, $argument);
 
-		$this->diagnosticService->sendJobStart(static::class, $this->getId());
-		$this->diagnosticService->sendHeartbeat(static::class, $this->getId());
-
-		$i = 0;
-		foreach ($this->storageService->getFilesInMount($storageId, $overrideRoot ?? $rootId, $lastFileId, self::BATCH_SIZE) as $fileId) {
-			$queueFile = new QueueFile();
-			$queueFile->setStorageId($storageId);
-			$queueFile->setRootId($rootId);
-			$queueFile->setFileId($fileId);
-			$queueFile->setUpdate(false);
+		try {
+			$this->diagnosticService->sendJobStart(static::class, $this->getId());
 			$this->diagnosticService->sendHeartbeat(static::class, $this->getId());
-			try {
-				$this->queue->insertIntoQueue($queueFile);
-			} catch (Exception $e) {
-				$this->logger->error('[StorageCrawlJob] Failed to add file to queue', [
-					'fileId' => $fileId,
-					'exception' => $e,
+
+			$i = 0;
+			foreach ($this->storageService->getFilesInMount($storageId, $overrideRoot ?? $rootId, $lastFileId, self::BATCH_SIZE) as $fileId) {
+				$queueFile = new QueueFile();
+				$queueFile->setStorageId($storageId);
+				$queueFile->setRootId($rootId);
+				$queueFile->setFileId($fileId);
+				$queueFile->setUpdate(false);
+				$this->diagnosticService->sendHeartbeat(static::class, $this->getId());
+				try {
+					$this->queue->insertIntoQueue($queueFile);
+				} catch (Exception $e) {
+					$this->logger->error('[StorageCrawlJob] Failed to add file to queue', [
+						'fileId' => $fileId,
+						'exception' => $e,
+						'storage_id' => $storageId,
+						'root_id' => $rootId,
+						'override_root' => $overrideRoot,
+						'last_file_id' => $lastFileId
+					]);
+				}
+				$i++;
+			}
+
+			if ($i > 0) {
+				// Schedule next iteration after 5 minutes
+				$this->jobList->scheduleAfter(self::class, $this->time->getTime() + $this->getJobInterval(), [
 					'storage_id' => $storageId,
 					'root_id' => $rootId,
 					'override_root' => $overrideRoot,
-					'last_file_id' => $lastFileId
+					'last_file_id' => $queueFile->getFileId(),
 				]);
-			}
-			$i++;
-		}
 
-		if ($i > 0) {
-			// Schedule next iteration after 5 minutes
-			$this->jobList->scheduleAfter(self::class, $this->time->getTime() + $this->getJobInterval(), [
-				'storage_id' => $storageId,
-				'root_id' => $rootId,
-				'override_root' => $overrideRoot,
-				'last_file_id' => $queueFile->getFileId(),
-			]);
-
-			// the last job to set this value will win
-			$this->appConfig->setValueInt(Application::APP_ID, 'last_indexed_file_id', $queueFile->getFileId());
+				// the last job to set this value will win
+				$this->appConfig->setValueInt(Application::APP_ID, 'last_indexed_file_id', $queueFile->getFileId());
+			}
+		} finally {
+			$this->diagnosticService->sendJobEnd(static::class, $this->getId());
 		}
-		$this->diagnosticService->sendJobEnd(static::class, $this->getId());
 	}
 
 	protected function getJobInterval(): int {
diff --git a/lib/BackgroundJobs/SubmitContentJob.php b/lib/BackgroundJobs/SubmitContentJob.php
index 3dc86491..4bb195c5 100644
--- a/lib/BackgroundJobs/SubmitContentJob.php
+++ b/lib/BackgroundJobs/SubmitContentJob.php
@@ -14,6 +14,7 @@
 use OCA\ContextChat\Db\QueueContentItemMapper;
 use OCA\ContextChat\Exceptions\RetryIndexException;
 use OCA\ContextChat\Logger;
+use OCA\ContextChat\Service\DiagnosticService;
 use OCA\ContextChat\Service\LangRopeService;
 use OCA\ContextChat\Service\ProviderConfigService;
 use OCA\ContextChat\Type\Source;
@@ -21,6 +22,7 @@
 use OCP\AppFramework\Utility\ITimeFactory;
 use OCP\BackgroundJob\IJobList;
 use OCP\BackgroundJob\QueuedJob;
+use OCP\DB\Exception;
 
 class SubmitContentJob extends QueuedJob {
 	private const BATCH_SIZE = 20;
@@ -32,6 +34,7 @@ public function __construct(
 		private IJobList $jobList,
 		private Logger $logger,
 		private IAppConfig $appConfig,
+		private DiagnosticService $diagnosticService,
 	) {
 		parent::__construct($timeFactory);
 	}
@@ -48,56 +51,64 @@ protected function run($argument): void {
 			return;
 		}
 
-		$sources = array_map(function (QueueContentItem $item) use ($maxSize) {
-			$contentSize = mb_strlen($item->getContent(), '8bit');
-			if ($contentSize > $maxSize) {
-				$this->logger->warning('[SubmitContentJob] Content too large to index', [
-					'contentSize' => $contentSize,
-					'maxSize' => $maxSize,
-					'itemId' => $item->getItemId(),
-					'providerId' => $item->getProviderId(),
-					'appId' => $item->getAppId(),
+		try {
+			$this->diagnosticService->sendJobStart(static::class, $this->getId());
+			$this->diagnosticService->sendHeartbeat(static::class, $this->getId());
+
+			$sources = array_map(function (QueueContentItem $item) use ($maxSize) {
+				$contentSize = mb_strlen($item->getContent(), '8bit');
+				if ($contentSize > $maxSize) {
+					$this->logger->warning('[SubmitContentJob] Content too large to index', [
+						'contentSize' => $contentSize,
+						'maxSize' => $maxSize,
+						'itemId' => $item->getItemId(),
+						'providerId' => $item->getProviderId(),
+						'appId' => $item->getAppId(),
+					]);
+					return null;
+				}
+
+				$providerKey = ProviderConfigService::getConfigKey($item->getAppId(), $item->getProviderId());
+				$sourceId = ProviderConfigService::getSourceId($item->getItemId(), $providerKey);
+				return new Source(
+					explode(',', $item->getUsers()),
+					$sourceId,
+					$item->getTitle(),
+					$item->getContent(),
+					$item->getLastModified()->getTimeStamp(),
+					$item->getDocumentType(),
+					$providerKey,
+				);
+			}, $entities);
+			$sources = array_filter($sources);
+
+			try {
+				$loadSourcesResult = $this->service->indexSources($sources);
+				$this->logger->info('[SubmitContentJob] Indexed sources for providers', [
+					'count' => count($loadSourcesResult['loaded_sources']),
+					'loaded_sources' => $loadSourcesResult['loaded_sources'],
+					'sources_to_retry' => $loadSourcesResult['sources_to_retry'],
 				]);
-				return null;
+			} catch (RetryIndexException $e) {
+				$this->logger->debug('[SubmitContentJob] At least one source is already being processed from another request, trying again soon', ['exception' => $e]);
+				return;
 			}
 
-			$providerKey = ProviderConfigService::getConfigKey($item->getAppId(), $item->getProviderId());
-			$sourceId = ProviderConfigService::getSourceId($item->getItemId(), $providerKey);
-			return new Source(
-				explode(',', $item->getUsers()),
-				$sourceId,
-				$item->getTitle(),
-				$item->getContent(),
-				$item->getLastModified()->getTimeStamp(),
-				$item->getDocumentType(),
-				$providerKey,
-			);
-		}, $entities);
-		$sources = array_filter($sources);
-
-		try {
-			$loadSourcesResult = $this->service->indexSources($sources);
-			$this->logger->info('[SubmitContentJob] Indexed sources for providers', [
-				'count' => count($loadSourcesResult['loaded_sources']),
-				'loaded_sources' => $loadSourcesResult['loaded_sources'],
-				'sources_to_retry' => $loadSourcesResult['sources_to_retry'],
-			]);
-		} catch (RetryIndexException $e) {
-			$this->logger->debug('[SubmitContentJob] At least one source is already being processed from another request, trying again soon', ['exception' => $e]);
+			foreach ($entities as $entity) {
+				$providerKey = ProviderConfigService::getConfigKey($entity->getAppId(), $entity->getProviderId());
+				$sourceId = ProviderConfigService::getSourceId($entity->getItemId(), $providerKey);
+				if (!in_array($sourceId, $loadSourcesResult['sources_to_retry'], true)) {
+					try {
+						$this->mapper->removeFromQueue($entity);
+					} catch (Exception $e) {
+						$this->logger->error('[SubmitContentJob] Failed to remove item from queue', ['exception' => $e]);
+					}
+				}
+			}
+		} finally {
+			$this->diagnosticService->sendJobEnd(static::class, $this->getId());
 			// schedule in 5mins
 			$this->jobList->scheduleAfter(static::class, $this->time->getTime() + 5 * 60);
-			return;
-		}
-
-		foreach ($entities as $entity) {
-			$providerKey = ProviderConfigService::getConfigKey($entity->getAppId(), $entity->getProviderId());
-			$sourceId = ProviderConfigService::getSourceId($entity->getItemId(), $providerKey);
-			if (!in_array($sourceId, $loadSourcesResult['sources_to_retry'])) {
-				$this->mapper->removeFromQueue($entity);
-			}
 		}
-
-		// schedule in 5mins
-		$this->jobList->scheduleAfter(static::class, $this->time->getTime() + 5 * 60);
 	}
 }
diff --git a/lib/Service/ActionScheduler.php b/lib/Service/ActionScheduler.php
index cc63f8eb..c99cafdb 100644
--- a/lib/Service/ActionScheduler.php
+++ b/lib/Service/ActionScheduler.php
@@ -9,7 +9,6 @@
 
 namespace OCA\ContextChat\Service;
 
-use OCA\ContextChat\BackgroundJobs\ActionJob;
 use OCA\ContextChat\Db\QueueAction;
 use OCA\ContextChat\Db\QueueActionMapper;
 use OCA\ContextChat\Logger;

From d8a31e95c8a8b167b211a81c7e87a6d3caec452b Mon Sep 17 00:00:00 2001
From: Marcel Klehr <mklehr@gmx.net>
Date: Fri, 25 Jul 2025 13:04:26 +0200
Subject: [PATCH 12/30] fix(SubmitContentJob): Improve error handling

Signed-off-by: Marcel Klehr <mklehr@gmx.net>
---
 lib/BackgroundJobs/SubmitContentJob.php | 24 ++++++++++++++++++++----
 lib/Service/LangRopeService.php         |  2 +-
 2 files changed, 21 insertions(+), 5 deletions(-)

diff --git a/lib/BackgroundJobs/SubmitContentJob.php b/lib/BackgroundJobs/SubmitContentJob.php
index 4bb195c5..7f0570f6 100644
--- a/lib/BackgroundJobs/SubmitContentJob.php
+++ b/lib/BackgroundJobs/SubmitContentJob.php
@@ -18,6 +18,7 @@
 use OCA\ContextChat\Service\LangRopeService;
 use OCA\ContextChat\Service\ProviderConfigService;
 use OCA\ContextChat\Type\Source;
+use OCP\App\IAppManager;
 use OCP\AppFramework\Services\IAppConfig;
 use OCP\AppFramework\Utility\ITimeFactory;
 use OCP\BackgroundJob\IJobList;
@@ -35,6 +36,7 @@ public function __construct(
 		private Logger $logger,
 		private IAppConfig $appConfig,
 		private DiagnosticService $diagnosticService,
+		private IAppManager $appManager,
 	) {
 		parent::__construct($timeFactory);
 	}
@@ -44,17 +46,29 @@ public function __construct(
 	 * @return void
 	 */
 	protected function run($argument): void {
-		$entities = $this->mapper->getFromQueue(static::BATCH_SIZE);
-		$maxSize = $this->appConfig->getAppValueInt('indexing_max_size', Application::CC_MAX_SIZE);
-
-		if (empty($entities)) {
+		if (!$this->appManager->isInstalled('app_api')) {
+			$this->logger->warning('SubmitContentJob is skipped as app_api is disabled');
 			return;
 		}
 
 		try {
+
 			$this->diagnosticService->sendJobStart(static::class, $this->getId());
 			$this->diagnosticService->sendHeartbeat(static::class, $this->getId());
 
+			try {
+				$entities = $this->mapper->getFromQueue(static::BATCH_SIZE);
+			} catch (Exception $e) {
+				$this->logger->warning('Error fetching queue content items in SubmitContentJob : ' . $e->getMessage(), ['exception' => $e]);
+				return;
+			}
+
+			$maxSize = $this->appConfig->getAppValueInt('indexing_max_size', Application::CC_MAX_SIZE);
+
+			if (empty($entities)) {
+				return;
+			}
+
 			$sources = array_map(function (QueueContentItem $item) use ($maxSize) {
 				$contentSize = mb_strlen($item->getContent(), '8bit');
 				if ($contentSize > $maxSize) {
@@ -105,6 +119,8 @@ protected function run($argument): void {
 					}
 				}
 			}
+		} catch (\Throwable $e) {
+			$this->logger->error('[SubmitContentJob] Error in SubmitContentJob : ' . $e->getMessage(), ['exception' => $e]);
 		} finally {
 			$this->diagnosticService->sendJobEnd(static::class, $this->getId());
 			// schedule in 5mins
diff --git a/lib/Service/LangRopeService.php b/lib/Service/LangRopeService.php
index f4d8da55..31851d5f 100644
--- a/lib/Service/LangRopeService.php
+++ b/lib/Service/LangRopeService.php
@@ -274,7 +274,7 @@ public function updateAccessDeclarative(array $userIds, string $sourceId): void
 	/**
 	 * @param Source[] $sources
 	 * @return array{loaded_sources: array<string>, sources_to_retry: array<string>}
-	 * @throws RuntimeException
+	 * @throws RuntimeException|RetryIndexException
 	 */
 	public function indexSources(array $sources): array {
 		if (count($sources) === 0) {

From f20798a5bad33f6ab5cfd904a2efbe0a73f88874 Mon Sep 17 00:00:00 2001
From: Marcel Klehr <mklehr@gmx.net>
Date: Fri, 25 Jul 2025 13:07:39 +0200
Subject: [PATCH 13/30] tests: Fix auth for sharing files

Signed-off-by: Marcel Klehr <mklehr@gmx.net>
---
 .github/workflows/integration-test.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml
index 669a8cbf..c35e0179 100644
--- a/.github/workflows/integration-test.yml
+++ b/.github/workflows/integration-test.yml
@@ -789,9 +789,9 @@ jobs:
 
       - name: Share some files
         run: |
-          ./occ user:add --generate-password -- test
+          OC_PASS=test ./occ user:add --password-from-env -- test
           # share developer_manual to test user
-          curl -X POST http://localhost:8080/ocs/v2.php/apps/files_sharing/api/v1/shares?shareType=0&shareWith=test&path=admin/files/developer_manual
+          curl -X POST -u 'test:test' http://localhost:8080/ocs/v2.php/apps/files_sharing/api/v1/shares?shareType=0&shareWith=test&path=admin/files/developer_manual
 
       - name: Run indexer cron
         run: |

From e8b79a3c05fbf8a3021fa68681763e99f0e4ac9d Mon Sep 17 00:00:00 2001
From: Marcel Klehr <mklehr@gmx.net>
Date: Fri, 25 Jul 2025 14:10:21 +0200
Subject: [PATCH 14/30] tests: Reduce matrix size

Signed-off-by: Marcel Klehr <mklehr@gmx.net>
---
 .github/workflows/integration-test.yml | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml
index c35e0179..9b2ed22e 100644
--- a/.github/workflows/integration-test.yml
+++ b/.github/workflows/integration-test.yml
@@ -544,7 +544,25 @@ jobs:
         php-versions: [ '8.1' ]
         databases: [ 'pgsql' ]
         server-versions: [ 'stable30', 'stable31', 'master' ]
-        file-deletion-method: ['occ', 'os']
+        file-deletion-method: [ 'occ', 'os' ]
+        include:
+          - server-versions: stable30
+            file-deletion-method: occ
+            database: pgsql
+            php-versions: 8.1
+          - server-versions: stable31
+            file-deletion-method: occ
+            database: pgsql
+            php-versions: 8.1
+          - server-version: master
+            file-deletion-method: occ
+            database: pgsql
+            php-versions: 8.1
+          - server-version: master
+            file-deletion-method: os
+            database: pgsql
+            php-versions: 8.1
+
 
     name: Integration test with file listener on ${{ matrix.server-versions }} php@${{ matrix.php-versions }} (deleting files using ${{ matrix.file-deletion-method }})
 

From 90c4dc22a7640059cdfae44add5b40c0c4c6deff Mon Sep 17 00:00:00 2001
From: Marcel Klehr <mklehr@gmx.net>
Date: Fri, 25 Jul 2025 14:10:35 +0200
Subject: [PATCH 15/30] tests: Fix OCS-APIRequest

Signed-off-by: Marcel Klehr <mklehr@gmx.net>
---
 .github/workflows/integration-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml
index 9b2ed22e..5f41f168 100644
--- a/.github/workflows/integration-test.yml
+++ b/.github/workflows/integration-test.yml
@@ -809,7 +809,7 @@ jobs:
         run: |
           OC_PASS=test ./occ user:add --password-from-env -- test
           # share developer_manual to test user
-          curl -X POST -u 'test:test' http://localhost:8080/ocs/v2.php/apps/files_sharing/api/v1/shares?shareType=0&shareWith=test&path=admin/files/developer_manual
+          curl -X POST -u 'test:test' -H 'OCS-APIRequest: true' http://localhost:8080/ocs/v2.php/apps/files_sharing/api/v1/shares?shareType=0&shareWith=test&path=admin/files/developer_manual
 
       - name: Run indexer cron
         run: |

From f7b00b8d8a3ebef3980f2b0b27f0c4bcc1089ae7 Mon Sep 17 00:00:00 2001
From: Marcel Klehr <mklehr@gmx.net>
Date: Fri, 25 Jul 2025 14:16:14 +0200
Subject: [PATCH 16/30] tests: Reduce matrix size

Signed-off-by: Marcel Klehr <mklehr@gmx.net>
---
 .github/workflows/integration-test.yml | 12 ++----------
 1 file changed, 2 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml
index 5f41f168..45fc22b9 100644
--- a/.github/workflows/integration-test.yml
+++ b/.github/workflows/integration-test.yml
@@ -545,20 +545,12 @@ jobs:
         databases: [ 'pgsql' ]
         server-versions: [ 'stable30', 'stable31', 'master' ]
         file-deletion-method: [ 'occ', 'os' ]
-        include:
+        exclude:
           - server-versions: stable30
-            file-deletion-method: occ
+            file-deletion-method: os
             database: pgsql
             php-versions: 8.1
           - server-versions: stable31
-            file-deletion-method: occ
-            database: pgsql
-            php-versions: 8.1
-          - server-version: master
-            file-deletion-method: occ
-            database: pgsql
-            php-versions: 8.1
-          - server-version: master
             file-deletion-method: os
             database: pgsql
             php-versions: 8.1

From 4e237a64e4a33403b08b21800f1523a16dd2b33e Mon Sep 17 00:00:00 2001
From: Marcel Klehr <mklehr@gmx.net>
Date: Fri, 25 Jul 2025 15:09:19 +0200
Subject: [PATCH 17/30] tests: Fix share API call

Signed-off-by: Marcel Klehr <mklehr@gmx.net>
---
 .github/workflows/integration-test.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml
index 45fc22b9..31970f0c 100644
--- a/.github/workflows/integration-test.yml
+++ b/.github/workflows/integration-test.yml
@@ -801,7 +801,7 @@ jobs:
         run: |
           OC_PASS=test ./occ user:add --password-from-env -- test
           # share developer_manual to test user
-          curl -X POST -u 'test:test' -H 'OCS-APIRequest: true' http://localhost:8080/ocs/v2.php/apps/files_sharing/api/v1/shares?shareType=0&shareWith=test&path=admin/files/developer_manual
+          curl -X POST -u 'admin:admin' -H 'OCS-APIRequest: true' 'http://localhost:8080/ocs/v2.php/apps/files_sharing/api/v1/shares?shareType=0&shareWith=test&path=developer_manual'
 
       - name: Run indexer cron
         run: |
@@ -826,7 +826,7 @@ jobs:
           OUT2=$(./occ context_chat:prompt test "Welche Faktoren beeinflussen das Ethical AI Rating?")
           OUT3=$(./occ context_chat:search test "Welche Faktoren beeinflussen das Ethical AI Rating?")
           echo "$OUT1" | grep -q -v "If all of these points are met, we give a Green label." && echo 'Test user does not see AI overview' || exit 1
-          echo "$OUT2" | grep -q -v "If all of these points are met, we give a Green label." && echo 'Test user does not see AI overview' || exit 1
+          echo "$OUT2" | grep -q -v "If all of these points are met, we give a Green label." && echo 'Test user does not see AI overview, after german question' || exit 1
           echo "$OUT3" | grep -q -v "overview.rst" && echo 'Test user does not see AI overview' || exit 1
           
           OUT1=$(./occ context_chat:prompt test "How do I set my TimedJob to be time insensitive?")

From 11118f57d1204e9c7e1a9444cfc9a031e8f52d24 Mon Sep 17 00:00:00 2001
From: Marcel Klehr <mklehr@gmx.net>
Date: Fri, 25 Jul 2025 15:32:35 +0200
Subject: [PATCH 18/30] tests: Fix typo

Signed-off-by: Marcel Klehr <mklehr@gmx.net>
---
 .github/workflows/integration-test.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml
index 31970f0c..e12d6bc9 100644
--- a/.github/workflows/integration-test.yml
+++ b/.github/workflows/integration-test.yml
@@ -548,11 +548,11 @@ jobs:
         exclude:
           - server-versions: stable30
             file-deletion-method: os
-            database: pgsql
+            databases: pgsql
             php-versions: 8.1
           - server-versions: stable31
             file-deletion-method: os
-            database: pgsql
+            databases: pgsql
             php-versions: 8.1
 
 

From 04c3e0253e663c245d83dfc04fb2d7a710bb6ee2 Mon Sep 17 00:00:00 2001
From: Marcel Klehr <mklehr@gmx.net>
Date: Fri, 25 Jul 2025 17:52:11 +0200
Subject: [PATCH 19/30] fix(tests)

Signed-off-by: Marcel Klehr <mklehr@gmx.net>
---
 .github/workflows/integration-test.yml | 56 +++++++++++++++++---------
 1 file changed, 37 insertions(+), 19 deletions(-)

diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml
index e12d6bc9..99027cfe 100644
--- a/.github/workflows/integration-test.yml
+++ b/.github/workflows/integration-test.yml
@@ -785,23 +785,24 @@ jobs:
           echo "$OUT2" | grep -q "If all of these points are met, we give a Green label." || exit 1
           echo "$OUT3" | grep -q "overview.rst" || exit 1
 
-      - name: Remove some files using ${{ matrix.file-deletion-method }}
+      - name: Remove some files using occ
         if: ${{ matrix.file-deletion-method == 'occ'}}
         run: |
           ./occ files:delete admin/files/admin_manual
 
-      - name: Remove some files using ${{ matrix.file-deletion-method }}
+      - name: Remove some files using os
         if: ${{ matrix.file-deletion-method == 'os'}}
         run: |
           cd data/admin/files 
           rm -rf ./admin_manual # this + files:scan sadly doesn't work, currently see https://github.com/nextcloud/context_chat/issues/153
+          cd ../../..
           ./occ files:scan admin # We run the scan again to check if the files are successfully removed from the vectordb
 
       - name: Share some files
         run: |
           OC_PASS=test ./occ user:add --password-from-env -- test
           # share developer_manual to test user
-          curl -X POST -u 'admin:admin' -H 'OCS-APIRequest: true' 'http://localhost:8080/ocs/v2.php/apps/files_sharing/api/v1/shares?shareType=0&shareWith=test&path=developer_manual'
+          curl -X POST -u 'admin:password' -H 'OCS-APIRequest: true' 'http://localhost:8080/ocs/v2.php/apps/files_sharing/api/v1/shares?shareType=0&shareWith=test&path=developer_manual'
 
       - name: Run indexer cron
         run: |
@@ -815,24 +816,41 @@ jobs:
         run: |
           ./occ background-job:worker 'OC\TaskProcessing\SynchronousBackgroundJob' &
           ./occ background-job:worker 'OC\TaskProcessing\SynchronousBackgroundJob' &
-          OUT1=$(./occ context_chat:prompt admin "Which factors are taken into account for the Ethical AI Rating?")
-          OUT2=$(./occ context_chat:prompt admin "Welche Faktoren beeinflussen das Ethical AI Rating?")
-          OUT3=$(./occ context_chat:search admin "Welche Faktoren beeinflussen das Ethical AI Rating?")
-          echo "$OUT1" | grep -q -v "If all of these points are met, we give a Green label." && echo 'Admin does not see AI overview anymore' || exit 1
-          echo "$OUT2" | grep -q -v "If all of these points are met, we give a Green label." && echo 'Admin does not see AI overview anymore, after german question' || exit 1
-          echo "$OUT3" | grep -q -v "overview.rst" && echo 'Admin does not see AI overview anymore, when using search' || exit 1
+          # Check for user admin: Should be there
+          OUT_ADMIN_ENGLISH_PROMPT_AI=$(./occ context_chat:prompt admin "Which factors are taken into account for the Ethical AI Rating?")
+          OUT_ADMIN_ENGLISH_PROMPT_AI=$(./occ context_chat:prompt admin "Welche Faktoren beeinflussen das Ethical AI Rating?")
+          OUT_ADMIN_SEARCH_AI=$(./occ context_chat:search admin "Welche Faktoren beeinflussen das Ethical AI Rating?")
+          
+          # Check for user test: Shouldn't be there
+          OUT_TEST_ENGLISH_PROMPT_AI=$(./occ context_chat:prompt test "Which factors are taken into account for the Ethical AI Rating?")
+          OUT_TEST_GERMAN_PROMPT_AI=$(./occ context_chat:prompt test "Welche Faktoren beeinflussen das Ethical AI Rating?")
+          OUT_TEST_SEARCH_AI=$(./occ context_chat:search test "Welche Faktoren beeinflussen das Ethical AI Rating?")
+          
+          # Check for user test: Should be there, because it was shared
+          OUT_TEST_PROMPT_JOBS=$(./occ context_chat:prompt test "How do I set my TimedJob to be time insensitive?")
+          OUT_TEST_SEARCH_JOBS=$(./occ context_chat:search test "How do I set my TimedJob to be time insensitive?")
+          
+          set -x
+          
+          echo "$OUT_ADMIN_ENGLISH_PROMPT_AI"
+          echo "$OUT_ADMIN_ENGLISH_PROMPT_AI"
+          echo "$OUT_ADMIN_SEARCH_AI"
+          echo "$OUT_TEST_ENGLISH_PROMPT_AI"
+          echo "$OUT_TEST_GERMAN_PROMPT_AI"
+          echo "$OUT_TEST_SEARCH_AI"
+          echo "$OUT_TEST_PROMPT_JOBS"
+          echo "$OUT_TEST_SEARCH_JOBS"
           
-          OUT1=$(./occ context_chat:prompt test "Which factors are taken into account for the Ethical AI Rating?")
-          OUT2=$(./occ context_chat:prompt test "Welche Faktoren beeinflussen das Ethical AI Rating?")
-          OUT3=$(./occ context_chat:search test "Welche Faktoren beeinflussen das Ethical AI Rating?")
-          echo "$OUT1" | grep -q -v "If all of these points are met, we give a Green label." && echo 'Test user does not see AI overview' || exit 1
-          echo "$OUT2" | grep -q -v "If all of these points are met, we give a Green label." && echo 'Test user does not see AI overview, after german question' || exit 1
-          echo "$OUT3" | grep -q -v "overview.rst" && echo 'Test user does not see AI overview' || exit 1
+          set +x
           
-          OUT1=$(./occ context_chat:prompt test "How do I set my TimedJob to be time insensitive?")
-          OUT3=$(./occ context_chat:search test "How do I set my TimedJob to be time insensitive?")
-          echo "$OUT1" | grep -q "This allows the Nextcloud to delay the job until a given nightly time window" && echo 'Test user does see BackgroundJobs docs' || exit 1
-          echo "$OUT3" | grep -q "backgroundjobs.rst" && echo 'Test user does see BackgroundJobs docs' || exit 1
+          echo "$OUT_ADMIN_ENGLISH_PROMPT_AI" | grep -q -v "If all of these points are met, we give a Green label." && echo 'Admin does not see AI overview anymore' || exit 1
+          echo "$OUT_ADMIN_ENGLISH_PROMPT_AI" | grep -q -v "If all of these points are met, we give a Green label." && echo 'Admin does not see AI overview anymore, after german question' || exit 1
+          echo "$OUT_ADMIN_SEARCH_AI" | grep -q -v "overview.rst" && echo 'Admin does not see AI overview anymore, when using search' || exit 1
+          echo "$OUT_TEST_ENGLISH_PROMPT_AI" | grep -q -v "If all of these points are met, we give a Green label." && echo 'Test user does not see AI overview' || exit 1
+          echo "$OUT_TEST_GERMAN_PROMPT_AI" | grep -q -v "If all of these points are met, we give a Green label." && echo 'Test user does not see AI overview, after german question' || exit 1
+          echo "$OUT_TEST_SEARCH_AI" | grep -q -v "overview.rst" && echo 'Test user does not see AI overview, when using search' || exit 1
+          echo "$OUT_TEST_PROMPT_JOBS" | grep -q "This allows the Nextcloud to delay the job until a given nightly time window" && echo 'Test user does see BackgroundJobs docs' || exit 1
+          echo "$OUT_TEST_SEARCH_JOBS" | grep -q "backgroundjobs.rst" && echo 'Test user does see BackgroundJobs docs' || exit 1
 
       - name: Show nextcloud logs
         if: always()

From 4d2baa495fc456bf3c89e8f27cd187df413da42e Mon Sep 17 00:00:00 2001
From: Marcel Klehr <mklehr@gmx.net>
Date: Sat, 26 Jul 2025 06:23:57 +0200
Subject: [PATCH 20/30] fix(tests)

Signed-off-by: Marcel Klehr <mklehr@gmx.net>
---
 .github/workflows/integration-test.yml       | 15 ++++++++----
 lib/BackgroundJobs/FileSystemListenerJob.php | 24 ++++++++++----------
 2 files changed, 22 insertions(+), 17 deletions(-)

diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml
index 99027cfe..b2f7dd40 100644
--- a/.github/workflows/integration-test.yml
+++ b/.github/workflows/integration-test.yml
@@ -803,6 +803,8 @@ jobs:
           OC_PASS=test ./occ user:add --password-from-env -- test
           # share developer_manual to test user
           curl -X POST -u 'admin:password' -H 'OCS-APIRequest: true' 'http://localhost:8080/ocs/v2.php/apps/files_sharing/api/v1/shares?shareType=0&shareWith=test&path=developer_manual'
+          # list files to trigger mounting of shares
+          curl -X PROPFIND -H "Depth: 0" -u 'test:test' 'http://localhost:8080/remote.php/webdav/test/'
 
       - name: Run indexer cron
         run: |
@@ -816,9 +818,12 @@ jobs:
         run: |
           ./occ background-job:worker 'OC\TaskProcessing\SynchronousBackgroundJob' &
           ./occ background-job:worker 'OC\TaskProcessing\SynchronousBackgroundJob' &
+          
+          set +e
+          
           # Check for user admin: Should be there
           OUT_ADMIN_ENGLISH_PROMPT_AI=$(./occ context_chat:prompt admin "Which factors are taken into account for the Ethical AI Rating?")
-          OUT_ADMIN_ENGLISH_PROMPT_AI=$(./occ context_chat:prompt admin "Welche Faktoren beeinflussen das Ethical AI Rating?")
+          OUT_ADMIN_GERMAN_PROMPT_AI=$(./occ context_chat:prompt admin "Welche Faktoren beeinflussen das Ethical AI Rating?")
           OUT_ADMIN_SEARCH_AI=$(./occ context_chat:search admin "Welche Faktoren beeinflussen das Ethical AI Rating?")
           
           # Check for user test: Shouldn't be there
@@ -833,7 +838,7 @@ jobs:
           set -x
           
           echo "$OUT_ADMIN_ENGLISH_PROMPT_AI"
-          echo "$OUT_ADMIN_ENGLISH_PROMPT_AI"
+          echo "$OUT_ADMIN_GERMAN_PROMPT_AI"
           echo "$OUT_ADMIN_SEARCH_AI"
           echo "$OUT_TEST_ENGLISH_PROMPT_AI"
           echo "$OUT_TEST_GERMAN_PROMPT_AI"
@@ -844,13 +849,13 @@ jobs:
           set +x
           
           echo "$OUT_ADMIN_ENGLISH_PROMPT_AI" | grep -q -v "If all of these points are met, we give a Green label." && echo 'Admin does not see AI overview anymore' || exit 1
-          echo "$OUT_ADMIN_ENGLISH_PROMPT_AI" | grep -q -v "If all of these points are met, we give a Green label." && echo 'Admin does not see AI overview anymore, after german question' || exit 1
+          echo "$OUT_ADMIN_GERMAN_PROMPT_AI" | grep -q -v "If all of these points are met, we give a Green label." && echo 'Admin does not see AI overview anymore, after german question' || exit 1
           echo "$OUT_ADMIN_SEARCH_AI" | grep -q -v "overview.rst" && echo 'Admin does not see AI overview anymore, when using search' || exit 1
           echo "$OUT_TEST_ENGLISH_PROMPT_AI" | grep -q -v "If all of these points are met, we give a Green label." && echo 'Test user does not see AI overview' || exit 1
           echo "$OUT_TEST_GERMAN_PROMPT_AI" | grep -q -v "If all of these points are met, we give a Green label." && echo 'Test user does not see AI overview, after german question' || exit 1
           echo "$OUT_TEST_SEARCH_AI" | grep -q -v "overview.rst" && echo 'Test user does not see AI overview, when using search' || exit 1
-          echo "$OUT_TEST_PROMPT_JOBS" | grep -q "This allows the Nextcloud to delay the job until a given nightly time window" && echo 'Test user does see BackgroundJobs docs' || exit 1
-          echo "$OUT_TEST_SEARCH_JOBS" | grep -q "backgroundjobs.rst" && echo 'Test user does see BackgroundJobs docs' || exit 1
+          echo "$OUT_TEST_PROMPT_JOBS" | grep -q "The ``TimedJob`` has a method ``setInterval`` where you can set the time minimum time in seconds between the jobs (from the constructor)." && echo 'Test user does see BackgroundJobs docs' || exit 1
+          echo "$OUT_TEST_SEARCH_JOBS" | grep -q "backgroundjobs.md" && echo 'Test user does see BackgroundJobs docs' || exit 1
 
       - name: Show nextcloud logs
         if: always()
diff --git a/lib/BackgroundJobs/FileSystemListenerJob.php b/lib/BackgroundJobs/FileSystemListenerJob.php
index 85631eb8..eb7d8e47 100644
--- a/lib/BackgroundJobs/FileSystemListenerJob.php
+++ b/lib/BackgroundJobs/FileSystemListenerJob.php
@@ -83,19 +83,19 @@ protected function run($argument): void {
 				}
 
 				try {
-					try {
-						switch ($fsEvent->getTypeObject()) {
-							case FsEventType::CREATE:
-								$this->fsEventService->onInsert($node);
-								break;
-							case FsEventType::ACCESS_UPDATE_DECL:
-								$this->fsEventService->onAccessUpdateDecl($node);
-								break;
-						}
-						$this->diagnosticService->sendHeartbeat(static::class, $this->getId());
-					} catch (\RuntimeException $e) {
-						$this->logger->warning('Error handling fs event "' . $fsEvent->getType() . '": ' . $e->getMessage(), ['exception' => $e]);
+					switch ($fsEvent->getTypeObject()) {
+						case FsEventType::CREATE:
+							$this->fsEventService->onInsert($node);
+							break;
+						case FsEventType::ACCESS_UPDATE_DECL:
+							$this->fsEventService->onAccessUpdateDecl($node);
+							break;
 					}
+					$this->diagnosticService->sendHeartbeat(static::class, $this->getId());
+				} catch (\Throwable $e) {
+					$this->logger->warning('Error handling fs event "' . $fsEvent->getType() . '": ' . $e->getMessage(), ['exception' => $e]);
+				}
+				try {
 					$this->fsEventMapper->delete($fsEvent);
 				} catch (Exception $e) {
 					$this->logger->warning('Error deleting fs event "' . $fsEvent->getType() . '": ' . $e->getMessage(), ['exception' => $e]);

From 03b5abc1abc031034eb9dd99b446def82a9dbfec Mon Sep 17 00:00:00 2001
From: Marcel Klehr <mklehr@gmx.net>
Date: Sat, 26 Jul 2025 11:10:01 +0200
Subject: [PATCH 21/30] test: Test with SSE enabled

Signed-off-by: Marcel Klehr <mklehr@gmx.net>
---
 .github/workflows/integration-test.yml | 33 ++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)

diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml
index b2f7dd40..b0e19ce7 100644
--- a/.github/workflows/integration-test.yml
+++ b/.github/workflows/integration-test.yml
@@ -32,6 +32,7 @@ jobs:
         php-versions: [ '8.1' ]
         databases: [ 'pgsql' ]
         server-versions: [ 'stable30', 'stable31', 'master' ]
+        encryption: ['on', 'off']
 
     name: Integration test on ${{ matrix.server-versions }} php@${{ matrix.php-versions }}
 
@@ -136,6 +137,12 @@ jobs:
       - name: Enable context_chat, app_api and testing
         run: ./occ app:enable -vvv -f ${{ env.APP_NAME }} app_api testing
 
+      - name: Enable encryption
+        if: ${{ matrix.encryption == 'on' }}
+        run: |
+          ./occ app:enable -vvv -f encryption
+          ./occ encryption:enable
+
       - name: Checkout documentation
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
@@ -293,6 +300,7 @@ jobs:
         php-versions: [ '8.1' ]
         databases: [ 'pgsql' ]
         server-versions: [ 'stable30', 'stable31', 'master' ]
+        encryption: ['on', 'off']
 
     name: Integration test with cron on ${{ matrix.server-versions }} php@${{ matrix.php-versions }}
 
@@ -397,6 +405,12 @@ jobs:
       - name: Enable app_api and testing
         run: ./occ app:enable -vvv -f app_api testing
 
+      - name: Enable encryption
+        if: ${{ matrix.encryption == 'on' }}
+        run: |
+          ./occ app:enable -vvv -f encryption
+          ./occ encryption:enable
+
       - name: Setup python 3.11
         uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
         with:
@@ -544,16 +558,29 @@ jobs:
         php-versions: [ '8.1' ]
         databases: [ 'pgsql' ]
         server-versions: [ 'stable30', 'stable31', 'master' ]
+        encryption: ['on', 'off']
         file-deletion-method: [ 'occ', 'os' ]
         exclude:
           - server-versions: stable30
             file-deletion-method: os
             databases: pgsql
             php-versions: 8.1
+            encryption: on
+          - server-versions: stable31
+            file-deletion-method: os
+            databases: pgsql
+            php-versions: 8.1
+            encryption: on
+          - server-versions: stable30
+            file-deletion-method: os
+            databases: pgsql
+            php-versions: 8.1
+            encryption: off
           - server-versions: stable31
             file-deletion-method: os
             databases: pgsql
             php-versions: 8.1
+            encryption: off
 
 
     name: Integration test with file listener on ${{ matrix.server-versions }} php@${{ matrix.php-versions }} (deleting files using ${{ matrix.file-deletion-method }})
@@ -659,6 +686,12 @@ jobs:
       - name: Enable app_api and testing
         run: ./occ app:enable -vvv -f app_api testing
 
+      - name: Enable encryption
+        if: ${{ matrix.encryption == 'on' }}
+        run: |
+          ./occ app:enable -vvv -f encryption
+          ./occ encryption:enable
+
       - name: Setup python 3.11
         uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
         with:

From edd8af16cb14b0f0e86f7503b40f5141d076ac27 Mon Sep 17 00:00:00 2001
From: Marcel Klehr <mklehr@gmx.net>
Date: Sat, 26 Jul 2025 11:40:58 +0200
Subject: [PATCH 22/30] fix(tests)

Signed-off-by: Marcel Klehr <mklehr@gmx.net>
---
 .github/workflows/integration-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml
index b0e19ce7..27a6ea24 100644
--- a/.github/workflows/integration-test.yml
+++ b/.github/workflows/integration-test.yml
@@ -887,7 +887,7 @@ jobs:
           echo "$OUT_TEST_ENGLISH_PROMPT_AI" | grep -q -v "If all of these points are met, we give a Green label." && echo 'Test user does not see AI overview' || exit 1
           echo "$OUT_TEST_GERMAN_PROMPT_AI" | grep -q -v "If all of these points are met, we give a Green label." && echo 'Test user does not see AI overview, after german question' || exit 1
           echo "$OUT_TEST_SEARCH_AI" | grep -q -v "overview.rst" && echo 'Test user does not see AI overview, when using search' || exit 1
-          echo "$OUT_TEST_PROMPT_JOBS" | grep -q "The ``TimedJob`` has a method ``setInterval`` where you can set the time minimum time in seconds between the jobs (from the constructor)." && echo 'Test user does see BackgroundJobs docs' || exit 1
+          echo "$OUT_TEST_PROMPT_JOBS" | grep -q "The ``TimedJob`` has a method ``setInterval``" && echo 'Test user does see BackgroundJobs docs' || exit 1
           echo "$OUT_TEST_SEARCH_JOBS" | grep -q "backgroundjobs.md" && echo 'Test user does see BackgroundJobs docs' || exit 1
 
       - name: Show nextcloud logs

From 114407d9c721e29ef06d5e8039390411df756cf3 Mon Sep 17 00:00:00 2001
From: Marcel Klehr <mklehr@gmx.net>
Date: Sat, 26 Jul 2025 12:44:23 +0200
Subject: [PATCH 23/30] fix: More emojis

Signed-off-by: Marcel Klehr <mklehr@gmx.net>
---
 .github/workflows/integration-test.yml | 34 ++++++++++++++------------
 1 file changed, 19 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml
index 27a6ea24..e0b35bfc 100644
--- a/.github/workflows/integration-test.yml
+++ b/.github/workflows/integration-test.yml
@@ -34,7 +34,7 @@ jobs:
         server-versions: [ 'stable30', 'stable31', 'master' ]
         encryption: ['on', 'off']
 
-    name: Integration test on ${{ matrix.server-versions }} php@${{ matrix.php-versions }}
+    name: Integration test ☁️${{ matrix.server-versions }} 🐘${{ matrix.php-versions }} 🔐:${{ matrix.encryption }}
 
     env:
       MYSQL_PORT: 4444
@@ -302,7 +302,7 @@ jobs:
         server-versions: [ 'stable30', 'stable31', 'master' ]
         encryption: ['on', 'off']
 
-    name: Integration test with cron on ${{ matrix.server-versions }} php@${{ matrix.php-versions }}
+    name: Integration test with cron ☁️${{ matrix.server-versions }} 🐘${{ matrix.php-versions }} 🔐:${{ matrix.encryption }}
 
     env:
       MYSQL_PORT: 4444
@@ -583,7 +583,7 @@ jobs:
             encryption: off
 
 
-    name: Integration test with file listener on ${{ matrix.server-versions }} php@${{ matrix.php-versions }} (deleting files using ${{ matrix.file-deletion-method }})
+    name: Integration test with file listener ☁️${{ matrix.server-versions }} 🐘${{ matrix.php-versions }} 🗑️${{ matrix.file-deletion-method }}) 🔐:${{ matrix.encryption }}
 
     env:
       MYSQL_PORT: 4444
@@ -868,27 +868,31 @@ jobs:
           OUT_TEST_PROMPT_JOBS=$(./occ context_chat:prompt test "How do I set my TimedJob to be time insensitive?")
           OUT_TEST_SEARCH_JOBS=$(./occ context_chat:search test "How do I set my TimedJob to be time insensitive?")
           
-          set -x
-          
+          echo "OUT_ADMIN_ENGLISH_PROMPT_AI"
           echo "$OUT_ADMIN_ENGLISH_PROMPT_AI"
+          echo "OUT_ADMIN_GERMAN_PROMPT_AI"
           echo "$OUT_ADMIN_GERMAN_PROMPT_AI"
+          echo "OUT_ADMIN_SEARCH_AI"
           echo "$OUT_ADMIN_SEARCH_AI"
+          echo "OUT_TEST_ENGLISH_PROMPT_AI"
           echo "$OUT_TEST_ENGLISH_PROMPT_AI"
+          echo "OUT_TEST_GERMAN_PROMPT_AI"
           echo "$OUT_TEST_GERMAN_PROMPT_AI"
+          echo "OUT_TEST_SEARCH_AI"
           echo "$OUT_TEST_SEARCH_AI"
+          echo "OUT_TEST_PROMPT_JOBS"
           echo "$OUT_TEST_PROMPT_JOBS"
+          echo "OUT_TEST_SEARCH_JOBS"
           echo "$OUT_TEST_SEARCH_JOBS"
           
-          set +x
-          
-          echo "$OUT_ADMIN_ENGLISH_PROMPT_AI" | grep -q -v "If all of these points are met, we give a Green label." && echo 'Admin does not see AI overview anymore' || exit 1
-          echo "$OUT_ADMIN_GERMAN_PROMPT_AI" | grep -q -v "If all of these points are met, we give a Green label." && echo 'Admin does not see AI overview anymore, after german question' || exit 1
-          echo "$OUT_ADMIN_SEARCH_AI" | grep -q -v "overview.rst" && echo 'Admin does not see AI overview anymore, when using search' || exit 1
-          echo "$OUT_TEST_ENGLISH_PROMPT_AI" | grep -q -v "If all of these points are met, we give a Green label." && echo 'Test user does not see AI overview' || exit 1
-          echo "$OUT_TEST_GERMAN_PROMPT_AI" | grep -q -v "If all of these points are met, we give a Green label." && echo 'Test user does not see AI overview, after german question' || exit 1
-          echo "$OUT_TEST_SEARCH_AI" | grep -q -v "overview.rst" && echo 'Test user does not see AI overview, when using search' || exit 1
-          echo "$OUT_TEST_PROMPT_JOBS" | grep -q "The ``TimedJob`` has a method ``setInterval``" && echo 'Test user does see BackgroundJobs docs' || exit 1
-          echo "$OUT_TEST_SEARCH_JOBS" | grep -q "backgroundjobs.md" && echo 'Test user does see BackgroundJobs docs' || exit 1
+          echo "$OUT_ADMIN_ENGLISH_PROMPT_AI" | grep -q -v "If all of these points are met, we give a Green label." && echo '✅ Admin does not see AI overview anymore' || exit 1
+          echo "$OUT_ADMIN_GERMAN_PROMPT_AI" | grep -q -v "If all of these points are met, we give a Green label." && echo '✅ Admin does not see AI overview anymore, after german question' || exit 1
+          echo "$OUT_ADMIN_SEARCH_AI" | grep -q -v "overview.rst" && echo '✅ Admin does not see AI overview anymore, when using search' || exit 1
+          echo "$OUT_TEST_ENGLISH_PROMPT_AI" | grep -q -v "If all of these points are met, we give a Green label." && echo '✅ Test user does not see AI overview' || exit 1
+          echo "$OUT_TEST_GERMAN_PROMPT_AI" | grep -q -v "If all of these points are met, we give a Green label." && echo '✅Test user does not see AI overview, after german question' || exit 1
+          echo "$OUT_TEST_SEARCH_AI" | grep -q -v "overview.rst" && echo '✅Test user does not see AI overview, when using search' || exit 1
+          echo "$OUT_TEST_PROMPT_JOBS" | grep -q "background jobs" && echo '✅Test user does see BackgroundJobs docs' || exit 1
+          echo "$OUT_TEST_SEARCH_JOBS" | grep -q "backgroundjobs.md" && echo '✅Test user does see BackgroundJobs docs' || exit 1
 
       - name: Show nextcloud logs
         if: always()

From d98594055e1983c23bddd7f4e8dbc2980ebfa33e Mon Sep 17 00:00:00 2001
From: Marcel Klehr <mklehr@gmx.net>
Date: Sat, 26 Jul 2025 15:06:51 +0200
Subject: [PATCH 24/30] fix: Reduce matrix

Signed-off-by: Marcel Klehr <mklehr@gmx.net>
---
 .github/workflows/integration-test.yml | 22 ++++++++++++++++++++--
 1 file changed, 20 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml
index e0b35bfc..7f23e233 100644
--- a/.github/workflows/integration-test.yml
+++ b/.github/workflows/integration-test.yml
@@ -33,6 +33,15 @@ jobs:
         databases: [ 'pgsql' ]
         server-versions: [ 'stable30', 'stable31', 'master' ]
         encryption: ['on', 'off']
+        exclude:
+          - server-versions: stable30
+            databases: pgsql
+            php-versions: 8.1
+            encryption: on
+          - server-versions: stable31
+            databases: pgsql
+            php-versions: 8.1
+            encryption: on
 
     name: Integration test ☁️${{ matrix.server-versions }} 🐘${{ matrix.php-versions }} 🔐:${{ matrix.encryption }}
 
@@ -301,6 +310,15 @@ jobs:
         databases: [ 'pgsql' ]
         server-versions: [ 'stable30', 'stable31', 'master' ]
         encryption: ['on', 'off']
+        exclude:
+          - server-versions: stable30
+            databases: pgsql
+            php-versions: 8.1
+            encryption: on
+          - server-versions: stable31
+            databases: pgsql
+            php-versions: 8.1
+            encryption: on
 
     name: Integration test with cron ☁️${{ matrix.server-versions }} 🐘${{ matrix.php-versions }} 🔐:${{ matrix.encryption }}
 
@@ -583,7 +601,7 @@ jobs:
             encryption: off
 
 
-    name: Integration test with file listener ☁️${{ matrix.server-versions }} 🐘${{ matrix.php-versions }} 🗑️${{ matrix.file-deletion-method }}) 🔐:${{ matrix.encryption }}
+    name: Integration test with file listener ☁️${{ matrix.server-versions }} 🐘${{ matrix.php-versions }} 🗑️${{ matrix.file-deletion-method }} 🔐:${{ matrix.encryption }}
 
     env:
       MYSQL_PORT: 4444
@@ -837,7 +855,7 @@ jobs:
           # share developer_manual to test user
           curl -X POST -u 'admin:password' -H 'OCS-APIRequest: true' 'http://localhost:8080/ocs/v2.php/apps/files_sharing/api/v1/shares?shareType=0&shareWith=test&path=developer_manual'
           # list files to trigger mounting of shares
-          curl -X PROPFIND -H "Depth: 0" -u 'test:test' 'http://localhost:8080/remote.php/webdav/test/'
+          curl -X PROPFIND -H "Depth: 0" -u 'test:test' 'http://localhost:8080/remote.php/webdav/'
 
       - name: Run indexer cron
         run: |

From 043574f458085f7ffa2d64d0bdb450e4323443b0 Mon Sep 17 00:00:00 2001
From: Marcel Klehr <mklehr@gmx.net>
Date: Sat, 26 Jul 2025 16:20:34 +0200
Subject: [PATCH 25/30] fix: More assertions for the record

Signed-off-by: Marcel Klehr <mklehr@gmx.net>
---
 .github/workflows/integration-test.yml | 55 ++++++++++++++++++++------
 1 file changed, 42 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml
index 7f23e233..8c8a1df7 100644
--- a/.github/workflows/integration-test.yml
+++ b/.github/workflows/integration-test.yml
@@ -823,18 +823,47 @@ jobs:
         run: |
           ./occ background-job:worker 'OC\TaskProcessing\SynchronousBackgroundJob' &
           ./occ background-job:worker 'OC\TaskProcessing\SynchronousBackgroundJob' &
-          OUT1=$(./occ context_chat:prompt admin "Which factors are taken into account for the Ethical AI Rating?")
-          echo "$OUT1"
-          echo '--------------------------------------------------'
-          OUT2=$(./occ context_chat:prompt admin "Welche Faktoren beeinflussen das Ethical AI Rating?")
-          echo "$OUT2"
-          echo '--------------------------------------------------'
-
-          OUT3=$(./occ context_chat:search admin "Welche Faktoren beeinflussen das Ethical AI Rating?")
-          echo "$OUT3"
-          echo "$OUT1" | grep -q "If all of these points are met, we give a Green label." || exit 1
-          echo "$OUT2" | grep -q "If all of these points are met, we give a Green label." || exit 1
-          echo "$OUT3" | grep -q "overview.rst" || exit 1
+          set +e
+          
+          # Check for user admin: Should be there
+          OUT_ADMIN_ENGLISH_PROMPT_AI=$(./occ context_chat:prompt admin "Which factors are taken into account for the Ethical AI Rating?")
+          OUT_ADMIN_GERMAN_PROMPT_AI=$(./occ context_chat:prompt admin "Welche Faktoren beeinflussen das Ethical AI Rating?")
+          OUT_ADMIN_SEARCH_AI=$(./occ context_chat:search admin "Welche Faktoren beeinflussen das Ethical AI Rating?")
+          
+          # Check for user test: Shouldn't be there
+          OUT_TEST_ENGLISH_PROMPT_AI=$(./occ context_chat:prompt test "Which factors are taken into account for the Ethical AI Rating?")
+          OUT_TEST_GERMAN_PROMPT_AI=$(./occ context_chat:prompt test "Welche Faktoren beeinflussen das Ethical AI Rating?")
+          OUT_TEST_SEARCH_AI=$(./occ context_chat:search test "Welche Faktoren beeinflussen das Ethical AI Rating?")
+          
+          # Check for user test: Shouldn't be there, because it was not shared yet
+          OUT_TEST_PROMPT_JOBS=$(./occ context_chat:prompt test "How do I set my TimedJob to be time insensitive?")
+          OUT_TEST_SEARCH_JOBS=$(./occ context_chat:search test "How do I set my TimedJob to be time insensitive?")
+          
+          echo "OUT_ADMIN_ENGLISH_PROMPT_AI"
+          echo "$OUT_ADMIN_ENGLISH_PROMPT_AI"
+          echo "OUT_ADMIN_GERMAN_PROMPT_AI"
+          echo "$OUT_ADMIN_GERMAN_PROMPT_AI"
+          echo "OUT_ADMIN_SEARCH_AI"
+          echo "$OUT_ADMIN_SEARCH_AI"
+          echo "OUT_TEST_ENGLISH_PROMPT_AI"
+          echo "$OUT_TEST_ENGLISH_PROMPT_AI"
+          echo "OUT_TEST_GERMAN_PROMPT_AI"
+          echo "$OUT_TEST_GERMAN_PROMPT_AI"
+          echo "OUT_TEST_SEARCH_AI"
+          echo "$OUT_TEST_SEARCH_AI"
+          echo "OUT_TEST_PROMPT_JOBS"
+          echo "$OUT_TEST_PROMPT_JOBS"
+          echo "OUT_TEST_SEARCH_JOBS"
+          echo "$OUT_TEST_SEARCH_JOBS"
+          
+          echo "$OUT_ADMIN_ENGLISH_PROMPT_AI" | grep -q  "If all of these points are met, we give a Green label." && echo '✅ Admin does see AI overview' || exit 1
+          echo "$OUT_ADMIN_GERMAN_PROMPT_AI" | grep -q "If all of these points are met, we give a Green label." && echo '✅ Admin does see AI overview, after german question' || exit 1
+          echo "$OUT_ADMIN_SEARCH_AI" | grep -q "overview.rst" && echo '✅ Admin does see AI overview, when using search' || exit 1
+          echo "$OUT_TEST_ENGLISH_PROMPT_AI" | grep -q -v "If all of these points are met, we give a Green label." && echo '✅ Test user does not see AI overview' || exit 1
+          echo "$OUT_TEST_GERMAN_PROMPT_AI" | grep -q -v "If all of these points are met, we give a Green label." && echo '✅Test user does not see AI overview, after german question' || exit 1
+          echo "$OUT_TEST_SEARCH_AI" | grep -q -v "overview.rst" && echo '✅Test user does not see AI overview, when using search' || exit 1
+          echo "$OUT_TEST_PROMPT_JOBS" | grep -q -v "background jobs" && echo '✅Test user does not see BackgroundJobs docs' || exit 1
+          echo "$OUT_TEST_SEARCH_JOBS" | grep -q -v "backgroundjobs.md" && echo '✅Test user does not see BackgroundJobs docs' || exit 1
 
       - name: Remove some files using occ
         if: ${{ matrix.file-deletion-method == 'occ'}}
@@ -872,7 +901,7 @@ jobs:
           
           set +e
           
-          # Check for user admin: Should be there
+          # Check for user admin: Shouldn't be there
           OUT_ADMIN_ENGLISH_PROMPT_AI=$(./occ context_chat:prompt admin "Which factors are taken into account for the Ethical AI Rating?")
           OUT_ADMIN_GERMAN_PROMPT_AI=$(./occ context_chat:prompt admin "Welche Faktoren beeinflussen das Ethical AI Rating?")
           OUT_ADMIN_SEARCH_AI=$(./occ context_chat:search admin "Welche Faktoren beeinflussen das Ethical AI Rating?")

From 3cddb9089e9391a0d5bfbbed250ec78307682256 Mon Sep 17 00:00:00 2001
From: Marcel Klehr <mklehr@gmx.net>
Date: Tue, 29 Jul 2025 13:51:24 +0200
Subject: [PATCH 26/30] tests: Make sure bg workers die

so we don't have old state hanging around

Signed-off-by: Marcel Klehr <mklehr@gmx.net>
---
 .github/workflows/integration-test.yml | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml
index 8c8a1df7..e42dc3ea 100644
--- a/.github/workflows/integration-test.yml
+++ b/.github/workflows/integration-test.yml
@@ -235,8 +235,8 @@ jobs:
 
       - name: Run the prompts
         run: |
-          ./occ background-job:worker 'OC\TaskProcessing\SynchronousBackgroundJob' &
-          ./occ background-job:worker 'OC\TaskProcessing\SynchronousBackgroundJob' &
+          ./occ background-job:worker --stop_after=2m 'OC\TaskProcessing\SynchronousBackgroundJob' &
+          ./occ background-job:worker --stop_after=2m 'OC\TaskProcessing\SynchronousBackgroundJob' &
           OUT1=$(./occ context_chat:prompt admin "Which factors are taken into account for the Ethical AI Rating?")
           echo "$OUT1"
           echo '--------------------------------------------------'
@@ -533,8 +533,8 @@ jobs:
 
       - name: Run the prompts
         run: |
-          ./occ background-job:worker 'OC\TaskProcessing\SynchronousBackgroundJob' &
-          ./occ background-job:worker 'OC\TaskProcessing\SynchronousBackgroundJob' &
+          ./occ background-job:worker --stop_after=2m 'OC\TaskProcessing\SynchronousBackgroundJob' &
+          ./occ background-job:worker --stop_after=2m 'OC\TaskProcessing\SynchronousBackgroundJob' &
           OUT1=$(./occ context_chat:prompt admin "Which factors are taken into account for the Ethical AI Rating?")
           echo "$OUT1"
           echo '--------------------------------------------------'
@@ -821,8 +821,8 @@ jobs:
 
       - name: Run the prompts
         run: |
-          ./occ background-job:worker 'OC\TaskProcessing\SynchronousBackgroundJob' &
-          ./occ background-job:worker 'OC\TaskProcessing\SynchronousBackgroundJob' &
+          ./occ background-job:worker --stop_after=2m 'OC\TaskProcessing\SynchronousBackgroundJob' &
+          ./occ background-job:worker --stop_after=2m 'OC\TaskProcessing\SynchronousBackgroundJob' &
           set +e
           
           # Check for user admin: Should be there
@@ -896,8 +896,8 @@ jobs:
 
       - name: Run the prompts again
         run: |
-          ./occ background-job:worker 'OC\TaskProcessing\SynchronousBackgroundJob' &
-          ./occ background-job:worker 'OC\TaskProcessing\SynchronousBackgroundJob' &
+          ./occ background-job:worker --stop_after=2m 'OC\TaskProcessing\SynchronousBackgroundJob' &
+          ./occ background-job:worker --stop_after=2m 'OC\TaskProcessing\SynchronousBackgroundJob' &
           
           set +e
           

From bd98a2b76eb225a8c892dfde1ea333c8b2087ec0 Mon Sep 17 00:00:00 2001
From: Marcel Klehr <mklehr@gmx.net>
Date: Tue, 29 Jul 2025 13:58:06 +0200
Subject: [PATCH 27/30] fix(FsEventScheduler): Try to fix weird type error

Signed-off-by: Marcel Klehr <mklehr@gmx.net>
---
 lib/Db/FsEvent.php               | 3 +--
 lib/Service/FsEventScheduler.php | 2 +-
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/lib/Db/FsEvent.php b/lib/Db/FsEvent.php
index 7cfdb536..ff2bad20 100644
--- a/lib/Db/FsEvent.php
+++ b/lib/Db/FsEvent.php
@@ -35,10 +35,9 @@ class FsEvent extends Entity {
 
 	public function __construct() {
 		// add types in constructor
-		$this->addType('id', Types::BIGINT);
 		$this->addType('type', Types::STRING);
 		$this->addType('userId', Types::STRING);
-		$this->addType('nodeId', Types::BIGINT);
+		$this->addType('nodeId', 'integer'); // stable30 does not support Types::BIGINT here
 	}
 
 	/**
diff --git a/lib/Service/FsEventScheduler.php b/lib/Service/FsEventScheduler.php
index 0b001526..683240d1 100644
--- a/lib/Service/FsEventScheduler.php
+++ b/lib/Service/FsEventScheduler.php
@@ -46,7 +46,7 @@ private function getOwnerIdForNode(Node $node): string {
 	private function scheduleEvent(FsEventType $type, string $userId, int $nodeId): void {
 		$item = new FsEvent();
 		$item->setUserId($userId);
-		$item->setType($type);
+		$item->setType($type->value);
 		$item->setNodeId($nodeId);
 
 		// do not catch DB exceptions

From 888fdcd4d92f7782e482835ad72b0ab8c8a8b942 Mon Sep 17 00:00:00 2001
From: Marcel Klehr <mklehr@gmx.net>
Date: Thu, 31 Jul 2025 08:37:44 +0200
Subject: [PATCH 28/30] tests: `sleep 60` for backend to get ready

Signed-off-by: Marcel Klehr <mklehr@gmx.net>
---
 .github/workflows/integration-test.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml
index e42dc3ea..f1207063 100644
--- a/.github/workflows/integration-test.yml
+++ b/.github/workflows/integration-test.yml
@@ -197,6 +197,7 @@ jobs:
           python3 -u ./main_em.py > em_backend_logs 2>&1 &
           python3 -u ./main.py > backend_logs 2>&1 &
           echo $! > ../pid.txt          # Save the process ID (PID)
+          sleep 60 # Wait for the backend to get ready
 
       - name: Register backend
         run: |
@@ -446,6 +447,7 @@ jobs:
           python3 -u ./main_em.py > em_backend_logs 2>&1 &
           python3 -u ./main.py > backend_logs 2>&1 &
           echo $! > ../pid.txt          # Save the process ID (PID)
+          sleep 60 # Wait for the backend to get ready
 
       - name: Register backend
         run: |
@@ -727,6 +729,7 @@ jobs:
           python3 -u ./main_em.py > em_backend_logs 2>&1 &
           python3 -u ./main.py > backend_logs 2>&1 &
           echo $! > ../pid.txt          # Save the process ID (PID)
+          sleep 60 # Wait for the backend to get ready
 
       - name: Register backend
         run: |

From 93235a5bac4f77f5e7a89030665e5ff501d2e5c0 Mon Sep 17 00:00:00 2001
From: Marcel Klehr <mklehr@gmx.net>
Date: Thu, 31 Jul 2025 08:45:04 +0200
Subject: [PATCH 29/30] tests: Reduce number of files

Signed-off-by: Marcel Klehr <mklehr@gmx.net>
---
 .github/workflows/integration-test.yml | 84 ++++++++++++++++++++++++++
 1 file changed, 84 insertions(+)

diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml
index f1207063..22f21347 100644
--- a/.github/workflows/integration-test.yml
+++ b/.github/workflows/integration-test.yml
@@ -180,6 +180,34 @@ jobs:
           cd ..
           rm -rf documentation
 
+      - name: Reduce number of files
+        run: |
+          cd data/admin/files
+          LIMIT=100
+        
+          # Find all files in the directory and its subdirectories
+          FILES=($(find papers/ -type f))
+          
+          # Count the number of files
+          FILE_COUNT=${#FILES[@]}
+          
+          # Check if the current number of files exceeds the limit
+          if [ "$FILE_COUNT" -le "$LIMIT" ]; then
+            echo "No files need to be deleted. Current count: $FILE_COUNT"
+            exit 0
+          fi
+
+          # Calculate how many files to delete
+          FILES_TO_DELETE=$((FILE_COUNT - LIMIT))
+          
+          # Sort files by modification time (oldest first) and delete the oldest ones
+          for FILE in $(ls -t "${FILES[@]}" | tail -n "$FILES_TO_DELETE"); do
+            echo "Deleting: $FILE"
+            rm "$FILE"
+          done
+
+          echo "Deleted $FILES_TO_DELETE files. Current count: $((FILE_COUNT - FILES_TO_DELETE))"
+
       - name: Setup python 3.11
         uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
         with:
@@ -486,6 +514,34 @@ jobs:
           cd ..
           rm -rf documentation
 
+      - name: Reduce number of files
+        run: |
+          cd data/admin/files
+          LIMIT=100
+          
+          # Find all files in the directory and its subdirectories
+          FILES=($(find papers/ -type f))
+          
+          # Count the number of files
+          FILE_COUNT=${#FILES[@]}
+          
+          # Check if the current number of files exceeds the limit
+          if [ "$FILE_COUNT" -le "$LIMIT" ]; then
+            echo "No files need to be deleted. Current count: $FILE_COUNT"
+            exit 0
+          fi
+
+          # Calculate how many files to delete
+          FILES_TO_DELETE=$((FILE_COUNT - LIMIT))
+          
+          # Sort files by modification time (oldest first) and delete the oldest ones
+          for FILE in $(ls -t "${FILES[@]}" | tail -n "$FILES_TO_DELETE"); do
+            echo "Deleting: $FILE"
+            rm "$FILE"
+          done
+
+          echo "Deleted $FILES_TO_DELETE files. Current count: $((FILE_COUNT - FILES_TO_DELETE))"
+
       - name: Run files scan
         run: |
           ./occ files:scan admin # Do the scan before enabling context chat
@@ -783,6 +839,34 @@ jobs:
           cd ..
           rm -rf documentation
 
+      - name: Reduce number of files
+        run: |
+          cd data/admin/files
+          LIMIT=100
+          
+          # Find all files in the directory and its subdirectories
+          FILES=($(find papers/ -type f))
+          
+          # Count the number of files
+          FILE_COUNT=${#FILES[@]}
+          
+          # Check if the current number of files exceeds the limit
+          if [ "$FILE_COUNT" -le "$LIMIT" ]; then
+            echo "No files need to be deleted. Current count: $FILE_COUNT"
+            exit 0
+          fi
+
+          # Calculate how many files to delete
+          FILES_TO_DELETE=$((FILE_COUNT - LIMIT))
+          
+          # Sort files by modification time (oldest first) and delete the oldest ones
+          for FILE in $(ls -t "${FILES[@]}" | tail -n "$FILES_TO_DELETE"); do
+            echo "Deleting: $FILE"
+            rm "$FILE"
+          done
+
+          echo "Deleted $FILES_TO_DELETE files. Current count: $((FILE_COUNT - FILES_TO_DELETE))"
+
       - name: Run files scan
         run: |
           ./occ files:scan admin # We do the scan after enabling context chat here, so we can test the file listeners

From 19e6f6bf07ded633ecb6ee604ef0c6c1c419c1a7 Mon Sep 17 00:00:00 2001
From: Marcel Klehr <mklehr@gmx.net>
Date: Thu, 31 Jul 2025 08:47:52 +0200
Subject: [PATCH 30/30] tests: Add summary job

Signed-off-by: Marcel Klehr <mklehr@gmx.net>
---
 .github/workflows/integration-test.yml | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml
index 22f21347..41f8993d 100644
--- a/.github/workflows/integration-test.yml
+++ b/.github/workflows/integration-test.yml
@@ -1045,3 +1045,18 @@ jobs:
           cat context_chat_backend/em_backend_logs || echo "No embedding server logs"
           echo '--------------------------------------------------'
           tail -v -n +1 context_chat_backend/persistent_storage/logs/* || echo "No logs in logs directory"
+
+  summary:
+    permissions:
+      contents: none
+    runs-on: ubuntu-latest-low
+    needs: [scan-test, cron-test, listener-test]
+
+    if: always()
+
+    # This is the summary, we just avoid to rename it so that branch protection rules still match
+    name: integration-test
+
+    steps:
+      - name: Summary status
+        run: if ${{ needs.scan-test.rest != 'success' && needs.cron-test.result != 'success'  && needs.listener-test.result != 'success' }}; then exit 1; fi
\ No newline at end of file