I gotta have moar tests. Plus some hardenings

appinfo/info.xml

@@ -45,6 +45,7 @@ Refer to the [Context Chat Backend's readme](https://github.com/nextcloud/contex
     <background-jobs>
 		<job>OCA\ContextChat\BackgroundJobs\SchedulerJob</job>
 		<job>OCA\ContextChat\BackgroundJobs\FileSystemListenerJob</job>
+		<job>OCA\ContextChat\BackgroundJobs\ActionJob</job>
 		<job>OCA\ContextChat\BackgroundJobs\RotateLogsJob</job>
 	</background-jobs>
 	<commands>

lib/BackgroundJobs/ActionJob.php

@@ -17,9 +17,11 @@
 use OCP\App\IAppManager;
 use OCP\AppFramework\Utility\ITimeFactory;
 use OCP\BackgroundJob\IJobList;
-use OCP\BackgroundJob\QueuedJob;
+use OCP\BackgroundJob\TimedJob;
+use OCP\DB\Exception;
+use OCP\IConfig;
 
-class ActionJob extends QueuedJob {
+class ActionJob extends TimedJob {
 	private const BATCH_SIZE = 1000;
 
 	public function __construct(
@@ -30,8 +32,15 @@ public function __construct(
 		private Logger $logger,
 		private DiagnosticService $diagnosticService,
 		private IAppManager $appManager,
+		private IConfig $config,
 	) {
 		parent::__construct($timeFactory);
+		$this->setAllowParallelRuns(false);
+		$this->setInterval($this->getJobInterval());
+	}
+
+	private function getJobInterval(): int {
+		return intval($this->config->getAppValue('context_chat', 'action_job_interval', (string)(5 * 60))); // 5 minutes
 	}
 
 	protected function run($argument): void {
@@ -40,15 +49,20 @@ protected function run($argument): void {
 			return;
 		}
 
-		$this->diagnosticService->sendJobStart(static::class, $this->getId());
-		$this->diagnosticService->sendHeartbeat(static::class, $this->getId());
-		$entities = $this->actionMapper->getFromQueue(static::BATCH_SIZE);
+		try {
+			$this->diagnosticService->sendJobStart(static::class, $this->getId());
+			$this->diagnosticService->sendHeartbeat(static::class, $this->getId());
+			try {
+				$entities = $this->actionMapper->getFromQueue(static::BATCH_SIZE);
+			} catch (Exception $e) {
+				$this->logger->warning('Error fetching actions in action Job : ' . $e->getMessage(), ['exception' => $e]);
+				return;
+			}
 
-		if (empty($entities)) {
-			return;
-		}
+			if (empty($entities)) {
+				return;
+			}
 
-		try {
 			foreach ($entities as $entity) {
 				$this->diagnosticService->sendHeartbeat(static::class, $this->getId());
 
@@ -118,13 +132,10 @@ protected function run($argument): void {
 				}
 			}
 		} catch (\Throwable $e) {
-			// schedule in 5mins
-			$this->jobList->scheduleAfter(static::class, $this->time->getTime() + 5 * 60);
+			$this->logger->warning('Error in action Job : ' . $e->getMessage(), ['exception' => $e]);
 			throw $e;
+		} finally {
+			$this->diagnosticService->sendJobEnd(static::class, $this->getId());
 		}
-
-		// schedule in 5mins
-		$this->jobList->scheduleAfter(static::class, $this->time->getTime() + 5 * 60);
-		$this->diagnosticService->sendJobEnd(static::class, $this->getId());
 	}
 }

lib/BackgroundJobs/FileSystemListenerJob.php

@@ -16,26 +16,31 @@
 use OCA\ContextChat\Type\FsEventType;
 use OCP\App\IAppManager;
 use OCP\AppFramework\Utility\ITimeFactory;
-use OCP\BackgroundJob\IJobList;
 use OCP\BackgroundJob\TimedJob;
+use OCP\DB\Exception;
 use OCP\Files\IRootFolder;
+use OCP\IConfig;
 
 class FileSystemListenerJob extends TimedJob {
 	private const BATCH_SIZE = 500;
 
 	public function __construct(
 		ITimeFactory $timeFactory,
 		private FsEventMapper $fsEventMapper,
-		private IJobList $jobList,
 		private Logger $logger,
 		private DiagnosticService $diagnosticService,
 		private IAppManager $appManager,
 		private FsEventService $fsEventService,
 		private IRootFolder $rootFolder,
+		private IConfig $config,
 	) {
 		parent::__construct($timeFactory);
 		$this->allowParallelRuns = false;
-		$this->setInterval(5 * 60); // 5 minutes
+		$this->setInterval($this->getJobInterval());
+	}
+
+	private function getJobInterval(): int {
+		return intval($this->config->getAppValue('context_chat', 'fs_listener_job_interval', (string)(5 * 60))); // 5 minutes
 	}
 
 	protected function run($argument): void {
@@ -44,45 +49,60 @@ protected function run($argument): void {
 			return;
 		}
 
-		$this->diagnosticService->sendJobStart(static::class, $this->getId());
-		$this->diagnosticService->sendHeartbeat(static::class, $this->getId());
-		$fsEvents = $this->fsEventMapper->getFromQueue(static::BATCH_SIZE);
-
-		if (empty($fsEvents)) {
-			return;
-		}
-
-		foreach ($fsEvents as $fsEvent) {
+		try {
+			$this->diagnosticService->sendJobStart(static::class, $this->getId());
 			$this->diagnosticService->sendHeartbeat(static::class, $this->getId());
-
 			try {
-				$node = current($this->rootFolder->getUserFolder($fsEvent->getUserId())->getById($fsEvent->getNodeId()));
-			} catch (\Exception $e) {
-				$this->logger->warning('Error retrieving node for fs event "' . $fsEvent->getType() . '": ' . $e->getMessage(), ['exception' => $e]);
-				$node = false;
+				$fsEvents = $this->fsEventMapper->getFromQueue(static::BATCH_SIZE);
+			} catch (Exception $e) {
+				$this->logger->warning('Error fetching fs events: ' . $e->getMessage(), ['exception' => $e]);
+				return;
 			}
-			if ($node === false) {
-				$this->logger->warning('Node with ID ' . $fsEvent->getNodeId() . ' not found for fs event "' . $fsEvent->getType() . '"');
-				$this->fsEventMapper->delete($fsEvent);
-				continue;
+
+			if (empty($fsEvents)) {
+				return;
 			}
 
-			try {
-				switch ($fsEvent->getTypeObject()) {
-					case FsEventType::CREATE:
-						$this->fsEventService->onInsert($node);
-						break;
-					case FsEventType::ACCESS_UPDATE_DECL:
-						$this->fsEventService->onAccessUpdateDecl($node);
-						break;
-				}
+			foreach ($fsEvents as $fsEvent) {
 				$this->diagnosticService->sendHeartbeat(static::class, $this->getId());
-				$this->fsEventMapper->delete($fsEvent);
-			} catch (\RuntimeException $e) {
-				$this->logger->warning('Error handling fs event "' . $fsEvent->getType() . '": ' . $e->getMessage(), ['exception' => $e]);
+
+				try {
+					$node = current($this->rootFolder->getUserFolder($fsEvent->getUserId())->getById($fsEvent->getNodeId()));
+				} catch (\Exception $e) {
+					$this->logger->warning('Error retrieving node for fs event "' . $fsEvent->getType() . '": ' . $e->getMessage(), ['exception' => $e]);
+					$node = false;
+				}
+				if ($node === false) {
+					$this->logger->warning('Node with ID ' . $fsEvent->getNodeId() . ' not found for fs event "' . $fsEvent->getType() . '"');
+					try {
+						$this->fsEventMapper->delete($fsEvent);
+					} catch (Exception $e) {
+						$this->logger->warning('Error deleting fs event "' . $fsEvent->getType() . '": ' . $e->getMessage(), ['exception' => $e]);
+					}
+					continue;
+				}
+
+				try {
+					switch ($fsEvent->getTypeObject()) {
+						case FsEventType::CREATE:
+							$this->fsEventService->onInsert($node);
+							break;
+						case FsEventType::ACCESS_UPDATE_DECL:
+							$this->fsEventService->onAccessUpdateDecl($node);
+							break;
+					}
+					$this->diagnosticService->sendHeartbeat(static::class, $this->getId());
+				} catch (\Throwable $e) {
+					$this->logger->warning('Error handling fs event "' . $fsEvent->getType() . '": ' . $e->getMessage(), ['exception' => $e]);
+				}
+				try {
+					$this->fsEventMapper->delete($fsEvent);
+				} catch (Exception $e) {
+					$this->logger->warning('Error deleting fs event "' . $fsEvent->getType() . '": ' . $e->getMessage(), ['exception' => $e]);
+				}
 			}
+		} finally {
+			$this->diagnosticService->sendJobEnd(static::class, $this->getId());
 		}
-
-		$this->diagnosticService->sendJobEnd(static::class, $this->getId());
 	}
 }

lib/BackgroundJobs/IndexerJob.php

@@ -103,31 +103,31 @@ public function run($argument): void {
 			return;
 		}
 
-		$this->diagnosticService->sendJobStart(static::class, $this->getId());
-		$this->diagnosticService->sendHeartbeat(static::class, $this->getId());
+		try {
+			$this->diagnosticService->sendJobStart(static::class, $this->getId());
+			$this->diagnosticService->sendHeartbeat(static::class, $this->getId());
 
-		// Setup Filesystem for a users that can access this mount
-		$mounts = array_values(array_filter($this->userMountCache->getMountsForStorageId($this->storageId), function (ICachedMountInfo $mount) {
-			return $mount->getRootId() === $this->rootId;
-		}));
+			// Setup Filesystem for a users that can access this mount
+			$mounts = array_values(array_filter($this->userMountCache->getMountsForStorageId($this->storageId), function (ICachedMountInfo $mount) {
+				return $mount->getRootId() === $this->rootId;
+			}));
 
-		if (count($mounts) > 0) {
-			\OC_Util::setupFS($mounts[0]->getUser()->getUID());
-		}
+			if (count($mounts) > 0) {
+				\OC_Util::setupFS($mounts[0]->getUser()->getUID());
+			}
 
-		try {
-			$this->logger->debug('[IndexerJob] Running indexing', ['storageId' => $this->storageId, 'rootId' => $this->rootId]);
-			$this->index($files);
-		} catch (\RuntimeException $e) {
-			$this->logger->warning('[IndexerJob] Temporary problem with indexing', ['exception' => $e, 'storageId' => $this->storageId, 'rootId' => $this->rootId]);
-		} catch (\ErrorException $e) {
-			$this->logger->warning('[IndexerJob]  Problem with indexing', ['exception' => $e, 'storageId' => $this->storageId, 'rootId' => $this->rootId]);
-			$this->logger->info('[IndexerJob] Removing ' . static::class . ' with argument ' . var_export($argument, true) . 'from oc_jobs');
-			$this->jobList->remove(static::class, $argument);
-			throw $e;
-		}
+			try {
+				$this->logger->debug('[IndexerJob] Running indexing', ['storageId' => $this->storageId, 'rootId' => $this->rootId]);
+				$this->index($files);
+			} catch (\RuntimeException $e) {
+				$this->logger->warning('[IndexerJob] Temporary problem with indexing', ['exception' => $e, 'storageId' => $this->storageId, 'rootId' => $this->rootId]);
+			} catch (\ErrorException $e) {
+				$this->logger->warning('[IndexerJob]  Problem with indexing', ['exception' => $e, 'storageId' => $this->storageId, 'rootId' => $this->rootId]);
+				$this->logger->info('[IndexerJob] Removing ' . static::class . ' with argument ' . var_export($argument, true) . 'from oc_jobs');
+				$this->jobList->remove(static::class, $argument);
+				throw $e;
+			}
 
-		try {
 			// If there is at least one file left in the queue, reschedule this job
 			$files = $this->queue->getFromQueue($this->storageId, $this->rootId, 1);
 			$indexerJobCount = $this->getJobCount(IndexerJob::class);
@@ -138,13 +138,14 @@ public function run($argument): void {
 				$this->setInitialIndexCompletion();
 			} elseif (count($files) === 0) {
 				$this->logger->debug('[IndexerJob] No files left in queue, but we keep the job around to wait for potential StorageCrawlJob instances to finish');
-
 			}
 		} catch (Exception $e) {
 			$this->logger->error('[IndexerJob] Cannot retrieve items from queue', ['exception' => $e, 'storageId' => $this->storageId, 'rootId' => $this->rootId]);
-			return;
+		} catch (\Throwable $e) {
+			$this->logger->error('[IndexerJob] Failure during job run', ['exception' => $e, 'storageId' => $this->storageId, 'rootId' => $this->rootId]);
+		} finally {
+			$this->diagnosticService->sendJobEnd(static::class, $this->getId());
 		}
-		$this->diagnosticService->sendJobEnd(static::class, $this->getId());
 	}
 
 	protected function getBatchSize(): int {

lib/BackgroundJobs/StorageCrawlJob.php

@@ -51,45 +51,48 @@ protected function run($argument): void {
 		// Remove current iteration
 		$this->jobList->remove(self::class, $argument);
 
-		$this->diagnosticService->sendJobStart(static::class, $this->getId());
-		$this->diagnosticService->sendHeartbeat(static::class, $this->getId());
-
-		$i = 0;
-		foreach ($this->storageService->getFilesInMount($storageId, $overrideRoot ?? $rootId, $lastFileId, self::BATCH_SIZE) as $fileId) {
-			$queueFile = new QueueFile();
-			$queueFile->setStorageId($storageId);
-			$queueFile->setRootId($rootId);
-			$queueFile->setFileId($fileId);
-			$queueFile->setUpdate(false);
+		try {
+			$this->diagnosticService->sendJobStart(static::class, $this->getId());
 			$this->diagnosticService->sendHeartbeat(static::class, $this->getId());
-			try {
-				$this->queue->insertIntoQueue($queueFile);
-			} catch (Exception $e) {
-				$this->logger->error('[StorageCrawlJob] Failed to add file to queue', [
-					'fileId' => $fileId,
-					'exception' => $e,
+
+			$i = 0;
+			foreach ($this->storageService->getFilesInMount($storageId, $overrideRoot ?? $rootId, $lastFileId, self::BATCH_SIZE) as $fileId) {
+				$queueFile = new QueueFile();
+				$queueFile->setStorageId($storageId);
+				$queueFile->setRootId($rootId);
+				$queueFile->setFileId($fileId);
+				$queueFile->setUpdate(false);
+				$this->diagnosticService->sendHeartbeat(static::class, $this->getId());
+				try {
+					$this->queue->insertIntoQueue($queueFile);
+				} catch (Exception $e) {
+					$this->logger->error('[StorageCrawlJob] Failed to add file to queue', [
+						'fileId' => $fileId,
+						'exception' => $e,
+						'storage_id' => $storageId,
+						'root_id' => $rootId,
+						'override_root' => $overrideRoot,
+						'last_file_id' => $lastFileId
+					]);
+				}
+				$i++;
+			}
+
+			if ($i > 0) {
+				// Schedule next iteration after 5 minutes
+				$this->jobList->scheduleAfter(self::class, $this->time->getTime() + $this->getJobInterval(), [
 					'storage_id' => $storageId,
 					'root_id' => $rootId,
 					'override_root' => $overrideRoot,
-					'last_file_id' => $lastFileId
+					'last_file_id' => $queueFile->getFileId(),
 				]);
-			}
-			$i++;
-		}
 
-		if ($i > 0) {
-			// Schedule next iteration after 5 minutes
-			$this->jobList->scheduleAfter(self::class, $this->time->getTime() + $this->getJobInterval(), [
-				'storage_id' => $storageId,
-				'root_id' => $rootId,
-				'override_root' => $overrideRoot,
-				'last_file_id' => $queueFile->getFileId(),
-			]);
-
-			// the last job to set this value will win
-			$this->appConfig->setValueInt(Application::APP_ID, 'last_indexed_file_id', $queueFile->getFileId());
+				// the last job to set this value will win
+				$this->appConfig->setValueInt(Application::APP_ID, 'last_indexed_file_id', $queueFile->getFileId());
+			}
+		} finally {
+			$this->diagnosticService->sendJobEnd(static::class, $this->getId());
 		}
-		$this->diagnosticService->sendJobEnd(static::class, $this->getId());
 	}
 
 	protected function getJobInterval(): int {