refactor(FileListener): Process file events asynchronously

.github/workflows/cluster-faces-test.yml

@@ -206,6 +206,7 @@ jobs:
       - name: Run clustering
         if: steps.clustering-cache.outputs.cache-hit != 'true'
         run: |
+          ./occ upgrade # in case server master has new migrations in the meantime
           ./occ recognize:cluster-faces -b 10000
           ./occ recognize:cluster-faces -b 10000
           ./occ recognize:cluster-faces -b 10000

.github/workflows/files-scan-test.yml

@@ -140,9 +140,10 @@ jobs:
           GITHUB_REF: ${{ github.ref }}
         run: |
           ./occ files:scan admin -vvv
+          for run in {1..100}; do php cron.php -v; done
           rm data/admin/files/res/alpine.JPG
           ./occ files:scan admin -vvv
-          for run in {1..100}; do php cron.php; done
+          for run in {1..100}; do php cron.php -v; done
           [ ! -z "$(./occ tag:list)" ]
 
       - name: Print tags

.github/workflows/phpunit-mariadb.yml

@@ -155,6 +155,7 @@ jobs:
           mkdir data
           ./occ maintenance:install --verbose --database=mysql --database-name=nextcloud --database-host=127.0.0.1 --database-port=$DB_PORT --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass admin
           ./occ app:enable --force ${{ env.APP_NAME }}
+          ./occ app:enable files_external
 
       - name: Check PHPUnit script is defined
         id: check_phpunit

.github/workflows/phpunit-mysql.yml

@@ -153,6 +153,7 @@ jobs:
           mkdir data
           ./occ maintenance:install --verbose --database=mysql --database-name=nextcloud --database-host=127.0.0.1 --database-port=$DB_PORT --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass admin
           ./occ app:enable --force ${{ env.APP_NAME }}
+          ./occ app:enable files_external
 
       - name: Check PHPUnit script is defined
         id: check_phpunit

.github/workflows/phpunit-pgsql.yml

@@ -151,6 +151,7 @@ jobs:
           mkdir data
           ./occ maintenance:install --verbose --database=pgsql --database-name=nextcloud --database-host=127.0.0.1 --database-port=$DB_PORT --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass admin
           ./occ app:enable --force ${{ env.APP_NAME }}
+          ./occ app:enable files_external
 
       - name: Check PHPUnit script is defined
         id: check_phpunit

.github/workflows/phpunit-sqlite.yml

@@ -140,6 +140,7 @@ jobs:
           mkdir data
           ./occ maintenance:install --verbose --database=sqlite --database-name=nextcloud --database-host=127.0.0.1 --database-port=$DB_PORT --database-user=root --database-pass=rootpassword --admin-user admin --admin-pass admin
           ./occ app:enable --force ${{ env.APP_NAME }}
+          ./occ app:enable files_external
 
       - name: Check PHPUnit script is defined
         id: check_phpunit

composer.json

@@ -22,9 +22,9 @@
         "lint": "find . -name \\*.php -not -path './vendor/*' -print0 | xargs -0 -n1 php -l",
         "cs:check": "php-cs-fixer fix --dry-run --diff",
         "cs:fix": "php-cs-fixer fix",
-        "psalm": "psalm.phar",
-        "psalm:update-baseline": "psalm.phar --threads=1 --update-baseline",
-        "psalm:update-baseline:force": "psalm.phar --threads=1 --update-baseline --set-baseline=psalm-baseline.xml",
+        "psalm": "psalm",
+        "psalm:update-baseline": "psalm --threads=1 --update-baseline",
+        "psalm:update-baseline:force": "psalm --threads=1 --update-baseline --set-baseline=psalm-baseline.xml",
         "test:unit": "phpunit --config tests/phpunit.xml",
         "post-install-cmd": [
             "@composer bin all install --ansi",

lib/AppInfo/Application.php

@@ -23,8 +23,6 @@
 use OCP\Files\Events\Node\NodeRenamedEvent;
 use OCP\Files\Events\NodeRemovedFromCache;
 use OCP\SabrePluginEvent;
-use OCP\Share\Events\ShareCreatedEvent;
-use OCP\Share\Events\ShareDeletedEvent;
 
 final class Application extends App implements IBootstrap {
 	public const APP_ID = 'recognize';
@@ -42,16 +40,10 @@ public function __construct() {
 		$dispatcher->addServiceListener(BeforeNodeRenamedEvent::class, FileListener::class);
 		$dispatcher->addServiceListener(CacheEntryInsertedEvent::class, FileListener::class);
 		$dispatcher->addServiceListener(NodeRemovedFromCache::class, FileListener::class);
-		// These events were added mid-way through NC 30, 31
-		if (class_exists('OCP\Files\Config\Event\UserMountAddedEvent')) {
-			$dispatcher->addServiceListener('OCP\Files\Config\Event\UserMountAddedEvent', FileListener::class);
-			$dispatcher->addServiceListener('OCP\Files\Config\Event\UserMountRemovedEvent', FileListener::class);
-			// it is not fired as of now, Added and Removed events are fired instead in that order
-			// $context->addServiceListener('OCP\Files\Config\Event\UserMountUpdatedEvent', FileListener::class);
-		} else {
-			$dispatcher->addServiceListener(ShareCreatedEvent::class, FileListener::class);
-			$dispatcher->addServiceListener(ShareDeletedEvent::class, FileListener::class);
-		}
+		$dispatcher->addServiceListener('OCP\Files\Config\Event\UserMountAddedEvent', FileListener::class);
+		$dispatcher->addServiceListener('OCP\Files\Config\Event\UserMountRemovedEvent', FileListener::class);
+		// it is not fired as of now, Added and Removed events are fired instead in that order
+		// $context->addServiceListener('OCP\Files\Config\Event\UserMountUpdatedEvent', FileListener::class);
 	}
 
 	public function register(IRegistrationContext $context): void {

lib/BackgroundJobs/ProcessFsActionsJob.php

@@ -0,0 +1,68 @@
+<?php
+
+/*
+ * Copyright (c) 2021-2022 The Recognize contributors.
+ * This file is licensed under the Affero General Public License version 3 or later. See the COPYING file.
+ */
+declare(strict_types=1);
+namespace OCA\Recognize\BackgroundJobs;
+
+use OCA\Recognize\Db\FsAccessUpdate;
+use OCA\Recognize\Db\FsActionMapper;
+use OCA\Recognize\Db\FsCreation;
+use OCA\Recognize\Db\FsDeletion;
+use OCA\Recognize\Db\FsMove;
+use OCA\Recognize\Service\FsActionService;
+use OCP\AppFramework\Utility\ITimeFactory;
+use OCP\BackgroundJob\IJobList;
+use OCP\BackgroundJob\TimedJob;
+use OCP\DB\Exception;
+use Psr\Log\LoggerInterface;
+
+final class ProcessFsActionsJob extends TimedJob {
+
+	public function __construct(
+		ITimeFactory            $timeFactory,
+		private FsActionService $accessUpdateService,
+		private IJobList        $jobList,
+		private FsActionMapper  $accessUpdateMapper,
+		private LoggerInterface $logger,
+	) {
+		parent::__construct($timeFactory);
+		$this->setInterval(5 * 60);
+		$this->setTimeSensitivity(self::TIME_SENSITIVE);
+	}
+
+	/**
+	 * @param array{storage_id:int, type: class-string<FsAccessUpdate|FsCreation|FsDeletion|FsMove>} $argument
+	 * @return void
+	 */
+	protected function run($argument): void {
+		$storageId = $argument['storage_id'] ?? null;
+		$className = $argument['type'];
+
+		if (isset($storageId)) {
+			$this->accessUpdateService->processActionsByClassAndStorageId($className, $storageId);
+			try {
+				$remainingCount = $this->accessUpdateMapper->countByStorageId($className, $storageId);
+			} catch (Exception $e) {
+				$this->logger->error('Failed to count fs actions: ' . $e->getMessage(), ['exception' => $e]);
+				$remainingCount = 1;
+			}
+		} else {
+			$this->accessUpdateService->processActionsByClass($className);
+			try {
+				$remainingCount = $this->accessUpdateMapper->count($className);
+			} catch (Exception $e) {
+				$this->logger->error('Failed to count fs actions: ' . $e->getMessage(), ['exception' => $e]);
+				$remainingCount = 1;
+			}
+		}
+
+
+		if ($remainingCount === 0) {
+			// Remove job from queue
+			$this->jobList->remove(self::class, $argument);
+		}
+	}
+}

lib/Clustering/DualTreeClique.php

@@ -64,7 +64,7 @@ public function propagateSetChanges(array &$labelToSetId) {
 
 		$labels = $this->dataset->labels();
 		$lastLabel = array_pop($labels);
-		if (!isset($labelToSetId[$lastLabel])) {
+		if (!isset($lastLabel) || !isset($labelToSetId[$lastLabel])) {
 			return null;
 		}
 		$setId = $labelToSetId[$lastLabel];

lib/Db/FsAccessUpdate.php

@@ -0,0 +1,43 @@
+<?php
+
+/*
+ * Copyright (c) 2022 The Recognize contributors.
+ * This file is licensed under the Affero General Public License version 3 or later. See the COPYING file.
+ */
+declare(strict_types=1);
+namespace OCA\Recognize\Db;
+
+use OCP\AppFramework\Db\Entity;
+
+/**
+ * Class FsAccessUpdate
+ *
+ * @package OCA\Recognize\Db
+ * @method int getStorageId()
+ * @method setStorageId(int $storageId)
+ * @method int getRootId()
+ * @method setRootId(int $rootId)
+ */
+final class FsAccessUpdate extends Entity {
+	protected ?int $storageId = null;
+	protected ?int $rootId = null;
+
+	/**
+	 * @var string[]
+	 */
+	public static array $columns = ['id', 'storage_id', 'root_id'];
+
+	/**
+	 * @var string[]
+	 */
+	public static array $fields = ['id', 'storageId', 'rootId'];
+
+	public static string $tableName = 'recognize_fs_access_updates';
+
+	public function __construct() {
+		// add types in constructor
+		$this->addType('id', 'integer');
+		$this->addType('storageId', 'integer');
+		$this->addType('rootId', 'integer');
+	}
+}