fix: Add RemoveDuplicateFaceDetections repair step

Signed-off-by: Marcel Klehr <mklehr@gmx.net>

Author: marcelklehr

lib/Db/FaceDetectionMapper.php

@@ -11,6 +11,7 @@
 use OCP\AppFramework\Db\DoesNotExistException;
 use OCP\AppFramework\Db\Entity;
 use OCP\AppFramework\Db\QBMapper;
+use OCP\DB\Exception;
 use OCP\DB\QueryBuilder\IQueryBuilder;
 use OCP\IConfig;
 use OCP\IDBConnection;
@@ -64,6 +65,13 @@ public function insert(Entity $entity): FaceDetection {
 		return $duplicates[0];
 	}
 
+	/**
+	 * @throws Exception
+	 */
+	public function insertWithoutDeduplication(Entity $entity): FaceDetection {
+		return parent::insert($entity);
+	}
+
 	/**
 	 * @throws \OCP\DB\Exception
 	 */

lib/Migration/RemoveDuplicateFaceDetections.php

@@ -0,0 +1,61 @@
+<?php
+
+declare(strict_types=1);
+/**
+ * @copyright Copyright (c) 2025, Marcel Klehr <mklehr@gmx.net>
+ *
+ * @author Marcel Klehr <mklehr@gmx.net>
+ *
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+namespace OCA\Recognize\Migration;
+
+use OCP\IDBConnection;
+use OCP\Migration\IOutput;
+use OCP\Migration\IRepairStep;
+use Psr\Log\LoggerInterface;
+
+final class RemoveDuplicateFaceDetections implements IRepairStep {
+
+	public function __construct(
+		private IDBConnection $db,
+		private LoggerInterface $logger,
+	) {
+	}
+
+	public function getName(): string {
+		return 'Remove duplicate face detections';
+	}
+
+	public function run(IOutput $output): void {
+		try {
+			$subQuery = $this->db->getQueryBuilder();
+			$subQuery->select($subQuery->func()->min('id'))
+				->from('recognize_face_detections')
+				->groupBy('file_id', 'user_id', 'x', 'y', 'height', 'width');
+
+			$qb = $this->db->getQueryBuilder();
+			$qb->delete('recognize_face_detections')
+				->where($qb->expr()->notIn('id', $qb->createFunction('(' . $subQuery->getSQL() .')')));
+
+			$qb->executeStatement();
+		} catch (\Throwable $e) {
+			$output->warning('Failed to automatically remove duplicate face detections for recognize.');
+			$this->logger->error('Failed to automatically remove duplicate face detections', ['exception' => $e]);
+		}
+	}
+}

tests/RemoveDuplicateFaceDetectionsTest.php

@@ -0,0 +1,79 @@
+<?php
+
+use OCA\Recognize\Db\FaceDetectionMapper;
+use OCA\Recognize\Migration\RemoveDuplicateFaceDetections;
+use OCP\IDBConnection;
+use OCP\Server;
+use Test\TestCase;
+
+/**
+ * @group DB
+ */
+class RemoveDuplicateFaceDetectionsTest extends TestCase {
+	private IDBConnection $db;
+	private FaceDetectionMapper $faceDetectionMapper;
+
+	public function setUp(): void {
+		parent::setUp();
+		$this->db = Server::get(IDBConnection::class);
+		$this->faceDetectionMapper = Server::get(FaceDetectionMapper::class);
+
+		// Clear
+		$qb = $this->db->getQueryBuilder();
+		$qb->delete('recognize_face_detections')->executeStatement();
+
+		// Generate 11 face detections per file (1000 files per user; 100 users)
+		// = 1.100.000 face detections out of which 900.000 are superfluous duplicates to be removed
+		// After the repair step there should be 200.000 left
+		for ($k = 0; $k < 100; $k++) {
+			for ($j = 0; $j < 1000; $j++) {
+				$user = 'user' . $k;
+				$x = rand(0, 100) / 100;
+				$y = rand(0, 100) / 100;
+				$height = rand(0, 100) / 100;
+				$width = rand(0, 100) / 100;
+				for ($i = 0; $i < 10; $i++) {
+					$face = new \OCA\Recognize\Db\FaceDetection();
+					$face->setUserId($user);
+					$face->setX($x);
+					$face->setY($y);
+					$face->setHeight($height);
+					$face->setWidth($width);
+					$face->setFileId($j);
+					$face->setThreshold(0.5);
+					$face->setVector([1, 2, 3, 4, 5, 6, 7, 8, 9, 0]);
+					$this->faceDetectionMapper->insertWithoutDeduplication($face);
+				}
+				$face2 = new \OCA\Recognize\Db\FaceDetection();
+				$face2->setUserId($user);
+				$face2->setX(rand(0, 100) / 100);
+				$face2->setY(rand(0, 100) / 100);
+				$face2->setHeight(rand(0, 100) / 100);
+				$face2->setWidth(rand(0, 100) / 100);
+				$face2->setFileId($k * $j);
+				$face2->setThreshold(0.5);
+				$face2->setVector([1, 2, 3, 4, 5, 6, 7, 8, 9, 0]);
+				$this->faceDetectionMapper->insertWithoutDeduplication($face2);
+			}
+		}
+	}
+
+	public function testRepairStep() : void {
+		// Prepare
+		$repairStep = Server::get(RemoveDuplicateFaceDetections::class);
+		$output = $this->createMock(\OCP\Migration\IOutput::class);
+
+		// Check
+		$qb = $this->db->getQueryBuilder();
+		$count = $qb->select($qb->func()->count('*'))->from('recognize_face_detections')->executeQuery()->fetchOne();
+		$this->assertEquals(1100000, (int)$count);
+
+		// Run
+		$repairStep->run($output);
+
+		// Assert
+		$qb = $this->db->getQueryBuilder();
+		$count = $qb->select($qb->func()->count('*'))->from('recognize_face_detections')->executeQuery()->fetchOne();
+		$this->assertEquals(200000, (int)$count);
+	}
+}