Skip to content

Commit d55875e

Browse files
kyteinskykyteinsky
committed
feat: add support for multimodal indexing
Signed-off-by: Anupam Kumar <kyteinsky@gmail.com>
1 parent baa8c90 commit d55875e

8 files changed

Lines changed: 207 additions & 21 deletions

File tree

‎appinfo/info.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,11 +44,11 @@ Refer to the [Context Chat Backend's readme](https://github.com/nextcloud/contex
4444
</dependencies>
4545
<background-jobs>
4646
<job>OCA\ContextChat\BackgroundJobs\FileSystemListenerJob</job>
47-
<job>OCA\ContextChat\BackgroundJobs\ActionJob</job>
4847
<job>OCA\ContextChat\BackgroundJobs\RotateLogsJob</job>
4948
</background-jobs>
5049
<commands>
5150
<command>OCA\ContextChat\Command\Prompt</command>
51+
<command>OCA\ContextChat\Command\QueueMultimodalFiles</command>
5252
<command>OCA\ContextChat\Command\Search</command>
5353
<command>OCA\ContextChat\Command\Statistics</command>
5454
</commands>

‎lib/AppInfo/Application.php‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,35 @@ class Application extends App implements IBootstrap {
6161
'text/org',
6262
];
6363

64+
public const IMAGE_MIMETYPES = [
65+
'image/bmp',
66+
'image/bpg',
67+
'image/emf',
68+
'image/gif',
69+
'image/heic',
70+
'image/heif',
71+
'image/jp2',
72+
'image/jpeg',
73+
'image/png',
74+
'image/svg+xml',
75+
'image/tga',
76+
'image/tiff',
77+
'image/webp',
78+
'image/x-dcraw',
79+
'image/x-icon',
80+
];
81+
82+
public const AUDIO_MIMETYPES = [
83+
'audio/aac',
84+
'audio/flac',
85+
'audio/mp4',
86+
'audio/mpeg',
87+
'audio/ogg',
88+
'audio/wav',
89+
'audio/webm',
90+
'audio/x-scpls',
91+
];
92+
6493
public function __construct(array $urlParams = []) {
6594
parent::__construct(self::APP_ID, $urlParams);
6695
}

‎lib/BackgroundJobs/StorageCrawlJob.php‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,13 @@
1010

1111
namespace OCA\ContextChat\BackgroundJobs;
1212

13+
use OCA\ContextChat\AppInfo\Application;
1314
use OCA\ContextChat\Db\QueueFile;
1415
use OCA\ContextChat\Logger;
1516
use OCA\ContextChat\Service\DiagnosticService;
1617
use OCA\ContextChat\Service\QueueService;
1718
use OCA\ContextChat\Service\StorageService;
19+
use OCA\ContextChat\Service\TaskTypeService;
1820
use OCP\AppFramework\Services\IAppConfig;
1921
use OCP\AppFramework\Utility\ITimeFactory;
2022
use OCP\BackgroundJob\IJobList;
@@ -33,19 +35,22 @@ public function __construct(
3335
private StorageService $storageService,
3436
private DiagnosticService $diagnosticService,
3537
private IAppConfig $appConfig,
38+
private TaskTypeService $taskTypeService,
3639
) {
3740
parent::__construct($timeFactory);
3841
}
3942

4043
/**
41-
* @param array{storage_id:int, root_id:int, overridden_root:int|null, override_root:int|null, last_file_id:int} $argument
44+
* @param array{storage_id:int, root_id:int, overridden_root:int|null, override_root:int|null, last_file_id:int, only_non_textual?:bool} $argument
4245
* @return void
4346
*/
4447
protected function run($argument): void {
4548
$storageId = $argument['storage_id'];
4649
$rootId = $argument['root_id'];
4750
$overrideRoot = $argument['overridden_root'] ?? $argument['override_root'] ?? $rootId;
4851
$lastFileId = $argument['last_file_id'];
52+
$onlyNonTextual = $argument['only_non_textual'] ?? false;
53+
$mimeTypes = $this->taskTypeService->getMultimodalMimetypes(!$onlyNonTextual);
4954

5055
// Remove current iteration
5156
$this->jobList->remove(self::class, $argument);
@@ -56,7 +61,7 @@ protected function run($argument): void {
5661

5762
$mountFilesCount = 0;
5863
$lastSuccessfulFileId = -1;
59-
foreach ($this->storageService->getFilesInMount($storageId, $overrideRoot ?? $rootId, $lastFileId, self::BATCH_SIZE) as $fileId) {
64+
foreach ($this->storageService->getFilesInMount($storageId, $overrideRoot ?? $rootId, $lastFileId, self::BATCH_SIZE, $mimeTypes) as $fileId) {
6065
$queueFile = new QueueFile();
6166
$queueFile->setStorageId($storageId);
6267
$queueFile->setRootId($rootId);

‎lib/Command/QueueMultimodalFiles.php‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
<?php
2+
3+
/**
4+
* SPDX-FileCopyrightText: 2023 Nextcloud GmbH and Nextcloud contributors
5+
* SPDX-License-Identifier: AGPL-3.0-or-later
6+
*/
7+
8+
namespace OCA\ContextChat\Command;
9+
10+
use OCA\ContextChat\BackgroundJobs\StorageCrawlJob;
11+
use OCA\ContextChat\Logger;
12+
use OCA\ContextChat\Service\StorageService;
13+
use OCA\ContextChat\Service\TaskTypeService;
14+
use OCP\BackgroundJob\IJobList;
15+
use Symfony\Component\Console\Command\Command;
16+
use Symfony\Component\Console\Input\InputInterface;
17+
use Symfony\Component\Console\Output\OutputInterface;
18+
19+
class QueueMultimodalFiles extends Command {
20+
21+
public function __construct(
22+
private TaskTypeService $taskTypeService,
23+
private StorageService $storageService,
24+
private IJobList $jobList,
25+
private Logger $logger,
26+
) {
27+
parent::__construct();
28+
}
29+
30+
protected function configure() {
31+
$this->setName('context_chat:queue-multimodal-files')
32+
->setDescription(
33+
'Queue existing multimodal files (Images and Audio) for indexation.'
34+
. ' Each type of files is queued only if the required TaskProcessing task provider is available.'
35+
. ' OCR for Images and Speech-to-text for Audio.'
36+
. ' See https://docs.nextcloud.com/server/latest/admin_manual/ai/overview.html for more information.'
37+
);
38+
}
39+
40+
protected function execute(InputInterface $input, OutputInterface $output) {
41+
if (!$this->taskTypeService->isOcrTaskTypeAvailable()) {
42+
$output->writeln('<warning>OCR task type is not available.</warning>');
43+
}
44+
if (!$this->taskTypeService->isSpeechToTextTaskTypeAvailable()) {
45+
$output->writeln('<warning>Speech-to-text task type is not available.</warning>');
46+
}
47+
48+
try {
49+
foreach ($this->storageService->getMounts() as $mount) {
50+
$this->logger->debug('Scheduling StorageCrawlJob storage_id=' . $mount['storage_id'] . ' root_id=' . $mount['root_id' ] . 'override_root=' . $mount['overridden_root']);
51+
$this->jobList->add(StorageCrawlJob::class, [
52+
'storage_id' => $mount['storage_id'],
53+
'root_id' => $mount['root_id' ],
54+
'overridden_root' => $mount['overridden_root'],
55+
'last_file_id' => 0,
56+
'only_non_textual' => true,
57+
]);
58+
}
59+
} catch (\Exception $e) {
60+
$this->logger->error('Failed to schedule StorageCrawlJob to find files for indexation.', ['exception' => $e]);
61+
$output->writeln('<error>Failed to schedule StorageCrawlJob to find files for indexation: ' . $e->getMessage() . '</error>');
62+
return 1;
63+
}
64+
65+
$output->writeln('<info>Multimodal files have been scheduled to be queued for indexation.</info>');
66+
return 0;
67+
}
68+
}

‎lib/Listener/ShareListener.php‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,12 +10,12 @@
1010

1111
namespace OCA\ContextChat\Listener;
1212

13-
use OCA\ContextChat\AppInfo\Application;
1413
use OCA\ContextChat\Logger;
1514
use OCA\ContextChat\Public\UpdateAccessOp;
1615
use OCA\ContextChat\Service\ActionScheduler;
1716
use OCA\ContextChat\Service\ProviderConfigService;
1817
use OCA\ContextChat\Service\StorageService;
18+
use OCA\ContextChat\Service\TaskTypeService;
1919
use OCP\EventDispatcher\Event;
2020
use OCP\EventDispatcher\IEventListener;
2121
use OCP\Files\FileInfo;
@@ -37,6 +37,7 @@ public function __construct(
3737
private IManager $shareManager,
3838
private ActionScheduler $actionService,
3939
private IGroupManager $groupManager,
40+
private TaskTypeService $taskTypeService,
4041
) {
4142
}
4243

@@ -145,6 +146,7 @@ public function handle(Event $event): void {
145146

146147
private function allowedMimeType(Node $file): bool {
147148
$mimeType = $file->getMimeType();
148-
return in_array($mimeType, Application::MIMETYPES, true);
149+
$mimeTypes = $this->taskTypeService->getMultimodalMimetypes();
150+
return in_array($mimeType, $mimeTypes, true);
149151
}
150152
}

‎lib/Service/FsEventService.php‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,13 +7,11 @@
77

88
namespace OCA\ContextChat\Service;
99

10-
use OCA\ContextChat\AppInfo\Application;
1110
use OCA\ContextChat\Db\QueueFile;
1211
use OCA\ContextChat\Logger;
1312
use OCP\DB\Exception;
1413
use OCP\Files\Folder;
1514
use OCP\Files\InvalidPathException;
16-
use OCP\Files\IRootFolder;
1715
use OCP\Files\Node;
1816
use OCP\Files\NotFoundException;
1917

@@ -24,7 +22,7 @@ public function __construct(
2422
private QueueService $queue,
2523
private ActionScheduler $actionService,
2624
private StorageService $storageService,
27-
private IRootFolder $rootFolder,
25+
private TaskTypeService $taskTypeService,
2826
) {
2927

3028
}
@@ -134,7 +132,8 @@ public function onInsert(Node $node, bool $recurse = true, bool $update = false)
134132

135133
private function allowedMimeType(Node $file): bool {
136134
$mimeType = $file->getMimeType();
137-
return in_array($mimeType, Application::MIMETYPES, true);
135+
$mimeTypes = $this->taskTypeService->getMultimodalMimetypes();
136+
return in_array($mimeType, $mimeTypes, true);
138137
}
139138

140139
private function allowedPath(Node $file): bool {

‎lib/Service/StorageService.php‎

Lines changed: 36 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@
2020
use OCP\Files\Config\IUserMountCache;
2121
use OCP\Files\Folder;
2222
use OCP\Files\IMimeTypeLoader;
23-
use OCP\Files\IRootFolder;
2423
use OCP\Files\Node;
2524
use OCP\FilesMetadata\IFilesMetadataManager;
2625
use OCP\IDBConnection;
@@ -44,8 +43,8 @@ public function __construct(
4443
private IMimeTypeLoader $mimeTypes,
4544
private IUserMountCache $userMountCache,
4645
private IFilesMetadataManager $metadataManager,
47-
private IRootFolder $rootFolder,
4846
private IFileAccess $fileAccess,
47+
private TaskTypeService $taskTypeService,
4948
) {
5049
}
5150

@@ -84,7 +83,8 @@ public function countFilesInMount(int $storageId, int $rootId): int {
8483
return 0;
8584
}
8685

87-
$mimeTypes = array_map(fn ($mimeType) => $this->mimeTypes->getId($mimeType), Application::MIMETYPES);
86+
$mimeTypes = $this->taskTypeService->getMultimodalMimetypes();
87+
$mimeTypesIds = array_map(fn ($mimeType) => $this->mimeTypes->getId($mimeType), $mimeTypes);
8888

8989
$qb = $this->getCacheQueryBuilder();
9090

@@ -110,7 +110,7 @@ public function countFilesInMount(int $storageId, int $rootId): int {
110110
->andWhere($qb->expr()->notLike('filecache.path', $qb->createNamedParameter('files_versions/%')))
111111
->andWhere($qb->expr()->notLike('filecache.path', $qb->createNamedParameter('files_trashbin/%')))
112112
->andWhere($qb->expr()->eq('filecache.storage', $qb->createNamedParameter($storageId)))
113-
->andWhere($qb->expr()->in('filecache.mimetype', $qb->createNamedParameter($mimeTypes, IQueryBuilder::PARAM_INT_ARRAY)))
113+
->andWhere($qb->expr()->in('filecache.mimetype', $qb->createNamedParameter($mimeTypesIds, IQueryBuilder::PARAM_INT_ARRAY)))
114114
->andWhere($qb->expr()->lte('filecache.size', $qb->createNamedParameter(Application::CC_MAX_SIZE, IQueryBuilder::PARAM_INT)))
115115
->andWhere($qb->expr()->gt('filecache.size', $qb->createNamedParameter(0, IQueryBuilder::PARAM_INT)));
116116
$result = $qb->executeQuery();
@@ -199,25 +199,42 @@ private function getMountsOld(): \Generator {
199199
* @param int $rootId
200200
* @param int $lastFileId
201201
* @param int $maxResults
202+
* @param list<string> $mimeTypes
202203
* @return \Generator<int,int,mixed,void>
203204
*/
204-
public function getFilesInMount(int $storageId, int $rootId, int $lastFileId = 0, int $maxResults = 100): \Generator {
205+
public function getFilesInMount(
206+
int $storageId,
207+
int $rootId,
208+
int $lastFileId = 0,
209+
int $maxResults = 100,
210+
array $mimeTypes = [],
211+
): \Generator {
212+
if ($mimeTypes === []) {
213+
$mimeTypes = $this->taskTypeService->getMultimodalMimetypes();
214+
}
205215
if (!$this->isFileAccessAvailable()) {
206-
return $this->getFilesInMountOld($storageId, $rootId, $lastFileId, $maxResults);
216+
return $this->getFilesInMountOld($storageId, $rootId, $lastFileId, $maxResults, $mimeTypes);
207217
}
208218

209-
return $this->getFilesInMountUsingFileAccess($storageId, $rootId, $lastFileId, $maxResults);
219+
return $this->getFilesInMountUsingFileAccess($storageId, $rootId, $lastFileId, $maxResults, $mimeTypes);
210220
}
211221

212222
/**
213223
* @param int $storageId
214224
* @param int $rootId
215225
* @param int $lastFileId
216226
* @param int $maxResults
227+
* @param list<string> $mimeTypes
217228
* @return \Generator<int,int,mixed,void>
218229
*/
219-
private function getFilesInMountUsingFileAccess(int $storageId, int $rootId, int $lastFileId = 0, int $maxResults = 100): \Generator {
220-
$mimeTypeIds = array_map(fn ($mimeType) => $this->mimeTypes->getId($mimeType), Application::MIMETYPES);
230+
private function getFilesInMountUsingFileAccess(
231+
int $storageId,
232+
int $rootId,
233+
int $lastFileId = 0,
234+
int $maxResults = 100,
235+
array $mimeTypes = Application::MIMETYPES,
236+
): \Generator {
237+
$mimeTypeIds = array_map(fn ($mimeType) => $this->mimeTypes->getId($mimeType), $mimeTypes);
221238
foreach ($this->fileAccess->getByAncestorInStorage($storageId, $rootId, $lastFileId, $maxResults, $mimeTypeIds, false, true) as $cacheEntry) {
222239
yield $cacheEntry['fileid'];
223240
}
@@ -228,9 +245,16 @@ private function getFilesInMountUsingFileAccess(int $storageId, int $rootId, int
228245
* @param int $rootId
229246
* @param int $lastFileId
230247
* @param int $maxResults
248+
* @param list<string> $mimeTypes
231249
* @return \Generator<int,int,mixed,void>
232250
*/
233-
private function getFilesInMountOld(int $storageId, int $rootId, int $lastFileId = 0, int $maxResults = 100): \Generator {
251+
private function getFilesInMountOld(
252+
int $storageId,
253+
int $rootId,
254+
int $lastFileId = 0,
255+
int $maxResults = 100,
256+
array $mimeTypes = Application::MIMETYPES,
257+
): \Generator {
234258
$qb = $this->getCacheQueryBuilder();
235259
try {
236260
$qb->selectFileCache();
@@ -249,7 +273,7 @@ private function getFilesInMountOld(int $storageId, int $rootId, int $lastFileId
249273
return;
250274
}
251275

252-
$mimeTypes = array_map(fn ($mimeType) => $this->mimeTypes->getId($mimeType), Application::MIMETYPES);
276+
$mimeTypesIds = array_map(fn ($mimeType) => $this->mimeTypes->getId($mimeType), $mimeTypes);
253277

254278
$qb = $this->getCacheQueryBuilder();
255279

@@ -272,7 +296,7 @@ private function getFilesInMountOld(int $storageId, int $rootId, int $lastFileId
272296
->andWhere($qb->expr()->like('filecache.path', $qb->createNamedParameter($path . '%')))
273297
->andWhere($qb->expr()->eq('filecache.storage', $qb->createNamedParameter($storageId)))
274298
->andWhere($qb->expr()->gt('filecache.fileid', $qb->createNamedParameter($lastFileId)))
275-
->andWhere($qb->expr()->in('filecache.mimetype', $qb->createNamedParameter($mimeTypes, IQueryBuilder::PARAM_INT_ARRAY)));
299+
->andWhere($qb->expr()->in('filecache.mimetype', $qb->createNamedParameter($mimeTypesIds, IQueryBuilder::PARAM_INT_ARRAY)));
276300

277301
if ($maxResults !== 0) {
278302
$qb->setMaxResults($maxResults);

0 commit comments

Comments
 (0)