Skip to content

Commit 1261cb4

Browse files
CarlSchwanCarlSchwan
committed
perf(preview): Optimize retriving all previews from oc_filecache
Previoulsy we used a PATH LIKE expression which is fine for small instances but doesn't scale for big instance (timeout). Manually moving accross the tree with getFolderContentsById is significantly faster as we can use the index and this also reuse common APIs from OCP/Files instead of directly manipulating the filecache with the query builder. Assisted-by: ClaudeCode:claude-opus-4-8 Signed-off-by: Carl Schwan <carl@carlschwan.eu>
1 parent 05cc8d9 commit 1261cb4

3 files changed

Lines changed: 50 additions & 55 deletions

File tree

‎core/BackgroundJobs/PreviewMigrationJob.php‎

Lines changed: 48 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -9,16 +9,15 @@
99

1010
namespace OC\Core\BackgroundJobs;
1111

12-
use OC\Preview\Db\Preview;
1312
use OC\Preview\PreviewMigrationService;
1413
use OCP\AppFramework\Utility\ITimeFactory;
1514
use OCP\BackgroundJob\TimedJob;
16-
use OCP\DB\IResult;
15+
use OCP\Files\FileInfo;
1716
use OCP\Files\IRootFolder;
1817
use OCP\IAppConfig;
1918
use OCP\IConfig;
20-
use OCP\IDBConnection;
2119
use Override;
20+
use Psr\Log\LoggerInterface;
2221

2322
class PreviewMigrationJob extends TimedJob {
2423
private string $previewRootPath;
@@ -27,9 +26,9 @@ public function __construct(
2726
ITimeFactory $time,
2827
private readonly IAppConfig $appConfig,
2928
private readonly IConfig $config,
30-
private readonly IDBConnection $connection,
3129
private readonly IRootFolder $rootFolder,
3230
private readonly PreviewMigrationService $migrationService,
31+
private readonly LoggerInterface $logger,
3332
) {
3433
parent::__construct($time);
3534

@@ -44,23 +43,52 @@ protected function run(mixed $argument): void {
4443
return;
4544
}
4645

46+
$storage = $this->rootFolder->getMountPoint()->getStorage();
47+
if ($storage === null) {
48+
$this->appConfig->setValueBool('core', 'previewMovedDone', true);
49+
return;
50+
}
51+
52+
$cache = $storage->getCache();
53+
$previewRootId = $cache->getId(rtrim($this->previewRootPath, '/'));
54+
if ($previewRootId === -1) {
55+
// No previews have ever been generated on this instance.
56+
$this->appConfig->setValueBool('core', 'previewMovedDone', true);
57+
return;
58+
}
59+
4760
$startTime = time();
48-
while (true) {
49-
$qb = $this->connection->getQueryBuilder();
50-
$qb->select('path')
51-
->from('filecache')
52-
// Hierarchical preview folder structure
53-
->where($qb->expr()->like('path', $qb->createNamedParameter($this->previewRootPath . '%/%/%/%/%/%/%/%/%')))
54-
// Legacy flat preview folder structure
55-
->orWhere($qb->expr()->like('path', $qb->createNamedParameter($this->previewRootPath . '%/%.%')))
56-
->hintShardKey('storage', $this->rootFolder->getMountPoint()->getNumericStorageId())
57-
->setMaxResults(100);
58-
59-
$result = $qb->executeQuery();
60-
$foundPreviews = $this->processQueryResult($result);
61-
62-
if (!$foundPreviews) {
63-
break;
61+
62+
// Walk the preview folder tree via the `parent` column, which is indexed on
63+
// every supported database platform.
64+
//
65+
// Depth from the preview root tells us which structure a leaf folder holds:
66+
// - depth 1: legacy flat structure, e.g. preview/<fileid>/<size>.png
67+
// - depth 8: hierarchical structure, e.g. preview/a/b/c/d/e/f/g/<fileid>/<size>.png
68+
$foldersToVisit = [[$previewRootId, '', 0]];
69+
70+
while ($foldersToVisit !== []) {
71+
[$folderId, $folderName, $depth] = array_pop($foldersToVisit);
72+
73+
$hasPreviewFiles = false;
74+
foreach ($cache->getFolderContentsById($folderId) as $entry) {
75+
if ($entry->getMimeType() === FileInfo::MIMETYPE_FOLDER) {
76+
$foldersToVisit[] = [$entry->getId(), $entry->getName(), $depth + 1];
77+
} else {
78+
$hasPreviewFiles = true;
79+
}
80+
}
81+
82+
if (!$hasPreviewFiles || !ctype_digit($folderName)) {
83+
continue;
84+
}
85+
86+
try {
87+
$this->migrationService->migrateFileId((int)$folderName, flatPath: $depth === 1);
88+
} catch (\Exception $e) {
89+
$this->logger->error('Failed to migrate preview with fileId: ' . $folderName, [
90+
'exception' => $e,
91+
]);
6492
}
6593

6694
// Stop if execution time is more than one hour.
@@ -71,36 +99,4 @@ protected function run(mixed $argument): void {
7199

72100
$this->appConfig->setValueBool('core', 'previewMovedDone', true);
73101
}
74-
75-
private function processQueryResult(IResult $result): bool {
76-
$foundPreview = false;
77-
$fileIds = [];
78-
$flatFileIds = [];
79-
while ($row = $result->fetch()) {
80-
$pathSplit = explode('/', $row['path']);
81-
assert(count($pathSplit) >= 2);
82-
$fileId = (int)$pathSplit[count($pathSplit) - 2];
83-
if (count($pathSplit) === 11) {
84-
// Hierarchical structure
85-
if (!in_array($fileId, $fileIds)) {
86-
$fileIds[] = $fileId;
87-
}
88-
} else {
89-
// Flat structure
90-
if (!in_array($fileId, $flatFileIds)) {
91-
$flatFileIds[] = $fileId;
92-
}
93-
}
94-
$foundPreview = true;
95-
}
96-
97-
foreach ($fileIds as $fileId) {
98-
$this->migrationService->migrateFileId($fileId, flatPath: false);
99-
}
100-
101-
foreach ($flatFileIds as $fileId) {
102-
$this->migrationService->migrateFileId($fileId, flatPath: true);
103-
}
104-
return $foundPreview;
105-
}
106102
}

‎tests/lib/Preview/BackgroundCleanupJobTest.php‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ class BackgroundCleanupJobTest extends \Test\TestCase {
4141
private ITimeFactory $timeFactory;
4242
private PreviewService $previewService;
4343

44+
#[\Override]
4445
protected function setUp(): void {
4546
parent::setUp();
4647

@@ -66,6 +67,7 @@ protected function setUp(): void {
6667
$this->previewService = Server::get(PreviewService::class);
6768
}
6869

70+
#[\Override]
6971
protected function tearDown(): void {
7072
if ($this->trashEnabled) {
7173
$appManager = Server::get(IAppManager::class);

‎tests/lib/Preview/PreviewMigrationJobTest.php‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -116,7 +116,6 @@ public function testMigrationLegacyPath(): void {
116116
Server::get(ITimeFactory::class),
117117
$this->appConfig,
118118
$this->config,
119-
Server::get(IDBConnection::class),
120119
Server::get(IRootFolder::class),
121120
new PreviewMigrationService(
122121
$this->config,
@@ -153,7 +152,6 @@ public function testMigrationPath(): void {
153152
Server::get(ITimeFactory::class),
154153
$this->appConfig,
155154
$this->config,
156-
Server::get(IDBConnection::class),
157155
Server::get(IRootFolder::class),
158156
new PreviewMigrationService(
159157
$this->config,
@@ -198,7 +196,6 @@ public function testMigrationPathWithVersion(): void {
198196
Server::get(ITimeFactory::class),
199197
$this->appConfig,
200198
$this->config,
201-
Server::get(IDBConnection::class),
202199
Server::get(IRootFolder::class),
203200
new PreviewMigrationService(
204201
$this->config,

0 commit comments

Comments
 (0)