Skip to content

Commit 2b7c149

Browse files
CarlSchwanCarlSchwan
committed
perf(preview): Optimize retriving all previews from oc_filecache
Previoulsy we used a PATH LIKE expression which is fine for small instances but doesn't scale for big instance (timeout). Manually moving accross the tree with getFolderContentsById is significantly faster as we can use the index and this also reuse common APIs from OCP/Files instead of directly manipulating the filecache with the query builder. Assisted-by: ClaudeCode:claude-opus-4-8 Signed-off-by: Carl Schwan <carl@carlschwan.eu>
1 parent 05cc8d9 commit 2b7c149

2 files changed

Lines changed: 57 additions & 58 deletions

File tree

‎core/BackgroundJobs/PreviewMigrationJob.php‎

Lines changed: 48 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -9,16 +9,15 @@
99

1010
namespace OC\Core\BackgroundJobs;
1111

12-
use OC\Preview\Db\Preview;
1312
use OC\Preview\PreviewMigrationService;
1413
use OCP\AppFramework\Utility\ITimeFactory;
1514
use OCP\BackgroundJob\TimedJob;
16-
use OCP\DB\IResult;
15+
use OCP\Files\FileInfo;
1716
use OCP\Files\IRootFolder;
1817
use OCP\IAppConfig;
1918
use OCP\IConfig;
20-
use OCP\IDBConnection;
2119
use Override;
20+
use Psr\Log\LoggerInterface;
2221

2322
class PreviewMigrationJob extends TimedJob {
2423
private string $previewRootPath;
@@ -27,9 +26,9 @@ public function __construct(
2726
ITimeFactory $time,
2827
private readonly IAppConfig $appConfig,
2928
private readonly IConfig $config,
30-
private readonly IDBConnection $connection,
3129
private readonly IRootFolder $rootFolder,
3230
private readonly PreviewMigrationService $migrationService,
31+
private readonly LoggerInterface $logger,
3332
) {
3433
parent::__construct($time);
3534

@@ -44,23 +43,52 @@ protected function run(mixed $argument): void {
4443
return;
4544
}
4645

46+
$storage = $this->rootFolder->getMountPoint()->getStorage();
47+
if ($storage === null) {
48+
$this->appConfig->setValueBool('core', 'previewMovedDone', true);
49+
return;
50+
}
51+
52+
$cache = $storage->getCache();
53+
$previewRootId = $cache->getId(rtrim($this->previewRootPath, '/'));
54+
if ($previewRootId === -1) {
55+
// No previews have ever been generated on this instance.
56+
$this->appConfig->setValueBool('core', 'previewMovedDone', true);
57+
return;
58+
}
59+
4760
$startTime = time();
48-
while (true) {
49-
$qb = $this->connection->getQueryBuilder();
50-
$qb->select('path')
51-
->from('filecache')
52-
// Hierarchical preview folder structure
53-
->where($qb->expr()->like('path', $qb->createNamedParameter($this->previewRootPath . '%/%/%/%/%/%/%/%/%')))
54-
// Legacy flat preview folder structure
55-
->orWhere($qb->expr()->like('path', $qb->createNamedParameter($this->previewRootPath . '%/%.%')))
56-
->hintShardKey('storage', $this->rootFolder->getMountPoint()->getNumericStorageId())
57-
->setMaxResults(100);
58-
59-
$result = $qb->executeQuery();
60-
$foundPreviews = $this->processQueryResult($result);
61-
62-
if (!$foundPreviews) {
63-
break;
61+
62+
// Walk the preview folder tree via the `parent` column, which is indexed on
63+
// every supported database platform.
64+
//
65+
// Depth from the preview root tells us which structure a leaf folder holds:
66+
// - depth 1: legacy flat structure, e.g. preview/<fileid>/<size>.png
67+
// - depth 8: hierarchical structure, e.g. preview/a/b/c/d/e/f/g/<fileid>/<size>.png
68+
$foldersToVisit = [[$previewRootId, '', 0]];
69+
70+
while ($foldersToVisit !== []) {
71+
[$folderId, $folderName, $depth] = array_pop($foldersToVisit);
72+
73+
$hasPreviewFiles = false;
74+
foreach ($cache->getFolderContentsById($folderId) as $entry) {
75+
if ($entry->getMimeType() === FileInfo::MIMETYPE_FOLDER) {
76+
$foldersToVisit[] = [$entry->getId(), $entry->getName(), $depth + 1];
77+
} else {
78+
$hasPreviewFiles = true;
79+
}
80+
}
81+
82+
if (!$hasPreviewFiles || !ctype_digit($folderName)) {
83+
continue;
84+
}
85+
86+
try {
87+
$this->migrationService->migrateFileId((int)$folderName, flatPath: $depth === 1);
88+
} catch (\Exception $e) {
89+
$this->logger->error('Failed to migrate preview with fileId: ' . $folderName, [
90+
'exception' => $e,
91+
]);
6492
}
6593

6694
// Stop if execution time is more than one hour.
@@ -71,36 +99,4 @@ protected function run(mixed $argument): void {
7199

72100
$this->appConfig->setValueBool('core', 'previewMovedDone', true);
73101
}
74-
75-
private function processQueryResult(IResult $result): bool {
76-
$foundPreview = false;
77-
$fileIds = [];
78-
$flatFileIds = [];
79-
while ($row = $result->fetch()) {
80-
$pathSplit = explode('/', $row['path']);
81-
assert(count($pathSplit) >= 2);
82-
$fileId = (int)$pathSplit[count($pathSplit) - 2];
83-
if (count($pathSplit) === 11) {
84-
// Hierarchical structure
85-
if (!in_array($fileId, $fileIds)) {
86-
$fileIds[] = $fileId;
87-
}
88-
} else {
89-
// Flat structure
90-
if (!in_array($fileId, $flatFileIds)) {
91-
$flatFileIds[] = $fileId;
92-
}
93-
}
94-
$foundPreview = true;
95-
}
96-
97-
foreach ($fileIds as $fileId) {
98-
$this->migrationService->migrateFileId($fileId, flatPath: false);
99-
}
100-
101-
foreach ($flatFileIds as $fileId) {
102-
$this->migrationService->migrateFileId($fileId, flatPath: true);
103-
}
104-
return $foundPreview;
105-
}
106102
}

‎tests/lib/Preview/PreviewMigrationJobTest.php‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ class PreviewMigrationJobTest extends TestCase {
4242
private IMimeTypeDetector&MockObject $mimeTypeDetector;
4343
private LoggerInterface&MockObject $logger;
4444

45+
#[\Override]
4546
public function setUp(): void {
4647
parent::setUp();
4748
$this->previewAppData = Server::get(IAppDataFactory::class)->get('preview');
@@ -91,6 +92,7 @@ public function setUp(): void {
9192
$this->logger = $this->createMock(LoggerInterface::class);
9293
}
9394

95+
#[\Override]
9496
public function tearDown(): void {
9597
foreach ($this->previewAppData->getDirectoryListing() as $folder) {
9698
$folder->delete();
@@ -101,6 +103,7 @@ public function tearDown(): void {
101103
$qb->delete('filecache')
102104
->where($qb->expr()->eq('fileid', $qb->createNamedParameter(5)))
103105
->executeStatement();
106+
parent::tearDown();
104107
}
105108

106109
#[TestDox('Test the migration from the legacy flat hierarchy to the new database format')]
@@ -116,7 +119,6 @@ public function testMigrationLegacyPath(): void {
116119
Server::get(ITimeFactory::class),
117120
$this->appConfig,
118121
$this->config,
119-
Server::get(IDBConnection::class),
120122
Server::get(IRootFolder::class),
121123
new PreviewMigrationService(
122124
$this->config,
@@ -128,7 +130,8 @@ public function testMigrationLegacyPath(): void {
128130
$this->previewMapper,
129131
$this->storageFactory,
130132
Server::get(IAppDataFactory::class),
131-
)
133+
),
134+
$this->logger,
132135
);
133136
$this->invokePrivate($job, 'run', [[]]);
134137
$this->assertEquals(0, count($this->previewAppData->getDirectoryListing()));
@@ -153,7 +156,6 @@ public function testMigrationPath(): void {
153156
Server::get(ITimeFactory::class),
154157
$this->appConfig,
155158
$this->config,
156-
Server::get(IDBConnection::class),
157159
Server::get(IRootFolder::class),
158160
new PreviewMigrationService(
159161
$this->config,
@@ -165,7 +167,8 @@ public function testMigrationPath(): void {
165167
$this->previewMapper,
166168
$this->storageFactory,
167169
Server::get(IAppDataFactory::class),
168-
)
170+
),
171+
$this->logger,
169172
);
170173
$this->invokePrivate($job, 'run', [[]]);
171174
$this->assertEquals(0, count($this->previewAppData->getDirectoryListing()));
@@ -198,7 +201,6 @@ public function testMigrationPathWithVersion(): void {
198201
Server::get(ITimeFactory::class),
199202
$this->appConfig,
200203
$this->config,
201-
Server::get(IDBConnection::class),
202204
Server::get(IRootFolder::class),
203205
new PreviewMigrationService(
204206
$this->config,
@@ -210,7 +212,8 @@ public function testMigrationPathWithVersion(): void {
210212
$this->previewMapper,
211213
$this->storageFactory,
212214
Server::get(IAppDataFactory::class),
213-
)
215+
),
216+
$this->logger,
214217
);
215218
$this->invokePrivate($job, 'run', [[]]);
216219
$previews = iterator_to_array($this->previewMapper->getAvailablePreviewsForFile(5));

0 commit comments

Comments
 (0)