Skip to content

Commit f1ecb50

Browse files
AndyScherzingerAndyScherzinger
authored
Merge pull request #62611 from nextcloud/backport/62601/stable34
[stable34] perf(preview): Optimize retriving all previews from oc_filecache
2 parents 803e484 + 1909e73 commit f1ecb50

7 files changed

Lines changed: 260 additions & 143 deletions

File tree

‎core/BackgroundJobs/PreviewMigrationJob.php‎

Lines changed: 43 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -13,11 +13,10 @@
1313
use OC\Preview\PreviewMigrationService;
1414
use OCP\AppFramework\Utility\ITimeFactory;
1515
use OCP\BackgroundJob\TimedJob;
16-
use OCP\DB\IResult;
16+
use OCP\Files\FileInfo;
1717
use OCP\Files\IRootFolder;
1818
use OCP\IAppConfig;
1919
use OCP\IConfig;
20-
use OCP\IDBConnection;
2120
use Override;
2221
use Psr\Log\LoggerInterface;
2322

@@ -28,7 +27,6 @@ public function __construct(
2827
ITimeFactory $time,
2928
private readonly IAppConfig $appConfig,
3029
private readonly IConfig $config,
31-
private readonly IDBConnection $connection,
3230
private readonly IRootFolder $rootFolder,
3331
private readonly PreviewMigrationService $migrationService,
3432
private readonly LoggerInterface $logger,
@@ -46,79 +44,62 @@ protected function run(mixed $argument): void {
4644
return;
4745
}
4846

49-
$startTime = time();
50-
while (true) {
51-
$qb = $this->connection->getQueryBuilder();
52-
$qb->select('path')
53-
->from('filecache')
54-
->where($qb->expr()->orX(
55-
// Hierarchical preview folder structure
56-
$qb->expr()->like('path', $qb->createNamedParameter($this->previewRootPath . '%/%/%/%/%/%/%/%/%')),
57-
// Legacy flat preview folder structure
58-
$qb->expr()->like('path', $qb->createNamedParameter($this->previewRootPath . '%/%.%'))
59-
))->andWhere(
60-
$qb->expr()->eq('storage', $qb->createNamedParameter($this->rootFolder->getMountPoint()->getNumericStorageId()))
61-
)
62-
->hintShardKey('storage', $this->rootFolder->getMountPoint()->getNumericStorageId())
63-
->setMaxResults(100);
64-
65-
$result = $qb->executeQuery();
66-
$foundPreviews = $this->processQueryResult($result);
67-
68-
if (!$foundPreviews) {
69-
break;
70-
}
47+
$storage = $this->rootFolder->getMountPoint()->getStorage();
48+
if ($storage === null) {
49+
$this->appConfig->setValueBool('core', 'previewMovedDone', true);
50+
return;
51+
}
7152

72-
// Stop if execution time is more than one hour.
73-
if (time() - $startTime > 3600) {
74-
return;
75-
}
53+
$cache = $storage->getCache();
54+
$previewRootId = $cache->getId(rtrim($this->previewRootPath, '/'));
55+
if ($previewRootId === -1) {
56+
// No previews have ever been generated on this instance.
57+
$this->appConfig->setValueBool('core', 'previewMovedDone', true);
58+
return;
7659
}
7760

78-
$this->appConfig->setValueBool('core', 'previewMovedDone', true);
79-
}
61+
$startTime = time();
8062

81-
private function processQueryResult(IResult $result): bool {
82-
$foundPreview = false;
83-
$fileIds = [];
84-
$flatFileIds = [];
85-
while ($row = $result->fetch()) {
86-
$pathSplit = explode('/', $row['path']);
87-
assert(count($pathSplit) >= 2);
88-
$fileId = (int)$pathSplit[count($pathSplit) - 2];
89-
if (count($pathSplit) === 11) {
90-
// Hierarchical structure
91-
if (!in_array($fileId, $fileIds)) {
92-
$fileIds[] = $fileId;
93-
}
94-
} else {
95-
// Flat structure
96-
if (!in_array($fileId, $flatFileIds)) {
97-
$flatFileIds[] = $fileId;
63+
// Walk the preview folder tree via the `parent` column, which is indexed on
64+
// every supported database platform.
65+
//
66+
// Depth from the preview root tells us which structure a leaf folder holds:
67+
// - depth 1: legacy flat structure, e.g. preview/<fileid>/<size>.png
68+
// - depth 8: hierarchical structure, e.g. preview/a/b/c/d/e/f/g/<fileid>/<size>.png
69+
$foldersToVisit = [[$previewRootId, '', 0]];
70+
71+
while ($foldersToVisit !== []) {
72+
[$folderId, $folderName, $depth] = array_pop($foldersToVisit);
73+
74+
// Collect the actual preview files here so migrateFileId() doesn't need to
75+
// list this folder's contents a second time.
76+
$previewEntries = [];
77+
foreach ($cache->getFolderContentsById($folderId) as $entry) {
78+
if ($entry->getMimeType() === FileInfo::MIMETYPE_FOLDER) {
79+
$foldersToVisit[] = [$entry->getId(), $entry->getName(), $depth + 1];
80+
} else {
81+
$previewEntries[] = $entry;
9882
}
9983
}
100-
$foundPreview = true;
101-
}
10284

103-
foreach ($fileIds as $fileId) {
104-
try {
105-
$this->migrationService->migrateFileId($fileId, flatPath: false);
106-
} catch (\Exception $e) {
107-
$this->logger->error('Failed to migrate preview with fileId: ' . $fileId . ' (hierarchical file structure)', [
108-
'exception' => $e,
109-
]);
85+
if ($previewEntries === [] || !ctype_digit($folderName)) {
86+
continue;
11087
}
111-
}
11288

113-
foreach ($flatFileIds as $fileId) {
11489
try {
115-
$this->migrationService->migrateFileId($fileId, flatPath: true);
90+
$this->migrationService->migrateFileId((int)$folderName, flatPath: $depth === 1, entries: $previewEntries);
11691
} catch (\Exception $e) {
117-
$this->logger->error('Failed to migrate preview with fileId: ' . $fileId . ' (legacy file structure)', [
92+
$this->logger->error('Failed to migrate preview with fileId: ' . $folderName, [
11893
'exception' => $e,
11994
]);
12095
}
96+
97+
// Stop if execution time is more than one hour.
98+
if (time() - $startTime > 3600) {
99+
return;
100+
}
121101
}
122-
return $foundPreview;
102+
103+
$this->appConfig->setValueBool('core', 'previewMovedDone', true);
123104
}
124105
}

‎lib/private/Preview/PreviewMigrationService.php‎

Lines changed: 101 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,9 @@
1414
use OC\Preview\Db\PreviewMapper;
1515
use OC\Preview\Storage\StorageFactory;
1616
use OCP\DB\Exception;
17+
use OCP\DB\QueryBuilder\IQueryBuilder;
1718
use OCP\Files\AppData\IAppDataFactory;
19+
use OCP\Files\Cache\ICacheEntry;
1820
use OCP\Files\IAppData;
1921
use OCP\Files\IMimeTypeDetector;
2022
use OCP\Files\IMimeTypeLoader;
@@ -44,44 +46,43 @@ public function __construct(
4446
}
4547

4648
/**
47-
* @param array<string|int, string[]> $previewFolders
49+
* @param list<ICacheEntry|SimpleFile>|null $entries Preview file entries already fetched by the caller.
4850
* @return Preview[]
4951
*/
50-
public function migrateFileId(int $fileId, bool $flatPath): array {
52+
public function migrateFileId(int $fileId, bool $flatPath, ?array $entries = null): array {
5153
$previews = [];
5254
$internalPath = $this->getInternalFolder((string)$fileId, $flatPath);
53-
try {
54-
$folder = $this->appData->getFolder($internalPath);
55-
} catch (NotFoundException) {
56-
return [];
55+
56+
if ($entries === null) {
57+
try {
58+
$entries = $this->appData->getFolder($internalPath)->getDirectoryListing();
59+
} catch (NotFoundException) {
60+
return [];
61+
}
5762
}
5863

5964
/**
60-
* @var list<array{file: SimpleFile, preview: Preview}> $previewFiles
65+
* @var list<Preview> $previewsToInsert
6166
*/
62-
$previewFiles = [];
67+
$previewsToInsert = [];
6368

64-
foreach ($folder->getDirectoryListing() as $previewFile) {
65-
$path = $fileId . '/' . $previewFile->getName();
66-
/** @var SimpleFile $previewFile */
69+
foreach ($entries as $entry) {
70+
$path = $fileId . '/' . $entry->getName();
6771
$preview = Preview::fromPath($path, $this->mimeTypeDetector);
6872
if ($preview === false) {
6973
$this->logger->error('Unable to import old preview at path.');
7074
continue;
7175
}
7276
$preview->generateId();
73-
$preview->setSize($previewFile->getSize());
74-
$preview->setMtime($previewFile->getMtime());
75-
$preview->setOldFileId($previewFile->getId());
77+
$preview->setSize($entry->getSize());
78+
$preview->setMtime($entry->getMTime());
79+
$preview->setOldFileId($entry->getId());
7680
$preview->setEncrypted(false);
7781

78-
$previewFiles[] = [
79-
'file' => $previewFile,
80-
'preview' => $preview,
81-
];
82+
$previewsToInsert[] = $preview;
8283
}
8384

84-
if (empty($previewFiles)) {
85+
if (empty($previewsToInsert)) {
8586
$this->deleteFolder($internalPath);
8687

8788
return $previews;
@@ -98,56 +99,51 @@ public function migrateFileId(int $fileId, bool $flatPath): array {
9899
$cursor->closeCursor();
99100

100101
if ($result !== false) {
101-
foreach ($previewFiles as $previewFile) {
102-
/** @var Preview $preview */
103-
$preview = $previewFile['preview'];
104-
/** @var SimpleFile $file */
105-
$file = $previewFile['file'];
106-
$preview->setStorageId($result['storage']);
107-
$preview->setEtag($result['etag']);
108-
$preview->setSourceMimeType($this->mimeTypeLoader->getMimetypeById((int)$result['mimetype']));
109-
$preview->generateId();
110-
try {
111-
$preview = $this->previewMapper->insert($preview);
112-
} catch (Exception $e) {
113-
if ($e->getReason() !== Exception::REASON_UNIQUE_CONSTRAINT_VIOLATION) {
114-
throw $e;
102+
$oldFileIdsToDelete = [];
103+
try {
104+
foreach ($previewsToInsert as $preview) {
105+
$preview->setStorageId($result['storage']);
106+
$preview->setEtag($result['etag']);
107+
$preview->setSourceMimeType($this->mimeTypeLoader->getMimetypeById((int)$result['mimetype']));
108+
$preview->generateId();
109+
try {
110+
$preview = $this->previewMapper->insert($preview);
111+
} catch (Exception $e) {
112+
if ($e->getReason() !== Exception::REASON_UNIQUE_CONSTRAINT_VIOLATION) {
113+
throw $e;
114+
}
115+
116+
// We already have this preview in the preview table, skip
117+
$oldFileIdsToDelete[] = $preview->getOldFileId();
118+
continue;
115119
}
116120

117-
$delete = $this->connection->getQueryBuilder();
118-
// We already have this preview in the preview table, skip
119-
$delete->delete('filecache')
120-
->where($delete->expr()->eq('fileid', $delete->createNamedParameter($file->getId())))
121-
->hintShardKey('storage', $this->rootFolder->getMountPoint()->getNumericStorageId())
122-
->executeStatement();
123-
continue;
124-
}
121+
try {
122+
$this->storageFactory->migratePreview($preview);
123+
// Do not delete the old file via a Node here, as that would also
124+
// delete it from the file system; only its filecache row is stale.
125+
} catch (\Exception $e) {
126+
$this->previewMapper->delete($preview);
127+
throw $e;
128+
}
125129

126-
try {
127-
$this->storageFactory->migratePreview($preview, $file);
128-
$qb = $this->connection->getQueryBuilder();
129-
$qb->delete('filecache')
130-
->where($qb->expr()->eq('fileid', $qb->createNamedParameter($file->getId())))
131-
->hintShardKey('storage', $this->rootFolder->getMountPoint()->getNumericStorageId())
132-
->executeStatement();
133-
// Do not call $file->delete() as this will also delete the file from the file system
134-
} catch (\Exception $e) {
135-
$this->previewMapper->delete($preview);
136-
throw $e;
130+
$oldFileIdsToDelete[] = $preview->getOldFileId();
131+
$previews[] = $preview;
137132
}
138-
139-
$previews[] = $preview;
133+
} finally {
134+
$this->deleteOldFileCacheEntries($oldFileIdsToDelete);
140135
}
141136
} else {
142-
// No matching fileId, delete preview
137+
// No matching fileId, delete the orphaned preview files themselves.
143138
try {
139+
$folder = $this->appData->getFolder($internalPath);
144140
$this->connection->beginTransaction();
145-
foreach ($previewFiles as $previewFile) {
146-
/** @var SimpleFile $file */
147-
$file = $previewFile['file'];
141+
foreach ($folder->getDirectoryListing() as $file) {
148142
$file->delete();
149143
}
150144
$this->connection->commit();
145+
} catch (NotFoundException) {
146+
// Folder already gone, nothing to clean up.
151147
} catch (Exception) {
152148
$this->connection->rollback();
153149
}
@@ -165,6 +161,23 @@ private static function getInternalFolder(string $name, bool $flatPath): string
165161
return implode('/', str_split(substr(md5($name), 0, 7))) . '/' . $name;
166162
}
167163

164+
/**
165+
* @param list<int> $fileIds
166+
*/
167+
private function deleteOldFileCacheEntries(array $fileIds): void {
168+
if ($fileIds === []) {
169+
return;
170+
}
171+
172+
foreach (array_chunk($fileIds, 1000) as $chunk) {
173+
$qb = $this->connection->getQueryBuilder();
174+
$qb->delete('filecache')
175+
->where($qb->expr()->in('fileid', $qb->createNamedParameter($chunk, IQueryBuilder::PARAM_INT_ARRAY)))
176+
->hintShardKey('storage', $this->rootFolder->getMountPoint()->getNumericStorageId())
177+
->executeStatement();
178+
}
179+
}
180+
168181
private function deleteFolder(string $path): void {
169182
$current = $path;
170183

@@ -185,14 +198,38 @@ private function deleteFolder(string $path): void {
185198
break;
186199
}
187200

188-
try {
189-
$folder = $this->appData->getFolder($current);
190-
} catch (NotFoundException) {
191-
break;
192-
}
193-
if (count($folder->getDirectoryListing()) !== 0) {
201+
if ($this->folderHasChildren($rootFolderId, $this->previewRootPath . $current)) {
194202
break;
195203
}
196204
}
197205
}
206+
207+
private function folderHasChildren(int $storageId, string $path): bool {
208+
$qb = $this->connection->getQueryBuilder();
209+
$qb->select('fileid')
210+
->from('filecache')
211+
->where($qb->expr()->eq('path_hash', $qb->createNamedParameter(md5($path))))
212+
->andWhere($qb->expr()->eq('storage', $qb->createNamedParameter($storageId)))
213+
->setMaxResults(1);
214+
$cursor = $qb->executeQuery();
215+
$folderId = $cursor->fetchOne();
216+
$cursor->closeCursor();
217+
218+
if ($folderId === false) {
219+
// The folder itself is already gone, nothing to check.
220+
return false;
221+
}
222+
223+
$qb = $this->connection->getQueryBuilder();
224+
$qb->select('fileid')
225+
->from('filecache')
226+
->where($qb->expr()->eq('parent', $qb->createNamedParameter((int)$folderId)))
227+
->andWhere($qb->expr()->eq('storage', $qb->createNamedParameter($storageId)))
228+
->setMaxResults(1);
229+
$cursor = $qb->executeQuery();
230+
$hasChild = $cursor->fetchOne() !== false;
231+
$cursor->closeCursor();
232+
233+
return $hasChild;
234+
}
198235
}

‎lib/private/Preview/Storage/IPreviewStorage.php‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@
1010
namespace OC\Preview\Storage;
1111

1212
use Exception;
13-
use OC\Files\SimpleFS\SimpleFile;
1413
use OC\Preview\Db\Preview;
1514
use OCP\Files\NotFoundException;
1615
use OCP\Files\NotPermittedException;
@@ -42,7 +41,7 @@ public function deletePreview(Preview $preview): void;
4241
* To remove at some point
4342
* @throws Exception
4443
*/
45-
public function migratePreview(Preview $preview, SimpleFile $file): void;
44+
public function migratePreview(Preview $preview): void;
4645

4746
/**
4847
* @throws NotPermittedException

0 commit comments

Comments
 (0)