Skip to content

Commit 2aa5ede

Browse files
AndyScherzingerAndyScherzinger
authored
Merge pull request #62603 from nextcloud/backport/62601/stable33
[stable33] perf(preview): Optimize retriving all previews from oc_filecache
2 parents 3945c7f + d194927 commit 2aa5ede

7 files changed

Lines changed: 281 additions & 130 deletions

File tree

‎core/BackgroundJobs/PreviewMigrationJob.php‎

Lines changed: 50 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -9,16 +9,15 @@
99

1010
namespace OC\Core\BackgroundJobs;
1111

12-
use OC\Preview\Db\Preview;
1312
use OC\Preview\PreviewMigrationService;
1413
use OCP\AppFramework\Utility\ITimeFactory;
1514
use OCP\BackgroundJob\TimedJob;
16-
use OCP\DB\IResult;
15+
use OCP\Files\FileInfo;
1716
use OCP\Files\IRootFolder;
1817
use OCP\IAppConfig;
1918
use OCP\IConfig;
20-
use OCP\IDBConnection;
2119
use Override;
20+
use Psr\Log\LoggerInterface;
2221

2322
class PreviewMigrationJob extends TimedJob {
2423
private string $previewRootPath;
@@ -27,9 +26,9 @@ public function __construct(
2726
ITimeFactory $time,
2827
private readonly IAppConfig $appConfig,
2928
private readonly IConfig $config,
30-
private readonly IDBConnection $connection,
3129
private readonly IRootFolder $rootFolder,
3230
private readonly PreviewMigrationService $migrationService,
31+
private readonly LoggerInterface $logger,
3332
) {
3433
parent::__construct($time);
3534

@@ -44,23 +43,54 @@ protected function run(mixed $argument): void {
4443
return;
4544
}
4645

46+
$storage = $this->rootFolder->getMountPoint()->getStorage();
47+
if ($storage === null) {
48+
$this->appConfig->setValueBool('core', 'previewMovedDone', true);
49+
return;
50+
}
51+
52+
$cache = $storage->getCache();
53+
$previewRootId = $cache->getId(rtrim($this->previewRootPath, '/'));
54+
if ($previewRootId === -1) {
55+
// No previews have ever been generated on this instance.
56+
$this->appConfig->setValueBool('core', 'previewMovedDone', true);
57+
return;
58+
}
59+
4760
$startTime = time();
48-
while (true) {
49-
$qb = $this->connection->getQueryBuilder();
50-
$qb->select('path')
51-
->from('filecache')
52-
// Hierarchical preview folder structure
53-
->where($qb->expr()->like('path', $qb->createNamedParameter($this->previewRootPath . '%/%/%/%/%/%/%/%/%')))
54-
// Legacy flat preview folder structure
55-
->orWhere($qb->expr()->like('path', $qb->createNamedParameter($this->previewRootPath . '%/%.%')))
56-
->hintShardKey('storage', $this->rootFolder->getMountPoint()->getNumericStorageId())
57-
->setMaxResults(100);
58-
59-
$result = $qb->executeQuery();
60-
$foundPreviews = $this->processQueryResult($result);
61-
62-
if (!$foundPreviews) {
63-
break;
61+
62+
// Walk the preview folder tree via the `parent` column, which is indexed on
63+
// every supported database platform.
64+
//
65+
// Depth from the preview root tells us which structure a leaf folder holds:
66+
// - depth 1: legacy flat structure, e.g. preview/<fileid>/<size>.png
67+
// - depth 8: hierarchical structure, e.g. preview/a/b/c/d/e/f/g/<fileid>/<size>.png
68+
$foldersToVisit = [[$previewRootId, '', 0]];
69+
70+
while ($foldersToVisit !== []) {
71+
[$folderId, $folderName, $depth] = array_pop($foldersToVisit);
72+
73+
// Collect the actual preview files here so migrateFileId() doesn't need to
74+
// list this folder's contents a second time.
75+
$previewEntries = [];
76+
foreach ($cache->getFolderContentsById($folderId) as $entry) {
77+
if ($entry->getMimeType() === FileInfo::MIMETYPE_FOLDER) {
78+
$foldersToVisit[] = [$entry->getId(), $entry->getName(), $depth + 1];
79+
} else {
80+
$previewEntries[] = $entry;
81+
}
82+
}
83+
84+
if ($previewEntries === [] || !ctype_digit($folderName)) {
85+
continue;
86+
}
87+
88+
try {
89+
$this->migrationService->migrateFileId((int)$folderName, flatPath: $depth === 1, entries: $previewEntries);
90+
} catch (\Exception $e) {
91+
$this->logger->error('Failed to migrate preview with fileId: ' . $folderName, [
92+
'exception' => $e,
93+
]);
6494
}
6595

6696
// Stop if execution time is more than one hour.
@@ -71,36 +101,4 @@ protected function run(mixed $argument): void {
71101

72102
$this->appConfig->setValueBool('core', 'previewMovedDone', true);
73103
}
74-
75-
private function processQueryResult(IResult $result): bool {
76-
$foundPreview = false;
77-
$fileIds = [];
78-
$flatFileIds = [];
79-
while ($row = $result->fetch()) {
80-
$pathSplit = explode('/', $row['path']);
81-
assert(count($pathSplit) >= 2);
82-
$fileId = (int)$pathSplit[count($pathSplit) - 2];
83-
if (count($pathSplit) === 11) {
84-
// Hierarchical structure
85-
if (!in_array($fileId, $fileIds)) {
86-
$fileIds[] = $fileId;
87-
}
88-
} else {
89-
// Flat structure
90-
if (!in_array($fileId, $flatFileIds)) {
91-
$flatFileIds[] = $fileId;
92-
}
93-
}
94-
$foundPreview = true;
95-
}
96-
97-
foreach ($fileIds as $fileId) {
98-
$this->migrationService->migrateFileId($fileId, flatPath: false);
99-
}
100-
101-
foreach ($flatFileIds as $fileId) {
102-
$this->migrationService->migrateFileId($fileId, flatPath: true);
103-
}
104-
return $foundPreview;
105-
}
106104
}

‎lib/private/Preview/PreviewMigrationService.php‎

Lines changed: 106 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,9 @@
1414
use OC\Preview\Db\PreviewMapper;
1515
use OC\Preview\Storage\StorageFactory;
1616
use OCP\DB\Exception;
17+
use OCP\DB\QueryBuilder\IQueryBuilder;
1718
use OCP\Files\AppData\IAppDataFactory;
19+
use OCP\Files\Cache\ICacheEntry;
1820
use OCP\Files\IAppData;
1921
use OCP\Files\IMimeTypeDetector;
2022
use OCP\Files\IMimeTypeLoader;
@@ -44,44 +46,45 @@ public function __construct(
4446
}
4547

4648
/**
47-
* @param array<string|int, string[]> $previewFolders
49+
* @param list<ICacheEntry|SimpleFile>|null $entries Preview file entries already fetched by the caller.
4850
* @return Preview[]
4951
*/
50-
public function migrateFileId(int $fileId, bool $flatPath): array {
52+
public function migrateFileId(int $fileId, bool $flatPath, ?array $entries = null): array {
5153
$previews = [];
5254
$internalPath = $this->getInternalFolder((string)$fileId, $flatPath);
53-
try {
54-
$folder = $this->appData->getFolder($internalPath);
55-
} catch (NotFoundException) {
56-
return [];
55+
56+
if ($entries === null) {
57+
try {
58+
$entries = $this->appData->getFolder($internalPath)->getDirectoryListing();
59+
} catch (NotFoundException) {
60+
return [];
61+
}
5762
}
5863

5964
/**
60-
* @var list<array{file: SimpleFile, preview: Preview}> $previewFiles
65+
* @var list<Preview> $previewsToInsert
6166
*/
62-
$previewFiles = [];
67+
$previewsToInsert = [];
6368

64-
foreach ($folder->getDirectoryListing() as $previewFile) {
65-
$path = $fileId . '/' . $previewFile->getName();
66-
/** @var SimpleFile $previewFile */
69+
foreach ($entries as $entry) {
70+
$path = $fileId . '/' . $entry->getName();
6771
$preview = Preview::fromPath($path, $this->mimeTypeDetector);
6872
if ($preview === false) {
6973
$this->logger->error('Unable to import old preview at path.');
7074
continue;
7175
}
7276
$preview->generateId();
73-
$preview->setSize($previewFile->getSize());
74-
$preview->setMtime($previewFile->getMtime());
75-
$preview->setOldFileId($previewFile->getId());
77+
$preview->setSize($entry->getSize());
78+
$preview->setMtime($entry->getMTime());
79+
$preview->setOldFileId($entry->getId());
7680
$preview->setEncrypted(false);
7781

78-
$previewFiles[] = [
79-
'file' => $previewFile,
80-
'preview' => $preview,
81-
];
82+
$previewsToInsert[] = $preview;
8283
}
8384

84-
if (empty($previewFiles)) {
85+
if (empty($previewsToInsert)) {
86+
$this->deleteFolder($internalPath);
87+
8588
return $previews;
8689
}
8790

@@ -95,51 +98,51 @@ public function migrateFileId(int $fileId, bool $flatPath): array {
9598
$result = $result->fetchAssociative();
9699

97100
if ($result !== false) {
98-
foreach ($previewFiles as $previewFile) {
99-
/** @var Preview $preview */
100-
$preview = $previewFile['preview'];
101-
/** @var SimpleFile $file */
102-
$file = $previewFile['file'];
103-
$preview->setStorageId($result['storage']);
104-
$preview->setEtag($result['etag']);
105-
$preview->setSourceMimeType($this->mimeTypeLoader->getMimetypeById((int)$result['mimetype']));
106-
$preview->generateId();
107-
try {
108-
$preview = $this->previewMapper->insert($preview);
109-
} catch (Exception) {
110-
// We already have this preview in the preview table, skip
111-
$qb->delete('filecache')
112-
->where($qb->expr()->eq('fileid', $qb->createNamedParameter($file->getId())))
113-
->hintShardKey('storage', $this->rootFolder->getMountPoint()->getNumericStorageId())
114-
->executeStatement();
115-
continue;
116-
}
117-
118-
try {
119-
$this->storageFactory->migratePreview($preview, $file);
120-
$qb = $this->connection->getQueryBuilder();
121-
$qb->delete('filecache')
122-
->where($qb->expr()->eq('fileid', $qb->createNamedParameter($file->getId())))
123-
->hintShardKey('storage', $this->rootFolder->getMountPoint()->getNumericStorageId())
124-
->executeStatement();
125-
// Do not call $file->delete() as this will also delete the file from the file system
126-
} catch (\Exception $e) {
127-
$this->previewMapper->delete($preview);
128-
throw $e;
101+
$oldFileIdsToDelete = [];
102+
try {
103+
foreach ($previewsToInsert as $preview) {
104+
$preview->setStorageId($result['storage']);
105+
$preview->setEtag($result['etag']);
106+
$preview->setSourceMimeType($this->mimeTypeLoader->getMimetypeById((int)$result['mimetype']));
107+
$preview->generateId();
108+
try {
109+
$preview = $this->previewMapper->insert($preview);
110+
} catch (Exception $e) {
111+
if ($e->getReason() !== Exception::REASON_UNIQUE_CONSTRAINT_VIOLATION) {
112+
throw $e;
113+
}
114+
115+
// We already have this preview in the preview table, skip
116+
$oldFileIdsToDelete[] = $preview->getOldFileId();
117+
continue;
118+
}
119+
120+
try {
121+
$this->storageFactory->migratePreview($preview);
122+
// Do not delete the old file via a Node here, as that would also
123+
// delete it from the file system; only its filecache row is stale.
124+
} catch (\Exception $e) {
125+
$this->previewMapper->delete($preview);
126+
throw $e;
127+
}
128+
129+
$oldFileIdsToDelete[] = $preview->getOldFileId();
130+
$previews[] = $preview;
129131
}
130-
131-
$previews[] = $preview;
132+
} finally {
133+
$this->deleteOldFileCacheEntries($oldFileIdsToDelete);
132134
}
133135
} else {
134-
// No matching fileId, delete preview
136+
// No matching fileId, delete the orphaned preview files themselves.
135137
try {
138+
$folder = $this->appData->getFolder($internalPath);
136139
$this->connection->beginTransaction();
137-
foreach ($previewFiles as $previewFile) {
138-
/** @var SimpleFile $file */
139-
$file = $previewFile['file'];
140+
foreach ($folder->getDirectoryListing() as $file) {
140141
$file->delete();
141142
}
142143
$this->connection->commit();
144+
} catch (NotFoundException) {
145+
// Folder already gone, nothing to clean up.
143146
} catch (Exception) {
144147
$this->connection->rollback();
145148
}
@@ -157,6 +160,23 @@ private static function getInternalFolder(string $name, bool $flatPath): string
157160
return implode('/', str_split(substr(md5($name), 0, 7))) . '/' . $name;
158161
}
159162

163+
/**
164+
* @param list<int> $fileIds
165+
*/
166+
private function deleteOldFileCacheEntries(array $fileIds): void {
167+
if ($fileIds === []) {
168+
return;
169+
}
170+
171+
foreach (array_chunk($fileIds, 1000) as $chunk) {
172+
$qb = $this->connection->getQueryBuilder();
173+
$qb->delete('filecache')
174+
->where($qb->expr()->in('fileid', $qb->createNamedParameter($chunk, IQueryBuilder::PARAM_INT_ARRAY)))
175+
->hintShardKey('storage', $this->rootFolder->getMountPoint()->getNumericStorageId())
176+
->executeStatement();
177+
}
178+
}
179+
160180
private function deleteFolder(string $path): void {
161181
$current = $path;
162182

@@ -177,10 +197,38 @@ private function deleteFolder(string $path): void {
177197
break;
178198
}
179199

180-
$folder = $this->appData->getFolder($current);
181-
if (count($folder->getDirectoryListing()) !== 0) {
200+
if ($this->folderHasChildren($rootFolderId, $this->previewRootPath . $current)) {
182201
break;
183202
}
184203
}
185204
}
205+
206+
private function folderHasChildren(int $storageId, string $path): bool {
207+
$qb = $this->connection->getQueryBuilder();
208+
$qb->select('fileid')
209+
->from('filecache')
210+
->where($qb->expr()->eq('path_hash', $qb->createNamedParameter(md5($path))))
211+
->andWhere($qb->expr()->eq('storage', $qb->createNamedParameter($storageId)))
212+
->setMaxResults(1);
213+
$cursor = $qb->executeQuery();
214+
$folderId = $cursor->fetchOne();
215+
$cursor->closeCursor();
216+
217+
if ($folderId === false) {
218+
// The folder itself is already gone, nothing to check.
219+
return false;
220+
}
221+
222+
$qb = $this->connection->getQueryBuilder();
223+
$qb->select('fileid')
224+
->from('filecache')
225+
->where($qb->expr()->eq('parent', $qb->createNamedParameter((int)$folderId)))
226+
->andWhere($qb->expr()->eq('storage', $qb->createNamedParameter($storageId)))
227+
->setMaxResults(1);
228+
$cursor = $qb->executeQuery();
229+
$hasChild = $cursor->fetchOne() !== false;
230+
$cursor->closeCursor();
231+
232+
return $hasChild;
233+
}
186234
}

‎lib/private/Preview/Storage/IPreviewStorage.php‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,6 @@
1010
namespace OC\Preview\Storage;
1111

1212
use Exception;
13-
use OC\Files\SimpleFS\SimpleFile;
1413
use OC\Preview\Db\Preview;
1514
use OCP\Files\NotFoundException;
1615
use OCP\Files\NotPermittedException;
@@ -42,7 +41,7 @@ public function deletePreview(Preview $preview): void;
4241
* To remove at some point
4342
* @throws Exception
4443
*/
45-
public function migratePreview(Preview $preview, SimpleFile $file): void;
44+
public function migratePreview(Preview $preview): void;
4645

4746
/**
4847
* @throws NotPermittedException

0 commit comments

Comments
 (0)