Skip to content

Commit 37ac045

Browse files
CarlSchwanCarlSchwan
committed
perf(version): Batch propagator for multi-write operations
Signed-off-by: Carl Schwan <carl@carlschwan.eu>
1 parent d80671d commit 37ac045

1 file changed

Lines changed: 81 additions & 54 deletions

File tree

‎apps/files_versions/lib/Storage.php‎

Lines changed: 81 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
use OCP\AppFramework\Db\DoesNotExistException;
2626
use OCP\Command\IBus;
2727
use OCP\EventDispatcher\IEventDispatcher;
28+
use OCP\Files\Cache\IPropagator;
2829
use OCP\Files\FileInfo;
2930
use OCP\Files\Folder;
3031
use OCP\Files\IMimeTypeDetector;
@@ -234,6 +235,14 @@ public static function markDeletedFile($path) {
234235
'filename' => $filename];
235236
}
236237

238+
private static function getUserStoragePropagator(string $uid): ?IPropagator {
239+
try {
240+
return Server::get(IRootFolder::class)->getUserFolder($uid)->getStorage()->getPropagator();
241+
} catch (\Exception) {
242+
return null;
243+
}
244+
}
245+
237246
/**
238247
* delete the version from the storage and cache
239248
*
@@ -264,10 +273,16 @@ public static function delete($path) {
264273

265274
$versions = self::getVersions($uid, $filename);
266275
if (!empty($versions)) {
267-
foreach ($versions as $v) {
268-
\OC_Hook::emit('\OCP\Versions', 'preDelete', ['path' => $path . $v['version'], 'trigger' => self::DELETE_TRIGGER_MASTER_REMOVED]);
269-
self::deleteVersion($view, $filename . '.v' . $v['version']);
270-
\OC_Hook::emit('\OCP\Versions', 'delete', ['path' => $path . $v['version'], 'trigger' => self::DELETE_TRIGGER_MASTER_REMOVED]);
276+
$propagator = self::getUserStoragePropagator($uid);
277+
$propagator?->beginBatch();
278+
try {
279+
foreach ($versions as $v) {
280+
\OC_Hook::emit('\OCP\Versions', 'preDelete', ['path' => $path . $v['version'], 'trigger' => self::DELETE_TRIGGER_MASTER_REMOVED]);
281+
self::deleteVersion($view, $filename . '.v' . $v['version']);
282+
\OC_Hook::emit('\OCP\Versions', 'delete', ['path' => $path . $v['version'], 'trigger' => self::DELETE_TRIGGER_MASTER_REMOVED]);
283+
}
284+
} finally {
285+
$propagator?->commitBatch();
271286
}
272287
}
273288
}
@@ -619,21 +634,27 @@ public static function expireOlderThanMaxForUser($uid) {
619634
return $version < $threshold;
620635
});
621636

622-
foreach ($versions as $version) {
623-
$internalPath = $version->getInternalPath();
624-
\OC_Hook::emit('\OCP\Versions', 'preDelete', ['path' => $internalPath, 'trigger' => self::DELETE_TRIGGER_RETENTION_CONSTRAINT]);
637+
$propagator = self::getUserStoragePropagator($uid);
638+
$propagator?->beginBatch();
639+
try {
640+
foreach ($versions as $version) {
641+
$internalPath = $version->getInternalPath();
642+
\OC_Hook::emit('\OCP\Versions', 'preDelete', ['path' => $internalPath, 'trigger' => self::DELETE_TRIGGER_RETENTION_CONSTRAINT]);
625643

626-
$versionEntity = isset($versionEntities[$version->getId()]) ? $versionEntities[$version->getId()] : null;
627-
if (!is_null($versionEntity)) {
628-
$versionsMapper->delete($versionEntity);
629-
}
644+
$versionEntity = isset($versionEntities[$version->getId()]) ? $versionEntities[$version->getId()] : null;
645+
if (!is_null($versionEntity)) {
646+
$versionsMapper->delete($versionEntity);
647+
}
630648

631-
try {
632-
$version->delete();
633-
\OC_Hook::emit('\OCP\Versions', 'delete', ['path' => $internalPath, 'trigger' => self::DELETE_TRIGGER_RETENTION_CONSTRAINT]);
634-
} catch (NotPermittedException $e) {
635-
Server::get(LoggerInterface::class)->error("Missing permissions to delete version for user {$uid}: {$internalPath}", ['app' => 'files_versions', 'exception' => $e]);
649+
try {
650+
$version->delete();
651+
\OC_Hook::emit('\OCP\Versions', 'delete', ['path' => $internalPath, 'trigger' => self::DELETE_TRIGGER_RETENTION_CONSTRAINT]);
652+
} catch (NotPermittedException $e) {
653+
Server::get(LoggerInterface::class)->error("Missing permissions to delete version for user {$uid}: {$internalPath}", ['app' => 'files_versions', 'exception' => $e]);
654+
}
636655
}
656+
} finally {
657+
$propagator?->commitBatch();
637658
}
638659
}
639660

@@ -935,47 +956,53 @@ public static function expire($filename, $uid) {
935956
$versionsSize = $versionsSize - $sizeOfDeletedVersions;
936957
}
937958

938-
foreach ($toDelete as $key => $path) {
939-
// Make sure to cleanup version table relations as expire does not pass deleteVersion
940-
try {
941-
/** @var VersionsMapper $versionsMapper */
942-
$versionsMapper = Server::get(VersionsMapper::class);
943-
$file = Server::get(IRootFolder::class)->getUserFolder($uid)->get($filename);
944-
$pathparts = pathinfo($path);
945-
$timestamp = (int)substr($pathparts['extension'] ?? '', 1);
946-
$versionEntity = $versionsMapper->findVersionForFileId($file->getId(), $timestamp);
947-
if ($versionEntity->getMetadataValue('label') !== null && $versionEntity->getMetadataValue('label') !== '') {
948-
continue;
959+
$propagator = self::getUserStoragePropagator($uid);
960+
$propagator?->beginBatch();
961+
try {
962+
foreach ($toDelete as $key => $path) {
963+
// Make sure to cleanup version table relations as expire does not pass deleteVersion
964+
try {
965+
/** @var VersionsMapper $versionsMapper */
966+
$versionsMapper = Server::get(VersionsMapper::class);
967+
$file = Server::get(IRootFolder::class)->getUserFolder($uid)->get($filename);
968+
$pathparts = pathinfo($path);
969+
$timestamp = (int)substr($pathparts['extension'] ?? '', 1);
970+
$versionEntity = $versionsMapper->findVersionForFileId($file->getId(), $timestamp);
971+
if ($versionEntity->getMetadataValue('label') !== null && $versionEntity->getMetadataValue('label') !== '') {
972+
continue;
973+
}
974+
$versionsMapper->delete($versionEntity);
975+
} catch (DoesNotExistException $e) {
949976
}
950-
$versionsMapper->delete($versionEntity);
951-
} catch (DoesNotExistException $e) {
952-
}
953977

954-
\OC_Hook::emit('\OCP\Versions', 'preDelete', ['path' => $path, 'trigger' => self::DELETE_TRIGGER_QUOTA_EXCEEDED]);
955-
self::deleteVersion($versionsFileview, $path);
956-
\OC_Hook::emit('\OCP\Versions', 'delete', ['path' => $path, 'trigger' => self::DELETE_TRIGGER_QUOTA_EXCEEDED]);
957-
unset($allVersions[$key]); // update array with the versions we keep
958-
$logger->info('Expire: ' . $path, ['app' => 'files_versions']);
959-
}
978+
\OC_Hook::emit('\OCP\Versions', 'preDelete', ['path' => $path, 'trigger' => self::DELETE_TRIGGER_QUOTA_EXCEEDED]);
979+
self::deleteVersion($versionsFileview, $path);
980+
\OC_Hook::emit('\OCP\Versions', 'delete', ['path' => $path, 'trigger' => self::DELETE_TRIGGER_QUOTA_EXCEEDED]);
981+
unset($allVersions[$key]); // update array with the versions we keep
982+
$logger->info('Expire: ' . $path, ['app' => 'files_versions']);
983+
}
960984

961-
// Check if enough space is available after versions are rearranged.
962-
// If not we delete the oldest versions until we meet the size limit for versions,
963-
// but always keep the two latest versions
964-
$numOfVersions = count($allVersions) - 2 ;
965-
$i = 0;
966-
// sort oldest first and make sure that we start at the first element
967-
ksort($allVersions);
968-
reset($allVersions);
969-
while ($availableSpace < 0 && $i < $numOfVersions) {
970-
$version = current($allVersions);
971-
\OC_Hook::emit('\OCP\Versions', 'preDelete', ['path' => $version['path'] . '.v' . $version['version'], 'trigger' => self::DELETE_TRIGGER_QUOTA_EXCEEDED]);
972-
self::deleteVersion($versionsFileview, $version['path'] . '.v' . $version['version']);
973-
\OC_Hook::emit('\OCP\Versions', 'delete', ['path' => $version['path'] . '.v' . $version['version'], 'trigger' => self::DELETE_TRIGGER_QUOTA_EXCEEDED]);
974-
$logger->info('running out of space! Delete oldest version: ' . $version['path'] . '.v' . $version['version'], ['app' => 'files_versions']);
975-
$versionsSize -= $version['size'];
976-
$availableSpace += $version['size'];
977-
next($allVersions);
978-
$i++;
985+
// Check if enough space is available after versions are rearranged.
986+
// If not we delete the oldest versions until we meet the size limit for versions,
987+
// but always keep the two latest versions
988+
$numOfVersions = count($allVersions) - 2 ;
989+
$i = 0;
990+
// sort oldest first and make sure that we start at the first element
991+
ksort($allVersions);
992+
reset($allVersions);
993+
while ($availableSpace < 0 && $i < $numOfVersions) {
994+
$version = current($allVersions);
995+
\OC_Hook::emit('\OCP\Versions', 'preDelete', ['path' => $version['path'] . '.v' . $version['version'], 'trigger' => self::DELETE_TRIGGER_QUOTA_EXCEEDED]);
996+
self::deleteVersion($versionsFileview, $version['path'] . '.v' . $version['version']);
997+
\OC_Hook::emit('\OCP\Versions', 'delete', ['path' => $version['path'] . '.v' . $version['version'], 'trigger' => self::DELETE_TRIGGER_QUOTA_EXCEEDED]);
998+
$logger->info('running out of space! Delete oldest version: ' . $version['path'] . '.v' . $version['version'], ['app' => 'files_versions']);
999+
$versionsSize -= $version['size'];
1000+
$availableSpace += $version['size'];
1001+
next($allVersions);
1002+
$i++;
1003+
}
1004+
} finally {
1005+
$propagator?->commitBatch();
9791006
}
9801007

9811008
return $versionsSize; // finally return the new size of the version history

0 commit comments

Comments
 (0)