Skip to content

Commit 2b7dfba

Browse files
CarlSchwanbackportbot[bot]
CarlSchwan
authored and
backportbot[bot]
committed
perf(version): Batch propagator for multi-write operations
perf(version): Batch propagator for multi-write operations Signed-off-by: Carl Schwan <carl@carlschwan.eu> [skip ci]
1 parent defb6e8 commit 2b7dfba

1 file changed

Lines changed: 80 additions & 54 deletions

File tree

‎apps/files_versions/lib/Storage.php‎

Lines changed: 80 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -229,6 +229,14 @@ public static function markDeletedFile($path) {
229229
'filename' => $filename];
230230
}
231231

232+
private static function getUserStoragePropagator(string $uid): ?IPropagator {
233+
try {
234+
return Server::get(IRootFolder::class)->getUserFolder($uid)->getStorage()->getPropagator();
235+
} catch (\Exception) {
236+
return null;
237+
}
238+
}
239+
232240
/**
233241
* delete the version from the storage and cache
234242
*
@@ -259,10 +267,16 @@ public static function delete($path) {
259267

260268
$versions = self::getVersions($uid, $filename);
261269
if (!empty($versions)) {
262-
foreach ($versions as $v) {
263-
\OC_Hook::emit('\OCP\Versions', 'preDelete', ['path' => $path . $v['version'], 'trigger' => self::DELETE_TRIGGER_MASTER_REMOVED]);
264-
self::deleteVersion($view, $filename . '.v' . $v['version']);
265-
\OC_Hook::emit('\OCP\Versions', 'delete', ['path' => $path . $v['version'], 'trigger' => self::DELETE_TRIGGER_MASTER_REMOVED]);
270+
$propagator = self::getUserStoragePropagator($uid);
271+
$propagator?->beginBatch();
272+
try {
273+
foreach ($versions as $v) {
274+
\OC_Hook::emit('\OCP\Versions', 'preDelete', ['path' => $path . $v['version'], 'trigger' => self::DELETE_TRIGGER_MASTER_REMOVED]);
275+
self::deleteVersion($view, $filename . '.v' . $v['version']);
276+
\OC_Hook::emit('\OCP\Versions', 'delete', ['path' => $path . $v['version'], 'trigger' => self::DELETE_TRIGGER_MASTER_REMOVED]);
277+
}
278+
} finally {
279+
$propagator?->commitBatch();
266280
}
267281
}
268282
}
@@ -614,21 +628,27 @@ public static function expireOlderThanMaxForUser($uid) {
614628
return $version < $threshold;
615629
});
616630

617-
foreach ($versions as $version) {
618-
$internalPath = $version->getInternalPath();
619-
\OC_Hook::emit('\OCP\Versions', 'preDelete', ['path' => $internalPath, 'trigger' => self::DELETE_TRIGGER_RETENTION_CONSTRAINT]);
631+
$propagator = self::getUserStoragePropagator($uid);
632+
$propagator?->beginBatch();
633+
try {
634+
foreach ($versions as $version) {
635+
$internalPath = $version->getInternalPath();
636+
\OC_Hook::emit('\OCP\Versions', 'preDelete', ['path' => $internalPath, 'trigger' => self::DELETE_TRIGGER_RETENTION_CONSTRAINT]);
620637

621-
$versionEntity = isset($versionEntities[$version->getId()]) ? $versionEntities[$version->getId()] : null;
622-
if (!is_null($versionEntity)) {
623-
$versionsMapper->delete($versionEntity);
624-
}
638+
$versionEntity = isset($versionEntities[$version->getId()]) ? $versionEntities[$version->getId()] : null;
639+
if (!is_null($versionEntity)) {
640+
$versionsMapper->delete($versionEntity);
641+
}
625642

626-
try {
627-
$version->delete();
628-
\OC_Hook::emit('\OCP\Versions', 'delete', ['path' => $internalPath, 'trigger' => self::DELETE_TRIGGER_RETENTION_CONSTRAINT]);
629-
} catch (NotPermittedException $e) {
630-
Server::get(LoggerInterface::class)->error("Missing permissions to delete version for user {$uid}: {$internalPath}", ['app' => 'files_versions', 'exception' => $e]);
643+
try {
644+
$version->delete();
645+
\OC_Hook::emit('\OCP\Versions', 'delete', ['path' => $internalPath, 'trigger' => self::DELETE_TRIGGER_RETENTION_CONSTRAINT]);
646+
} catch (NotPermittedException $e) {
647+
Server::get(LoggerInterface::class)->error("Missing permissions to delete version for user {$uid}: {$internalPath}", ['app' => 'files_versions', 'exception' => $e]);
648+
}
631649
}
650+
} finally {
651+
$propagator?->commitBatch();
632652
}
633653
}
634654

@@ -929,47 +949,53 @@ public static function expire($filename, $uid) {
929949
$versionsSize = $versionsSize - $sizeOfDeletedVersions;
930950
}
931951

932-
foreach ($toDelete as $key => $path) {
933-
// Make sure to cleanup version table relations as expire does not pass deleteVersion
934-
try {
935-
/** @var VersionsMapper $versionsMapper */
936-
$versionsMapper = Server::get(VersionsMapper::class);
937-
$file = Server::get(IRootFolder::class)->getUserFolder($uid)->get($filename);
938-
$pathparts = pathinfo($path);
939-
$timestamp = (int)substr($pathparts['extension'] ?? '', 1);
940-
$versionEntity = $versionsMapper->findVersionForFileId($file->getId(), $timestamp);
941-
if ($versionEntity->getMetadataValue('label') !== null && $versionEntity->getMetadataValue('label') !== '') {
942-
continue;
952+
$propagator = self::getUserStoragePropagator($uid);
953+
$propagator?->beginBatch();
954+
try {
955+
foreach ($toDelete as $key => $path) {
956+
// Make sure to cleanup version table relations as expire does not pass deleteVersion
957+
try {
958+
/** @var VersionsMapper $versionsMapper */
959+
$versionsMapper = Server::get(VersionsMapper::class);
960+
$file = Server::get(IRootFolder::class)->getUserFolder($uid)->get($filename);
961+
$pathparts = pathinfo($path);
962+
$timestamp = (int)substr($pathparts['extension'] ?? '', 1);
963+
$versionEntity = $versionsMapper->findVersionForFileId($file->getId(), $timestamp);
964+
if ($versionEntity->getMetadataValue('label') !== null && $versionEntity->getMetadataValue('label') !== '') {
965+
continue;
966+
}
967+
$versionsMapper->delete($versionEntity);
968+
} catch (DoesNotExistException $e) {
943969
}
944-
$versionsMapper->delete($versionEntity);
945-
} catch (DoesNotExistException $e) {
946-
}
947970

948-
\OC_Hook::emit('\OCP\Versions', 'preDelete', ['path' => $path, 'trigger' => self::DELETE_TRIGGER_QUOTA_EXCEEDED]);
949-
self::deleteVersion($versionsFileview, $path);
950-
\OC_Hook::emit('\OCP\Versions', 'delete', ['path' => $path, 'trigger' => self::DELETE_TRIGGER_QUOTA_EXCEEDED]);
951-
unset($allVersions[$key]); // update array with the versions we keep
952-
$logger->info('Expire: ' . $path, ['app' => 'files_versions']);
953-
}
971+
\OC_Hook::emit('\OCP\Versions', 'preDelete', ['path' => $path, 'trigger' => self::DELETE_TRIGGER_QUOTA_EXCEEDED]);
972+
self::deleteVersion($versionsFileview, $path);
973+
\OC_Hook::emit('\OCP\Versions', 'delete', ['path' => $path, 'trigger' => self::DELETE_TRIGGER_QUOTA_EXCEEDED]);
974+
unset($allVersions[$key]); // update array with the versions we keep
975+
$logger->info('Expire: ' . $path, ['app' => 'files_versions']);
976+
}
954977

955-
// Check if enough space is available after versions are rearranged.
956-
// If not we delete the oldest versions until we meet the size limit for versions,
957-
// but always keep the two latest versions
958-
$numOfVersions = count($allVersions) - 2 ;
959-
$i = 0;
960-
// sort oldest first and make sure that we start at the first element
961-
ksort($allVersions);
962-
reset($allVersions);
963-
while ($availableSpace < 0 && $i < $numOfVersions) {
964-
$version = current($allVersions);
965-
\OC_Hook::emit('\OCP\Versions', 'preDelete', ['path' => $version['path'] . '.v' . $version['version'], 'trigger' => self::DELETE_TRIGGER_QUOTA_EXCEEDED]);
966-
self::deleteVersion($versionsFileview, $version['path'] . '.v' . $version['version']);
967-
\OC_Hook::emit('\OCP\Versions', 'delete', ['path' => $version['path'] . '.v' . $version['version'], 'trigger' => self::DELETE_TRIGGER_QUOTA_EXCEEDED]);
968-
$logger->info('running out of space! Delete oldest version: ' . $version['path'] . '.v' . $version['version'], ['app' => 'files_versions']);
969-
$versionsSize -= $version['size'];
970-
$availableSpace += $version['size'];
971-
next($allVersions);
972-
$i++;
978+
// Check if enough space is available after versions are rearranged.
979+
// If not we delete the oldest versions until we meet the size limit for versions,
980+
// but always keep the two latest versions
981+
$numOfVersions = count($allVersions) - 2 ;
982+
$i = 0;
983+
// sort oldest first and make sure that we start at the first element
984+
ksort($allVersions);
985+
reset($allVersions);
986+
while ($availableSpace < 0 && $i < $numOfVersions) {
987+
$version = current($allVersions);
988+
\OC_Hook::emit('\OCP\Versions', 'preDelete', ['path' => $version['path'] . '.v' . $version['version'], 'trigger' => self::DELETE_TRIGGER_QUOTA_EXCEEDED]);
989+
self::deleteVersion($versionsFileview, $version['path'] . '.v' . $version['version']);
990+
\OC_Hook::emit('\OCP\Versions', 'delete', ['path' => $version['path'] . '.v' . $version['version'], 'trigger' => self::DELETE_TRIGGER_QUOTA_EXCEEDED]);
991+
$logger->info('running out of space! Delete oldest version: ' . $version['path'] . '.v' . $version['version'], ['app' => 'files_versions']);
992+
$versionsSize -= $version['size'];
993+
$availableSpace += $version['size'];
994+
next($allVersions);
995+
$i++;
996+
}
997+
} finally {
998+
$propagator?->commitBatch();
973999
}
9741000

9751001
return $versionsSize; // finally return the new size of the version history

0 commit comments

Comments
 (0)