Skip to content

fix(files): purge files and refund quota per tenant #1419

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
iammukeshm merged 1 commit into from
Sep 30, 2026
+439 −70
Merged

fix(files): purge files and refund quota per tenant #1419

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
129 changes: 89 additions & 40 deletions src/Modules/Files/Modules.Files/Jobs/PurgeDeletedFilesJob.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,20 +1,26 @@
using Finbuckle.MultiTenant;
using Finbuckle.MultiTenant.Abstractions;
using FSH.Framework.Persistence;
using FSH.Framework.Quota;
using FSH.Framework.Shared.Multitenancy;
using FSH.Framework.Shared.Quota;
using FSH.Framework.Storage.Services;
using FSH.Modules.Files.Data;
using Hangfire;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;

namespace FSH.Modules.Files.Jobs;

/// <summary>
/// Daily purge of soft-deleted FileAsset rows past the retention window. Hard-deletes the row,
/// removes the bytes from storage, and refunds the quota (the bytes were debited at finalize time).
/// Daily purge of soft-deleted FileAsset rows past the retention window, in every tenant. Hard-deletes
/// the row, removes the bytes from storage, and refunds the owning tenant's quota (the bytes were
/// debited at finalize time).
/// </summary>
public sealed class PurgeDeletedFilesJob(
FilesDbContext db,
IServiceScopeFactory scopeFactory,
IStorageService storage,
IQuotaService quotas,
IOptions<FilesOptions> options,
Expand All @@ -23,60 +29,103 @@ public sealed class PurgeDeletedFilesJob(
[AutomaticRetry(Attempts = 2, DelaysInSeconds = [300, 1800])]
public async Task RunAsync(CancellationToken cancellationToken)
{
// The recurring job is registered without a tenant, so none is resolved. Each tenant is purged
// inside its own context, which also points FilesDbContext at a tenant's dedicated database.
List<AppTenantInfo> tenants;
using (var scope = scopeFactory.CreateScope())
{
var tenantStore = scope.ServiceProvider.GetRequiredService<IMultiTenantStore<AppTenantInfo>>();
tenants = (await tenantStore.GetAllAsync().ConfigureAwait(false)).ToList();
}

var cutoff = DateTimeOffset.UtcNow.AddDays(-options.Value.SoftDeleteRetentionDays);
var candidates = await db.FileAssets
.IgnoreQueryFilters()
.Where(f => f.IsDeleted && f.DeletedOnUtc != null && f.DeletedOnUtc < cutoff)
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
int total = 0;
foreach (var tenant in tenants)
{
cancellationToken.ThrowIfCancellationRequested();
total += await PurgeTenantAsync(tenant, cutoff, cancellationToken).ConfigureAwait(false);
}

if (candidates.Count == 0)
if (total > 0 && logger.IsEnabled(LogLevel.Information))
{
return;
logger.LogInformation("Hard-purged {Total} soft-deleted file assets across {TenantCount} tenants",
total, tenants.Count);
}
}

// Best-effort byte removal per file. Schema-per-tenant means all rows share one tenant,
// and Hangfire wires the job per-tenant for multi-tenant deployments.
foreach (var f in candidates)
private async Task<int> PurgeTenantAsync(AppTenantInfo tenant, DateTimeOffset cutoff, CancellationToken cancellationToken)
{
try
{
try
using var scope = scopeFactory.CreateScope();
scope.ServiceProvider.GetRequiredService<IMultiTenantContextSetter>()
.MultiTenantContext = new MultiTenantContext<AppTenantInfo>(tenant);
var db = scope.ServiceProvider.GetRequiredService<FilesDbContext>();

// Soft-deleted rows are hidden by the soft-delete filter; lift only that one so the tenant
// filter stays on.
var candidates = await db.FileAssets
.IgnoreQueryFilters([QueryFilters.SoftDelete])
.Where(f => f.IsDeleted && f.DeletedOnUtc != null && f.DeletedOnUtc < cutoff)
.ToListAsync(cancellationToken)
.ConfigureAwait(false);

if (candidates.Count == 0)
{
await storage.RemoveAsync(f.StorageKey, cancellationToken).ConfigureAwait(false);
return 0;
}
catch (Exception ex)

// Best-effort byte removal per file.
foreach (var f in candidates)
{
logger.LogWarning(ex, "Storage remove failed for {Key}", f.StorageKey);
try
{
await storage.RemoveAsync(f.StorageKey, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex) when (ex is not OperationCanceledException)
{
logger.LogWarning(ex, "Storage remove failed for {Key} for tenant {TenantId}",
f.StorageKey, tenant.Id);
}
}
}

// Quota refund — group bytes once per tenant. In schema-per-tenant the resolved tenant
// matches every row's logical tenant; the framework's QuotaService is tenant-scoped via DI.
var totalBytes = candidates.Sum(f => f.SizeBytes);
if (totalBytes > 0)
{
// Empty tenant id satisfies the contract; QuotaService resolves the tenant from DI.
// Falls back gracefully with no tenant (the refund is simply lost).
try
var ids = candidates.Select(f => f.Id).ToList();
await db.FileAssets
.IgnoreQueryFilters([QueryFilters.SoftDelete])
.Where(f => ids.Contains(f.Id))
.ExecuteDeleteAsync(cancellationToken)
.ConfigureAwait(false);

// Refund the bytes debited at finalize, under the tenant that owns them. Done after the
// rows are gone so a failed delete can't refund bytes that are still counted as stored.
var totalBytes = candidates.Sum(f => f.SizeBytes);
if (totalBytes > 0)
{
await quotas.RecordAsync("", QuotaResource.StorageBytes, -totalBytes, cancellationToken).ConfigureAwait(false);
try
{
await quotas.RecordAsync(tenant.Id, QuotaResource.StorageBytes, -totalBytes, cancellationToken)
.ConfigureAwait(false);
}
catch (Exception ex) when (ex is not OperationCanceledException)
{
logger.LogWarning(ex, "Quota refund of {Bytes} bytes failed for tenant {TenantId}",
totalBytes, tenant.Id);
}
}
catch (Exception ex)

if (logger.IsEnabled(LogLevel.Information))
{
logger.LogWarning(ex, "Quota refund failed for {Bytes} bytes", totalBytes);
logger.LogInformation("Hard-purged {Count} soft-deleted file assets ({Bytes} bytes) for tenant {TenantId}",
candidates.Count, totalBytes, tenant.Id);
}
}

var ids = candidates.Select(f => f.Id).ToList();
await db.FileAssets
.IgnoreQueryFilters()
.Where(f => ids.Contains(f.Id))
.ExecuteDeleteAsync(cancellationToken)
.ConfigureAwait(false);

if (logger.IsEnabled(LogLevel.Information))
return candidates.Count;
}
catch (Exception ex) when (ex is not OperationCanceledException)
{
logger.LogInformation("Hard-purged {Count} soft-deleted file assets ({Bytes} bytes total)",
candidates.Count, totalBytes);
// One tenant's database being unreachable must not keep the other tenants' files around.
logger.LogError(ex, "Deleted file purge failed for tenant {TenantId}", tenant.Id);
return 0;
}
}
}
109 changes: 79 additions & 30 deletions src/Modules/Files/Modules.Files/Jobs/PurgeOrphanedFilesJob.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,65 +1,114 @@
using Finbuckle.MultiTenant;
using Finbuckle.MultiTenant.Abstractions;
using FSH.Framework.Persistence;
using FSH.Framework.Shared.Multitenancy;
using FSH.Framework.Storage.Services;
using FSH.Modules.Files.Contracts.v1.DTOs;
using FSH.Modules.Files.Data;
using FSH.Modules.Files.Domain;
using Hangfire;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;

namespace FSH.Modules.Files.Jobs;

/// <summary>
/// Hourly purge of FileAsset rows stuck in PendingUpload past their UploadDeadline. Best-effort
/// removal of any bytes that did make it to storage. No quota effect — those bytes were never
/// debited.
/// Hourly purge of FileAsset rows stuck in PendingUpload past their UploadDeadline, in every tenant.
/// Best-effort removal of any bytes that did make it to storage. No quota effect — those bytes were
/// never debited.
/// </summary>
public sealed class PurgeOrphanedFilesJob(
FilesDbContext db,
IServiceScopeFactory scopeFactory,
IStorageService storage,
ILogger<PurgeOrphanedFilesJob> logger)
{
[AutomaticRetry(Attempts = 3, DelaysInSeconds = [30, 120, 600])]
public async Task RunAsync(CancellationToken cancellationToken)
{
// The recurring job is registered without a tenant, so none is resolved. Each tenant is purged
// inside its own context, which also points FilesDbContext at a tenant's dedicated database.
List<AppTenantInfo> tenants;
using (var scope = scopeFactory.CreateScope())
{
var tenantStore = scope.ServiceProvider.GetRequiredService<IMultiTenantStore<AppTenantInfo>>();
tenants = (await tenantStore.GetAllAsync().ConfigureAwait(false)).ToList();
}

var now = DateTimeOffset.UtcNow;
var orphans = await db.FileAssets
.IgnoreQueryFilters()
.Where(f => f.Status == FileAssetStatus.PendingUpload
&& f.UploadDeadline != null
&& f.UploadDeadline < now)
.ToListAsync(cancellationToken)
.ConfigureAwait(false);
int total = 0;
foreach (var tenant in tenants)
{
cancellationToken.ThrowIfCancellationRequested();
total += await PurgeTenantAsync(tenant, now, cancellationToken).ConfigureAwait(false);
}

if (orphans.Count == 0)
if (total > 0 && logger.IsEnabled(LogLevel.Information))
{
return;
logger.LogInformation("Purged {Total} orphaned file assets across {TenantCount} tenants",
total, tenants.Count);
}
}

foreach (var f in orphans)
private async Task<int> PurgeTenantAsync(AppTenantInfo tenant, DateTimeOffset now, CancellationToken cancellationToken)
{
try
{
try
using var scope = scopeFactory.CreateScope();
scope.ServiceProvider.GetRequiredService<IMultiTenantContextSetter>()
.MultiTenantContext = new MultiTenantContext<AppTenantInfo>(tenant);
var db = scope.ServiceProvider.GetRequiredService<FilesDbContext>();

// Only the soft-delete filter is lifted; the tenant filter stays on.
var orphans = await db.FileAssets
.IgnoreQueryFilters([QueryFilters.SoftDelete])
.Where(f => f.Status == FileAssetStatus.PendingUpload
&& f.UploadDeadline != null
&& f.UploadDeadline < now)
.ToListAsync(cancellationToken)
.ConfigureAwait(false);

if (orphans.Count == 0)
{
await storage.RemoveAsync(f.StorageKey, cancellationToken).ConfigureAwait(false);
return 0;
}
catch (Exception ex)

foreach (var f in orphans)
{
logger.LogWarning(ex, "Failed to remove orphan storage object {Key}", f.StorageKey);
try
{
await storage.RemoveAsync(f.StorageKey, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex) when (ex is not OperationCanceledException)
{
logger.LogWarning(ex, "Failed to remove orphan storage object {Key} for tenant {TenantId}",
f.StorageKey, tenant.Id);
}
}
// Hard delete (row never reached Available, so soft-delete doesn't apply). FileAsset is ISoftDeletable,
// so Remove() would become UPDATE IsDeleted=true — we use the bulk ExecuteDelete below to bypass the interceptor instead.
}

// Bulk hard delete — bypasses the soft-delete interceptor.
var ids = orphans.Select(f => f.Id).ToList();
await db.FileAssets
.IgnoreQueryFilters()
.Where(f => ids.Contains(f.Id))
.ExecuteDeleteAsync(cancellationToken)
.ConfigureAwait(false);
// Hard delete (the row never reached Available, so soft-delete doesn't apply). FileAsset is
// ISoftDeletable, so Remove() would become UPDATE IsDeleted=true — the bulk ExecuteDelete
// bypasses the interceptor instead.
var ids = orphans.Select(f => f.Id).ToList();
await db.FileAssets
.IgnoreQueryFilters([QueryFilters.SoftDelete])
.Where(f => ids.Contains(f.Id))
.ExecuteDeleteAsync(cancellationToken)
.ConfigureAwait(false);

if (logger.IsEnabled(LogLevel.Information))
if (logger.IsEnabled(LogLevel.Information))
{
logger.LogInformation("Purged {Count} orphaned file assets for tenant {TenantId}",
orphans.Count, tenant.Id);
}

return orphans.Count;
}
catch (Exception ex) when (ex is not OperationCanceledException)
{
logger.LogInformation("Purged {Count} orphaned file assets", orphans.Count);
// One tenant's database being unreachable must not keep the other tenants' orphans around.
logger.LogError(ex, "Orphaned file purge failed for tenant {TenantId}", tenant.Id);
return 0;
}
}
}
Loading
Loading