From 3ce93edb0f7ba4c4d2ed0d190dc55532416f962b Mon Sep 17 00:00:00 2001 From: iammukeshm Date: Wed, 30 Sep 2026 09:45:16 +0530 Subject: [PATCH] fix(files): purge files and refund quota per tenant PurgeOrphanedFilesJob and PurgeDeletedFilesJob run from Hangfire with no tenant context, so FilesDbContext only ever reached the shared database (tenants with a dedicated database were never purged), and the deleted-files purge refunded storage quota under an empty tenant id, which the quota service rejects, so the refund was lost. Both jobs now load every tenant from the tenant store, open a scope per tenant, set the Finbuckle tenant context, and purge that tenant's FilesDbContext. Only the SoftDelete filter is lifted; the tenant filter stays on. The quota refund is recorded under the owning tenant's id after the rows are deleted. Failures are logged per tenant so one bad tenant does not block the rest. Closes #1418 Co-Authored-By: Claude Opus 5.5 (1M context) --- .../Jobs/PurgeDeletedFilesJob.cs | 129 ++++++--- .../Jobs/PurgeOrphanedFilesJob.cs | 109 +++++-- .../Tests/Files/PurgeJobsPerTenantTests.cs | 271 ++++++++++++++++++ 3 files changed, 439 insertions(+), 70 deletions(-) create mode 100644 src/Tests/Integration.Tests/Tests/Files/PurgeJobsPerTenantTests.cs diff --git a/src/Modules/Files/Modules.Files/Jobs/PurgeDeletedFilesJob.cs b/src/Modules/Files/Modules.Files/Jobs/PurgeDeletedFilesJob.cs index 6824fd3213..feae358297 100644 --- a/src/Modules/Files/Modules.Files/Jobs/PurgeDeletedFilesJob.cs +++ b/src/Modules/Files/Modules.Files/Jobs/PurgeDeletedFilesJob.cs @@ -1,20 +1,26 @@ +using Finbuckle.MultiTenant; +using Finbuckle.MultiTenant.Abstractions; +using FSH.Framework.Persistence; using FSH.Framework.Quota; +using FSH.Framework.Shared.Multitenancy; using FSH.Framework.Shared.Quota; using FSH.Framework.Storage.Services; using FSH.Modules.Files.Data; using Hangfire; using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; namespace FSH.Modules.Files.Jobs; /// -/// Daily purge of soft-deleted FileAsset rows past the retention window. Hard-deletes the row, -/// removes the bytes from storage, and refunds the quota (the bytes were debited at finalize time). +/// Daily purge of soft-deleted FileAsset rows past the retention window, in every tenant. Hard-deletes +/// the row, removes the bytes from storage, and refunds the owning tenant's quota (the bytes were +/// debited at finalize time). /// public sealed class PurgeDeletedFilesJob( - FilesDbContext db, + IServiceScopeFactory scopeFactory, IStorageService storage, IQuotaService quotas, IOptions options, @@ -23,60 +29,103 @@ public sealed class PurgeDeletedFilesJob( [AutomaticRetry(Attempts = 2, DelaysInSeconds = [300, 1800])] public async Task RunAsync(CancellationToken cancellationToken) { + // The recurring job is registered without a tenant, so none is resolved. Each tenant is purged + // inside its own context, which also points FilesDbContext at a tenant's dedicated database. + List tenants; + using (var scope = scopeFactory.CreateScope()) + { + var tenantStore = scope.ServiceProvider.GetRequiredService>(); + tenants = (await tenantStore.GetAllAsync().ConfigureAwait(false)).ToList(); + } + var cutoff = DateTimeOffset.UtcNow.AddDays(-options.Value.SoftDeleteRetentionDays); - var candidates = await db.FileAssets - .IgnoreQueryFilters() - .Where(f => f.IsDeleted && f.DeletedOnUtc != null && f.DeletedOnUtc < cutoff) - .ToListAsync(cancellationToken) - .ConfigureAwait(false); + int total = 0; + foreach (var tenant in tenants) + { + cancellationToken.ThrowIfCancellationRequested(); + total += await PurgeTenantAsync(tenant, cutoff, cancellationToken).ConfigureAwait(false); + } - if (candidates.Count == 0) + if (total > 0 && logger.IsEnabled(LogLevel.Information)) { - return; + logger.LogInformation("Hard-purged {Total} soft-deleted file assets across {TenantCount} tenants", + total, tenants.Count); } + } - // Best-effort byte removal per file. Schema-per-tenant means all rows share one tenant, - // and Hangfire wires the job per-tenant for multi-tenant deployments. - foreach (var f in candidates) + private async Task PurgeTenantAsync(AppTenantInfo tenant, DateTimeOffset cutoff, CancellationToken cancellationToken) + { + try { - try + using var scope = scopeFactory.CreateScope(); + scope.ServiceProvider.GetRequiredService() + .MultiTenantContext = new MultiTenantContext(tenant); + var db = scope.ServiceProvider.GetRequiredService(); + + // Soft-deleted rows are hidden by the soft-delete filter; lift only that one so the tenant + // filter stays on. + var candidates = await db.FileAssets + .IgnoreQueryFilters([QueryFilters.SoftDelete]) + .Where(f => f.IsDeleted && f.DeletedOnUtc != null && f.DeletedOnUtc < cutoff) + .ToListAsync(cancellationToken) + .ConfigureAwait(false); + + if (candidates.Count == 0) { - await storage.RemoveAsync(f.StorageKey, cancellationToken).ConfigureAwait(false); + return 0; } - catch (Exception ex) + + // Best-effort byte removal per file. + foreach (var f in candidates) { - logger.LogWarning(ex, "Storage remove failed for {Key}", f.StorageKey); + try + { + await storage.RemoveAsync(f.StorageKey, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + logger.LogWarning(ex, "Storage remove failed for {Key} for tenant {TenantId}", + f.StorageKey, tenant.Id); + } } - } - // Quota refund — group bytes once per tenant. In schema-per-tenant the resolved tenant - // matches every row's logical tenant; the framework's QuotaService is tenant-scoped via DI. - var totalBytes = candidates.Sum(f => f.SizeBytes); - if (totalBytes > 0) - { - // Empty tenant id satisfies the contract; QuotaService resolves the tenant from DI. - // Falls back gracefully with no tenant (the refund is simply lost). - try + var ids = candidates.Select(f => f.Id).ToList(); + await db.FileAssets + .IgnoreQueryFilters([QueryFilters.SoftDelete]) + .Where(f => ids.Contains(f.Id)) + .ExecuteDeleteAsync(cancellationToken) + .ConfigureAwait(false); + + // Refund the bytes debited at finalize, under the tenant that owns them. Done after the + // rows are gone so a failed delete can't refund bytes that are still counted as stored. + var totalBytes = candidates.Sum(f => f.SizeBytes); + if (totalBytes > 0) { - await quotas.RecordAsync("", QuotaResource.StorageBytes, -totalBytes, cancellationToken).ConfigureAwait(false); + try + { + await quotas.RecordAsync(tenant.Id, QuotaResource.StorageBytes, -totalBytes, cancellationToken) + .ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + logger.LogWarning(ex, "Quota refund of {Bytes} bytes failed for tenant {TenantId}", + totalBytes, tenant.Id); + } } - catch (Exception ex) + + if (logger.IsEnabled(LogLevel.Information)) { - logger.LogWarning(ex, "Quota refund failed for {Bytes} bytes", totalBytes); + logger.LogInformation("Hard-purged {Count} soft-deleted file assets ({Bytes} bytes) for tenant {TenantId}", + candidates.Count, totalBytes, tenant.Id); } - } - var ids = candidates.Select(f => f.Id).ToList(); - await db.FileAssets - .IgnoreQueryFilters() - .Where(f => ids.Contains(f.Id)) - .ExecuteDeleteAsync(cancellationToken) - .ConfigureAwait(false); - - if (logger.IsEnabled(LogLevel.Information)) + return candidates.Count; + } + catch (Exception ex) when (ex is not OperationCanceledException) { - logger.LogInformation("Hard-purged {Count} soft-deleted file assets ({Bytes} bytes total)", - candidates.Count, totalBytes); + // One tenant's database being unreachable must not keep the other tenants' files around. + logger.LogError(ex, "Deleted file purge failed for tenant {TenantId}", tenant.Id); + return 0; } } } diff --git a/src/Modules/Files/Modules.Files/Jobs/PurgeOrphanedFilesJob.cs b/src/Modules/Files/Modules.Files/Jobs/PurgeOrphanedFilesJob.cs index b280ffd585..bdfcc7ab79 100644 --- a/src/Modules/Files/Modules.Files/Jobs/PurgeOrphanedFilesJob.cs +++ b/src/Modules/Files/Modules.Files/Jobs/PurgeOrphanedFilesJob.cs @@ -1,65 +1,114 @@ +using Finbuckle.MultiTenant; +using Finbuckle.MultiTenant.Abstractions; +using FSH.Framework.Persistence; +using FSH.Framework.Shared.Multitenancy; using FSH.Framework.Storage.Services; using FSH.Modules.Files.Contracts.v1.DTOs; using FSH.Modules.Files.Data; using FSH.Modules.Files.Domain; using Hangfire; using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; namespace FSH.Modules.Files.Jobs; /// -/// Hourly purge of FileAsset rows stuck in PendingUpload past their UploadDeadline. Best-effort -/// removal of any bytes that did make it to storage. No quota effect — those bytes were never -/// debited. +/// Hourly purge of FileAsset rows stuck in PendingUpload past their UploadDeadline, in every tenant. +/// Best-effort removal of any bytes that did make it to storage. No quota effect — those bytes were +/// never debited. /// public sealed class PurgeOrphanedFilesJob( - FilesDbContext db, + IServiceScopeFactory scopeFactory, IStorageService storage, ILogger logger) { [AutomaticRetry(Attempts = 3, DelaysInSeconds = [30, 120, 600])] public async Task RunAsync(CancellationToken cancellationToken) { + // The recurring job is registered without a tenant, so none is resolved. Each tenant is purged + // inside its own context, which also points FilesDbContext at a tenant's dedicated database. + List tenants; + using (var scope = scopeFactory.CreateScope()) + { + var tenantStore = scope.ServiceProvider.GetRequiredService>(); + tenants = (await tenantStore.GetAllAsync().ConfigureAwait(false)).ToList(); + } + var now = DateTimeOffset.UtcNow; - var orphans = await db.FileAssets - .IgnoreQueryFilters() - .Where(f => f.Status == FileAssetStatus.PendingUpload - && f.UploadDeadline != null - && f.UploadDeadline < now) - .ToListAsync(cancellationToken) - .ConfigureAwait(false); + int total = 0; + foreach (var tenant in tenants) + { + cancellationToken.ThrowIfCancellationRequested(); + total += await PurgeTenantAsync(tenant, now, cancellationToken).ConfigureAwait(false); + } - if (orphans.Count == 0) + if (total > 0 && logger.IsEnabled(LogLevel.Information)) { - return; + logger.LogInformation("Purged {Total} orphaned file assets across {TenantCount} tenants", + total, tenants.Count); } + } - foreach (var f in orphans) + private async Task PurgeTenantAsync(AppTenantInfo tenant, DateTimeOffset now, CancellationToken cancellationToken) + { + try { - try + using var scope = scopeFactory.CreateScope(); + scope.ServiceProvider.GetRequiredService() + .MultiTenantContext = new MultiTenantContext(tenant); + var db = scope.ServiceProvider.GetRequiredService(); + + // Only the soft-delete filter is lifted; the tenant filter stays on. + var orphans = await db.FileAssets + .IgnoreQueryFilters([QueryFilters.SoftDelete]) + .Where(f => f.Status == FileAssetStatus.PendingUpload + && f.UploadDeadline != null + && f.UploadDeadline < now) + .ToListAsync(cancellationToken) + .ConfigureAwait(false); + + if (orphans.Count == 0) { - await storage.RemoveAsync(f.StorageKey, cancellationToken).ConfigureAwait(false); + return 0; } - catch (Exception ex) + + foreach (var f in orphans) { - logger.LogWarning(ex, "Failed to remove orphan storage object {Key}", f.StorageKey); + try + { + await storage.RemoveAsync(f.StorageKey, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + logger.LogWarning(ex, "Failed to remove orphan storage object {Key} for tenant {TenantId}", + f.StorageKey, tenant.Id); + } } - // Hard delete (row never reached Available, so soft-delete doesn't apply). FileAsset is ISoftDeletable, - // so Remove() would become UPDATE IsDeleted=true — we use the bulk ExecuteDelete below to bypass the interceptor instead. - } - // Bulk hard delete — bypasses the soft-delete interceptor. - var ids = orphans.Select(f => f.Id).ToList(); - await db.FileAssets - .IgnoreQueryFilters() - .Where(f => ids.Contains(f.Id)) - .ExecuteDeleteAsync(cancellationToken) - .ConfigureAwait(false); + // Hard delete (the row never reached Available, so soft-delete doesn't apply). FileAsset is + // ISoftDeletable, so Remove() would become UPDATE IsDeleted=true — the bulk ExecuteDelete + // bypasses the interceptor instead. + var ids = orphans.Select(f => f.Id).ToList(); + await db.FileAssets + .IgnoreQueryFilters([QueryFilters.SoftDelete]) + .Where(f => ids.Contains(f.Id)) + .ExecuteDeleteAsync(cancellationToken) + .ConfigureAwait(false); - if (logger.IsEnabled(LogLevel.Information)) + if (logger.IsEnabled(LogLevel.Information)) + { + logger.LogInformation("Purged {Count} orphaned file assets for tenant {TenantId}", + orphans.Count, tenant.Id); + } + + return orphans.Count; + } + catch (Exception ex) when (ex is not OperationCanceledException) { - logger.LogInformation("Purged {Count} orphaned file assets", orphans.Count); + // One tenant's database being unreachable must not keep the other tenants' orphans around. + logger.LogError(ex, "Orphaned file purge failed for tenant {TenantId}", tenant.Id); + return 0; } } } diff --git a/src/Tests/Integration.Tests/Tests/Files/PurgeJobsPerTenantTests.cs b/src/Tests/Integration.Tests/Tests/Files/PurgeJobsPerTenantTests.cs new file mode 100644 index 0000000000..ea08ae591a --- /dev/null +++ b/src/Tests/Integration.Tests/Tests/Files/PurgeJobsPerTenantTests.cs @@ -0,0 +1,271 @@ +using System.Security.Cryptography; +using Finbuckle.MultiTenant; +using Finbuckle.MultiTenant.Abstractions; +using FSH.Framework.Quota; +using FSH.Framework.Shared.Multitenancy; +using FSH.Framework.Shared.Quota; +using FSH.Framework.Storage.Services; +using FSH.Modules.Files.Contracts.v1.DTOs; +using FSH.Modules.Files.Data; +using FSH.Modules.Files.Jobs; +using Integration.Tests.Infrastructure; +using Integration.Tests.Infrastructure.Extensions; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Npgsql; + +namespace Integration.Tests.Tests.Files; + +/// +/// The Files purge jobs are Hangfire recurring jobs registered without a tenant, so they run with no +/// ambient tenant. Purgeable files are seeded in three tenants — root, a tenant on the shared database +/// and a tenant with its own dedicated database — and each job is resolved from a fresh, tenant-less +/// scope (the shape Hangfire's activator gives it) and run once. Every tenant's rows and blobs must be +/// purged, and the deleted-files purge must refund each tenant's storage quota under that tenant's id. +/// +[Collection(FshCollectionDefinition.Name)] +public sealed class PurgeJobsPerTenantTests +{ + private const string FilesBasePath = "/api/v1/files"; + private const int FileSize = 256; + + private readonly FshWebApplicationFactory _factory; + private readonly AuthHelper _auth; + + public PurgeJobsPerTenantTests(FshWebApplicationFactory factory) + { + _factory = factory; + _auth = new AuthHelper(factory); + } + + [Fact] + public async Task PurgeDeletedFilesJob_Should_PurgeAndRefundQuota_InEveryTenant_IncludingDedicatedDatabase() + { + // Arrange + var tenants = await CreateTenantClientsAsync("files-del"); + var quotaStore = new InMemoryQuotaStore(); + var quotas = CreateQuotaService(quotaStore); + + var seeded = new List<(string TenantId, Guid Id, string Key)>(); + foreach (var (tenantId, client) in tenants) + { + var (id, key) = await UploadFinalizeAndDeleteAsync(tenantId, client, "purge-me.pdf"); + await BackdateAsync(tenantId, id, deletedOnUtc: DateTimeOffset.UtcNow.AddDays(-90), uploadDeadline: null); + + // Quota is disabled in the harness, so the finalize debit is replayed here against the + // in-memory counter the job is handed. + await quotas.RecordAsync(tenantId, QuotaResource.StorageBytes, FileSize); + seeded.Add((tenantId, id, key)); + } + + // Act — fresh scope, no tenant set, as FshJobActivator gives a job registered via AddOrUpdate. + using (var jobScope = _factory.Services.CreateScope()) + { + var job = ActivatorUtilities.CreateInstance(jobScope.ServiceProvider, quotas); + await job.RunAsync(CancellationToken.None); + } + + // Assert + foreach (var (tenantId, id, key) in seeded) + { + (await RowExistsIgnoringFiltersAsync(tenantId, id)) + .ShouldBeFalse($"soft-deleted file past retention in tenant '{tenantId}' must be hard-purged"); + (await ObjectExistsAsync(key)) + .ShouldBeFalse($"storage object of the purged file in tenant '{tenantId}' must be removed"); + (await quotas.GetCurrentAsync(tenantId, QuotaResource.StorageBytes)) + .ShouldBe(0, $"the purged bytes must be refunded to tenant '{tenantId}'"); + } + + quotaStore.Counters.Keys.ShouldNotContain(k => k.StartsWith("quota::", StringComparison.Ordinal), + "no refund may be recorded against an empty tenant id"); + } + + [Fact] + public async Task PurgeOrphanedFilesJob_Should_PurgeExpiredPendingUploads_InEveryTenant_IncludingDedicatedDatabase() + { + // Arrange + var tenants = await CreateTenantClientsAsync("files-orph"); + + var seeded = new List<(string TenantId, Guid Id, string Key)>(); + foreach (var (tenantId, client) in tenants) + { + var (id, key) = await RequestUploadAndPutBytesAsync(tenantId, client, "orphan.pdf"); + await BackdateAsync(tenantId, id, deletedOnUtc: null, uploadDeadline: DateTimeOffset.UtcNow.AddHours(-2)); + seeded.Add((tenantId, id, key)); + } + + // Act — fresh scope, no tenant set. + using (var jobScope = _factory.Services.CreateScope()) + { + var job = ActivatorUtilities.CreateInstance(jobScope.ServiceProvider); + await job.RunAsync(CancellationToken.None); + } + + // Assert + foreach (var (tenantId, id, key) in seeded) + { + (await RowExistsIgnoringFiltersAsync(tenantId, id)) + .ShouldBeFalse($"expired pending upload in tenant '{tenantId}' must be hard-purged"); + (await ObjectExistsAsync(key)) + .ShouldBeFalse($"orphaned storage object in tenant '{tenantId}' must be removed"); + } + } + + // ─── tenants ───────────────────────────────────────────────────── + + /// Root, a shared-database tenant and a dedicated-database tenant, each with an admin client. + private async Task> CreateTenantClientsAsync(string prefix) + { + using var rootClient = await _auth.CreateRootAdminClientAsync(); + var uniqueId = Guid.NewGuid().ToString("N")[..8]; + + var sharedTenantId = $"{prefix}-shared-{uniqueId}"; + var dedicatedTenantId = $"{prefix}-dedicated-{uniqueId}"; + + var defaultConnection = _factory.Services.GetRequiredService()["DatabaseOptions:ConnectionString"]; + defaultConnection.ShouldNotBeNullOrWhiteSpace(); + var dedicatedConnection = new NpgsqlConnectionStringBuilder(defaultConnection) + { + Database = $"fsh_{prefix.Replace('-', '_')}_{uniqueId}", + }.ConnectionString; + + await CreateTenantAsync(rootClient, sharedTenantId, connectionString: null); + await CreateTenantAsync(rootClient, dedicatedTenantId, dedicatedConnection); + await TenantProvisioningWait.WaitForProvisioningAsync(rootClient, sharedTenantId); + await TenantProvisioningWait.WaitForProvisioningAsync(rootClient, dedicatedTenantId); + + return + [ + (TestConstants.RootTenantId, await _auth.CreateRootAdminClientAsync()), + (sharedTenantId, await _auth.CreateAuthenticatedClientAsync(AdminEmail(sharedTenantId), TestConstants.DefaultPassword, sharedTenantId)), + (dedicatedTenantId, await _auth.CreateAuthenticatedClientAsync(AdminEmail(dedicatedTenantId), TestConstants.DefaultPassword, dedicatedTenantId)), + ]; + } + + private static string AdminEmail(string tenantId) => $"admin-{tenantId}@tenant.com"; + + private static async Task CreateTenantAsync(HttpClient rootClient, string tenantId, string? connectionString) + { + using var response = await rootClient.PostAsJsonAsync(TestConstants.TenantsBasePath, new + { + id = tenantId, + name = $"Tenant {tenantId}", + connectionString, + adminEmail = AdminEmail(tenantId), + adminPassword = TestConstants.DefaultPassword, + issuer = $"{tenantId}.issuer" + }); + var body = await response.Content.ReadAsStringAsync(); + response.StatusCode.ShouldBe(HttpStatusCode.Created, $"Create tenant failed: {body}"); + } + + private static InMemoryQuotaService CreateQuotaService(InMemoryQuotaStore store) + { + var options = new QuotaOptions { Enabled = true }; + return new InMemoryQuotaService(store, options, new QuotaPlanResolver(options), [], TimeProvider.System); + } + + // ─── files ─────────────────────────────────────────────────────── + + private async Task<(Guid Id, string StorageKey)> UploadFinalizeAndDeleteAsync( + string tenantId, HttpClient client, string fileName) + { + var (id, storageKey) = await RequestUploadAndPutBytesAsync(tenantId, client, fileName); + + using var finalize = await client.PostAsync($"{FilesBasePath}/{id}/finalize", null); + finalize.EnsureSuccessStatusCode(); + + using var del = await client.DeleteAsync($"{FilesBasePath}/{id}"); + del.StatusCode.ShouldBe(HttpStatusCode.NoContent); + + return (id, storageKey); + } + + private async Task<(Guid Id, string StorageKey)> RequestUploadAndPutBytesAsync( + string tenantId, HttpClient client, string fileName) + { + using var response = await client.PostAsJsonAsync($"{FilesBasePath}/upload-url", new + { + ownerType = "MyFiles", + ownerId = (Guid?)null, + fileName, + contentType = "application/pdf", + sizeBytes = FileSize, + visibility = 1, + category = "Document", + }); + var body = await response.Content.ReadAsStringAsync(); + response.StatusCode.ShouldBe(HttpStatusCode.OK, $"upload-url failed in tenant '{tenantId}': {body}"); + var presigned = await response.DeserializeAsync(); + + byte[] bytes = new byte[FileSize]; + RandomNumberGenerator.Fill(bytes); + using var raw = new HttpClient(); + using var put = new HttpRequestMessage(HttpMethod.Put, presigned.UploadUrl) + { + Content = new ByteArrayContent(bytes) + { + Headers = { ContentType = new MediaTypeHeaderValue("application/pdf") } + } + }; + using var putResp = await raw.SendAsync(put); + putResp.EnsureSuccessStatusCode(); + + var storageKey = await ReadStorageKeyAsync(tenantId, presigned.FileAssetId); + return (presigned.FileAssetId, storageKey); + } + + // Tenant context is an AsyncLocal, so it is set in the same method as each DbContext call. + + private async Task ReadStorageKeyAsync(string tenantId, Guid id) + { + using var scope = _factory.Services.CreateScope(); + var tenant = await scope.ServiceProvider.GetRequiredService>().GetAsync(tenantId); + tenant.ShouldNotBeNull(); + scope.ServiceProvider.GetRequiredService().MultiTenantContext = + new MultiTenantContext(tenant); + + var db = scope.ServiceProvider.GetRequiredService(); + var key = await db.FileAssets.IgnoreQueryFilters() + .Where(f => f.Id == id) + .Select(f => f.StorageKey) + .FirstOrDefaultAsync(); + key.ShouldNotBeNull(); + return key; + } + + private async Task BackdateAsync(string tenantId, Guid id, DateTimeOffset? deletedOnUtc, DateTimeOffset? uploadDeadline) + { + using var scope = _factory.Services.CreateScope(); + var tenant = await scope.ServiceProvider.GetRequiredService>().GetAsync(tenantId); + tenant.ShouldNotBeNull(); + scope.ServiceProvider.GetRequiredService().MultiTenantContext = + new MultiTenantContext(tenant); + + var db = scope.ServiceProvider.GetRequiredService(); + var query = db.FileAssets.IgnoreQueryFilters().Where(f => f.Id == id); + int updated = deletedOnUtc is { } deleted + ? await query.ExecuteUpdateAsync(s => s.SetProperty(f => f.DeletedOnUtc, deleted)) + : await query.ExecuteUpdateAsync(s => s.SetProperty(f => f.UploadDeadline, uploadDeadline)); + updated.ShouldBe(1); + } + + private async Task RowExistsIgnoringFiltersAsync(string tenantId, Guid id) + { + using var scope = _factory.Services.CreateScope(); + var tenant = await scope.ServiceProvider.GetRequiredService>().GetAsync(tenantId); + tenant.ShouldNotBeNull(); + scope.ServiceProvider.GetRequiredService().MultiTenantContext = + new MultiTenantContext(tenant); + + var db = scope.ServiceProvider.GetRequiredService(); + return await db.FileAssets.IgnoreQueryFilters().AnyAsync(f => f.Id == id); + } + + private async Task ObjectExistsAsync(string storageKey) + { + using var scope = _factory.Services.CreateScope(); + return await scope.ServiceProvider.GetRequiredService().ExistsAsync(storageKey); + } +}