diff --git a/DocAnalytics.Api/BackgroundServices/ExtractionWorker.cs b/DocAnalytics.Api/BackgroundServices/ExtractionWorker.cs index 4fc49d5..78fe543 100644 --- a/DocAnalytics.Api/BackgroundServices/ExtractionWorker.cs +++ b/DocAnalytics.Api/BackgroundServices/ExtractionWorker.cs @@ -74,6 +74,9 @@ await notifier.NotifyFileStateChangedAsync(file.SiteId, try { + + var validateStart = DateTime.UtcNow; + // ── SECURITY GATE 1: GuardDuty malware verdict FIRST ── // Runs BEFORE download/extraction so we never spend Bedrock cost on an unscanned/malicious file. // GuardDuty writes the tag asynchronously after upload, so poll briefly. @@ -133,10 +136,17 @@ await FailFileAsync(db, notifier, file, txn, now, } // ───────────────────────────────────────────────────────────────────────────── + var validateEnd = DateTime.UtcNow; + var extractStart = DateTime.UtcNow; + + var result = await extractor.ExtractAsync(bytes, detectedType, ct); var v = validator.Validate(result); + var extractEnd = DateTime.UtcNow; + var loadStart = DateTime.UtcNow; + // ⚠️ GOTCHA #2: idempotent → clear existing line items first var old = await db.InvoiceLineItems.Where(i => i.FileId == file.Id).ToListAsync(ct); db.RemoveRange(old); @@ -215,6 +225,16 @@ await FailFileAsync(db, notifier, file, txn, now, file.ExtractionConfidence = v.Confidence; file.LastUpdatedAt = done; + db.Add(new FileStepHistory + { + Id = Guid.NewGuid(), + FileId = file.Id, + DocumentTypeId = file.DocumentTypeId, + StepName = "Validate", + Status = "Success", + StartedAt = validateStart, + CompletedAt = validateEnd + }); db.Add(new FileStepHistory { Id = Guid.NewGuid(), @@ -222,11 +242,21 @@ await FailFileAsync(db, notifier, file, txn, now, DocumentTypeId = file.DocumentTypeId, StepName = "Extract", Status = failed ? "Failed" : "Success", - StartedAt = now, - CompletedAt = done, + StartedAt = extractStart, + CompletedAt = extractEnd, ErrorCode = failed ? v.ErrorCode : null, ErrorMessage = failed ? "Extraction confidence too low." : null }); + db.Add(new FileStepHistory + { + Id = Guid.NewGuid(), + FileId = file.Id, + DocumentTypeId = file.DocumentTypeId, + StepName = "Load", + Status = failed ? "Failed" : "Success", + StartedAt = loadStart, + CompletedAt = done + }); txn.ProcessingCount = Math.Max(0, txn.ProcessingCount - 1); if (failed) txn.FailedCount += 1; else txn.CompletedCount += 1; diff --git a/DocAnalytics.Service.Tests/Analytics/AnalyticsServiceTests.cs b/DocAnalytics.Service.Tests/Analytics/AnalyticsServiceTests.cs index 72c738d..fc21f42 100644 --- a/DocAnalytics.Service.Tests/Analytics/AnalyticsServiceTests.cs +++ b/DocAnalytics.Service.Tests/Analytics/AnalyticsServiceTests.cs @@ -86,7 +86,7 @@ public async Task GetStepPercentilesAsync_computes_durations() new FileStepHistory { Id = Guid.NewGuid(), StepName = "Upload", Status = "Success", StartedAt = start, CompletedAt = start.AddSeconds(10) }, new FileStepHistory { Id = Guid.NewGuid(), StepName = "Upload", Status = "Success", StartedAt = start, CompletedAt = start.AddSeconds(20) }), }; - var result = await Sut(files).GetStepPercentilesAsync(); + var result = await Sut(files).GetStepPercentilesAsync(CancellationToken.None); var upload = result.Single(r => r.Step == "Upload"); Assert.Equal(2, upload.SampleCount); Assert.InRange(upload.P50Seconds, 10, 20); diff --git a/DocAnalytics.Service/Analytics/AnalyticsService.cs b/DocAnalytics.Service/Analytics/AnalyticsService.cs index 483cb63..e97cf3b 100644 --- a/DocAnalytics.Service/Analytics/AnalyticsService.cs +++ b/DocAnalytics.Service/Analytics/AnalyticsService.cs @@ -120,24 +120,27 @@ public async Task GetErrorTrendAsync(DateTime? from, DateTime? to, Ca } /// - public async Task> GetStepPercentilesAsync(CancellationToken ct = default) + public async Task> GetStepPercentilesAsync(CancellationToken ct) { - // Drive from Files (ITenantScoped → tenant_id + site_id auto-applied), - // navigate out to its steps → isolation guaranteed without touching FileStepHistory directly. - var raw = await _db.Files - .AsNoTracking() - .SelectMany(f => f.Steps) - .Where(s => s.StartedAt != null && s.CompletedAt != null) // only completed steps + // Navigate through FileRecord (which carries the global tenant+site query filter) + // so we only read steps belonging to the current tenant/site. + var stepData = await _db.Files + .SelectMany(f => f.Steps, (_, s) => s) + .Where(s => s.Status == "Success" + && s.StartedAt != null + && s.CompletedAt != null) .Select(s => new { s.StepName, s.StartedAt, s.CompletedAt }) .ToListAsync(ct); - return raw - .GroupBy(x => x.StepName) + if (!stepData.Any()) + return new List(); + + return stepData + .GroupBy(s => s.StepName) .Select(g => { var durations = g - .Select(x => (x.CompletedAt!.Value - x.StartedAt!.Value).TotalSeconds) - .Where(d => d >= 0) + .Select(s => (s.CompletedAt!.Value - s.StartedAt!.Value).TotalSeconds) .OrderBy(d => d) .ToList(); @@ -150,20 +153,23 @@ public async Task> GetStepPercentilesAsync(CancellationT P99Seconds = Math.Round(Percentile(durations, 0.99), 1), }; }) - .OrderBy(r => StepOrder(r.Step)) // Upload → Validate → Transform → Load + .OrderBy(s => StepOrder(s.Step)) .ToList(); } - // Linear-interpolation percentile (same method Postgres percentile_cont uses). - private static double Percentile(IReadOnlyList sorted, double p) + // Linear interpolation (same as numpy/Excel PERCENTILE.INC) + private static double Percentile(List sorted, double p) { if (sorted.Count == 0) return 0; if (sorted.Count == 1) return sorted[0]; - var rank = p * (sorted.Count - 1); - var lo = (int)Math.Floor(rank); - var hi = (int)Math.Ceiling(rank); + + double idx = p * (sorted.Count - 1); + int lo = (int)Math.Floor(idx); + int hi = (int)Math.Ceiling(idx); if (lo == hi) return sorted[lo]; - return sorted[lo] + (sorted[hi] - sorted[lo]) * (rank - lo); + + double frac = idx - lo; + return sorted[lo] * (1 - frac) + sorted[hi] * frac; } private static int StepOrder(string step) => step switch @@ -172,8 +178,9 @@ private static double Percentile(IReadOnlyList sorted, double p) "Validate" => 1, "Transform" => 2, "Load" => 3, - _ => 99 + _ => 99, }; + } diff --git a/DocAnalytics.Service/Uploads/UploadService.cs b/DocAnalytics.Service/Uploads/UploadService.cs index b168ee3..43b6877 100644 --- a/DocAnalytics.Service/Uploads/UploadService.cs +++ b/DocAnalytics.Service/Uploads/UploadService.cs @@ -207,11 +207,30 @@ public async Task DeleteBatchAsync(Guid batchId, CancellationToken ct = de foreach (var f in files.Where(f => !string.IsNullOrEmpty(f.StorageKey))) await _storage.DeleteAsync(f.StorageKey!, ct); + // 2) DB: files cascade to headers/line-items/errors; then the batch itself // 2) DB: files cascade to headers/line-items/errors; then the batch itself _db.Files.RemoveRange(files); _db.Remove(txn); + + // ── audit trail: write BEFORE SaveChanges so it commits in the same transaction ── + _db.Add(new DocAnalytics.Domain.Entities.ActivityLog + { + Id = Guid.NewGuid(), + TenantId = txn.TenantId, + SiteId = txn.SiteId, + EventType = "BATCH_DELETED", + EntityType = "Batch", + EntityId = txn.Id, + EntityName = txn.SourceSystem, + OldState = txn.State, + NewState = "Deleted", + TriggeredBy = _me.UserId.ToString(), + CreatedAt = DateTime.UtcNow + }); + await _db.SaveChangesAsync(ct); return true; + } diff --git a/docanalytics-web/src/app/features/activity-log/activity-log.component.css b/docanalytics-web/src/app/features/activity-log/activity-log.component.css index 6165ce9..1b93ff9 100644 --- a/docanalytics-web/src/app/features/activity-log/activity-log.component.css +++ b/docanalytics-web/src/app/features/activity-log/activity-log.component.css @@ -129,3 +129,23 @@ background: var(--badge-file-bg, rgba(34, 197, 94, 0.12)); color: var(--badge-file-fg, #4ade80); } + +.al-evt-deleted { + opacity: 0.55; + text-decoration: line-through; +} + +.al-deleted-badge { + display: inline-block; + margin-left: 8px; + padding: 2px 8px; + border-radius: 4px; + font-size: 0.7rem; + font-weight: 600; + background: rgba(192, 57, 43, 0.12); + color: var(--text-error, #c0392b); +} + +:host ::ng-deep .row-orphaned td { + opacity: 0.4; +} diff --git a/docanalytics-web/src/app/features/activity-log/activity-log.component.html b/docanalytics-web/src/app/features/activity-log/activity-log.component.html index 4dd7008..ecef0e6 100644 --- a/docanalytics-web/src/app/features/activity-log/activity-log.component.html +++ b/docanalytics-web/src/app/features/activity-log/activity-log.component.html @@ -22,6 +22,7 @@

Activity Log

Activity Log - {{ eventLabel(row.event_type) }} + + {{ eventLabel(row.event_type) }} + + @if (row.event_type === 'BATCH_DELETED') { + 🗑️ Deleted + } diff --git a/docanalytics-web/src/app/features/activity-log/activity-log.component.ts b/docanalytics-web/src/app/features/activity-log/activity-log.component.ts index dfe3ab3..e5e35d6 100644 --- a/docanalytics-web/src/app/features/activity-log/activity-log.component.ts +++ b/docanalytics-web/src/app/features/activity-log/activity-log.component.ts @@ -65,12 +65,16 @@ export class ActivityLogComponent { return false; } + protected readonly rowClass = (row: ActivityLogItem): string => + row.entity_type === 'File' && !row.batch_id ? 'row-orphaned' : ''; + protected readonly eventTypeOptions: FilterOption[] = [ { value: 'all', label: 'All events' }, { value: 'FILE_STATE_CHANGED', label: 'File state changed' }, { value: 'BATCH_SUBMITTED', label: 'Batch submitted' }, { value: 'BATCH_COMPLETED', label: 'Batch completed' }, { value: 'BATCH_FAILED', label: 'Batch failed' }, + { value: 'BATCH_DELETED', label: 'Batch deleted' }, ]; // 'transition' has no backing field → rendered via the dtCell template (not sortable) @@ -87,6 +91,7 @@ export class ActivityLogComponent { BATCH_SUBMITTED: 'Batch submitted', BATCH_COMPLETED: 'Batch completed', BATCH_FAILED: 'Batch failed', + BATCH_DELETED: 'Batch Deleted', }; constructor() { diff --git a/docanalytics-web/src/app/shared/components/data-table/data-table.component.html b/docanalytics-web/src/app/shared/components/data-table/data-table.component.html index 3ec466c..8cf6dd4 100644 --- a/docanalytics-web/src/app/shared/components/data-table/data-table.component.html +++ b/docanalytics-web/src/app/shared/components/data-table/data-table.component.html @@ -40,7 +40,11 @@ } @else { @for (row of rows(); track key(row, $index)) { - + @for (col of columns(); track col.key) { @if (cellTemplates().get(col.key); as tpl) { diff --git a/docanalytics-web/src/app/shared/components/data-table/data-table.component.ts b/docanalytics-web/src/app/shared/components/data-table/data-table.component.ts index 9832515..4db3818 100644 --- a/docanalytics-web/src/app/shared/components/data-table/data-table.component.ts +++ b/docanalytics-web/src/app/shared/components/data-table/data-table.component.ts @@ -8,7 +8,7 @@ import { input, output, } from '@angular/core'; -import { NgTemplateOutlet } from '@angular/common'; +import { NgClass, NgTemplateOutlet } from '@angular/common'; export type SortDir = 'asc' | 'desc'; export interface SortState { @@ -34,7 +34,7 @@ export class DtCellDirective { @Component({ selector: 'app-data-table', - imports: [NgTemplateOutlet], + imports: [NgTemplateOutlet, NgClass], templateUrl: './data-table.component.html', styleUrl: './data-table.component.css', @@ -58,6 +58,8 @@ export class DataTableComponent { readonly clickable = input(false); readonly rowId = input<((row: T) => string | number) | null>(null); + readonly rowClass = input<((row: T) => string) | null>(null); + readonly sortChange = output(); readonly pageChange = output(); readonly pageSizeChange = output();