From de2f3d7108a89f08808eb5334c5d11bc7a9d50e3 Mon Sep 17 00:00:00 2001 From: Akash Goswami Date: Fri, 24 Jul 2026 12:33:38 +0530 Subject: [PATCH 1/5] fix: compute step percentiles from real DB timestamps instead of hardcoded 120s --- .../BackgroundServices/ExtractionWorker.cs | 34 +++++++++++++- .../Analytics/AnalyticsService.cs | 46 +++++++++++-------- 2 files changed, 59 insertions(+), 21 deletions(-) diff --git a/DocAnalytics.Api/BackgroundServices/ExtractionWorker.cs b/DocAnalytics.Api/BackgroundServices/ExtractionWorker.cs index 4fc49d5..78fe543 100644 --- a/DocAnalytics.Api/BackgroundServices/ExtractionWorker.cs +++ b/DocAnalytics.Api/BackgroundServices/ExtractionWorker.cs @@ -74,6 +74,9 @@ await notifier.NotifyFileStateChangedAsync(file.SiteId, try { + + var validateStart = DateTime.UtcNow; + // ── SECURITY GATE 1: GuardDuty malware verdict FIRST ── // Runs BEFORE download/extraction so we never spend Bedrock cost on an unscanned/malicious file. // GuardDuty writes the tag asynchronously after upload, so poll briefly. @@ -133,10 +136,17 @@ await FailFileAsync(db, notifier, file, txn, now, } // ───────────────────────────────────────────────────────────────────────────── + var validateEnd = DateTime.UtcNow; + var extractStart = DateTime.UtcNow; + + var result = await extractor.ExtractAsync(bytes, detectedType, ct); var v = validator.Validate(result); + var extractEnd = DateTime.UtcNow; + var loadStart = DateTime.UtcNow; + // ⚠️ GOTCHA #2: idempotent → clear existing line items first var old = await db.InvoiceLineItems.Where(i => i.FileId == file.Id).ToListAsync(ct); db.RemoveRange(old); @@ -215,6 +225,16 @@ await FailFileAsync(db, notifier, file, txn, now, file.ExtractionConfidence = v.Confidence; file.LastUpdatedAt = done; + db.Add(new FileStepHistory + { + Id = Guid.NewGuid(), + FileId = file.Id, + DocumentTypeId = file.DocumentTypeId, + StepName = "Validate", + Status = "Success", + StartedAt = validateStart, + CompletedAt = validateEnd + }); db.Add(new FileStepHistory { Id = Guid.NewGuid(), @@ -222,11 +242,21 @@ await FailFileAsync(db, notifier, file, txn, now, DocumentTypeId = file.DocumentTypeId, StepName = "Extract", Status = failed ? "Failed" : "Success", - StartedAt = now, - CompletedAt = done, + StartedAt = extractStart, + CompletedAt = extractEnd, ErrorCode = failed ? v.ErrorCode : null, ErrorMessage = failed ? "Extraction confidence too low." : null }); + db.Add(new FileStepHistory + { + Id = Guid.NewGuid(), + FileId = file.Id, + DocumentTypeId = file.DocumentTypeId, + StepName = "Load", + Status = failed ? "Failed" : "Success", + StartedAt = loadStart, + CompletedAt = done + }); txn.ProcessingCount = Math.Max(0, txn.ProcessingCount - 1); if (failed) txn.FailedCount += 1; else txn.CompletedCount += 1; diff --git a/DocAnalytics.Service/Analytics/AnalyticsService.cs b/DocAnalytics.Service/Analytics/AnalyticsService.cs index 483cb63..0cfcdfc 100644 --- a/DocAnalytics.Service/Analytics/AnalyticsService.cs +++ b/DocAnalytics.Service/Analytics/AnalyticsService.cs @@ -1,4 +1,5 @@ using DocAnalytics.Data; +using DocAnalytics.Domain.Entities; using DocAnalytics.Service.Common; using Microsoft.EntityFrameworkCore; @@ -120,24 +121,27 @@ public async Task GetErrorTrendAsync(DateTime? from, DateTime? to, Ca } /// - public async Task> GetStepPercentilesAsync(CancellationToken ct = default) + public async Task> GetStepPercentilesAsync(CancellationToken ct) { - // Drive from Files (ITenantScoped → tenant_id + site_id auto-applied), - // navigate out to its steps → isolation guaranteed without touching FileStepHistory directly. - var raw = await _db.Files - .AsNoTracking() - .SelectMany(f => f.Steps) - .Where(s => s.StartedAt != null && s.CompletedAt != null) // only completed steps + // Navigate through FileRecord (which carries the global tenant+site query filter) + // so we only read steps belonging to the current tenant/site. + var stepData = await _db.Set() + .SelectMany(f => f.Steps, (_, s) => s) + .Where(s => s.Status == "Success" + && s.StartedAt != null + && s.CompletedAt != null) .Select(s => new { s.StepName, s.StartedAt, s.CompletedAt }) .ToListAsync(ct); - return raw - .GroupBy(x => x.StepName) + if (!stepData.Any()) + return new List(); + + return stepData + .GroupBy(s => s.StepName) .Select(g => { var durations = g - .Select(x => (x.CompletedAt!.Value - x.StartedAt!.Value).TotalSeconds) - .Where(d => d >= 0) + .Select(s => (s.CompletedAt!.Value - s.StartedAt!.Value).TotalSeconds) .OrderBy(d => d) .ToList(); @@ -150,20 +154,23 @@ public async Task> GetStepPercentilesAsync(CancellationT P99Seconds = Math.Round(Percentile(durations, 0.99), 1), }; }) - .OrderBy(r => StepOrder(r.Step)) // Upload → Validate → Transform → Load + .OrderBy(s => StepOrder(s.Step)) .ToList(); } - // Linear-interpolation percentile (same method Postgres percentile_cont uses). - private static double Percentile(IReadOnlyList sorted, double p) + // Linear interpolation (same as numpy/Excel PERCENTILE.INC) + private static double Percentile(List sorted, double p) { if (sorted.Count == 0) return 0; if (sorted.Count == 1) return sorted[0]; - var rank = p * (sorted.Count - 1); - var lo = (int)Math.Floor(rank); - var hi = (int)Math.Ceiling(rank); + + double idx = p * (sorted.Count - 1); + int lo = (int)Math.Floor(idx); + int hi = (int)Math.Ceiling(idx); if (lo == hi) return sorted[lo]; - return sorted[lo] + (sorted[hi] - sorted[lo]) * (rank - lo); + + double frac = idx - lo; + return sorted[lo] * (1 - frac) + sorted[hi] * frac; } private static int StepOrder(string step) => step switch @@ -172,8 +179,9 @@ private static double Percentile(IReadOnlyList sorted, double p) "Validate" => 1, "Transform" => 2, "Load" => 3, - _ => 99 + _ => 99, }; + } From e25c1a5656f27a02c281ba54a48d798b032059c0 Mon Sep 17 00:00:00 2001 From: Akash Goswami Date: Fri, 24 Jul 2026 12:39:11 +0530 Subject: [PATCH 2/5] fix: use Sut(files) in GetStepPercentilesAsync test instead of undefined service variable --- DocAnalytics.Service.Tests/Analytics/AnalyticsServiceTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DocAnalytics.Service.Tests/Analytics/AnalyticsServiceTests.cs b/DocAnalytics.Service.Tests/Analytics/AnalyticsServiceTests.cs index 72c738d..fc21f42 100644 --- a/DocAnalytics.Service.Tests/Analytics/AnalyticsServiceTests.cs +++ b/DocAnalytics.Service.Tests/Analytics/AnalyticsServiceTests.cs @@ -86,7 +86,7 @@ public async Task GetStepPercentilesAsync_computes_durations() new FileStepHistory { Id = Guid.NewGuid(), StepName = "Upload", Status = "Success", StartedAt = start, CompletedAt = start.AddSeconds(10) }, new FileStepHistory { Id = Guid.NewGuid(), StepName = "Upload", Status = "Success", StartedAt = start, CompletedAt = start.AddSeconds(20) }), }; - var result = await Sut(files).GetStepPercentilesAsync(); + var result = await Sut(files).GetStepPercentilesAsync(CancellationToken.None); var upload = result.Single(r => r.Step == "Upload"); Assert.Equal(2, upload.SampleCount); Assert.InRange(upload.P50Seconds, 10, 20); From 964c89cfcee02f9d5ad54b3a19ed27d51d4222e7 Mon Sep 17 00:00:00 2001 From: Akash Goswami Date: Fri, 24 Jul 2026 14:20:32 +0530 Subject: [PATCH 3/5] fix: use _db.Files instead of Set() in GetStepPercentilesAsync for mock compatibility --- DocAnalytics.Service/Analytics/AnalyticsService.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/DocAnalytics.Service/Analytics/AnalyticsService.cs b/DocAnalytics.Service/Analytics/AnalyticsService.cs index 0cfcdfc..e97cf3b 100644 --- a/DocAnalytics.Service/Analytics/AnalyticsService.cs +++ b/DocAnalytics.Service/Analytics/AnalyticsService.cs @@ -1,5 +1,4 @@ using DocAnalytics.Data; -using DocAnalytics.Domain.Entities; using DocAnalytics.Service.Common; using Microsoft.EntityFrameworkCore; @@ -125,7 +124,7 @@ public async Task> GetStepPercentilesAsync(CancellationT { // Navigate through FileRecord (which carries the global tenant+site query filter) // so we only read steps belonging to the current tenant/site. - var stepData = await _db.Set() + var stepData = await _db.Files .SelectMany(f => f.Steps, (_, s) => s) .Where(s => s.Status == "Success" && s.StartedAt != null From 8147d01230dea6b8bc2b49cc771c4cdb0b0c04e4 Mon Sep 17 00:00:00 2001 From: Akash Goswami Date: Fri, 24 Jul 2026 15:18:18 +0530 Subject: [PATCH 4/5] feat: real step processing times + BATCH_DELETED audit trail with orphaned row dimming --- DocAnalytics.Service/Uploads/UploadService.cs | 19 +++++ .../activity-log/activity-log.component.css | 21 ++++++ .../activity-log/activity-log.component.html | 71 ++++++++++--------- .../activity-log/activity-log.component.ts | 8 +++ .../data-table/data-table.component.html | 24 +++---- .../data-table/data-table.component.ts | 6 +- 6 files changed, 100 insertions(+), 49 deletions(-) diff --git a/DocAnalytics.Service/Uploads/UploadService.cs b/DocAnalytics.Service/Uploads/UploadService.cs index b168ee3..43b6877 100644 --- a/DocAnalytics.Service/Uploads/UploadService.cs +++ b/DocAnalytics.Service/Uploads/UploadService.cs @@ -207,11 +207,30 @@ public async Task DeleteBatchAsync(Guid batchId, CancellationToken ct = de foreach (var f in files.Where(f => !string.IsNullOrEmpty(f.StorageKey))) await _storage.DeleteAsync(f.StorageKey!, ct); + // 2) DB: files cascade to headers/line-items/errors; then the batch itself // 2) DB: files cascade to headers/line-items/errors; then the batch itself _db.Files.RemoveRange(files); _db.Remove(txn); + + // ── audit trail: write BEFORE SaveChanges so it commits in the same transaction ── + _db.Add(new DocAnalytics.Domain.Entities.ActivityLog + { + Id = Guid.NewGuid(), + TenantId = txn.TenantId, + SiteId = txn.SiteId, + EventType = "BATCH_DELETED", + EntityType = "Batch", + EntityId = txn.Id, + EntityName = txn.SourceSystem, + OldState = txn.State, + NewState = "Deleted", + TriggeredBy = _me.UserId.ToString(), + CreatedAt = DateTime.UtcNow + }); + await _db.SaveChangesAsync(ct); return true; + } diff --git a/docanalytics-web/src/app/features/activity-log/activity-log.component.css b/docanalytics-web/src/app/features/activity-log/activity-log.component.css index 6165ce9..0a66ca6 100644 --- a/docanalytics-web/src/app/features/activity-log/activity-log.component.css +++ b/docanalytics-web/src/app/features/activity-log/activity-log.component.css @@ -129,3 +129,24 @@ background: var(--badge-file-bg, rgba(34, 197, 94, 0.12)); color: var(--badge-file-fg, #4ade80); } + +.al-evt-deleted { + opacity: 0.55; + text-decoration: line-through; +} + +.al-deleted-badge { + display: inline-block; + margin-left: 8px; + padding: 2px 8px; + border-radius: 4px; + font-size: 0.7rem; + font-weight: 600; + background: rgba(192, 57, 43, 0.12); + color: var(--text-error, #c0392b); +} + +:host ::ng-deep .row-orphaned td { + opacity: 0.4; +} + diff --git a/docanalytics-web/src/app/features/activity-log/activity-log.component.html b/docanalytics-web/src/app/features/activity-log/activity-log.component.html index 4dd7008..58ce649 100644 --- a/docanalytics-web/src/app/features/activity-log/activity-log.component.html +++ b/docanalytics-web/src/app/features/activity-log/activity-log.component.html @@ -20,47 +20,50 @@

Activity Log

(changed)="onFilters($event)" /> - + {{ row.ts | date: 'medium' }} - {{ eventLabel(row.event_type) }} + + {{ eventLabel(row.event_type) }} + + @if (row.event_type === 'BATCH_DELETED') { + 🗑️ Deleted + } + @if (isNavigable(row)) { - + } @else { - {{ row.entity }} + {{ row.entity }} } - + {{ row.entity_type }} @@ -68,15 +71,15 @@

Activity Log

@if (row.old_state && row.new_state) { - - - - - - } @else if (row.new_state) { + + + + + } @else if (row.new_state) { + } @else { - + } diff --git a/docanalytics-web/src/app/features/activity-log/activity-log.component.ts b/docanalytics-web/src/app/features/activity-log/activity-log.component.ts index dfe3ab3..be3474c 100644 --- a/docanalytics-web/src/app/features/activity-log/activity-log.component.ts +++ b/docanalytics-web/src/app/features/activity-log/activity-log.component.ts @@ -65,12 +65,19 @@ export class ActivityLogComponent { return false; } + protected readonly rowClass = (row: ActivityLogItem): string => + row.entity_type === 'File' && !row.batch_id + ? 'row-orphaned' + : ''; + + protected readonly eventTypeOptions: FilterOption[] = [ { value: 'all', label: 'All events' }, { value: 'FILE_STATE_CHANGED', label: 'File state changed' }, { value: 'BATCH_SUBMITTED', label: 'Batch submitted' }, { value: 'BATCH_COMPLETED', label: 'Batch completed' }, { value: 'BATCH_FAILED', label: 'Batch failed' }, + { value: 'BATCH_DELETED', label: 'Batch deleted' }, ]; // 'transition' has no backing field → rendered via the dtCell template (not sortable) @@ -87,6 +94,7 @@ export class ActivityLogComponent { BATCH_SUBMITTED: 'Batch submitted', BATCH_COMPLETED: 'Batch completed', BATCH_FAILED: 'Batch failed', + BATCH_DELETED: 'Batch Deleted', }; constructor() { diff --git a/docanalytics-web/src/app/shared/components/data-table/data-table.component.html b/docanalytics-web/src/app/shared/components/data-table/data-table.component.html index 3ec466c..ff8f1e7 100644 --- a/docanalytics-web/src/app/shared/components/data-table/data-table.component.html +++ b/docanalytics-web/src/app/shared/components/data-table/data-table.component.html @@ -40,20 +40,18 @@ } @else { @for (row of rows(); track key(row, $index)) { - - @for (col of columns(); track col.key) { - - @if (cellTemplates().get(col.key); as tpl) { - - } @else { - {{ display(row, col) }} - } - + + @for (col of columns(); track col.key) { + + @if (cellTemplates().get(col.key); as tpl) { + + } @else { + {{ display(row, col) }} } - + + } + } } diff --git a/docanalytics-web/src/app/shared/components/data-table/data-table.component.ts b/docanalytics-web/src/app/shared/components/data-table/data-table.component.ts index 9832515..4db3818 100644 --- a/docanalytics-web/src/app/shared/components/data-table/data-table.component.ts +++ b/docanalytics-web/src/app/shared/components/data-table/data-table.component.ts @@ -8,7 +8,7 @@ import { input, output, } from '@angular/core'; -import { NgTemplateOutlet } from '@angular/common'; +import { NgClass, NgTemplateOutlet } from '@angular/common'; export type SortDir = 'asc' | 'desc'; export interface SortState { @@ -34,7 +34,7 @@ export class DtCellDirective { @Component({ selector: 'app-data-table', - imports: [NgTemplateOutlet], + imports: [NgTemplateOutlet, NgClass], templateUrl: './data-table.component.html', styleUrl: './data-table.component.css', @@ -58,6 +58,8 @@ export class DataTableComponent { readonly clickable = input(false); readonly rowId = input<((row: T) => string | number) | null>(null); + readonly rowClass = input<((row: T) => string) | null>(null); + readonly sortChange = output(); readonly pageChange = output(); readonly pageSizeChange = output(); From bc9d045963fea08ede042179c782a0d2c629929f Mon Sep 17 00:00:00 2001 From: Akash Goswami Date: Fri, 24 Jul 2026 15:24:41 +0530 Subject: [PATCH 5/5] prettier fix --- .../activity-log/activity-log.component.css | 1 - .../activity-log/activity-log.component.html | 67 ++++++++++--------- .../activity-log/activity-log.component.ts | 5 +- .../data-table/data-table.component.html | 28 +++++--- 4 files changed, 53 insertions(+), 48 deletions(-) diff --git a/docanalytics-web/src/app/features/activity-log/activity-log.component.css b/docanalytics-web/src/app/features/activity-log/activity-log.component.css index 0a66ca6..1b93ff9 100644 --- a/docanalytics-web/src/app/features/activity-log/activity-log.component.css +++ b/docanalytics-web/src/app/features/activity-log/activity-log.component.css @@ -149,4 +149,3 @@ :host ::ng-deep .row-orphaned td { opacity: 0.4; } - diff --git a/docanalytics-web/src/app/features/activity-log/activity-log.component.html b/docanalytics-web/src/app/features/activity-log/activity-log.component.html index 58ce649..ecef0e6 100644 --- a/docanalytics-web/src/app/features/activity-log/activity-log.component.html +++ b/docanalytics-web/src/app/features/activity-log/activity-log.component.html @@ -20,22 +20,24 @@

Activity Log

(changed)="onFilters($event)" /> - + {{ row.ts | date: 'medium' }} @@ -45,25 +47,26 @@

Activity Log

{{ eventLabel(row.event_type) }}
@if (row.event_type === 'BATCH_DELETED') { - 🗑️ Deleted + 🗑️ Deleted }
- @if (isNavigable(row)) { - + } @else { - {{ row.entity }} + {{ row.entity }} } - + {{ row.entity_type }} @@ -71,15 +74,15 @@

Activity Log

@if (row.old_state && row.new_state) { - - - - - + + + + + } @else if (row.new_state) { - + } @else { - + } diff --git a/docanalytics-web/src/app/features/activity-log/activity-log.component.ts b/docanalytics-web/src/app/features/activity-log/activity-log.component.ts index be3474c..e5e35d6 100644 --- a/docanalytics-web/src/app/features/activity-log/activity-log.component.ts +++ b/docanalytics-web/src/app/features/activity-log/activity-log.component.ts @@ -66,10 +66,7 @@ export class ActivityLogComponent { } protected readonly rowClass = (row: ActivityLogItem): string => - row.entity_type === 'File' && !row.batch_id - ? 'row-orphaned' - : ''; - + row.entity_type === 'File' && !row.batch_id ? 'row-orphaned' : ''; protected readonly eventTypeOptions: FilterOption[] = [ { value: 'all', label: 'All events' }, diff --git a/docanalytics-web/src/app/shared/components/data-table/data-table.component.html b/docanalytics-web/src/app/shared/components/data-table/data-table.component.html index ff8f1e7..8cf6dd4 100644 --- a/docanalytics-web/src/app/shared/components/data-table/data-table.component.html +++ b/docanalytics-web/src/app/shared/components/data-table/data-table.component.html @@ -40,18 +40,24 @@ } @else { @for (row of rows(); track key(row, $index)) { - - @for (col of columns(); track col.key) { - - @if (cellTemplates().get(col.key); as tpl) { - - } @else { - {{ display(row, col) }} + + @for (col of columns(); track col.key) { + + @if (cellTemplates().get(col.key); as tpl) { + + } @else { + {{ display(row, col) }} + } + } - - } - + } }