Skip to content

Commit 01b6de8

Browse files
authored
Merge pull request #458 from PiwiTests/claude/audit-attempt-diffing
feat(app): diff a flaky test's failing and passing attempts
2 parents a12a112 + 98afdb3 commit 01b6de8

15 files changed

Lines changed: 1061 additions & 8 deletions

File tree

Lines changed: 160 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,160 @@
1+
<script setup lang="ts">
2+
/**
3+
* The Attempts tab for a flaky test: a strip of every attempt (retry number,
4+
* status, duration, a "this one" marker on the opened execution), and below it
5+
* "what differed" between the failing attempt and the attempt that passed on
6+
* retry — the flakiness fingerprint. The diff is loaded lazily from
7+
* `/attempt-diff` when the tab is first opened (this card mounts under a `v-if`).
8+
*
9+
* Each difference cites an evidence section; the chip switches to that evidence
10+
* tab through the page's section locator — the same mechanism a clue uses.
11+
*/
12+
import type { AttemptDiffEntry } from '#shared/attempt-diff';
13+
import type { AttemptDiffResult } from '#shared/handlers/test-cases';
14+
import { useClusterSectionLocator } from '~/composables/useClusterSectionLocator';
15+
16+
const props = defineProps<{
17+
testRunsCaseId: number;
18+
/** Every attempt of this execution, already fetched at page level. */
19+
attempts: Array<{ retry: number; status: string; duration: number | null; executionId: number | null }>;
20+
}>();
21+
22+
const { data, status } = await useFetch<AttemptDiffResult>(`/api/test-run-cases/${props.testRunsCaseId}/attempt-diff`);
23+
24+
const locator = useClusterSectionLocator();
25+
26+
// The endpoint's attempt list is authoritative (it unions every attempt row);
27+
// the page-level `attempts` prop is the fallback while the diff is loading.
28+
const orderedAttempts = computed(() => {
29+
const source = data.value?.attempts?.length ? data.value.attempts : (props.attempts ?? []);
30+
return [...source].sort((a, b) => (a.retry ?? 0) - (b.retry ?? 0));
31+
});
32+
33+
const differences = computed<AttemptDiffEntry[]>(() => data.value?.differences ?? []);
34+
const applicable = computed(() => data.value?.applicable === true);
35+
36+
// ── Per-kind presentation ──────────────────────────────────────────────────
37+
const KIND_ICON: Record<AttemptDiffEntry['kind'], string> = {
38+
error: 'i-lucide-circle-x',
39+
network: 'i-lucide-arrow-left-right',
40+
console: 'i-lucide-terminal',
41+
step: 'i-lucide-list-checks',
42+
duration: 'i-lucide-timer',
43+
'page-state': 'i-lucide-database',
44+
aria: 'i-lucide-scan-text',
45+
};
46+
47+
/** Where a difference's citation jumps — evidence section id → readable tab name. */
48+
const SECTION_LABEL: Record<string, string> = {
49+
executionError: 'Error',
50+
networkRequests: 'Network',
51+
console: 'Console',
52+
steps: 'Timeline',
53+
appState: 'State',
54+
ariaSnapshot: 'Screen',
55+
};
56+
57+
function onlyLabel(entry: AttemptDiffEntry): { text: string; class: string } {
58+
if (entry.only === 'failing') {
59+
return { text: 'only on the failing attempt', class: 'text-red-600 dark:text-red-400 bg-red-500/10' };
60+
}
61+
if (entry.only === 'passing') {
62+
return { text: 'only on the passing attempt', class: 'text-green-600 dark:text-green-400 bg-green-500/10' };
63+
}
64+
return { text: 'changed', class: 'text-amber-600 dark:text-amber-400 bg-amber-500/10' };
65+
}
66+
67+
function citationLabel(entry: AttemptDiffEntry): string | null {
68+
const section = entry.ref?.section;
69+
if (!section || !locator.canLocate(section)) return null;
70+
return SECTION_LABEL[section] ?? null;
71+
}
72+
73+
function reveal(entry: AttemptDiffEntry) {
74+
const section = entry.ref?.section;
75+
if (section && locator.canLocate(section)) locator.open(section);
76+
}
77+
78+
function attemptLabel(retry: number): string {
79+
return retry === 0 ? 'Attempt 1' : `Retry ${retry}`;
80+
}
81+
</script>
82+
83+
<template>
84+
<div class="space-y-4" data-shot="attempts-diff">
85+
<!-- ── Attempt strip ──────────────────────────────────────────────────── -->
86+
<SectionCard embedded icon="i-lucide-repeat" title="Attempts" help="case.attempts">
87+
<ul class="flex flex-col sm:flex-row sm:flex-wrap gap-2">
88+
<li
89+
v-for="attempt in orderedAttempts"
90+
:key="attempt.retry"
91+
class="flex items-center gap-2 rounded-md border border-default px-2.5 py-1.5 text-sm"
92+
:class="attempt.executionId === testRunsCaseId ? 'bg-primary/5 border-primary/40' : ''"
93+
>
94+
<span class="font-medium whitespace-nowrap">{{ attemptLabel(attempt.retry) }}</span>
95+
<StatusChip :status="attempt.status" size="xs" />
96+
<DurationValue :ms="attempt.duration" class="text-muted tabular-nums" />
97+
<span v-if="attempt.executionId === testRunsCaseId" class="text-xs text-primary font-medium whitespace-nowrap"
98+
>this one</span
99+
>
100+
<ULink
101+
v-else-if="attempt.executionId"
102+
:to="`/test-run-cases/${attempt.executionId}`"
103+
class="text-xs text-primary hover:underline whitespace-nowrap"
104+
>open</ULink
105+
>
106+
</li>
107+
</ul>
108+
</SectionCard>
109+
110+
<!-- ── What differed ──────────────────────────────────────────────────── -->
111+
<SectionCard embedded icon="i-lucide-git-compare" title="What differed" help="case.attempts">
112+
<LoadingState v-if="status === 'pending'" text="Comparing attempts…" />
113+
114+
<EmptyState
115+
v-else-if="!applicable"
116+
icon="i-lucide-repeat"
117+
text="No failing-and-passing pair to compare — this needs one attempt that failed and one that passed."
118+
/>
119+
120+
<EmptyState
121+
v-else-if="differences.length === 0"
122+
icon="i-lucide-equal"
123+
text="The failing and passing attempts left no different evidence — the flakiness is not visible in the captured signals."
124+
/>
125+
126+
<ul v-else class="space-y-2.5">
127+
<li
128+
v-for="(entry, i) in differences"
129+
:key="i"
130+
class="flex items-start gap-2.5 rounded-md border border-default p-2.5"
131+
>
132+
<UIcon :name="KIND_ICON[entry.kind]" class="size-4 shrink-0 mt-0.5 text-muted" />
133+
<div class="min-w-0 flex-1 space-y-1">
134+
<div class="flex flex-wrap items-center gap-2">
135+
<span
136+
class="rounded px-1.5 py-0.5 text-xs font-medium whitespace-nowrap"
137+
:class="onlyLabel(entry).class"
138+
>{{ onlyLabel(entry).text }}</span
139+
>
140+
<span class="text-sm font-medium break-words">{{ entry.summary }}</span>
141+
</div>
142+
<pre
143+
v-if="entry.detail"
144+
class="text-xs text-muted font-mono whitespace-pre-wrap break-words max-h-32 overflow-y-auto"
145+
>{{ entry.detail }}</pre>
146+
<button
147+
v-if="citationLabel(entry)"
148+
type="button"
149+
class="inline-flex items-center gap-1 text-xs text-primary hover:underline"
150+
@click="reveal(entry)"
151+
>
152+
<UIcon name="i-lucide-arrow-up-right" class="size-3" />
153+
View in {{ citationLabel(entry) }}
154+
</button>
155+
</div>
156+
</li>
157+
</ul>
158+
</SectionCard>
159+
</div>
160+
</template>

apps/application/app/components/test-case/EvidenceTabs.vue

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,12 @@ const stateHasData = computed(() => Boolean(pageState.value));
104104
const performanceHasData = computed(() => Boolean(webVitals.value) || performanceHints.value.length > 0);
105105
const timelineHasData = computed(() => steps.value.length > 0);
106106
107+
// Every attempt of this execution (each retry is its own row), already fetched.
108+
const attemptsList = computed<
109+
Array<{ retry: number; status: string; duration: number | null; executionId: number | null }>
110+
>(() => props.testCase?.attempts ?? []);
111+
const hasMultipleAttempts = computed(() => attemptsList.value.length > 1);
112+
107113
interface TabDef {
108114
value: TabValue;
109115
label: string;
@@ -113,6 +119,13 @@ interface TabDef {
113119
}
114120
const tabs = computed<TabDef[]>(() => [
115121
{ value: 'timeline', label: 'Timeline', icon: 'i-lucide-activity', hasData: timelineHasData.value, count: null },
122+
{
123+
value: 'attempts',
124+
label: 'Attempts',
125+
icon: 'i-lucide-repeat',
126+
hasData: hasMultipleAttempts.value,
127+
count: hasMultipleAttempts.value ? attemptsList.value.length : null,
128+
},
116129
{ value: 'screen', label: 'Screen', icon: 'i-lucide-camera', hasData: screenHasData.value, count: null },
117130
{ value: 'source', label: 'Source', icon: 'i-lucide-file-code-2', hasData: sourceHasData.value, count: null },
118131
{
@@ -282,6 +295,12 @@ defineExpose({ canLocate, revealSection, selectTab: (t: TabValue) => (activeTab.
282295
</SectionCard>
283296
</div>
284297

298+
<!-- ── Attempts ─────────────────────────────────────────────── -->
299+
<!-- Lazy: this card mounts only when the tab opens, fetching the diff then. -->
300+
<div v-else-if="activeTab === 'attempts'" class="scroll-mt-4">
301+
<AttemptsCard :test-runs-case-id="testRunsCaseId" :attempts="attemptsList" />
302+
</div>
303+
285304
<!-- ── Screen ───────────────────────────────────────────────── -->
286305
<div v-else-if="activeTab === 'screen'" class="space-y-4">
287306
<div ref="screenEvidenceWrap" class="scroll-mt-4 space-y-4">

apps/application/app/demo/api/router.ts

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,7 @@ import {
9797
getTestCaseStabilityTrend,
9898
getFailureTimeline,
9999
getFailureClues,
100+
getAttemptDiff,
100101
} from '#shared/handlers/test-cases';
101102
import {
102103
getFailureCluster,
@@ -968,6 +969,14 @@ const routes: RouteEntry[] = [
968969
return getFailureClues(await getDemoDb(), +m[1]!);
969970
},
970971
},
972+
{
973+
method: 'GET',
974+
pattern: /^\/api\/test-run-cases\/(\d+)\/attempt-diff$/,
975+
handler: async (m, _b, _q, ctx) => {
976+
await assertDemoEntityScope(ctx, 'execution', +m[1]!);
977+
return getAttemptDiff(await getDemoDb(), +m[1]!);
978+
},
979+
},
971980
{
972981
method: 'GET',
973982
pattern: /^\/api\/test-run-cases\/(\d+)\/dom-snapshot$/,

apps/application/app/utils/evidence-sections.ts

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,15 @@
44
* time, whether a citation is locatable). Kept out of the component so the
55
* page's section locator gives the same answer during SSR and on the client.
66
*/
7-
export type EvidenceTabValue = 'timeline' | 'screen' | 'source' | 'network' | 'console' | 'state' | 'performance';
7+
export type EvidenceTabValue =
8+
| 'timeline'
9+
| 'attempts'
10+
| 'screen'
11+
| 'source'
12+
| 'network'
13+
| 'console'
14+
| 'state'
15+
| 'performance';
816

917
export const EVIDENCE_SECTION_TAB: Record<string, EvidenceTabValue> = {
1018
steps: 'timeline',

apps/application/app/utils/help-content.ts

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -331,6 +331,11 @@ export const HELP_TOPICS = {
331331
text: 'A snapshot of the accessibility tree at the moment of failure — what assistive tech saw, and useful grounding for AI diagnosis. An empty card says whether it was not captured (add the capture fixtures) or captured with nothing to snapshot; with a trace and no fixtures it is recovered from the trace\'s error context and marked "derived from the trace".',
332332
doc: 'ai-diagnosis#what-a-diagnosis-contains',
333333
},
334+
'case.attempts': {
335+
title: 'Attempts',
336+
text: 'When a test failed then passed on retry, this compares the failing attempt against the passing one and lists what differed — the error that was there then gone, a request that failed on only one attempt, a console error, a slower step, a duration or page-state change. Each difference links to the evidence it came from. That delta is the flakiness fingerprint, and it feeds the root-cause classifier.',
337+
doc: 'flaky-tests#flaky-test-detection',
338+
},
334339

335340
// ── Test case across runs ─────────────────────────────────────────────
336341
'case.history-chart': {

apps/application/scripts/take-feature-screenshots.mjs

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -362,6 +362,19 @@ const SCENES = [
362362
},
363363

364364
// ── Feature states (report artifacts) ─────────────────────────────────────
365+
{
366+
name: 'attempt-diff',
367+
description: 'Attempts tab: every attempt, and what differed between the failing and passing attempt',
368+
// Execution 21 is a flaky test that passed on retry, so the Attempts tab holds a diff.
369+
route: '/test-run-cases/21',
370+
viewport: { width: 1280, height: 1000 },
371+
of: '[data-shot="attempts-diff"]',
372+
pad: 12,
373+
async run({ shoot, openTab }) {
374+
await openTab('Attempts');
375+
await shoot();
376+
},
377+
},
365378
{
366379
name: 'execution-history',
367380
description: 'Execution page opened straight onto its History tab (duration trend + executions)',
Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
import { getAttemptDiff } from '#shared/handlers/test-cases';
2+
import {
3+
requireResolvedProjectAccess,
4+
requireRouteId,
5+
resolveTestRunCaseProjectId,
6+
} from '../../../utils/project-access';
7+
8+
defineRouteMeta({
9+
openAPI: {
10+
tags: ['Test Run Cases'],
11+
summary: "Diff a flaky test's failing and passing attempts",
12+
description:
13+
'Compares the failing and passing attempts of one flaky execution — the failing attempt this id belongs to against the first later attempt that passed, or, when this id is the passing attempt, the last prior failing one. Returns an ordered list of what differed (the error present on the failing attempt and gone on the pass, a request that failed on only one attempt, a console error/warning on only one, a step that errored or slowed, a duration delta, a page-state/URL change, an ARIA structural change), most-diagnostic first, plus a compact summary of each compared attempt. `applicable` is false when the execution has only one attempt or no failing/passing pair exists.',
14+
parameters: [
15+
{ name: 'id', in: 'path', required: true, schema: { type: 'integer' }, description: 'Test run case id' },
16+
],
17+
'x-required-roles': ['administrator', 'reporter', 'user'],
18+
},
19+
});
20+
21+
export default eventHandler(async (event) => {
22+
const id = requireRouteId(event, 'id', 'test run case ID');
23+
24+
// Authorize by the execution's own project: this id may be opened from the
25+
// cluster page, where it can belong to a run in another project.
26+
const { db } = await requireResolvedProjectAccess(event, id, resolveTestRunCaseProjectId, 'Test run case');
27+
28+
// Don't 404 — "not applicable" (one attempt, or no pair) is a valid answer.
29+
return getAttemptDiff(db, id);
30+
});

0 commit comments

Comments
 (0)