Merge pull request #688 from PiwiTests/claude/flake-lab-verified

feat(app): mark verified flake fixes on the flaky list and propose quarantine release

Author: PhenX

apps/application/app/components/project/FlakyTestsList.vue

@@ -1,5 +1,5 @@
 <script setup lang="ts">
-import type { FlakyTest } from '~~/types/api';
+import type { FlakyTest, VerifiedFixedFlakyTest } from '~~/types/api';
 import type { TopFlakeSuspect } from '#shared/handlers/flake-profile';
 import type { FlakeLabSummary } from '#shared/handlers/flake-lab';
 import { buildTestRowBadges } from '~/utils/test-row-badges';
@@ -21,7 +21,7 @@ const quarantiningId = ref<number | null>(null);
 const runsWindow = ref(50);
 const rootCauseFilter = ref<string[]>([]);
 
-const { data: tests, status } = await useFetch(
+const { data: flaky, status } = await useFetch(
   () => {
     const params = new URLSearchParams({ runs: String(runsWindow.value) });
     if (props.environment) params.set('environment', props.environment);
@@ -32,9 +32,12 @@ const { data: tests, status } = await useFetch(
     lazy: true,
     server: false,
     watch: [runsWindow, () => props.environment, () => props.branch],
-    transform: (r: { items: FlakyTest[] }) => r.items,
+    transform: (r: { items: FlakyTest[]; verifiedFixed?: VerifiedFixedFlakyTest[] }) => r,
   },
 );
+const tests = computed(() => flaky.value?.items ?? null);
+/** Tests a verified fix took off the ranking; they return to it at their next retry-pass. */
+const verifiedFixed = computed(() => flaky.value?.verifiedFixed ?? []);
 // The server renders before the client-only fetch starts (`idle`), so idle reads as loading too.
 const loading = computed(() => status.value === 'idle' || status.value === 'pending');
 
@@ -264,5 +267,41 @@ function flakyBadges(test: FlakyTest) {
     <p v-if="!loading && filteredTests.length === 0" class="text-sm text-gray-500 py-4 text-center">
       No flaky tests detected in the last {{ runsWindow }} runs.
     </p>
+
+    <div v-if="verifiedFixed.length" class="mt-4 space-y-1.5" data-testid="flaky-verified-fixed">
+      <p class="text-xs text-muted inline-flex items-center gap-1">
+        <UIcon name="i-lucide-badge-check" class="size-3.5 text-success" />
+        Verified fixed: off the ranking until they retry-pass again
+      </p>
+      <ul class="rounded-lg border border-default divide-y divide-default">
+        <li
+          v-for="test in verifiedFixed"
+          :key="test.testCaseId"
+          class="flex flex-wrap items-center gap-x-3 gap-y-1 px-3 py-2"
+          data-testid="flaky-verified-row"
+        >
+          <NuxtLink
+            :to="`/test-cases/${test.testCaseId}?tab=flakiness`"
+            class="min-w-0 text-sm font-medium text-highlighted hover:underline break-words"
+          >
+            {{ test.title }}
+          </NuxtLink>
+          <span class="min-w-0 text-xs text-muted font-mono truncate">{{ test.filePath }}</span>
+          <span class="flex-1" />
+          <UBadge
+            color="success"
+            variant="soft"
+            size="xs"
+            :title="`Flake Lab verified the fix ${formatRelativeTime(test.verifiedFix.verifiedAt)}`"
+            data-testid="flaky-verified-badge"
+          >
+            Verified fixed{{ test.verifiedFix.commit ? ` on ${test.verifiedFix.commit.slice(0, 7)}` : '' }}
+          </UBadge>
+          <span class="text-xs text-muted tabular-nums">
+            {{ test.retryPassRuns }} retry pass{{ test.retryPassRuns === 1 ? '' : 'es' }} before it
+          </span>
+        </li>
+      </ul>
+    </div>
   </UCard>
 </template>

apps/application/app/components/project/QuarantineTable.vue

@@ -4,7 +4,8 @@
  *
  * The streak column is the point of the view: a quarantined test still runs, so
  * it can earn its way out, and the table says when it has rather than waiting
- * to be asked.
+ * to be asked: after a streak of passes, or at once when a Flake Lab verify
+ * experiment proved the fix after the quarantine.
  */
 import { buildTestRowBadges } from '~/utils/test-row-badges';
 
@@ -29,6 +30,8 @@ interface QuarantineEntry {
   ageMs: number;
   consecutivePasses: number;
   releaseProposed: boolean;
+  releaseReason: 'streak' | 'verified-fix' | null;
+  verifiedFix: { commit: string | null; verifiedAt: string } | null;
   runsSinceQuarantine: number;
 }
 
@@ -138,7 +141,18 @@ async function release(testCaseId: number) {
             :project-name="projectName"
           >
             <template #metrics>
-              <UBadge v-if="entry.releaseProposed" color="success" variant="soft" size="xs">
+              <UBadge
+                v-if="entry.releaseReason === 'verified-fix'"
+                color="success"
+                variant="soft"
+                size="xs"
+                :title="`Flake Lab verified the fix ${formatRelativeTime(entry.verifiedFix!.verifiedAt)}`"
+                data-testid="quarantine-verified-fix"
+              >
+                Verified fixed{{ entry.verifiedFix?.commit ? ` on ${entry.verifiedFix.commit.slice(0, 7)}` : '' }} —
+                ready
+              </UBadge>
+              <UBadge v-else-if="entry.releaseProposed" color="success" variant="soft" size="xs">
                 {{ entry.consecutivePasses }} green — ready
               </UBadge>
               <span v-else-if="entry.runsSinceQuarantine === 0" class="text-xs">not run yet</span>

apps/application/app/components/test-case/FlakinessTab.vue

@@ -13,22 +13,26 @@
  */
 import type { FlakeProfile, FlakeSuspect } from '#shared/handlers/flake-profile';
 import { latestSuspectResults, type FlakeExperimentRecord, type FlakeSuspectResult } from '#shared/flake-lab';
+import type { VerifiedFix } from '#shared/handlers/flake-verified';
 
 const props = defineProps<{ testCaseId: number; projectId?: number | null; projectLabel?: string | null }>();
 
 const route = useRoute();
 const profile = ref<FlakeProfile | null>(null);
 const experiments = ref<FlakeExperimentRecord[]>([]);
+/** The verify experiment that marks the test verified fixed, holding or not. */
+const verifiedFix = ref<VerifiedFix | null>(null);
 const loading = ref(true);
 const failed = ref(false);
 
 /** Re-read the experiments, when a lab session run from the desktop app ends. */
 async function reloadExperiments() {
   try {
-    const res = await $fetch<{ items: FlakeExperimentRecord[] }>(
+    const res = await $fetch<{ items: FlakeExperimentRecord[]; verifiedFix?: VerifiedFix | null }>(
       `/api/test-cases/${props.testCaseId}/flake-experiments`,
     );
     experiments.value = res.items;
+    verifiedFix.value = res.verifiedFix ?? null;
   } catch {
     // The list stays as it was; the next visit reads it again.
   }
@@ -41,11 +45,14 @@ watch(
     failed.value = false;
     const [p, e] = await Promise.allSettled([
       $fetch<FlakeProfile>(`/api/test-cases/${id}/flake-profile`),
-      $fetch<{ items: FlakeExperimentRecord[] }>(`/api/test-cases/${id}/flake-experiments`),
+      $fetch<{ items: FlakeExperimentRecord[]; verifiedFix?: VerifiedFix | null }>(
+        `/api/test-cases/${id}/flake-experiments`,
+      ),
     ]);
     if (p.status === 'fulfilled') profile.value = p.value;
     else failed.value = true;
     experiments.value = e.status === 'fulfilled' ? e.value.items : [];
+    verifiedFix.value = e.status === 'fulfilled' ? (e.value.verifiedFix ?? null) : null;
     loading.value = false;
   },
   { immediate: true },
@@ -232,6 +239,37 @@ function detailLine(s: FlakeSuspect): string | null {
       data-shot="flake-experiments"
     >
       <div class="space-y-3" data-testid="flake-experiments">
+        <p
+          v-if="verifiedFix"
+          class="flex flex-wrap items-center gap-x-2 gap-y-1 text-sm"
+          data-testid="flake-verified-fix"
+          :data-holding="verifiedFix.flakedAgainAt ? 'false' : 'true'"
+        >
+          <UIcon
+            :name="verifiedFix.flakedAgainAt ? 'i-lucide-rotate-ccw' : 'i-lucide-badge-check'"
+            :class="verifiedFix.flakedAgainAt ? 'text-warning' : 'text-success'"
+            class="size-4 shrink-0"
+          />
+          <template v-if="!verifiedFix.flakedAgainAt">
+            <span class="font-medium text-highlighted"
+              >Verified fixed<template v-if="verifiedFix.commit">
+                on <span class="font-mono">{{ verifiedFix.commit.slice(0, 7) }}</span></template
+              ></span
+            >
+            <span class="text-muted"
+              >· {{ formatRelativeTime(verifiedFix.verifiedAt) }} · off the flaky ranking until it retry-passes
+              again</span
+            >
+          </template>
+          <template v-else>
+            <span class="text-highlighted"
+              >Verified fixed<template v-if="verifiedFix.commit">
+                on <span class="font-mono">{{ verifiedFix.commit.slice(0, 7) }}</span></template
+              >, then it retry-passed again {{ formatRelativeTime(verifiedFix.flakedAgainAt) }}</span
+            >
+            <span class="text-muted">· back on the flaky ranking</span>
+          </template>
+        </p>
         <p v-if="!loading && experiments.length === 0" class="text-sm text-highlighted leading-relaxed">
           None yet. Run the lab from the project root: it applies each suspect’s condition next to a control, with
           retries off, until the failure reproduces.

apps/application/app/demo/api/router.ts

@@ -101,7 +101,7 @@ import {
   createProject,
   getProjectMenu,
   deleteProjectData,
-  getProjectFlakyTests,
+  getProjectFlakyTestsWithVerified,
   getProjectsOverview,
   getProjectSpecHealth,
 } from '#shared/handlers/projects';
@@ -191,6 +191,7 @@ import {
   type FlakeResultsInput,
 } from '#shared/handlers/flake-lab';
 import { buildExecutionReproduce } from '#shared/handlers/reproduce';
+import { getVerifiedFixes } from '#shared/handlers/flake-verified';
 import {
   getFailureCluster,
   getOpenFailureClusters,
@@ -742,16 +743,14 @@ const routes: RouteEntry[] = [
         : undefined;
       // CODEOWNERS resolution needs an SCM client the browser cannot reach —
       // ownership stays annotation-only here (seeded cases carry `piwi:` owners).
-      return {
-        items: await getProjectFlakyTests(
-          await getDemoDb(),
-          +m[1]!,
-          runs,
-          environment,
-          { tags, owner, priority },
-          branch,
-        ),
-      };
+      return getProjectFlakyTestsWithVerified(
+        await getDemoDb(),
+        +m[1]!,
+        runs,
+        environment,
+        { tags, owner, priority },
+        branch,
+      );
     },
   },
   {
@@ -1275,7 +1274,11 @@ const routes: RouteEntry[] = [
     handler: async (m, _b, q, ctx) => {
       await assertDemoEntityScope(ctx, 'case', +m[1]!);
       const limit = Number(q?.get('limit')) || undefined;
-      return { items: await listFlakeExperiments(await getDemoDb(), +m[1]!, { limit }) };
+      const db = await getDemoDb();
+      return {
+        items: await listFlakeExperiments(db, +m[1]!, { limit }),
+        verifiedFix: (await getVerifiedFixes(db, [+m[1]!])).get(+m[1]!) ?? null,
+      };
     },
   },
   {

apps/application/server/api/projects/[id]/flaky-tests.get.ts

@@ -1,7 +1,7 @@
 import { requireProjectAccess, requireRouteId } from '../../../utils/project-access';
 import { optionalIntQuery } from '../../../utils/query-params';
 import { getDatabase } from '../../../database';
-import { getProjectFlakyTests } from '#shared/handlers/projects';
+import { getProjectFlakyTestsWithVerified } from '#shared/handlers/projects';
 import { parseTagFilter } from '#shared/utils/tag-filter';
 import { withResolvedOwners } from '../../../utils/scm/ownership';
 import { TEST_PRIORITIES } from '@piwitests/core/test-meta';
@@ -11,7 +11,7 @@ defineRouteMeta({
     tags: ['Analytics'],
     summary: 'Flaky test analysis',
     description:
-      'Analyzes test flakiness across recent runs using retry-pass detection and pass/fail alternation scoring. Pass an environment and/or branch to scope the analysis to runs from that deployment environment or SCM branch.',
+      'Analyzes test flakiness across recent runs using retry-pass detection and pass/fail alternation scoring. Pass an environment and/or branch to scope the analysis to runs from that deployment environment or SCM branch. A test whose Flake Lab verify experiment held, and that has not retry-passed in a run started since, leaves `items` and is listed in `verifiedFixed` (`{ testCaseId, title, filePath, retryPassRuns, lastFlakeAt, verifiedFix }`) until it retry-passes again.',
     parameters: [
       { name: 'id', in: 'path', required: true, schema: { type: 'integer' } },
       { name: 'runs', in: 'query', required: false, schema: { type: 'integer' } },
@@ -63,10 +63,17 @@ export default eventHandler(async (event) => {
   const db = await getDatabase();
 
   try {
-    const rows = await getProjectFlakyTests(db, projectId, runsLimit, environment, filter, branch);
+    const { items, verifiedFixed } = await getProjectFlakyTestsWithVerified(
+      db,
+      projectId,
+      runsLimit,
+      environment,
+      filter,
+      branch,
+    );
     // Fill in the owner from CODEOWNERS for tests that declare none, so the
     // leaderboard can be read per team without anyone annotating a test.
-    return { items: await withResolvedOwners(db, projectId, rows) };
+    return { items: await withResolvedOwners(db, projectId, items), verifiedFixed };
   } catch (e: any) {
     if (e?.message === 'Project not found') {
       throw apiError({ statusCode: 404, message: 'Project not found' });

apps/application/server/api/test-cases/[id]/flake-experiments.get.ts

@@ -1,11 +1,12 @@
 import { FLAKE_EXPERIMENTS_MAX, listFlakeExperiments } from '#shared/handlers/flake-lab';
+import { getVerifiedFixes } from '#shared/handlers/flake-verified';
 import { requireResolvedProjectAccess, requireRouteId, resolveCaseProjectId } from '../../../utils/project-access';
 
 defineRouteMeta({
   openAPI: {
     tags: ['Test Cases'],
     summary: 'Flake-lab experiments of a test case',
-    description: `The test’s finished \`piwi flake\` and \`piwi flake verify\` experiments, newest first (at most ${FLAKE_EXPERIMENTS_MAX}): kind, verdict (reproduce: \`reproduced\`, \`amplified\`, \`not-reproduced\`; verify: \`verified\`, \`still-fails\`, \`inconclusive\`), the commit it ran and the commit of the failures, source, machine, Playwright project, and each arm with its conditions, runs, matching and other failures, discarded rounds, whether it stopped early, and the p-value and verdict the server computed against the control.`,
+    description: `The test’s finished \`piwi flake\` and \`piwi flake verify\` experiments, newest first (at most ${FLAKE_EXPERIMENTS_MAX}): kind, verdict (reproduce: \`reproduced\`, \`amplified\`, \`not-reproduced\`; verify: \`verified\`, \`still-fails\`, \`inconclusive\`), the commit it ran and the commit of the failures, source, machine, Playwright project, and each arm with its conditions, runs, matching and other failures, discarded rounds, whether it stopped early, and the p-value and verdict the server computed against the control. \`verifiedFix\` is the verify experiment that marks the test verified fixed (\`{ testCaseId, experimentId, commit, verifiedAt, flakedAgainAt }\`, \`flakedAgainAt\` set once it retry-passed in a run started after \`verifiedAt\`), or null.`,
     'x-required-roles': ['administrator', 'reporter', 'user'],
     parameters: [
       { name: 'id', in: 'path', required: true, schema: { type: 'integer' } },
@@ -18,5 +19,8 @@ export default eventHandler(async (event) => {
   const testCaseId = requireRouteId(event, 'id', 'test case ID');
   const { db } = await requireResolvedProjectAccess(event, testCaseId, resolveCaseProjectId, 'Test case');
   const limit = Number(getQuery(event).limit) || undefined;
-  return { items: await listFlakeExperiments(db, testCaseId, { limit }) };
+  return {
+    items: await listFlakeExperiments(db, testCaseId, { limit }),
+    verifiedFix: (await getVerifiedFixes(db, [testCaseId])).get(testCaseId) ?? null,
+  };
 });