Replace index based lookups with text

Author: kyle-ssg

.claude/commands/e2e-create.md

@@ -6,12 +6,20 @@ Create a new E2E test following the existing patterns in the codebase.
 
 - Test files location: `frontend/e2e/tests/*.pw.ts`
 - Test helpers: `frontend/e2e/helpers.playwright.ts`
+- Test results: Individual test directories in `frontend/e2e/test-results/`
+- **For EACH failed test**, a directory is created with these files:
+  - `failed.json` - Summary of failures with error messages and stack traces
+  - `error-context.md` - **MOST VALUABLE** - Page snapshot showing DOM state, field values, button states
+  - `trace.zip` - Detailed execution trace (unzip to get action logs)
+  - Screenshots (`.png`) and videos (`.webm`)
 - Existing tests to reference for patterns
 - Tests use Playwright with Firefox
 - Tests are tagged with `@oss` or `@enterprise`
 
 ## Workflow
 
+**IMPORTANT: Always start by changing to the frontend directory** - the `.env` file and dependencies are located there.
+
 1. **Ask the user what they want to test:**
    - What feature/page/flow to test
    - Any specific scenarios or edge cases
@@ -47,10 +55,36 @@ Create a new E2E test following the existing patterns in the codebase.
    - Include proper test descriptions and tags
    - Add comments explaining complex test logic
 
-6. **Report what was created:**
+6. **Run the new test to verify it works:**
+   - Run the test 2-3 times to ensure it's stable:
+     ```bash
+     cd frontend
+     SKIP_BUNDLE=1 E2E_CONCURRENCY=1 npm run test -- tests/new-test.pw.ts --quiet
+     ```
+   - **If the test fails, ALWAYS READ TRACES FIRST:**
+     1. **Read error-context.md** in the failed test directory:
+        - Shows exact DOM state when test failed
+        - YAML tree with all elements, their states, and data-test attributes
+        - Check if expected elements exist and their actual values
+        - Verify selector is correct by searching for the data-test attribute
+     2. **If error-context.md doesn't show the issue**, unzip and check trace files:
+        ```bash
+        cd frontend/e2e/test-results/<failed-test-directory>
+        unzip -q trace.zip
+        grep -i "error\|failed" 0-trace.network  # Check for network errors
+        ```
+     3. **Only after analyzing traces**, fix the issue:
+        - Wrong selector → Update to match actual DOM from error-context.md
+        - Missing `data-test` attribute → Add it to the component
+        - Element hidden → Filter for visible elements or wait for visibility
+        - Missing wait → Add appropriate `waitFor*` calls
+     - Re-run until it passes consistently
+
+7. **Report what was created:**
    - Show the test file path
    - List any `data-test` attributes that were added
-   - Suggest how to run the test
+   - Report test stability (how many runs passed/failed)
+   - If there were failures, explain what was fixed
 
 ## Important Notes
 

.claude/commands/e2e.md

@@ -9,50 +9,108 @@ Run E2E tests with the following configuration:
 ## Context
 
 - Test files location: `frontend/e2e/tests/*.pw.ts`
-- Test results: `frontend/e2e/test-results/results.json`
+- Test results: Individual test directories in `frontend/e2e/test-results/`
+- **For EACH failed test**, a directory is created with these files:
+
+## ALWAYS read these files in this order when debugging failures:
+
+1. **`failed.json`** (in `test-results/` root) - START HERE
+   - Contains only failed tests with error messages, stack traces, file paths, line numbers
+   - Much smaller than results.json - only failures
+
+2. **`error-context.md`** (in each failed test directory) - READ THIS SECOND
+   - **THIS IS THE MOST VALUABLE FILE** - shows exact page state when test failed
+   - YAML tree structure of the entire DOM with element states
+   - Shows which buttons are disabled/enabled
+   - Shows form field values
+   - Shows what's visible on the page
+   - Example: Can see if a button is `[disabled]` or a field is empty
+
+3. **Trace files** (if needed for more detail):
+   - Unzip `trace.zip` to get:
+     - `0-trace.trace` - Every action taken with timestamps and selectors
+     - `0-trace.network` - Network requests
+     - Screenshots (jpegs) showing visual state at each step
+
+4. **Full JSON report**: `results.json` - contains ALL test results (passed + failed), only use if needed
+
+- **HTML report**: `frontend/e2e/playwright-report/` - browsable HTML interface (for humans, not programmatic use)
 - Tests use Playwright with Firefox
 - The retry logic is built into the test runner (via `run-with-retry.ts`)
 - Tests run against a local Docker environment (API + DB)
 
 ## Workflow
 
+**IMPORTANT: Always start by changing to the frontend directory** - the `.env` file and dependencies are located there.
+
 1. **Run tests** from the frontend directory:
    ```bash
    cd frontend
    SKIP_BUNDLE=1 E2E_CONCURRENCY=20 npm run test -- --grep-invert @enterprise --quiet
    ```
 
-2. **If ANY tests fail (even after automatic retries):**
+2. **ALWAYS check for flaky tests after test run completes:**
+
+   **CRITICAL:** Tests that fail initially but pass on retry are FLAKY and MUST be investigated, even if the final result shows all tests passed.
+
+   a. Check the test output for any tests that failed on first run (look for the initial failure messages before "Retrying")
+
+   b. For EACH test that failed initially (even if it passed on retry):
+      - Parse the error from the test output:
+        * Error type (e.g., `TimeoutError`, `AssertionError`)
+        * Error message (e.g., `"waiting for locator('#button') to be visible"`)
+        * File path and line number where it failed
+        * Stack trace showing the call chain
+      - Read the test file at the failing line number to understand what was being tested
+      - **Report these as FLAKY TESTS** - they indicate timing issues, race conditions, or environmental problems
+      - **Analyze the root cause**:
+        * Timeout errors → likely missing waits or race conditions
+        * Assertion errors → check if value is correct or if timing is off
+        * Element not found → selector may have changed or element loads slowly
+
+   c. **If ANY tests are still failing after automatic retries:**
+
+      For EACH failed test, **ALWAYS READ TRACES FIRST:**
+
+      1. **Read error-context.md** in the failed test directory:
+         - Shows exact DOM state when test failed
+         - YAML tree with all elements, their states, and data-test attributes
+         - Check if expected elements exist and their actual values
+         - Verify selector is correct by searching for the data-test attribute
 
-   a. Parse the test output and `e2e/test-results/results.json` to identify failures
+      2. **If error-context.md doesn't show the issue**, unzip and check trace files:
+         ```bash
+         cd frontend/e2e/test-results/<failed-test-directory>
+         unzip -q trace.zip
+         grep -i "error\|failed" 0-trace.network  # Check for network errors
+         ```
 
-   b. For EACH failed test:
-      - Read the test file
-      - Read the error message and stack trace
-      - Analyze the root cause
-      - **Attempt to fix the issue** if it's one of these:
+      3. **Only after analyzing traces**, read the test file and fix:
+        - Wrong selector → Update to match actual DOM from error-context.md
         - Missing `data-test` attribute → Add it to the component
-        - Selector changed → Update the test to use the new selector
+        - Element hidden → Filter for visible elements or wait for visibility
         - Missing wait → Add appropriate `waitFor*` calls
         - Race condition → Add network waits, increase timeouts, or use more specific waits
         - Flaky element interaction → Add `scrollIntoView` or `waitForVisible` before clicking
 
-   c. **After making fixes**, re-run ONLY the failed tests:
+   d. **After making fixes**, re-run ONLY the failed tests:
       ```bash
       cd frontend
       SKIP_BUNDLE=1 E2E_CONCURRENCY=1 npm run test -- tests/flag-tests.pw.ts tests/invite-test.pw.ts
       ```
       - Use concurrency=1 to avoid race conditions
       - Only run the specific test files that failed
 
-   d. **If tests still fail after fixes:**
+   e. **If tests still fail after fixes:**
       - Try a second round of fixes if the error changed
       - Otherwise, report the issue with details on what was attempted
 
 3. **Report final results:**
-   - List which tests passed/failed
+   - **ALWAYS report flaky tests first** (tests that failed initially but passed on retry)
+   - List which tests passed/failed after all retries
    - Document any fixes that were applied
    - For unfixable issues, explain why and suggest manual investigation
+   - Show the error message and line number for any failures
 
 ## Important Notes
 

.claude/commands/optimise-e2e.md

@@ -6,13 +6,22 @@ Clean up E2E tests by removing unnecessary timeouts/waits and documenting necess
 
 - Test files location: `frontend/e2e/tests/*.pw.ts`
 - Test helpers: `frontend/e2e/helpers.playwright.ts`
+- Test results: Individual test directories in `frontend/e2e/test-results/`
+- **For EACH failed test**, a directory is created with these files:
+  - `failed.json` - Summary of failures with error messages and stack traces
+  - `error-context.md` - **MOST VALUABLE** - Page snapshot showing DOM state, field values, button states
+  - `trace.zip` - Detailed execution trace (unzip to get action logs)
+  - Screenshots (`.png`) and videos (`.webm`)
 - Tests should be fast, reliable, and well-documented
 
 ## Workflow
 
+**IMPORTANT: Always start by changing to the frontend directory** - the `.env` file and dependencies are located there.
+
 1. **Scan all test files:**
    ```bash
-   ls frontend/e2e/tests/*.pw.ts
+   cd frontend
+   ls e2e/tests/*.pw.ts
    ```
 
 2. **For each test file, look for:**
@@ -73,9 +82,24 @@ Clean up E2E tests by removing unnecessary timeouts/waits and documenting necess
        SKIP_BUNDLE=1 E2E_CONCURRENCY=1 npm run test -- tests/modified-test.pw.ts --quiet
      done
      ```
-   - If any run fails, investigate whether the optimisation caused it
-   - If failures are due to removed waits, add them back with proper comments
-   - Only keep optimisations that pass all 3 runs
+   - **If any run fails, ALWAYS READ TRACES FIRST:**
+     1. **Read error-context.md** in the failed test directory:
+        - Shows exact DOM state when test failed
+        - YAML tree with all elements, their states, and data-test attributes
+        - Check if expected elements exist and their actual values
+        - Verify selector is correct by searching for the data-test attribute
+     2. **If error-context.md doesn't show the issue**, unzip and check trace files:
+        ```bash
+        cd frontend/e2e/test-results/<failed-test-directory>
+        unzip -q trace.zip
+        grep -i "error\|failed" 0-trace.network  # Check for network errors
+        ```
+     3. **Only after analyzing traces**, investigate whether the optimisation caused it:
+        - Wrong selector → Update to match actual DOM from error-context.md
+        - Missing wait → If optimisation removed a necessary wait, add it back with proper comment explaining why it's needed
+        - Element hidden → Check if removed wait was allowing element to become visible
+        - Race condition → If timeout removal caused timing issue, use event-based wait instead
+   - Only keep optimisations that pass all 3 runs consistently
 
 6. **Report changes:**
    - List files modified and verified

frontend/e2e/global-setup.playwright.ts

@@ -2,10 +2,49 @@ import { FullConfig } from '@playwright/test';
 import fetch from 'node-fetch';
 import flagsmith from 'flagsmith/isomorphic';
 import Project from '../common/project';
+import * as fs from 'fs';
+import * as path from 'path';
 
 async function globalSetup(config: FullConfig) {
   console.log('Starting global setup for E2E tests...');
 
+  // Clear previous test results and reports (skip on retries to preserve failed.json)
+  const testResultsDir = path.join(__dirname, 'test-results');
+  const reportDir = path.join(__dirname, 'playwright-report');
+
+  if (!process.env.E2E_SKIP_CLEANUP) {
+    if (fs.existsSync(testResultsDir)) {
+      // Preserve error-context.md files from previous runs for debugging
+      // Only delete test result directories that don't contain error-context.md
+      const entries = fs.readdirSync(testResultsDir, { withFileTypes: true });
+      for (const entry of entries) {
+        if (entry.isDirectory()) {
+          const errorContextPath = path.join(testResultsDir, entry.name, 'error-context.md');
+          if (!fs.existsSync(errorContextPath)) {
+            // No error context - safe to delete
+            fs.rmSync(path.join(testResultsDir, entry.name), { recursive: true, force: true });
+          }
+        } else {
+          // Delete files in test-results root (like .last-run.json, results.json)
+          fs.unlinkSync(path.join(testResultsDir, entry.name));
+        }
+      }
+      console.log('Cleared previous test results (preserved error contexts)');
+    }
+
+    if (fs.existsSync(reportDir)) {
+      fs.rmSync(reportDir, { recursive: true, force: true });
+      console.log('Cleared previous HTML report');
+    }
+  } else {
+    console.log('Skipping cleanup (retry attempt)');
+  }
+
+  // Ensure test-results directory exists for the JSON reporter
+  if (!fs.existsSync(testResultsDir)) {
+    fs.mkdirSync(testResultsDir, { recursive: true });
+  }
+
   const e2eTestApi = `${process.env.FLAGSMITH_API_URL || Project.api}e2etests/teardown/`;
   const token = process.env.E2E_TEST_TOKEN
       ? process.env.E2E_TEST_TOKEN
@@ -26,33 +65,52 @@ async function globalSetup(config: FullConfig) {
     fetch,
   });
 
-  // Teardown previous test data
+  // Teardown previous test data with retry logic
   if (token) {
-    try {
-      const res = await fetch(e2eTestApi, {
-        body: JSON.stringify({}),
-        headers: {
-          'Accept': 'application/json',
-          'Content-Type': 'application/json',
-          'X-E2E-Test-Auth-Token': token.trim(),
-        },
-        method: 'POST',
-      });
+    const maxAttempts = 3;
+    const delayMs = 2000;
+    let teardownSuccess = false;
+
+    for (let attempt = 0; attempt < maxAttempts; attempt++) {
+      if (attempt > 0) {
+        console.log(`\x1b[33m%s\x1b[0m`, `Retrying teardown (attempt ${attempt + 1}/${maxAttempts})...`);
+        await new Promise(resolve => setTimeout(resolve, delayMs));
+      }
 
-      if (res.ok) {
-        console.log('\n', '\x1b[32m', 'e2e teardown successful', '\x1b[0m', '\n');
-      } else {
-        const errorMsg = `e2e teardown failed with status ${res.status}`;
-        console.error('\n', '\x1b[31m', errorMsg, '\x1b[0m', '\n');
-        if (process.env.E2E_LOCAL !== 'true' && process.env.E2E_DEV !== 'true') {
-          throw new Error(errorMsg); // Fail tests early in CI if teardown fails
+      try {
+        const res = await fetch(e2eTestApi, {
+          body: JSON.stringify({}),
+          headers: {
+            'Accept': 'application/json',
+            'Content-Type': 'application/json',
+            'X-E2E-Test-Auth-Token': token.trim(),
+          },
+          method: 'POST',
+        });
+
+        if (res.ok) {
+          console.log('\n', '\x1b[32m', 'e2e teardown successful', '\x1b[0m', '\n');
+          teardownSuccess = true;
+          break;
+        } else {
+          console.error('\x1b[31m%s\x1b[0m', `✗ E2E teardown failed: ${res.status}`);
+          if (attempt < maxAttempts - 1) {
+            console.log('');
+          }
+        }
+      } catch (error) {
+        console.error('\x1b[31m%s\x1b[0m', `✗ E2E teardown error: ${error.message || String(error)}`);
+        if (attempt < maxAttempts - 1) {
+          console.log('');
         }
       }
-    } catch (error) {
-      const errorMsg = `e2e teardown error: ${error.message || String(error)}`;
+    }
+
+    if (!teardownSuccess) {
+      const errorMsg = `e2e teardown failed after ${maxAttempts} attempts`;
       console.error('\n', '\x1b[31m', errorMsg, '\x1b[0m', '\n');
       if (process.env.E2E_LOCAL !== 'true' && process.env.E2E_DEV !== 'true') {
-        throw error; // Fail tests early in CI if teardown errors
+        throw new Error(errorMsg); // Fail tests early in CI if teardown fails
       }
     }
   } else {

frontend/e2e/global-teardown.playwright.ts

@@ -2,6 +2,7 @@ import { FullConfig } from '@playwright/test';
 import * as fs from 'fs';
 import * as path from 'path';
 import * as archiver from 'archiver';
+import { extractFailedTests } from './extract-failed-tests';
 
 let upload: ((file: string) => Promise<void>) | null = null;
 try {
@@ -30,6 +31,9 @@ async function zipDirectory(sourceDir: string, outPath: string): Promise<void> {
 async function globalTeardown(config: FullConfig) {
   console.log('Running global teardown for E2E tests...');
 
+  // Extract failed tests to a smaller JSON file for easier debugging
+  extractFailedTests(__dirname);
+
   // Upload screenshots/videos if they exist and not in dev mode
   const dir = path.join(__dirname, 'test-results');