Better retrying / add quiet mode and skip bundle

Author: kyle-ssg

frontend/e2e/helpers.playwright.ts

@@ -4,7 +4,7 @@ import fetch from 'node-fetch';
 import flagsmith from 'flagsmith/isomorphic';
 import { IFlagsmith } from 'flagsmith/types';
 
-export const LONG_TIMEOUT = 15000;
+export const LONG_TIMEOUT = 20000;
 
 // Browser debugging - console and network logging
 export const setupBrowserLogging = (page: Page) => {
@@ -97,7 +97,10 @@ export const setupBrowserLogging = (page: Page) => {
           url.includes('/usage-data/') ||
           url.includes('/invite-links/') ||
           url.includes('/roles/') ||
-          url.includes('/change-requests/')) {
+          url.includes('/change-requests/') ||
+          url.includes('/api-keys/') ||
+          url.includes('/metrics/') ||
+          url.includes('/invites/')) {
         return;
       }
     }
@@ -214,11 +217,11 @@ export class E2EHelpers {
     }
   }
 
-  async waitForElementVisible(selector: string) {
+  async waitForElementVisible(selector: string, timeout: number = LONG_TIMEOUT) {
     logUsingLastSection(`Waiting element visible ${selector}`);
     await this.page.locator(selector).first().waitFor({
       state: 'visible',
-      timeout: LONG_TIMEOUT
+      timeout
     });
   }
 
@@ -249,9 +252,45 @@ export class E2EHelpers {
     await expect(this.page.locator(selector)).toHaveCount(0, { timeout: 10000 });
   }
 
+  async waitForNetworkIdle() {
+    await this.page.waitForLoadState('networkidle', { timeout: 10000 }).catch(() => {
+      // Silently continue if timeout - page might already be idle
+    });
+  }
+
+  async waitForDomContentLoaded() {
+    await this.page.waitForLoadState('domcontentloaded', { timeout: 10000 }).catch(() => {
+      // Silently continue if timeout - DOM might already be loaded
+    });
+  }
+
+  async waitForPageFullyLoaded() {
+    await this.waitForDomContentLoaded();
+    await this.waitForNetworkIdle();
+  }
+
+  async getInputValue(selector: string): Promise<string> {
+    const element = this.page.locator(selector).first();
+    await element.waitFor({ state: 'visible', timeout: LONG_TIMEOUT });
+    return await element.inputValue();
+  }
+
+  async scrollBy(x: number, y: number) {
+    await this.page.evaluate(({ x, y }) => {
+      window.scrollBy(x, y);
+    }, { x, y });
+  }
+
+  async scrollToBottom() {
+    await this.page.evaluate(() => {
+      window.scrollTo(0, document.body.scrollHeight);
+    });
+  }
+
   async gotoFeatures() {
     await this.click('#features-link');
     await this.waitForElementVisible('#show-create-feature-btn');
+    await this.waitForPageFullyLoaded();
   }
 
   async click(selector: string) {
@@ -449,11 +488,11 @@ export const setText = async (page: Page, selector: string, text: string) => {
 };
 
 // Wait for an element to be visible
-export const waitForElementVisible = async (page: Page, selector: string) => {
+export const waitForElementVisible = async (page: Page, selector: string, timeout: number = LONG_TIMEOUT) => {
   logUsingLastSection(`Waiting element visible ${selector}`);
   await page.locator(selector).first().waitFor({
     state: 'visible',
-    timeout: LONG_TIMEOUT
+    timeout
   });
 };
 
@@ -488,10 +527,52 @@ export const waitForElementNotExist = async (page: Page, selector: string) => {
   await expect(page.locator(selector)).toHaveCount(0, { timeout: 10000 });
 };
 
+// Wait for network to be idle
+export const waitForNetworkIdle = async (page: Page) => {
+  await page.waitForLoadState('networkidle', { timeout: 10000 }).catch(() => {
+    // Silently continue if timeout - page might already be idle
+  });
+};
+
+// Wait for DOM content to be loaded
+export const waitForDomContentLoaded = async (page: Page) => {
+  await page.waitForLoadState('domcontentloaded', { timeout: 10000 }).catch(() => {
+    // Silently continue if timeout - DOM might already be loaded
+  });
+};
+
+// Wait for page to be fully loaded (DOM + network)
+export const waitForPageFullyLoaded = async (page: Page) => {
+  await waitForDomContentLoaded(page);
+  await waitForNetworkIdle(page);
+};
+
+// Get input value from an element
+export const getInputValue = async (page: Page, selector: string): Promise<string> => {
+  const element = page.locator(selector).first();
+  await element.waitFor({ state: 'visible', timeout: LONG_TIMEOUT });
+  return await element.inputValue();
+};
+
+// Scroll the page by x and y pixels
+export const scrollBy = async (page: Page, x: number, y: number) => {
+  await page.evaluate(({ x, y }) => {
+    window.scrollBy(x, y);
+  }, { x, y });
+};
+
+// Scroll to bottom of page
+export const scrollToBottom = async (page: Page) => {
+  await page.evaluate(() => {
+    window.scrollTo(0, document.body.scrollHeight);
+  });
+};
+
 // Navigate to features page
 export const gotoFeatures = async (page: Page) => {
   await click(page, '#features-link');
   await waitForElementVisible(page, '#show-create-feature-btn');
+  await waitForPageFullyLoaded(page);
 };
 
 // Click an element
@@ -869,9 +950,7 @@ export const createRemoteConfig = async (
     await page.waitForTimeout(100);
   }
   // Wait for form validation to complete after all variations added
-  if (mvs.length > 0) {
-    await page.waitForTimeout(500);
-  }
+  await page.waitForTimeout(500);
   await click(page, byId('create-feature-btn'));
   // Wait for the feature to be created and modal to close
   await page.waitForTimeout(2000); // Increased wait for API to process
@@ -989,6 +1068,7 @@ export const createFeature = async (
   if (value) {
     await click(page, byId('toggle-feature-button'));
   }
+  await page.waitForTimeout(500);
   await click(page, byId('create-feature-btn'));
   // Wait for feature creation to complete
   await page.waitForTimeout(1500);
@@ -1159,4 +1239,4 @@ export const setUserPermission = async (
     await click(page, byId(`permission-switch-${permission}`));
   }
   await closeModal(page);
-};
+};

frontend/e2e/run-with-retry.ts

@@ -0,0 +1,125 @@
+import { execSync } from 'child_process';
+import fetch from 'node-fetch';
+import Project from '../common/project';
+
+require('dotenv').config();
+
+const RETRIES = parseInt(process.env.E2E_RETRIES || '1', 10);
+
+async function runTeardown() {
+  console.log('\n\x1b[36m%s\x1b[0m\n', 'Running E2E teardown...');
+
+  const e2eTestApi = `${process.env.FLAGSMITH_API_URL || Project.api}e2etests/teardown/`;
+  const token = process.env.E2E_TEST_TOKEN
+    ? process.env.E2E_TEST_TOKEN
+    : process.env[`E2E_TEST_TOKEN_${Project.env.toUpperCase()}`];
+
+  if (!token) {
+    console.error('\x1b[31m%s\x1b[0m\n', 'Error: No E2E_TEST_TOKEN found');
+    return false;
+  }
+
+  try {
+    const res = await fetch(e2eTestApi, {
+      body: JSON.stringify({}),
+      headers: {
+        'Accept': 'application/json',
+        'Content-Type': 'application/json',
+        'X-E2E-Test-Auth-Token': token.trim(),
+      },
+      method: 'POST',
+    });
+
+    if (res.ok) {
+      console.log('\x1b[32m%s\x1b[0m\n', '✓ E2E teardown successful');
+      return true;
+    } else {
+      console.error('\x1b[31m%s\x1b[0m\n', `✗ E2E teardown failed: ${res.status}`);
+      return false;
+    }
+  } catch (error) {
+    console.error('\x1b[31m%s\x1b[0m\n', `✗ E2E teardown error: ${error}`);
+    return false;
+  }
+}
+
+function runPlaywright(args: string[], quietMode: boolean): boolean {
+  try {
+    // Quote arguments that contain spaces
+    const quotedArgs = args.map(arg => arg.includes(' ') ? `"${arg}"` : arg);
+    const playwrightCmd = ['npx', 'cross-env', 'NODE_ENV=production', 'E2E=true', 'playwright', 'test', ...quotedArgs];
+    if (!quietMode) console.log('Running:', playwrightCmd.join(' '));
+    execSync(playwrightCmd.join(' '), {
+      stdio: 'inherit',
+      env: process.env,
+      shell: true,
+    });
+    return true;
+  } catch (error) {
+    return false;
+  }
+}
+
+async function main() {
+  let attempt = 0;
+
+  // Get additional args passed to the script (e.g., test file names, -g patterns)
+  const extraArgs = process.argv.slice(2);
+  const quietMode = process.env.QUIET === '1';
+
+  while (attempt <= RETRIES) {
+    if (attempt > 0) {
+      if (!quietMode) {
+        console.log('\n==========================================');
+        console.log(`Test attempt ${attempt} failed, running teardown and retrying failed tests only...`);
+        console.log('==========================================\n');
+      }
+      await runTeardown();
+    }
+
+    const playwrightArgs = attempt > 0 ? ['--last-failed', ...extraArgs] : extraArgs;
+
+    // Add --quiet flag if QUIET is set
+    if (quietMode && !playwrightArgs.includes('--quiet')) {
+      playwrightArgs.push('--quiet');
+    }
+
+    // First attempt: build bundle and run tests
+    if (attempt === 0 && !process.env.SKIP_BUNDLE) {
+      if (!quietMode) console.log('Building test bundle...');
+      try {
+        execSync('npm run test:bundle', { stdio: quietMode ? 'ignore' : 'inherit' });
+      } catch (error) {
+        console.error('Failed to build test bundle');
+        process.exit(1);
+      }
+    } else if (attempt === 0 && process.env.SKIP_BUNDLE) {
+      if (!quietMode) console.log('Skipping bundle build (SKIP_BUNDLE=1)');
+    }
+
+    if (!quietMode) console.log(attempt > 0 ? 'Running failed tests...' : 'Running all tests...');
+    const success = runPlaywright(playwrightArgs, quietMode);
+
+    if (success) {
+      if (!quietMode) {
+        console.log('\n==========================================');
+        console.log(attempt > 0
+          ? `Tests passed on attempt ${attempt} (after retrying failed tests)`
+          : `Tests passed on attempt ${attempt}`);
+        console.log('==========================================\n');
+      }
+      process.exit(0);
+    }
+
+    attempt++;
+  }
+
+  if (!quietMode) {
+    console.log('\n==========================================');
+    console.log(`Tests failed after ${RETRIES} retries`);
+    console.log('==========================================\n');
+  }
+  process.exit(1);
+}
+
+main();

frontend/e2e/tests/flag-tests.pw.ts

@@ -10,8 +10,10 @@ import {
   log,
   login,
   parseTryItResults,
+  scrollBy,
   toggleFeature,
   waitForElementVisible,
+  waitForNetworkIdle,
   createHelpers,
 } from '../helpers.playwright';
 import { E2E_USER, PASSWORD } from '../config';
@@ -35,15 +37,11 @@ test.describe('Flag Tests', () => {
 
   log('Create Short Life Feature')
   await createFeature(page, 3, 'short_life_feature', false)
-  await page.evaluate(() => {
-    window.scrollBy(0, 15000)
-  })
+  await scrollBy(page, 0, 15000)
 
   log('Delete Short Life Feature')
   await deleteFeature(page, 3, 'short_life_feature')
-  await page.evaluate(() => {
-    window.scrollBy(0, 30000)
-  })
+  await scrollBy(page, 0, 30000)
 
   log('Toggle Feature')
   await toggleFeature(page, 0, true)
@@ -80,10 +78,8 @@ test.describe('Flag Tests', () => {
   log('Switch environment')
   // Navigate back to features list so environment switcher is visible in navbar
   await helpers.gotoFeatures()
-  // Add a wait to ensure the page has fully loaded and environment switcher is ready
-  await page.waitForTimeout(1500)
-  await helpers.waitForElementVisible(byId('switch-environment-production'))
-  // Additional wait to ensure element is interactive
+  await waitForNetworkIdle(page)
+  await waitForElementVisible(page, byId('switch-environment-production'))
   await page.waitForTimeout(500)
   await helpers.click(byId('switch-environment-production'))
 
@@ -95,4 +91,4 @@ test.describe('Flag Tests', () => {
   await deleteFeature(page, 1, 'header_size')
   await deleteFeature(page, 0, 'header_enabled')
   });
-});
+});

frontend/e2e/tests/invite-test.pw.ts

@@ -1,5 +1,5 @@
 import { test } from '../test-setup';
-import { assertTextContent, byId, createHelpers, log } from '../helpers.playwright';
+import { assertTextContent, byId, createHelpers, getInputValue, log } from '../helpers.playwright';
 import { E2E_CHANGE_MAIL, E2E_USER, PASSWORD } from '../config';
 
 const invitePrefix = `flagsmith${new Date().valueOf()}`
@@ -14,9 +14,9 @@ test.describe('Invite Tests', () => {
   await helpers.click(byId('organisation-link'))
   await helpers.waitForElementVisible(byId('org-settings-link'))
   await helpers.click(byId('org-settings-link'))
-  await page.locator(byId('organisation-name')).inputValue()
+  await getInputValue(page, byId('organisation-name'))
   await helpers.click(byId('users-and-permissions'))
-  const inviteLink = await page.locator(byId('invite-link')).inputValue()
+  const inviteLink = await getInputValue(page, byId('invite-link'))
   log('Accept invite')
   await page.goto(inviteLink)
   await helpers.setText('[name="email"]', inviteEmail)

frontend/e2e/tests/organisation-test.pw.ts

@@ -117,8 +117,7 @@ test.describe('Organisation Tests', () => {
 
   log('Test empty organisation name validation')
   await helpers.waitForElementVisible("[data-test='organisation-name']")
-  const orgNameInput = page.locator("[data-test='organisation-name']")
-  const originalName = await orgNameInput.inputValue()
+  const originalName = await helpers.getInputValue("[data-test='organisation-name']")
 
   log('Clear organisation name')
   await helpers.setText("[data-test='organisation-name']", '')