From 54423381f6d403a74932d09e9c1edf7d0c2f345a Mon Sep 17 00:00:00 2001 From: deacon Date: Mon, 16 Mar 2026 10:53:04 -0400 Subject: [PATCH 1/3] Add exhaustive Playwright E2E tests for debrief plugin UI Covers plugin page load, operation selection, report tabs (stats, agents, steps, tactics, facts), PDF download, JSON export, D3 graph rendering, graph settings modal, and error states. Tests run against a full Caldera instance via CALDERA_URL env var. --- package.json | 13 + playwright.config.js | 30 ++ tests/e2e/debrief.spec.js | 852 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 895 insertions(+) create mode 100644 package.json create mode 100644 playwright.config.js create mode 100644 tests/e2e/debrief.spec.js diff --git a/package.json b/package.json new file mode 100644 index 0000000..7a236d3 --- /dev/null +++ b/package.json @@ -0,0 +1,13 @@ +{ + "name": "debrief-e2e-tests", + "version": "1.0.0", + "private": true, + "scripts": { + "test:e2e": "npx playwright test", + "test:e2e:headed": "npx playwright test --headed", + "test:e2e:debug": "npx playwright test --debug" + }, + "devDependencies": { + "@playwright/test": "^1.49.0" + } +} diff --git a/playwright.config.js b/playwright.config.js new file mode 100644 index 0000000..6d89a67 --- /dev/null +++ b/playwright.config.js @@ -0,0 +1,30 @@ +// @ts-check +const { defineConfig, devices } = require('@playwright/test'); + +const CALDERA_URL = process.env.CALDERA_URL || 'http://localhost:8888'; + +module.exports = defineConfig({ + testDir: './tests/e2e', + fullyParallel: false, + forbidOnly: !!process.env.CI, + retries: process.env.CI ? 2 : 0, + workers: 1, + reporter: [['html', { open: 'never' }], ['list']], + timeout: 60_000, + use: { + baseURL: CALDERA_URL, + trace: 'on-first-retry', + screenshot: 'only-on-failure', + headless: true, + httpCredentials: { + username: process.env.CALDERA_USER || 'admin', + password: process.env.CALDERA_PASS || 'admin', + }, + }, + projects: [ + { + name: 'chromium', + use: { ...devices['Desktop Chrome'] }, + }, + ], +}); diff --git a/tests/e2e/debrief.spec.js b/tests/e2e/debrief.spec.js new file mode 100644 index 0000000..3283885 --- /dev/null +++ b/tests/e2e/debrief.spec.js @@ -0,0 +1,852 @@ +// @ts-check +const { test, expect } = require('@playwright/test'); + +const CALDERA_URL = process.env.CALDERA_URL || 'http://localhost:8888'; +const PLUGIN_ROUTE = '/#/plugins/debrief'; + +// --------------------------------------------------------------------------- +// Helper: navigate to the debrief plugin page inside magma +// --------------------------------------------------------------------------- +async function navigateToDebrief(page) { + await page.goto(`${CALDERA_URL}${PLUGIN_ROUTE}`, { waitUntil: 'networkidle' }); +} + +// Fake operations payload for mocking +const MOCK_OPERATIONS = [ + { id: 'op-001', name: 'Test Operation Alpha', state: 'finished', planner: { name: 'atomic' }, objective: { name: 'default' }, start: '2025-01-01 00:00:00' }, + { id: 'op-002', name: 'Test Operation Beta', state: 'running', planner: { name: 'batch' }, objective: { name: 'custom-obj' }, start: '2025-01-02 12:00:00' }, +]; + +const MOCK_REPORT = { + operations: [ + { + name: 'Test Operation Alpha', + state: 'finished', + planner: { name: 'atomic' }, + objective: { name: 'default' }, + start: '2025-01-01 00:00:00', + host_group: [ + { paw: 'abc123', host: 'workstation1', platform: 'windows', username: 'admin', privilege: 'Elevated', exe_name: 'sandcat.exe' }, + ], + chain: [ + { id: 'link-1', status: 0, finish: '2025-01-01 00:01:00', ability: { name: 'whoami' }, paw: 'abc123', command: 'd2hvYW1p' }, + ], + }, + ], + ttps: {}, +}; + +const MOCK_SECTIONS = { + report_sections: [ + { key: 'main-summary', name: 'Main Summary' }, + { key: 'statistics', name: 'Statistics' }, + { key: 'agents', name: 'Agents' }, + { key: 'attackpath-graph', name: 'Attack Path Graph' }, + { key: 'steps-graph', name: 'Steps Graph' }, + { key: 'tactic-graph', name: 'Tactic Graph' }, + { key: 'technique-graph', name: 'Technique Graph' }, + { key: 'fact-graph', name: 'Fact Graph' }, + { key: 'tactic-technique-table', name: 'Tactic/Technique Table' }, + { key: 'steps-table', name: 'Steps Table' }, + { key: 'facts-table', name: 'Facts Table' }, + ], +}; + +// =========================================================================== +// 1. Plugin page loads +// =========================================================================== +test.describe('Debrief plugin page load', () => { + test('should display the Debrief heading', async ({ page }) => { + await navigateToDebrief(page); + const heading = page.locator('h2', { hasText: 'Debrief' }); + await expect(heading).toBeVisible({ timeout: 15_000 }); + }); + + test('should display campaign analytics description', async ({ page }) => { + await navigateToDebrief(page); + await expect( + page.locator('text=Campaign Analytics') + ).toBeVisible({ timeout: 15_000 }); + }); + + test('should render Graph Settings button', async ({ page }) => { + await navigateToDebrief(page); + const btn = page.locator('button', { hasText: 'Graph Settings' }); + await expect(btn).toBeVisible({ timeout: 15_000 }); + }); + + test('should render Download PDF Report button', async ({ page }) => { + await navigateToDebrief(page); + const btn = page.locator('button', { hasText: 'Download PDF Report' }); + await expect(btn).toBeVisible({ timeout: 15_000 }); + }); + + test('should render Download Operation JSON button', async ({ page }) => { + await navigateToDebrief(page); + const btn = page.locator('button', { hasText: 'Download Operation JSON' }); + await expect(btn).toBeVisible({ timeout: 15_000 }); + }); + + test('should have an operation select dropdown', async ({ page }) => { + await navigateToDebrief(page); + const select = page.locator('select').first(); + await expect(select).toBeVisible({ timeout: 15_000 }); + }); + + test('should have a horizontal rule separator', async ({ page }) => { + await navigateToDebrief(page); + await expect(page.locator('hr').first()).toBeVisible({ timeout: 15_000 }); + }); +}); + +// =========================================================================== +// 2. Operation selection for debrief +// =========================================================================== +test.describe('Debrief operation selection', () => { + test('should fetch operations from the API', async ({ page }) => { + const response = await page.request.get(`${CALDERA_URL}/api/v2/operations`); + expect(response.ok()).toBeTruthy(); + const ops = await response.json(); + expect(Array.isArray(ops)).toBeTruthy(); + }); + + test('buttons should be disabled when no operation is selected', async ({ page }) => { + await navigateToDebrief(page); + const pdfBtn = page.locator('button', { hasText: 'Download PDF Report' }); + await expect(pdfBtn).toBeDisabled({ timeout: 15_000 }); + const jsonBtn = page.locator('button', { hasText: 'Download Operation JSON' }); + await expect(jsonBtn).toBeDisabled(); + const settingsBtn = page.locator('button', { hasText: 'Graph Settings' }); + await expect(settingsBtn).toBeDisabled(); + }); + + test('operations dropdown should populate from API', async ({ page }) => { + await page.route('**/api/v2/operations', (route) => + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify(MOCK_OPERATIONS), + }) + ); + await navigateToDebrief(page); + const options = page.locator('select option'); + // Should have at least the mock operations plus the default disabled option + await expect(options).toHaveCount(3, { timeout: 15_000 }); // 1 disabled + 2 ops + }); + + test('selecting an operation should trigger report loading', async ({ page }) => { + let reportRequested = false; + await page.route('**/api/v2/operations', (route) => + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify(MOCK_OPERATIONS), + }) + ); + await page.route('**/plugin/debrief/report', (route) => { + reportRequested = true; + return route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify(MOCK_REPORT), + }); + }); + await page.route('**/plugin/debrief/graph**', (route) => + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify({ nodes: [], links: [] }), + }) + ); + await page.route('**/plugin/debrief/sections', (route) => + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify(MOCK_SECTIONS), + }) + ); + await page.route('**/plugin/debrief/logos', (route) => + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify({ logos: [] }), + }) + ); + await navigateToDebrief(page); + const select = page.locator('select').first(); + await select.selectOption({ index: 1 }); + await page.waitForTimeout(2000); + expect(reportRequested).toBeTruthy(); + }); +}); + +// =========================================================================== +// 3. Report section display (tabs: stats, agents, steps, tactics, facts) +// =========================================================================== +test.describe('Debrief report section display', () => { + test('should show tabs when an operation is selected (mocked)', async ({ page }) => { + await page.route('**/api/v2/operations', (route) => + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify(MOCK_OPERATIONS), + }) + ); + await page.route('**/plugin/debrief/report', (route) => + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify(MOCK_REPORT), + }) + ); + await page.route('**/plugin/debrief/graph**', (route) => + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify({ nodes: [], links: [] }), + }) + ); + await page.route('**/plugin/debrief/sections', (route) => + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify(MOCK_SECTIONS), + }) + ); + await page.route('**/plugin/debrief/logos', (route) => + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify({ logos: [] }), + }) + ); + await navigateToDebrief(page); + const select = page.locator('select').first(); + await select.selectOption({ index: 1 }); + await page.waitForTimeout(1000); + + // Tab labels should be visible + await expect(page.locator('a', { hasText: 'Stats' })).toBeVisible(); + await expect(page.locator('a', { hasText: 'Agents' })).toBeVisible(); + await expect(page.locator('a', { hasText: 'Steps' })).toBeVisible(); + await expect(page.locator('a', { hasText: 'Tactics & Techniques' })).toBeVisible(); + await expect(page.locator('a', { hasText: 'Fact Graph' })).toBeVisible(); + }); + + test('Stats tab should show operation statistics table headers', async ({ page }) => { + await page.route('**/api/v2/operations', (route) => + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify(MOCK_OPERATIONS), + }) + ); + await page.route('**/plugin/debrief/report', (route) => + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify(MOCK_REPORT), + }) + ); + await page.route('**/plugin/debrief/graph**', (route) => + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify({ nodes: [], links: [] }), + }) + ); + await page.route('**/plugin/debrief/sections', (route) => + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify(MOCK_SECTIONS), + }) + ); + await page.route('**/plugin/debrief/logos', (route) => + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify({ logos: [] }), + }) + ); + await navigateToDebrief(page); + const select = page.locator('select').first(); + await select.selectOption({ index: 1 }); + await page.waitForTimeout(1000); + + await expect(page.locator('th', { hasText: 'Name' }).first()).toBeVisible(); + await expect(page.locator('th', { hasText: 'State' }).first()).toBeVisible(); + await expect(page.locator('th', { hasText: 'Planner' }).first()).toBeVisible(); + await expect(page.locator('th', { hasText: 'Objective' }).first()).toBeVisible(); + await expect(page.locator('th', { hasText: 'Time' }).first()).toBeVisible(); + }); +}); + +// =========================================================================== +// 4. PDF download button +// =========================================================================== +test.describe('Debrief PDF download', () => { + test('PDF button should be disabled without operation selection', async ({ page }) => { + await navigateToDebrief(page); + const btn = page.locator('button', { hasText: 'Download PDF Report' }); + await expect(btn).toBeDisabled({ timeout: 15_000 }); + }); + + test('clicking PDF button should open report modal', async ({ page }) => { + await page.route('**/api/v2/operations', (route) => + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify(MOCK_OPERATIONS), + }) + ); + await page.route('**/plugin/debrief/report', (route) => + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify(MOCK_REPORT), + }) + ); + await page.route('**/plugin/debrief/graph**', (route) => + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify({ nodes: [], links: [] }), + }) + ); + await page.route('**/plugin/debrief/sections', (route) => + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify(MOCK_SECTIONS), + }) + ); + await page.route('**/plugin/debrief/logos', (route) => + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify({ logos: [] }), + }) + ); + await navigateToDebrief(page); + const select = page.locator('select').first(); + await select.selectOption({ index: 1 }); + await page.waitForTimeout(1000); + + const pdfBtn = page.locator('button', { hasText: 'Download PDF Report' }); + await pdfBtn.click(); + await expect(page.locator('.modal.is-active')).toBeVisible({ timeout: 5_000 }); + await expect(page.locator('text=Download Report as PDF')).toBeVisible(); + }); + + test('report modal should have Report Sections heading', async ({ page }) => { + await page.route('**/api/v2/operations', (route) => + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify(MOCK_OPERATIONS), + }) + ); + await page.route('**/plugin/debrief/report', (route) => + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify(MOCK_REPORT), + }) + ); + await page.route('**/plugin/debrief/graph**', (route) => + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify({ nodes: [], links: [] }), + }) + ); + await page.route('**/plugin/debrief/sections', (route) => + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify(MOCK_SECTIONS), + }) + ); + await page.route('**/plugin/debrief/logos', (route) => + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify({ logos: [] }), + }) + ); + await navigateToDebrief(page); + const select = page.locator('select').first(); + await select.selectOption({ index: 1 }); + await page.waitForTimeout(1000); + + await page.locator('button', { hasText: 'Download PDF Report' }).click(); + await expect(page.locator('h5', { hasText: 'Report Sections' })).toBeVisible({ timeout: 5_000 }); + }); + + test('report modal should show custom logo checkbox', async ({ page }) => { + await page.route('**/api/v2/operations', (route) => + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify(MOCK_OPERATIONS), + }) + ); + await page.route('**/plugin/debrief/report', (route) => + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify(MOCK_REPORT), + }) + ); + await page.route('**/plugin/debrief/graph**', (route) => + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify({ nodes: [], links: [] }), + }) + ); + await page.route('**/plugin/debrief/sections', (route) => + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify(MOCK_SECTIONS), + }) + ); + await page.route('**/plugin/debrief/logos', (route) => + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify({ logos: [] }), + }) + ); + await navigateToDebrief(page); + const select = page.locator('select').first(); + await select.selectOption({ index: 1 }); + await page.waitForTimeout(1000); + + await page.locator('button', { hasText: 'Download PDF Report' }).click(); + await expect(page.locator('text=Use custom logo')).toBeVisible({ timeout: 5_000 }); + }); + + test('report modal Download button triggers PDF API call', async ({ page }) => { + let pdfRequested = false; + await page.route('**/api/v2/operations', (route) => + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify(MOCK_OPERATIONS), + }) + ); + await page.route('**/plugin/debrief/report', (route) => + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify(MOCK_REPORT), + }) + ); + await page.route('**/plugin/debrief/graph**', (route) => + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify({ nodes: [], links: [] }), + }) + ); + await page.route('**/plugin/debrief/sections', (route) => + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify(MOCK_SECTIONS), + }) + ); + await page.route('**/plugin/debrief/logos', (route) => + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify({ logos: [] }), + }) + ); + await page.route('**/plugin/debrief/pdf', (route) => { + pdfRequested = true; + return route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify({ filename: 'debrief-report.pdf', pdf_bytes: '' }), + }); + }); + await navigateToDebrief(page); + const select = page.locator('select').first(); + await select.selectOption({ index: 1 }); + await page.waitForTimeout(1000); + + await page.locator('button', { hasText: 'Download PDF Report' }).click(); + await page.waitForTimeout(500); + // Click the Download button inside the modal + const downloadBtn = page.locator('.modal.is-active button', { hasText: 'Download' }); + await downloadBtn.click(); + await page.waitForTimeout(2000); + expect(pdfRequested).toBeTruthy(); + }); +}); + +// =========================================================================== +// 5. JSON export +// =========================================================================== +test.describe('Debrief JSON export', () => { + test('JSON button should be disabled without operation selection', async ({ page }) => { + await navigateToDebrief(page); + const btn = page.locator('button', { hasText: 'Download Operation JSON' }); + await expect(btn).toBeDisabled({ timeout: 15_000 }); + }); + + test('clicking JSON button should trigger JSON API call', async ({ page }) => { + let jsonRequested = false; + await page.route('**/api/v2/operations', (route) => + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify(MOCK_OPERATIONS), + }) + ); + await page.route('**/plugin/debrief/report', (route) => + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify(MOCK_REPORT), + }) + ); + await page.route('**/plugin/debrief/graph**', (route) => + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify({ nodes: [], links: [] }), + }) + ); + await page.route('**/plugin/debrief/sections', (route) => + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify(MOCK_SECTIONS), + }) + ); + await page.route('**/plugin/debrief/logos', (route) => + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify({ logos: [] }), + }) + ); + await page.route('**/plugin/debrief/json', (route) => { + jsonRequested = true; + return route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify({ filename: 'debrief-report', operations: [] }), + }); + }); + await navigateToDebrief(page); + const select = page.locator('select').first(); + await select.selectOption({ index: 1 }); + await page.waitForTimeout(1000); + + await page.locator('button', { hasText: 'Download Operation JSON' }).click(); + await page.waitForTimeout(2000); + expect(jsonRequested).toBeTruthy(); + }); +}); + +// =========================================================================== +// 6. D3 graph rendering +// =========================================================================== +test.describe('Debrief D3 graph rendering', () => { + test('should have SVG containers for all graph types', async ({ page }) => { + await navigateToDebrief(page); + await expect(page.locator('#debrief-steps-svg')).toBeAttached({ timeout: 15_000 }); + await expect(page.locator('#debrief-attackpath-svg')).toBeAttached(); + await expect(page.locator('#debrief-tactic-svg')).toBeAttached(); + await expect(page.locator('#debrief-technique-svg')).toBeAttached(); + }); + + test('should have graph type selector with all options', async ({ page }) => { + await navigateToDebrief(page); + const graphSelect = page.locator('select', { has: page.locator('option[value="attackpath"]') }); + await expect(graphSelect).toBeAttached({ timeout: 15_000 }); + await expect(page.locator('option[value="steps"]')).toBeAttached(); + await expect(page.locator('option[value="tactic"]')).toBeAttached(); + await expect(page.locator('option[value="technique"]')).toBeAttached(); + }); + + test('should have fact graph SVG container', async ({ page }) => { + await navigateToDebrief(page); + await expect(page.locator('#debrief-fact-svg')).toBeAttached({ timeout: 15_000 }); + }); + + test('should have playback control buttons', async ({ page }) => { + await navigateToDebrief(page); + // Playback buttons: fast-backward, backward, play/pause, forward, fast-forward + const buttons = page.locator('#debrief-graph .buttons button'); + await expect(buttons).toHaveCount(6, { timeout: 15_000 }); + }); + + test('should have a legend toggle button', async ({ page }) => { + await navigateToDebrief(page); + const legendBtn = page.locator('button', { hasText: /Legend/ }); + await expect(legendBtn).toBeAttached({ timeout: 15_000 }); + }); + + test('graph settings modal should show display options', async ({ page }) => { + await page.route('**/api/v2/operations', (route) => + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify(MOCK_OPERATIONS), + }) + ); + await page.route('**/plugin/debrief/report', (route) => + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify(MOCK_REPORT), + }) + ); + await page.route('**/plugin/debrief/graph**', (route) => + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify({ nodes: [], links: [] }), + }) + ); + await page.route('**/plugin/debrief/sections', (route) => + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify(MOCK_SECTIONS), + }) + ); + await page.route('**/plugin/debrief/logos', (route) => + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify({ logos: [] }), + }) + ); + await navigateToDebrief(page); + const select = page.locator('select').first(); + await select.selectOption({ index: 1 }); + await page.waitForTimeout(1000); + + await page.locator('button', { hasText: 'Graph Settings' }).click(); + await expect(page.locator('.modal.is-active')).toBeVisible({ timeout: 5_000 }); + await expect(page.locator('text=Display Options')).toBeVisible(); + await expect(page.locator('text=Show labels')).toBeVisible(); + await expect(page.locator('text=Show icons')).toBeVisible(); + await expect(page.locator('text=Data Options')).toBeVisible(); + await expect(page.locator('text=Show operation steps')).toBeVisible(); + await expect(page.locator('text=Show steps as tactics')).toBeVisible(); + }); +}); + +// =========================================================================== +// 7. Error states +// =========================================================================== +test.describe('Debrief error states', () => { + test('should handle no operations gracefully', async ({ page }) => { + await page.route('**/api/v2/operations', (route) => + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify([]), + }) + ); + await page.route('**/plugin/debrief/sections', (route) => + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify(MOCK_SECTIONS), + }) + ); + await page.route('**/plugin/debrief/logos', (route) => + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify({ logos: [] }), + }) + ); + await navigateToDebrief(page); + // Heading should still show + await expect(page.locator('h2', { hasText: 'Debrief' })).toBeVisible({ timeout: 15_000 }); + // No options in dropdown + const options = page.locator('select option:not([disabled])'); + await expect(options).toHaveCount(0); + }); + + test('should handle operations API failure gracefully', async ({ page }) => { + await page.route('**/api/v2/operations', (route) => + route.fulfill({ status: 500, body: 'Server Error' }) + ); + await page.route('**/plugin/debrief/sections', (route) => + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify(MOCK_SECTIONS), + }) + ); + await navigateToDebrief(page); + // Page should still render without crashing + await expect(page.locator('h2', { hasText: 'Debrief' })).toBeVisible({ timeout: 15_000 }); + }); + + test('should handle report API failure gracefully', async ({ page }) => { + await page.route('**/api/v2/operations', (route) => + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify(MOCK_OPERATIONS), + }) + ); + await page.route('**/plugin/debrief/report', (route) => + route.fulfill({ status: 500, body: 'Report generation failed' }) + ); + await page.route('**/plugin/debrief/graph**', (route) => + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify({ nodes: [], links: [] }), + }) + ); + await page.route('**/plugin/debrief/sections', (route) => + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify(MOCK_SECTIONS), + }) + ); + await page.route('**/plugin/debrief/logos', (route) => + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify({ logos: [] }), + }) + ); + await navigateToDebrief(page); + const select = page.locator('select').first(); + await select.selectOption({ index: 1 }); + await page.waitForTimeout(2000); + // Page should not crash + await expect(page.locator('h2', { hasText: 'Debrief' })).toBeVisible(); + }); + + test('should handle PDF download failure gracefully', async ({ page }) => { + await page.route('**/api/v2/operations', (route) => + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify(MOCK_OPERATIONS), + }) + ); + await page.route('**/plugin/debrief/report', (route) => + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify(MOCK_REPORT), + }) + ); + await page.route('**/plugin/debrief/graph**', (route) => + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify({ nodes: [], links: [] }), + }) + ); + await page.route('**/plugin/debrief/sections', (route) => + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify(MOCK_SECTIONS), + }) + ); + await page.route('**/plugin/debrief/logos', (route) => + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify({ logos: [] }), + }) + ); + await page.route('**/plugin/debrief/pdf', (route) => + route.fulfill({ status: 500, body: 'PDF generation failed' }) + ); + await navigateToDebrief(page); + const select = page.locator('select').first(); + await select.selectOption({ index: 1 }); + await page.waitForTimeout(1000); + await page.locator('button', { hasText: 'Download PDF Report' }).click(); + await page.waitForTimeout(500); + const downloadBtn = page.locator('.modal.is-active button', { hasText: 'Download' }); + await downloadBtn.click(); + await page.waitForTimeout(2000); + // Page should not crash + await expect(page.locator('h2', { hasText: 'Debrief' })).toBeVisible(); + }); + + test('should handle JSON export failure gracefully', async ({ page }) => { + await page.route('**/api/v2/operations', (route) => + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify(MOCK_OPERATIONS), + }) + ); + await page.route('**/plugin/debrief/report', (route) => + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify(MOCK_REPORT), + }) + ); + await page.route('**/plugin/debrief/graph**', (route) => + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify({ nodes: [], links: [] }), + }) + ); + await page.route('**/plugin/debrief/sections', (route) => + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify(MOCK_SECTIONS), + }) + ); + await page.route('**/plugin/debrief/logos', (route) => + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify({ logos: [] }), + }) + ); + await page.route('**/plugin/debrief/json', (route) => + route.fulfill({ status: 500, body: 'JSON export failed' }) + ); + await navigateToDebrief(page); + const select = page.locator('select').first(); + await select.selectOption({ index: 1 }); + await page.waitForTimeout(1000); + await page.locator('button', { hasText: 'Download Operation JSON' }).click(); + await page.waitForTimeout(2000); + // Page should not crash + await expect(page.locator('h2', { hasText: 'Debrief' })).toBeVisible(); + }); + + test('should handle network timeout on operations API', async ({ page }) => { + await page.route('**/api/v2/operations', (route) => route.abort('timedout')); + await page.route('**/plugin/debrief/sections', (route) => + route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify(MOCK_SECTIONS), + }) + ); + await navigateToDebrief(page); + await expect(page.locator('h2', { hasText: 'Debrief' })).toBeVisible({ timeout: 15_000 }); + }); +}); From 6515d8cc618d801590fe447c15534c68e5de22a3 Mon Sep 17 00:00:00 2001 From: deacon Date: Mon, 16 Mar 2026 17:00:48 -0400 Subject: [PATCH 2/3] Refactor E2E tests: extract shared helper, fix flakiness, fix codebase bugs - Extract mockDebriefRoutes() helper to eliminate ~400 lines of copy-pasted route mocking across 15 tests - Replace all waitForTimeout() calls with deterministic waits (waitForResponse, waitFor on locators) - Replace networkidle with domcontentloaded + explicit heading wait - Remove unmocked live API test that would fail without running server - Fix playback button count comment (6 buttons including legend toggle) - Use baseURL from playwright config instead of duplicating CALDERA_URL - Fix file handle leak in c_story.py adjust_icon_svgs - Add viewBox null check in c_story.py - Fix stale op_id bug in debrief_svc.py build_steps_d3 --- app/debrief_svc.py | 8 +- app/objects/c_story.py | 8 +- tests/e2e/debrief.spec.js | 708 +++++++------------------------------- 3 files changed, 138 insertions(+), 586 deletions(-) diff --git a/app/debrief_svc.py b/app/debrief_svc.py index daf4c17..c956782 100644 --- a/app/debrief_svc.py +++ b/app/debrief_svc.py @@ -25,14 +25,14 @@ async def build_steps_d3(self, operation_ids): for operation in operations: # Add operation node - graph_output['nodes'].append(dict(name=operation.name, type='operation', id=op_id, img='operation', + graph_output['nodes'].append(dict(name=operation.name, type='operation', id=operation.id, img='operation', timestamp=self._format_timestamp(operation.created))) # Add agents for this operation agents = [x for x in operation.agents if x] self._add_agents_to_d3(agents, id_store, graph_output) for agent in agents: - graph_output['links'].append(dict(source=op_id, + graph_output['links'].append(dict(source=operation.id, target=id_store['agent' + agent.unique], type='has_agent')) @@ -42,12 +42,12 @@ async def build_steps_d3(self, operation_ids): link_graph_id = id_store['link' + link.unique] = max(id_store.values()) + 1 display_name = link.ability.name + (' (cleanup)' if link.cleanup else '') graph_output['nodes'].append(dict(type='link', name='link:'+link.unique, id=link_graph_id, - status=link.status, operation=op_id, img=link.ability.tactic, + status=link.status, operation=operation.id, img=link.ability.tactic, attrs=dict(status=link.status, name=display_name), timestamp=self._format_timestamp(link.created))) if not previous_link_graph_id: - graph_output['links'].append(dict(source=op_id, target=link_graph_id, type='next_link')) + graph_output['links'].append(dict(source=operation.id, target=link_graph_id, type='next_link')) else: graph_output['links'].append(dict(source=previous_link_graph_id, target=link_graph_id, type='next_link')) diff --git a/app/objects/c_story.py b/app/objects/c_story.py index 0109405..b1a6023 100644 --- a/app/objects/c_story.py +++ b/app/objects/c_story.py @@ -99,12 +99,16 @@ def adjust_icon_svgs(path): for icon_svg in svg.getroot().iter("{http://www.w3.org/2000/svg}svg"): if icon_svg.get('id') == 'copy-svg': continue - viewbox = [int(float(val)) for val in icon_svg.get('viewBox').split()] + viewbox_attr = icon_svg.get('viewBox') + if not viewbox_attr: + continue + viewbox = [int(float(val)) for val in viewbox_attr.split()] aspect = viewbox[2] / viewbox[3] icon_svg.set('width', str(round(float(icon_svg.get('height')) * aspect))) if not icon_svg.get('id') or 'legend' not in icon_svg.get('id'): icon_svg.set('x', '-' + str(int(icon_svg.get('width')) / 2)) - svg.write(open(path, 'wb')) + with open(path, 'wb') as f: + svg.write(f) @staticmethod def get_table_object(val): diff --git a/tests/e2e/debrief.spec.js b/tests/e2e/debrief.spec.js index 3283885..1c122f4 100644 --- a/tests/e2e/debrief.spec.js +++ b/tests/e2e/debrief.spec.js @@ -1,17 +1,11 @@ // @ts-check const { test, expect } = require('@playwright/test'); -const CALDERA_URL = process.env.CALDERA_URL || 'http://localhost:8888'; const PLUGIN_ROUTE = '/#/plugins/debrief'; // --------------------------------------------------------------------------- -// Helper: navigate to the debrief plugin page inside magma +// Shared mock data // --------------------------------------------------------------------------- -async function navigateToDebrief(page) { - await page.goto(`${CALDERA_URL}${PLUGIN_ROUTE}`, { waitUntil: 'networkidle' }); -} - -// Fake operations payload for mocking const MOCK_OPERATIONS = [ { id: 'op-001', name: 'Test Operation Alpha', state: 'finished', planner: { name: 'atomic' }, objective: { name: 'default' }, start: '2025-01-01 00:00:00' }, { id: 'op-002', name: 'Test Operation Beta', state: 'running', planner: { name: 'batch' }, objective: { name: 'custom-obj' }, start: '2025-01-02 12:00:00' }, @@ -52,45 +46,79 @@ const MOCK_SECTIONS = { ], }; +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +/** + * Mock all standard debrief API routes. Pass `overrides` to replace or add + * route handlers (keyed by glob pattern). + */ +async function mockDebriefRoutes(page, overrides = {}) { + const defaults = { + '**/api/v2/operations': (route) => + route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify(MOCK_OPERATIONS) }), + '**/plugin/debrief/report': (route) => + route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify(MOCK_REPORT) }), + '**/plugin/debrief/graph**': (route) => + route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify({ nodes: [], links: [] }) }), + '**/plugin/debrief/sections': (route) => + route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify(MOCK_SECTIONS) }), + '**/plugin/debrief/logos': (route) => + route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify({ logos: [] }) }), + }; + const routes = { ...defaults, ...overrides }; + for (const [pattern, handler] of Object.entries(routes)) { + await page.route(pattern, handler); + } +} + +/** Navigate to the debrief plugin page and wait for the heading to appear. */ +async function navigateToDebrief(page) { + await page.goto(PLUGIN_ROUTE, { waitUntil: 'domcontentloaded' }); + await page.locator('h2', { hasText: 'Debrief' }).waitFor({ state: 'visible', timeout: 15_000 }); +} + +/** Select the first operation from the dropdown and wait for the report to load. */ +async function selectFirstOperation(page) { + const select = page.locator('select').first(); + await select.selectOption({ index: 1 }); + // Wait for the report API response to arrive + await page.waitForResponse((resp) => resp.url().includes('/plugin/debrief/report'), { timeout: 10_000 }).catch(() => {}); +} + // =========================================================================== // 1. Plugin page loads // =========================================================================== test.describe('Debrief plugin page load', () => { test('should display the Debrief heading', async ({ page }) => { await navigateToDebrief(page); - const heading = page.locator('h2', { hasText: 'Debrief' }); - await expect(heading).toBeVisible({ timeout: 15_000 }); + await expect(page.locator('h2', { hasText: 'Debrief' })).toBeVisible(); }); test('should display campaign analytics description', async ({ page }) => { await navigateToDebrief(page); - await expect( - page.locator('text=Campaign Analytics') - ).toBeVisible({ timeout: 15_000 }); + await expect(page.locator('text=Campaign Analytics')).toBeVisible({ timeout: 15_000 }); }); test('should render Graph Settings button', async ({ page }) => { await navigateToDebrief(page); - const btn = page.locator('button', { hasText: 'Graph Settings' }); - await expect(btn).toBeVisible({ timeout: 15_000 }); + await expect(page.locator('button', { hasText: 'Graph Settings' })).toBeVisible({ timeout: 15_000 }); }); test('should render Download PDF Report button', async ({ page }) => { await navigateToDebrief(page); - const btn = page.locator('button', { hasText: 'Download PDF Report' }); - await expect(btn).toBeVisible({ timeout: 15_000 }); + await expect(page.locator('button', { hasText: 'Download PDF Report' })).toBeVisible({ timeout: 15_000 }); }); test('should render Download Operation JSON button', async ({ page }) => { await navigateToDebrief(page); - const btn = page.locator('button', { hasText: 'Download Operation JSON' }); - await expect(btn).toBeVisible({ timeout: 15_000 }); + await expect(page.locator('button', { hasText: 'Download Operation JSON' })).toBeVisible({ timeout: 15_000 }); }); test('should have an operation select dropdown', async ({ page }) => { await navigateToDebrief(page); - const select = page.locator('select').first(); - await expect(select).toBeVisible({ timeout: 15_000 }); + await expect(page.locator('select').first()).toBeVisible({ timeout: 15_000 }); }); test('should have a horizontal rule separator', async ({ page }) => { @@ -103,79 +131,31 @@ test.describe('Debrief plugin page load', () => { // 2. Operation selection for debrief // =========================================================================== test.describe('Debrief operation selection', () => { - test('should fetch operations from the API', async ({ page }) => { - const response = await page.request.get(`${CALDERA_URL}/api/v2/operations`); - expect(response.ok()).toBeTruthy(); - const ops = await response.json(); - expect(Array.isArray(ops)).toBeTruthy(); - }); - test('buttons should be disabled when no operation is selected', async ({ page }) => { await navigateToDebrief(page); - const pdfBtn = page.locator('button', { hasText: 'Download PDF Report' }); - await expect(pdfBtn).toBeDisabled({ timeout: 15_000 }); - const jsonBtn = page.locator('button', { hasText: 'Download Operation JSON' }); - await expect(jsonBtn).toBeDisabled(); - const settingsBtn = page.locator('button', { hasText: 'Graph Settings' }); - await expect(settingsBtn).toBeDisabled(); + await expect(page.locator('button', { hasText: 'Download PDF Report' })).toBeDisabled({ timeout: 15_000 }); + await expect(page.locator('button', { hasText: 'Download Operation JSON' })).toBeDisabled(); + await expect(page.locator('button', { hasText: 'Graph Settings' })).toBeDisabled(); }); test('operations dropdown should populate from API', async ({ page }) => { - await page.route('**/api/v2/operations', (route) => - route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify(MOCK_OPERATIONS), - }) - ); + await mockDebriefRoutes(page); await navigateToDebrief(page); const options = page.locator('select option'); - // Should have at least the mock operations plus the default disabled option - await expect(options).toHaveCount(3, { timeout: 15_000 }); // 1 disabled + 2 ops + // 1 disabled placeholder + 2 mock operations + await expect(options).toHaveCount(3, { timeout: 15_000 }); }); test('selecting an operation should trigger report loading', async ({ page }) => { let reportRequested = false; - await page.route('**/api/v2/operations', (route) => - route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify(MOCK_OPERATIONS), - }) - ); - await page.route('**/plugin/debrief/report', (route) => { - reportRequested = true; - return route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify(MOCK_REPORT), - }); + await mockDebriefRoutes(page, { + '**/plugin/debrief/report': (route) => { + reportRequested = true; + return route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify(MOCK_REPORT) }); + }, }); - await page.route('**/plugin/debrief/graph**', (route) => - route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify({ nodes: [], links: [] }), - }) - ); - await page.route('**/plugin/debrief/sections', (route) => - route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify(MOCK_SECTIONS), - }) - ); - await page.route('**/plugin/debrief/logos', (route) => - route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify({ logos: [] }), - }) - ); await navigateToDebrief(page); - const select = page.locator('select').first(); - await select.selectOption({ index: 1 }); - await page.waitForTimeout(2000); + await selectFirstOperation(page); expect(reportRequested).toBeTruthy(); }); }); @@ -184,49 +164,12 @@ test.describe('Debrief operation selection', () => { // 3. Report section display (tabs: stats, agents, steps, tactics, facts) // =========================================================================== test.describe('Debrief report section display', () => { - test('should show tabs when an operation is selected (mocked)', async ({ page }) => { - await page.route('**/api/v2/operations', (route) => - route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify(MOCK_OPERATIONS), - }) - ); - await page.route('**/plugin/debrief/report', (route) => - route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify(MOCK_REPORT), - }) - ); - await page.route('**/plugin/debrief/graph**', (route) => - route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify({ nodes: [], links: [] }), - }) - ); - await page.route('**/plugin/debrief/sections', (route) => - route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify(MOCK_SECTIONS), - }) - ); - await page.route('**/plugin/debrief/logos', (route) => - route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify({ logos: [] }), - }) - ); + test('should show tabs when an operation is selected', async ({ page }) => { + await mockDebriefRoutes(page); await navigateToDebrief(page); - const select = page.locator('select').first(); - await select.selectOption({ index: 1 }); - await page.waitForTimeout(1000); + await selectFirstOperation(page); - // Tab labels should be visible - await expect(page.locator('a', { hasText: 'Stats' })).toBeVisible(); + await expect(page.locator('a', { hasText: 'Stats' })).toBeVisible({ timeout: 10_000 }); await expect(page.locator('a', { hasText: 'Agents' })).toBeVisible(); await expect(page.locator('a', { hasText: 'Steps' })).toBeVisible(); await expect(page.locator('a', { hasText: 'Tactics & Techniques' })).toBeVisible(); @@ -234,47 +177,11 @@ test.describe('Debrief report section display', () => { }); test('Stats tab should show operation statistics table headers', async ({ page }) => { - await page.route('**/api/v2/operations', (route) => - route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify(MOCK_OPERATIONS), - }) - ); - await page.route('**/plugin/debrief/report', (route) => - route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify(MOCK_REPORT), - }) - ); - await page.route('**/plugin/debrief/graph**', (route) => - route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify({ nodes: [], links: [] }), - }) - ); - await page.route('**/plugin/debrief/sections', (route) => - route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify(MOCK_SECTIONS), - }) - ); - await page.route('**/plugin/debrief/logos', (route) => - route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify({ logos: [] }), - }) - ); + await mockDebriefRoutes(page); await navigateToDebrief(page); - const select = page.locator('select').first(); - await select.selectOption({ index: 1 }); - await page.waitForTimeout(1000); + await selectFirstOperation(page); - await expect(page.locator('th', { hasText: 'Name' }).first()).toBeVisible(); + await expect(page.locator('th', { hasText: 'Name' }).first()).toBeVisible({ timeout: 10_000 }); await expect(page.locator('th', { hasText: 'State' }).first()).toBeVisible(); await expect(page.locator('th', { hasText: 'Planner' }).first()).toBeVisible(); await expect(page.locator('th', { hasText: 'Objective' }).first()).toBeVisible(); @@ -288,142 +195,32 @@ test.describe('Debrief report section display', () => { test.describe('Debrief PDF download', () => { test('PDF button should be disabled without operation selection', async ({ page }) => { await navigateToDebrief(page); - const btn = page.locator('button', { hasText: 'Download PDF Report' }); - await expect(btn).toBeDisabled({ timeout: 15_000 }); + await expect(page.locator('button', { hasText: 'Download PDF Report' })).toBeDisabled({ timeout: 15_000 }); }); test('clicking PDF button should open report modal', async ({ page }) => { - await page.route('**/api/v2/operations', (route) => - route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify(MOCK_OPERATIONS), - }) - ); - await page.route('**/plugin/debrief/report', (route) => - route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify(MOCK_REPORT), - }) - ); - await page.route('**/plugin/debrief/graph**', (route) => - route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify({ nodes: [], links: [] }), - }) - ); - await page.route('**/plugin/debrief/sections', (route) => - route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify(MOCK_SECTIONS), - }) - ); - await page.route('**/plugin/debrief/logos', (route) => - route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify({ logos: [] }), - }) - ); + await mockDebriefRoutes(page); await navigateToDebrief(page); - const select = page.locator('select').first(); - await select.selectOption({ index: 1 }); - await page.waitForTimeout(1000); + await selectFirstOperation(page); - const pdfBtn = page.locator('button', { hasText: 'Download PDF Report' }); - await pdfBtn.click(); + await page.locator('button', { hasText: 'Download PDF Report' }).click(); await expect(page.locator('.modal.is-active')).toBeVisible({ timeout: 5_000 }); await expect(page.locator('text=Download Report as PDF')).toBeVisible(); }); test('report modal should have Report Sections heading', async ({ page }) => { - await page.route('**/api/v2/operations', (route) => - route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify(MOCK_OPERATIONS), - }) - ); - await page.route('**/plugin/debrief/report', (route) => - route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify(MOCK_REPORT), - }) - ); - await page.route('**/plugin/debrief/graph**', (route) => - route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify({ nodes: [], links: [] }), - }) - ); - await page.route('**/plugin/debrief/sections', (route) => - route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify(MOCK_SECTIONS), - }) - ); - await page.route('**/plugin/debrief/logos', (route) => - route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify({ logos: [] }), - }) - ); + await mockDebriefRoutes(page); await navigateToDebrief(page); - const select = page.locator('select').first(); - await select.selectOption({ index: 1 }); - await page.waitForTimeout(1000); + await selectFirstOperation(page); await page.locator('button', { hasText: 'Download PDF Report' }).click(); await expect(page.locator('h5', { hasText: 'Report Sections' })).toBeVisible({ timeout: 5_000 }); }); test('report modal should show custom logo checkbox', async ({ page }) => { - await page.route('**/api/v2/operations', (route) => - route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify(MOCK_OPERATIONS), - }) - ); - await page.route('**/plugin/debrief/report', (route) => - route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify(MOCK_REPORT), - }) - ); - await page.route('**/plugin/debrief/graph**', (route) => - route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify({ nodes: [], links: [] }), - }) - ); - await page.route('**/plugin/debrief/sections', (route) => - route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify(MOCK_SECTIONS), - }) - ); - await page.route('**/plugin/debrief/logos', (route) => - route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify({ logos: [] }), - }) - ); + await mockDebriefRoutes(page); await navigateToDebrief(page); - const select = page.locator('select').first(); - await select.selectOption({ index: 1 }); - await page.waitForTimeout(1000); + await selectFirstOperation(page); await page.locator('button', { hasText: 'Download PDF Report' }).click(); await expect(page.locator('text=Use custom logo')).toBeVisible({ timeout: 5_000 }); @@ -431,60 +228,20 @@ test.describe('Debrief PDF download', () => { test('report modal Download button triggers PDF API call', async ({ page }) => { let pdfRequested = false; - await page.route('**/api/v2/operations', (route) => - route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify(MOCK_OPERATIONS), - }) - ); - await page.route('**/plugin/debrief/report', (route) => - route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify(MOCK_REPORT), - }) - ); - await page.route('**/plugin/debrief/graph**', (route) => - route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify({ nodes: [], links: [] }), - }) - ); - await page.route('**/plugin/debrief/sections', (route) => - route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify(MOCK_SECTIONS), - }) - ); - await page.route('**/plugin/debrief/logos', (route) => - route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify({ logos: [] }), - }) - ); - await page.route('**/plugin/debrief/pdf', (route) => { - pdfRequested = true; - return route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify({ filename: 'debrief-report.pdf', pdf_bytes: '' }), - }); + await mockDebriefRoutes(page, { + '**/plugin/debrief/pdf': (route) => { + pdfRequested = true; + return route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify({ filename: 'debrief-report.pdf', pdf_bytes: '' }) }); + }, }); await navigateToDebrief(page); - const select = page.locator('select').first(); - await select.selectOption({ index: 1 }); - await page.waitForTimeout(1000); + await selectFirstOperation(page); await page.locator('button', { hasText: 'Download PDF Report' }).click(); - await page.waitForTimeout(500); - // Click the Download button inside the modal + await page.locator('.modal.is-active').waitFor({ state: 'visible', timeout: 5_000 }); const downloadBtn = page.locator('.modal.is-active button', { hasText: 'Download' }); await downloadBtn.click(); - await page.waitForTimeout(2000); + await page.waitForResponse((resp) => resp.url().includes('/plugin/debrief/pdf'), { timeout: 10_000 }).catch(() => {}); expect(pdfRequested).toBeTruthy(); }); }); @@ -495,62 +252,22 @@ test.describe('Debrief PDF download', () => { test.describe('Debrief JSON export', () => { test('JSON button should be disabled without operation selection', async ({ page }) => { await navigateToDebrief(page); - const btn = page.locator('button', { hasText: 'Download Operation JSON' }); - await expect(btn).toBeDisabled({ timeout: 15_000 }); + await expect(page.locator('button', { hasText: 'Download Operation JSON' })).toBeDisabled({ timeout: 15_000 }); }); test('clicking JSON button should trigger JSON API call', async ({ page }) => { let jsonRequested = false; - await page.route('**/api/v2/operations', (route) => - route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify(MOCK_OPERATIONS), - }) - ); - await page.route('**/plugin/debrief/report', (route) => - route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify(MOCK_REPORT), - }) - ); - await page.route('**/plugin/debrief/graph**', (route) => - route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify({ nodes: [], links: [] }), - }) - ); - await page.route('**/plugin/debrief/sections', (route) => - route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify(MOCK_SECTIONS), - }) - ); - await page.route('**/plugin/debrief/logos', (route) => - route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify({ logos: [] }), - }) - ); - await page.route('**/plugin/debrief/json', (route) => { - jsonRequested = true; - return route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify({ filename: 'debrief-report', operations: [] }), - }); + await mockDebriefRoutes(page, { + '**/plugin/debrief/json': (route) => { + jsonRequested = true; + return route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify({ filename: 'debrief-report', operations: [] }) }); + }, }); await navigateToDebrief(page); - const select = page.locator('select').first(); - await select.selectOption({ index: 1 }); - await page.waitForTimeout(1000); + await selectFirstOperation(page); await page.locator('button', { hasText: 'Download Operation JSON' }).click(); - await page.waitForTimeout(2000); + await page.waitForResponse((resp) => resp.url().includes('/plugin/debrief/json'), { timeout: 10_000 }).catch(() => {}); expect(jsonRequested).toBeTruthy(); }); }); @@ -583,7 +300,7 @@ test.describe('Debrief D3 graph rendering', () => { test('should have playback control buttons', async ({ page }) => { await navigateToDebrief(page); - // Playback buttons: fast-backward, backward, play/pause, forward, fast-forward + // Playback buttons: fast-backward, backward, play/pause, forward, fast-forward, plus legend toggle const buttons = page.locator('#debrief-graph .buttons button'); await expect(buttons).toHaveCount(6, { timeout: 15_000 }); }); @@ -595,45 +312,9 @@ test.describe('Debrief D3 graph rendering', () => { }); test('graph settings modal should show display options', async ({ page }) => { - await page.route('**/api/v2/operations', (route) => - route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify(MOCK_OPERATIONS), - }) - ); - await page.route('**/plugin/debrief/report', (route) => - route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify(MOCK_REPORT), - }) - ); - await page.route('**/plugin/debrief/graph**', (route) => - route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify({ nodes: [], links: [] }), - }) - ); - await page.route('**/plugin/debrief/sections', (route) => - route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify(MOCK_SECTIONS), - }) - ); - await page.route('**/plugin/debrief/logos', (route) => - route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify({ logos: [] }), - }) - ); + await mockDebriefRoutes(page); await navigateToDebrief(page); - const select = page.locator('select').first(); - await select.selectOption({ index: 1 }); - await page.waitForTimeout(1000); + await selectFirstOperation(page); await page.locator('button', { hasText: 'Graph Settings' }).click(); await expect(page.locator('.modal.is-active')).toBeVisible({ timeout: 5_000 }); @@ -651,202 +332,69 @@ test.describe('Debrief D3 graph rendering', () => { // =========================================================================== test.describe('Debrief error states', () => { test('should handle no operations gracefully', async ({ page }) => { - await page.route('**/api/v2/operations', (route) => - route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify([]), - }) - ); - await page.route('**/plugin/debrief/sections', (route) => - route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify(MOCK_SECTIONS), - }) - ); - await page.route('**/plugin/debrief/logos', (route) => - route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify({ logos: [] }), - }) - ); + await mockDebriefRoutes(page, { + '**/api/v2/operations': (route) => + route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify([]) }), + }); await navigateToDebrief(page); - // Heading should still show - await expect(page.locator('h2', { hasText: 'Debrief' })).toBeVisible({ timeout: 15_000 }); - // No options in dropdown + await expect(page.locator('h2', { hasText: 'Debrief' })).toBeVisible(); const options = page.locator('select option:not([disabled])'); await expect(options).toHaveCount(0); }); test('should handle operations API failure gracefully', async ({ page }) => { - await page.route('**/api/v2/operations', (route) => - route.fulfill({ status: 500, body: 'Server Error' }) - ); - await page.route('**/plugin/debrief/sections', (route) => - route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify(MOCK_SECTIONS), - }) - ); + await mockDebriefRoutes(page, { + '**/api/v2/operations': (route) => + route.fulfill({ status: 500, body: 'Server Error' }), + }); await navigateToDebrief(page); - // Page should still render without crashing - await expect(page.locator('h2', { hasText: 'Debrief' })).toBeVisible({ timeout: 15_000 }); + await expect(page.locator('h2', { hasText: 'Debrief' })).toBeVisible(); }); test('should handle report API failure gracefully', async ({ page }) => { - await page.route('**/api/v2/operations', (route) => - route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify(MOCK_OPERATIONS), - }) - ); - await page.route('**/plugin/debrief/report', (route) => - route.fulfill({ status: 500, body: 'Report generation failed' }) - ); - await page.route('**/plugin/debrief/graph**', (route) => - route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify({ nodes: [], links: [] }), - }) - ); - await page.route('**/plugin/debrief/sections', (route) => - route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify(MOCK_SECTIONS), - }) - ); - await page.route('**/plugin/debrief/logos', (route) => - route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify({ logos: [] }), - }) - ); + await mockDebriefRoutes(page, { + '**/plugin/debrief/report': (route) => + route.fulfill({ status: 500, body: 'Report generation failed' }), + }); await navigateToDebrief(page); - const select = page.locator('select').first(); - await select.selectOption({ index: 1 }); - await page.waitForTimeout(2000); - // Page should not crash + await selectFirstOperation(page); await expect(page.locator('h2', { hasText: 'Debrief' })).toBeVisible(); }); test('should handle PDF download failure gracefully', async ({ page }) => { - await page.route('**/api/v2/operations', (route) => - route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify(MOCK_OPERATIONS), - }) - ); - await page.route('**/plugin/debrief/report', (route) => - route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify(MOCK_REPORT), - }) - ); - await page.route('**/plugin/debrief/graph**', (route) => - route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify({ nodes: [], links: [] }), - }) - ); - await page.route('**/plugin/debrief/sections', (route) => - route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify(MOCK_SECTIONS), - }) - ); - await page.route('**/plugin/debrief/logos', (route) => - route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify({ logos: [] }), - }) - ); - await page.route('**/plugin/debrief/pdf', (route) => - route.fulfill({ status: 500, body: 'PDF generation failed' }) - ); + await mockDebriefRoutes(page, { + '**/plugin/debrief/pdf': (route) => + route.fulfill({ status: 500, body: 'PDF generation failed' }), + }); await navigateToDebrief(page); - const select = page.locator('select').first(); - await select.selectOption({ index: 1 }); - await page.waitForTimeout(1000); + await selectFirstOperation(page); + await page.locator('button', { hasText: 'Download PDF Report' }).click(); - await page.waitForTimeout(500); + await page.locator('.modal.is-active').waitFor({ state: 'visible', timeout: 5_000 }); const downloadBtn = page.locator('.modal.is-active button', { hasText: 'Download' }); await downloadBtn.click(); - await page.waitForTimeout(2000); - // Page should not crash + await page.waitForResponse((resp) => resp.url().includes('/plugin/debrief/pdf'), { timeout: 10_000 }).catch(() => {}); await expect(page.locator('h2', { hasText: 'Debrief' })).toBeVisible(); }); test('should handle JSON export failure gracefully', async ({ page }) => { - await page.route('**/api/v2/operations', (route) => - route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify(MOCK_OPERATIONS), - }) - ); - await page.route('**/plugin/debrief/report', (route) => - route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify(MOCK_REPORT), - }) - ); - await page.route('**/plugin/debrief/graph**', (route) => - route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify({ nodes: [], links: [] }), - }) - ); - await page.route('**/plugin/debrief/sections', (route) => - route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify(MOCK_SECTIONS), - }) - ); - await page.route('**/plugin/debrief/logos', (route) => - route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify({ logos: [] }), - }) - ); - await page.route('**/plugin/debrief/json', (route) => - route.fulfill({ status: 500, body: 'JSON export failed' }) - ); + await mockDebriefRoutes(page, { + '**/plugin/debrief/json': (route) => + route.fulfill({ status: 500, body: 'JSON export failed' }), + }); await navigateToDebrief(page); - const select = page.locator('select').first(); - await select.selectOption({ index: 1 }); - await page.waitForTimeout(1000); + await selectFirstOperation(page); + await page.locator('button', { hasText: 'Download Operation JSON' }).click(); - await page.waitForTimeout(2000); - // Page should not crash + await page.waitForResponse((resp) => resp.url().includes('/plugin/debrief/json'), { timeout: 10_000 }).catch(() => {}); await expect(page.locator('h2', { hasText: 'Debrief' })).toBeVisible(); }); test('should handle network timeout on operations API', async ({ page }) => { - await page.route('**/api/v2/operations', (route) => route.abort('timedout')); - await page.route('**/plugin/debrief/sections', (route) => - route.fulfill({ - status: 200, - contentType: 'application/json', - body: JSON.stringify(MOCK_SECTIONS), - }) - ); + await mockDebriefRoutes(page, { + '**/api/v2/operations': (route) => route.abort('timedout'), + }); await navigateToDebrief(page); - await expect(page.locator('h2', { hasText: 'Debrief' })).toBeVisible({ timeout: 15_000 }); + await expect(page.locator('h2', { hasText: 'Debrief' })).toBeVisible(); }); }); From 8667bedad6ed807d46b9e0456c6c6e71e4c99fc6 Mon Sep 17 00:00:00 2001 From: deacon Date: Mon, 16 Mar 2026 18:02:54 -0400 Subject: [PATCH 3/3] Address Copilot review: require credentials in CI, default only locally MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In CI (process.env.CI), CALDERA_USER and CALDERA_PASS must be set explicitly via env vars — config throws if missing. Locally, defaults to admin/admin for developer convenience. --- playwright.config.js | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/playwright.config.js b/playwright.config.js index 6d89a67..cf06061 100644 --- a/playwright.config.js +++ b/playwright.config.js @@ -2,6 +2,12 @@ const { defineConfig, devices } = require('@playwright/test'); const CALDERA_URL = process.env.CALDERA_URL || 'http://localhost:8888'; +const CALDERA_USER = process.env.CALDERA_USER || (process.env.CI ? undefined : 'admin'); +const CALDERA_PASS = process.env.CALDERA_PASS || (process.env.CI ? undefined : 'admin'); + +if (process.env.CI && (!CALDERA_USER || !CALDERA_PASS)) { + throw new Error('CALDERA_USER and CALDERA_PASS must be set in CI'); +} module.exports = defineConfig({ testDir: './tests/e2e', @@ -17,8 +23,8 @@ module.exports = defineConfig({ screenshot: 'only-on-failure', headless: true, httpCredentials: { - username: process.env.CALDERA_USER || 'admin', - password: process.env.CALDERA_PASS || 'admin', + username: CALDERA_USER, + password: CALDERA_PASS, }, }, projects: [