diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 3d4f849ba..e330b87cc 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -231,10 +231,10 @@ jobs: CDP_WALLET_SECRET: ${{ env.E2E_CDP_WALLET_SECRET }} GA_EXTRA: ${{ steps.changed.outputs.ga_extra }} # GA_EXTRA is a space-separated test-file list; left unquoted intentionally so it word-splits into vitest args. - # --reporter=verbose keeps the console log; --reporter=junit adds a machine-readable - # report surfaced in the GH Actions test-summary tab and uploaded as an artifact below. + # --reporter=verbose keeps the sanitized console log; the custom JUnit reporter adds a + # machine-readable report without copying console streams into the public artifact. run: - npx vitest run --project e2e --reporter=verbose --reporter=junit + npx vitest run --project e2e --reporter=verbose --reporter=./scripts/safe-junit-reporter.ts --outputFile.junit=test-results/e2e-ga.junit.xml e2e-tests/strands-bedrock.test.ts e2e-tests/payment-strands-bedrock.test.ts $GA_EXTRA @@ -253,11 +253,21 @@ jobs: # HARNESS_EXTRA is a space-separated test-file list; left unquoted intentionally so it word-splits into vitest args. # Separate --outputFile from the GA step so the two vitest runs don't clobber each other's report. run: - npx vitest run --project e2e --reporter=verbose --reporter=junit + npx vitest run --project e2e --reporter=verbose --reporter=./scripts/safe-junit-reporter.ts --outputFile.junit=test-results/e2e-harness.junit.xml e2e-tests/harness-bedrock.test.ts $HARNESS_EXTRA - - name: Upload E2E test report + - name: Sanitize E2E test reports + id: sanitize-reports + if: always() + run: npm run test-artifacts:sanitize -- test-results/ + + - name: Verify E2E test reports contain no secrets + id: scan-reports if: always() + run: npm run test-artifacts:check -- test-results/ + + - name: Upload E2E test report + if: always() && steps.sanitize-reports.outcome == 'success' && steps.scan-reports.outcome == 'success' uses: actions/upload-artifact@v7 with: name: e2e-test-report diff --git a/package.json b/package.json index 792c59891..ba2d20dc9 100644 --- a/package.json +++ b/package.json @@ -68,6 +68,8 @@ "test:integ": "npm run build && vitest run --project integ", "test:unit": "vitest run --project unit --coverage", "test:e2e": "vitest run --project e2e", + "test-artifacts:sanitize": "tsx scripts/sanitize-test-artifacts.ts --redact", + "test-artifacts:check": "tsx scripts/sanitize-test-artifacts.ts --check", "test:update-snapshots": "vitest run --project unit --update", "test:tui": "npm run build:harness && vitest run --project tui", "test:browser": "npx playwright test --config browser-tests/playwright.config.ts", diff --git a/scripts/safe-junit-reporter.ts b/scripts/safe-junit-reporter.ts new file mode 100644 index 000000000..513b266d8 --- /dev/null +++ b/scripts/safe-junit-reporter.ts @@ -0,0 +1,7 @@ +import { JUnitReporter } from 'vitest/node'; + +export default class SafeJUnitReporter extends JUnitReporter { + constructor() { + super({ includeConsoleOutput: false }); + } +} diff --git a/scripts/sanitize-test-artifacts.ts b/scripts/sanitize-test-artifacts.ts new file mode 100644 index 000000000..891bcb513 --- /dev/null +++ b/scripts/sanitize-test-artifacts.ts @@ -0,0 +1,65 @@ +import { redactTestOutput, sensitiveEnvironmentValues } from '../src/test-utils/test-output-redaction'; +import { lstat, readFile, readdir, writeFile } from 'node:fs/promises'; +import { relative, resolve } from 'node:path'; + +type Mode = 'check' | 'redact'; + +async function collectFiles(inputPath: string): Promise { + const absolutePath = resolve(inputPath); + + try { + const fileStat = await lstat(absolutePath); + if (fileStat.isSymbolicLink()) { + throw new Error(`Symbolic links are not allowed in test artifacts: ${inputPath}`); + } + if (fileStat.isFile()) return [absolutePath]; + if (!fileStat.isDirectory()) return []; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return []; + throw error; + } + + const entries = await readdir(absolutePath, { withFileTypes: true }); + const files = await Promise.all(entries.map(entry => collectFiles(resolve(absolutePath, entry.name)))); + return files.flat(); +} + +async function main(): Promise { + const [modeArgument, ...inputPaths] = process.argv.slice(2); + if ((modeArgument !== '--redact' && modeArgument !== '--check') || inputPaths.length === 0) { + throw new Error('Usage: tsx scripts/sanitize-test-artifacts.ts <--redact|--check> '); + } + + const mode = modeArgument.slice(2) as Mode; + const files = (await Promise.all(inputPaths.map(collectFiles))).flat(); + const secretValues = sensitiveEnvironmentValues(); + const affectedFiles: string[] = []; + let redactionCount = 0; + + for (const file of files) { + const original = await readFile(file, 'utf8'); + const result = redactTestOutput(original, secretValues); + if (result.redactions === 0) continue; + + affectedFiles.push(relative(process.cwd(), file)); + redactionCount += result.redactions; + if (mode === 'redact') await writeFile(file, result.text, 'utf8'); + } + + if (mode === 'check' && affectedFiles.length > 0) { + console.error(`Sensitive output detected in ${affectedFiles.length} artifact file(s): ${affectedFiles.join(', ')}`); + process.exitCode = 1; + return; + } + + if (mode === 'redact') { + console.log(`Redacted ${redactionCount} sensitive value(s) from ${affectedFiles.length} artifact file(s).`); + } else { + console.log(`No sensitive output detected in ${files.length} artifact file(s).`); + } +} + +void main().catch((error: unknown) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; +}); diff --git a/src/test-utils/test-output-redaction.test.ts b/src/test-utils/test-output-redaction.test.ts new file mode 100644 index 000000000..74d0009f5 --- /dev/null +++ b/src/test-utils/test-output-redaction.test.ts @@ -0,0 +1,72 @@ +import { containsSensitiveTestOutput, redactTestOutput, sensitiveEnvironmentValues } from './test-output-redaction'; +import { describe, expect, it } from 'vitest'; + +describe('test output redaction', () => { + it('redacts supported provider, AWS, and GitHub credential formats', () => { + const credentials = [ + `AKIA${'A'.repeat(16)}`, + `AIza${'b'.repeat(35)}`, + `ghp_${'c'.repeat(36)}`, + `github_pat_${'d'.repeat(32)}`, + `sk-ant-${'e'.repeat(32)}`, + `sk-proj-${'f'.repeat(32)}`, + `eyJ${'g'.repeat(12)}.${'h'.repeat(12)}.${'i'.repeat(12)}`, + ]; + + const result = redactTestOutput(credentials.join('\n')); + + expect(result.redactions).toBe(credentials.length); + for (const credential of credentials) { + expect(result.text).not.toContain(credential); + } + }); + + it('redacts private keys, bearer tokens, and credential assignments', () => { + const privateKey = ['-----BEGIN PRIVATE KEY-----', 'not-real-key-material', '-----END PRIVATE KEY-----'].join('\n'); + const input = [privateKey, `Authorization: Bearer ${'a'.repeat(24)}`, `client_secret=${'b'.repeat(24)}`].join('\n'); + + const result = redactTestOutput(input); + + expect(result.redactions).toBe(3); + expect(result.text).not.toContain('not-real-key-material'); + expect(result.text).toContain('Authorization: Bearer [REDACTED]'); + expect(result.text).toContain('client_secret=[REDACTED]'); + }); + + it('redacts exact environment secrets without treating identifiers as secrets', () => { + const environment = { + OPENAI_API_KEY: 'provider-value-without-a-known-prefix', + APP_PRIVATE_KEY: 'private-key-value', + CDP_API_KEY_ID: 'non-secret-identifier', + APP_ID: 'non-secret-app-id', + SHORT_TOKEN: 'short', + }; + + const secretValues = sensitiveEnvironmentValues(environment); + const result = redactTestOutput(Object.values(environment).join(' '), secretValues); + + expect(secretValues).toEqual(['provider-value-without-a-known-prefix', 'private-key-value']); + expect(result.text).not.toContain('provider-value-without-a-known-prefix'); + expect(result.text).not.toContain('private-key-value'); + expect(result.text).toContain('non-secret-identifier'); + expect(result.text).toContain('non-secret-app-id'); + expect(result.text).toContain('short'); + }); + + it('leaves ordinary test output unchanged', () => { + const input = 'Harness deployment completed in us-east-1'; + + expect(redactTestOutput(input)).toEqual({ text: input, redactions: 0 }); + expect(containsSensitiveTestOutput(input)).toBe(false); + }); + + it('does not detect its own redaction marker', () => { + const firstPass = redactTestOutput(`api_key=${'x'.repeat(32)}`); + + expect(firstPass.redactions).toBe(1); + expect(redactTestOutput(firstPass.text)).toEqual({ + text: 'api_key=[REDACTED]', + redactions: 0, + }); + }); +}); diff --git a/src/test-utils/test-output-redaction.ts b/src/test-utils/test-output-redaction.ts new file mode 100644 index 000000000..6db97ed3b --- /dev/null +++ b/src/test-utils/test-output-redaction.ts @@ -0,0 +1,90 @@ +const REDACTED = '[REDACTED]'; +const MIN_SECRET_LENGTH = 8; +const PRIVATE_KEY_TYPES = ['', 'RSA ', 'EC ', 'OPENSSH '] as const; + +interface SecretPattern { + pattern: RegExp; + replacement?: (match: string, ...groups: string[]) => string; +} + +const SECRET_PATTERNS: SecretPattern[] = [ + { pattern: /\b(?:gh[pousr]_[A-Za-z0-9_]{20,255}|github_pat_[A-Za-z0-9_]{20,255})\b/g }, + { pattern: /\b(?:AKIA|ASIA)[A-Z0-9]{16}\b/g }, + { pattern: /\bAIza[0-9A-Za-z_-]{35}\b/g }, + { pattern: /\bsk-ant-[A-Za-z0-9_-]{20,}\b/g }, + { pattern: /\bsk-(?:proj-|svcacct-)?[A-Za-z0-9_-]{20,}\b/g }, + { pattern: /\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b/g }, + { + pattern: /\bbearer[ \t]+[A-Za-z0-9._~+/-]{16,2048}={0,2}/gi, + replacement: () => `Bearer ${REDACTED}`, + }, + { + pattern: + /(\b(?:api[_-]?key|access[_-]?key|client[_-]?secret|private[_-]?key|password|secret|session[_-]?token|token)\b\s*[:=]\s*["']?)([^"',;<>{}[\]\s]{8,})/gi, + replacement: (_match, prefix) => `${prefix}${REDACTED}`, + }, +]; + +const SENSITIVE_ENV_KEY = + /(?:^|_)(?:ACCESS_KEY_ID|API_KEY|API_KEY_SECRET|APP_PRIVATE_KEY|CLIENT_SECRET|PASSWORD|PRIVATE_KEY|SECRET|SECRET_ACCESS_KEY|SESSION_TOKEN|TOKEN|WALLET_SECRET)$/i; +const REFERENCE_ENV_KEY = /(?:^|_)(?:API_KEY_ID|APP_ID|AUTHORIZATION_ID|CLIENT_ID)$/i; + +export interface RedactionResult { + text: string; + redactions: number; +} + +export function sensitiveEnvironmentValues(environment: NodeJS.ProcessEnv = process.env): string[] { + const values = Object.entries(environment) + .filter((entry): entry is [string, string] => { + const [key, value] = entry; + return ( + typeof value === 'string' && + value.length >= MIN_SECRET_LENGTH && + SENSITIVE_ENV_KEY.test(key) && + !REFERENCE_ENV_KEY.test(key) + ); + }) + .map(([, value]) => value); + + return [...new Set(values)].sort((left, right) => right.length - left.length); +} + +export function redactTestOutput(input: string, secretValues: readonly string[] = []): RedactionResult { + let text = input; + let redactions = 0; + + for (const keyType of PRIVATE_KEY_TYPES) { + const beginMarker = `-----BEGIN ${keyType}PRIVATE KEY-----`; + const endMarker = `-----END ${keyType}PRIVATE KEY-----`; + let beginIndex = text.indexOf(beginMarker); + + while (beginIndex >= 0) { + const endIndex = text.indexOf(endMarker, beginIndex + beginMarker.length); + if (endIndex < 0) break; + text = text.slice(0, beginIndex) + REDACTED + text.slice(endIndex + endMarker.length); + redactions += 1; + beginIndex = text.indexOf(beginMarker, beginIndex + REDACTED.length); + } + } + + for (const secret of [...new Set(secretValues)].sort((left, right) => right.length - left.length)) { + if (secret.length < MIN_SECRET_LENGTH || !text.includes(secret)) continue; + const occurrences = text.split(secret).length - 1; + text = text.split(secret).join(REDACTED); + redactions += occurrences; + } + + for (const { pattern, replacement } of SECRET_PATTERNS) { + text = text.replace(pattern, (match: string, ...groups: string[]) => { + redactions += 1; + return replacement ? replacement(match, ...groups) : REDACTED; + }); + } + + return { text, redactions }; +} + +export function containsSensitiveTestOutput(input: string, secretValues: readonly string[] = []): boolean { + return redactTestOutput(input, secretValues).redactions > 0; +} diff --git a/vitest.config.ts b/vitest.config.ts index 44c08c51e..20b18ee63 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -1,3 +1,4 @@ +import { containsSensitiveTestOutput, sensitiveEnvironmentValues } from './src/test-utils/test-output-redaction'; import * as fs from 'fs'; import * as path from 'path'; import { defineConfig } from 'vitest/config'; @@ -25,6 +26,8 @@ const textLoaderPlugin = { }, }; +const testSecretValues = sensitiveEnvironmentValues(); + export default defineConfig({ resolve: { alias: { @@ -82,6 +85,9 @@ export default defineConfig({ hookTimeout: 120000, globals: false, reporters: ['verbose'], + onConsoleLog(log) { + return !containsSensitiveTestOutput(log, testSecretValues); + }, coverage: { provider: 'v8', reporter: ['text', 'text-summary', 'json', 'json-summary', 'html', 'lcov'],