From 18255126da343500e921db76757fd261b64e709f Mon Sep 17 00:00:00 2001 From: gmegidish Date: Tue, 8 Sep 2026 20:40:39 +0200 Subject: [PATCH] feat: add expect.soft for non-fatal assertions --- README.md | 4 + packages/mobilewright-core/src/expect.test.ts | 84 ++++++++++++++++- packages/mobilewright-core/src/expect.ts | 94 +++++++++++++++---- packages/mobilewright-core/src/index.ts | 2 +- packages/test/src/expect-soft.test.ts | 37 ++++++++ packages/test/src/fixtures.ts | 12 ++- 6 files changed, 211 insertions(+), 22 deletions(-) create mode 100644 packages/test/src/expect-soft.test.ts diff --git a/README.md b/README.md index 28c8e2b..3e39cee 100644 --- a/README.md +++ b/README.md @@ -323,6 +323,10 @@ await expect(locator).toHaveValue(/@example\.com$/); await expect(locator).toHaveCount(3); // number of matching elements await expect(locator).toBeVisible({ timeout: 10_000 }); + +// Soft assertions record the failure but let the test keep running (Playwright semantics) +await expect.soft(locator).toHaveText('Success'); +await expect.soft(locator, 'eta should be shown').not.toBeEmpty(); ``` ### Role Mapping diff --git a/packages/mobilewright-core/src/expect.test.ts b/packages/mobilewright-core/src/expect.test.ts index b8d9b59..78beba4 100644 --- a/packages/mobilewright-core/src/expect.test.ts +++ b/packages/mobilewright-core/src/expect.test.ts @@ -8,7 +8,7 @@ import type { } from '@mobilewright/protocol'; import { Locator } from './locator.js'; import type { StepFn } from './locator.js'; -import { expect as mwExpect, ExpectError } from './expect.js'; +import { expect as mwExpect, ExpectError, setSoftFailureHandler } from './expect.js'; function node( overrides: Partial & { type: string }, @@ -939,3 +939,85 @@ test.describe('custom message as the step title', () => { expect(titles).not.toContain('expect.not.toBeVisible()'); }); }); + +test.describe('expect.soft', () => { + const recorded: ExpectError[] = []; + const recordInsteadOfThrowing = (error: ExpectError): void => { recorded.push(error); }; + const throwLikeAHardAssertion = (error: ExpectError): void => { throw error; }; + + test.beforeEach(() => { + recorded.length = 0; + setSoftFailureHandler(recordInsteadOfThrowing); + }); + + test.afterAll(() => { + setSoftFailureHandler(throwLikeAHardAssertion); + }); + + test('hands a failed sync assertion to the handler instead of throwing', () => { + mwExpect.soft(1).toBe(2); + + expect(recorded).toHaveLength(1); + expect(recorded[0]).toBeInstanceOf(ExpectError); + expect(recorded[0].message).toContain('Expected 2, but received 1'); + }); + + test('hands a failed async locator assertion to the handler instead of rejecting', async () => { + const driver = createMockDriver(hierarchy); + const locator = new Locator(driver, { kind: 'testId', value: 'hiddenBtn' }); + + await mwExpect.soft(locator).toBeVisible({ timeout: 200 }); + + expect(recorded).toHaveLength(1); + }); + + test('keeps running so later soft failures are collected too', () => { + mwExpect.soft(1).toBe(2); + mwExpect.soft('a').toBe('b'); + + expect(recorded).toHaveLength(2); + }); + + test('records nothing when the assertion passes', async () => { + const driver = createMockDriver(hierarchy); + const locator = new Locator(driver, { kind: 'testId', value: 'submitBtn' }); + + mwExpect.soft(1).toBe(1); + await mwExpect.soft(locator).toBeVisible(); + + expect(recorded).toHaveLength(0); + }); + + test('survives negation via not', async () => { + const driver = createMockDriver(hierarchy); + const locator = new Locator(driver, { kind: 'testId', value: 'submitBtn' }); + + await mwExpect.soft(locator).not.toBeVisible({ timeout: 200 }); + + expect(recorded).toHaveLength(1); + }); + + test('keeps the custom message in the recorded failure', () => { + mwExpect.soft(1, 'counts must match').toBe(2); + + expect(recorded[0].message).toMatch(/^counts must match\n\nExpected 2, but received 1/); + }); + + test('marks the reporter step title as soft', async () => { + const { stepFn, titles } = recordingStepFn(); + const driver = createMockDriver(hierarchy); + const locator = new Locator(driver, { kind: 'testId', value: 'submitBtn' }); + locator._stepFn = stepFn; + + await mwExpect.soft(locator).toBeVisible(); + await mwExpect.soft(locator).not.toBeHidden(); + + expect(titles).toContain('expect.soft.toBeVisible()'); + expect(titles).toContain('expect.soft.not.toBeHidden()'); + }); + + test('throws like a hard assertion when no runner installed a handler', () => { + setSoftFailureHandler(throwLikeAHardAssertion); + expect(() => mwExpect.soft(1).toBe(2)).toThrow(ExpectError); + }); +}); diff --git a/packages/mobilewright-core/src/expect.ts b/packages/mobilewright-core/src/expect.ts index 15e65d8..061f94b 100644 --- a/packages/mobilewright-core/src/expect.ts +++ b/packages/mobilewright-core/src/expect.ts @@ -16,10 +16,23 @@ export interface ExpectOptions { // argument. Its object form also carries `timeout`, which we do not support yet. export type ExpectMessage = string | { message?: string }; -// Carries the custom message from the expect() Proxy down to wrapAssertion, so a -// message also renames the reporter step (as Playwright does). -interface MessageCarrier { +// Carries the custom message and soft flag from the expect() Proxies down to +// wrapAssertion, so both show up in the reporter step title (as Playwright does). +interface StepCarrier { _message?: string; + _soft?: boolean; +} + +// Called with the ExpectError of a failed expect.soft() assertion. The test runner +// installs a handler that records the failure and lets the test continue; without +// a runner, soft failures throw like hard ones. +export type SoftFailureHandler = (error: ExpectError) => void; + +// ponytail: module-level hook, only one runner per process +let softFailureHandler: SoftFailureHandler = (error) => { throw error; }; + +export function setSoftFailureHandler(handler: SoftFailureHandler): void { + softFailureHandler = handler; } /** @@ -31,6 +44,7 @@ interface MessageCarrier { * expect(webLocator).toHaveText('Hello') * expect(42).toBe(42) * expect(locator, 'checkout button should appear').toBeVisible() + * expect.soft(locator).toBeVisible() // records the failure, test keeps running */ export function expect(actual: Page, message?: ExpectMessage): PageAssertions; export function expect(actual: WebLocator, message?: ExpectMessage): WebLocatorAssertions; @@ -45,6 +59,29 @@ export function expect(actual: unknown, message?: ExpectMessage): any { return withMessage(assertions, resolved); } +function soft(actual: Page, message?: ExpectMessage): PageAssertions; +function soft(actual: WebLocator, message?: ExpectMessage): WebLocatorAssertions; +function soft(actual: Locator, message?: ExpectMessage): LocatorAssertions; +function soft(actual: T, message?: ExpectMessage): ValueAssertions; +function soft(actual: unknown, message?: ExpectMessage): any { + return withSoft(expect(actual as Page, message)); +} + +function withSoft(assertions: T): T { + (assertions as StepCarrier)._soft = true; + return interceptErrors(assertions, swallowSoftFailure, withSoft); +} + +function swallowSoftFailure(error: unknown): unknown { + if (error instanceof ExpectError) { + softFailureHandler(error); + return undefined; + } + return error; +} + +expect.soft = soft; + function createAssertions(actual: unknown): object { if (actual instanceof Page) { return new PageAssertions(actual, false); } if (actual instanceof WebLocator) { return new WebLocatorAssertions(actual, false); } @@ -54,22 +91,37 @@ function createAssertions(actual: unknown): object { return new ValueAssertions(actual, false); } -// Every failure funnels through ExpectError, so a single Proxy over the assertions -// object can prefix the custom message without touching any individual matcher. -// Matchers are a mix of sync (ValueAssertions) and async (everything else), hence -// both the try/catch and the promise catch. function withMessage(assertions: T, message: string): T { // Fresh instance per expect() call (and per `.not`), so tagging it is safe and // lets wrapAssertion use the message as the step title. - (assertions as MessageCarrier)._message = message; + (assertions as StepCarrier)._message = message; + return interceptErrors(assertions, (e) => prefixMessage(e, message), (child) => withMessage(child, message)); +} + +// Every failure funnels through ExpectError, so a single Proxy over the assertions +// object can post-process failures without touching any individual matcher. +// `onError` returns the error to throw, or undefined to swallow it (soft mode). +// Matchers are a mix of sync (ValueAssertions) and async (everything else), hence +// both the try/catch and the promise catch. +function interceptErrors( + assertions: T, + onError: (error: unknown) => unknown, + wrapChild: (child: object) => object = (child) => interceptErrors(child, onError), +): T { + const rethrow = (e: unknown): void => { + const error = onError(e); + if (error !== undefined) { + throw error; + } + }; return new Proxy(assertions, { get(target, prop, receiver): unknown { const value = Reflect.get(target, prop, receiver); - // `.not` returns another assertions object — re-wrap so the message survives chaining. + // `.not` returns another assertions object — re-wrap so the interception survives chaining. if (value !== null && typeof value === 'object') { - return withMessage(value, message); + return wrapChild(value); } if (typeof value !== 'function') { @@ -80,11 +132,12 @@ function withMessage(assertions: T, message: string): T { try { const result = Reflect.apply(value, target, args); if (result instanceof Promise) { - return result.catch((e: unknown) => { throw prefixMessage(e, message); }); + return result.catch(rethrow); } return result; } catch (e) { - throw prefixMessage(e, message); + rethrow(e); + return undefined; } }; }, @@ -119,10 +172,10 @@ function wrapAssertion( negated: boolean, method: string, fn: () => Promise, - message?: string, + carrier: StepCarrier, ): Promise { - const defaultTitle = negated ? `expect.not.${method}()` : `expect.${method}()`; - return runStep(stepFn, message ?? defaultTitle, fn); + const prefix = ['expect', carrier._soft && 'soft', negated && 'not'].filter(Boolean).join('.'); + return runStep(stepFn, carrier._message ?? `${prefix}.${method}()`, fn); } // Poll until `predicate` holds (or the timeout elapses), re-raising any failure @@ -146,8 +199,9 @@ class LocatorAssertions { protected readonly negated: boolean, ) {} - // Set by withMessage() when expect() was given a custom message. + // Set by withMessage() / withSoft() on the expect() Proxy. _message?: string; + _soft?: boolean; get not(): LocatorAssertions { return new LocatorAssertions(this.locator, !this.negated); @@ -158,7 +212,7 @@ class LocatorAssertions { } protected _wrapAssertion(method: string, fn: () => Promise): Promise { - return wrapAssertion(this.locator._stepFn, this.negated, method, fn, this._message); + return wrapAssertion(this.locator._stepFn, this.negated, method, fn, this); } async toBeVisible(opts?: ExpectOptions): Promise { @@ -513,13 +567,14 @@ class PageAssertions { // Set by withMessage() when expect() was given a custom message. _message?: string; + _soft?: boolean; get not(): PageAssertions { return new PageAssertions(this.page, !this.negated); } private _wrapAssertion(method: string, fn: () => Promise): Promise { - return wrapAssertion(this.page._stepFn, this.negated, method, fn, this._message); + return wrapAssertion(this.page._stepFn, this.negated, method, fn, this); } // Applies negation so callers pass the plain "does it match?" predicate. @@ -565,6 +620,7 @@ class WebLocatorAssertions { // Set by withMessage() when expect() was given a custom message. _message?: string; + _soft?: boolean; get not(): WebLocatorAssertions { return new WebLocatorAssertions(this.webLocator, !this.negated); @@ -600,7 +656,7 @@ class WebLocatorAssertions { : `Expected ${method} to match, but it did not (received ${got})`; }, ); - }, this._message); + }, this); } toBeVisible(opts?: ExpectOptions): Promise { diff --git a/packages/mobilewright-core/src/index.ts b/packages/mobilewright-core/src/index.ts index 9c3f8cd..b93801f 100644 --- a/packages/mobilewright-core/src/index.ts +++ b/packages/mobilewright-core/src/index.ts @@ -4,7 +4,7 @@ export { Screen, type GetByWebViewOptions } from './screen.js'; export { Device, type DeviceOptions } from './device.js'; export { MobileWebViewPage, MobileWebViewPage as Page } from './page.js'; export { MobileWebViewLocator, MobileWebViewLocator as WebLocator } from './web-locator.js'; -export { expect, ExpectError, type ExpectOptions } from './expect.js'; +export { expect, ExpectError, setSoftFailureHandler, type ExpectOptions, type SoftFailureHandler } from './expect.js'; export { queryAll, ROLE_TYPE_MAP, bareTypeName, type LocatorStrategy, type Role } from './query-engine.js'; export { sleep } from './sleep.js'; export type { HardwareButton } from '@mobilewright/protocol'; diff --git a/packages/test/src/expect-soft.test.ts b/packages/test/src/expect-soft.test.ts new file mode 100644 index 0000000..cc47387 --- /dev/null +++ b/packages/test/src/expect-soft.test.ts @@ -0,0 +1,37 @@ +import { test, expect } from './fixtures.js'; + +// A soft failure marks the test failed, so each test undoes that once it has +// inspected what the runner recorded. +function takeRecordedErrors(): { message?: string }[] { + const info = test.info(); + const errors = [...info.errors]; + info.errors.length = 0; + info.status = 'passed'; + return errors; +} + +test.describe('expect.soft with the Playwright runner', () => { + test('records the failure on the test and keeps running', async () => { + expect.soft(1).toBe(2); + const reachedNextLine = true; + + const errors = takeRecordedErrors(); + expect(reachedNextLine).toBe(true); + expect(errors).toHaveLength(1); + expect(errors[0].message).toContain('Expected 2, but received 1'); + }); + + test('collects every soft failure, not just the first', async () => { + expect.soft(1).toBe(2); + expect.soft('a', 'letters must match').toBe('b'); + + const errors = takeRecordedErrors(); + expect(errors).toHaveLength(2); + expect(errors[1].message).toContain('letters must match'); + }); + + test('records nothing when the soft assertion passes', async () => { + expect.soft(1).toBe(1); + expect(takeRecordedErrors()).toHaveLength(0); + }); +}); diff --git a/packages/test/src/fixtures.ts b/packages/test/src/fixtures.ts index 88f81b5..1575353 100644 --- a/packages/test/src/fixtures.ts +++ b/packages/test/src/fixtures.ts @@ -12,11 +12,21 @@ import { toArray, type DevicePoolClient, } from 'mobilewright'; -import { expect } from '@mobilewright/core'; +import { expect, setSoftFailureHandler } from '@mobilewright/core'; import type { Device, Screen } from '@mobilewright/core'; const debug = createDebug('mw:test:fixtures'); +// ponytail: same private Playwright API its own expect.soft goes through +interface SoftFailureReporter { + _failWithError(error: Error): void; +} + +// expect.soft(): mark the test failed and record the error, but keep running. +setSoftFailureHandler((error) => { + (base.info() as unknown as SoftFailureReporter)._failWithError(error); +}); + const ZIP_MAGIC = Buffer.from([0x50, 0x4B, 0x03, 0x04]); function assertValidZipFile(path: string): void {