From b57ed7d696b2b676b1e421d4d1f099569a312298 Mon Sep 17 00:00:00 2001 From: Alexander Prokhorov Date: Fri, 8 May 2026 19:25:28 +0400 Subject: [PATCH 1/3] Inline mocha-reporter-spec-with-retries package --- .../mocha-reporter-spec-with-retries.js | 165 ++++++++++++++++++ gulp/helpers/test-functional.js | 14 +- .../suite1.js | 45 +++++ .../suite2.js | 45 +++++ .../mocha-reporter-spec-with-retries-test.js | 54 ++++++ 5 files changed, 317 insertions(+), 6 deletions(-) create mode 100644 gulp/helpers/mocha-reporter-spec-with-retries.js create mode 100644 test/server/data/mocha-reporter-spec-with-retries/suite1.js create mode 100644 test/server/data/mocha-reporter-spec-with-retries/suite2.js create mode 100644 test/server/mocha-reporter-spec-with-retries-test.js diff --git a/gulp/helpers/mocha-reporter-spec-with-retries.js b/gulp/helpers/mocha-reporter-spec-with-retries.js new file mode 100644 index 00000000000..dd3d4c50882 --- /dev/null +++ b/gulp/helpers/mocha-reporter-spec-with-retries.js @@ -0,0 +1,165 @@ +const Mocha = require('mocha'); +const milliseconds = require('ms'); + +const { + EVENT_RUN_BEGIN, + EVENT_RUN_END, + EVENT_SUITE_BEGIN, + EVENT_SUITE_END, + EVENT_TEST_FAIL, + EVENT_TEST_PASS, + EVENT_TEST_PENDING, + EVENT_TEST_RETRY, +} = Mocha.Runner.constants; + +const { inherits } = Mocha.utils; + +const Base = Mocha.reporters.Base; +const color = Base.color; + +exports = module.exports = SpecWithRetries; + +function SpecWithRetries (runner, options) { + Base.call(this, runner, options); + + this.stats.unstables = []; + + const self = this; + let indents = 0; + let n = 0; + + function indent () { + return Array(indents).join(' '); + } + + function groupBy (collection, predicate) { + return collection.reduce((r, v, i, a, k = predicate(v)) => ((r[k] || (r[k] = [])).push(v), r), {}); // eslint-disable-line no-sequences + } + + function epilogue () { + const stats = this.stats; + let fmt; + + Base.consoleLog(); + + // passes + fmt = + color('bright pass', ' ') + + color('green', ' %d passing') + + color('light', ' (%s)'); + + Base.consoleLog(fmt, stats.passes || 0, milliseconds(stats.duration)); + + // pending + if (stats.pending) { + fmt = color('pending', ' ') + color('pending', ' %d pending'); + + Base.consoleLog(fmt, stats.pending); + } + + // failures + if (stats.failures) { + fmt = color('fail', ' %d failing'); + + Base.consoleLog(fmt, stats.failures); + + Base.list(this.failures); + } + + // unstable tests + if (stats.unstables.length) { + Base.consoleLog(); + + fmt = color('bright yellow', ' Unstable test(s):'); + + Base.consoleLog(fmt); + + const groupedByFile = groupBy(stats.unstables, unstable => unstable.file); + + Object.entries(groupedByFile) + .forEach(([key, value]) => { + Base.consoleLog(color('bright yellow', ' %s'), key); + + value.forEach(unstableTest => { + Base.consoleLog(color('bright yellow', ' %s'), unstableTest.title); + }); + }); + } + + Base.consoleLog(); + } + + function findTestIndex (collection, test) { + return collection.findIndex(item => { + return item.file === test.file && + item.title === test.title; + }); + } + + function isInUnstables (test) { + return findTestIndex(this.stats.unstables, test) > -1; + } + + runner.on(EVENT_RUN_BEGIN, function () { + Base.consoleLog(); + }); + + runner.on(EVENT_SUITE_BEGIN, function (suite) { + ++indents; + Base.consoleLog(color('suite', '%s%s'), indent(), suite.title); + }); + + runner.on(EVENT_SUITE_END, function () { + --indents; + if (indents === 1) + Base.consoleLog(); + }); + + runner.on(EVENT_TEST_PENDING, function (test) { + const fmt = indent() + color('pending', ' - %s'); + + Base.consoleLog(fmt, test.title); + }); + + runner.on(EVENT_TEST_PASS, function (test) { + let fmt; + + if (test.speed === 'fast') { + fmt = + indent() + + color('checkmark', ' ' + Base.symbols.ok) + + color('pass', ' %s'); + Base.consoleLog(fmt, test.title); + } + else { + fmt = + indent() + + color('checkmark', ' ' + Base.symbols.ok) + + color('pass', ' %s') + + color(test.speed, ' (%dms)'); + Base.consoleLog(fmt, test.title, test.duration); + } + }); + + runner.on(EVENT_TEST_FAIL, function (test) { + Base.consoleLog(indent() + color('fail', ' %d) %s'), ++n, test.title); + + const index = findTestIndex(this.stats.unstables, test); + + if (index > -1) + this.stats.unstables.splice(index, 1); + }); + + runner.on(EVENT_TEST_RETRY, test => { + if (!isInUnstables.call(self, test)) + this.stats.unstables.push(test); + }); + + runner.once(EVENT_RUN_END, () => { + epilogue.call(self); + }); +} + +inherits(SpecWithRetries, Base); + +SpecWithRetries.description = 'hierarchical & verbose & display retried tests'; diff --git a/gulp/helpers/test-functional.js b/gulp/helpers/test-functional.js index 7e36e1f17fe..c6d3198f3a7 100644 --- a/gulp/helpers/test-functional.js +++ b/gulp/helpers/test-functional.js @@ -1,8 +1,9 @@ -const { castArray } = require('lodash'); -const getTimeout = require('./get-timeout'); -const chai = require('chai'); -const globby = require('globby'); -const Mocha = require('mocha'); +const { castArray } = require('lodash'); +const getTimeout = require('./get-timeout'); +const chai = require('chai'); +const globby = require('globby'); +const Mocha = require('mocha'); +const SpecWithRetries = require('./mocha-reporter-spec-with-retries'); const { TESTS_GLOB, @@ -51,7 +52,8 @@ module.exports = async function testFunctional (src, testingEnvironmentName, { n tests.unshift(SETUP_TESTS_GLOB); const opts = { - timeout: getTimeout(3 * 60 * 1000), + reporter: SpecWithRetries, + timeout: getTimeout(3 * 60 * 1000), }; if (process.env.RETRY_FAILED_TESTS === 'true') diff --git a/test/server/data/mocha-reporter-spec-with-retries/suite1.js b/test/server/data/mocha-reporter-spec-with-retries/suite1.js new file mode 100644 index 00000000000..09e1c6c21c3 --- /dev/null +++ b/test/server/data/mocha-reporter-spec-with-retries/suite1.js @@ -0,0 +1,45 @@ +const assert = require('assert'); + +describe('Test suite 1', function () { + let unstable1RunCount = 0; + let unstable2RunCount = 0; + + this.retries(3); + + it('Passed', () => { + assert.ok(true); + }); + + it('Failed', () => { + let isCorrect = false; + + try { + assert.ok(false); + } + catch (_) { + isCorrect = true; + } + + assert.ok(isCorrect); + }); + + it('Pending'); + + it('Unstable - 1', () => { + unstable1RunCount++; + + if (unstable1RunCount === 2) + assert.ok(true); + else + assert.ok(false); + }); + + it('Unstable - 2', () => { + unstable2RunCount++; + + if (unstable2RunCount === 2) + assert.ok(true); + else + assert.ok(false); + }); +}); diff --git a/test/server/data/mocha-reporter-spec-with-retries/suite2.js b/test/server/data/mocha-reporter-spec-with-retries/suite2.js new file mode 100644 index 00000000000..fb788a8b696 --- /dev/null +++ b/test/server/data/mocha-reporter-spec-with-retries/suite2.js @@ -0,0 +1,45 @@ +const assert = require('assert'); + +describe('Test suite 2', function () { + let unstable1RunCount = 0; + let unstable2RunCount = 0; + + this.retries(3); + + it('Passed', () => { + assert.ok(true); + }); + + it('Failed', () => { + let isCorrect = false; + + try { + assert.ok(false); + } + catch (_) { + isCorrect = true; + } + + assert.ok(isCorrect); + }); + + it('Pending'); + + it('Unstable - 1', () => { + unstable1RunCount++; + + if (unstable1RunCount === 2) + assert.ok(true); + else + assert.ok(false); + }); + + it('Unstable - 2', () => { + unstable2RunCount++; + + if (unstable2RunCount === 2) + assert.ok(true); + else + assert.ok(false); + }); +}); diff --git a/test/server/mocha-reporter-spec-with-retries-test.js b/test/server/mocha-reporter-spec-with-retries-test.js new file mode 100644 index 00000000000..e5d044d09cd --- /dev/null +++ b/test/server/mocha-reporter-spec-with-retries-test.js @@ -0,0 +1,54 @@ +const { expect } = require('chai'); +const path = require('path'); +const util = require('util'); +const Mocha = require('mocha'); +const SpecWithRetries = require('../../gulp/helpers/mocha-reporter-spec-with-retries'); + +function runSuitesWithReporter (files) { + return new Promise((resolve, reject) => { + const output = []; + const originalConsoleLog = Mocha.reporters.Base.consoleLog; + const mocha = new Mocha({ + reporter: SpecWithRetries, + timeout: 2000, + color: false, + }); + + files.forEach(file => { + mocha.addFile(file); + }); + + Mocha.reporters.Base.consoleLog = (format, ...args) => { + if (!format) + output.push(''); + else + output.push(util.format(format, ...args)); + }; + + mocha.run(failures => { + Mocha.reporters.Base.consoleLog = originalConsoleLog; + + if (failures) + reject(new Error(`${failures} test(s) failed`)); + else + resolve(output.join('\n')); + }); + }); +} + +describe('Mocha reporter spec with retries', () => { + it('Should include unstable test section grouped by source file', async () => { + const dataDir = path.join(__dirname, 'data/mocha-reporter-spec-with-retries'); + const report = await runSuitesWithReporter([ + path.join(dataDir, 'suite1.js'), + path.join(dataDir, 'suite2.js'), + ]); + + expect(report).contains('Unstable test(s):'); + expect(report).match(/suite1\.js/); + expect(report).match(/suite2\.js/); + + expect((report.match(/Unstable - 1/g) || []).length).gte(2); + expect((report.match(/Unstable - 2/g) || []).length).gte(2); + }); +}); From cc372f76b9787fe57341031fe1525cd90713d3e5 Mon Sep 17 00:00:00 2001 From: Alexander Prokhorov Date: Fri, 8 May 2026 20:13:10 +0400 Subject: [PATCH 2/3] copilot notes fixed --- .../mocha-reporter-spec-with-retries.js | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/gulp/helpers/mocha-reporter-spec-with-retries.js b/gulp/helpers/mocha-reporter-spec-with-retries.js index dd3d4c50882..adfbb245c94 100644 --- a/gulp/helpers/mocha-reporter-spec-with-retries.js +++ b/gulp/helpers/mocha-reporter-spec-with-retries.js @@ -1,5 +1,11 @@ -const Mocha = require('mocha'); -const milliseconds = require('ms'); +const Mocha = require('mocha'); + +function formatMilliseconds (duration) { + if (duration < 1000) + return `${duration}ms`; + + return `${(duration / 1000).toFixed(2).replace(/\.?0+$/, '')}s`; +} const { EVENT_RUN_BEGIN, @@ -48,7 +54,7 @@ function SpecWithRetries (runner, options) { color('green', ' %d passing') + color('light', ' (%s)'); - Base.consoleLog(fmt, stats.passes || 0, milliseconds(stats.duration)); + Base.consoleLog(fmt, stats.passes || 0, formatMilliseconds(stats.duration)); // pending if (stats.pending) { @@ -144,15 +150,15 @@ function SpecWithRetries (runner, options) { runner.on(EVENT_TEST_FAIL, function (test) { Base.consoleLog(indent() + color('fail', ' %d) %s'), ++n, test.title); - const index = findTestIndex(this.stats.unstables, test); + const index = findTestIndex(self.stats.unstables, test); if (index > -1) - this.stats.unstables.splice(index, 1); + self.stats.unstables.splice(index, 1); }); runner.on(EVENT_TEST_RETRY, test => { if (!isInUnstables.call(self, test)) - this.stats.unstables.push(test); + self.stats.unstables.push(test); }); runner.once(EVENT_RUN_END, () => { @@ -162,4 +168,4 @@ function SpecWithRetries (runner, options) { inherits(SpecWithRetries, Base); -SpecWithRetries.description = 'hierarchical & verbose & display retried tests'; +SpecWithRetries.description = 'hierarchical & verbose & displays retried tests'; From eca7f2b1eab24e183db60a2defe91e40f1637527 Mon Sep 17 00:00:00 2001 From: Alexander Prokhorov Date: Mon, 11 May 2026 12:24:00 +0400 Subject: [PATCH 3/3] try to stable iframe switchig tests --- .../api/es-next/iframe-switching/pages/second.html | 2 ++ .../testcafe-fixtures/iframe-switching-test.js | 7 +++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/test/functional/fixtures/api/es-next/iframe-switching/pages/second.html b/test/functional/fixtures/api/es-next/iframe-switching/pages/second.html index bf316045d96..a10474cbbf7 100644 --- a/test/functional/fixtures/api/es-next/iframe-switching/pages/second.html +++ b/test/functional/fixtures/api/es-next/iframe-switching/pages/second.html @@ -7,6 +7,8 @@