Inline mocha-reporter-spec-with-retries package

gulp/helpers/mocha-reporter-spec-with-retries.js

@@ -0,0 +1,171 @@
+const Mocha = require('mocha');
+
+function formatMilliseconds (duration) {
+    if (duration < 1000)
+        return `${duration}ms`;
+
+    return `${(duration / 1000).toFixed(2).replace(/\.?0+$/, '')}s`;
+}
+
+const {
+    EVENT_RUN_BEGIN,
+    EVENT_RUN_END,
+    EVENT_SUITE_BEGIN,
+    EVENT_SUITE_END,
+    EVENT_TEST_FAIL,
+    EVENT_TEST_PASS,
+    EVENT_TEST_PENDING,
+    EVENT_TEST_RETRY,
+} = Mocha.Runner.constants;
+
+const { inherits } = Mocha.utils;
+
+const Base  = Mocha.reporters.Base;
+const color = Base.color;
+
+exports = module.exports = SpecWithRetries;
+
+function SpecWithRetries (runner, options) {
+    Base.call(this, runner, options);
+
+    this.stats.unstables = [];
+
+    const self = this;
+    let indents = 0;
+    let n       = 0;
+
+    function indent () {
+        return Array(indents).join('  ');
+    }
+
+    function groupBy (collection, predicate) {
+        return collection.reduce((r, v, i, a, k = predicate(v)) => ((r[k] || (r[k] = [])).push(v), r), {}); // eslint-disable-line no-sequences
+    }
+
+    function epilogue () {
+        const stats = this.stats;
+        let fmt;
+
+        Base.consoleLog();
+
+        // passes
+        fmt =
+            color('bright pass', ' ') +
+            color('green', ' %d passing') +
+            color('light', ' (%s)');
+
+        Base.consoleLog(fmt, stats.passes || 0, formatMilliseconds(stats.duration));
+
+        // pending
+        if (stats.pending) {
+            fmt = color('pending', ' ') + color('pending', ' %d pending');
+
+            Base.consoleLog(fmt, stats.pending);
+        }
+
+        // failures
+        if (stats.failures) {
+            fmt = color('fail', '  %d failing');
+
+            Base.consoleLog(fmt, stats.failures);
+
+            Base.list(this.failures);
+        }
+
+        // unstable tests
+        if (stats.unstables.length) {
+            Base.consoleLog();
+
+            fmt = color('bright yellow', '  Unstable test(s):');
+
+            Base.consoleLog(fmt);
+
+            const groupedByFile = groupBy(stats.unstables, unstable => unstable.file);
+
+            Object.entries(groupedByFile)
+                .forEach(([key, value]) => {
+                    Base.consoleLog(color('bright yellow', '    %s'), key);
+
+                    value.forEach(unstableTest => {
+                        Base.consoleLog(color('bright yellow', '      %s'), unstableTest.title);
+                    });
+                });
+        }
+
+        Base.consoleLog();
+    }
+
+    function findTestIndex (collection, test) {
+        return collection.findIndex(item => {
+            return item.file === test.file &&
+                item.title === test.title;
+        });
+    }
+
+    function isInUnstables (test) {
+        return findTestIndex(this.stats.unstables, test) > -1;
+    }
+
+    runner.on(EVENT_RUN_BEGIN, function () {
+        Base.consoleLog();
+    });
+
+    runner.on(EVENT_SUITE_BEGIN, function (suite) {
+        ++indents;
+        Base.consoleLog(color('suite', '%s%s'), indent(), suite.title);
+    });
+
+    runner.on(EVENT_SUITE_END, function () {
+        --indents;
+        if (indents === 1)
+            Base.consoleLog();
+    });
+
+    runner.on(EVENT_TEST_PENDING, function (test) {
+        const fmt = indent() + color('pending', '  - %s');
+
+        Base.consoleLog(fmt, test.title);
+    });
+
+    runner.on(EVENT_TEST_PASS, function (test) {
+        let fmt;
+
+        if (test.speed === 'fast') {
+            fmt =
+                indent() +
+                color('checkmark', '  ' + Base.symbols.ok) +
+                color('pass', ' %s');
+            Base.consoleLog(fmt, test.title);
+        }
+        else {
+            fmt =
+                indent() +
+                color('checkmark', '  ' + Base.symbols.ok) +
+                color('pass', ' %s') +
+                color(test.speed, ' (%dms)');
+            Base.consoleLog(fmt, test.title, test.duration);
+        }
+    });
+
+    runner.on(EVENT_TEST_FAIL, function (test) {
+        Base.consoleLog(indent() + color('fail', '  %d) %s'), ++n, test.title);
+
+        const index = findTestIndex(self.stats.unstables, test);
+
+        if (index > -1)
+            self.stats.unstables.splice(index, 1);
+    });
+
+    runner.on(EVENT_TEST_RETRY, test => {
+        if (!isInUnstables.call(self, test))
+            self.stats.unstables.push(test);
+    });
+
+    runner.once(EVENT_RUN_END, () => {
+        epilogue.call(self);
+    });
+}
+
+inherits(SpecWithRetries, Base);
+
+SpecWithRetries.description = 'hierarchical & verbose & displays retried tests';

gulp/helpers/test-functional.js

@@ -1,8 +1,9 @@
-const { castArray }  = require('lodash');
-const getTimeout     = require('./get-timeout');
-const chai           = require('chai');
-const globby         = require('globby');
-const Mocha          = require('mocha');
+const { castArray }   = require('lodash');
+const getTimeout      = require('./get-timeout');
+const chai            = require('chai');
+const globby          = require('globby');
+const Mocha           = require('mocha');
+const SpecWithRetries = require('./mocha-reporter-spec-with-retries');
 
 const {
     TESTS_GLOB,
@@ -51,7 +52,8 @@ module.exports = async function testFunctional (src, testingEnvironmentName, { n
     tests.unshift(SETUP_TESTS_GLOB);
 
     const opts = {
-        timeout: getTimeout(3 * 60 * 1000),
+        reporter: SpecWithRetries,
+        timeout:  getTimeout(3 * 60 * 1000),
     };
 
     if (process.env.RETRY_FAILED_TESTS === 'true')

test/functional/fixtures/api/es-next/iframe-switching/pages/second.html

@@ -7,6 +7,8 @@
 <button id="second-page-btn">button</button>
 
 <script>
+    window.secondPageBtnClickCount = 0;
+
     document.getElementById('second-page-btn').addEventListener('click', function () {
         window.secondPageBtnClickCount = (window.secondPageBtnClickCount || 0) + 1;
     });

test/functional/fixtures/api/es-next/iframe-switching/testcafe-fixtures/iframe-switching-test.js

@@ -133,6 +133,8 @@ test('Click in a removed iframe', async t => {
     await t
         .switchToIframe('#iframe')
         .click('#remove-from-parent-btn')
+        // NOTE: allow the setTimeout(0) removal handler to run before the next action
+        .wait(500)
         .click('#btn');
 });
 
@@ -213,9 +215,11 @@ test('Click in a cross-domain iframe with redirect', async t => {
     await t
         .switchToIframe('#cross-domain-iframe')
         .click('#link')
+        // NOTE: this ensures the redirected page script initialized before we click
+        .expect(getSecondPageBtnClickCount()).eql(0)
         .click('#second-page-btn');
 
-    const secondPageBtnClickCount = await getSecondPageBtnClickCount();
+    await t.expect(getSecondPageBtnClickCount()).eql(1);
 
     await t
         .switchToMainWindow()
@@ -224,7 +228,6 @@ test('Click in a cross-domain iframe with redirect', async t => {
     const btnClickCount = await getBtnClickCount();
 
     expect(btnClickCount).eql(1);
-    expect(secondPageBtnClickCount).eql(1);
 });
 
 test("Click in a iframe that's loading too slowly", async t => {

test/server/data/mocha-reporter-spec-with-retries/suite1.js

@@ -0,0 +1,45 @@
+const assert = require('assert');
+
+describe('Test suite 1', function () {
+    let unstable1RunCount = 0;
+    let unstable2RunCount = 0;
+
+    this.retries(3);
+
+    it('Passed', () => {
+        assert.ok(true);
+    });
+
+    it('Failed', () => {
+        let isCorrect = false;
+
+        try {
+            assert.ok(false);
+        }
+        catch (_) {
+            isCorrect = true;
+        }
+
+        assert.ok(isCorrect);
+    });
+
+    it('Pending');
+
+    it('Unstable - 1', () => {
+        unstable1RunCount++;
+
+        if (unstable1RunCount === 2)
+            assert.ok(true);
+        else
+            assert.ok(false);
+    });
+
+    it('Unstable - 2', () => {
+        unstable2RunCount++;
+
+        if (unstable2RunCount === 2)
+            assert.ok(true);
+        else
+            assert.ok(false);
+    });
+});

test/server/data/mocha-reporter-spec-with-retries/suite2.js

@@ -0,0 +1,45 @@
+const assert = require('assert');
+
+describe('Test suite 2', function () {
+    let unstable1RunCount = 0;
+    let unstable2RunCount = 0;
+
+    this.retries(3);
+
+    it('Passed', () => {
+        assert.ok(true);
+    });
+
+    it('Failed', () => {
+        let isCorrect = false;
+
+        try {
+            assert.ok(false);
+        }
+        catch (_) {
+            isCorrect = true;
+        }
+
+        assert.ok(isCorrect);
+    });
+
+    it('Pending');
+
+    it('Unstable - 1', () => {
+        unstable1RunCount++;
+
+        if (unstable1RunCount === 2)
+            assert.ok(true);
+        else
+            assert.ok(false);
+    });
+
+    it('Unstable - 2', () => {
+        unstable2RunCount++;
+
+        if (unstable2RunCount === 2)
+            assert.ok(true);
+        else
+            assert.ok(false);
+    });
+});

test/server/mocha-reporter-spec-with-retries-test.js

@@ -0,0 +1,54 @@
+const { expect }      = require('chai');
+const path            = require('path');
+const util            = require('util');
+const Mocha           = require('mocha');
+const SpecWithRetries = require('../../gulp/helpers/mocha-reporter-spec-with-retries');
+
+function runSuitesWithReporter (files) {
+    return new Promise((resolve, reject) => {
+        const output              = [];
+        const originalConsoleLog  = Mocha.reporters.Base.consoleLog;
+        const mocha               = new Mocha({
+            reporter: SpecWithRetries,
+            timeout:  2000,
+            color:    false,
+        });
+
+        files.forEach(file => {
+            mocha.addFile(file);
+        });
+
+        Mocha.reporters.Base.consoleLog = (format, ...args) => {
+            if (!format)
+                output.push('');
+            else
+                output.push(util.format(format, ...args));
+        };
+
+        mocha.run(failures => {
+            Mocha.reporters.Base.consoleLog = originalConsoleLog;
+
+            if (failures)
+                reject(new Error(`${failures} test(s) failed`));
+            else
+                resolve(output.join('\n'));
+        });
+    });
+}
+
+describe('Mocha reporter spec with retries', () => {
+    it('Should include unstable test section grouped by source file', async () => {
+        const dataDir = path.join(__dirname, 'data/mocha-reporter-spec-with-retries');
+        const report  = await runSuitesWithReporter([
+            path.join(dataDir, 'suite1.js'),
+            path.join(dataDir, 'suite2.js'),
+        ]);
+
+        expect(report).contains('Unstable test(s):');
+        expect(report).match(/suite1\.js/);
+        expect(report).match(/suite2\.js/);
+
+        expect((report.match(/Unstable - 1/g) || []).length).gte(2);
+        expect((report.match(/Unstable - 2/g) || []).length).gte(2);
+    });
+});