diff --git a/.gitignore b/.gitignore index de1abdbc..a49db639 100644 --- a/.gitignore +++ b/.gitignore @@ -40,3 +40,5 @@ test/test-app/js/jasmine-html.js #Test-suite test-runner.json test/test-suite/data +test/test-suite/Apps/wwTest +test/test-suite/Apps/wwTest-automation diff --git a/build/build/utils.js b/build/build/utils.js index 1187341c..1d190038 100644 --- a/build/build/utils.js +++ b/build/build/utils.js @@ -45,6 +45,11 @@ module.exports = { }, + cp: function (srcFile, dstFile) { + var fileBuffer = fs.readFileSync(srcFile); + fs.writeFileSync(dstFile, fileBuffer); + }, + listFiles: function (directory, filter) { var files = wrench.readdirSyncRecursive(directory), filteredFiles = []; diff --git a/build/test-suite.js b/build/test-suite.js index 4d0d3589..4e6c558f 100644 --- a/build/test-suite.js +++ b/build/test-suite.js @@ -13,6 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + +var utils = require('./build/utils'), + CLITest = require('../test/test-suite/helpers/CLITest'); + module.exports = function (done, custom) { var jasmine = require('jasmine-node'), verbose = false, @@ -60,10 +64,23 @@ module.exports = function (done, custom) { } } - jasmine.executeSpecsInFolder(specs, function (runner, log) { - var failed = runner.results().failedCount === 0 ? 0 : 1; - setTimeout(function () { - (typeof done !== "function" ? process.exit : done)(failed); - }, 10); - }, verbose, colored); + utils.displayOutput("STARTING TEST SUITE"); + + jasmine.executeSpecsInFolder(specs, null, verbose, colored); + + function noop () {} + + jasmine.getEnv().reporter = { + log: noop, + reportSpecStarting: noop, + reportRunnerStarting: noop, + reportSuiteResults: noop, + reportSpecResults: noop, + reportRunnerResults: function (runner) { + failed = runner.results().failedCount === 0 ? 0 : 1; + CLITest.reportFinalResults(); + utils.displayOutput("TEST SUITE COMPLETED"); + process.exit(failed); + } + }; }; diff --git a/test/test-app/automatic/SpecRunner.htm b/test/test-app/automatic/SpecRunner.htm index 06d37dfa..0ee538d9 100644 --- a/test/test-app/automatic/SpecRunner.htm +++ b/test/test-app/automatic/SpecRunner.htm @@ -22,11 +22,73 @@ diff --git a/test/test-app/automation/SpecRunner.htm b/test/test-app/automation/SpecRunner.htm index b1f60706..166349a6 100644 --- a/test/test-app/automation/SpecRunner.htm +++ b/test/test-app/automation/SpecRunner.htm @@ -26,26 +26,94 @@ + + + function takeScreenshot() { + internal.automation.takeScreenshot("/accounts/1000/shared/camera/WebWorksScreenShot.bmp"); + } + diff --git a/test/test-app/config.xml b/test/test-app/config.xml index 547b33c2..a9480310 100644 --- a/test/test-app/config.xml +++ b/test/test-app/config.xml @@ -120,6 +120,8 @@ + + Licensed under the Apache License, Version 2.0 (the "License"); #you may not use this file except in compliance with the License. diff --git a/test/test-suite/Apps/DisableWebSecurity/config.xml b/test/test-suite/Apps/DisableWebSecurity/config.xml index 5fe06324..c6520037 100644 --- a/test/test-suite/Apps/DisableWebSecurity/config.xml +++ b/test/test-suite/Apps/DisableWebSecurity/config.xml @@ -14,9 +14,6 @@ - - - Research In Motion Ltd. diff --git a/test/test-suite/Apps/DisableWebSecurity/index.html b/test/test-suite/Apps/DisableWebSecurity/index.html index 1f4f7511..e734633f 100644 --- a/test/test-suite/Apps/DisableWebSecurity/index.html +++ b/test/test-suite/Apps/DisableWebSecurity/index.html @@ -1,88 +1,100 @@ - - - Jasmine Spec Runner - - - - - - - - - - - - - + + + Jasmine Spec Runner + + + + + + + + + + + + + diff --git a/test/test-suite/Apps/SubfolderStartupPage/a/index.html b/test/test-suite/Apps/SubfolderStartupPage/a/index.html index 7679ee0e..4874dd5b 100644 --- a/test/test-suite/Apps/SubfolderStartupPage/a/index.html +++ b/test/test-suite/Apps/SubfolderStartupPage/a/index.html @@ -12,77 +12,88 @@ - - - - + + + + - diff --git a/test/test-suite/Apps/WildcardDomain/config.xml b/test/test-suite/Apps/WildcardDomain/config.xml index 66f0c3cf..15aaf370 100644 --- a/test/test-suite/Apps/WildcardDomain/config.xml +++ b/test/test-suite/Apps/WildcardDomain/config.xml @@ -12,8 +12,8 @@ Research In Motion Ltd. - - + + diff --git a/test/test-suite/Apps/WildcardDomain/index.html b/test/test-suite/Apps/WildcardDomain/index.html index 7ba8b6c2..b902af2e 100644 --- a/test/test-suite/Apps/WildcardDomain/index.html +++ b/test/test-suite/Apps/WildcardDomain/index.html @@ -1,51 +1,100 @@ - - - Jasmine Spec Runner + + + Jasmine Spec Runner - - - - + + + + - - + + - + jasmineEnv.addReporter(htmlReporter); - + jasmineEnv.specFilter = function(spec) { + return htmlReporter.specFilter(spec); + }; - - + function execJasmine() { + jasmineEnv.execute(); + } + + document.addEventListener("webworksready", function () { + execJasmine(); + }); + + + + diff --git a/test/output/.gitkeep b/test/test-suite/Apps/wwTest-automation/.gitkeep similarity index 100% rename from test/output/.gitkeep rename to test/test-suite/Apps/wwTest-automation/.gitkeep diff --git a/test/test-suite/Apps/wwTest/.gitkeep b/test/test-suite/Apps/wwTest/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/test/test-suite/helpers/CLITest.js b/test/test-suite/helpers/CLITest.js new file mode 100644 index 00000000..831a8022 --- /dev/null +++ b/test/test-suite/helpers/CLITest.js @@ -0,0 +1,160 @@ +/* + * Copyright 2012 Research In Motion Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +var childProcess = require("child_process"), + path = require("path"), + sys = require("sys"), + CLITest; + +CLITest = function () { + this.option = []; + + this.log_ = []; + this.specResults = []; + this.columnCounter_ = 0; + this.start_ = 0; + this.elapsed_ = 0; + this.suites_ = 0; + this.assertions_ = 0; + this.failures_ = 0; + + this.execCmd = function (cmd, done) { + console.log("EXECUTING " + cmd); + var c = childProcess.exec(cmd, function (error, stdout, stderr) { + done(); + }); + c.stdout.on('data', function (data) { + console.log(data); + }); + + c.stderr.on('data', function (data) { + console.log(data); + }); + }; + + this.addOption = function (name, value) { + this.option.push(name + " " + (value || "")); + }; + this.run = function (packager, done) { + var cmd = this.option.reduce(function (previousValue, currentValue, index, array) { + return (index === 0 ? "" : " ") + previousValue + " " + currentValue; + }, "node " + path.join(packager, "lib", "bbwp.js")); + this.execCmd(cmd, done); + this.option = []; + }; + + this.reportRunnerStarting = function (name) { + sys.print('Started ' + name); + sys.print('\n'); + this.start_ = new Date(); + }; + + this.reportRunnerFinished = function (name, runner) { + var currentElaped = ((new Date() - this.start_) / 1000); + + this.elapsed_ += currentElaped; + this.suites_ += runner.suites.length; + this.assertions_ += runner.results.totalCount; + this.failures_ += runner.results.failedCount; + + sys.print('\n'); + sys.print('Finished ' + name + ' in ' + currentElaped + ' seconds'); + sys.print('\n'); + sys.print('\n'); + sys.print('\n'); + }; + + this.reportFinalResults = function () { + var summary = ''; + + sys.print('\n'); + sys.print('\n'); + + sys.print('Repeat of final results'); + sys.print('\n'); + this.columnCounter_ = 0; + this.specResults.forEach(function (result) { + sys.print(result); + if (this.columnCounter_++ === 50) { + this.columnCounter_ = 0; + sys.print('\n'); + } + }); + sys.print('\n'); + sys.print('\n'); + + this.log_.forEach(function(entry) { + if (typeof entry === "string" && entry.length) { + sys.print(entry); + sys.print('\n'); + } + }); + + sys.print('Finished all suites in ' + this.elapsed_ + ' seconds '); + + summary += this.suites_ + ' test' + ((this.suites_.length === 1) ? '' : 's') + ', '; + summary += this.assertions_ + ' assertion' + ((this.assertions_ === 1) ? '' : 's') + ', '; + summary += this.failures_ + ' failure' + ((this.failures_ === 1) ? '' : 's') + '\n'; + sys.print(summary); + sys.print('\n'); + }; + + this.reportSpecResults = function (spec) { + var result = spec.results, + msg = ''; + + if (result.passedCount === result.totalCount) { + msg = "."; + // } else if (result.skipped) { TODO: Research why "result.skipped" returns false when "xit" is called on a spec? + // msg = (colors) ? (ansi.yellow + '*' + ansi.none) : '*'; + } else { + msg = 'F'; + } + + this.specResults.push(msg); + sys.print(msg); + + if (this.columnCounter_++ === 50) { + this.columnCounter_ = 0; + sys.print('\n'); + } + }; + + this.reportSuiteResults = function (suite) { + var specResults = suite.results, + path = [], + description; + + while(suite) { + path.unshift(suite.description); + suite = suite.parentSuite; + } + + description = path.join(' '); + + outerThis = this; + specResults.items_.forEach(function(spec){ + if (spec.failedCount > 0 && spec.description) { + outerThis.log_.push(' it ' + spec.description); + spec.items_.forEach(function(result){ + if (result.trace && result.trace.stack) { + outerThis.log_.push(' ' + result.trace.stack + '\n'); + } + }); + } + }); + }; +}; +module.exports = new CLITest(); diff --git a/test/test-suite/test/DeployTest.js b/test/test-suite/helpers/DeployTest.js similarity index 100% rename from test/test-suite/test/DeployTest.js rename to test/test-suite/helpers/DeployTest.js diff --git a/test/test-suite/test/server.js b/test/test-suite/helpers/server.js similarity index 100% rename from test/test-suite/test/server.js rename to test/test-suite/helpers/server.js diff --git a/test/test-suite/test/CLITest.js b/test/test-suite/test/CLITest.js deleted file mode 100644 index 17435b95..00000000 --- a/test/test-suite/test/CLITest.js +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright 2012 Research In Motion Limited. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -var childProcess = require("child_process"), - CLITest; - -CLITest = function (face) { - var cli = face, - option = []; - - this.addOption = function (name, value) { - option.push(name + " " + (value || "")); - }; - this.run = function (done) { - var cmd = option.reduce(function (previousValue, currentValue, index, array) { - return (index === 0 ? "" : " ") + previousValue + " " + currentValue; - }, cli); - console.log(cmd); - childProcess.exec(cmd, function (error, stdout, stderr) { - console.log(stderr); - console.log(stdout); - done(); - }); - }; -}; -module.exports = CLITest; diff --git a/test/test-suite/test/automation-test-runner.js b/test/test-suite/test/automation-test-runner.js new file mode 100644 index 00000000..99ad9117 --- /dev/null +++ b/test/test-suite/test/automation-test-runner.js @@ -0,0 +1,109 @@ +/* + * Copyright 2012 Research In Motion Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +var input = __dirname + "/../Apps", + wrench = require("wrench"), + output = __dirname + "/../data/output", + path = require("path"), + cliTest, + cliTest = require('../helpers/CLITest'), + DeployTest = require('../helpers/DeployTest'), + utils = require('../../../build/build/utils'), + flag, + settings = require('../../../test-runner.json'); + +describe("functional tests", function () { + beforeEach(function () { + flag = false; + wrench.rmdirSyncRecursive(output, true); + wrench.mkdirSyncRecursive(output, "0755"); + }); + + afterEach(function () { + wrench.rmdirSyncRecursive(output, true); + wrench.mkdirSyncRecursive(output, "0755"); + }); + + it("runs the functional test app", function () { + flag = false; + cliTest.execCmd("jake test-app", function () { + flag = true; + }); + waitsFor(function () { + return flag; + }, "Something", 100000); + + runs(function () { + var testAppSrc = path.normalize(__dirname + "/../../test-app/"), + testAppDst = path.normalize(input + "/wwTest-automation/"), + flag = false; + + utils.copyFolder(path.join(testAppSrc, "js"), path.join(testAppDst, "js"), undefined, true); + utils.copyFolder(path.join(testAppSrc, "css"), path.join(testAppDst, "css"), undefined, true); + utils.copyFile(path.join(testAppSrc, "config.xml"), testAppDst, undefined, true); + utils.cp(path.join(testAppSrc, "automation","SpecRunner.htm"), path.join(testAppDst, "index.html")); + wrench.copyDirSyncRecursive(path.join(testAppSrc, "automation", "spec"), path.join(testAppDst, "spec")); + + cliTest.addOption(testAppDst); + cliTest.addOption("-d"); + cliTest.addOption("-o", output); + cliTest.run(settings.packager, function () { + flag = true; + }); + + waitsFor(function () { + return flag; + }, "Something", 100000); + + runs(function () { + var reqData = [], + flag = false, + dt; + runs(function () { + dt = new DeployTest(output + "/device/wwTest-automation.bar"); + dt.listen(function (request) { + if (request.status === 'finished') { + cliTest.reportRunnerFinished("Functional Tests", request.data); + flag = true; + } else if (request.status === "SuiteResults") { + cliTest.reportSuiteResults(request.data); + } else if (request.status === "SpecResults") { + cliTest.reportSpecResults(request.data); + } else if (request.status === "RunnerStarting") { + cliTest.reportRunnerStarting("Functional Tests"); + } + }); + dt.startServer(); + dt.deploy(); + }); + waitsFor(function () { + return flag; + }, "Something", 1000000); + + runs(function () { + var flag = false; + dt.terminate(function () { + flag = true; + }); + waitsFor(function () { + return flag; + }); + runs(function () { + }); + }); + }); + }); + }); +}); diff --git a/test/test-suite/test/functional-test-runner.js b/test/test-suite/test/functional-test-runner.js new file mode 100644 index 00000000..7c1cea44 --- /dev/null +++ b/test/test-suite/test/functional-test-runner.js @@ -0,0 +1,109 @@ +/* + * Copyright 2012 Research In Motion Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +var input = __dirname + "/../Apps", + wrench = require("wrench"), + output = __dirname + "/../data/output", + path = require("path"), + cliTest, + cliTest = require('../helpers/CLITest'), + DeployTest = require('../helpers/DeployTest'), + utils = require('../../../build/build/utils'), + flag, + settings = require('../../../test-runner.json'); + +describe("functional tests", function () { + beforeEach(function () { + flag = false; + wrench.rmdirSyncRecursive(output, true); + wrench.mkdirSyncRecursive(output, "0755"); + }); + + afterEach(function () { + wrench.rmdirSyncRecursive(output, true); + wrench.mkdirSyncRecursive(output, "0755"); + }); + + it("runs the functional test app", function () { + flag = false; + cliTest.execCmd("jake test-app", function () { + flag = true; + }); + waitsFor(function () { + return flag; + }, "Something", 100000); + + runs(function () { + var testAppSrc = path.normalize(__dirname + "/../../test-app/"), + testAppDst = path.normalize(input + "/wwTest/"), + flag = false; + + utils.copyFolder(path.join(testAppSrc, "js"), path.join(testAppDst, "js"), undefined, true); + utils.copyFolder(path.join(testAppSrc, "css"), path.join(testAppDst, "css"), undefined, true); + utils.copyFile(path.join(testAppSrc, "config.xml"), testAppDst, undefined, true); + utils.cp(path.join(testAppSrc, "automatic","SpecRunner.htm"), path.join(testAppDst, "index.html")); + wrench.copyDirSyncRecursive(path.join(testAppSrc, "automatic", "spec"), path.join(testAppDst, "spec")); + + cliTest.addOption(testAppDst); + cliTest.addOption("-d"); + cliTest.addOption("-o", output); + cliTest.run(settings.packager, function () { + flag = true; + }); + + waitsFor(function () { + return flag; + }, "Something", 100000); + + runs(function () { + var reqData = [], + flag = false, + dt; + runs(function () { + dt = new DeployTest(output + "/device/wwTest.bar"); + dt.listen(function (request) { + if (request.status === 'finished') { + cliTest.reportRunnerFinished("Functional Tests", request.data); + flag = true; + } else if (request.status === "SuiteResults") { + cliTest.reportSuiteResults(request.data); + } else if (request.status === "SpecResults") { + cliTest.reportSpecResults(request.data); + } else if (request.status === "RunnerStarting") { + cliTest.reportRunnerStarting("Functional Tests"); + } + }); + dt.startServer(); + dt.deploy(); + }); + waitsFor(function () { + return flag; + }, "Something", 1000000); + + runs(function () { + var flag = false; + dt.terminate(function () { + flag = true; + }); + waitsFor(function () { + return flag; + }); + runs(function () { + }); + }); + }); + }); + }); +}); diff --git a/test/test-suite/test/whitelist-suite-runner.js b/test/test-suite/test/whitelist-suite-runner.js index f600ed20..b07909c8 100644 --- a/test/test-suite/test/whitelist-suite-runner.js +++ b/test/test-suite/test/whitelist-suite-runner.js @@ -19,14 +19,12 @@ var path = require("path"), input = path.join(__dirname, "..", "Apps"), wrench = require("wrench"), output = path.normalize(path.join(conf.ROOT, conf.COMMAND_DEFAULTS.output_folder)), - cliTest, - CLITest = require('./CLITest'), - DeployTest = require('./DeployTest'), + cliTest = require('../helpers/CLITest'), + DeployTest = require('../helpers/DeployTest'), flag; describe("whitelist", function () { beforeEach(function () { - cliTest = new CLITest(settings.packager + "bbwp"); flag = false; wrench.rmdirSyncRecursive(output, true); wrench.mkdirSyncRecursive(output, "0755"); @@ -41,7 +39,7 @@ describe("whitelist", function () { cliTest.addOption(path.join(input, "DisableWebSecurity")); cliTest.addOption("-d"); cliTest.addOption("-o", output); - cliTest.run(function () { + cliTest.run(settings.packager, function () { flag = true; }); @@ -57,9 +55,14 @@ describe("whitelist", function () { dt = new DeployTest(path.join(output, "device", "DisableWebSecurity.bar")); dt.listen(function (request) { if (request.status === 'finished') { + cliTest.reportRunnerFinished("DisableWebSecurity", request.data); flag = true; - } else { - reqData.push(request.data); + } else if (request.status === "SuiteResults") { + cliTest.reportSuiteResults(request.data); + } else if (request.status === "SpecResults") { + cliTest.reportSpecResults(request.data); + } else if (request.status === "RunnerStarting") { + cliTest.reportRunnerStarting("DisableWebSecurity"); } }); dt.startServer(); @@ -70,7 +73,7 @@ describe("whitelist", function () { }, "Something", 100000); runs(function () { - flag = false; + var flag = false; dt.terminate(function () { flag = true; }); @@ -78,22 +81,6 @@ describe("whitelist", function () { return flag; }); runs(function () { - reqData.forEach(function (spec) { - if (spec.failedCount === 0) { - console.log("Success - " + spec.description); - } else { - console.log("Failed - " + spec.description); - spec.items.forEach(function (item) { - if (!item.passed_) { - console.log(item.message); - if (item.trace && item.trace.stack) { - console.log(item.trace.stack); - } - } - }); - } - console.log(""); - }); }); }); }); @@ -104,7 +91,7 @@ describe("whitelist", function () { cliTest.addOption(path.join(input, "SubfolderStartupPage")); cliTest.addOption("-d"); cliTest.addOption("-o", output); - cliTest.run(function () { + cliTest.run(settings.packager, function () { flag = true; }); @@ -120,9 +107,14 @@ describe("whitelist", function () { dt = new DeployTest(path.join(output, "device", "SubfolderStartupPage.bar")); dt.listen(function (request) { if (request.status === 'finished') { + cliTest.reportRunnerFinished("SubfolderStartupPage", request.data); flag = true; - } else { - reqData.push(request.data); + } else if (request.status === "SuiteResults") { + cliTest.reportSuiteResults(request.data); + } else if (request.status === "SpecResults") { + cliTest.reportSpecResults(request.data); + } else if (request.status === "RunnerStarting") { + cliTest.reportRunnerStarting("SubfolderStartupPage"); } }); dt.startServer(); @@ -133,7 +125,7 @@ describe("whitelist", function () { }, "Something", 100000); runs(function () { - flag = false; + var flag = false; dt.terminate(function () { flag = true; }); @@ -141,25 +133,59 @@ describe("whitelist", function () { return flag; }); runs(function () { - reqData.forEach(function (spec) { - if (spec.failedCount === 0) { - console.log("Success - " + spec.description); - } else { - console.log("Failed - " + spec.description); - spec.items.forEach(function (item) { - if (!item.passed_) { - console.log(item.message); - if (item.trace && item.trace.stack) { - console.log(item.trace.stack); - } - } - }); - } - console.log(""); - }); }); }); }); + }); + + it("runs the wildcard domain tests", function () { + cliTest.addOption(path.join(input, "WildcardDomain")); + cliTest.addOption("-d"); + cliTest.addOption("-o", output); + cliTest.run(settings.packager, function () { + flag = true; + }); + + waitsFor(function () { + return flag; + }, "Something", 100000); + + runs(function () { + var reqData = [], + flag = false, + dt; + runs(function () { + dt = new DeployTest(output + "/device/WildcardDomain.bar"); + dt.listen(function (request) { + if (request.status === 'finished') { + cliTest.reportRunnerFinished("WildcardDomain", request.data); + flag = true; + } else if (request.status === "SuiteResults") { + cliTest.reportSuiteResults(request.data); + } else if (request.status === "SpecResults") { + cliTest.reportSpecResults(request.data); + } else if (request.status === "RunnerStarting") { + cliTest.reportRunnerStarting("WildcardDomain"); + } + }); + dt.startServer(); + dt.deploy(); + }); + waitsFor(function () { + return flag; + }, "Something", 100000); + runs(function () { + var flag = false; + dt.terminate(function () { + flag = true; + }); + waitsFor(function () { + return flag; + }); + runs(function () { + }); + }); + }); }); });