From 3f3fa484b263a3856fcf142c88fa24cb5f3c52b1 Mon Sep 17 00:00:00 2001 From: Marcio Junior Date: Sat, 26 Sep 2015 11:25:33 -0300 Subject: [PATCH] Allow tests to run in random order --- grunt/config/concat.js | 1 + lib/jasmine-core/boot.js | 1 + lib/jasmine-core/boot/boot.js | 9 ++ spec/core/TreeProcessorSpec.js | 63 +++++++++++ spec/core/integration/EnvSpec.js | 1 - spec/core/integration/SpecRunningSpec.js | 99 +++++++++++++++++ spec/html/HtmlReporterSpec.js | 136 +++++++++++++++++++++++ src/core/Env.js | 32 +++++- src/core/Order.js | 46 ++++++++ src/core/TreeProcessor.js | 13 ++- src/core/requireCore.js | 1 + src/html/HtmlReporter.js | 31 +++++- 12 files changed, 423 insertions(+), 10 deletions(-) create mode 100644 src/core/Order.js diff --git a/grunt/config/concat.js b/grunt/config/concat.js index 555a2f05..d8ab504e 100644 --- a/grunt/config/concat.js +++ b/grunt/config/concat.js @@ -26,6 +26,7 @@ module.exports = { 'src/core/base.js', 'src/core/util.js', 'src/core/Spec.js', + 'src/core/Order.js', 'src/core/Env.js', 'src/core/JsApiReporter.js', 'src/core/PrettyPrinter', diff --git a/lib/jasmine-core/boot.js b/lib/jasmine-core/boot.js index 8b66e001..4305d672 100644 --- a/lib/jasmine-core/boot.js +++ b/lib/jasmine-core/boot.js @@ -85,6 +85,7 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. env: env, onRaiseExceptionsClick: function() { queryString.navigateWithNewParam("catch", !env.catchingExceptions()); }, onThrowExpectationsClick: function() { queryString.navigateWithNewParam("throwFailures", !env.throwingExpectationFailures()); }, + onRandomClick: function() { queryString.navigateWithNewParam("random", !env.randomTests()); }, addToExistingQueryString: function(key, value) { return queryString.fullStringWithNewParam(key, value); }, getContainer: function() { return document.body; }, createElement: function() { return document.createElement.apply(document, arguments); }, diff --git a/lib/jasmine-core/boot/boot.js b/lib/jasmine-core/boot/boot.js index 04ed64c1..a99774d0 100644 --- a/lib/jasmine-core/boot/boot.js +++ b/lib/jasmine-core/boot/boot.js @@ -55,6 +55,14 @@ var throwingExpectationFailures = queryString.getParam("throwFailures"); env.throwOnExpectationFailure(throwingExpectationFailures); + var random = queryString.getParam("random"); + env.randomizeTests(random); + + var seed = queryString.getParam("seed"); + if (seed) { + env.seed(seed); + } + /** * ## Reporters * The `HtmlReporter` builds all of the HTML UI for the runner page. This reporter paints the dots, stars, and x's for specs, as well as all spec names and all failures (if any). @@ -63,6 +71,7 @@ env: env, onRaiseExceptionsClick: function() { queryString.navigateWithNewParam("catch", !env.catchingExceptions()); }, onThrowExpectationsClick: function() { queryString.navigateWithNewParam("throwFailures", !env.throwingExpectationFailures()); }, + onRandomClick: function() { queryString.navigateWithNewParam("random", !env.randomTests()); }, addToExistingQueryString: function(key, value) { return queryString.fullStringWithNewParam(key, value); }, getContainer: function() { return document.body; }, createElement: function() { return document.createElement.apply(document, arguments); }, diff --git a/spec/core/TreeProcessorSpec.js b/spec/core/TreeProcessorSpec.js index 02e17f40..3dea3ee8 100644 --- a/spec/core/TreeProcessorSpec.js +++ b/spec/core/TreeProcessorSpec.js @@ -690,4 +690,67 @@ describe("TreeProcessor", function() { queueableFns[10].fn(); expect(leaf11.execute).toHaveBeenCalled(); }); + + it("runs nodes in a custom order when orderChildren is overrided", function() { + var leaf1 = new Leaf(), + leaf2 = new Leaf(), + leaf3 = new Leaf(), + leaf4 = new Leaf(), + leaf5 = new Leaf(), + leaf6 = new Leaf(), + leaf7 = new Leaf(), + leaf8 = new Leaf(), + leaf9 = new Leaf(), + leaf10 = new Leaf(), + leaf11 = new Leaf(), + root = new Node({ children: [leaf1, leaf2, leaf3, leaf4, leaf5, leaf6, leaf7, leaf8, leaf9, leaf10, leaf11] }), + queueRunner = jasmine.createSpy('queueRunner'), + processor = new j$.TreeProcessor({ + tree: root, + runnableIds: [root.id], + queueRunnerFactory: queueRunner, + orderChildren: function(node) { + var children = node.children.slice(); + return children.reverse(); + } + }); + + processor.execute(); + var queueableFns = queueRunner.calls.mostRecent().args[0].queueableFns; + expect(queueableFns.length).toBe(11); + + queueableFns[0].fn(); + expect(leaf11.execute).toHaveBeenCalled(); + + queueableFns[1].fn(); + expect(leaf10.execute).toHaveBeenCalled(); + + queueableFns[2].fn(); + expect(leaf9.execute).toHaveBeenCalled(); + + queueableFns[3].fn(); + expect(leaf8.execute).toHaveBeenCalled(); + + queueableFns[4].fn(); + expect(leaf7.execute).toHaveBeenCalled(); + + queueableFns[5].fn(); + expect(leaf6.execute).toHaveBeenCalled(); + + queueableFns[6].fn(); + expect(leaf5.execute).toHaveBeenCalled(); + + queueableFns[7].fn(); + expect(leaf4.execute).toHaveBeenCalled(); + + queueableFns[8].fn(); + expect(leaf3.execute).toHaveBeenCalled(); + + queueableFns[9].fn(); + expect(leaf2.execute).toHaveBeenCalled(); + + queueableFns[10].fn(); + expect(leaf1.execute).toHaveBeenCalled(); + + }); }); diff --git a/spec/core/integration/EnvSpec.js b/spec/core/integration/EnvSpec.js index 0b24610a..08c13be4 100644 --- a/spec/core/integration/EnvSpec.js +++ b/spec/core/integration/EnvSpec.js @@ -207,7 +207,6 @@ describe("Env integration", function() { var env = new j$.Env(); env.addReporter({jasmineDone: done}); - env.describe("tests", function() { var firstTimeThrough = true, firstSpecContext, secondSpecContext; diff --git a/spec/core/integration/SpecRunningSpec.js b/spec/core/integration/SpecRunningSpec.js index 9ea82c16..59ddfdae 100644 --- a/spec/core/integration/SpecRunningSpec.js +++ b/spec/core/integration/SpecRunningSpec.js @@ -696,4 +696,103 @@ describe("jasmine spec running", function () { env.execute([spec2.id, spec3.id, spec1.id]); }).toThrowError(/afterAll/); }); + + it("should run the tests in a consistent order when a seed is supplied", function(done) { + var actions = []; + env.randomizeTests(true); + env.seed('123456'); + + env.beforeEach(function () { + actions.push('topSuite beforeEach'); + }); + + env.afterEach(function () { + actions.push('topSuite afterEach'); + }); + + env.describe('Something', function() { + env.beforeEach(function() { + actions.push('outer beforeEach'); + }); + + env.afterEach(function() { + actions.push('outer afterEach'); + }); + + env.it('does it 1', function() { + actions.push('outer it 1'); + }); + + env.describe('Inner 1', function() { + env.beforeEach(function() { + actions.push('inner 1 beforeEach'); + }); + + env.afterEach(function() { + actions.push('inner 1 afterEach'); + }); + + env.it('does it 2', function() { + actions.push('inner 1 it'); + }); + }); + + env.it('does it 3', function() { + actions.push('outer it 2'); + }); + + env.describe('Inner 2', function() { + env.beforeEach(function() { + actions.push('inner 2 beforeEach'); + }); + + env.afterEach(function() { + actions.push('inner 2 afterEach'); + }); + + env.it('does it 2', function() { + actions.push('inner 2 it'); + }); + }); + }); + + var assertions = function() { + var expected = [ + 'topSuite beforeEach', + 'outer beforeEach', + 'outer it 2', + 'outer afterEach', + 'topSuite afterEach', + + 'topSuite beforeEach', + 'outer beforeEach', + 'inner 2 beforeEach', + 'inner 2 it', + 'inner 2 afterEach', + 'outer afterEach', + 'topSuite afterEach', + + 'topSuite beforeEach', + 'outer beforeEach', + 'inner 1 beforeEach', + 'inner 1 it', + 'inner 1 afterEach', + 'outer afterEach', + 'topSuite afterEach', + + 'topSuite beforeEach', + 'outer beforeEach', + 'outer it 1', + 'outer afterEach', + 'topSuite afterEach' + ]; + expect(actions).toEqual(expected); + done(); + }; + + env.addReporter({jasmineDone: assertions}); + + env.execute(); + }); + }); diff --git a/spec/html/HtmlReporterSpec.js b/spec/html/HtmlReporterSpec.js index 9334d945..ccfdf5c5 100644 --- a/spec/html/HtmlReporterSpec.js +++ b/spec/html/HtmlReporterSpec.js @@ -556,6 +556,142 @@ describe("New HtmlReporter", function() { }); }); + describe("UI for running tests in random order", function() { + it("should be unchecked if not randomizing", function() { + var env = new j$.Env(), + container = document.createElement("div"), + getContainer = function() { + return container; + }, + reporter = new j$.HtmlReporter({ + env: env, + getContainer: getContainer, + createElement: function() { + return document.createElement.apply(document, arguments); + }, + createTextNode: function() { + return document.createTextNode.apply(document, arguments); + } + }); + + reporter.initialize(); + reporter.jasmineDone({}); + + var randomUI = container.querySelector(".jasmine-random"); + expect(randomUI.checked).toBe(false); + }); + + it("should be checked if randomizing", function() { + var env = new j$.Env(), + container = document.createElement("div"), + getContainer = function() { + return container; + }, + reporter = new j$.HtmlReporter({ + env: env, + getContainer: getContainer, + createElement: function() { + return document.createElement.apply(document, arguments); + }, + createTextNode: function() { + return document.createTextNode.apply(document, arguments); + } + }); + + env.randomizeTests(true); + reporter.initialize(); + reporter.jasmineDone({}); + + var throwingExpectationsUI = container.querySelector(".jasmine-random"); + expect(throwingExpectationsUI.checked).toBe(true); + }); + + it("should affect the query param for random tests", function() { + var env = new j$.Env(), + container = document.createElement("div"), + randomHandler = jasmine.createSpy('randomHandler'), + getContainer = function() { + return container; + }, + reporter = new j$.HtmlReporter({ + env: env, + getContainer: getContainer, + onRandomClick: randomHandler, + createElement: function() { + return document.createElement.apply(document, arguments); + }, + createTextNode: function() { + return document.createTextNode.apply(document, arguments); + } + }); + + reporter.initialize(); + reporter.jasmineDone({}); + + var randomUI = container.querySelector(".jasmine-random"); + randomUI.click(); + + expect(randomHandler).toHaveBeenCalled(); + }); + + it("should show the seed bar if randomizing", function() { + var env = new j$.Env(), + container = document.createElement("div"), + getContainer = function() { + return container; + }, + reporter = new j$.HtmlReporter({ + env: env, + getContainer: getContainer, + createElement: function() { + return document.createElement.apply(document, arguments); + }, + createTextNode: function() { + return document.createTextNode.apply(document, arguments); + } + }); + + reporter.initialize(); + reporter.jasmineDone({ + order: { + random: true, + seed: '424242' + } + }); + + var seedBar = container.querySelector(".jasmine-seed-bar"); + var seedBarText = 'textContent' in seedBar ? seedBar.textContent : seedBar.innerText; + expect(seedBarText).toBe(', randomized with seed 424242'); + var seedLink = container.querySelector(".jasmine-seed-bar a"); + expect(seedLink.getAttribute('href')).toBe('?seed=424242'); + }); + + it("should not show the current seed bar if not randomizing", function() { + var env = new j$.Env(), + container = document.createElement("div"), + getContainer = function() { + return container; + }, + reporter = new j$.HtmlReporter({ + env: env, + getContainer: getContainer, + createElement: function() { + return document.createElement.apply(document, arguments); + }, + createTextNode: function() { + return document.createTextNode.apply(document, arguments); + } + }); + + reporter.initialize(); + reporter.jasmineDone(); + + var seedBar = container.querySelector(".jasmine-seed-bar"); + expect(seedBar).toBeNull(); + }); + + }); + it("shows a message if no specs are run", function(){ var env, container, reporter; env = new j$.Env(); diff --git a/src/core/Env.js b/src/core/Env.js index d6d0198f..aad1b1dd 100644 --- a/src/core/Env.js +++ b/src/core/Env.js @@ -20,6 +20,8 @@ getJasmineRequireObj().Env = function(j$) { var currentlyExecutingSuites = []; var currentDeclarationSuite = null; var throwOnExpectationFailure = false; + var random = false; + var seed = null; var currentSuite = function() { return currentlyExecutingSuites[currentlyExecutingSuites.length - 1]; @@ -169,6 +171,21 @@ getJasmineRequireObj().Env = function(j$) { return throwOnExpectationFailure; }; + this.randomizeTests = function(value) { + random = !!value; + }; + + this.randomTests = function() { + return random; + }; + + this.seed = function(value) { + if (value) { + seed = value; + } + return seed; + }; + var queueRunnerFactory = function(options) { options.catchException = catchException; options.clearStack = options.clearStack || clearStack; @@ -200,6 +217,12 @@ getJasmineRequireObj().Env = function(j$) { runnablesToRun = [topSuite.id]; } } + + var order = new j$.Order({ + random: random, + seed: seed + }); + var processor = new j$.TreeProcessor({ tree: topSuite, runnableIds: runnablesToRun, @@ -215,6 +238,9 @@ getJasmineRequireObj().Env = function(j$) { } currentlyExecutingSuites.pop(); reporter.suiteDone(result); + }, + orderChildren: function(node) { + return order.sort(node.children); } }); @@ -226,7 +252,11 @@ getJasmineRequireObj().Env = function(j$) { totalSpecsDefined: totalSpecsDefined }); - processor.execute(reporter.jasmineDone); + processor.execute(function() { + reporter.jasmineDone({ + order: order + }); + }); }; this.addReporter = function(reporterToAdd) { diff --git a/src/core/Order.js b/src/core/Order.js new file mode 100644 index 00000000..ccebd5e3 --- /dev/null +++ b/src/core/Order.js @@ -0,0 +1,46 @@ +/*jshint bitwise: false*/ + +getJasmineRequireObj().Order = function() { + function Order(options) { + this.random = 'random' in options ? options.random : true; + var seed = this.seed = options.seed || generateSeed(); + this.sort = this.random ? randomOrder : naturalOrder; + + function naturalOrder(items) { + return items; + } + + function randomOrder(items) { + var copy = items.slice(); + copy.sort(function(a, b) { + return jenkinsHash(seed + a.id) - jenkinsHash(seed + b.id); + }); + return copy; + } + + function generateSeed() { + return String(Math.random()).slice(-5); + } + + // Bob Jenkins One-at-a-Time Hash algorithm is a non-cryptographic hash function + // used to get a different output when the key changes slighly. + // We use your return to sort the children randomly in a consistent way when + // used in conjunction with a seed + + function jenkinsHash(key) { + var hash, i; + for(hash = i = 0; i < key.length; ++i) { + hash += key.charCodeAt(i); + hash += (hash << 10); + hash ^= (hash >> 6); + } + hash += (hash << 3); + hash ^= (hash >> 11); + hash += (hash << 15); + return hash; + } + + } + + return Order; +}; diff --git a/src/core/TreeProcessor.js b/src/core/TreeProcessor.js index 775f1d59..a1f5f4cb 100644 --- a/src/core/TreeProcessor.js +++ b/src/core/TreeProcessor.js @@ -5,6 +5,7 @@ getJasmineRequireObj().TreeProcessor = function() { queueRunnerFactory = attrs.queueRunnerFactory, nodeStart = attrs.nodeStart || function() {}, nodeComplete = attrs.nodeComplete || function() {}, + orderChildren = attrs.orderChildren || function(node) { return node.children; }, stats = { valid: true }, processed = false, defaultMin = Infinity, @@ -68,8 +69,10 @@ getJasmineRequireObj().TreeProcessor = function() { } else { var hasExecutableChild = false; - for (var i = 0; i < node.children.length; i++) { - var child = node.children[i]; + var orderedChildren = orderChildren(node); + + for (var i = 0; i < orderedChildren.length; i++) { + var child = orderedChildren[i]; processNode(child, parentEnabled); @@ -86,7 +89,7 @@ getJasmineRequireObj().TreeProcessor = function() { executable: hasExecutableChild }; - segmentChildren(node, stats[node.id], executableIndex); + segmentChildren(node, orderedChildren, stats[node.id], executableIndex); if (!node.canBeReentered() && stats[node.id].segments.length > 1) { stats = { valid: false }; @@ -102,11 +105,11 @@ getJasmineRequireObj().TreeProcessor = function() { return executableIndex === undefined ? defaultMax : executableIndex; } - function segmentChildren(node, nodeStats, executableIndex) { + function segmentChildren(node, orderedChildren, nodeStats, executableIndex) { var currentSegment = { index: 0, owner: node, nodes: [], min: startingMin(executableIndex), max: startingMax(executableIndex) }, result = [currentSegment], lastMax = defaultMax, - orderedChildSegments = orderChildSegments(node.children); + orderedChildSegments = orderChildSegments(orderedChildren); function isSegmentBoundary(minIndex) { return lastMax !== defaultMax && minIndex !== defaultMin && lastMax < minIndex - 1; diff --git a/src/core/requireCore.js b/src/core/requireCore.js index 9f2490d5..571cf48a 100644 --- a/src/core/requireCore.js +++ b/src/core/requireCore.js @@ -50,6 +50,7 @@ var getJasmineRequireObj = (function (jasmineGlobal) { j$.Timer = jRequire.Timer(); j$.TreeProcessor = jRequire.TreeProcessor(); j$.version = jRequire.version(); + j$.Order = jRequire.Order(); j$.matchers = jRequire.requireMatchers(jRequire, j$); diff --git a/src/html/HtmlReporter.js b/src/html/HtmlReporter.js index 396bf2ba..d80674a7 100644 --- a/src/html/HtmlReporter.js +++ b/src/html/HtmlReporter.js @@ -12,6 +12,7 @@ jasmineRequire.HtmlReporter = function(j$) { createTextNode = options.createTextNode, onRaiseExceptionsClick = options.onRaiseExceptionsClick || function() {}, onThrowExpectationsClick = options.onThrowExpectationsClick || function() {}, + onRandomClick = options.onRandomClick || function() {}, addToExistingQueryString = options.addToExistingQueryString || defaultQueryString, timer = options.timer || noopTimer, results = [], @@ -117,9 +118,10 @@ jasmineRequire.HtmlReporter = function(j$) { } }; - this.jasmineDone = function() { + this.jasmineDone = function(doneResult) { var banner = find('.jasmine-banner'); var alert = find('.jasmine-alert'); + var order = doneResult && doneResult.order; alert.appendChild(createDom('span', {className: 'jasmine-duration'}, 'finished in ' + timer.elapsed() / 1000 + 's')); banner.appendChild( @@ -139,7 +141,14 @@ jasmineRequire.HtmlReporter = function(j$) { id: 'jasmine-throw-failures', type: 'checkbox' }), - createDom('label', { className: 'jasmine-label', 'for': 'jasmine-throw-failures' }, 'stop spec on expectation failure')) + createDom('label', { className: 'jasmine-label', 'for': 'jasmine-throw-failures' }, 'stop spec on expectation failure')), + createDom('div', { className: 'jasmine-random-order' }, + createDom('input', { + className: 'jasmine-random', + id: 'jasmine-random-order', + type: 'checkbox' + }), + createDom('label', { className: 'jasmine-label', 'for': 'jasmine-random-order' }, 'run tests in random order')) ) )); @@ -152,6 +161,10 @@ jasmineRequire.HtmlReporter = function(j$) { throwCheckbox.checked = env.throwingExpectationFailures(); throwCheckbox.onclick = onThrowExpectationsClick; + var randomCheckbox = find('#jasmine-random-order'); + randomCheckbox.checked = env.randomTests(); + randomCheckbox.onclick = onRandomClick; + var optionsMenu = find('.jasmine-run-options'), optionsTrigger = optionsMenu.querySelector('.jasmine-trigger'), optionsPayload = optionsMenu.querySelector('.jasmine-payload'), @@ -185,7 +198,15 @@ jasmineRequire.HtmlReporter = function(j$) { statusBarMessage += 'No specs found'; } - alert.appendChild(createDom('span', {className: statusBarClassName}, statusBarMessage)); + var seedBar; + if (order && order.random) { + seedBar = createDom('span', {class: 'jasmine-seed-bar'}, + ', randomized with seed ', + createDom('a', {title: 'randomized with seed ' + order.seed, href: seedHref(order.seed)}, order.seed) + ); + } + + alert.appendChild(createDom('span', {className: statusBarClassName}, statusBarMessage, seedBar)); for(i = 0; i < failedSuites.length; i++) { var failedSuite = failedSuites[i]; @@ -316,6 +337,10 @@ jasmineRequire.HtmlReporter = function(j$) { return addToExistingQueryString('spec', result.fullName); } + function seedHref(seed) { + return addToExistingQueryString('seed', seed); + } + function defaultQueryString(key, value) { return '?' + key + '=' + value; }