Report start and end events for specs/suites that are skipped due to a beforeAll error

This is noisier, but it maintains compatibility with reporters that assume
(quite reasonably) that all specs and suites are either filtered out or
reported.

lib/jasmine-core/jasmine.js

@@ -1880,7 +1880,14 @@ getJasmineRequireObj().Env = function(j$) {
             hasFailures = true;
           }
           suite.endTimer();
-          reporter.suiteDone(result, next);
+
+          if (suite.hadBeforeAllFailure) {
+            reportChildrenOfBeforeAllFailure(suite).then(function() {
+              reporter.suiteDone(result, next);
+            });
+          } else {
+            reporter.suiteDone(result, next);
+          }
         },
         orderChildren: function(node) {
           return order.sort(node.children);
@@ -1926,51 +1933,92 @@ getJasmineRequireObj().Env = function(j$) {
             currentlyExecutingSuites.push(topSuite);
 
             processor.execute(function() {
-              clearResourcesForRunnable(topSuite.id);
-              currentlyExecutingSuites.pop();
-              var overallStatus, incompleteReason;
+              (async function() {
+                if (topSuite.hadBeforeAllFailure) {
+                  await reportChildrenOfBeforeAllFailure(topSuite);
+                }
 
-              if (
-                hasFailures ||
-                topSuite.result.failedExpectations.length > 0
-              ) {
-                overallStatus = 'failed';
-              } else if (focusedRunnables.length > 0) {
-                overallStatus = 'incomplete';
-                incompleteReason = 'fit() or fdescribe() was found';
-              } else if (totalSpecsDefined === 0) {
-                overallStatus = 'incomplete';
-                incompleteReason = 'No specs found';
-              } else {
-                overallStatus = 'passed';
-              }
+                clearResourcesForRunnable(topSuite.id);
+                currentlyExecutingSuites.pop();
+                var overallStatus, incompleteReason;
 
-              /**
-               * Information passed to the {@link Reporter#jasmineDone} event.
-               * @typedef JasmineDoneInfo
-               * @property {OverallStatus} overallStatus - The overall result of the suite: 'passed', 'failed', or 'incomplete'.
-               * @property {Int} totalTime - The total time (in ms) that it took to execute the suite
-               * @property {IncompleteReason} incompleteReason - Explanation of why the suite was incomplete.
-               * @property {Order} order - Information about the ordering (random or not) of this execution of the suite.
-               * @property {Expectation[]} failedExpectations - List of expectations that failed in an {@link afterAll} at the global level.
-               * @property {Expectation[]} deprecationWarnings - List of deprecation warnings that occurred at the global level.
-               * @since 2.4.0
-               */
-              const jasmineDoneInfo = {
-                overallStatus: overallStatus,
-                totalTime: jasmineTimer.elapsed(),
-                incompleteReason: incompleteReason,
-                order: order,
-                failedExpectations: topSuite.result.failedExpectations,
-                deprecationWarnings: topSuite.result.deprecationWarnings
-              };
-              reporter.jasmineDone(jasmineDoneInfo, function() {
-                done(jasmineDoneInfo);
-              });
+                if (
+                  hasFailures ||
+                  topSuite.result.failedExpectations.length > 0
+                ) {
+                  overallStatus = 'failed';
+                } else if (focusedRunnables.length > 0) {
+                  overallStatus = 'incomplete';
+                  incompleteReason = 'fit() or fdescribe() was found';
+                } else if (totalSpecsDefined === 0) {
+                  overallStatus = 'incomplete';
+                  incompleteReason = 'No specs found';
+                } else {
+                  overallStatus = 'passed';
+                }
+
+                /**
+                 * Information passed to the {@link Reporter#jasmineDone} event.
+                 * @typedef JasmineDoneInfo
+                 * @property {OverallStatus} overallStatus - The overall result of the suite: 'passed', 'failed', or 'incomplete'.
+                 * @property {Int} totalTime - The total time (in ms) that it took to execute the suite
+                 * @property {IncompleteReason} incompleteReason - Explanation of why the suite was incomplete.
+                 * @property {Order} order - Information about the ordering (random or not) of this execution of the suite.
+                 * @property {Expectation[]} failedExpectations - List of expectations that failed in an {@link afterAll} at the global level.
+                 * @property {Expectation[]} deprecationWarnings - List of deprecation warnings that occurred at the global level.
+                 * @since 2.4.0
+                 */
+                const jasmineDoneInfo = {
+                  overallStatus: overallStatus,
+                  totalTime: jasmineTimer.elapsed(),
+                  incompleteReason: incompleteReason,
+                  order: order,
+                  failedExpectations: topSuite.result.failedExpectations,
+                  deprecationWarnings: topSuite.result.deprecationWarnings
+                };
+                reporter.jasmineDone(jasmineDoneInfo, function() {
+                  done(jasmineDoneInfo);
+                });
+              })();
             });
           }
         );
       }
+
+      async function reportChildrenOfBeforeAllFailure(suite) {
+        for (const child of suite.children) {
+          if (child instanceof j$.Suite) {
+            await new Promise(function(resolve) {
+              reporter.suiteStarted(child.result, resolve);
+            });
+            await reportChildrenOfBeforeAllFailure(child);
+            markNotRun(child);
+            await new Promise(function(resolve) {
+              reporter.suiteDone(child.result, resolve);
+            });
+          } /* a spec */ else {
+            await new Promise(function(resolve) {
+              reporter.specStarted(child.result, resolve);
+            });
+            await new Promise(function(resolve) {
+              markNotRun(child);
+              reporter.specDone(child.result, resolve);
+            });
+          }
+        }
+
+        function markNotRun(runnable) {
+          runnable.addExpectationResult(
+            false,
+            {
+              passed: false,
+              message: 'Not run because a beforeAll function failed'
+            },
+            true
+          );
+          runnable.result.status = 'failed';
+        }
+      }
     };
 
     /**
@@ -8636,6 +8684,12 @@ getJasmineRequireObj().SkipAfterBeforeAllErrorPolicy = function(j$) {
   SkipAfterBeforeAllErrorPolicy.prototype.fnErrored = function(fnIx) {
     if (this.queueableFns_[fnIx].type === 'beforeAll') {
       this.skipping_ = true;
+      // Failures need to be reported for each contained spec. But we can't do
+      // that from here because reporting is async. This function isn't async
+      // (and can't be without greatly complicating QueueRunner). Mark the
+      // failure so that the code that reports the suite result (which is
+      // already async) can detect the failure and report the specs.
+      this.queueableFns_[fnIx].suite.hadBeforeAllFailure = true;
     }
   };
 
@@ -9604,7 +9658,7 @@ getJasmineRequireObj().Suite = function(j$) {
   };
 
   Suite.prototype.beforeAll = function(fn) {
-    this.beforeAllFns.push({ ...fn, type: 'beforeAll' });
+    this.beforeAllFns.push({ ...fn, type: 'beforeAll', suite: this });
   };
 
   Suite.prototype.afterEach = function(fn) {

spec/core/SkipAfterBeforeAllErrorPolicySpec.js

@@ -3,8 +3,7 @@ describe('SkipAfterBeforeAllErrorPolicy', function() {
     describe('When nothing has errored', function() {
       it('does not skip anything', function() {
         const policy = new jasmineUnderTest.SkipAfterBeforeAllErrorPolicy(
-          arrayOfArbitraryFns(4),
-          2
+          arrayOfArbitraryFns(4)
         );
 
         expect(policy.skipTo(0)).toEqual(1);
@@ -17,8 +16,7 @@ describe('SkipAfterBeforeAllErrorPolicy', function() {
     describe('When anything but a beforeAll has errored', function() {
       it('does not skip anything', function() {
         const policy = new jasmineUnderTest.SkipAfterBeforeAllErrorPolicy(
-          arrayOfArbitraryFns(4),
-          2
+          arrayOfArbitraryFns(4)
         );
 
         policy.fnErrored(0);
@@ -34,17 +32,15 @@ describe('SkipAfterBeforeAllErrorPolicy', function() {
 
     describe('When a beforeAll has errored', function() {
       it('skips subsequent functions other than afterAll', function() {
+        const suite = {};
         const fns = [
-          { type: 'beforeAll', fn: () => {} },
+          { type: 'beforeAll', fn: () => {}, suite },
           { fn: () => {} },
           { fn: () => {} },
           { type: 'afterAll', fn: () => {} },
           { type: 'afterAll', fn: () => {} }
         ];
-        const policy = new jasmineUnderTest.SkipAfterBeforeAllErrorPolicy(
-          fns,
-          2
-        );
+        const policy = new jasmineUnderTest.SkipAfterBeforeAllErrorPolicy(fns);
 
         policy.fnErrored(0);
         expect(policy.skipTo(0)).toEqual(3);
@@ -52,6 +48,29 @@ describe('SkipAfterBeforeAllErrorPolicy', function() {
       });
     });
   });
+
+  describe('#fnErrored', function() {
+    describe('When the fn is a beforeAll', function() {
+      it("sets the suite's hadBeforeAllFailure property to true", function() {
+        const suite = {};
+        const fns = [{ type: 'beforeAll', fn: () => {}, suite }];
+        const policy = new jasmineUnderTest.SkipAfterBeforeAllErrorPolicy(fns);
+
+        policy.fnErrored(0);
+
+        expect(suite.hadBeforeAllFailure).toBeTrue();
+      });
+    });
+
+    describe('When the fn is not a beforeAll', function() {
+      it('does not try to access the suite, which is probably not there', function() {
+        const fns = [{ fn: () => {} /* no suite */ }];
+        const policy = new jasmineUnderTest.SkipAfterBeforeAllErrorPolicy(fns);
+
+        expect(() => policy.fnErrored(0)).not.toThrow();
+      });
+    });
+  });
 });
 
 function arrayOfArbitraryFns(n) {

spec/core/SuiteSpec.js

@@ -71,9 +71,20 @@ describe('Suite', function() {
     suite.beforeAll(outerBefore);
     suite.beforeAll(innerBefore);
 
+    function sameInstance(expected) {
+      return {
+        asymmetricMatch: function(actual) {
+          return actual === expected;
+        },
+        jasmineToString: function() {
+          return `<same instance as ${expected}>`;
+        }
+      };
+    }
+
     expect(suite.beforeAllFns).toEqual([
-      { fn: outerBefore.fn, type: 'beforeAll' },
-      { fn: innerBefore.fn, type: 'beforeAll' }
+      { fn: outerBefore.fn, type: 'beforeAll', suite: sameInstance(suite) },
+      { fn: innerBefore.fn, type: 'beforeAll', suite: sameInstance(suite) }
     ]);
   });
 

spec/core/integration/SpecRunningSpec.js

@@ -1065,8 +1065,8 @@ describe('spec running', function() {
     });
   });
 
-  describe('When a beforeAll function fails', function() {
-    it('skips contained specs and suites', async function() {
+  describe('When a top-level beforeAll function fails', function() {
+    it('skips and reports contained specs', async function() {
       const outerBeforeEach = jasmine.createSpy('outerBeforeEach');
       const nestedBeforeEach = jasmine.createSpy('nestedBeforeEach');
       const outerAfterEach = jasmine.createSpy('outerAfterEach');
@@ -1089,6 +1089,14 @@ describe('spec running', function() {
       });
       env.afterEach(outerAfterEach);
 
+      const reporter = jasmine.createSpyObj('reporter', [
+        'suiteStarted',
+        'suiteDone',
+        'specStarted',
+        'specDone'
+      ]);
+      env.addReporter(reporter);
+
       await env.execute();
 
       expect(outerBeforeEach).not.toHaveBeenCalled();
@@ -1098,6 +1106,162 @@ describe('spec running', function() {
       expect(nestedIt).not.toHaveBeenCalled();
       expect(nestedAfterEach).not.toHaveBeenCalled();
       expect(outerAfterEach).not.toHaveBeenCalled();
+
+      expect(reporter.suiteStarted).toHaveBeenCalledWith(
+        jasmine.objectContaining({
+          fullName: 'a nested suite'
+        })
+      );
+
+      expect(reporter.suiteDone).toHaveBeenCalledWith(
+        jasmine.objectContaining({
+          fullName: 'a nested suite',
+          status: 'failed',
+          failedExpectations: [
+            jasmine.objectContaining({
+              passed: false,
+              message: 'Not run because a beforeAll function failed'
+            })
+          ]
+        })
+      );
+
+      expect(reporter.specStarted).toHaveBeenCalledWith(
+        jasmine.objectContaining({
+          fullName: 'a spec'
+        })
+      );
+      expect(reporter.specDone).toHaveBeenCalledWith(
+        jasmine.objectContaining({
+          fullName: 'a spec',
+          status: 'failed',
+          failedExpectations: [
+            jasmine.objectContaining({
+              passed: false,
+              message: 'Not run because a beforeAll function failed'
+            })
+          ]
+        })
+      );
+
+      expect(reporter.specStarted).toHaveBeenCalledWith(
+        jasmine.objectContaining({
+          fullName: 'a nested suite a nested spec'
+        })
+      );
+      expect(reporter.specDone).toHaveBeenCalledWith(
+        jasmine.objectContaining({
+          fullName: 'a nested suite a nested spec',
+          status: 'failed',
+          failedExpectations: [
+            jasmine.objectContaining({
+              passed: false,
+              message: 'Not run because a beforeAll function failed'
+            })
+          ]
+        })
+      );
+    });
+  });
+
+  describe('When a suite beforeAll function fails', function() {
+    it('skips and reports contained specs', async function() {
+      const outerBeforeEach = jasmine.createSpy('outerBeforeEach');
+      const nestedBeforeEach = jasmine.createSpy('nestedBeforeEach');
+      const outerAfterEach = jasmine.createSpy('outerAfterEach');
+      const nestedAfterEach = jasmine.createSpy('nestedAfterEach');
+      const outerIt = jasmine.createSpy('outerIt');
+      const nestedIt = jasmine.createSpy('nestedIt');
+      const nestedBeforeAll = jasmine.createSpy('nestedBeforeAll');
+
+      env.describe('a suite', function() {
+        env.beforeAll(function() {
+          throw new Error('nope');
+        });
+
+        env.beforeEach(outerBeforeEach);
+        env.it('a spec', outerIt);
+        env.describe('a nested suite', function() {
+          env.beforeAll(nestedBeforeAll);
+          env.beforeEach(nestedBeforeEach);
+          env.it('a nested spec', nestedIt);
+          env.afterEach(nestedAfterEach);
+        });
+        env.afterEach(outerAfterEach);
+      });
+
+      const reporter = jasmine.createSpyObj('reporter', [
+        'suiteStarted',
+        'suiteDone',
+        'specStarted',
+        'specDone'
+      ]);
+      env.addReporter(reporter);
+
+      await env.execute();
+
+      expect(outerBeforeEach).not.toHaveBeenCalled();
+      expect(outerIt).not.toHaveBeenCalled();
+      expect(nestedBeforeAll).not.toHaveBeenCalled();
+      expect(nestedBeforeEach).not.toHaveBeenCalled();
+      expect(nestedIt).not.toHaveBeenCalled();
+      expect(nestedAfterEach).not.toHaveBeenCalled();
+      expect(outerAfterEach).not.toHaveBeenCalled();
+
+      expect(reporter.suiteStarted).toHaveBeenCalledWith(
+        jasmine.objectContaining({
+          fullName: 'a suite a nested suite'
+        })
+      );
+
+      expect(reporter.suiteDone).toHaveBeenCalledWith(
+        jasmine.objectContaining({
+          fullName: 'a suite a nested suite',
+          status: 'failed',
+          failedExpectations: [
+            jasmine.objectContaining({
+              passed: false,
+              message: 'Not run because a beforeAll function failed'
+            })
+          ]
+        })
+      );
+
+      expect(reporter.specStarted).toHaveBeenCalledWith(
+        jasmine.objectContaining({
+          fullName: 'a suite a spec'
+        })
+      );
+      expect(reporter.specDone).toHaveBeenCalledWith(
+        jasmine.objectContaining({
+          fullName: 'a suite a spec',
+          status: 'failed',
+          failedExpectations: [
+            jasmine.objectContaining({
+              passed: false,
+              message: 'Not run because a beforeAll function failed'
+            })
+          ]
+        })
+      );
+
+      expect(reporter.specStarted).toHaveBeenCalledWith(
+        jasmine.objectContaining({
+          fullName: 'a suite a nested suite a nested spec'
+        })
+      );
+      expect(reporter.specDone).toHaveBeenCalledWith(
+        jasmine.objectContaining({
+          fullName: 'a suite a nested suite a nested spec',
+          status: 'failed',
+          failedExpectations: [
+            jasmine.objectContaining({
+              passed: false,
+              message: 'Not run because a beforeAll function failed'
+            })
+          ]
+        })
+      );
     });
 
     it('runs afterAll functions in the current suite and outer scopes', async function() {

src/core/Env.js

@@ -729,7 +729,14 @@ getJasmineRequireObj().Env = function(j$) {
             hasFailures = true;
           }
           suite.endTimer();
-          reporter.suiteDone(result, next);
+
+          if (suite.hadBeforeAllFailure) {
+            reportChildrenOfBeforeAllFailure(suite).then(function() {
+              reporter.suiteDone(result, next);
+            });
+          } else {
+            reporter.suiteDone(result, next);
+          }
         },
         orderChildren: function(node) {
           return order.sort(node.children);
@@ -775,51 +782,92 @@ getJasmineRequireObj().Env = function(j$) {
             currentlyExecutingSuites.push(topSuite);
 
             processor.execute(function() {
-              clearResourcesForRunnable(topSuite.id);
-              currentlyExecutingSuites.pop();
-              var overallStatus, incompleteReason;
+              (async function() {
+                if (topSuite.hadBeforeAllFailure) {
+                  await reportChildrenOfBeforeAllFailure(topSuite);
+                }
 
-              if (
-                hasFailures ||
-                topSuite.result.failedExpectations.length > 0
-              ) {
-                overallStatus = 'failed';
-              } else if (focusedRunnables.length > 0) {
-                overallStatus = 'incomplete';
-                incompleteReason = 'fit() or fdescribe() was found';
-              } else if (totalSpecsDefined === 0) {
-                overallStatus = 'incomplete';
-                incompleteReason = 'No specs found';
-              } else {
-                overallStatus = 'passed';
-              }
+                clearResourcesForRunnable(topSuite.id);
+                currentlyExecutingSuites.pop();
+                var overallStatus, incompleteReason;
 
-              /**
-               * Information passed to the {@link Reporter#jasmineDone} event.
-               * @typedef JasmineDoneInfo
-               * @property {OverallStatus} overallStatus - The overall result of the suite: 'passed', 'failed', or 'incomplete'.
-               * @property {Int} totalTime - The total time (in ms) that it took to execute the suite
-               * @property {IncompleteReason} incompleteReason - Explanation of why the suite was incomplete.
-               * @property {Order} order - Information about the ordering (random or not) of this execution of the suite.
-               * @property {Expectation[]} failedExpectations - List of expectations that failed in an {@link afterAll} at the global level.
-               * @property {Expectation[]} deprecationWarnings - List of deprecation warnings that occurred at the global level.
-               * @since 2.4.0
-               */
-              const jasmineDoneInfo = {
-                overallStatus: overallStatus,
-                totalTime: jasmineTimer.elapsed(),
-                incompleteReason: incompleteReason,
-                order: order,
-                failedExpectations: topSuite.result.failedExpectations,
-                deprecationWarnings: topSuite.result.deprecationWarnings
-              };
-              reporter.jasmineDone(jasmineDoneInfo, function() {
-                done(jasmineDoneInfo);
-              });
+                if (
+                  hasFailures ||
+                  topSuite.result.failedExpectations.length > 0
+                ) {
+                  overallStatus = 'failed';
+                } else if (focusedRunnables.length > 0) {
+                  overallStatus = 'incomplete';
+                  incompleteReason = 'fit() or fdescribe() was found';
+                } else if (totalSpecsDefined === 0) {
+                  overallStatus = 'incomplete';
+                  incompleteReason = 'No specs found';
+                } else {
+                  overallStatus = 'passed';
+                }
+
+                /**
+                 * Information passed to the {@link Reporter#jasmineDone} event.
+                 * @typedef JasmineDoneInfo
+                 * @property {OverallStatus} overallStatus - The overall result of the suite: 'passed', 'failed', or 'incomplete'.
+                 * @property {Int} totalTime - The total time (in ms) that it took to execute the suite
+                 * @property {IncompleteReason} incompleteReason - Explanation of why the suite was incomplete.
+                 * @property {Order} order - Information about the ordering (random or not) of this execution of the suite.
+                 * @property {Expectation[]} failedExpectations - List of expectations that failed in an {@link afterAll} at the global level.
+                 * @property {Expectation[]} deprecationWarnings - List of deprecation warnings that occurred at the global level.
+                 * @since 2.4.0
+                 */
+                const jasmineDoneInfo = {
+                  overallStatus: overallStatus,
+                  totalTime: jasmineTimer.elapsed(),
+                  incompleteReason: incompleteReason,
+                  order: order,
+                  failedExpectations: topSuite.result.failedExpectations,
+                  deprecationWarnings: topSuite.result.deprecationWarnings
+                };
+                reporter.jasmineDone(jasmineDoneInfo, function() {
+                  done(jasmineDoneInfo);
+                });
+              })();
             });
           }
         );
       }
+
+      async function reportChildrenOfBeforeAllFailure(suite) {
+        for (const child of suite.children) {
+          if (child instanceof j$.Suite) {
+            await new Promise(function(resolve) {
+              reporter.suiteStarted(child.result, resolve);
+            });
+            await reportChildrenOfBeforeAllFailure(child);
+            markNotRun(child);
+            await new Promise(function(resolve) {
+              reporter.suiteDone(child.result, resolve);
+            });
+          } /* a spec */ else {
+            await new Promise(function(resolve) {
+              reporter.specStarted(child.result, resolve);
+            });
+            await new Promise(function(resolve) {
+              markNotRun(child);
+              reporter.specDone(child.result, resolve);
+            });
+          }
+        }
+
+        function markNotRun(runnable) {
+          runnable.addExpectationResult(
+            false,
+            {
+              passed: false,
+              message: 'Not run because a beforeAll function failed'
+            },
+            true
+          );
+          runnable.result.status = 'failed';
+        }
+      }
     };
 
     /**

src/core/SkipAfterBeforeAllErrorPolicy.js

@@ -25,6 +25,12 @@ getJasmineRequireObj().SkipAfterBeforeAllErrorPolicy = function(j$) {
   SkipAfterBeforeAllErrorPolicy.prototype.fnErrored = function(fnIx) {
     if (this.queueableFns_[fnIx].type === 'beforeAll') {
       this.skipping_ = true;
+      // Failures need to be reported for each contained spec. But we can't do
+      // that from here because reporting is async. This function isn't async
+      // (and can't be without greatly complicating QueueRunner). Mark the
+      // failure so that the code that reports the suite result (which is
+      // already async) can detect the failure and report the specs.
+      this.queueableFns_[fnIx].suite.hadBeforeAllFailure = true;
     }
   };
 

src/core/Suite.js

@@ -103,7 +103,7 @@ getJasmineRequireObj().Suite = function(j$) {
   };
 
   Suite.prototype.beforeAll = function(fn) {
-    this.beforeAllFns.push({ ...fn, type: 'beforeAll' });
+    this.beforeAllFns.push({ ...fn, type: 'beforeAll', suite: this });
   };
 
   Suite.prototype.afterEach = function(fn) {