Changeset 243953 in webkit


Ignore:
Timestamp:
Apr 5, 2019 4:39:16 PM (5 years ago)
Author:
Devin Rousso
Message:

Web Inspector: TestSuite test cases should have their own timeout to ensure tests fail with output instead of timeout by test runner
​https://bugs.webkit.org/show_bug.cgi?id=162814
<rdar://problem/28574102>

Reviewed by Brian Burg.

Source/WebInspectorUI:

A 10s timer is started for every test case added to an async suite. The timer is cleared
when the test finishes, but if the timer fires, the test is forcibly ended with an error.

This timer can be configured by setting a timeout value when adding the test case. Values
are expected to be in milliseconds. The value -1 will prevent a timer from being set.

This change also relaxes the expectation that any individual test case failure will stop the
rest of the suite from running. Since timers are set per test case, it is possible to
recover from a "bad" test case to still run the remaining test cases.

NOTE: there may be unexpected behaviour if a test times out, as the timer doesn't actually
stop the execution of the test, so it may still run and log information, which may appear
"out of nowhere" in the middle of other tests.

  • UserInterface/Test/TestSuite.js:

(TestSuite.prototype.get passCount):
(AsyncTestSuite.prototype.runTestCases):
(SyncTestSuite.prototype.runTestCases):

LayoutTests:

  • inspector/unit-tests/async-test-suite.html:
  • inspector/unit-tests/async-test-suite-expected.txt:
  • inspector/unit-tests/sync-test-suite.html:
  • inspector/unit-tests/sync-test-suite-expected.txt:
  • http/tests/inspector/network/set-resource-caching-disabled-disk-cache-expected.txt:
  • inspector/canvas/recording-2d.html:
  • inspector/canvas/recording-webgl-snapshots.html:
  • inspector/canvas/recording-webgl.html:
  • inspector/canvas/resources/shaderProgram-utilities.js:

(TestPage.registerInitializer.whenProgramAdded): Added.
(TestPage.registerInitializer.whenProgramRemoved): Added.
(TestPage.registerInitializer.window.initializeTestSuite):
(TestPage.registerInitializer.window.addSimpleTestCase):
(TestPage.registerInitializer.window.addParentCanvasRemovedTestCase):
(TestPage.registerInitializer.awaitProgramAdded): Added.
(TestPage.registerInitializer.awaitProgramRemoved): Added.

  • inspector/console/command-line-api-expected.txt:
  • inspector/console/heap-snapshot.html:
  • inspector/debugger/async-stack-trace-truncate-expected.txt:
  • inspector/debugger/pause-for-internal-scripts-expected.txt:
  • inspector/formatting/resources/utilities.js:

(TestPage.registerInitializer.window.addFormattingTests):

Location:
trunk
Files:
17 edited

Legend:

Unmodified
Added
Removed

  • trunk/LayoutTests/ChangeLog

    r243945 r243953  
     12019-04-05  Devin Rousso  <drousso@apple.com>
     2
     3        Web Inspector: TestSuite test cases should have their own timeout to ensure tests fail with output instead of timeout by test runner
     4        https://bugs.webkit.org/show_bug.cgi?id=162814
     5        <rdar://problem/28574102>
     6
     7        Reviewed by Brian Burg.
     8
     9        * inspector/unit-tests/async-test-suite.html:
     10        * inspector/unit-tests/async-test-suite-expected.txt:
     11        * inspector/unit-tests/sync-test-suite.html:
     12        * inspector/unit-tests/sync-test-suite-expected.txt:
     13
     14        * http/tests/inspector/network/set-resource-caching-disabled-disk-cache-expected.txt:
     15        * inspector/canvas/recording-2d.html:
     16        * inspector/canvas/recording-webgl-snapshots.html:
     17        * inspector/canvas/recording-webgl.html:
     18        * inspector/canvas/resources/shaderProgram-utilities.js:
     19        (TestPage.registerInitializer.whenProgramAdded): Added.
     20        (TestPage.registerInitializer.whenProgramRemoved): Added.
     21        (TestPage.registerInitializer.window.initializeTestSuite):
     22        (TestPage.registerInitializer.window.addSimpleTestCase):
     23        (TestPage.registerInitializer.window.addParentCanvasRemovedTestCase):
     24        (TestPage.registerInitializer.awaitProgramAdded): Added.
     25        (TestPage.registerInitializer.awaitProgramRemoved): Added.
     26        * inspector/console/command-line-api-expected.txt:
     27        * inspector/console/heap-snapshot.html:
     28        * inspector/debugger/async-stack-trace-truncate-expected.txt:
     29        * inspector/debugger/pause-for-internal-scripts-expected.txt:
     30        * inspector/formatting/resources/utilities.js:
     31        (TestPage.registerInitializer.window.addFormattingTests):
     32
    1332019-04-05  Ryan Haddad  <ryanhaddad@apple.com>
    234

  • trunk/LayoutTests/http/tests/inspector/network/set-resource-caching-disabled-disk-cache-expected.txt

    r214494 r243953  
    77PASS: Resource should be created.
    88PASS: Resource should receive a Response.
     9
    910-- Running test setup.
    10 
    1111-- Running test case: SetResourceCachingDisabled.DiskCache
    1212PASS: Resource should be created.

  • trunk/LayoutTests/inspector/canvas/recording-2d.html

    r243400 r243953  
    445445            startRecording(WI.Canvas.ContextType.Canvas2D, resolve, reject, {frameCount: 1});
    446446        },
     447        timeout: -1,
    447448    });
    448449
    … …  
    453454            startRecording(WI.Canvas.ContextType.Canvas2D, resolve, reject);
    454455        },
     456        timeout: -1,
    455457    });
    456458
    … …  
    461463            startRecording(WI.Canvas.ContextType.Canvas2D, resolve, reject, {memoryLimit: 10});
    462464        },
     465        timeout: -1,
    463466    });
    464467

  • trunk/LayoutTests/inspector/canvas/recording-webgl-snapshots.html

    r242225 r243953  
    103103            startRecording(WI.Canvas.ContextType.WebGL, resolve, reject, {frameCount: 1, checkForContentChange: true});
    104104        },
     105        timeout: -1,
    105106    });
    106107

  • trunk/LayoutTests/inspector/canvas/recording-webgl.html

    r243400 r243953  
    527527            startRecording(WI.Canvas.ContextType.WebGL, resolve, reject, {frameCount: 1});
    528528        },
     529        timeout: -1,
    529530    });
    530531
    … …  
    535536            startRecording(WI.Canvas.ContextType.WebGL, resolve, reject);
    536537        },
     538        timeout: -1,
    537539    });
    538540
    … …  
    543545            startRecording(WI.Canvas.ContextType.WebGL, resolve, reject, {memoryLimit: 10});
    544546        },
     547        timeout: -1,
    545548    });
    546549

  • trunk/LayoutTests/inspector/canvas/resources/shaderProgram-utilities.js

    r236972 r243953  
    4747    let suite = null;
    4848
    49     function awaitProgramAdded() {
     49    function whenProgramAdded(callback) {
    5050        InspectorTest.assert(WI.canvasManager.canvases.length === 1, "There should only be one canvas.");
    51         return WI.canvasManager.canvases[0].shaderProgramCollection.awaitEvent(WI.Collection.Event.ItemAdded)
    52         .then((event) => {
     51        WI.canvasManager.canvases[0].shaderProgramCollection.singleFireEventListener(WI.Collection.Event.ItemAdded, (event) => {
    5352            let program = event.data.item;
    5453            InspectorTest.expectThat(program instanceof WI.ShaderProgram, "Added ShaderProgram.");
    5554            InspectorTest.expectThat(program.canvas instanceof WI.Canvas, "ShaderProgram should have a parent Canvas.");
    56             return program;
     55            callback(program);
    5756        });
    5857    }
    5958
    60     function awaitProgramRemoved() {
     59    function whenProgramRemoved(callback) {
    6160        InspectorTest.assert(WI.canvasManager.canvases.length === 1, "There should only be one canvas.");
    62         return WI.canvasManager.canvases[0].shaderProgramCollection.awaitEvent(WI.Collection.Event.ItemRemoved)
    63         .then((event) => {
     61        WI.canvasManager.canvases[0].shaderProgramCollection.singleFireEventListener(WI.Collection.Event.ItemRemoved, (event) => {
    6462            let program = event.data.item;
    6563            InspectorTest.expectThat(program instanceof WI.ShaderProgram, "Removed ShaderProgram.");
    6664            InspectorTest.expectThat(program.canvas instanceof WI.Canvas, "ShaderProgram should have a parent Canvas.");
    67             return program;
     65            callback(program);
    6866        });
    6967    }
    … …  
    7876                // This can't use `awaitEvent` since the promise resolution happens on the next tick.
    7977                WI.canvasManager.singleFireEventListener(WI.CanvasManager.Event.CanvasAdded, (event) => {
    80                     awaitProgramAdded().then(resolve, reject);
     78                    whenProgramAdded((program) => {
     79                        resolve();
     80                    });
    8181                });
    8282
    … …  
    9393            description: "Check that added/removed events are sent.",
    9494            test(resolve, reject) {
    95                 awaitProgramAdded()
    96                 .then((addedProgram) => {
    97                     awaitProgramRemoved()
    98                     .then((removedProgram) => {
     95                whenProgramAdded((addedProgram) => {
     96                    whenProgramRemoved((removedProgram) => {
    9997                        InspectorTest.expectEqual(removedProgram, addedProgram, "Removed the previously added ShaderProgram.");
    100                     })
    101                     .then(resolve, reject);
     98                        resolve();
     99                    });
    102100
    103101                    InspectorTest.evaluateInPage(`deleteProgram()`);
    … …  
    114112            description: "Check that the ShaderProgram is removed before it's parent Canvas.",
    115113            test(resolve, reject) {
    116                 Promise.race([
    117                     awaitProgramRemoved()
    118                     .then(() => {
    119                         InspectorTest.pass("Removed ShaderProgram before Canvas.");
    120                         resolve();
    121                     }),
    122                     WI.canvasManager.awaitEvent(WI.CanvasManager.Event.CanvasRemoved)
    123                     .then(reject)
    124                 ])
    125                 .catch(() => { InspectorTest.fail("Removed Canvas before ShaderProgram."); });
     114                let canvasRemoved = false;
     115
     116                WI.canvasManager.singleFireEventListener(WI.CanvasManager.Event.CanvasRemoved, (event) => {
     117                    canvasRemoved = true;
     118                });
     119
     120                whenProgramRemoved((program) => {
     121                    InspectorTest.expectFalse(canvasRemoved, "Removed ShaderProgram before Canvas.");
     122                    resolve();
     123                });
    126124
    127125                InspectorTest.evaluateInPage(`createProgram("${contextType}")`);

  • trunk/LayoutTests/inspector/console/command-line-api-expected.txt

    r200533 r243953  
    1111Input: values([3,4])
    1212Output: 3,4
     13
    1314-- Running test setup.
    14 
    1515-- Running test case: EvaluateDollarZero
    1616Input: $0
    1717Output: [object HTMLParagraphElement]
     18
    1819-- Running test setup.
    19 
    2020-- Running test case: EvaluateInvalidSelector
    2121Input: $('foo')

  • trunk/LayoutTests/inspector/console/heapSnapshot.html

    r210062 r243953  
    2828
    2929            ProtocolTest.evaluateInPage("triggerHeapSnapshotNoTitle()");
    30         }
     30        },
     31        timeout: -1,
    3132    });
    3233
    … …  
    4344
    4445            ProtocolTest.evaluateInPage("triggerHeapSnapshotWithTitle()");
    45         }
     46        },
     47        timeout: -1,
    4648    });
    4749

  • trunk/LayoutTests/inspector/debugger/async-stack-trace-truncate-expected.txt

    r234635 r243953  
    1111PASS: Async stack trace should be null.
    1212-- Running test teardown.
     13
    1314-- Running test setup.
    1415Set maximum stack trace depth = 10.
    15 
    1616-- Running test case: AsyncStackTrace.CheckTruncated
    1717PAUSED
    … …  
    3232PASS: Async stack trace should be truncated.
    3333-- Running test teardown.
     34
    3435-- Running test setup.
    3536Set maximum stack trace depth = 10.
    36 
    3737-- Running test case: AsyncStackTrace.CheckNotTruncated
    3838PAUSED

  • trunk/LayoutTests/inspector/debugger/pause-for-internal-scripts-expected.txt

    r233655 r243953  
    6666RESUMED
    6767PASS: Should have used all steps.
     68
    6869-- Running test setup.
    69 
    7070-- Running test case: Debugger.setPauseForInternalScripts.Enabled
    7171EXPRESSION: setTimeout(entryConsoleLog)

  • trunk/LayoutTests/inspector/formatting/resources/utilities.js

    r236766 r243953  
    4444    window.addFormattingTests = function(suite, mode, tests) {
    4545        let testPageURL = WI.networkManager.mainFrame.mainResource.url;
    46         let testPageResourcesURL = testPageURL.substring(0, testPageURL.lastIndexOf("/"));           
     46        let testPageResourcesURL = testPageURL.substring(0, testPageURL.lastIndexOf("/"));
    4747
    4848        for (let test of tests) {
    … …  
    5353                test(resolve, reject) {
    5454                    runFormattingTest(mode, testName, testURL).then(resolve).catch(reject);
    55                 }
     55                },
     56                timeout: -1,
    5657            });
    5758        }

  • trunk/LayoutTests/inspector/unit-tests/async-test-suite-expected.txt

    r225820 r243953  
    1 PASS: instantiating AsyncTestSuite requires name argument.
    2 PASS: instantiating AsyncTestSuite requires string name argument.
    3 PASS: instantiating AsyncTestSuite requires non-whitespace name argument.
    4 PASS: instantiating AsyncTestSuite requires test harness argument.
    5 PASS: should not be able to add empty test case.
    6 PASS: should not be able to add non-object test case.
    7 PASS: test case should require string name.
    8 PASS: test case should require non-whitespace name.
    9 PASS: test case should require test function.
    10 PASS: should not be able to specify non-Function `setup` parameter.
    11 PASS: should not be able to specify non-Function `setup` parameter.
    12 PASS: should not be able to specify non-Function `setup` parameter.
    13 PASS: should not be able to specify non-Function `teardown` parameter.
    14 PASS: should not be able to specify non-Function `teardown` parameter.
    15 PASS: should not be able to specify non-Function `teardown` parameter.
     1PASS: Should produce an exception.
     2Error: Must pass the test's harness as the first argument.
     3PASS: Should produce an exception.
     4Error: Must pass the test's harness as the first argument.
     5PASS: Should produce an exception.
     6Error: Must pass the test's harness as the first argument.
     7PASS: Should produce an exception.
     8Error: Must pass the test's harness as the first argument.
     9PASS: Should produce an exception.
     10Error: Tried to add non-object test case.
     11PASS: Should produce an exception.
     12Error: Tried to add non-object test case.
     13PASS: Should produce an exception.
     14Error: Tried to add test case without a name.
     15PASS: Should produce an exception.
     16Error: Tried to add test case without a name.
     17PASS: Should produce an exception.
     18Error: Tried to add test case without `test` function.
     19PASS: Should produce an exception.
     20Error: Tried to add test case with invalid `setup` parameter (must be a function).
     21PASS: Should produce an exception.
     22Error: Tried to add test case with invalid `setup` parameter (must be a function).
     23PASS: Should produce an exception.
     24Error: Tried to add test case with invalid `setup` parameter (must be a function).
     25PASS: Should produce an exception.
     26Error: Tried to add test case with invalid `teardown` parameter (must be a function).
     27PASS: Should produce an exception.
     28Error: Tried to add test case with invalid `teardown` parameter (must be a function).
     29PASS: Should produce an exception.
     30Error: Tried to add test case with invalid `teardown` parameter (must be a function).
    1631PASS: should not be able to run empty test suite.
    1732
    1833== Running test suite: AsyncTestSuite.RunTwiceSuite
    19 PASS: should not be able to run a test suite twice.
     34PASS: Should produce an exception.
     35Error: Tried to call runTestCases() more than once.
    2036-- Running test case: DummyTest0
    21 
    22 == Running test suite: AsyncTestSuite.SequentialExecution
    23 PASS: AsyncTestSuite.RunTestCases() should return a Promise.
    24 -- Running test case: DummyTest1
    25 -- Running test case: DummyTest2
    26 -- Running test case: DummyTest3
    27 -- Running test case: FailingTest4
    28 !! EXCEPTION: [object Object]
    29 Stack Trace: (suppressed)
    30 PASS: Promise from sequentialExecutionSuite.runTestCases() should reject when a test case fails.
    31 PASS: Promise from sequentialExecutionSuite.runTestCases() should reject without altering its result value.
    32 PASS: sequentialExecutionSuite should have executed four tests.
    33 PASS: sequentialExecutionSuite should have passed three tests.
    34 PASS: sequentialExecutionSuite should have failed 1 test.
    35 PASS: sequentialExecutionSuite should have skipped zero tests.
    36 
    37 == Running test suite: AsyncTestSuite.AbortOnFailure
    38 -- Running test case: PassingTest5
    39 -- Running test case: FailingTest6
    40 !! EXCEPTION: {"token":666}
    41 Stack Trace: (suppressed)
    42 PASS: Promise from abortOnFailureSuite.runTestCases() should reject when a test case fails.
    43 PASS: Promise from abortOnFailureSuite.runTestCases() should reject without altering its result value.
    44 PASS: abortOnFailureSuite should have executed two tests.
    45 PASS: abortOnFailureSuite should have passed one test.
    46 PASS: abortOnFailureSuite should have failed one test.
    47 PASS: abortOnFailureSuite should have skipped one test.
    48 
    49 == Running test suite: AsyncTestSuite.SetupAndTeardown
     37PASS: DummyTest0 should only run once.
     38
     39== Running test suite: AsyncTestSuite.PromiseFunctionSuccess
     40-- Running test case: PromiseFunctionSuccess
     41
     42PASS: Promise from AsyncTestSuite.PromiseFunctionSuccess.runTestCases() should resolve even if a test case fails.
     43PASS: AsyncTestSuite.PromiseFunctionSuccess should have executed 1 tests.
     44PASS: AsyncTestSuite.PromiseFunctionSuccess should have passed 1 tests.
     45PASS: AsyncTestSuite.PromiseFunctionSuccess should have failed 0 tests.
     46PASS: AsyncTestSuite.PromiseFunctionSuccess should have skipped 0 tests.
     47
     48== Running test suite: PromiseTestSuite.PromiseFunctionException
     49-- Running test case: PromiseFunctionException
     50Throwing...
     51!! EXCEPTION: PromiseFunctionException throw
     52Stack Trace: (suppressed)
     53
     54PASS: Promise from PromiseTestSuite.PromiseFunctionException.runTestCases() should resolve even if a test case fails.
     55PASS: PromiseTestSuite.PromiseFunctionException should have executed 1 tests.
     56PASS: PromiseTestSuite.PromiseFunctionException should have passed 0 tests.
     57PASS: PromiseTestSuite.PromiseFunctionException should have failed 1 tests.
     58PASS: PromiseTestSuite.PromiseFunctionException should have skipped 0 tests.
     59
     60== Running test suite: PromiseTestSuite.PromiseFunctionFailure
     61-- Running test case: PromiseFunctionFailure
     62Rejecting...
     63!! EXCEPTION: PromiseFunctionFailure reject
     64Stack Trace: (suppressed)
     65
     66PASS: Promise from PromiseTestSuite.PromiseFunctionFailure.runTestCases() should resolve even if a test case fails.
     67PASS: PromiseTestSuite.PromiseFunctionFailure should have executed 1 tests.
     68PASS: PromiseTestSuite.PromiseFunctionFailure should have passed 0 tests.
     69PASS: PromiseTestSuite.PromiseFunctionFailure should have failed 1 tests.
     70PASS: PromiseTestSuite.PromiseFunctionFailure should have skipped 0 tests.
     71
     72== Running test suite: AsyncTestSuite.PromiseSequentialExecution
     73-- Running test case: 1 (Pass)
     74-- Running test case: 2 (Pass)
     75-- Running test case: 3 (Pass)
     76-- Running test case: 4 (Pass)
     77
     78PASS: Promise from AsyncTestSuite.PromiseSequentialExecution.runTestCases() should resolve even if a test case fails.
     79PASS: AsyncTestSuite.PromiseSequentialExecution should have executed 4 tests.
     80PASS: AsyncTestSuite.PromiseSequentialExecution should have passed 4 tests.
     81PASS: AsyncTestSuite.PromiseSequentialExecution should have failed 0 tests.
     82PASS: AsyncTestSuite.PromiseSequentialExecution should have skipped 0 tests.
     83
     84== Running test suite: AsyncTestSuite.PromiseContinueOnFailure
     85-- Running test case: 1 (Pass)
     86-- Running test case: 2 (Fail)
     87Throwing...
     88!! EXCEPTION: {"x":"PromiseContinueOnFailure throw"}
     89Stack Trace: (suppressed)
     90
     91-- Running test case: 3 (Pass)
     92-- Running test case: 4 (Fail)
     93Rejecting...
     94!! EXCEPTION: {"x":"PromiseContinueOnFailure reject"}
     95Stack Trace: (suppressed)
     96
     97PASS: Promise from AsyncTestSuite.PromiseContinueOnFailure.runTestCases() should resolve even if a test case fails.
     98PASS: AsyncTestSuite.PromiseContinueOnFailure should have executed 4 tests.
     99PASS: AsyncTestSuite.PromiseContinueOnFailure should have passed 2 tests.
     100PASS: AsyncTestSuite.PromiseContinueOnFailure should have failed 2 tests.
     101PASS: AsyncTestSuite.PromiseContinueOnFailure should have skipped 0 tests.
     102
     103== Running test suite: AsyncTestSuite.PromiseSetupAndTeardown
    50104-- Running test setup.
    51105-- Running test case: TestWithSetupAndTeardown
    52 PASS: Test should see side effects of running setup() action.
    53 -- Running test teardown.
    54 PASS: Teardown should see side effects of running setup() action.
    55 
     106-- Running test teardown.
    56107-- Running test case: TestRunningAfterTeardown
    57 PASS: Test should see side effects of previous test's teardown() action.
    58 PASS: Promise from setupAndTeardownTestSuite.runTestCases() should resolve.
    59 
    60 == Running test suite: AsyncTestSuite.SetupException
    61 -- Running test setup.
    62 !! EXCEPTION:
    63 Stack Trace: (suppressed)
    64 PASS: Promise from setupExceptionTestSuite.runTestCases() should reject.
    65 
    66 == Running test suite: AsyncTestSuite.SetupFailure
    67 -- Running test setup.
    68 !! EXCEPTION: undefined
    69 Stack Trace: (suppressed)
    70 PASS: Promise from setupFailureTestSuite.runTestCases() should reject.
    71 
    72 == Running test suite: AsyncTestSuite.TeardownException
     108
     109PASS: Promise from AsyncTestSuite.PromiseSetupAndTeardown.runTestCases() should resolve even if a test case fails.
     110PASS: AsyncTestSuite.PromiseSetupAndTeardown should have executed 2 tests.
     111PASS: AsyncTestSuite.PromiseSetupAndTeardown should have passed 2 tests.
     112PASS: AsyncTestSuite.PromiseSetupAndTeardown should have failed 0 tests.
     113PASS: AsyncTestSuite.PromiseSetupAndTeardown should have skipped 0 tests.
     114
     115== Running test suite: AsyncTestSuite.PromiseSetupException
     116-- Running test setup.
     117Throwing...
     118!! EXCEPTION: PromiseSetupException throw
     119Stack Trace: (suppressed)
     120
     121-- Running test setup.
     122PASS: Setup action should still execute if previous test's setup action threw an exception.
     123-- Running test case: TestAfterSetupException
     124PASS: Test should still execute if previous test's setup action threw an exception.
     125-- Running test teardown.
     126PASS: Teardown action should still execute if previous test's setup action threw an exception.
     127
     128PASS: Promise from AsyncTestSuite.PromiseSetupException.runTestCases() should resolve even if a test case fails.
     129PASS: AsyncTestSuite.PromiseSetupException should have executed 1 tests.
     130PASS: AsyncTestSuite.PromiseSetupException should have passed 1 tests.
     131PASS: AsyncTestSuite.PromiseSetupException should have failed 1 tests.
     132PASS: AsyncTestSuite.PromiseSetupException should have skipped 1 tests.
     133
     134== Running test suite: AsyncTestSuite.PromiseSetupFailure
     135-- Running test setup.
     136Rejecting...
     137!! EXCEPTION: PromiseSetupFailure reject
     138Stack Trace: (suppressed)
     139
     140-- Running test setup.
     141PASS: Setup action should still execute if previous test's setup action failed.
     142-- Running test case: TestAfterSetupException
     143PASS: Test should still execute if previous test's setup action failed.
     144-- Running test teardown.
     145PASS: Teardown action should still execute if previous test's setup action failed.
     146
     147PASS: Promise from AsyncTestSuite.PromiseSetupFailure.runTestCases() should resolve even if a test case fails.
     148PASS: AsyncTestSuite.PromiseSetupFailure should have executed 1 tests.
     149PASS: AsyncTestSuite.PromiseSetupFailure should have passed 1 tests.
     150PASS: AsyncTestSuite.PromiseSetupFailure should have failed 1 tests.
     151PASS: AsyncTestSuite.PromiseSetupFailure should have skipped 1 tests.
     152
     153== Running test suite: AsyncTestSuite.PromiseTeardownException
    73154-- Running test case: TestWithExceptionDuringTeardown
    74155-- Running test teardown.
    75 !! EXCEPTION:
    76 Stack Trace: (suppressed)
    77 PASS: Promise from teardownExceptionTestSuite.runTestCases() should reject.
    78 
    79 == Running test suite: AsyncTestSuite.TeardownFailure
     156Throwing...
     157!! EXCEPTION: PromiseTeardownException throw
     158Stack Trace: (suppressed)
     159
     160-- Running test setup.
     161PASS: Setup action should still execute if previous test's teardown action threw an exception.
     162-- Running test case: TestAfterTeardownException
     163PASS: Test should still execute if previous test's teardown action threw an exception.
     164-- Running test teardown.
     165PASS: Teardown action should still execute if previous test's teardown action threw an exception.
     166
     167PASS: Promise from AsyncTestSuite.PromiseTeardownException.runTestCases() should resolve even if a test case fails.
     168PASS: AsyncTestSuite.PromiseTeardownException should have executed 2 tests.
     169PASS: AsyncTestSuite.PromiseTeardownException should have passed 1 tests.
     170PASS: AsyncTestSuite.PromiseTeardownException should have failed 1 tests.
     171PASS: AsyncTestSuite.PromiseTeardownException should have skipped 0 tests.
     172
     173== Running test suite: AsyncTestSuite.PromiseTeardownFailure
    80174-- Running test case: TestWithExceptionDuringTeardown
    81175-- Running test teardown.
    82 !! EXCEPTION: undefined
    83 Stack Trace: (suppressed)
    84 PASS: Promise from teardownFailureTestSuite.runTestCases() should reject.
     176Rejecting...
     177!! EXCEPTION: PromiseTeardownFailure reject
     178Stack Trace: (suppressed)
     179
     180-- Running test setup.
     181PASS: Setup action should still execute if previous test's teardown action failed.
     182-- Running test case: TestAfterTeardownException
     183PASS: Test should still execute if previous test's teardown action failed.
     184-- Running test teardown.
     185PASS: Teardown action should still execute if previous test's teardown action failed.
     186
     187PASS: Promise from AsyncTestSuite.PromiseTeardownFailure.runTestCases() should resolve even if a test case fails.
     188PASS: AsyncTestSuite.PromiseTeardownFailure should have executed 2 tests.
     189PASS: AsyncTestSuite.PromiseTeardownFailure should have passed 1 tests.
     190PASS: AsyncTestSuite.PromiseTeardownFailure should have failed 1 tests.
     191PASS: AsyncTestSuite.PromiseTeardownFailure should have skipped 0 tests.
     192
     193== Running test suite: AsyncTestSuite.PromiseTimeout
     194-- Running test case: PromiseTestWithTimeout
     195Timeout...
     196!! TIMEOUT: took longer than 5ms
     197
     198-- Running test setup.
     199PASS: Setup action should still execute if previous test timed out.
     200-- Running test case: PromiseTestAfterTimeout
     201PASS: Test should still execute if previous test timed out.
     202-- Running test teardown.
     203PASS: Teardown action should still execute if previous test timed out.
     204
     205PASS: Promise from AsyncTestSuite.PromiseTimeout.runTestCases() should resolve even if a test case fails.
     206PASS: AsyncTestSuite.PromiseTimeout should have executed 2 tests.
     207PASS: AsyncTestSuite.PromiseTimeout should have passed 1 tests.
     208PASS: AsyncTestSuite.PromiseTimeout should have failed 1 tests.
     209PASS: AsyncTestSuite.PromiseTimeout should have skipped 0 tests.
    85210
    86211== Running test suite: AsyncTestSuite.AsyncFunctionSuccess
    87212-- Running test case: AsyncFunctionSuccess
    88 PASS: Promise from asyncFunctionSuccessTestSuite.runTestCases() should succeed.
    89 PASS: Promise did evaluate the async test function.
    90 PASS: Resolved value should be 42.
    91 
    92 == Running test suite: AsyncTestSuite.AsyncFunctionExplicitFailure
    93 -- Running test case: AsyncFunctionFailure
    94 !! EXCEPTION: AsyncFunctionFailure Exception Message
    95 Stack Trace: (suppressed)
    96 PASS: Promise from asyncFunctionExplicitFailureTestSuite.runTestCases() should reject.
    97 PASS: Promise did evaluate the async test function.
    98 PASS: Rejected value should be thrown exception.
    99 
    100 == Running test suite: AsyncTestSuite.AsyncFunctionRuntimeFailure
    101 -- Running test case: AsyncFunctionFailure
     213
     214PASS: Promise from AsyncTestSuite.AsyncFunctionSuccess.runTestCases() should resolve even if a test case fails.
     215PASS: AsyncTestSuite.AsyncFunctionSuccess should have executed 1 tests.
     216PASS: AsyncTestSuite.AsyncFunctionSuccess should have passed 1 tests.
     217PASS: AsyncTestSuite.AsyncFunctionSuccess should have failed 0 tests.
     218PASS: AsyncTestSuite.AsyncFunctionSuccess should have skipped 0 tests.
     219
     220== Running test suite: AsyncTestSuite.AsyncFunctionExplicitException
     221-- Running test case: AsyncFunctionExplicitException
     222Throwing...
     223!! EXCEPTION: AsyncFunctionExplicitException throw
     224Stack Trace: (suppressed)
     225
     226PASS: Promise from AsyncTestSuite.AsyncFunctionExplicitException.runTestCases() should resolve even if a test case fails.
     227PASS: AsyncTestSuite.AsyncFunctionExplicitException should have executed 1 tests.
     228PASS: AsyncTestSuite.AsyncFunctionExplicitException should have passed 0 tests.
     229PASS: AsyncTestSuite.AsyncFunctionExplicitException should have failed 1 tests.
     230PASS: AsyncTestSuite.AsyncFunctionExplicitException should have skipped 0 tests.
     231
     232== Running test suite: AsyncTestSuite.AsyncFunctionRuntimeException
     233-- Running test case: AsyncFunctionRuntimeException
     234Throwing...
    102235!! EXCEPTION: undefined is not an object (evaluating '({}).x.x')
    103236Stack Trace: (suppressed)
    104 PASS: Promise from asyncFunctionRuntimeFailureTestSuite.runTestCases() should reject.
    105 PASS: Promise did evaluate the async test function.
    106 PASS: Rejected value should be a runtime exception.
    107 
    108 == Running test suite: AsyncTestSuite.AsyncSetupAndAsyncTeardown
    109 -- Running test setup.
    110 -- Running test case: TestWithSetupAndTeardown
    111 PASS: Test should see side effects of running setup() action.
    112 -- Running test teardown.
    113 PASS: Teardown should see side effects of running setup() action.
    114 
    115 -- Running test case: TestRunningAfterTeardown
    116 PASS: Test should see side effects of previous test's teardown() action.
    117 PASS: Promise from asyncSetupAndAsyncTeardownTestSuite.runTestCases() should resolve.
    118 
    119 == Running test suite: AsyncTestSuite.AsyncSetupExplicitFailure
    120 -- Running test case: AsyncFunctionFailure
    121 !! EXCEPTION: AsyncFunctionFailure Exception Message
    122 Stack Trace: (suppressed)
    123 PASS: Promise from asyncSetupExplicitFailureTestSuite.runTestCases() should reject.
    124 PASS: Promise did evaluate the async setup function.
    125 PASS: Rejected value should be thrown exception.
    126 
    127 == Running test suite: AsyncTestSuite.AsyncSetupRuntimeFailure
    128 -- Running test setup.
     237
     238PASS: Promise from AsyncTestSuite.AsyncFunctionRuntimeException.runTestCases() should resolve even if a test case fails.
     239PASS: AsyncTestSuite.AsyncFunctionRuntimeException should have executed 1 tests.
     240PASS: AsyncTestSuite.AsyncFunctionRuntimeException should have passed 0 tests.
     241PASS: AsyncTestSuite.AsyncFunctionRuntimeException should have failed 1 tests.
     242PASS: AsyncTestSuite.AsyncFunctionRuntimeException should have skipped 0 tests.
     243
     244== Running test suite: AsyncTestSuite.AsyncFunctionFailure
     245-- Running test case: AsyncFunctionException
     246Rejecting...
     247!! EXCEPTION: AsyncFunctionFailure reject
     248Stack Trace: (suppressed)
     249
     250PASS: Promise from AsyncTestSuite.AsyncFunctionFailure.runTestCases() should resolve even if a test case fails.
     251PASS: AsyncTestSuite.AsyncFunctionFailure should have executed 1 tests.
     252PASS: AsyncTestSuite.AsyncFunctionFailure should have passed 0 tests.
     253PASS: AsyncTestSuite.AsyncFunctionFailure should have failed 1 tests.
     254PASS: AsyncTestSuite.AsyncFunctionFailure should have skipped 0 tests.
     255
     256== Running test suite: AsyncTestSuite.AsyncSequentialExecution
     257-- Running test case: 1 (Pass)
     258-- Running test case: 2 (Pass)
     259-- Running test case: 3 (Pass)
     260-- Running test case: 4 (Pass)
     261
     262PASS: Promise from AsyncTestSuite.AsyncSequentialExecution.runTestCases() should resolve even if a test case fails.
     263PASS: AsyncTestSuite.AsyncSequentialExecution should have executed 4 tests.
     264PASS: AsyncTestSuite.AsyncSequentialExecution should have passed 4 tests.
     265PASS: AsyncTestSuite.AsyncSequentialExecution should have failed 0 tests.
     266PASS: AsyncTestSuite.AsyncSequentialExecution should have skipped 0 tests.
     267
     268== Running test suite: AsyncTestSuite.AsyncContinueOnFailure
     269-- Running test case: 1 (Pass)
     270-- Running test case: 2 (Fail)
     271Throwing...
     272!! EXCEPTION: {"x":"AsyncContinueOnFailure throw"}
     273Stack Trace: (suppressed)
     274
     275-- Running test case: 3 (Pass)
     276-- Running test case: 4 (Fail)
     277Throwing...
    129278!! EXCEPTION: undefined is not an object (evaluating '({}).x.x')
    130279Stack Trace: (suppressed)
    131 PASS: Promise from asyncSetupRuntimeFailureTestSuite.runTestCases() should reject.
    132 PASS: Promise did evaluate the async setup function.
    133 PASS: Rejected value should be a runtime exception.
    134 
    135 == Running test suite: AsyncTestSuite.AsyncTeardownExplicitFailure
    136 -- Running test case: AsyncFunctionFailure
    137 -- Running test teardown.
    138 !! EXCEPTION: AsyncFunctionFailure Exception Message
    139 Stack Trace: (suppressed)
    140 PASS: Promise from asyncTeardownExplicitFailureTestSuite.runTestCases() should reject.
    141 PASS: Promise did evaluate the async teardown function.
    142 PASS: Rejected value should be thrown exception.
    143 
    144 == Running test suite: AsyncTestSuite.AsyncTeardownRuntimeFailure
    145 -- Running test case: AsyncFunctionFailure
    146 -- Running test teardown.
     280
     281-- Running test case: 5 (Pass)
     282-- Running test case: 6 (Fail)
     283Rejecting...
     284!! EXCEPTION: AsyncContinueOnFailure reject
     285Stack Trace: (suppressed)
     286
     287PASS: Promise from AsyncTestSuite.AsyncContinueOnFailure.runTestCases() should resolve even if a test case fails.
     288PASS: AsyncTestSuite.AsyncContinueOnFailure should have executed 6 tests.
     289PASS: AsyncTestSuite.AsyncContinueOnFailure should have passed 3 tests.
     290PASS: AsyncTestSuite.AsyncContinueOnFailure should have failed 3 tests.
     291PASS: AsyncTestSuite.AsyncContinueOnFailure should have skipped 0 tests.
     292
     293== Running test suite: AsyncTestSuite.AsyncSetupAndTeardown
     294-- Running test setup.
     295-- Running test case: AsyncTestWithSetupAndTeardown
     296-- Running test teardown.
     297-- Running test case: AsyncTestRunningAfterTeardown
     298
     299PASS: Promise from AsyncTestSuite.AsyncSetupAndTeardown.runTestCases() should resolve even if a test case fails.
     300PASS: AsyncTestSuite.AsyncSetupAndTeardown should have executed 2 tests.
     301PASS: AsyncTestSuite.AsyncSetupAndTeardown should have passed 2 tests.
     302PASS: AsyncTestSuite.AsyncSetupAndTeardown should have failed 0 tests.
     303PASS: AsyncTestSuite.AsyncSetupAndTeardown should have skipped 0 tests.
     304
     305== Running test suite: AsyncTestSuite.AsyncSetupExplicitException
     306-- Running test setup.
     307Throwing...
     308!! EXCEPTION: AsyncSetupExplicitException throw
     309Stack Trace: (suppressed)
     310
     311-- Running test setup.
     312PASS: Setup action should still execute if previous test's setup action threw an exception.
     313-- Running test case: AsyncTestAfterSetupExplicitException
     314PASS: Test should still execute if previous test's setup action threw an exception.
     315-- Running test teardown.
     316PASS: Teardown action should still execute if previous test's setup action threw an exception.
     317
     318PASS: Promise from AsyncTestSuite.AsyncSetupExplicitException.runTestCases() should resolve even if a test case fails.
     319PASS: AsyncTestSuite.AsyncSetupExplicitException should have executed 1 tests.
     320PASS: AsyncTestSuite.AsyncSetupExplicitException should have passed 1 tests.
     321PASS: AsyncTestSuite.AsyncSetupExplicitException should have failed 1 tests.
     322PASS: AsyncTestSuite.AsyncSetupExplicitException should have skipped 1 tests.
     323
     324== Running test suite: AsyncTestSuite.AsyncSetupRuntimeException
     325-- Running test setup.
     326Throwing...
    147327!! EXCEPTION: undefined is not an object (evaluating '({}).x.x')
    148328Stack Trace: (suppressed)
    149 PASS: Promise from asyncTeardownRuntimeFailureTestSuite.runTestCases() should reject.
    150 PASS: Promise did evaluate the async teardown function.
    151 PASS: Rejected value should be a runtime exception.
    152 
     329
     330-- Running test setup.
     331PASS: Setup action should still execute if previous test's setup action threw an exception.
     332-- Running test case: AsyncTestAfterSetupRuntimeException
     333PASS: Test should still execute if previous test's setup action threw an exception.
     334-- Running test teardown.
     335PASS: Teardown action should still execute if previous test's setup action threw an exception.
     336
     337PASS: Promise from AsyncTestSuite.AsyncSetupRuntimeException.runTestCases() should resolve even if a test case fails.
     338PASS: AsyncTestSuite.AsyncSetupRuntimeException should have executed 1 tests.
     339PASS: AsyncTestSuite.AsyncSetupRuntimeException should have passed 1 tests.
     340PASS: AsyncTestSuite.AsyncSetupRuntimeException should have failed 1 tests.
     341PASS: AsyncTestSuite.AsyncSetupRuntimeException should have skipped 1 tests.
     342
     343== Running test suite: AsyncTestSuite.AsyncSetupFailure
     344-- Running test setup.
     345Rejecting...
     346!! EXCEPTION: AsyncSetupFailure reject
     347Stack Trace: (suppressed)
     348
     349-- Running test setup.
     350PASS: Setup action should still execute if previous test's setup action failed.
     351-- Running test case: AsyncTestAfterSetupFailure
     352PASS: Test should still execute if previous test's setup action failed.
     353-- Running test teardown.
     354PASS: Setup action should still execute if previous test's setup action failed.
     355
     356PASS: Promise from AsyncTestSuite.AsyncSetupFailure.runTestCases() should resolve even if a test case fails.
     357PASS: AsyncTestSuite.AsyncSetupFailure should have executed 1 tests.
     358PASS: AsyncTestSuite.AsyncSetupFailure should have passed 1 tests.
     359PASS: AsyncTestSuite.AsyncSetupFailure should have failed 1 tests.
     360PASS: AsyncTestSuite.AsyncSetupFailure should have skipped 1 tests.
     361
     362== Running test suite: AsyncTestSuite.AsyncTeardownExplicitException
     363-- Running test case: AsyncTestWithExplicitExceptionDuringTeardown
     364-- Running test teardown.
     365Throwing...
     366!! EXCEPTION: AsyncTeardownExplicitException throw
     367Stack Trace: (suppressed)
     368
     369-- Running test setup.
     370PASS: Setup action should still execute if previous test's teardown action threw an exception.
     371-- Running test case: AsyncTestAfterTeardownExplicitException
     372PASS: Test should still execute if previous test's teardown action threw an exception.
     373-- Running test teardown.
     374PASS: Teardown action should still execute if previous test's teardown action threw an exception.
     375
     376PASS: Promise from AsyncTestSuite.AsyncTeardownExplicitException.runTestCases() should resolve even if a test case fails.
     377PASS: AsyncTestSuite.AsyncTeardownExplicitException should have executed 2 tests.
     378PASS: AsyncTestSuite.AsyncTeardownExplicitException should have passed 1 tests.
     379PASS: AsyncTestSuite.AsyncTeardownExplicitException should have failed 1 tests.
     380PASS: AsyncTestSuite.AsyncTeardownExplicitException should have skipped 0 tests.
     381
     382== Running test suite: AsyncTestSuite.AsyncTeardownRuntimeException
     383-- Running test case: AsyncTestWithRuntimeExceptionDuringTeardown
     384-- Running test teardown.
     385Throwing...
     386!! EXCEPTION: undefined is not an object (evaluating '({}).x.x')
     387Stack Trace: (suppressed)
     388
     389-- Running test setup.
     390PASS: Setup action should still execute if previous test's teardown action threw an exception.
     391-- Running test case: AsyncTestAfterTeardownRuntimeException
     392PASS: Test should still execute if previous test's teardown action threw an exception.
     393-- Running test teardown.
     394PASS: Teardown action should still execute if previous test's teardown action threw an exception.
     395
     396PASS: Promise from AsyncTestSuite.AsyncTeardownRuntimeException.runTestCases() should resolve even if a test case fails.
     397PASS: AsyncTestSuite.AsyncTeardownRuntimeException should have executed 2 tests.
     398PASS: AsyncTestSuite.AsyncTeardownRuntimeException should have passed 1 tests.
     399PASS: AsyncTestSuite.AsyncTeardownRuntimeException should have failed 1 tests.
     400PASS: AsyncTestSuite.AsyncTeardownRuntimeException should have skipped 0 tests.
     401
     402== Running test suite: AsyncTestSuite.AsyncTeardownFailure
     403-- Running test case: AsyncTestWithFailureDuringTeardown
     404-- Running test teardown.
     405Rejecting...
     406!! EXCEPTION: AsyncTeardownFailure reject
     407Stack Trace: (suppressed)
     408
     409-- Running test setup.
     410PASS: Setup action should still execute if previous test's teardown action failed.
     411-- Running test case: AsyncTestAfterTeardownFailure
     412PASS: Test should still execute if previous test's teardown action failed.
     413-- Running test teardown.
     414PASS: Teardown action should still execute if previous test's teardown action failed.
     415
     416PASS: Promise from AsyncTestSuite.AsyncTeardownFailure.runTestCases() should resolve even if a test case fails.
     417PASS: AsyncTestSuite.AsyncTeardownFailure should have executed 2 tests.
     418PASS: AsyncTestSuite.AsyncTeardownFailure should have passed 1 tests.
     419PASS: AsyncTestSuite.AsyncTeardownFailure should have failed 1 tests.
     420PASS: AsyncTestSuite.AsyncTeardownFailure should have skipped 0 tests.
     421
     422== Running test suite: AsyncTestSuite.AsyncTimeout
     423-- Running test case: AsyncTestWithTimeout
     424Timeout...
     425!! TIMEOUT: took longer than 5ms
     426
     427-- Running test setup.
     428PASS: Setup action should still execute if previous test timed out.
     429-- Running test case: AsyncTestAfterTimeout
     430PASS: Test should still execute if previous test timed out.
     431-- Running test teardown.
     432PASS: Teardown action should still execute if previous test timed out.
     433
     434PASS: Promise from AsyncTestSuite.AsyncTimeout.runTestCases() should resolve even if a test case fails.
     435PASS: AsyncTestSuite.AsyncTimeout should have executed 2 tests.
     436PASS: AsyncTestSuite.AsyncTimeout should have passed 1 tests.
     437PASS: AsyncTestSuite.AsyncTimeout should have failed 1 tests.
     438PASS: AsyncTestSuite.AsyncTimeout should have skipped 0 tests.
     439

  • trunk/LayoutTests/inspector/unit-tests/async-test-suite.html

    r225820 r243953  
    1 <!doctype html>
     1<!DOCTYPE html>
    22<html>
    33<head>
    … …  
    88    ProtocolTest.suppressStackTraces = true;
    99
    10     try {
    11         let result = new AsyncTestSuite(this);
    12         ProtocolTest.fail("instantiating AsyncTestSuite requires name argument.");
    13     } catch (e) {
    14         ProtocolTest.pass("instantiating AsyncTestSuite requires name argument.");
    15     }
    16 
    17     try {
    18         let result = new AsyncTestSuite(this, {});
    19         ProtocolTest.fail("instantiating AsyncTestSuite requires string name argument.");
    20     } catch (e) {
    21         ProtocolTest.pass("instantiating AsyncTestSuite requires string name argument.");
    22     }
    23 
    24     try {
    25         let result = new AsyncTestSuite(this, "      ");
    26         ProtocolTest.fail("instantiating AsyncTestSuite requires non-whitespace name argument.");
    27     } catch (e) {
    28         ProtocolTest.pass("instantiating AsyncTestSuite requires non-whitespace name argument.");
    29     }
    30 
    31     try {
    32         let result = new AsyncTestSuite("something", {});
    33         ProtocolTest.fail("instantiating AsyncTestSuite requires test harness argument.");
    34     } catch (e) {
    35         ProtocolTest.pass("instantiating AsyncTestSuite requires test harness argument.");
    36     }
     10    ProtocolTest.expectException(() => {
     11        new AsyncTestSuite(this);
     12    });
     13
     14    ProtocolTest.expectException(() => {
     15        new AsyncTestSuite(this, {});
     16    });
     17
     18    ProtocolTest.expectException(() => {
     19        new AsyncTestSuite(this, "      ");
     20    });
     21
     22    ProtocolTest.expectException(() => {
     23        new AsyncTestSuite("something", {});
     24    });
    3725
    3826    let badArgsSuite = ProtocolTest.createAsyncSuite("dummy");
    39     try {
     27    ProtocolTest.expectException(() => {
    4028        badArgsSuite.addTestCase();
    41         ProtocolTest.fail("should not be able to add empty test case.");
    42     } catch (e) {
    43         ProtocolTest.pass("should not be able to add empty test case.");
    44     }
    45     try {
     29    });
     30    ProtocolTest.expectException(() => {
    4631        badArgsSuite.addTestCase("string");
    47         ProtocolTest.fail("should not be able to add non-object test case.");
    48     } catch (e) {
    49         ProtocolTest.pass("should not be able to add non-object test case.");
    50     }
    51     try {
     32    });
     33    ProtocolTest.expectException(() => {
    5234        badArgsSuite.addTestCase({
    5335            name: {},
    54             test() {},
    55         });
    56         ProtocolTest.fail("test case should require string name.");
    57     } catch (e) {
    58         ProtocolTest.pass("test case should require string name.");
    59     }
    60     try {
     36            test() {
     37            },
     38        });
     39    });
     40    ProtocolTest.expectException(() => {
    6141        badArgsSuite.addTestCase({
    6242            name: "        ",
    63             test() {},
    64         });
    65         ProtocolTest.fail("test case should require non-whitespace name.");
    66     } catch (e) {
    67         ProtocolTest.pass("test case should require non-whitespace name.");
    68     }
    69     try {
     43            test() {
     44            },
     45        });
     46    });
     47    ProtocolTest.expectException(() => {
    7048        badArgsSuite.addTestCase({
    7149            name: "foo",
    7250            test: null,
    7351        });
    74         ProtocolTest.fail("test case should require test function.");
    75     } catch (e) {
    76         ProtocolTest.pass("test case should require test function.");
    77     }
    78     try {
     52    });
     53    ProtocolTest.expectException(() => {
    7954        badArgsSuite.addTestCase({
    8055            name: "foo",
    81             test() {},
    82             setup: "astd"
    83         });
    84         ProtocolTest.fail("should not be able to specify non-Function `setup` parameter.");
    85     } catch (e) {
    86         ProtocolTest.pass("should not be able to specify non-Function `setup` parameter.");
    87     }
    88     try {
     56            test() {
     57            },
     58            setup: "astd",
     59        });
     60    });
     61    ProtocolTest.expectException(() => {
    8962        badArgsSuite.addTestCase({
    9063            name: "foo",
    91             test() {},
    92             setup: 123
    93         });
    94         ProtocolTest.fail("should not be able to specify non-Function `setup` parameter.");
    95     } catch (e) {
    96         ProtocolTest.pass("should not be able to specify non-Function `setup` parameter.");
    97     }
    98     try {
     64            test() {
     65            },
     66            setup: 123,
     67        });
     68    });
     69    ProtocolTest.expectException(() => {
    9970        badArgsSuite.addTestCase({
    10071            name: "foo",
    101             test() {},
    102             setup: {}
    103         });
    104         ProtocolTest.fail("should not be able to specify non-Function `setup` parameter.");
    105     } catch (e) {
    106         ProtocolTest.pass("should not be able to specify non-Function `setup` parameter.");
    107     }
    108     try {
     72            test() {
     73            },
     74            setup: {},
     75        });
     76    });
     77    ProtocolTest.expectException(() => {
    10978        badArgsSuite.addTestCase({
    11079            name: "foo",
    111             test() {},
    112             teardown: "astd"
    113         });
    114         ProtocolTest.fail("should not be able to specify non-Function `teardown` parameter.");
    115     } catch (e) {
    116         ProtocolTest.pass("should not be able to specify non-Function `teardown` parameter.");
    117     }
    118     try {
     80            test() {
     81            },
     82            teardown: "astd",
     83        });
     84    });
     85    ProtocolTest.expectException(() => {
    11986        badArgsSuite.addTestCase({
    12087            name: "foo",
    121             test() {},
    122             teardown: 123
    123         });
    124         ProtocolTest.fail("should not be able to specify non-Function `teardown` parameter.");
    125     } catch (e) {
    126         ProtocolTest.pass("should not be able to specify non-Function `teardown` parameter.");
    127     }
    128     try {
     88            test() {
     89            },
     90            teardown: 123,
     91        });
     92    });
     93    ProtocolTest.expectException(() => {
    12994        badArgsSuite.addTestCase({
    13095            name: "foo",
    131             test() {},
    132             teardown: {}
    133         });
    134         ProtocolTest.fail("should not be able to specify non-Function `teardown` parameter.");
    135     } catch (e) {
    136         ProtocolTest.pass("should not be able to specify non-Function `teardown` parameter.");
    137     }
     96            test() {
     97            },
     98            teardown: {},
     99        });
     100    });
    138101
    139102    let runEmptySuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.RunEmptySuite");
    … …  
    145108    }
    146109
     110    let runTwiceSuiteRunCount = 0;
    147111    let runTwiceSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.RunTwiceSuite");
    148112    runTwiceSuite.addTestCase({
    149113        name: "DummyTest0",
    150         description: "Check that a suite can't run more than once.",
    151         test(resolve, reject) { resolve(); }
     114        test(resolve, reject) {
     115            ProtocolTest.expectEqual(++runTwiceSuiteRunCount, 1, "DummyTest0 should only run once.");
     116            resolve();
     117        },
    152118    });
    153119
    154120    let result = runTwiceSuite.runTestCases();
    155     try {
     121    ProtocolTest.expectException(() => {
    156122        // Test cases won't run in this event loop; this call should still throw.
    157123        // Later tests are chained to this suite to avoid nondeterminism.
    158124        runTwiceSuite.runTestCases();
    159         ProtocolTest.fail("should not be able to run a test suite twice.");
    160     } catch (e) {
    161         ProtocolTest.pass("should not be able to run a test suite twice.");
     125    });
     126
     127    function checkResult(suite, expectedCounts) {
     128        result = result.then(() => suite.runTestCases());
     129
     130        let message = `Promise from ${suite.name}.runTestCases() should resolve even if a test case fails.`;
     131
     132        result = result.then(function resolved() {
     133            ProtocolTest.log("");
     134            ProtocolTest.pass(message);
     135
     136            ProtocolTest.expectEqual(expectedCounts.runCount, suite.runCount, `${suite.name} should have executed ${expectedCounts.runCount} tests.`);
     137            ProtocolTest.expectEqual(expectedCounts.passCount, suite.passCount, `${suite.name} should have passed ${expectedCounts.passCount} tests.`);
     138            ProtocolTest.expectEqual(expectedCounts.failCount, suite.failCount, `${suite.name} should have failed ${expectedCounts.failCount} tests.`);
     139            ProtocolTest.expectEqual(expectedCounts.skipCount, suite.skipCount, `${suite.name} should have skipped ${expectedCounts.skipCount} tests.`);
     140        }, function rejected(e) {
     141            ProtocolTest.log("");
     142            ProtocolTest.fail(message);
     143
     144            ProtocolTest.log(e.message);
     145        });
    162146    }
    163147
    164     let rejectToken = {"token": 666};
    165     let thrownError = new Error(rejectToken);
    166 
    167     let sequentialExecutionSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.SequentialExecution");
    168     sequentialExecutionSuite.addTestCase({
    169         name: "DummyTest1",
    170         description: "Check test case execution order.",
    171         test(resolve, reject) { resolve(); }
    172     });
    173     sequentialExecutionSuite.addTestCase({
    174         name: "DummyTest2",
    175         description: "Check test case execution order.",
    176         test(resolve, reject) { resolve(); }
    177     });
    178     sequentialExecutionSuite.addTestCase({
    179         name: "DummyTest3",
    180         description: "Check test case execution order.",
    181         test(resolve, reject) { resolve(); }
    182     });
    183     sequentialExecutionSuite.addTestCase({
    184         name: "FailingTest4",
    185         description: "Check that test fails by throwing an Error instance.",
    186         test(resolve, reject) { throw thrownError; }
    187     });
    188 
    189     let abortOnFailureSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.AbortOnFailure");
    190     abortOnFailureSuite.addTestCase({
    191         name: "PassingTest5",
    192         description: "This test is a dummy.",
    193         test(resolve, reject) { resolve(); }
    194     });
    195     abortOnFailureSuite.addTestCase({
    196         name: "FailingTest6",
    197         description: "This test should fail by explicitly calling the `reject` callback.",
    198         test(resolve, reject) { reject(rejectToken); }
    199     });
    200     abortOnFailureSuite.addTestCase({
    201         name: "PassingTest7",
    202         description: "This test should not executed when the preceding test fails.",
    203         test(resolve, reject) { resolve(); }
    204     });
    205 
     148    // Promise test functions.
     149
     150    let promiseFunctionSuccessSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.PromiseFunctionSuccess");
     151    promiseFunctionSuccessSuite.addTestCase({
     152        name: "PromiseFunctionSuccess",
     153        test(resolve, reject) {
     154            resolve();
     155        },
     156    });
     157    checkResult(promiseFunctionSuccessSuite, {
     158        runCount: 1,
     159        passCount: 1,
     160        failCount: 0,
     161        skipCount: 0,
     162    });
     163
     164    let promiseFunctionExceptionSuite = ProtocolTest.createAsyncSuite("PromiseTestSuite.PromiseFunctionException");
     165    promiseFunctionExceptionSuite.addTestCase({
     166        name: "PromiseFunctionException",
     167        test(resolve, reject) {
     168            ProtocolTest.log("Throwing...");
     169            throw "PromiseFunctionException throw";
     170        },
     171    });
     172    checkResult(promiseFunctionExceptionSuite, {
     173        runCount: 1,
     174        passCount: 0,
     175        failCount: 1,
     176        skipCount: 0,
     177    });
     178
     179    let promiseFunctionFailureSuite = ProtocolTest.createAsyncSuite("PromiseTestSuite.PromiseFunctionFailure");
     180    promiseFunctionFailureSuite.addTestCase({
     181        name: "PromiseFunctionFailure",
     182        test(resolve, reject) {
     183            ProtocolTest.log("Rejecting...");
     184            reject("PromiseFunctionFailure reject");
     185        },
     186    });
     187    checkResult(promiseFunctionFailureSuite, {
     188        runCount: 1,
     189        passCount: 0,
     190        failCount: 1,
     191        skipCount: 0,
     192    });
     193
     194    let promiseSequentialExecutionPhase = 0;
     195    let promiseSequentialExecutionSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.PromiseSequentialExecution");
     196    promiseSequentialExecutionSuite.addTestCase({
     197        name: "1 (Pass)",
     198        test(resolve, reject) {
     199            ProtocolTest.assert(promiseSequentialExecutionPhase === 0);
     200            promiseSequentialExecutionPhase = 1;
     201            resolve();
     202        },
     203    });
     204    promiseSequentialExecutionSuite.addTestCase({
     205        name: "2 (Pass)",
     206        test(resolve, reject) {
     207            ProtocolTest.assert(promiseSequentialExecutionPhase === 1);
     208            promiseSequentialExecutionPhase = 2;
     209            resolve();
     210        },
     211    });
     212    promiseSequentialExecutionSuite.addTestCase({
     213        name: "3 (Pass)",
     214        test(resolve, reject) {
     215            ProtocolTest.assert(promiseSequentialExecutionPhase === 2);
     216            promiseSequentialExecutionPhase = 3;
     217            resolve();
     218        },
     219    });
     220    promiseSequentialExecutionSuite.addTestCase({
     221        name: "4 (Pass)",
     222        test(resolve, reject) {
     223            ProtocolTest.assert(promiseSequentialExecutionPhase === 3);
     224            promiseSequentialExecutionPhase = 4;
     225            resolve();
     226        },
     227    });
     228    checkResult(promiseSequentialExecutionSuite, {
     229        runCount: 4,
     230        passCount: 4,
     231        failCount: 0,
     232        skipCount: 0,
     233    });
    206234    result = result.then(() => {
    207         let promise = sequentialExecutionSuite.runTestCases();
    208         ProtocolTest.expectThat(result instanceof Promise, "AsyncTestSuite.RunTestCases() should return a Promise.");
    209         return promise;
    210     });
    211     result = result.then(function resolved() {
    212         ProtocolTest.fail("Promise from sequentialExecutionSuite.runTestCases() should reject when a test case fails.");
    213         return Promise.resolve(); // Continue this test.
    214     }, function rejected(e) {
    215         ProtocolTest.pass("Promise from sequentialExecutionSuite.runTestCases() should reject when a test case fails.");
    216         ProtocolTest.expectThat(e === thrownError, "Promise from sequentialExecutionSuite.runTestCases() should reject without altering its result value.");
    217 
    218         ProtocolTest.expectThat(sequentialExecutionSuite.runCount === 4, "sequentialExecutionSuite should have executed four tests.");
    219         ProtocolTest.expectThat(sequentialExecutionSuite.passCount === 3, "sequentialExecutionSuite should have passed three tests.");
    220         ProtocolTest.expectThat(sequentialExecutionSuite.failCount === 1, "sequentialExecutionSuite should have failed 1 test.");
    221         ProtocolTest.expectThat(sequentialExecutionSuite.skipCount === 0, "sequentialExecutionSuite should have skipped zero tests.");
    222         return Promise.resolve(); // Continue this test.
    223     });
    224 
     235        ProtocolTest.assert(promiseSequentialExecutionPhase === 4);
     236    });
     237
     238    let promiseContinueOnFailureSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.PromiseContinueOnFailure");
     239    promiseContinueOnFailureSuite.addTestCase({
     240        name: "1 (Pass)",
     241        test(resolve, reject) {
     242            resolve();
     243        },
     244    });
     245    promiseContinueOnFailureSuite.addTestCase({
     246        name: "2 (Fail)",
     247        test(resolve, reject) {
     248            ProtocolTest.log("Throwing...");
     249            throw {x: "PromiseContinueOnFailure throw"};
     250        },
     251    });
     252    promiseContinueOnFailureSuite.addTestCase({
     253        name: "3 (Pass)",
     254        test(resolve, reject) {
     255            resolve();
     256        },
     257    });
     258    promiseContinueOnFailureSuite.addTestCase({
     259        name: "4 (Fail)",
     260        test(resolve, reject) {
     261            ProtocolTest.log("Rejecting...");
     262            reject({x: "PromiseContinueOnFailure reject"});
     263        },
     264    });
     265    checkResult(promiseContinueOnFailureSuite, {
     266        runCount: 4,
     267        passCount: 2,
     268        failCount: 2,
     269        skipCount: 0,
     270    });
     271
     272    let promiseSetupAndTeardownPhase = 0;
     273    let promiseSetupAndTeardownSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.PromiseSetupAndTeardown");
     274    promiseSetupAndTeardownSuite.addTestCase({
     275        name: "TestWithSetupAndTeardown",
     276        setup(resolve, reject) {
     277            ProtocolTest.assert(promiseSetupAndTeardownPhase === 0);
     278            promiseSetupAndTeardownPhase = 1;
     279            resolve();
     280        },
     281        test(resolve, reject) {
     282            ProtocolTest.assert(promiseSetupAndTeardownPhase === 1);
     283            promiseSetupAndTeardownPhase = 2;
     284            resolve();
     285        },
     286        teardown(resolve, reject) {
     287            ProtocolTest.assert(promiseSetupAndTeardownPhase === 2);
     288            promiseSetupAndTeardownPhase = 3;
     289            resolve();
     290        },
     291    });
     292    promiseSetupAndTeardownSuite.addTestCase({
     293        name: "TestRunningAfterTeardown",
     294        test(resolve, reject) {
     295            ProtocolTest.assert(promiseSetupAndTeardownPhase === 3);
     296            resolve();
     297        },
     298    });
     299    checkResult(promiseSetupAndTeardownSuite, {
     300        runCount: 2,
     301        passCount: 2,
     302        failCount: 0,
     303        skipCount: 0,
     304    });
     305
     306    let promiseSetupExceptionSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.PromiseSetupException");
     307    promiseSetupExceptionSuite.addTestCase({
     308        name: "TestWithExceptionDuringSetup",
     309        setup(resolve, reject) {
     310            ProtocolTest.log("Throwing...");
     311            throw "PromiseSetupException throw";
     312        },
     313        test(resolve, reject) {
     314            ProtocolTest.fail("Test should not execute if its setup action threw an exception.");
     315            reject();
     316        },
     317        teardown(resolve, reject) {
     318            ProtocolTest.fail("Teardown action should not execute if its setup action threw an exception.");
     319            reject();
     320        },
     321    });
     322    promiseSetupExceptionSuite.addTestCase({
     323        name: "TestAfterSetupException",
     324        setup(resolve, reject) {
     325            ProtocolTest.pass("Setup action should still execute if previous test's setup action threw an exception.");
     326            resolve();
     327        },
     328        test(resolve, reject) {
     329            ProtocolTest.pass("Test should still execute if previous test's setup action threw an exception.");
     330            resolve();
     331        },
     332        teardown(resolve, reject) {
     333            ProtocolTest.pass("Teardown action should still execute if previous test's setup action threw an exception.");
     334            resolve();
     335        },
     336    });
     337    checkResult(promiseSetupExceptionSuite, {
     338        runCount: 1,
     339        passCount: 1,
     340        failCount: 1,
     341        skipCount: 1,
     342    });
     343
     344    let promiseSetupFailureSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.PromiseSetupFailure");
     345    promiseSetupFailureSuite.addTestCase({
     346        name: "TestWithFailureDuringSetup",
     347        setup(resolve, reject) {
     348            ProtocolTest.log("Rejecting...");
     349            reject("PromiseSetupFailure reject");
     350        },
     351        test(resolve, reject) {
     352            ProtocolTest.fail("Test should not execute if its setup action failed.");
     353            reject();
     354        },
     355        teardown(resolve, reject) {
     356            ProtocolTest.fail("Teardown action should not execute if its setup action failed.");
     357            reject();
     358        },
     359    });
     360    promiseSetupFailureSuite.addTestCase({
     361        name: "TestAfterSetupException",
     362        setup(resolve, reject) {
     363            ProtocolTest.pass("Setup action should still execute if previous test's setup action failed.");
     364            resolve();
     365        },
     366        test(resolve, reject) {
     367            ProtocolTest.pass("Test should still execute if previous test's setup action failed.");
     368            resolve();
     369        },
     370        teardown(resolve, reject) {
     371            ProtocolTest.pass("Teardown action should still execute if previous test's setup action failed.");
     372            resolve();
     373        },
     374    });
     375    checkResult(promiseSetupFailureSuite, {
     376        runCount: 1,
     377        passCount: 1,
     378        failCount: 1,
     379        skipCount: 1,
     380    });
     381
     382    let promiseTeardownExceptionSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.PromiseTeardownException");
     383    promiseTeardownExceptionSuite.addTestCase({
     384        name: "TestWithExceptionDuringTeardown",
     385        test(resolve, reject) {
     386            resolve();
     387        },
     388        teardown(resolve, reject) {
     389            ProtocolTest.log("Throwing...");
     390            throw "PromiseTeardownException throw";
     391        },
     392    });
     393    promiseTeardownExceptionSuite.addTestCase({
     394        name: "TestAfterTeardownException",
     395        setup(resolve, reject) {
     396            ProtocolTest.pass("Setup action should still execute if previous test's teardown action threw an exception.");
     397            resolve();
     398        },
     399        test(resolve, reject) {
     400            ProtocolTest.pass("Test should still execute if previous test's teardown action threw an exception.");
     401            resolve();
     402        },
     403        teardown(resolve, reject) {
     404            ProtocolTest.pass("Teardown action should still execute if previous test's teardown action threw an exception.");
     405            resolve();
     406        },
     407    });
     408    checkResult(promiseTeardownExceptionSuite, {
     409        runCount: 2,
     410        passCount: 1,
     411        failCount: 1,
     412        skipCount: 0,
     413    });
     414
     415    let promiseTeardownFailureSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.PromiseTeardownFailure");
     416    promiseTeardownFailureSuite.addTestCase({
     417        name: "TestWithExceptionDuringTeardown",
     418        test(resolve, reject) {
     419            resolve();
     420        },
     421        teardown(resolve, reject) {
     422            ProtocolTest.log("Rejecting...");
     423            reject("PromiseTeardownFailure reject");
     424        },
     425    });
     426    promiseTeardownFailureSuite.addTestCase({
     427        name: "TestAfterTeardownException",
     428        setup(resolve, reject) {
     429            ProtocolTest.pass("Setup action should still execute if previous test's teardown action failed.");
     430            resolve();
     431        },
     432        test(resolve, reject) {
     433            ProtocolTest.pass("Test should still execute if previous test's teardown action failed.");
     434            resolve();
     435        },
     436        teardown(resolve, reject) {
     437            ProtocolTest.pass("Teardown action should still execute if previous test's teardown action failed.");
     438            resolve();
     439        },
     440    });
     441    checkResult(promiseTeardownFailureSuite, {
     442        runCount: 2,
     443        passCount: 1,
     444        failCount: 1,
     445        skipCount: 0,
     446    });
     447
     448    let promiseTimeoutSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.PromiseTimeout");
     449    promiseTimeoutSuite.addTestCase({
     450        name: "PromiseTestWithTimeout",
     451        test(resolve, reject) {
     452            ProtocolTest.log("Timeout...");
     453            setTimeout(() => {
     454                resolve();
     455            }, 50);
     456        },
     457        timeout: 5,
     458    });
     459    promiseTimeoutSuite.addTestCase({
     460        name: "PromiseTestAfterTimeout",
     461        setup(resolve, reject) {
     462            ProtocolTest.pass("Setup action should still execute if previous test timed out.");
     463            resolve();
     464        },
     465        test(resolve, reject) {
     466            ProtocolTest.pass("Test should still execute if previous test timed out.");
     467            resolve();
     468        },
     469        teardown(resolve, reject) {
     470            ProtocolTest.pass("Teardown action should still execute if previous test timed out.");
     471            resolve();
     472        },
     473    });
     474    checkResult(promiseTimeoutSuite, {
     475        runCount: 2,
     476        passCount: 1,
     477        failCount: 1,
     478        skipCount: 0,
     479    });
     480
     481    // Async test functions.
     482
     483    let asyncFunctionSuccessSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.AsyncFunctionSuccess");
     484    asyncFunctionSuccessSuite.addTestCase({
     485        name: "AsyncFunctionSuccess",
     486        async test() {
     487        },
     488    });
     489    checkResult(asyncFunctionSuccessSuite, {
     490        runCount: 1,
     491        passCount: 1,
     492        failCount: 0,
     493        skipCount: 0,
     494    });
     495
     496    let asyncFunctionExplicitExceptionSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.AsyncFunctionExplicitException");
     497    asyncFunctionExplicitExceptionSuite.addTestCase({
     498        name: "AsyncFunctionExplicitException",
     499        async test() {
     500            ProtocolTest.log("Throwing...");
     501            throw "AsyncFunctionExplicitException throw";
     502        },
     503    });
     504    checkResult(asyncFunctionExplicitExceptionSuite, {
     505        runCount: 1,
     506        passCount: 0,
     507        failCount: 1,
     508        skipCount: 0,
     509    });
     510
     511    let asyncFunctionRuntimeExceptionSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.AsyncFunctionRuntimeException");
     512    asyncFunctionRuntimeExceptionSuite.addTestCase({
     513        name: "AsyncFunctionRuntimeException",
     514        async test() {
     515            ProtocolTest.log("Throwing...");
     516            ({}).x.x.x;
     517        },
     518    });
     519    checkResult(asyncFunctionRuntimeExceptionSuite, {
     520        runCount: 1,
     521        passCount: 0,
     522        failCount: 1,
     523        skipCount: 0,
     524    });
     525
     526    let asyncFunctionFailureSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.AsyncFunctionFailure");
     527    asyncFunctionFailureSuite.addTestCase({
     528        name: "AsyncFunctionException",
     529        async test() {
     530            ProtocolTest.log("Rejecting...");
     531            return Promise.reject("AsyncFunctionFailure reject");
     532        },
     533    });
     534    checkResult(asyncFunctionFailureSuite, {
     535        runCount: 1,
     536        passCount: 0,
     537        failCount: 1,
     538        skipCount: 0,
     539    });
     540
     541    let asyncSequentialExecutionPhase = 0;
     542    let asyncSequentialExecutionSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.AsyncSequentialExecution");
     543    asyncSequentialExecutionSuite.addTestCase({
     544        name: "1 (Pass)",
     545        async test() {
     546            ProtocolTest.assert(asyncSequentialExecutionPhase === 0);
     547            asyncSequentialExecutionPhase = 1;
     548        },
     549    });
     550    asyncSequentialExecutionSuite.addTestCase({
     551        name: "2 (Pass)",
     552        async test() {
     553            ProtocolTest.assert(asyncSequentialExecutionPhase === 1);
     554            asyncSequentialExecutionPhase = 2;
     555        },
     556    });
     557    asyncSequentialExecutionSuite.addTestCase({
     558        name: "3 (Pass)",
     559        async test() {
     560            ProtocolTest.assert(asyncSequentialExecutionPhase === 2);
     561            asyncSequentialExecutionPhase = 3;
     562        },
     563    });
     564    asyncSequentialExecutionSuite.addTestCase({
     565        name: "4 (Pass)",
     566        async test() {
     567            ProtocolTest.assert(asyncSequentialExecutionPhase === 3);
     568            asyncSequentialExecutionPhase = 4;
     569        },
     570    });
     571    checkResult(asyncSequentialExecutionSuite, {
     572        runCount: 4,
     573        passCount: 4,
     574        failCount: 0,
     575        skipCount: 0,
     576    });
    225577    result = result.then(() => {
    226         return abortOnFailureSuite.runTestCases();
    227     }).then(function resolved() {
    228         ProtocolTest.fail("Promise from abortOnFailureSuite.runTestCases() should reject when a test case fails.");
    229         return Promise.resolve(); // Continue this test.
    230     }, function rejected(e) {
    231         ProtocolTest.pass("Promise from abortOnFailureSuite.runTestCases() should reject when a test case fails.");
    232         ProtocolTest.expectThat(e === rejectToken, "Promise from abortOnFailureSuite.runTestCases() should reject without altering its result value.");
    233         ProtocolTest.expectThat(abortOnFailureSuite.runCount === 2, "abortOnFailureSuite should have executed two tests.");
    234         ProtocolTest.expectThat(abortOnFailureSuite.passCount === 1, "abortOnFailureSuite should have passed one test.");
    235         ProtocolTest.expectThat(abortOnFailureSuite.failCount === 1, "abortOnFailureSuite should have failed one test.");
    236         ProtocolTest.expectThat(abortOnFailureSuite.skipCount === 1, "abortOnFailureSuite should have skipped one test.");
    237 
    238         return Promise.resolve(); // Continue this test.
    239     });
    240 
    241     var setupAndTeardownSymbol = Symbol("async-suite-setup-and-teardown-token");
    242     window[setupAndTeardownSymbol] = 0;
    243 
    244     let setupAndTeardownTestSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.SetupAndTeardown");
    245     setupAndTeardownTestSuite.addTestCase({
    246         name: "TestWithSetupAndTeardown",
    247         description: "Check execution order for setup and teardown actions.",
    248         setup: (resolve, reject) => {
    249             window[setupAndTeardownSymbol] = 1;
    250             resolve();
    251         },
    252         test(resolve, reject) {
    253             ProtocolTest.expectThat(window[setupAndTeardownSymbol] === 1, "Test should see side effects of running setup() action.");
    254             window[setupAndTeardownSymbol] = 2;
    255             resolve();
    256         },
    257         teardown: (resolve, reject) => {
    258             ProtocolTest.expectThat(window[setupAndTeardownSymbol] === 2, "Teardown should see side effects of running setup() action.");
    259             window[setupAndTeardownSymbol] = 3;
    260             resolve();
    261         }
    262     });
    263     setupAndTeardownTestSuite.addTestCase({
    264         name: "TestRunningAfterTeardown",
    265         description: "Check execution order for test after a teardown action.",
    266         test(resolve, reject) {
    267             ProtocolTest.expectThat(window[setupAndTeardownSymbol] === 3, "Test should see side effects of previous test's teardown() action.");
    268             resolve();
    269         },
    270     });
    271 
     578        ProtocolTest.assert(asyncSequentialExecutionPhase === 4);
     579    });
     580
     581    let asyncContinueOnFailureSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.AsyncContinueOnFailure");
     582    asyncContinueOnFailureSuite.addTestCase({
     583        name: "1 (Pass)",
     584        async test() {
     585        },
     586    });
     587    asyncContinueOnFailureSuite.addTestCase({
     588        name: "2 (Fail)",
     589        async test() {
     590            ProtocolTest.log("Throwing...");
     591            throw {x: "AsyncContinueOnFailure throw"};
     592        },
     593    });
     594    asyncContinueOnFailureSuite.addTestCase({
     595        name: "3 (Pass)",
     596        async test() {
     597        },
     598    });
     599    asyncContinueOnFailureSuite.addTestCase({
     600        name: "4 (Fail)",
     601        async test() {
     602            ProtocolTest.log("Throwing...");
     603            ({}).x.x.x;
     604        },
     605    });
     606    asyncContinueOnFailureSuite.addTestCase({
     607        name: "5 (Pass)",
     608        async test() {
     609        },
     610    });
     611    asyncContinueOnFailureSuite.addTestCase({
     612        name: "6 (Fail)",
     613        async test() {
     614            ProtocolTest.log("Rejecting...");
     615            return Promise.reject("AsyncContinueOnFailure reject");
     616        },
     617    });
     618    checkResult(asyncContinueOnFailureSuite, {
     619        runCount: 6,
     620        passCount: 3,
     621        failCount: 3,
     622        skipCount: 0,
     623    });
     624
     625    let asyncSetupAndTeardownPhase = 0;
     626    let asyncSetupAndTeardownSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.AsyncSetupAndTeardown");
     627    asyncSetupAndTeardownSuite.addTestCase({
     628        name: "AsyncTestWithSetupAndTeardown",
     629        async setup() {
     630            ProtocolTest.assert(asyncSetupAndTeardownPhase === 0);
     631            asyncSetupAndTeardownPhase = 1;
     632        },
     633        async test() {
     634            ProtocolTest.assert(asyncSetupAndTeardownPhase === 1);
     635            asyncSetupAndTeardownPhase = 2;
     636        },
     637        async teardown() {
     638            ProtocolTest.assert(asyncSetupAndTeardownPhase === 2);
     639            asyncSetupAndTeardownPhase = 3;
     640        },
     641    });
     642    asyncSetupAndTeardownSuite.addTestCase({
     643        name: "AsyncTestRunningAfterTeardown",
     644        async test() {
     645            ProtocolTest.assert(asyncSetupAndTeardownPhase === 3);
     646        },
     647    });
     648    checkResult(asyncSetupAndTeardownSuite, {
     649        runCount: 2,
     650        passCount: 2,
     651        failCount: 0,
     652        skipCount: 0,
     653    });
     654
     655    let asyncSetupExplicitExceptionSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.AsyncSetupExplicitException");
     656    asyncSetupExplicitExceptionSuite.addTestCase({
     657        name: "AsyncTestWithExplicitExceptionDuringSetup",
     658        async setup() {
     659            ProtocolTest.log("Throwing...");
     660            throw "AsyncSetupExplicitException throw";
     661        },
     662        async test() {
     663            ProtocolTest.fail("Test should not execute if its setup action threw an exception.");
     664        },
     665        async teardown() {
     666            ProtocolTest.fail("Teardown action should not execute if its setup action threw an exception.");
     667        },
     668    });
     669    asyncSetupExplicitExceptionSuite.addTestCase({
     670        name: "AsyncTestAfterSetupExplicitException",
     671        async setup() {
     672            ProtocolTest.pass("Setup action should still execute if previous test's setup action threw an exception.");
     673        },
     674        async test() {
     675            ProtocolTest.pass("Test should still execute if previous test's setup action threw an exception.");
     676        },
     677        async teardown() {
     678            ProtocolTest.pass("Teardown action should still execute if previous test's setup action threw an exception.");
     679        },
     680    });
     681    checkResult(asyncSetupExplicitExceptionSuite, {
     682        runCount: 1,
     683        passCount: 1,
     684        failCount: 1,
     685        skipCount: 1,
     686    });
     687
     688    let asyncSetupRuntimeExceptionSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.AsyncSetupRuntimeException");
     689    asyncSetupRuntimeExceptionSuite.addTestCase({
     690        name: "AsyncTestWithRuntimeExceptionDuringSetup",
     691        async setup() {
     692            ProtocolTest.log("Throwing...");
     693            ({}).x.x.x;
     694        },
     695        async test() {
     696            ProtocolTest.fail("Test should not execute if its setup action threw an exception.");
     697        },
     698        async teardown() {
     699            ProtocolTest.fail("Teardown action should not execute if its setup action threw an exception.");
     700        },
     701    });
     702    asyncSetupRuntimeExceptionSuite.addTestCase({
     703        name: "AsyncTestAfterSetupRuntimeException",
     704        async setup() {
     705            ProtocolTest.pass("Setup action should still execute if previous test's setup action threw an exception.");
     706        },
     707        async test() {
     708            ProtocolTest.pass("Test should still execute if previous test's setup action threw an exception.");
     709        },
     710        async teardown() {
     711            ProtocolTest.pass("Teardown action should still execute if previous test's setup action threw an exception.");
     712        },
     713    });
     714    checkResult(asyncSetupRuntimeExceptionSuite, {
     715        runCount: 1,
     716        passCount: 1,
     717        failCount: 1,
     718        skipCount: 1,
     719    });
     720
     721    let asyncSetupFailureSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.AsyncSetupFailure");
     722    asyncSetupFailureSuite.addTestCase({
     723        name: "AsyncTestWithFailureDuringSetup",
     724        async setup() {
     725            ProtocolTest.log("Rejecting...");
     726            return Promise.reject("AsyncSetupFailure reject");
     727        },
     728        async test() {
     729        },
     730    });
     731    asyncSetupFailureSuite.addTestCase({
     732        name: "AsyncTestAfterSetupFailure",
     733        async setup() {
     734            ProtocolTest.pass("Setup action should still execute if previous test's setup action failed.");
     735        },
     736        async test() {
     737            ProtocolTest.pass("Test should still execute if previous test's setup action failed.");
     738        },
     739        async teardown() {
     740            ProtocolTest.pass("Setup action should still execute if previous test's setup action failed.");
     741        },
     742    });
     743    checkResult(asyncSetupFailureSuite, {
     744        runCount: 1,
     745        passCount: 1,
     746        failCount: 1,
     747        skipCount: 1,
     748    });
     749
     750    let asyncTeardownExplicitExceptionSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.AsyncTeardownExplicitException");
     751    asyncTeardownExplicitExceptionSuite.addTestCase({
     752        name: "AsyncTestWithExplicitExceptionDuringTeardown",
     753        async test() {
     754        },
     755        async teardown() {
     756            ProtocolTest.log("Throwing...");
     757            throw "AsyncTeardownExplicitException throw";
     758        },
     759    });
     760    asyncTeardownExplicitExceptionSuite.addTestCase({
     761        name: "AsyncTestAfterTeardownExplicitException",
     762        async setup() {
     763            ProtocolTest.pass("Setup action should still execute if previous test's teardown action threw an exception.");
     764        },
     765        async test() {
     766            ProtocolTest.pass("Test should still execute if previous test's teardown action threw an exception.");
     767        },
     768        async teardown() {
     769            ProtocolTest.pass("Teardown action should still execute if previous test's teardown action threw an exception.");
     770        },
     771    });
     772    checkResult(asyncTeardownExplicitExceptionSuite, {
     773        runCount: 2,
     774        passCount: 1,
     775        failCount: 1,
     776        skipCount: 0,
     777    });
     778
     779    let asyncTeardownRuntimeExceptionSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.AsyncTeardownRuntimeException");
     780    asyncTeardownRuntimeExceptionSuite.addTestCase({
     781        name: "AsyncTestWithRuntimeExceptionDuringTeardown",
     782        async test() {
     783        },
     784        async teardown() {
     785            ProtocolTest.log("Throwing...");
     786            ({}).x.x.x;
     787        },
     788    });
     789    asyncTeardownRuntimeExceptionSuite.addTestCase({
     790        name: "AsyncTestAfterTeardownRuntimeException",
     791        async setup() {
     792            ProtocolTest.pass("Setup action should still execute if previous test's teardown action threw an exception.");
     793        },
     794        async test() {
     795            ProtocolTest.pass("Test should still execute if previous test's teardown action threw an exception.");
     796        },
     797        async teardown() {
     798            ProtocolTest.pass("Teardown action should still execute if previous test's teardown action threw an exception.");
     799        },
     800    });
     801    checkResult(asyncTeardownRuntimeExceptionSuite, {
     802        runCount: 2,
     803        passCount: 1,
     804        failCount: 1,
     805        skipCount: 0,
     806    });
     807
     808    let asyncTeardownFailureSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.AsyncTeardownFailure");
     809    asyncTeardownFailureSuite.addTestCase({
     810        name: "AsyncTestWithFailureDuringTeardown",
     811        async test() {
     812        },
     813        async teardown() {
     814            ProtocolTest.log("Rejecting...");
     815            return Promise.reject("AsyncTeardownFailure reject");
     816        },
     817    });
     818    asyncTeardownFailureSuite.addTestCase({
     819        name: "AsyncTestAfterTeardownFailure",
     820        async setup() {
     821            ProtocolTest.pass("Setup action should still execute if previous test's teardown action failed.");
     822        },
     823        async test() {
     824            ProtocolTest.pass("Test should still execute if previous test's teardown action failed.");
     825        },
     826        async teardown() {
     827            ProtocolTest.pass("Teardown action should still execute if previous test's teardown action failed.");
     828        },
     829    });
     830    checkResult(asyncTeardownFailureSuite, {
     831        runCount: 2,
     832        passCount: 1,
     833        failCount: 1,
     834        skipCount: 0,
     835    });
     836
     837    let asyncTimeoutSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.AsyncTimeout");
     838    asyncTimeoutSuite.addTestCase({
     839        name: "AsyncTestWithTimeout",
     840        async test() {
     841            ProtocolTest.log("Timeout...");
     842            await Promise.delay(50);
     843        },
     844        timeout: 5,
     845    });
     846    asyncTimeoutSuite.addTestCase({
     847        name: "AsyncTestAfterTimeout",
     848        async setup() {
     849            ProtocolTest.pass("Setup action should still execute if previous test timed out.");
     850        },
     851        async test() {
     852            ProtocolTest.pass("Test should still execute if previous test timed out.");
     853        },
     854        async teardown() {
     855            ProtocolTest.pass("Teardown action should still execute if previous test timed out.");
     856        },
     857    });
     858    checkResult(asyncTimeoutSuite, {
     859        runCount: 2,
     860        passCount: 1,
     861        failCount: 1,
     862        skipCount: 0,
     863    });
     864
     865    // This will finish the test whether the chain was resolved or rejected.
    272866    result = result.then(() => {
    273         return setupAndTeardownTestSuite.runTestCases();
    274     }).then(function resolved() {
    275         ProtocolTest.pass("Promise from setupAndTeardownTestSuite.runTestCases() should resolve.");
    276         return Promise.resolve(); // Continue this test.
    277     }, function rejected(e) {
    278         ProtocolTest.fail("Promise from setupAndTeardownTestSuite.runTestCases() should resolve.");
    279         return Promise.resolve(); // Continue this test.
    280     });
    281 
    282     let setupExceptionTestSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.SetupException");
    283     setupExceptionTestSuite.addTestCase({
    284         name: "TestWithExceptionDuringSetup",
    285         description: "Check execution order for setup action that throws an exception.",
    286         setup: (resolve, reject) => { throw new Error() },
    287         test(resolve, reject) {
    288             ProtocolTest.assert(false, "Test should not execute if its setup action threw an exception.");
    289             reject();
    290         },
    291         teardown: (resolve, reject) => {
    292             ProtocolTest.assert(false, "Teardown action should not execute if its setup action threw an exception.");
    293             reject();
    294         }
    295     });
    296 
    297     result = result.then(() => {
    298         return setupExceptionTestSuite.runTestCases();
    299     }).then(function resolved() {
    300         ProtocolTest.fail("Promise from setupExceptionTestSuite.runTestCases() should reject.");
    301         return Promise.resolve(); // Continue this test.
    302     }, function rejected(e) {
    303         ProtocolTest.pass("Promise from setupExceptionTestSuite.runTestCases() should reject.");
    304         return Promise.resolve(); // Continue this test.
    305     });
    306 
    307     let setupFailureTestSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.SetupFailure");
    308     setupFailureTestSuite.addTestCase({
    309         name: "TestWithFailureDuringSetup",
    310         description: "Check execution order for setup action that has failed.",
    311         setup: (resolve, reject) => { reject(); },
    312         test(resolve, reject) {
    313             ProtocolTest.assert(false, "Test should not execute if its setup action failed.")
    314             reject();
    315         },
    316         teardown: (resolve, reject) => {
    317             ProtocolTest.assert(false, "Teardown action should not execute if its setup action failed.")
    318             reject();
    319         }
    320     });
    321 
    322     result = result.then(() => {
    323         return setupFailureTestSuite.runTestCases();
    324     }).then(function resolved() {
    325         ProtocolTest.fail("Promise from setupFailureTestSuite.runTestCases() should reject.");
    326         return Promise.resolve(); // Continue this test.
    327     }, function rejected(e) {
    328         ProtocolTest.pass("Promise from setupFailureTestSuite.runTestCases() should reject.");
    329         return Promise.resolve(); // Continue this test.
    330     });
    331 
    332     let teardownExceptionTestSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.TeardownException");
    333     teardownExceptionTestSuite.addTestCase({
    334         name: "TestWithExceptionDuringTeardown",
    335         description: "Check execution order for teardown action that throws an exception.",
    336         test(resolve, reject) { resolve(); },
    337         teardown: (resolve, reject) => { throw new Error() }
    338     });
    339     teardownExceptionTestSuite.addTestCase({
    340         name: "TestAfterTeardownException",
    341         descrption: "Check execution order for test after previous test's teardown throws an exception.",
    342         setup: (resolve, reject) => {
    343             ProtocolTest.assert(false, "Setup action should not execute if previous test's teardown action threw an exception.");
    344             reject();
    345         },
    346         test(resolve, reject) {
    347             ProtocolTest.assert(false, "Test should not execute if previous test's teardown action threw an exception.");
    348             reject();
    349         }
    350     });
    351 
    352     result = result.then(() => {
    353         return teardownExceptionTestSuite.runTestCases();
    354     }).then(function resolved() {
    355         ProtocolTest.fail("Promise from teardownExceptionTestSuite.runTestCases() should reject.");
    356         return Promise.resolve(); // Continue this test.
    357     }, function rejected(e) {
    358         ProtocolTest.pass("Promise from teardownExceptionTestSuite.runTestCases() should reject.");
    359         return Promise.resolve(); // Continue this test.
    360     });
    361 
    362     let teardownFailureTestSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.TeardownFailure");
    363     teardownFailureTestSuite.addTestCase({
    364         name: "TestWithExceptionDuringTeardown",
    365         description: "Check execution order for teardown action that has failed.",
    366         test(resolve, reject) { resolve(); },
    367         teardown: (resolve, reject) => { reject(); },
    368     });
    369     teardownFailureTestSuite.addTestCase({
    370         name: "TestAfterTeardownException",
    371         description: "Check execution order for test after previous test's teardown throws an exception",
    372         setup: (resolve, reject) => {
    373             ProtocolTest.assert(false, "Setup action should not execute if previous test's teardown action failed.");
    374             reject();
    375         },
    376         test(resolve, reject) {
    377             ProtocolTest.assert(false, "Test should not execute if previous test's teardown action failed.");
    378             reject();
    379         }
    380     });
    381 
    382     result = result.then(() => {
    383         return teardownFailureTestSuite.runTestCases();
    384     }).then(function resolved() {
    385         ProtocolTest.fail("Promise from teardownFailureTestSuite.runTestCases() should reject.");
    386         return Promise.resolve(); // Continue this test.
    387     }, function rejected(e) {
    388         ProtocolTest.pass("Promise from teardownFailureTestSuite.runTestCases() should reject.");
    389         return Promise.resolve(); // Continue this test.
    390     });
    391 
    392     // Async test functions.
    393     let asyncFunctionSuccessTestSuiteDidEvaluate = false;
    394     let asyncFunctionSuccessTestSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.AsyncFunctionSuccess");
    395     asyncFunctionSuccessTestSuite.addTestCase({
    396         name: "AsyncFunctionSuccess",
    397         description: "Check that an async suite with async test functions can succeed",
    398         async test() {
    399             asyncFunctionSuccessTestSuiteDidEvaluate = true;
    400             return 42;
    401         }
    402     });
    403 
    404     result = result.then(() => {
    405         return asyncFunctionSuccessTestSuite.runTestCases();
    406     }).then(function resolve(x) {
    407         ProtocolTest.pass("Promise from asyncFunctionSuccessTestSuite.runTestCases() should succeed.");
    408         ProtocolTest.expectThat(asyncFunctionSuccessTestSuiteDidEvaluate, "Promise did evaluate the async test function.");
    409         ProtocolTest.expectEqual(x, 42, "Resolved value should be 42.");
    410         return Promise.resolve(); // Continue this test.
    411     }, function rejected(e) {
    412         ProtocolTest.fail("Promise from asyncFunctionSuccessTestSuite.runTestCases() should succeed.");
    413         return Promise.resolve(); // Continue this test.
    414     });
    415 
    416     let asyncFunctionExplicitFailureTestSuiteDidEvaluate = false;
    417     let asyncFunctionExplicitFailureTestSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.AsyncFunctionExplicitFailure");
    418     asyncFunctionExplicitFailureTestSuite.addTestCase({
    419         name: "AsyncFunctionFailure",
    420         description: "Check that an async suite with async test functions that throws will reject",
    421         async test() {
    422             asyncFunctionExplicitFailureTestSuiteDidEvaluate = true;
    423             throw "AsyncFunctionFailure Exception Message";
    424         }
    425     });
    426 
    427     result = result.then(() => {
    428         return asyncFunctionExplicitFailureTestSuite.runTestCases();
    429     }).then(function resolve() {
    430         ProtocolTest.fail("Promise from asyncFunctionExplicitFailureTestSuite.runTestCases() should reject.");
    431         return Promise.resolve(); // Continue this test.
    432     }, function rejected(e) {
    433         ProtocolTest.pass("Promise from asyncFunctionExplicitFailureTestSuite.runTestCases() should reject.");
    434         ProtocolTest.expectThat(asyncFunctionExplicitFailureTestSuiteDidEvaluate, "Promise did evaluate the async test function.");
    435         ProtocolTest.expectEqual(e, "AsyncFunctionFailure Exception Message", "Rejected value should be thrown exception.");
    436         return Promise.resolve(); // Continue this test.
    437     });
    438 
    439     let asyncFunctionRuntimeFailureTestSuiteDidEvaluate = false;
    440     let asyncFunctionRuntimeFailureTestSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.AsyncFunctionRuntimeFailure");
    441     asyncFunctionRuntimeFailureTestSuite.addTestCase({
    442         name: "AsyncFunctionFailure",
    443         description: "Check that an async suite with async test functions that throws with a runtime error will reject",
    444         async test() {
    445             asyncFunctionRuntimeFailureTestSuiteDidEvaluate = true;
    446             ({}).x.x.x;
    447         }
    448     });
    449 
    450     result = result.then(() => {
    451         return asyncFunctionRuntimeFailureTestSuite.runTestCases();
    452     }).then(function resolve() {
    453         ProtocolTest.fail("Promise from asyncFunctionRuntimeFailureTestSuite.runTestCases() should reject.");
    454         return Promise.resolve(); // Continue this test.
    455     }, function rejected(e) {
    456         ProtocolTest.pass("Promise from asyncFunctionRuntimeFailureTestSuite.runTestCases() should reject.");
    457         ProtocolTest.expectThat(asyncFunctionRuntimeFailureTestSuiteDidEvaluate, "Promise did evaluate the async test function.");
    458         ProtocolTest.expectThat(e instanceof TypeError, "Rejected value should be a runtime exception.");
    459         return Promise.resolve(); // Continue this test.
    460     });
    461 
    462     // Async setup() and teardown() success test cases.
    463     const asyncSetupAndTeardownSymbol = Symbol("async-suite-async-setup-and-teardown-token");
    464     window[asyncSetupAndTeardownSymbol] = 0;
    465 
    466     let asyncSetupAndAsyncTeardownTestSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.AsyncSetupAndAsyncTeardown");
    467     asyncSetupAndAsyncTeardownTestSuite.addTestCase({
    468         name: "TestWithSetupAndTeardown",
    469         description: "Check execution order for setup and teardown actions.",
    470         async setup() {
    471             window[asyncSetupAndTeardownSymbol] = 1;
    472         },
    473         async test() {
    474             ProtocolTest.expectThat(window[asyncSetupAndTeardownSymbol] === 1, "Test should see side effects of running setup() action.");
    475             window[asyncSetupAndTeardownSymbol] = 2;
    476         },
    477         async teardown() {
    478             ProtocolTest.expectThat(window[asyncSetupAndTeardownSymbol] === 2, "Teardown should see side effects of running setup() action.");
    479             window[asyncSetupAndTeardownSymbol] = 3;
    480         }
    481     });
    482     asyncSetupAndAsyncTeardownTestSuite.addTestCase({
    483         name: "TestRunningAfterTeardown",
    484         description: "Check execution order for test after a teardown action.",
    485         test(resolve, reject) {
    486             ProtocolTest.expectThat(window[asyncSetupAndTeardownSymbol] === 3, "Test should see side effects of previous test's teardown() action.");
    487             resolve();
    488         },
    489     });
    490 
    491     result = result.then(() => {
    492         return asyncSetupAndAsyncTeardownTestSuite.runTestCases();
    493     }).then(function resolved() {
    494         ProtocolTest.pass("Promise from asyncSetupAndAsyncTeardownTestSuite.runTestCases() should resolve.");
    495         return Promise.resolve(); // Continue this test.
    496     }, function rejected(e) {
    497         ProtocolTest.fail("Promise from asyncSetupAndAsyncTeardownTestSuite.runTestCases() should resolve.");
    498         return Promise.resolve(); // Continue this test.
    499     });
    500 
    501     // Async setup() failure test cases.
    502     let asyncSetupExplicitFailureTestSuiteDidEvaluate = false;
    503     let asyncSetupExplicitFailureTestSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.AsyncSetupExplicitFailure");
    504     asyncSetupExplicitFailureTestSuite.addTestCase({
    505         name: "AsyncFunctionFailure",
    506         description: "Check that an async suite with async test functions that throws will reject",
    507         async test() {
    508             asyncSetupExplicitFailureTestSuiteDidEvaluate = true;
    509             throw "AsyncFunctionFailure Exception Message";
    510         }
    511     });
    512 
    513     result = result.then(() => {
    514         return asyncSetupExplicitFailureTestSuite.runTestCases();
    515     }).then(function resolve() {
    516         ProtocolTest.fail("Promise from asyncSetupExplicitFailureTestSuite.runTestCases() should reject.");
    517         return Promise.resolve(); // Continue this test.
    518     }, function rejected(e) {
    519         ProtocolTest.pass("Promise from asyncSetupExplicitFailureTestSuite.runTestCases() should reject.");
    520         ProtocolTest.expectThat(asyncSetupExplicitFailureTestSuiteDidEvaluate, "Promise did evaluate the async setup function.");
    521         ProtocolTest.expectEqual(e, "AsyncFunctionFailure Exception Message", "Rejected value should be thrown exception.");
    522         return Promise.resolve(); // Continue this test.
    523     });
    524 
    525     let asyncSetupRuntimeFailureTestSuiteDidEvaluate = false;
    526     let asyncSetupRuntimeFailureTestSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.AsyncSetupRuntimeFailure");
    527     asyncSetupRuntimeFailureTestSuite.addTestCase({
    528         name: "AsyncFunctionFailure",
    529         description: "Check that an async suite with an async setup function that throws with a runtime error will reject",
    530         async setup() {
    531             asyncSetupRuntimeFailureTestSuiteDidEvaluate = true;
    532             ({}).x.x.x;
    533         },
    534         async test() { return true; },
    535     });
    536 
    537     result = result.then(() => {
    538         return asyncSetupRuntimeFailureTestSuite.runTestCases();
    539     }).then(function resolve() {
    540         ProtocolTest.fail("Promise from asyncSetupRuntimeFailureTestSuite.runTestCases() should reject.");
    541         return Promise.resolve(); // Continue this test.
    542     }, function rejected(e) {
    543         ProtocolTest.pass("Promise from asyncSetupRuntimeFailureTestSuite.runTestCases() should reject.");
    544         ProtocolTest.expectThat(asyncSetupRuntimeFailureTestSuiteDidEvaluate, "Promise did evaluate the async setup function.");
    545         ProtocolTest.expectThat(e instanceof TypeError, "Rejected value should be a runtime exception.");
    546         return Promise.resolve(); // Continue this test.
    547     });
    548 
    549     // Async teardown() failure test cases.
    550     let asyncTeardownExplicitFailureTestSuiteDidEvaluate = false;
    551     let asyncTeardownExplicitFailureTestSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.AsyncTeardownExplicitFailure");
    552     asyncTeardownExplicitFailureTestSuite.addTestCase({
    553         name: "AsyncFunctionFailure",
    554         description: "Check that an async suite with async test functions that throws will reject",
    555         async test() { return true; },
    556         async teardown() {
    557             asyncTeardownExplicitFailureTestSuiteDidEvaluate = true;
    558             throw "AsyncFunctionFailure Exception Message";
    559         },
    560     });
    561 
    562     result = result.then(() => {
    563         return asyncTeardownExplicitFailureTestSuite.runTestCases();
    564     }).then(function resolve() {
    565         ProtocolTest.fail("Promise from asyncTeardownExplicitFailureTestSuite.runTestCases() should reject.");
    566         return Promise.resolve(); // Continue this test.
    567     }, function rejected(e) {
    568         ProtocolTest.pass("Promise from asyncTeardownExplicitFailureTestSuite.runTestCases() should reject.");
    569         ProtocolTest.expectThat(asyncTeardownExplicitFailureTestSuiteDidEvaluate, "Promise did evaluate the async teardown function.");
    570         ProtocolTest.expectEqual(e, "AsyncFunctionFailure Exception Message", "Rejected value should be thrown exception.");
    571         return Promise.resolve(); // Continue this test.
    572     });
    573 
    574     let asyncTeardownRuntimeFailureTestSuiteDidEvaluate = false;
    575     let asyncTeardownRuntimeFailureTestSuite = ProtocolTest.createAsyncSuite("AsyncTestSuite.AsyncTeardownRuntimeFailure");
    576     asyncTeardownRuntimeFailureTestSuite.addTestCase({
    577         name: "AsyncFunctionFailure",
    578         description: "Check that an async suite with an async teardown function that throws with a runtime error will reject",
    579         async test() { return true; },
    580         async teardown() {
    581             asyncTeardownRuntimeFailureTestSuiteDidEvaluate = true;
    582             ({}).x.x.x;
    583         },
    584     });
    585 
    586     result = result.then(() => {
    587         return asyncTeardownRuntimeFailureTestSuite.runTestCases();
    588     }).then(function resolve() {
    589         ProtocolTest.fail("Promise from asyncTeardownRuntimeFailureTestSuite.runTestCases() should reject.");
    590         return Promise.resolve(); // Continue this test.
    591     }, function rejected(e) {
    592         ProtocolTest.pass("Promise from asyncTeardownRuntimeFailureTestSuite.runTestCases() should reject.");
    593         ProtocolTest.expectThat(asyncTeardownRuntimeFailureTestSuiteDidEvaluate, "Promise did evaluate the async teardown function.");
    594         ProtocolTest.expectThat(e instanceof TypeError, "Rejected value should be a runtime exception.");
    595         return Promise.resolve(); // Continue this test.
    596     });
    597 
    598     // This will finish the test whether the chain was resolved or rejected.
    599     result = result.then(() => { ProtocolTest.completeTest(); });
     867        ProtocolTest.completeTest();
     868    });
    600869}
    601870</script>

  • trunk/LayoutTests/inspector/unit-tests/sync-test-suite-expected.txt

    r225814 r243953  
    1 PASS: instantiating SyncTestSuite requires name argument.
    2 PASS: instantiating SyncTestSuite requires string name argument.
    3 PASS: instantiating SyncTestSuite requires non-whitespace name argument.
    4 PASS: instantiating SyncTestSuite requires test harness argument.
    5 PASS: should not be able to add empty test case.
    6 PASS: should not be able to add non-object test case.
    7 PASS: test case should require string name.
    8 PASS: test case should require non-whitespace name.
    9 PASS: test case should require test function.
    10 PASS: should not be able to specify non-Function `setup` parameter.
    11 PASS: should not be able to specify non-Function `setup` parameter.
    12 PASS: should not be able to specify non-Function `setup` parameter.
    13 PASS: should not be able to specify non-Function `teardown` parameter.
    14 PASS: should not be able to specify non-Function `teardown` parameter.
    15 PASS: should not be able to specify non-Function `teardown` parameter.
    16 PASS: should not be able to specify async `test` parameter.
    17 PASS: should not be able to specify async `setup` parameter.
    18 PASS: should not be able to specify async `teardown` parameter.
    19 PASS: should not be able to run empty test suite.
     1PASS: Should produce an exception.
     2Error: Must pass the test's harness as the first argument.
     3PASS: Should produce an exception.
     4Error: Must pass the test's harness as the first argument.
     5PASS: Should produce an exception.
     6Error: Must pass the test's harness as the first argument.
     7PASS: Should produce an exception.
     8Error: Must pass the test's harness as the first argument.
     9PASS: Should produce an exception.
     10TypeError: undefined is not an object (evaluating 'testcase.setup')
     11PASS: Should produce an exception.
     12Error: Tried to add non-object test case.
     13PASS: Should produce an exception.
     14Error: Tried to add test case without a name.
     15PASS: Should produce an exception.
     16Error: Tried to add test case without a name.
     17PASS: Should produce an exception.
     18Error: Tried to add test case without `test` function.
     19PASS: Should produce an exception.
     20Error: Tried to add test case with invalid `setup` parameter (must be a function).
     21PASS: Should produce an exception.
     22Error: Tried to add test case with invalid `setup` parameter (must be a function).
     23PASS: Should produce an exception.
     24Error: Tried to add test case with invalid `setup` parameter (must be a function).
     25PASS: Should produce an exception.
     26Error: Tried to add test case with invalid `teardown` parameter (must be a function).
     27PASS: Should produce an exception.
     28Error: Tried to add test case with invalid `teardown` parameter (must be a function).
     29PASS: Should produce an exception.
     30Error: Tried to add test case with invalid `teardown` parameter (must be a function).
     31PASS: Should produce an exception.
     32Error: Tried to pass a test case with an async `setup`, `test`, or `teardown` function, but this is a synchronous test suite.
     33PASS: Should produce an exception.
     34Error: Tried to pass a test case with an async `setup`, `test`, or `teardown` function, but this is a synchronous test suite.
     35PASS: Should produce an exception.
     36Error: Tried to pass a test case with an async `setup`, `test`, or `teardown` function, but this is a synchronous test suite.
     37PASS: Should produce an exception.
     38Error: Tried to call runTestCases() for suite with no test cases
    2039
    2140== Running test suite: SyncTestSuite.RunTwiceSuite
    2241-- Running test case: DummyTest0
    2342PASS: Return value of runTwiceSuite.runTestCases() should be true when all tests pass.
    24 PASS: should not be able to run a test suite twice.
     43PASS: Should produce an exception.
     44Error: Tried to call runTestCases() more than once.
    2545
    2646== Running test suite: SyncTestSuite.SequentialExecution
    27 -- Running test case: DummyTest1
    28 -- Running test case: DummyTest2
    29 -- Running test case: DummyTest3
    30 -- Running test case: FailingTest4
    31 !! EXCEPTION: [object Object]
     47-- Running test case: 1 (Pass)
     48-- Running test case: 2 (Pass)
     49-- Running test case: 3 (Pass)
     50-- Running test case: 4 (Pass)
     51PASS: Return value of SyncTestSuite.SequentialExecution.runTestCases() should be true even if a test case fails.
     52PASS: SyncTestSuite.SequentialExecution should have executed 4 tests.
     53PASS: SyncTestSuite.SequentialExecution should have passed 4 tests.
     54PASS: SyncTestSuite.SequentialExecution should have failed 0 tests.
     55PASS: SyncTestSuite.SequentialExecution should have skipped 0 tests.
     56
     57== Running test suite: SyncTestSuite.ContinueOnFailure
     58-- Running test case: 1 (Pass)
     59-- Running test case: 2 (Fail)
     60Throwing...
     61!! EXCEPTION: {"x":"ContinueOnFailure throw"}
    3262Stack Trace: (suppressed)
    33 PASS: Return value of sequentialExecutionSuite.runTestCases() should be false when a test case fails.
    34 PASS: sequentialExecutionSuite should have executed four tests.
    35 PASS: sequentialExecutionSuite should have passed three tests.
    36 PASS: sequentialExecutionSuite should have failed 1 test.
    37 PASS: sequentialExecutionSuite should have skipped zero tests.
    3863
    39 == Running test suite: SyncTestSuite.AbortOnFailure
    40 -- Running test case: PassingTest5
    41 -- Running test case: FailingTest6
    42 PASS: Return value of abortOnFailureSuite.runTestCases() should be false when a test case fails.
    43 PASS: abortOnFailureSuite should have executed two tests.
    44 PASS: abortOnFailureSuite should have passed one test.
    45 PASS: abortOnFailureSuite should have failed one test.
    46 PASS: abortOnFailureSuite should have skipped one test.
     64-- Running test case: 3 (Pass)
     65-- Running test case: 4 (Fail)
     66Failing...
     67PASS: Return value of SyncTestSuite.ContinueOnFailure.runTestCases() should be true even if a test case fails.
     68PASS: SyncTestSuite.ContinueOnFailure should have executed 4 tests.
     69PASS: SyncTestSuite.ContinueOnFailure should have passed 2 tests.
     70PASS: SyncTestSuite.ContinueOnFailure should have failed 2 tests.
     71PASS: SyncTestSuite.ContinueOnFailure should have skipped 0 tests.
    4772
    4873== Running test suite: SyncTestSuite.SetupAndTeardown
    4974-- Running test setup.
    5075-- Running test case: TestWithSetupAndTeardown
    51 PASS: Test should see side effects of running setup() action.
    5276-- Running test teardown.
    53 PASS: Teardown should see side effects of running setup() action.
    54 
    5577-- Running test case: TestRunningAfterTeardown
    56 PASS: Test should see side effects of previous test's teardown() action.
     78PASS: Return value of SyncTestSuite.SetupAndTeardown.runTestCases() should be true even if a test case fails.
     79PASS: SyncTestSuite.SetupAndTeardown should have executed 2 tests.
     80PASS: SyncTestSuite.SetupAndTeardown should have passed 2 tests.
     81PASS: SyncTestSuite.SetupAndTeardown should have failed 0 tests.
     82PASS: SyncTestSuite.SetupAndTeardown should have skipped 0 tests.
    5783
    5884== Running test suite: SyncTestSuite.SetupException
    5985-- Running test setup.
    60 !! EXCEPTION:
     86Throwing...
     87!! EXCEPTION: SetupException throw
    6188Stack Trace: (suppressed)
     89
     90-- Running test setup.
     91PASS: Setup action should still execute if previous test's setup action threw an exception.
     92-- Running test case: TestAfterSetupException
     93PASS: Test should still execute if previous test's setup action threw an exception.
     94-- Running test teardown.
     95PASS: Teardown action should still execute if previous test's setup action threw an exception.
     96PASS: Return value of SyncTestSuite.SetupException.runTestCases() should be true even if a test case fails.
     97PASS: SyncTestSuite.SetupException should have executed 1 tests.
     98PASS: SyncTestSuite.SetupException should have passed 1 tests.
     99PASS: SyncTestSuite.SetupException should have failed 1 tests.
     100PASS: SyncTestSuite.SetupException should have skipped 1 tests.
    62101
    63102== Running test suite: SyncTestSuite.SetupFailure
    64103-- Running test setup.
     104Failing...
    65105!! SETUP FAILED
     106
     107-- Running test setup.
     108PASS: Setup action should still execute if previous test's setup action failed.
     109-- Running test case: TestAfterSetupException
     110PASS: Test should still execute if previous test's setup action failed.
     111-- Running test teardown.
     112PASS: Teardown action should still execute if previous test's setup action failed.
     113PASS: Return value of SyncTestSuite.SetupFailure.runTestCases() should be true even if a test case fails.
     114PASS: SyncTestSuite.SetupFailure should have executed 1 tests.
     115PASS: SyncTestSuite.SetupFailure should have passed 1 tests.
     116PASS: SyncTestSuite.SetupFailure should have failed 1 tests.
     117PASS: SyncTestSuite.SetupFailure should have skipped 1 tests.
    66118
    67119== Running test suite: SyncTestSuite.TeardownException
    68120-- Running test case: TestWithExceptionDuringTeardown
    69121-- Running test teardown.
    70 !! EXCEPTION:
     122Throwing...
     123!! EXCEPTION: TeardownException throw
    71124Stack Trace: (suppressed)
     125
     126-- Running test setup.
     127PASS: Setup action should still execute if previous test's teardown action threw an exception.
     128-- Running test case: TestAfterTeardownException
     129PASS: Test should still execute if previous test's teardown action threw an exception.
     130-- Running test teardown.
     131PASS: Teardown action should still execute if previous test's teardown action threw an exception.
     132PASS: Return value of SyncTestSuite.TeardownException.runTestCases() should be true even if a test case fails.
     133PASS: SyncTestSuite.TeardownException should have executed 2 tests.
     134PASS: SyncTestSuite.TeardownException should have passed 1 tests.
     135PASS: SyncTestSuite.TeardownException should have failed 1 tests.
     136PASS: SyncTestSuite.TeardownException should have skipped 0 tests.
    72137
    73138== Running test suite: SyncTestSuite.TeardownFailure
    74139-- Running test case: TestWithExceptionDuringTeardown
    75140-- Running test teardown.
     141Failing...
    76142!! TEARDOWN FAILED
    77143
     144-- Running test setup.
     145PASS: Setup action should still execute if previous test's teardown action failed.
     146-- Running test case: TestAfterTeardownException
     147PASS: Test should still execute if previous test's teardown action failed.
     148-- Running test teardown.
     149PASS: Teardown action should still execute if previous test's teardown action failed.
     150PASS: Return value of SyncTestSuite.TeardownFailure.runTestCases() should be true even if a test case fails.
     151PASS: SyncTestSuite.TeardownFailure should have executed 2 tests.
     152PASS: SyncTestSuite.TeardownFailure should have passed 1 tests.
     153PASS: SyncTestSuite.TeardownFailure should have failed 1 tests.
     154PASS: SyncTestSuite.TeardownFailure should have skipped 0 tests.
     155

  • trunk/LayoutTests/inspector/unit-tests/sync-test-suite.html

    r225814 r243953  
    1 <!doctype html>
     1<!DOCTYPE html>
    22<html>
    33<head>
    … …  
    88    ProtocolTest.suppressStackTraces = true;
    99
    10     try {
    11         let result = new SyncTestSuite(this);
    12         ProtocolTest.log("FAIL: instantiating SyncTestSuite requires name argument.");
    13     } catch (e) {
    14         ProtocolTest.log("PASS: instantiating SyncTestSuite requires name argument.");
    15     }
    16 
    17     try {
    18         let result = new SyncTestSuite(this, {});
    19         ProtocolTest.log("FAIL: instantiating SyncTestSuite requires string name argument.");
    20     } catch (e) {
    21         ProtocolTest.log("PASS: instantiating SyncTestSuite requires string name argument.");
    22     }
    23 
    24     try {
    25         let result = new SyncTestSuite(this, "      ");
    26         ProtocolTest.log("FAIL: instantiating SyncTestSuite requires non-whitespace name argument.");
    27     } catch (e) {
    28         ProtocolTest.log("PASS: instantiating SyncTestSuite requires non-whitespace name argument.");
    29     }
    30 
    31     try {
    32         let result = new SyncTestSuite("something", {});
    33         ProtocolTest.log("FAIL: instantiating SyncTestSuite requires test harness argument.");
    34     } catch (e) {
    35         ProtocolTest.log("PASS: instantiating SyncTestSuite requires test harness argument.");
    36     }
     10    ProtocolTest.expectException(() => {
     11        new SyncTestSuite(this);
     12    });
     13
     14    ProtocolTest.expectException(() => {
     15        new SyncTestSuite(this, {});
     16    });
     17
     18    ProtocolTest.expectException(() => {
     19        new SyncTestSuite(this, "      ");
     20    });
     21
     22    ProtocolTest.expectException(() => {
     23        new SyncTestSuite("something", {});
     24    });
    3725
    3826    let badArgsSuite = ProtocolTest.createSyncSuite("dummy");
    39     try {
     27    ProtocolTest.expectException(() => {
    4028        badArgsSuite.addTestCase();
    41         ProtocolTest.log("FAIL: should not be able to add empty test case.");
    42     } catch (e) {
    43         ProtocolTest.log("PASS: should not be able to add empty test case.");
    44     }
    45     try {
     29    });
     30    ProtocolTest.expectException(() => {
    4631        badArgsSuite.addTestCase("string");
    47         ProtocolTest.log("FAIL: should not be able to add non-object test case.");
    48     } catch (e) {
    49         ProtocolTest.log("PASS: should not be able to add non-object test case.");
    50     }
    51     try {
     32    });
     33    ProtocolTest.expectException(() => {
    5234        badArgsSuite.addTestCase({
    5335            name: {},
    54             test: () => true,
    55         });
    56         ProtocolTest.log("FAIL: test case should require string name.");
    57     } catch (e) {
    58         ProtocolTest.log("PASS: test case should require string name.");
    59     }
    60     try {
     36            test() {
     37                return true;
     38            },
     39        });
     40    });
     41    ProtocolTest.expectException(() => {
    6142        badArgsSuite.addTestCase({
    6243            name: "        ",
    63             test() {},
    64         });
    65         ProtocolTest.log("FAIL: test case should require non-whitespace name.");
    66     } catch (e) {
    67         ProtocolTest.log("PASS: test case should require non-whitespace name.");
    68     }
    69     try {
     44            test() {
     45            },
     46        });
     47    });
     48    ProtocolTest.expectException(() => {
    7049        badArgsSuite.addTestCase({
    7150            name: "foo",
    7251            test: null,
    7352        });
    74         ProtocolTest.log("FAIL: test case should require test function.");
    75     } catch (e) {
    76         ProtocolTest.log("PASS: test case should require test function.");
    77     }
    78     try {
    79         badArgsSuite.addTestCase({
    80             name: "foo",
    81             test() {},
     53    });
     54    ProtocolTest.expectException(() => {
     55        badArgsSuite.addTestCase({
     56            name: "foo",
     57            test() {
     58            },
    8259            setup: "astd"
    8360        });
    84         ProtocolTest.log("FAIL: should not be able to specify non-Function `setup` parameter.");
    85     } catch (e) {
    86         ProtocolTest.log("PASS: should not be able to specify non-Function `setup` parameter.");
    87     }
    88     try {
    89         badArgsSuite.addTestCase({
    90             name: "foo",
    91             test() {},
     61    });
     62    ProtocolTest.expectException(() => {
     63        badArgsSuite.addTestCase({
     64            name: "foo",
     65            test() {
     66            },
    9267            setup: 123
    9368        });
    94         ProtocolTest.log("FAIL: should not be able to specify non-Function `setup` parameter.");
    95     } catch (e) {
    96         ProtocolTest.log("PASS: should not be able to specify non-Function `setup` parameter.");
    97     }
    98     try {
    99         badArgsSuite.addTestCase({
    100             name: "foo",
    101             test() {},
     69    });
     70    ProtocolTest.expectException(() => {
     71        badArgsSuite.addTestCase({
     72            name: "foo",
     73            test() {
     74            },
    10275            setup: Promise.resolve()
    10376        });
    104         ProtocolTest.log("FAIL: should not be able to specify non-Function `setup` parameter.");
    105     } catch (e) {
    106         ProtocolTest.log("PASS: should not be able to specify non-Function `setup` parameter.");
    107     }
    108     try {
    109         badArgsSuite.addTestCase({
    110             name: "foo",
    111             test() {},
     77    });
     78    ProtocolTest.expectException(() => {
     79        badArgsSuite.addTestCase({
     80            name: "foo",
     81            test() {
     82            },
    11283            teardown: "astd"
    11384        });
    114         ProtocolTest.log("FAIL: should not be able to specify non-Function `teardown` parameter.");
    115     } catch (e) {
    116         ProtocolTest.log("PASS: should not be able to specify non-Function `teardown` parameter.");
    117     }
    118     try {
    119         badArgsSuite.addTestCase({
    120             name: "foo",
    121             test() {},
     85    });
     86    ProtocolTest.expectException(() => {
     87        badArgsSuite.addTestCase({
     88            name: "foo",
     89            test() {
     90            },
    12291            teardown: 123
    12392        });
    124         ProtocolTest.log("FAIL: should not be able to specify non-Function `teardown` parameter.");
    125     } catch (e) {
    126         ProtocolTest.log("PASS: should not be able to specify non-Function `teardown` parameter.");
    127     }
    128     try {
    129         badArgsSuite.addTestCase({
    130             name: "foo",
    131             test() {},
     93    });
     94    ProtocolTest.expectException(() => {
     95        badArgsSuite.addTestCase({
     96            name: "foo",
     97            test() {
     98            },
    13299            teardown: Promise.resolve()
    133100        });
    134         ProtocolTest.log("FAIL: should not be able to specify non-Function `teardown` parameter.");
    135     } catch (e) {
    136         ProtocolTest.log("PASS: should not be able to specify non-Function `teardown` parameter.");
    137     }
    138     try {
    139         badArgsSuite.addTestCase({
    140             name: "foo",
    141             async test() {},
    142         });
    143         ProtocolTest.log("FAIL: should not be able to specify async `test` parameter.");
    144     } catch (e) {
    145         ProtocolTest.log("PASS: should not be able to specify async `test` parameter.");
    146     }
    147     try {
    148         badArgsSuite.addTestCase({
    149             name: "foo",
    150             async setup() {},
    151             test() {},
    152         });
    153         ProtocolTest.log("FAIL: should not be able to specify async `setup` parameter.");
    154     } catch (e) {
    155         ProtocolTest.log("PASS: should not be able to specify async `setup` parameter.");
    156     }
    157     try {
    158         badArgsSuite.addTestCase({
    159             name: "foo",
    160             test() {},
    161             async teardown() {},
    162         });
    163         ProtocolTest.log("FAIL: should not be able to specify async `teardown` parameter.");
    164     } catch (e) {
    165         ProtocolTest.log("PASS: should not be able to specify async `teardown` parameter.");
    166     }
    167 
    168     let runEmptySuite = ProtocolTest.createSyncSuite("SyncTestSuite.RunEmptySuite");
    169     try {
     101    });
     102    ProtocolTest.expectException(() => {
     103        badArgsSuite.addTestCase({
     104            name: "foo",
     105            async test() {
     106            },
     107        });
     108    });
     109    ProtocolTest.expectException(() => {
     110        badArgsSuite.addTestCase({
     111            name: "foo",
     112            async setup() {
     113            },
     114            test() {
     115            },
     116        });
     117    });
     118    ProtocolTest.expectException(() => {
     119        badArgsSuite.addTestCase({
     120            name: "foo",
     121            test() {
     122            },
     123            async teardown() {
     124            },
     125        });
     126    });
     127
     128    ProtocolTest.expectException(() => {
     129        let runEmptySuite = ProtocolTest.createSyncSuite("SyncTestSuite.RunEmptySuite");
    170130        runEmptySuite.runTestCases();
    171         ProtocolTest.log("FAIL: should not be able to run empty test suite.");
    172     } catch (e) {
    173         ProtocolTest.log("PASS: should not be able to run empty test suite.");
    174     }
     131    });
    175132
    176133    let runTwiceSuite = ProtocolTest.createSyncSuite("SyncTestSuite.RunTwiceSuite");
    177134    runTwiceSuite.addTestCase({
    178135        name: "DummyTest0",
    179         description: "Check that a suite can't run more than once.",
    180         test: () => true,
    181     });
    182 
    183     try {
    184         let result = runTwiceSuite.runTestCases();
    185         ProtocolTest.expectThat(result === true, "Return value of runTwiceSuite.runTestCases() should be true when all tests pass.");
    186 
    187         runTwiceSuite.runTestCases(); // Try to trigger an error.
    188         ProtocolTest.log("FAIL: should not be able to run a test suite twice.");
    189     } catch (e) {
    190         ProtocolTest.log("PASS: should not be able to run a test suite twice.");
     136        test() {
     137            return true;
     138        },
     139    });
     140
     141    ProtocolTest.expectThat(runTwiceSuite.runTestCases() === true, "Return value of runTwiceSuite.runTestCases() should be true when all tests pass.");
     142
     143    ProtocolTest.expectException(() => {
     144        runTwiceSuite.runTestCases();
     145    });
     146
     147    function checkResult(suite, expectedCounts) {
     148        ProtocolTest.expectThat(suite.runTestCases(), `Return value of ${suite.name}.runTestCases() should be true even if a test case fails.`);
     149        ProtocolTest.expectEqual(expectedCounts.runCount, suite.runCount, `${suite.name} should have executed ${expectedCounts.runCount} tests.`);
     150        ProtocolTest.expectEqual(expectedCounts.passCount, suite.passCount, `${suite.name} should have passed ${expectedCounts.passCount} tests.`);
     151        ProtocolTest.expectEqual(expectedCounts.failCount, suite.failCount, `${suite.name} should have failed ${expectedCounts.failCount} tests.`);
     152        ProtocolTest.expectEqual(expectedCounts.skipCount, suite.skipCount, `${suite.name} should have skipped ${expectedCounts.skipCount} tests.`);
    191153    }
    192154
    193155    let thrownError = new Error({"token": 666});
    194156
     157    let sequentialExecutionPhase = 0;
    195158    let sequentialExecutionSuite = ProtocolTest.createSyncSuite("SyncTestSuite.SequentialExecution");
    196159    sequentialExecutionSuite.addTestCase({
    197         name: "DummyTest1",
    198         description: "Check test case execution order.",
    199         test: () => true
     160        name: "1 (Pass)",
     161        test() {
     162            ProtocolTest.assert(sequentialExecutionPhase === 0);
     163            sequentialExecutionPhase = 1;
     164            return true;
     165        },
    200166    });
    201167    sequentialExecutionSuite.addTestCase({
    202         name: "DummyTest2",
    203         description: "Check test case execution order.",
    204         test: () => true
     168        name: "2 (Pass)",
     169        test() {
     170            ProtocolTest.assert(sequentialExecutionPhase === 1);
     171            sequentialExecutionPhase = 2;
     172            return true;
     173        },
    205174    });
    206175    sequentialExecutionSuite.addTestCase({
    207         name: "DummyTest3",
    208         description: "Check test case execution order.",
    209         test: () => true
     176        name: "3 (Pass)",
     177        test() {
     178            ProtocolTest.assert(sequentialExecutionPhase === 2);
     179            sequentialExecutionPhase = 3;
     180            return true;
     181        },
    210182    });
    211183    sequentialExecutionSuite.addTestCase({
    212         name: "FailingTest4",
    213         description: "Check that test fails by throwing an Error instance.",
    214         test() { throw thrownError; }
    215     });
    216 
    217     let result = sequentialExecutionSuite.runTestCases();
    218     ProtocolTest.expectThat(result === false, "Return value of sequentialExecutionSuite.runTestCases() should be false when a test case fails.");
    219     ProtocolTest.expectThat(sequentialExecutionSuite.runCount === 4, "sequentialExecutionSuite should have executed four tests.");
    220     ProtocolTest.expectThat(sequentialExecutionSuite.passCount === 3, "sequentialExecutionSuite should have passed three tests.");
    221     ProtocolTest.expectThat(sequentialExecutionSuite.failCount === 1, "sequentialExecutionSuite should have failed 1 test.");
    222     ProtocolTest.expectThat(sequentialExecutionSuite.skipCount === 0, "sequentialExecutionSuite should have skipped zero tests.");
    223 
    224     let abortOnFailureSuite = ProtocolTest.createSyncSuite("SyncTestSuite.AbortOnFailure");
    225     abortOnFailureSuite.addTestCase({
    226         name: "PassingTest5",
    227         description: "This test is a dummy.",
    228         test: () => true
    229     });
    230     abortOnFailureSuite.addTestCase({
    231         name: "FailingTest6",
    232         description: "This test should fail by explicitly returning `false`.",
    233         test: () => false
    234     });
    235     abortOnFailureSuite.addTestCase({
    236         name: "PassingTest7",
    237         description: "This test should not executed when the preceding test fails.",
    238         test: () => true
    239     });
    240 
    241     abortOnFailureSuite.runTestCases();
    242     ProtocolTest.expectThat(result === false, "Return value of abortOnFailureSuite.runTestCases() should be false when a test case fails.");
    243     ProtocolTest.expectThat(abortOnFailureSuite.runCount === 2, "abortOnFailureSuite should have executed two tests.");
    244     ProtocolTest.expectThat(abortOnFailureSuite.passCount === 1, "abortOnFailureSuite should have passed one test.");
    245     ProtocolTest.expectThat(abortOnFailureSuite.failCount === 1, "abortOnFailureSuite should have failed one test.");
    246     ProtocolTest.expectThat(abortOnFailureSuite.skipCount === 1, "abortOnFailureSuite should have skipped one test.");
    247 
    248     let setupAndTeardownSymbol = Symbol("sync-suite-setup-and-teardown-token");
    249     window[setupAndTeardownSymbol] = 0;
    250 
    251     let setupAndTeardownTestSuite = ProtocolTest.createSyncSuite("SyncTestSuite.SetupAndTeardown");
    252     setupAndTeardownTestSuite.addTestCase({
     184        name: "4 (Pass)",
     185        test() {
     186            ProtocolTest.assert(sequentialExecutionPhase === 3);
     187            sequentialExecutionPhase = 4;
     188            return true;
     189        },
     190    });
     191    checkResult(sequentialExecutionSuite, {
     192        runCount: 4,
     193        passCount: 4,
     194        failCount: 0,
     195        skipCount: 0,
     196    });
     197
     198    let continueOnFailureSuite = ProtocolTest.createSyncSuite("SyncTestSuite.ContinueOnFailure");
     199    continueOnFailureSuite.addTestCase({
     200        name: "1 (Pass)",
     201        test() {
     202            return true;
     203        },
     204    });
     205    continueOnFailureSuite.addTestCase({
     206        name: "2 (Fail)",
     207        test() {
     208            ProtocolTest.log("Throwing...");
     209            throw {x: "ContinueOnFailure throw"};
     210        },
     211    });
     212    continueOnFailureSuite.addTestCase({
     213        name: "3 (Pass)",
     214        test() {
     215            return true;
     216        },
     217    });
     218    continueOnFailureSuite.addTestCase({
     219        name: "4 (Fail)",
     220        test() {
     221            ProtocolTest.log("Failing...");
     222            return false;
     223        },
     224    });
     225    checkResult(continueOnFailureSuite, {
     226        runCount: 4,
     227        passCount: 2,
     228        failCount: 2,
     229        skipCount: 0,
     230    });
     231
     232    let setupAndTeardownPhase = 0;
     233    let setupAndTeardownSuite = ProtocolTest.createSyncSuite("SyncTestSuite.SetupAndTeardown");
     234    setupAndTeardownSuite.addTestCase({
    253235        name: "TestWithSetupAndTeardown",
    254         description: "Check execution order for setup and teardown actions.",
    255         setup: () => {
    256             window[setupAndTeardownSymbol] = 1;
    257             return true;
    258         },
    259         test() {
    260             ProtocolTest.expectThat(window[setupAndTeardownSymbol] === 1, "Test should see side effects of running setup() action.");
    261             window[setupAndTeardownSymbol] = 2;
    262             return true;
    263         },
    264         teardown: () => {
    265             ProtocolTest.expectThat(window[setupAndTeardownSymbol] === 2, "Teardown should see side effects of running setup() action.");
    266             window[setupAndTeardownSymbol] = 3;
    267             return true;
    268         }
    269     });
    270     setupAndTeardownTestSuite.addTestCase({
     236        setup() {
     237            ProtocolTest.assert(setupAndTeardownPhase === 0);
     238            setupAndTeardownPhase = 1;
     239            return true;
     240        },
     241        test() {
     242            ProtocolTest.assert(setupAndTeardownPhase === 1);
     243            setupAndTeardownPhase = 2;
     244            return true;
     245        },
     246        teardown() {
     247            ProtocolTest.assert(setupAndTeardownPhase === 2);
     248            setupAndTeardownPhase = 3;
     249            return true;
     250        },
     251    });
     252    setupAndTeardownSuite.addTestCase({
    271253        name: "TestRunningAfterTeardown",
    272         description: "Check execution order for test after a teardown action.",
    273         test() {
    274             ProtocolTest.expectThat(window[setupAndTeardownSymbol] === 3, "Test should see side effects of previous test's teardown() action.");
    275             return true;
    276         },
    277     });
    278 
    279     setupAndTeardownTestSuite.runTestCases();
    280 
    281     let setupExceptionTestSuite = ProtocolTest.createSyncSuite("SyncTestSuite.SetupException");
    282     setupExceptionTestSuite.addTestCase({
     254        test() {
     255            ProtocolTest.assert(setupAndTeardownPhase === 3);
     256            return true;
     257        },
     258    });
     259    checkResult(setupAndTeardownSuite, {
     260        runCount: 2,
     261        passCount: 2,
     262        failCount: 0,
     263        skipCount: 0,
     264    });
     265
     266   
     267    let setupExceptionSuite = ProtocolTest.createSyncSuite("SyncTestSuite.SetupException");
     268    setupExceptionSuite.addTestCase({
    283269        name: "TestWithExceptionDuringSetup",
    284         description: "Check execution order for setup action that throws an exception.",
    285         setup: () => { throw new Error() },
    286         test() {
    287             ProtocolTest.assert(false, "Test should not execute if its setup action threw an exception.");
    288             return false;
    289         },
    290         teardown: () => {
    291             ProtocolTest.assert(false, "Teardown action should not execute if its setup action threw an exception.");
    292             return false;           
    293         }
    294     });
    295     setupExceptionTestSuite.runTestCases();
    296 
    297     let setupFailureTestSuite = ProtocolTest.createSyncSuite("SyncTestSuite.SetupFailure");
    298     setupFailureTestSuite.addTestCase({
     270        setup() {
     271            ProtocolTest.log("Throwing...");
     272            throw "SetupException throw";
     273        },
     274        test() {
     275            ProtocolTest.fail("Test should not execute if its setup action threw an exception.");
     276            return false;
     277        },
     278        teardown() {
     279            ProtocolTest.fail("Teardown action should not execute if its setup action threw an exception.");
     280            return false;
     281        },
     282    });
     283    setupExceptionSuite.addTestCase({
     284        name: "TestAfterSetupException",
     285        setup() {
     286            ProtocolTest.pass("Setup action should still execute if previous test's setup action threw an exception.");
     287            return true;
     288        },
     289        test() {
     290            ProtocolTest.pass("Test should still execute if previous test's setup action threw an exception.");
     291            return true;
     292        },
     293        teardown() {
     294            ProtocolTest.pass("Teardown action should still execute if previous test's setup action threw an exception.");
     295            return true;
     296        },
     297    });
     298    checkResult(setupExceptionSuite, {
     299        runCount: 1,
     300        passCount: 1,
     301        failCount: 1,
     302        skipCount: 1,
     303    });
     304
     305    let setupFailureSuite = ProtocolTest.createSyncSuite("SyncTestSuite.SetupFailure");
     306    setupFailureSuite.addTestCase({
    299307        name: "TestWithFailureDuringSetup",
    300         description: "Check execution order for setup action that has failed.",
    301         setup: () => false,
    302         test() {
    303             ProtocolTest.assert(false, "Test should not execute if its setup action returned false.")
    304             return false;
    305         },
    306         teardown: () => {
    307             ProtocolTest.assert(false, "Teardown action should not execute if its setup action returned false.")
    308             return false;           
    309         }
    310     });
    311     setupFailureTestSuite.runTestCases();
    312 
    313     let teardownExceptionTestSuite = ProtocolTest.createSyncSuite("SyncTestSuite.TeardownException");
    314     teardownExceptionTestSuite.addTestCase({
     308        setup() {
     309            ProtocolTest.log("Failing...");
     310            return false;
     311        },
     312        test() {
     313            ProtocolTest.fail("Test should not execute if its setup action failed.");
     314            return false;
     315        },
     316        teardown() {
     317            ProtocolTest.fail("Teardown action should not execute if its setup action failed.");
     318            return false;
     319        },
     320    });
     321    setupFailureSuite.addTestCase({
     322        name: "TestAfterSetupException",
     323        setup() {
     324            ProtocolTest.pass("Setup action should still execute if previous test's setup action failed.");
     325            return true;
     326        },
     327        test() {
     328            ProtocolTest.pass("Test should still execute if previous test's setup action failed.");
     329            return true;
     330        },
     331        teardown() {
     332            ProtocolTest.pass("Teardown action should still execute if previous test's setup action failed.");
     333            return true;
     334        },
     335    });
     336    checkResult(setupFailureSuite, {
     337        runCount: 1,
     338        passCount: 1,
     339        failCount: 1,
     340        skipCount: 1,
     341    });
     342
     343    let teardownExceptionSuite = ProtocolTest.createSyncSuite("SyncTestSuite.TeardownException");
     344    teardownExceptionSuite.addTestCase({
    315345        name: "TestWithExceptionDuringTeardown",
    316         description: "Check execution order for teardown action that throws an exception.",
    317         test: () => true,
    318         teardown: () => { throw new Error() }
    319     });
    320     teardownExceptionTestSuite.addTestCase({
     346        test() {
     347            return true;
     348        },
     349        teardown() {
     350            ProtocolTest.log("Throwing...");
     351            throw "TeardownException throw";
     352        },
     353    });
     354    teardownExceptionSuite.addTestCase({
    321355        name: "TestAfterTeardownException",
    322         descrption: "Check execution order for test after previous test's teardown throws an exception",
    323         setup: () => {
    324             ProtocolTest.assert(false, "Setup action should not execute if previous test's teardown action threw an exception.");
    325             return false;
    326         },
    327         test() {
    328             ProtocolTest.assert(false, "Test should not execute if previous test's teardown action threw an exception.");
    329             return false;
    330         }
    331     });
    332     teardownExceptionTestSuite.runTestCases();
    333 
    334     let teardownFailureTestSuite = ProtocolTest.createSyncSuite("SyncTestSuite.TeardownFailure");
    335     teardownFailureTestSuite.addTestCase({
     356        setup() {
     357            ProtocolTest.pass("Setup action should still execute if previous test's teardown action threw an exception.");
     358            return true;
     359        },
     360        test() {
     361            ProtocolTest.pass("Test should still execute if previous test's teardown action threw an exception.");
     362            return true;
     363        },
     364        teardown() {
     365            ProtocolTest.pass("Teardown action should still execute if previous test's teardown action threw an exception.");
     366            return true;
     367        },
     368    });
     369    checkResult(teardownExceptionSuite, {
     370        runCount: 2,
     371        passCount: 1,
     372        failCount: 1,
     373        skipCount: 0,
     374    });
     375
     376    let teardownFailureSuite = ProtocolTest.createSyncSuite("SyncTestSuite.TeardownFailure");
     377    teardownFailureSuite.addTestCase({
    336378        name: "TestWithExceptionDuringTeardown",
    337         description: "Check execution order for teardown action that has failed.",
    338         test: () => true,
    339         teardown: () => false,
    340     });
    341     teardownFailureTestSuite.addTestCase({
     379        test() {
     380            return true;
     381        },
     382        teardown() {
     383            ProtocolTest.log("Failing...");
     384            return false;
     385        },
     386    });
     387    teardownFailureSuite.addTestCase({
    342388        name: "TestAfterTeardownException",
    343         descrption: "Check execution order for test after previous test's teardown throws an exception",
    344         setup: () => {
    345             ProtocolTest.assert(false, "Setup action should not execute if previous test's teardown action failed.");
    346             return false;
    347         },
    348         test() {
    349             ProtocolTest.assert(false, "Test should not execute if previous test's teardown action failed.");
    350             return false;
    351         }
    352     });
    353     teardownFailureTestSuite.runTestCases();
     389        setup() {
     390            ProtocolTest.pass("Setup action should still execute if previous test's teardown action failed.");
     391            return true;
     392        },
     393        test() {
     394            ProtocolTest.pass("Test should still execute if previous test's teardown action failed.");
     395            return true;
     396        },
     397        teardown() {
     398            ProtocolTest.pass("Teardown action should still execute if previous test's teardown action failed.");
     399            return true;
     400        },
     401    });
     402    checkResult(teardownFailureSuite, {
     403        runCount: 2,
     404        passCount: 1,
     405        failCount: 1,
     406        skipCount: 0,
     407    });
    354408
    355409    ProtocolTest.completeTest();

  • trunk/Source/WebInspectorUI/ChangeLog

    r243826 r243953  
     12019-04-05  Devin Rousso  <drousso@apple.com>
     2
     3        Web Inspector: TestSuite test cases should have their own timeout to ensure tests fail with output instead of timeout by test runner
     4        https://bugs.webkit.org/show_bug.cgi?id=162814
     5        <rdar://problem/28574102>
     6
     7        Reviewed by Brian Burg.
     8
     9        A 10s timer is started for every test case added to an async suite. The timer is cleared
     10        when the test finishes, but if the timer fires, the test is forcibly ended with an error.
     11
     12        This timer can be configured by setting a `timeout` value when adding the test case. Values
     13        are expected to be in milliseconds. The value `-1` will prevent a timer from being set.
     14
     15        This change also relaxes the expectation that any individual test case failure will stop the
     16        rest of the suite from running. Since timers are set per test case, it is possible to
     17        recover from a "bad" test case to still run the remaining test cases.
     18
     19        NOTE: there may be unexpected behaviour if a test times out, as the timer doesn't actually
     20        stop the execution of the test, so it may still run and log information, which may appear
     21        "out of nowhere" in the middle of other tests.
     22
     23        * UserInterface/Test/TestSuite.js:
     24        (TestSuite.prototype.get passCount):
     25        (AsyncTestSuite.prototype.runTestCases):
     26        (SyncTestSuite.prototype.runTestCases):
     27
    1282019-04-03  Devin Rousso  <drousso@apple.com>
    229

  • trunk/Source/WebInspectorUI/UserInterface/Test/TestSuite.js

    r235873 r243953  
    5555    get passCount()
    5656    {
    57         return this.runCount - this.failCount;
     57        return this.runCount - (this.failCount - this.skipCount);
    5858    }
    5959
    … …  
    136136        // Avoid adding newlines if nothing was logged.
    137137        let priorLogCount = this._harness.logCount;
    138         let result = this.testcases.reduce((chain, testcase, i) => {
    139             if (testcase.setup) {
    140                 chain = chain.then(() => {
     138
     139        return Promise.resolve().then(() => Promise.chain(this.testcases.map((testcase, i) => () => new Promise(async (resolve, reject) => {
     140            if (i > 0 && priorLogCount < this._harness.logCount)
     141                this._harness.log("");
     142            priorLogCount = this._harness.logCount;
     143
     144            let hasTimeout = testcase.timeout !== -1;
     145            let timeoutId = undefined;
     146            if (hasTimeout) {
     147                let delay = testcase.timeout || 10000;
     148                timeoutId = setTimeout(() => {
     149                    if (!timeoutId)
     150                        return;
     151
     152                    timeoutId = undefined;
     153
     154                    this.failCount++;
     155                    this._harness.log(`!! TIMEOUT: took longer than ${delay}ms`);
     156
     157                    resolve();
     158                }, delay);
     159            }
     160
     161            try {
     162                if (testcase.setup) {
    141163                    this._harness.log("-- Running test setup.");
     164                    priorLogCount++;
     165
    142166                    if (testcase.setup[Symbol.toStringTag] === "AsyncFunction")
    143                         return testcase.setup();
    144                     return new Promise(testcase.setup);
    145                 });
    146             }
    147 
    148             chain = chain.then(() => {
    149                 if (i > 0 && priorLogCount + 1 < this._harness.logCount)
    150                     this._harness.log("");
    151 
    152                 priorLogCount = this._harness.logCount;
     167                        await testcase.setup();
     168                    else
     169                        await new Promise(testcase.setup);
     170                }
     171
     172                this.runCount++;
     173
    153174                this._harness.log(`-- Running test case: ${testcase.name}`);
    154                 this.runCount++;
     175                priorLogCount++;
     176
    155177                if (testcase.test[Symbol.toStringTag] === "AsyncFunction")
    156                     return testcase.test();
    157                 return new Promise(testcase.test);
    158             });
    159 
    160             if (testcase.teardown) {
    161                 chain = chain.then(() => {
     178                    await testcase.test();
     179                else
     180                    await new Promise(testcase.test);
     181
     182                if (testcase.teardown) {
    162183                    this._harness.log("-- Running test teardown.");
     184                    priorLogCount++;
     185
    163186                    if (testcase.teardown[Symbol.toStringTag] === "AsyncFunction")
    164                         return testcase.teardown();
    165                     return new Promise(testcase.teardown);
    166                 });
    167             }
    168             return chain;
    169         }, Promise.resolve());
    170 
    171         return result.catch((e) => {
    172             this.failCount++;
    173             this.logThrownObject(e);
    174 
    175             throw e; // Reject this promise by re-throwing the error.
    176         });
     187                        await testcase.teardown();
     188                    else
     189                        await new Promise(testcase.teardown);
     190                }
     191            } catch (e) {
     192                this.failCount++;
     193                this.logThrownObject(e);
     194            }
     195
     196            if (!hasTimeout || timeoutId) {
     197                clearTimeout(timeoutId);
     198                timeoutId = undefined;
     199
     200                resolve();
     201            }
     202        })))
     203        // Clear result value.
     204        .then(() => {}));
    177205    }
    178206};
    … …  
    209237        for (let i = 0; i < this.testcases.length; i++) {
    210238            let testcase = this.testcases[i];
    211             if (i > 0 && priorLogCount + 1 < this._harness.logCount)
     239
     240            if (i > 0 && priorLogCount < this._harness.logCount)
    212241                this._harness.log("");
    213 
    214242            priorLogCount = this._harness.logCount;
    215243
    216             // Run the setup action, if one was provided.
    217             if (testcase.setup) {
    218                 this._harness.log("-- Running test setup.");
    219                 try {
    220                     let result = testcase.setup.call(null);
    221                     if (result === false) {
     244            try {
     245                // Run the setup action, if one was provided.
     246                if (testcase.setup) {
     247                    this._harness.log("-- Running test setup.");
     248                    priorLogCount++;
     249
     250                    let setupResult = testcase.setup();
     251                    if (setupResult === false) {
    222252                        this._harness.log("!! SETUP FAILED");
    223                         return false;
     253                        this.failCount++;
     254                        continue;
    224255                    }
    225                 } catch (e) {
    226                     this.logThrownObject(e);
    227                     return false;
    228                 }
    229             }
    230 
    231             this._harness.log("-- Running test case: " + testcase.name);
    232             this.runCount++;
    233             try {
    234                 let result = testcase.test.call(null);
    235                 if (result === false) {
     256                }
     257
     258                this.runCount++;
     259
     260                this._harness.log(`-- Running test case: ${testcase.name}`);
     261                priorLogCount++;
     262
     263                let testResult = testcase.test();
     264                if (testResult === false) {
    236265                    this.failCount++;
    237                     return false;
     266                    continue;
     267                }
     268
     269                // Run the teardown action, if one was provided.
     270                if (testcase.teardown) {
     271                    this._harness.log("-- Running test teardown.");
     272                    priorLogCount++;
     273
     274                    let teardownResult = testcase.teardown();
     275                    if (teardownResult === false) {
     276                        this._harness.log("!! TEARDOWN FAILED");
     277                        this.failCount++;
     278                        continue;
     279                    }
    238280                }
    239281            } catch (e) {
    240282                this.failCount++;
    241283                this.logThrownObject(e);
    242                 return false;
    243             }
    244 
    245             // Run the teardown action, if one was provided.
    246             if (testcase.teardown) {
    247                 this._harness.log("-- Running test teardown.");
    248                 try {
    249                     let result = testcase.teardown.call(null);
    250                     if (result === false) {
    251                         this._harness.log("!! TEARDOWN FAILED");
    252                         return false;
    253                     }
    254                 } catch (e) {
    255                     this.logThrownObject(e);
    256                     return false;
    257                 }
    258284            }
    259285        }
Note: See TracChangeset for help on using the changeset viewer.