Changeset 93222 in webkit

Timestamp:

Aug 17, 2011, 11:13:12 AM (15 years ago)

Author:

Adam Roben

Message:

Teach TestFailures to ignore unbelievably short test runs

Fixes <​http://webkit.org/b/66385> TestFailures page thinks all tests passed in
​http://build.webkit.org/builders/Windows%207%20Release%20(Tests)/builds/14956

Reviewed by Dan Bates.

• BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/Builder.js:

(Builder.prototype.getNumberOfFailingTests): If it looks like all tests passed, but
run-webkit-tests took less than 10 seconds to run, assume that some weird error occurred
that caused it not to run any tests at all (as happened for a while due to
<​http://webkit.org/b/64988>). Bumped the cache version to evict old, buggy cached data.

• BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/Builder_unittests.js:

Test for the above.

• BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/LayoutTestResultsLoader.js:

(LayoutTestResultsLoader.prototype.start): Bumped the cache version to evict old, buggy cached data.

Location:

trunk/Tools

Files:

• BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/Builder.js (modified) (3 diffs)
• BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/Builder_unittests.js (modified) (2 diffs)
• BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/LayoutTestResultsLoader.js (modified) (1 diff)
• ChangeLog (modified) (1 diff)

trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/Builder.js

@@ -101 +101 @@
     getNumberOfFailingTests: function(buildNumber, callback) {
         var cacheKey = this.name + '_getNumberOfFailingTests_' + buildNumber;
-        const currentCachedDataVersion = 2;
+        const currentCachedDataVersion = 3;
         if (PersistentCache.contains(cacheKey)) {
             var cachedData = PersistentCache.get(cacheKey);
@@ -129 +129 @@
 
             if (!('results' in layoutTestStep) || layoutTestStep.results[0] === 0) {
+                if (!('times' in layoutTestStep) || layoutTestStep.times.length < 2 || layoutTestStep.times[1] - layoutTestStep.times[0] < self._minimumSuccessfulLayoutTestStepRunTime) {
+                    // Either something caused the start/stop times not to be recorded, or
+                    // run-webkit-tests ran so quickly that we can't believe there wasn't an error
+                    // (e.g., a bug in the script that made it not find any tests to run).
+                    PersistentCache.set(cacheKey, result);
+                    callback(result.failureCount, result.tooManyFailures);
+                    return;
+                }
+
                 // All tests passed.
                 result.failureCount = 0;
@@ -225 +234 @@
         });
     },
+
+    // Any successful runs of run-webkit-tests that took less than this number of seconds are
+    // assumed to be errors.
+    _minimumSuccessfulLayoutTestStepRunTime: 20,
 };

trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/Builder_unittests.js

@@ -75 +75 @@
                     ],
                 ],
+                times: [
+                    1310599204.1231229,
+                    1310600152.973659,
+                ]
             },
         ],
@@ -205 +209 @@
 });
 
+test("getNumberOfFailingTests treats successful but unbelievably short test runs as errors", 4, function() {
+    const jsonData = {
+        steps: [
+            {
+                isFinished: true, 
+                isStarted: true, 
+                name: "layout-test", 
+                step_number: 7, 
+                text: [
+                    "layout-test"
+                ], 
+                times: [
+                    1311288797.7207019, 
+                    1311288802.7791941
+                ]
+            }, 
+        ],
+    };
+
+    runGetNumberOfFailingTestsTest(jsonData, function(failureCount, tooManyFailures) {
+        equal(failureCount, -1);
+        equal(tooManyFailures, false);
+    });
+});
+
+test("getNumberOfFailingTests doesn't care if a failing run is unbelievably short", 4, function() {
+    const jsonData = {
+        steps: [
+            {
+                isFinished: true, 
+                isStarted: true, 
+                name: "layout-test", 
+                results: [
+                  2, 
+                  [
+                      "2011-07-13 04:38:46,315 11247 manager.py:780 WARNING Exiting early after 20 crashes and 0 timeouts. 2251 tests run.", 
+                      "20 failed"
+                  ]
+                ], 
+                step_number: 4, 
+                text: [
+                    "2011-07-13 04:38:46,315 11247 manager.py:780 WARNING Exiting early after 20 crashes and 0 timeouts. 2251 tests run.", 
+                    "20 failed"
+                ], 
+                times: [
+                    1310557115.793082, 
+                    1310557119.832104
+                ]
+            }, 
+        ],
+    };
+
+    runGetNumberOfFailingTestsTest(jsonData, function(failureCount, tooManyFailures) {
+        equal(failureCount, 20);
+        equal(tooManyFailures, true);
+    });
+});
+
 })();

trunk/Tools/BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/LayoutTestResultsLoader.js

@@ -31 +31 @@
     start: function(buildName, callback, errorCallback) {
         var cacheKey = 'LayoutTestResultsLoader.' + this._builder.name + '.' + buildName;
-        const currentCachedDataVersion = 7;
+        const currentCachedDataVersion = 8;
         if (PersistentCache.contains(cacheKey)) {
             var cachedData = PersistentCache.get(cacheKey);

trunk/Tools/ChangeLog

@@ +1 @@
+2011-08-17  Adam Roben  <aroben@apple.com>
+
+        Teach TestFailures to ignore unbelievably short test runs
+
+        Fixes <http://webkit.org/b/66385> TestFailures page thinks all tests passed in
+        http://build.webkit.org/builders/Windows%207%20Release%20(Tests)/builds/14956
+
+        Reviewed by Dan Bates.
+
+        * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/Builder.js:
+        (Builder.prototype.getNumberOfFailingTests): If it looks like all tests passed, but
+        run-webkit-tests took less than 10 seconds to run, assume that some weird error occurred
+        that caused it not to run any tests at all (as happened for a while due to
+        <http://webkit.org/b/64988>). Bumped the cache version to evict old, buggy cached data.
+
+        * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/Builder_unittests.js:
+        Test for the above.
+
+        * BuildSlaveSupport/build.webkit.org-config/public_html/TestFailures/LayoutTestResultsLoader.js:
+        (LayoutTestResultsLoader.prototype.start): Bumped the cache version to evict old, buggy cached data.
+
 2011-08-16  Adam Barth  <abarth@webkit.org>