Changeset 101303 in webkit

Timestamp:

Nov 28, 2011, 4:38:25 PM (15 years ago)

Author:

ojan@chromium.org

Message:

gtest normalization in the test results server is sometimes wrong
​https://bugs.webkit.org/show_bug.cgi?id=73262

Reviewed by Tony Chang.

If a gtest has it's modified changed (e.g. add FLAKY_) then it will have two
entries in the incremental JSON, one of the entries will have the correct data
from the run and the other will have dummy no-data values. Make sure to
always pick the one with real data.

• TestResultServer/model/jsonresults.py:

(JsonResults._remove_gtest_modifiers):

• TestResultServer/model/jsonresults_unittest.py:

(JsonResultsTest.test_remove_gtest_modifiers):

Location:

trunk/Tools

Files:

• ChangeLog (modified) (1 diff)
• TestResultServer/model/jsonresults.py (modified) (1 diff)
• TestResultServer/model/jsonresults_unittest.py (modified) (3 diffs)

trunk/Tools/ChangeLog

@@ +1 @@
+2011-11-28  Ojan Vafai  <ojan@chromium.org>
+
+        gtest normalization in the test results server is sometimes wrong
+        https://bugs.webkit.org/show_bug.cgi?id=73262
+
+        Reviewed by Tony Chang.
+
+        If a gtest has it's modified changed (e.g. add FLAKY_) then it will have two
+        entries in the incremental JSON, one of the entries will have the correct data
+        from the run and the other will have dummy no-data values. Make sure to
+        always pick the one with real data.
+
+        * TestResultServer/model/jsonresults.py:
+        (JsonResults._remove_gtest_modifiers):
+        * TestResultServer/model/jsonresults_unittest.py:
+        (JsonResultsTest.test_remove_gtest_modifiers):
+
 2011-11-28  Ojan Vafai  <ojan@chromium.org>
 

trunk/Tools/TestResultServer/model/jsonresults.py

@@ -236 +236 @@
         tests = json[builder][JSON_RESULTS_TESTS]
         new_tests = {}
-        # FIXME: This is wrong. If the test exists in the incremental results as both values, then one will overwrite the other.
-        # We should instead pick the one that doesn't have NO_DATA as its value.
-        # Alternately we could fix this by having the JSON generation code on the buildbot only include the test
-        # that was actually run.
-        for name, test in tests.iteritems():
+        for name, test in tests.items():
             new_name = name.replace('.FLAKY_', '.', 1)
             new_name = new_name.replace('.FAILS_', '.', 1)
             new_name = new_name.replace('.MAYBE_', '.', 1)
             new_name = new_name.replace('.DISABLED_', '.', 1)
-            new_tests[new_name] = test
+            if new_name not in new_tests or test[JSON_RESULTS_RESULTS][0][1] != JSON_RESULTS_NO_DATA:
+                new_tests[new_name] = test
+
         json[builder][JSON_RESULTS_TESTS] = new_tests
 

trunk/Tools/TestResultServer/model/jsonresults_unittest.py

@@ -690 +690 @@
             # Incremental results
             {"builds": ["3"],
-             "tests": {"foo.FLAKY_bar": {
+             "tests": {"foo.DISABLED_bar": {
                            "results": [[1,"F"]],
                            "times": [[1,0]]},
-                       "foo.DISABLED_bar2": {
+                       "foo.FLAKY_bar2": {
+                           "results": [[1,"N"]],
+                           "times": [[1,0]]},
+                       "foo.bar2": {
                            "results": [[1,"I"]],
                            "times": [[1,0]]},
                        "foo.bar3": {
-                           "results": [[1,"I"]],
+                           "results": [[1,"N"]],
                            "times": [[1,0]]},
                        "foo.FAILS_bar3": {
@@ -708 +711 @@
             # Expected results
             {"builds": ["3", "2", "1"],
-             "tests": {"foo.bar": {
+             "tests": {"foo.FAILS_bar3": {
+                           "results": [[1,"N"],[100,"I"]],
+                           "times": [[101,0]]},
+                       "foo.bar": {
                            "results": [[51,"F"]],
                            "times": [[51,0]]},
@@ -717 +723 @@
                            "results": [[1,"I"]],
                            "times": [[1,0]]},
-                       "foo.FAILS_bar3": {
-                              "results": [[1,"N"],[100,"I"]],
-                              "times": [[101,0]]},
                        "foo.bar4": {
                            "results": [[1,"I"]],