Changeset 85087 in webkit


Ignore:
Timestamp:
Apr 27, 2011 2:04:22 PM (13 years ago)
Author:
ojan@chromium.org
Message:

2011-04-22 Ojan Vafai <​ojan@chromium.org>

Reviewed by Adam Roben.

move times into their own json file
​https://bugs.webkit.org/show_bug.cgi?id=59239

The test runtimes are the only case where we need an entry for
every test that was run, including passing tests. Move them into
their own json file so we can prune the other json files to only
include failing tests.

Use a trie-like data structure to hold the tests to avoid including
full test paths. This cuts 52% of the filesize. The end result is ~700k
pre-gzip.

  • Scripts/webkitpy/common/net/resultsjsonparser.py: Exclude times from the parsed TestResults.
  • Scripts/webkitpy/common/net/resultsjsonparser_unittest.py:
  • Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py:
  • Scripts/webkitpy/layout_tests/layout_package/json_results_generator_unittest.py:
  • Scripts/webkitpy/layout_tests/layout_package/manager.py:
Location:
trunk/Tools
Files:
7 edited

Legend:

Unmodified
Added
Removed

  • trunk/Tools/ChangeLog

    r85083 r85087  
     12011-04-22  Ojan Vafai  <ojan@chromium.org>
     2
     3        Reviewed by Adam Roben.
     4
     5        move times into their own json file
     6        https://bugs.webkit.org/show_bug.cgi?id=59239
     7
     8        The test runtimes are the only case where we need an entry for
     9        every test that was run, including passing tests. Move them into
     10        their own json file so we can prune the other json files to only
     11        include failing tests.
     12
     13        Use a trie-like data structure to hold the tests to avoid including
     14        full test paths. This cuts 52% of the filesize. The end result is ~700k
     15        pre-gzip.
     16
     17        * Scripts/webkitpy/common/net/resultsjsonparser.py:
     18        Exclude times from the parsed TestResults.
     19        * Scripts/webkitpy/common/net/resultsjsonparser_unittest.py:
     20        * Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py:
     21        * Scripts/webkitpy/layout_tests/layout_package/json_results_generator_unittest.py:
     22        * Scripts/webkitpy/layout_tests/layout_package/manager.py:
     23
    1242011-04-27  Anders Carlsson  <andersca@apple.com>
    225

  • trunk/Tools/Scripts/webkitpy/common/net/resultsjsonparser.py

    r84633 r85087  
    4646        self._result_dict = result_dict
    4747
    48     def test_runtime(self):
    49         return self._result_dict.get('time_ms')
    50 
    5148    def did_pass(self):
    5249        return self._actual_as_expectation() == test_expectations.PASS
    … …  
    8481
    8582    def test_result(self):
    86         return test_results.TestResult(self._test_name, self._failures(), test_run_time=self.test_runtime())
     83        # FIXME: Optionally pull in the test runtime from times_ms.json.
     84        return test_results.TestResult(self._test_name, self._failures())
    8785
    8886

  • trunk/Tools/Scripts/webkitpy/common/net/resultsjsonparser_unittest.py

    r84633 r85087  
    4040        "fast\/dom\/prototype-inheritance.html": {
    4141            "expected": "PASS",
    42             "time_ms": 225,
    43             "actual": "TEXT",
    44             "has_stderr": false
     42            "actual": "TEXT"
    4543        },
    4644        "svg\/dynamic-updates\/SVGFEDropShadowElement-dom-stdDeviation-attr.html": {
    4745            "expected": "PASS",
    48             "time_ms": 110,
    4946            "actual": "IMAGE",
    50             "has_stderr": false
     47            "has_stderr": true
    5148        }
    5249    },
    … …  
    6562    def test_basic(self):
    6663        expected_results = [
    67             test_results.TestResult("svg/dynamic-updates/SVGFEDropShadowElement-dom-stdDeviation-attr.html", [test_failures.FailureImageHashMismatch()], 110),
    68             test_results.TestResult("fast/dom/prototype-inheritance.html", [test_failures.FailureTextMismatch()], 225),
     64            test_results.TestResult("svg/dynamic-updates/SVGFEDropShadowElement-dom-stdDeviation-attr.html", [test_failures.FailureImageHashMismatch()], 0),
     65            test_results.TestResult("fast/dom/prototype-inheritance.html", [test_failures.FailureTextMismatch()], 0),
    6966        ]
    7067        results = ResultsJSONParser.parse_results_json(self._example_unexpected_results_json)

  • trunk/Tools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py

    r84633 r85087  
    2828
    2929import logging
     30import os
    3031import subprocess
    3132import sys
    … …  
    6768    filesystem.write_text_file(file_path, json_string)
    6869
     70def test_timings_trie(port, individual_test_timings):
     71    """Breaks a filename into chunks by directory and puts the test time as a value in the lowest part, e.g.
     72    foo/bar/baz.html: 1ms
     73    foo/bar/baz1.html: 3ms
     74
     75    becomes
     76    foo: {
     77        bar: {
     78            baz.html: 1,
     79            baz1.html: 3
     80        }
     81    }
     82    """
     83    trie = {}
     84    for test_result in individual_test_timings:
     85        test = port.relative_test_filename(test_result.filename)
     86        parts = test.split(os.sep)
     87        current_map = trie
     88        for i, part in enumerate(parts):
     89            if i == (len(parts) - 1):
     90                current_map[part] = int(1000 * test_result.test_run_time)
     91                break
     92
     93            if part not in current_map:
     94                current_map[part] = {}
     95            current_map = current_map[part]
     96    return trie
     97
    6998# FIXME: We already have a TestResult class in test_results.py
    7099class TestResult(object):
    … …  
    75104
    76105    def __init__(self, name, failed=False, elapsed_time=0):
    77         self.name = name
     106        # FIXME: s/filename/name to be consistent with the rest of layout_package.
     107        self.filename = name
    78108        self.failed = failed
    79         self.time = elapsed_time
     109        self.test_run_time = elapsed_time
    80110
    81111        test_name = name
    … …  
    132162
    133163    RESULTS_FILENAME = "results.json"
    134     FULL_RESULTS_FILENAME = "full_results.json"
     164    TIMES_MS_FILENAME = "times_ms.json"
    135165    INCREMENTAL_RESULTS_FILENAME = "incremental_results.json"
    136166
    … …  
    189219        self._archived_results = None
    190220
    191     def generate_json_output(self):
     221    def generate_times_ms_file(self):
    192222        json = self.get_json()
    193223        if json:
    … …  
    195225            write_json(self._fs, json, file_path)
    196226
     227    # FIXME: Remove this function once the chromium buildbots stop calling it.
     228    def generate_json_output(self):
     229        self.generate_times_ms_file()
     230
    197231    def generate_full_results_file(self):
    198         # Use the same structure as the compacted version of TestRunner.summarize_results.
    199         # For now we only include the times as this is only used for treemaps and
    200         # expected/actual don't make sense for gtests.
    201         results = {}
    202         results['version'] = 1
    203 
    204         tests = {}
    205 
    206         for test in self._test_results_map:
    207             time_seconds = self._test_results_map[test].time
    208             tests[test] = {}
    209             tests[test]['time_ms'] = int(1000 * time_seconds)
    210 
    211         results['tests'] = tests
    212         file_path = self._fs.join(self._results_directory, self.FULL_RESULTS_FILENAME)
    213         write_json(self._fs, results, file_path)
     232        # FIXME: rename to generate_times_ms_file. This needs to be coordinated with
     233        # changing the calls to this on the chromium build slaves.
     234        times = test_timings_trie(self._port, self._test_results_map.values())
     235        file_path = self._fs.join(self._results_directory, self.TIMES_MS_FILENAME)
     236        write_json(self._fs, times, file_path)
    214237
    215238    def get_json(self):
    … …  
    292315        if test_name in self._test_results_map:
    293316            # Floor for now to get time in seconds.
    294             return int(self._test_results_map[test_name].time)
     317            return int(self._test_results_map[test_name].test_run_time)
    295318        return 0
    296319
    297320    def _get_failed_test_names(self):
    298321        """Returns a set of failed test names."""
    299         return set([r.name for r in self._test_results if r.failed])
     322        return set([r.filename for r in self._test_results if r.failed])
    300323
    301324    def _get_modifier_char(self, test_name):

  • trunk/Tools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator_unittest.py

    r79837 r85087  
    3636from webkitpy.layout_tests.layout_package import json_results_generator
    3737from webkitpy.layout_tests.layout_package import test_expectations
     38from webkitpy.layout_tests.port import test
    3839from webkitpy.thirdparty.mock import Mock
    39 
     40from webkitpy.thirdparty import simplejson
    4041
    4142class JSONGeneratorTest(unittest.TestCase):
    … …  
    8586
    8687        port = Mock()
     88        port.relative_test_filename = lambda filename: filename
    8789        port._filesystem = filesystem_mock.MockFileSystem()
    8890        generator = json_results_generator.JSONResultsGeneratorBase(port,
    … …  
    196198            ['A', 'FLAKY_E'])
    197199
     200    def test_test_timings_trie(self):
     201        test_port = test.TestPort()
     202        individual_test_timings = []
     203        individual_test_timings.append(json_results_generator.TestResult('/test.checkout/LayoutTests/foo/bar/baz.html', elapsed_time=1.2))
     204        individual_test_timings.append(json_results_generator.TestResult('/test.checkout/LayoutTests/bar.html', elapsed_time=0.0001))
     205        trie = json_results_generator.test_timings_trie(test_port, individual_test_timings)
     206       
     207        expected_trie = {
     208          'bar.html': 0,
     209          'foo': {
     210              'bar': {
     211                  'baz.html': 1200,
     212              }
     213          }
     214        }
     215       
     216        self.assertEqual(simplejson.dumps(trie), simplejson.dumps(expected_trie))
     217       
     218       
     219
    198220if __name__ == '__main__':
    199221    unittest.main()

  • trunk/Tools/Scripts/webkitpy/layout_tests/layout_package/manager.py

    r84594 r85087  
    8686        A dictionary containing a summary of the unexpected results from the
    8787        run, with the following fields:
    88         'version': a version indicator (1 in this version)
     88        'version': a version indicator
    8989        'fixable': # of fixable tests (NOW - PASS)
    9090        'skipped': # of skipped tests (NOW & SKIPPED)
    … …  
    9292        'num_flaky': # of flaky failures
    9393        'num_passes': # of unexpected passes
    94         'tests': a dict of tests -> {'expected': '...', 'actual': '...', 'time_ms': ...}
     94        'tests': a dict of tests -> {'expected': '...', 'actual': '...'}
    9595    """
    9696    results = {}
    97     results['version'] = 1
    98 
    99     test_timings_map = dict((test_result.filename, test_result.test_run_time) for test_result in test_timings)
     97    results['version'] = 2
    10098
    10199    tbe = result_summary.tests_by_expectation
    … …  
    128126        actual = [keywords[result_type]]
    129127
     128        # FIXME: only include passing tests that have stderr output.
    130129        if result_type == test_expectations.PASS:
    131130            num_passes += 1
    … …  
    149148        # FIXME: Set this correctly once https://webkit.org/b/37739 is fixed
    150149        # and only set it if there actually is stderr data.
    151         tests[test]['has_stderr'] = False
     150        if False:
     151            tests[test]['has_stderr'] = True
    152152
    153153        failure_types = [type(f) for f in result.failures]
    … …  
    170170        if test_failures.FailureMissingImage in failure_types or test_failures.FailureMissingImageHash in failure_types:
    171171            tests[test]['is_missing_image'] = True
    172 
    173         if filename in test_timings_map:
    174             time_seconds = test_timings_map[filename]
    175             tests[test]['time_ms'] = int(1000 * time_seconds)
    176172
    177173    results['tests'] = tests
    … …  
    910906        _log.debug("Writing JSON files in %s." % self._results_directory)
    911907
     908        times_trie = json_results_generator.test_timings_trie(self._port, individual_test_timings)
     909        times_json_path = self._fs.join(self._results_directory, "times_ms.json")
     910        json_results_generator.write_json(self._fs, times_trie, times_json_path)
     911
    912912        unexpected_json_path = self._fs.join(self._results_directory, "unexpected_results.json")
    913913        json_results_generator.write_json(self._fs, unexpected_results, unexpected_json_path)

  • trunk/Tools/Scripts/webkitpy/layout_tests/layout_package/test_results.py

    r79840 r85087  
    4040
    4141    def __init__(self, filename, failures=None, test_run_time=None):
     42        # FIXME: s/filename/name to be consistent with the rest of layout_package.
    4243        self.filename = filename
    4344        self.failures = failures or []
Note: See TracChangeset for help on using the changeset viewer.