Changeset 64786 in webkit
- Timestamp:
- Aug 5, 2010 2:24:00 PM (14 years ago)
- Location:
- trunk/WebKitTools
- Files:
-
- 5 edited
-
ChangeLog (modified) (1 diff)
-
Scripts/webkitpy/layout_tests/layout_package/json_layout_results_generator.py (modified) (2 diffs)
-
Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py (modified) (4 diffs)
-
Scripts/webkitpy/layout_tests/layout_package/json_results_generator_unittest.py (modified) (6 diffs)
-
Scripts/webkitpy/layout_tests/run_webkit_tests.py (modified) (3 diffs)
Legend:
- Unmodified
- Added
- Removed
-
trunk/WebKitTools/ChangeLog
r64763 r64786 1 2010-08-05 Victor Wang <victorw@chromium.org> 2 3 Reviewed by Ojan Vafai. 4 5 Add option to generate/upload incremental json results to test result server. 6 Also refactor the json results generator unittest code to test 7 incremental and aggregated json results. 8 9 https://bugs.webkit.org/show_bug.cgi?id=43519 10 11 * Scripts/webkitpy/layout_tests/layout_package/json_layout_results_generator.py: 12 * Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py: 13 * Scripts/webkitpy/layout_tests/layout_package/json_results_generator_unittest.py: 14 * Scripts/webkitpy/layout_tests/run_webkit_tests.py: 15 1 16 2010-08-05 Jian Li <jianli@chromium.org> 2 17 -
trunk/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_layout_results_generator.py
r62989 r64786 57 57 def __init__(self, port, builder_name, build_name, build_number, 58 58 results_file_base_path, builder_base_url, 59 test_timings, expectations, result_summary, all_tests): 59 test_timings, expectations, result_summary, all_tests, 60 generate_incremental_results=False): 60 61 """Modifies the results.json file. Grabs it off the archive directory 61 62 if it is not found locally. … … 67 68 super(JSONLayoutResultsGenerator, self).__init__( 68 69 builder_name, build_name, build_number, results_file_base_path, 69 builder_base_url, {}, port.test_repository_paths()) 70 builder_base_url, {}, port.test_repository_paths(), 71 generate_incremental_results) 70 72 71 73 self._port = port -
trunk/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator.py
r62989 r64786 83 83 84 84 RESULTS_FILENAME = "results.json" 85 INCREMENTAL_RESULTS_FILENAME = "incremental_results.json" 85 86 86 87 def __init__(self, builder_name, build_name, build_number, 87 88 results_file_base_path, builder_base_url, 88 test_results_map, svn_repositories=None): 89 test_results_map, svn_repositories=None, 90 generate_incremental_results=False): 89 91 """Modifies the results.json file. Grabs it off the archive directory 90 92 if it is not found locally. … … 109 111 self._results_file_path = os.path.join(results_file_base_path, 110 112 self.RESULTS_FILENAME) 113 self._incremental_results_file_path = os.path.join( 114 results_file_base_path, self.INCREMENTAL_RESULTS_FILENAME) 111 115 112 116 self._test_results_map = test_results_map 113 117 self._test_results = test_results_map.values() 118 self._generate_incremental_results = generate_incremental_results 114 119 115 120 self._svn_repositories = svn_repositories … … 118 123 119 124 self._json = None 125 self._archived_results = None 120 126 121 127 def generate_json_output(self): 122 128 """Generates the JSON output file.""" 129 130 # Generate the JSON output file that has full results. 131 # FIXME: stop writing out the full results file once all bots use 132 # incremental results. 123 133 if not self._json: 124 134 self._json = self.get_json() 125 135 if self._json: 126 # Specify separators in order to get compact encoding. 127 json_data = simplejson.dumps(self._json, separators=(',', ':')) 128 json_string = self.JSON_PREFIX + json_data + self.JSON_SUFFIX 129 130 results_file = codecs.open(self._results_file_path, "w", "utf-8") 131 results_file.write(json_string) 132 results_file.close() 133 134 def get_json(self): 136 self._generate_json_file(self._json, self._results_file_path) 137 138 # Generate the JSON output file that only has incremental results. 139 if self._generate_incremental_results: 140 json = self.get_json(incremental=True) 141 if json: 142 self._generate_json_file( 143 json, self._incremental_results_file_path) 144 145 def get_json(self, incremental=False): 135 146 """Gets the results for the results.json file.""" 136 if self._json: 137 return self._json 138 139 results_json, error = self._get_archived_json_results() 140 if error: 141 # If there was an error don't write a results.json 142 # file at all as it would lose all the information on the bot. 143 _log.error("Archive directory is inaccessible. Not modifying " 144 "or clobbering the results.json file: " + str(error)) 145 return None 147 if incremental: 148 results_json = {} 149 else: 150 if self._json: 151 return self._json 152 153 if not self._archived_results: 154 self._archived_results, error = \ 155 self._get_archived_json_results() 156 if error: 157 # If there was an error don't write a results.json 158 # file at all as it would lose all the information on the 159 # bot. 160 _log.error("Archive directory is inaccessible. Not " 161 "modifying or clobbering the results.json " 162 "file: " + str(error)) 163 return None 164 165 results_json = self._archived_results 146 166 147 167 builder_name = self._builder_name … … 169 189 self._insert_test_time_and_result(test, tests) 170 190 171 self._json = results_json 172 return self._json 191 return results_json 192 193 def set_archived_results(self, archived_results): 194 self._archived_results = archived_results 195 196 def _generate_json_file(self, json, file_path): 197 # Specify separators in order to get compact encoding. 198 json_data = simplejson.dumps(json, separators=(',', ':')) 199 json_string = self.JSON_PREFIX + json_data + self.JSON_SUFFIX 200 201 results_file = codecs.open(file_path, "w", "utf-8") 202 results_file.write(json_string) 203 results_file.close() 173 204 174 205 def _get_test_timing(self, test_name): -
trunk/WebKitTools/Scripts/webkitpy/layout_tests/layout_package/json_results_generator_unittest.py
r62989 r64786 46 46 self.build_name = 'DUMMY_BUILD_NAME' 47 47 self.build_number = 'DUMMY_BUILDER_NUMBER' 48 self._json = None 49 self._num_runs = 0 50 self._tests_list = set([]) 51 self._test_timings = {} 52 self._failed_tests = {} 53 self._passed_tests = set([]) 54 self._skipped_tests = set([]) 48 55 49 56 def _test_json_generation(self, passed_tests, failed_tests, skipped_tests): … … 53 60 tests_list = passed_tests | set(failed_tests.keys()) 54 61 test_timings = {} 62 i = 0 55 63 for test in tests_list: 56 test_timings[test] = float(random.randint(1, 10)) 64 test_timings[test] = float(self._num_runs * 100 + i) 65 i += 1 57 66 58 67 port_obj = port.get(None) 59 68 60 # Generate a JSON file.61 69 generator = json_results_generator.JSONResultsGenerator(port_obj, 62 70 self.builder_name, self.build_name, self.build_number, … … 69 77 tests_list) 70 78 71 json = generator.get_json() 79 # Test incremental json results 80 incremental_json = generator.get_json(incremental=True) 81 self._verify_json_results( 82 tests_list, 83 test_timings, 84 passed_tests, 85 failed_tests, 86 skipped_tests, 87 incremental_json, 88 1) 72 89 90 # Test aggregated json results 91 generator.set_archived_results(self._json) 92 json = generator.get_json(incremental=False) 93 self._json = json 94 self._num_runs += 1 95 self._tests_list |= tests_list 96 self._test_timings.update(test_timings) 97 self._failed_tests.update(failed_tests) 98 self._passed_tests |= passed_tests 99 self._skipped_tests |= skipped_tests 100 self._verify_json_results( 101 self._tests_list, 102 self._test_timings, 103 self._passed_tests, 104 self._failed_tests, 105 self._skipped_tests, 106 self._json, 107 self._num_runs) 108 109 def _verify_json_results(self, tests_list, test_timings, 110 passed_tests, failed_tests, 111 skipped_tests, json, num_runs): 73 112 # Aliasing to a short name for better access to its constants. 74 113 JRG = json_results_generator.JSONResultsGenerator … … 80 119 self.assertTrue(JRG.FIXABLE in buildinfo) 81 120 self.assertTrue(JRG.TESTS in buildinfo) 82 self.assertTrue(len(buildinfo[JRG.BUILD_NUMBERS]) == 1)121 self.assertTrue(len(buildinfo[JRG.BUILD_NUMBERS]) == num_runs) 83 122 self.assertTrue(buildinfo[JRG.BUILD_NUMBERS][0] == self.build_number) 84 123 85 if tests_list or skipped_tests: 86 fixable = buildinfo[JRG.FIXABLE][0] 124 if tests_list or skipped_tests: 125 fixable = {} 126 for fixable_items in buildinfo[JRG.FIXABLE]: 127 for (type, count) in fixable_items.iteritems(): 128 if type in fixable: 129 fixable[type] = fixable[type] + count 130 else: 131 fixable[type] = count 132 87 133 if passed_tests: 88 134 self.assertTrue(fixable[JRG.PASS_RESULT] == len(passed_tests)) … … 101 147 self.assertTrue(test_name in tests) 102 148 test = tests[test_name] 103 self.assertTrue(test[JRG.RESULTS][0][0] == 1) 104 self.assertTrue(test[JRG.RESULTS][0][1] == JRG.FAIL_RESULT) 105 self.assertTrue(test[JRG.TIMES][0][0] == 1) 106 self.assertTrue(test[JRG.TIMES][0][1] == 107 int(test_timings[test_name])) 149 150 failed = 0 151 for result in test[JRG.RESULTS]: 152 if result[1] == JRG.FAIL_RESULT: 153 failed = result[0] 154 self.assertTrue(failed == 1) 155 156 timing_count = 0 157 for timings in test[JRG.TIMES]: 158 if timings[1] == test_timings[test_name]: 159 timing_count = timings[0] 160 self.assertTrue(timing_count == 1) 108 161 109 162 fixable_count = len(skipped_tests) + len(failed_tests.keys()) 110 163 if skipped_tests or failed_tests: 111 self.assertTrue( buildinfo[JRG.FIXABLE_COUNT][0]== fixable_count)164 self.assertTrue(sum(buildinfo[JRG.FIXABLE_COUNT]) == fixable_count) 112 165 113 166 def test_json_generation(self): … … 115 168 116 169 self._test_json_generation([], {}, []) 117 self._test_json_generation(['A', 'B'], {}, []) 118 self._test_json_generation([], {'A': reason, 'B': reason}, []) 119 self._test_json_generation([], {}, ['A', 'B']) 120 self._test_json_generation(['A'], {'B': reason, 'C': reason}, []) 121 self._test_json_generation([], {'A': reason, 'B': reason}, ['C', 'D']) 122 self._test_json_generation(['A', 'B', 'C'], {'D': reason}, ['E', 'F']) 170 self._test_json_generation(['A1', 'B1'], {}, []) 171 self._test_json_generation([], {'A2': reason, 'B2': reason}, []) 172 self._test_json_generation([], {}, ['A3', 'B3']) 173 self._test_json_generation(['A4'], {'B4': reason, 'C4': reason}, []) 174 self._test_json_generation( 175 [], {'A5': reason, 'B5': reason}, ['C5', 'D5']) 176 self._test_json_generation( 177 ['A6', 'B6', 'C6'], {'D6': reason}, ['E6', 'F6']) 123 178 124 179 -
trunk/WebKitTools/Scripts/webkitpy/layout_tests/run_webkit_tests.py
r62728 r64786 890 890 self._options.build_number, self._options.results_directory, 891 891 BUILDER_BASE_URL, individual_test_timings, 892 self._expectations, result_summary, self._test_files_list) 892 self._expectations, result_summary, self._test_files_list, 893 self._options.upload_incremental_results) 893 894 894 895 _log.debug("Finished writing JSON files.") … … 901 902 self._options.builder_name) 902 903 903 attrs = [('builder', self._options.builder_name)] 904 json_files = ["expectations.json", "results.json"] 904 attrs = [("builder", self._options.builder_name)] 905 json_files = ["expectations.json"] 906 if self._options.upload_incremental_results: 907 json_files.append("incremental_results.json") 908 else: 909 json_files.append("results.json") 905 910 906 911 files = [(file, os.path.join(self._options.results_directory, file)) … … 1657 1662 help=("If specified, upload results json files to this appengine " 1658 1663 "server.")), 1664 optparse.make_option("--upload-incremental-results", 1665 action="store_true", 1666 default=False, 1667 help="If true, upload incremental json results to server."), 1659 1668 ] 1660 1669
Note: See TracChangeset
for help on using the changeset viewer.
