Changeset 263629 in webkit
- Timestamp:
- Jun 27, 2020 3:18:41 PM (4 years ago)
- Location:
- trunk/Tools
- Files:
-
- 5 edited
- 1 copied
Legend:
- Unmodified
- Added
- Removed
-
trunk/Tools/ChangeLog
r263627 r263629 1 2020-06-27 Keith Miller <keith_miller@apple.com> 2 3 compare-results should be able to parse multiple JSONs at once by merging them 4 https://bugs.webkit.org/show_bug.cgi?id=213685 5 6 Reviewed by Saam Barati. 7 8 This patch improves the compare-results script to enable it to 9 merge JSON output files. This is handy when running scripts by 10 hand where you'll get many a and b JSONs. To do the merging this 11 patch moves the logic of merge-result-jsons into a shared library. 12 13 compare-results can take multiple files sequentally or by passing 14 the '-a'/'-b' flags multiple times. e.g. `/compare-results -a 15 OG-1.json -a OG-2.json -a OG-3.json -a OG-4.json -b MC-1.json 16 MC-2.json MC-3.json MC-4.json` will behave as you'd expect; 17 combining all the OG JSONs and the MC JSONs before computing the 18 result. 19 20 Lastly, the benchmark_results script now can handle duplicates of 21 an aggregator in the aggregator list. This is useful because 22 otherwise the logic of merging JSONs is significantly more 23 complicated. 24 25 * Scripts/compare-results: 26 (getOptions): 27 (main): 28 * Scripts/merge-result-jsons: 29 (readJSONFile): 30 (deepAppend): Deleted. 31 (mergeJSONs): Deleted. 32 * Scripts/webkitpy/benchmark_runner/benchmark_json_merge.py: Copied from Tools/Scripts/merge-result-jsons. 33 (deepAppend): 34 (mergeJSONs): 35 * Scripts/webkitpy/benchmark_runner/benchmark_results.py: 36 (BenchmarkResults._aggregate_results_for_test): 37 (BenchmarkResults._lint_subtest_results): 38 * Scripts/webkitpy/benchmark_runner/benchmark_results_unittest.py: 39 1 40 2020-06-27 Jer Noble <jer.noble@apple.com> 2 41 -
trunk/Tools/Scripts/compare-results
r263124 r263629 30 30 import argparse 31 31 import json 32 import itertools 32 33 from webkitpy.benchmark_runner.benchmark_results import BenchmarkResults 34 from webkitpy.benchmark_runner.benchmark_json_merge import mergeJSONs 33 35 34 36 try: … … 396 398 397 399 def getOptions(): 398 parser = argparse.ArgumentParser(description="Compare two WebKit benchmark results. Pass in two JSON result files to compare them. This script prints the pValue along with the magnitude of the change.")400 parser = argparse.ArgumentParser(description="Compare two WebKit benchmark results. Pass in at least two JSON result files to compare them. This script prints the pValue along with the magnitude of the change. If more than one JSON is passed as a/b they will be merged when computing the breakdown.") 399 401 400 402 parser.add_argument("-a", 401 403 type=str, 402 404 required=True, 403 help="a of a/b. Path to JSON results file.") 405 nargs='+', 406 action="append", 407 help="a JSONs of a/b. Path to JSON results file. Takes multiple values and can be passed multiple times.") 404 408 405 409 parser.add_argument("-b", 406 410 type=str, 407 411 required=True, 408 help="b of a/b. Path to JSON results file.") 412 nargs='+', 413 action="append", 414 help="b JSONs of a/b. Path to JSON results file. Takes multiple values and can be passed multiple times.") 409 415 410 416 parser.add_argument("--csv", … … 422 428 args = getOptions() 423 429 424 a = readJSONFile(args.a) 425 b = readJSONFile(args.b) 430 # Flatten the list of lists of JSON files. 431 a = itertools.chain.from_iterable(args.a) 432 b = itertools.chain.from_iterable(args.b) 433 434 a = mergeJSONs(list(map(readJSONFile, a))) 435 b = mergeJSONs(list(map(readJSONFile, b))) 426 436 427 437 typeA = detectBenchmark(a) -
trunk/Tools/Scripts/merge-result-jsons
r244469 r263629 1 1 #!/usr/bin/env python -u 2 2 3 # Copyright (C) 2019 Apple Inc. All rights reserved.3 # Copyright (C) 2019-2020 Apple Inc. All rights reserved. 4 4 # 5 5 # Redistribution and use in source and binary forms, with or without … … 29 29 import argparse 30 30 import json 31 31 from webkitpy.benchmark_runner.benchmark_json_merge import mergeJSONs 32 32 33 33 def readJSONFile(path): 34 34 with open(path, 'r') as contents: 35 35 return json.load(contents) 36 37 38 def deepAppend(value1, value2, currentKey=None):39 if type(value1) != type(value2):40 raise TypeError("values have different types for key: {}, {} and {}".format(currentKey, type(value1), type(value2)))41 if isinstance(value1, list):42 return value1 + value243 44 result = {};45 for key in (value1.keys() + value2.keys()):46 if key not in result:47 result[key] = deepAppend(value1[key], value2[key], key)48 return result49 50 def mergeJSONs(jsons):51 if len(jsons) == 0:52 raise TypeError("no jsons to merge")53 54 last = jsons.pop()55 return reduce(deepAppend, jsons, last)56 36 57 37 def main(): -
trunk/Tools/Scripts/webkitpy/benchmark_runner/benchmark_json_merge.py
- Property svn:executable deleted
r263628 r263629 1 #!/usr/bin/env python -u 2 3 # Copyright (C) 2019 Apple Inc. All rights reserved. 1 # Copyright (C) 2019-2020 Apple Inc. All rights reserved. 4 2 # 5 3 # Redistribution and use in source and binary forms, with or without … … 27 25 # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 26 29 import argparse30 27 import json 31 32 33 def readJSONFile(path): 34 with open(path, 'r') as contents: 35 return json.load(contents) 28 from webkitpy.common.iteration_compatibility import iteritems 36 29 37 30 … … 42 35 return value1 + value2 43 36 44 result = {} ;37 result = {} 45 38 for key in (value1.keys() + value2.keys()): 46 39 if key not in result: 47 40 result[key] = deepAppend(value1[key], value2[key], key) 48 41 return result 42 49 43 50 44 def mergeJSONs(jsons): … … 54 48 last = jsons.pop() 55 49 return reduce(deepAppend, jsons, last) 56 57 def main():58 parser = argparse.ArgumentParser(description="Merge the resulting json files from multiple invocations of the run_benchmark script.")59 60 parser.add_argument("-o",61 type=str,62 required=False,63 help="File to put the merged json into prints to standard out if nothing is passed")64 parser.add_argument("jsons",65 type=str,66 nargs='+',67 help="The json files to be merged.")68 69 # parse_args will error on our list of incomming JSON files...70 args = parser.parse_args()71 72 result = mergeJSONs(list(map(readJSONFile, args.jsons)))73 74 if args.o:75 with open(args.o, 'w') as f:76 json.dump(result, f)77 else:78 print(json.dumps(result))79 80 81 if __name__ == "__main__":82 main() -
trunk/Tools/Scripts/webkitpy/benchmark_runner/benchmark_results.py
r261231 r263629 154 154 continue 155 155 156 aggregator_list = metric 156 # Filter duplicate aggregators that could have arisen from merging JSONs. 157 aggregator_list = list(set(metric)) 157 158 results[metric_name] = {} 158 159 for aggregator in aggregator_list: … … 217 218 for metric_name, metric in iteritems(metrics): 218 219 if isinstance(metric, list): 219 cls._lint_aggregator_list(test_name, metric_name, metric, parent_test, parent_aggregator_list) 220 aggregator_list = metric 220 # Filter duplicate aggregators that could have arisen from merging JSONs. 221 aggregator_list = list(set(metric)) 222 cls._lint_aggregator_list(test_name, metric_name, aggregator_list, parent_test, parent_aggregator_list) 221 223 elif isinstance(metric, dict): 222 224 cls._lint_configuration(test_name, metric_name, metric, parent_test, parent_aggregator_list, iteration_groups_by_config) -
trunk/Tools/Scripts/webkitpy/benchmark_runner/benchmark_results_unittest.py
r253129 r263629 284 284 self.assertTrue(BenchmarkResults._lint_results({'SomeTest': {'metrics': {'Time': {'current': [1, 2]}}}})) 285 285 286 self.assertTrue(BenchmarkResults._lint_results({'SomeTest': {'metrics': {'Time': ['Total', 'Total']}, 'tests': { 287 'SubTest1': {'metrics': {'Time': {'current': []}}}}}})) 288 286 289 with self.assertRaisesRegexp(TypeError, r'"Time" metric of "SomeTest" was not an aggregator list or a dictionary of configurations: 1'): 287 290 BenchmarkResults._lint_results({'SomeTest': {'metrics': {'Time': 1}}}) … … 298 301 with self.assertRaisesRegexp(TypeError, r'"OtherTest" requires aggregation but it has no subtests'): 299 302 BenchmarkResults._lint_results({'OtherTest': {'metrics': {'Time': ['Total']}}}) 300 301 with self.assertRaisesRegexp(TypeError, r'"Time" metric of "SomeTest" had invalid aggregator list: \["Total", "Total"\]'):302 BenchmarkResults._lint_results({'SomeTest': {'metrics': {'Time': ['Total', 'Total']}, 'tests': {303 'SubTest1': {'metrics': {'Time': {'current': []}}}}}})304 303 305 304 with self.assertRaisesRegexp(TypeError, r'"Time" metric of "SomeTest" uses unknown aggregator: KittenMean'):
Note: See TracChangeset
for help on using the changeset viewer.