Changeset 252430 in webkit


Ignore:
Timestamp:
Nov 13, 2019 1:51:48 PM (4 years ago)
Author:
aakash_jain@apple.com
Message:

[EWS] Parse jsc_results.json for JSC tests
​https://bugs.webkit.org/show_bug.cgi?id=204090

Reviewed by Jonathan Bedard.

  • BuildSlaveSupport/ews-build/steps.py:

(RunJavaScriptCoreTests):
(RunJavaScriptCoreTests.start): Initialize log_observer for json.
(RunJavaScriptCoreTests.commandComplete): Parse jsc_results.json and set properties accordingly.
(RunJavaScriptCoreTests.getResultSummary): Update the build-step summary string.
(ReRunJavaScriptCoreTests):
(RunJSCTestsWithoutPatch):

  • BuildSlaveSupport/ews-build/steps_unittest.py:

(TestRunJavaScriptCoreTests.setUp): Added sample json files for testing.
(TestRunJavaScriptCoreTests.configureStep):
(TestRunJavaScriptCoreTests.test_single_stress_test_failure): Added unit-tests.
(TestRunJavaScriptCoreTests.test_lot_of_stress_test_failure): Ditto.
(TestRunJavaScriptCoreTests.test_masm_failure): Ditto.
(TestRunJavaScriptCoreTests.test_b3_and_stress_test_failure): Ditto.
(TestRunJavaScriptCoreTests.test_dfg_air_and_stress_test_failure): Ditto.

Location:
trunk/Tools
Files:
3 edited

Legend:

Unmodified
Added
Removed

  • trunk/Tools/BuildSlaveSupport/ews-build/steps.py

    r252411 r252430  
    947947    logfiles = {'json': jsonFileName}
    948948    command = ['perl', 'Tools/Scripts/run-javascriptcore-tests', '--no-build', '--no-fail-fast', '--json-output={0}'.format(jsonFileName), WithProperties('--%(configuration)s')]
     949    prefix = 'jsc_'
     950    NUM_FAILURES_TO_DISPLAY_IN_STATUS = 5
    949951
    950952    def __init__(self, **kwargs):
    951953        shell.Test.__init__(self, logEnviron=False, **kwargs)
    952 
    953     def start(self):
     954        self.binaryFailures = []
     955        self.stressTestFailures = []
     956
     957    def start(self):
     958        self.log_observer_json = logobserver.BufferLogObserver()
     959        self.addLogObserver('json', self.log_observer_json)
     960
    954961        # add remotes configuration file path to the command line if needed
    955962        remotesfile = self.getProperty('remotes', False)
    … …  
    971978        return rc
    972979
     980    def commandComplete(self, cmd):
     981        shell.Test.commandComplete(self, cmd)
     982        logLines = self.log_observer_json.getStdout()
     983        json_text = ''.join([line for line in logLines.splitlines()])
     984        try:
     985            jsc_results = json.loads(json_text)
     986        except Exception as ex:
     987            self._addToLog('stderr', 'ERROR: unable to parse data, exception: {}'.format(ex))
     988            return
     989
     990        if jsc_results.get('allMasmTestsPassed') == False:
     991            self.binaryFailures.append('testmasm')
     992        if jsc_results.get('allAirTestsPassed') == False:
     993            self.binaryFailures.append('testair')
     994        if jsc_results.get('allB3TestsPassed') == False:
     995            self.binaryFailures.append('testb3')
     996        if jsc_results.get('allDFGTestsPassed') == False:
     997            self.binaryFailures.append('testdfg')
     998        if jsc_results.get('allApiTestsPassed') == False:
     999            self.binaryFailures.append('testapi')
     1000
     1001        self.stressTestFailures = jsc_results.get('stressTestFailures')
     1002        if self.stressTestFailures:
     1003            self.setProperty(self.prefix + 'stress_test_failures', self.stressTestFailures)
     1004        if self.binaryFailures:
     1005            self.setProperty(self.prefix + 'binary_failures', self.binaryFailures)
     1006
     1007    def getResultSummary(self):
     1008        if self.results != SUCCESS and (self.stressTestFailures or self.binaryFailures):
     1009            status = ''
     1010            if self.stressTestFailures:
     1011                num_failures = len(self.stressTestFailures)
     1012                pluralSuffix = 's' if num_failures > 1 else ''
     1013                failures_to_display = self.stressTestFailures[:self.NUM_FAILURES_TO_DISPLAY_IN_STATUS]
     1014                status = 'Found {} jsc stress test failure{}: '.format(num_failures, pluralSuffix) + ', '.join(failures_to_display)
     1015                if num_failures > self.NUM_FAILURES_TO_DISPLAY_IN_STATUS:
     1016                    status += ' ...'
     1017            if self.binaryFailures:
     1018                if status:
     1019                    status += ', '
     1020                pluralSuffix = 's' if len(self.binaryFailures) > 1 else ''
     1021                status += 'JSC test binary failure{}: {}'.format(pluralSuffix, ', '.join(self.binaryFailures))
     1022
     1023            return {u'step': unicode(status)}
     1024
     1025        return shell.Test.getResultSummary(self)
     1026
     1027    @defer.inlineCallbacks
     1028    def _addToLog(self, logName, message):
     1029        try:
     1030            log = self.getLog(logName)
     1031        except KeyError:
     1032            log = yield self.addLog(logName)
     1033        log.addStdout(message)
     1034
    9731035
    9741036class ReRunJavaScriptCoreTests(RunJavaScriptCoreTests):
    9751037    name = 'jscore-test-rerun'
     1038    prefix = 'jsc_rerun_'
    9761039
    9771040    def evaluateCommand(self, cmd):
    … …  
    9901053class RunJSCTestsWithoutPatch(RunJavaScriptCoreTests):
    9911054    name = 'jscore-test-without-patch'
    992     jsonFileName = 'jsc_results.json'
     1055    prefix = 'jsc_clean_tree_'
    9931056
    9941057    def evaluateCommand(self, cmd):

  • trunk/Tools/BuildSlaveSupport/ews-build/steps_unittest.py

    r252411 r252430  
    10001000        self.longMessage = True
    10011001        self.jsonFileName = 'jsc_results.json'
     1002        self.jsc_masm_failure = '''{"allDFGTestsPassed":true,"allMasmTestsPassed":false,"allB3TestsPassed":true,"allAirTestsPassed":true,"stressTestFailures":[],"allApiTestsPassed":true}\n'''
     1003        self.jsc_b3_and_stress_test_failure = '''{"allDFGTestsPassed":true,"allMasmTestsPassed":true,"allB3TestsPassed":false,"allAirTestsPassed":true,"allApiTestsPassed":true,"stressTestFailures":["stress/weakset-gc.js"]}\n'''
     1004        self.jsc_dfg_air_and_stress_test_failure = '''{"allDFGTestsPassed":false,"allMasmTestsPassed":true,"allB3TestsPassed":true,"allAirTestsPassed":false,"allApiTestsPassed":true,"stressTestFailures":["stress/weakset-gc.js"]}\n'''
     1005        self.jsc_single_stress_test_failure = '''{"allDFGTestsPassed":true,"allMasmTestsPassed":true,"allB3TestsPassed":true,"allAirTestsPassed":true,"stressTestFailures":["stress/switch-on-char-llint-rope.js.dfg-eager"],"allApiTestsPassed":true}\n'''
     1006        self.jsc_multiple_stress_test_failures = '''{"allDFGTestsPassed":true,"allMasmTestsPassed":true,"allB3TestsPassed":true,"allAirTestsPassed":true,"stressTestFailures":["stress/switch-on-char-llint-rope.js.dfg-eager","stress/switch-on-char-llint-rope.js.dfg-eager-no-cjit-validate","stress/switch-on-char-llint-rope.js.eager-jettison-no-cjit","stress/switch-on-char-llint-rope.js.ftl-eager","stress/switch-on-char-llint-rope.js.ftl-eager-no-cjit","stress/switch-on-char-llint-rope.js.ftl-eager-no-cjit-b3o1","stress/switch-on-char-llint-rope.js.ftl-no-cjit-b3o0","stress/switch-on-char-llint-rope.js.ftl-no-cjit-no-inline-validate","stress/switch-on-char-llint-rope.js.ftl-no-cjit-no-put-stack-validate","stress/switch-on-char-llint-rope.js.ftl-no-cjit-small-pool","stress/switch-on-char-llint-rope.js.ftl-no-cjit-validate-sampling-profiler","stress/switch-on-char-llint-rope.js.no-cjit-collect-continuously","stress/switch-on-char-llint-rope.js.no-cjit-validate-phases","stress/switch-on-char-llint-rope.js.no-ftl"],"allApiTestsPassed":true}\n'''
    10021007        return self.setUpBuildStep()
    10031008
    … …  
    10071012    def configureStep(self, platform=None, fullPlatform=None, configuration=None):
    10081013        self.setupStep(RunJavaScriptCoreTests())
     1014        self.prefix = RunJavaScriptCoreTests.prefix
    10091015        if platform:
    10101016            self.setProperty('platform', platform)
    … …  
    10551061        return self.runStep()
    10561062
     1063    def test_single_stress_test_failure(self):
     1064        self.configureStep(platform='mac', fullPlatform='mac-highsierra', configuration='debug')
     1065        self.expectRemoteCommands(
     1066            ExpectShell(workdir='wkdir',
     1067                        logEnviron=False,
     1068                        logfiles={'json': self.jsonFileName},
     1069                        command=['perl', 'Tools/Scripts/run-javascriptcore-tests', '--no-build', '--no-fail-fast', '--json-output={0}'.format(self.jsonFileName), '--debug'],
     1070                        )
     1071            + 2
     1072            + ExpectShell.log('json', stdout=self.jsc_single_stress_test_failure),
     1073        )
     1074        self.expectOutcome(result=FAILURE, state_string='Found 1 jsc stress test failure: stress/switch-on-char-llint-rope.js.dfg-eager')
     1075        rc = self.runStep()
     1076        self.assertEqual(self.getProperty(self.prefix + 'stress_test_failures'), ['stress/switch-on-char-llint-rope.js.dfg-eager'])
     1077        self.assertEqual(self.getProperty(self.prefix + 'binary_failures'), None)
     1078        return rc
     1079
     1080    def test_lot_of_stress_test_failure(self):
     1081        self.configureStep(platform='mac', fullPlatform='mac-highsierra', configuration='debug')
     1082        self.expectRemoteCommands(
     1083            ExpectShell(workdir='wkdir',
     1084                        logEnviron=False,
     1085                        logfiles={'json': self.jsonFileName},
     1086                        command=['perl', 'Tools/Scripts/run-javascriptcore-tests', '--no-build', '--no-fail-fast', '--json-output={0}'.format(self.jsonFileName), '--debug'],
     1087                        )
     1088            + 2
     1089            + ExpectShell.log('json', stdout=self.jsc_multiple_stress_test_failures),
     1090        )
     1091        self.expectOutcome(result=FAILURE, state_string='Found 14 jsc stress test failures: stress/switch-on-char-llint-rope.js.dfg-eager, stress/switch-on-char-llint-rope.js.dfg-eager-no-cjit-validate, stress/switch-on-char-llint-rope.js.eager-jettison-no-cjit, stress/switch-on-char-llint-rope.js.ftl-eager, stress/switch-on-char-llint-rope.js.ftl-eager-no-cjit ...')
     1092        rc = self.runStep()
     1093        self.assertEqual(self.getProperty(self.prefix + 'stress_test_failures'), ["stress/switch-on-char-llint-rope.js.dfg-eager", "stress/switch-on-char-llint-rope.js.dfg-eager-no-cjit-validate", "stress/switch-on-char-llint-rope.js.eager-jettison-no-cjit", "stress/switch-on-char-llint-rope.js.ftl-eager", "stress/switch-on-char-llint-rope.js.ftl-eager-no-cjit", "stress/switch-on-char-llint-rope.js.ftl-eager-no-cjit-b3o1", "stress/switch-on-char-llint-rope.js.ftl-no-cjit-b3o0", "stress/switch-on-char-llint-rope.js.ftl-no-cjit-no-inline-validate", "stress/switch-on-char-llint-rope.js.ftl-no-cjit-no-put-stack-validate", "stress/switch-on-char-llint-rope.js.ftl-no-cjit-small-pool", "stress/switch-on-char-llint-rope.js.ftl-no-cjit-validate-sampling-profiler", "stress/switch-on-char-llint-rope.js.no-cjit-collect-continuously", "stress/switch-on-char-llint-rope.js.no-cjit-validate-phases", "stress/switch-on-char-llint-rope.js.no-ftl"])
     1094        self.assertEqual(self.getProperty(self.prefix + 'binary_failures'), None)
     1095        return rc
     1096
     1097    def test_masm_failure(self):
     1098        self.configureStep(platform='mac', fullPlatform='mac-highsierra', configuration='debug')
     1099        self.expectRemoteCommands(
     1100            ExpectShell(workdir='wkdir',
     1101                        logEnviron=False,
     1102                        logfiles={'json': self.jsonFileName},
     1103                        command=['perl', 'Tools/Scripts/run-javascriptcore-tests', '--no-build', '--no-fail-fast', '--json-output={0}'.format(self.jsonFileName), '--debug'],
     1104                        )
     1105            + 2
     1106            + ExpectShell.log('json', stdout=self.jsc_masm_failure),
     1107        )
     1108        self.expectOutcome(result=FAILURE, state_string='JSC test binary failure: testmasm')
     1109        rc = self.runStep()
     1110        self.assertEqual(self.getProperty(self.prefix + 'stress_test_failures'), None)
     1111        self.assertEqual(self.getProperty(self.prefix + 'binary_failures'), ['testmasm'])
     1112        return rc
     1113
     1114    def test_b3_and_stress_test_failure(self):
     1115        self.configureStep(platform='mac', fullPlatform='mac-highsierra', configuration='release')
     1116        self.expectRemoteCommands(
     1117            ExpectShell(workdir='wkdir',
     1118                        logEnviron=False,
     1119                        logfiles={'json': self.jsonFileName},
     1120                        command=['perl', 'Tools/Scripts/run-javascriptcore-tests', '--no-build', '--no-fail-fast', '--json-output={0}'.format(self.jsonFileName), '--release'],
     1121                        )
     1122            + 2
     1123            + ExpectShell.log('json', stdout=self.jsc_b3_and_stress_test_failure),
     1124        )
     1125        self.expectOutcome(result=FAILURE, state_string='Found 1 jsc stress test failure: stress/weakset-gc.js, JSC test binary failure: testb3')
     1126        rc = self.runStep()
     1127        self.assertEqual(self.getProperty(self.prefix + 'stress_test_failures'), ['stress/weakset-gc.js'])
     1128        self.assertEqual(self.getProperty(self.prefix + 'binary_failures'), ['testb3'])
     1129        return rc
     1130
     1131    def test_dfg_air_and_stress_test_failure(self):
     1132        self.configureStep(platform='jsc-only', fullPlatform='jsc-only', configuration='release')
     1133        self.expectRemoteCommands(
     1134            ExpectShell(workdir='wkdir',
     1135                        logEnviron=False,
     1136                        logfiles={'json': self.jsonFileName},
     1137                        command=['perl', 'Tools/Scripts/run-javascriptcore-tests', '--no-build', '--no-fail-fast', '--json-output={0}'.format(self.jsonFileName), '--release', '--jsc-only'],
     1138                        )
     1139            + 2
     1140            + ExpectShell.log('json', stdout=self.jsc_dfg_air_and_stress_test_failure),
     1141        )
     1142        self.expectOutcome(result=FAILURE, state_string='Found 1 jsc stress test failure: stress/weakset-gc.js, JSC test binary failures: testair, testdfg')
     1143        rc = self.runStep()
     1144        self.assertEqual(self.getProperty(self.prefix + 'stress_test_failures'), ['stress/weakset-gc.js'])
     1145        self.assertEqual(self.getProperty(self.prefix + 'binary_failures'), ['testair', 'testdfg'])
     1146        return rc
     1147
    10571148
    10581149class TestReRunJavaScriptCoreTests(TestRunJavaScriptCoreTests):
    10591150    def configureStep(self, platform=None, fullPlatform=None, configuration=None):
    10601151        self.setupStep(ReRunJavaScriptCoreTests())
     1152        self.prefix = ReRunJavaScriptCoreTests.prefix
    10611153        if platform:
    10621154            self.setProperty('platform', platform)

  • trunk/Tools/ChangeLog

    r252418 r252430  
     12019-11-13  Aakash Jain  <aakash_jain@apple.com>
     2
     3        [EWS] Parse jsc_results.json for JSC tests
     4        https://bugs.webkit.org/show_bug.cgi?id=204090
     5
     6        Reviewed by Jonathan Bedard.
     7
     8        * BuildSlaveSupport/ews-build/steps.py:
     9        (RunJavaScriptCoreTests):
     10        (RunJavaScriptCoreTests.start): Initialize log_observer for json.
     11        (RunJavaScriptCoreTests.commandComplete): Parse jsc_results.json and set properties accordingly.
     12        (RunJavaScriptCoreTests.getResultSummary): Update the build-step summary string.
     13        (ReRunJavaScriptCoreTests):
     14        (RunJSCTestsWithoutPatch):
     15        * BuildSlaveSupport/ews-build/steps_unittest.py:
     16        (TestRunJavaScriptCoreTests.setUp): Added sample json files for testing.
     17        (TestRunJavaScriptCoreTests.configureStep):
     18        (TestRunJavaScriptCoreTests.test_single_stress_test_failure): Added unit-tests.
     19        (TestRunJavaScriptCoreTests.test_lot_of_stress_test_failure): Ditto.
     20        (TestRunJavaScriptCoreTests.test_masm_failure): Ditto.
     21        (TestRunJavaScriptCoreTests.test_b3_and_stress_test_failure): Ditto.
     22        (TestRunJavaScriptCoreTests.test_dfg_air_and_stress_test_failure): Ditto.
     23
    1242019-11-13  Youenn Fablet  <youenn@apple.com>
    225
Note: See TracChangeset for help on using the changeset viewer.