Changeset 267082 in webkit
- Timestamp:
- Sep 15, 2020 2:02:57 AM (4 years ago)
- Location:
- trunk/Tools
- Files:
-
- 2 edited
Legend:
- Unmodified
- Added
- Removed
-
trunk/Tools/ChangeLog
r267081 r267082 1 2020-09-15 Angelos Oikonomopoulos <angelos@igalia.com> 2 3 run-jsc-stress-tests: handle remotes that are down on startup 4 https://bugs.webkit.org/show_bug.cgi?id=216357 5 6 Reviewed by Yusuke Suzuki. 7 8 Previously, the GNU parallel runner would needlessly fail to run if a 9 remote host was down on startup, as it used forEachRemote for the 10 preparatory steps (which would instafail on error). 11 12 Change forEachRemote to (optionally) recover from failure by removing 13 the remote host from the array. This also handles the case where a 14 remote unexpectedly comes up during a test run (by not picking it up). 15 16 * Scripts/run-jsc-stress-tests: 17 1 18 2020-09-15 Youenn Fablet <youenn@apple.com> 2 19 -
trunk/Tools/Scripts/run-jsc-stress-tests
r266937 r267082 88 88 end 89 89 90 $ignoreNextCommandExecutionExitCode = false 91 92 def mysys(*cmd) 93 begin 94 printCommandArray(*cmd) if $verbosity >= 1 95 raise "Command failed: #{$?.inspect}" unless (system(*cmd) or $ignoreNextCommandExecutionExitCode) 96 ensure 97 $ignoreNextCommandExecutionExitCode = false 98 end 90 class CommandExecutionFailed < Exception 91 end 92 93 def mysys(commandArray, options={}) 94 printCommandArray(commandArray) if $verbosity >= 1 95 successful = system(*commandArray) 96 status = $? 97 if successful or options[:ignoreFailure] 98 return 99 end 100 raise CommandExecutionFailed, "Command failed: #{status.inspect}" 99 101 end 100 102 … … 2033 2035 2034 2036 $runnerDirMutex = Mutex.new 2035 def runAndMonitorTestRunnerCommand( *cmd)2037 def runAndMonitorTestRunnerCommand(cmd, options={}) 2036 2038 numberOfTests = 0 2037 2039 $runnerDirMutex.synchronize { … … 2042 2044 } 2043 2045 unless $progressMeter 2044 mysys(cmd.join(' ') )2046 mysys(cmd.join(' '), options) 2045 2047 else 2046 2048 running = {} … … 2098 2100 } 2099 2101 puts 2100 raise "Failed to run #{cmd}: #{$?.inspect}" unless $?.success? 2102 if not $?.success? and not options[:ignoreFailure] 2103 raise "Failed to run #{cmd}: #{$?.inspect}" 2104 end 2101 2105 end 2102 2106 end … … 2110 2114 2111 2115 def copyBundleToRemote(remoteHost) 2112 mysys( "ssh", "-o", "NoHostAuthenticationForLocalhost=yes", "-p", remoteHost.port.to_s, "#{remoteHost.user}@#{remoteHost.host}", "mkdir -p #{remoteHost.remoteDirectory}")2113 mysys( "scp", "-o", "NoHostAuthenticationForLocalhost=yes", "-P", remoteHost.port.to_s, ($outputDir.dirname + $tarFileName).to_s, "#{remoteHost.user}@#{remoteHost.host}:#{remoteHost.remoteDirectory}")2116 mysys(["ssh", "-o", "NoHostAuthenticationForLocalhost=yes", "-p", remoteHost.port.to_s, "#{remoteHost.user}@#{remoteHost.host}", "mkdir -p #{remoteHost.remoteDirectory}"]) 2117 mysys(["scp", "-o", "NoHostAuthenticationForLocalhost=yes", "-P", remoteHost.port.to_s, ($outputDir.dirname + $tarFileName).to_s, "#{remoteHost.user}@#{remoteHost.host}:#{remoteHost.remoteDirectory}"]) 2114 2118 end 2115 2119 … … 2140 2144 $envVars.each { |var| remoteScript += "export " << var << "\n" } 2141 2145 remoteScript += "#{testRunnerCommand(remoteIndex)}\"" 2142 runAndMonitorTestRunnerCommand( "ssh", "-o", "NoHostAuthenticationForLocalhost=yes", "-p", remoteHost.port.to_s, "#{remoteHost.user}@#{remoteHost.host}", remoteScript)2146 runAndMonitorTestRunnerCommand(["ssh", "-o", "NoHostAuthenticationForLocalhost=yes", "-p", remoteHost.port.to_s, "#{remoteHost.user}@#{remoteHost.host}", remoteScript]) 2143 2147 else 2144 2148 Dir.chdir($runnerDir) { 2145 runAndMonitorTestRunnerCommand( testRunnerCommand)2149 runAndMonitorTestRunnerCommand(Shellwords.shellsplit(testRunnerCommand)) 2146 2150 } 2147 2151 end … … 2307 2311 end 2308 2312 2309 def forEachRemote( &blk)2313 def forEachRemote(options={}, &blk) 2310 2314 threads = [] 2311 2315 $remoteHosts.each_index { … … 2316 2320 } 2317 2321 } 2318 threads.each { |thr| thr.join } 2322 failedRemotes = [] 2323 threads.each_index { 2324 | index | 2325 thread = threads[index] 2326 begin 2327 thread.join 2328 rescue CommandExecutionFailed 2329 if options[:dropOnFailure] 2330 failedRemotes << index 2331 else 2332 raise 2333 end 2334 end 2335 } 2336 failedRemotes.reverse.each { 2337 | index | 2338 if $verbosity > 0 2339 $stderr.puts("Dropping failed remote #{$remoteHosts[index]}") 2340 end 2341 $remoteHosts.delete_at(index) 2342 } 2319 2343 end 2320 2344 … … 2397 2421 end 2398 2422 2399 2400 def runSameCommandOnRemotes(cmd)2401 Tempfile.open('gnu-parallel-commands', $runnerDir) {2402 | commands |2403 commands.puts(cmd)2404 commands.flush2405 withGnuParallelSshLoginFile {2406 | slf |2407 cmd = [2408 "parallel",2409 "--nonall",2410 "-j0",2411 "--slf", slf2412 ] + ["-a", commands.path]2413 return mysys(*cmd)2414 }2415 }2416 end2417 2418 2423 def unpackBundleGnuParallel 2419 runSameCommandOnRemotes("rm -rf #{$outputDir.basename} && " + 2420 "tar xzf #{$tarFileName}") 2424 forEachRemote ({ :dropOnFailure => true }) { 2425 |remoteIndex, remoteHost| 2426 mysys(["ssh", "-o", "NoHostAuthenticationForLocalhost=yes", 2427 "-p", remoteHost.port.to_s, 2428 "#{remoteHost.user}@#{remoteHost.host}", 2429 "cd #{Shellwords.shellescape(remoteHost.remoteDirectory)} && rm -rf #{$outputDir.basename} && tar xzf #{$tarFileName}"]) 2430 } 2421 2431 end 2422 2432 … … 2446 2456 "sh \"" 2447 2457 ] 2448 $ignoreNextCommandExecutionExitCode = true 2449 runAndMonitorTestRunnerCommand(*cmd) 2458 runAndMonitorTestRunnerCommand(cmd, :ignoreFailure => true) 2450 2459 } 2451 2460 end … … 2456 2465 prepareTestRunner 2457 2466 compressBundle 2458 forEachRemote { 2467 # If the preparatory steps fail, drop the remote host from our 2468 # list. Otherwise, if it later comes up online, we'll try to run 2469 # test jobs on it, possibly using a previously-unpacked unrelated 2470 # bundle. 2471 forEachRemote ({ :dropOnFailure => true }) { 2459 2472 |remoteIndex, remoteHost| 2460 2473 getRemoteDirectoryIfNeeded(remoteIndex) … … 2462 2475 } 2463 2476 unpackBundleGnuParallel 2477 if $remoteHosts.size == 0 2478 raise "All remote hosts failed, giving up" 2479 end 2464 2480 runGnuParallelRunner 2465 2481 detectFailures
Note: See TracChangeset
for help on using the changeset viewer.