Changeset 220431 in webkit


Ignore:
Timestamp:
Aug 8, 2017 5:18:08 PM (7 years ago)
Author:
commit-queue@webkit.org
Message:

Separate jsc stress test script writer from run-jsc-stress-tests
​https://bugs.webkit.org/show_bug.cgi?id=175216

Patch by Stephan Szabo <​stephan.szabo@sony.com> on 2017-08-08
Reviewed by Mark Lam.

  • Scripts/run-jsc-stress-tests:
  • Scripts/webkitruby/jsc-stress-test-writer-default.rb: Added.
Location:
trunk/Tools
Files:
1 added
2 edited

Legend:

Unmodified
Added
Removed

  • trunk/Tools/ChangeLog

    r220425 r220431  
     12017-08-08  Stephan Szabo  <stephan.szabo@sony.com>
     2
     3        Separate jsc stress test script writer from run-jsc-stress-tests
     4        https://bugs.webkit.org/show_bug.cgi?id=175216
     5
     6        Reviewed by Mark Lam.
     7
     8        * Scripts/run-jsc-stress-tests:
     9        * Scripts/webkitruby/jsc-stress-test-writer-default.rb: Added.
     10
    1112017-08-08  Michael Catanzaro  <mcatanzaro@igalia.com>
    212

  • trunk/Tools/Scripts/run-jsc-stress-tests

    r220118 r220431  
    435435FTL_OPTIONS = ["--useFTLJIT=true"]
    436436
     437require_relative "webkitruby/jsc-stress-test-writer-default"
     438
    437439def shouldCollectContinuously?
    438440    $buildType == "release" or $forceCollectContinuously
    … …  
    472474end
    473475
    474 def prefixCommand(prefix)
    475     "awk " + Shellwords.shellescape("{ printf #{(prefix + ': ').inspect}; print }")
    476 end
    477 
    478 def redirectAndPrefixCommand(prefix)
    479     prefixCommand(prefix) + " 2>&1"
    480 end
    481 
    482 def pipeAndPrefixCommand(outputFilename, prefix)
    483     "tee " + Shellwords.shellescape(outputFilename.to_s) + " | " + prefixCommand(prefix)
    484 end
    485 
    486 # Output handler for tests that are expected to be silent.
    487 def silentOutputHandler
    488     Proc.new {
    489         | name |
    490         " | " + pipeAndPrefixCommand((Pathname("..") + (name + ".out")).to_s, name)
    491     }
    492 end
    493 
    494 # Output handler for tests that are expected to produce meaningful output.
    495 def noisyOutputHandler
    496     Proc.new {
    497         | name |
    498         " | cat > " + Shellwords.shellescape((Pathname("..") + (name + ".out")).to_s)
    499     }
    500 end
    501 
    502 # Error handler for tests that fail exactly when they return non-zero exit status.
    503 # This is useful when a test is expected to fail.
    504 def simpleErrorHandler
    505     Proc.new {
    506         | outp, plan |
    507         outp.puts "if test -e #{plan.failFile}"
    508         outp.puts "then"
    509         outp.puts "    (echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
    510         outp.puts "    " + plan.failCommand
    511         outp.puts "else"
    512         outp.puts "    " + plan.successCommand
    513         outp.puts "fi"
    514     }
    515 end
    516 
    517 # Error handler for tests that fail exactly when they return zero exit status.
    518 def expectedFailErrorHandler
    519     Proc.new {
    520         | outp, plan |
    521         outp.puts "if test -e #{plan.failFile}"
    522         outp.puts "then"
    523         outp.puts "    " + plan.successCommand
    524         outp.puts "else"
    525         outp.puts "    (echo ERROR: Unexpected exit code: 0) | " + redirectAndPrefixCommand(plan.name)
    526         outp.puts "    " + plan.failCommand
    527         outp.puts "fi"
    528     }
    529 end
    530 
    531 # Error handler for tests that fail exactly when they return non-zero exit status and produce
    532 # lots of spew. This will echo that spew when the test fails.
    533 def noisyErrorHandler
    534     Proc.new {
    535         | outp, plan |
    536         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
    537    
    538         outp.puts "if test -e #{plan.failFile}"
    539         outp.puts "then"
    540         outp.puts "    (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
    541         outp.puts "    " + plan.failCommand
    542         outp.puts "else"
    543         outp.puts "    " + plan.successCommand
    544         outp.puts "fi"
    545     }
    546 end
    547 
    548 # Error handler for tests that diff their output with some expectation.
    549 def diffErrorHandler(expectedFilename)
    550     Proc.new {
    551         | outp, plan |
    552         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
    553         diffFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".diff")).to_s)
    554        
    555         outp.puts "if test -e #{plan.failFile}"
    556         outp.puts "then"
    557         outp.puts "    (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
    558         outp.puts "    " + plan.failCommand
    559         outp.puts "elif test -e ../#{Shellwords.shellescape(expectedFilename)}"
    560         outp.puts "then"
    561         outp.puts "    diff --strip-trailing-cr -u ../#{Shellwords.shellescape(expectedFilename)} #{outputFilename} > #{diffFilename}"
    562         outp.puts "    if [ $? -eq 0 ]"
    563         outp.puts "    then"
    564         outp.puts "    " + plan.successCommand
    565         outp.puts "    else"
    566         outp.puts "        (echo \"DIFF FAILURE!\" && cat #{diffFilename}) | " + redirectAndPrefixCommand(plan.name)
    567         outp.puts "        " + plan.failCommand
    568         outp.puts "    fi"
    569         outp.puts "else"
    570         outp.puts "    (echo \"NO EXPECTATION!\" && cat #{outputFilename}) | " + redirectAndPrefixCommand(plan.name)
    571         outp.puts "    " + plan.failCommand
    572         outp.puts "fi"
    573     }
    574 end
    575 
    576 # Error handler for tests that report error by saying "failed!". This is used by Mozilla
    577 # tests.
    578 def mozillaErrorHandler
    579     Proc.new {
    580         | outp, plan |
    581         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
    582 
    583         outp.puts "if test -e #{plan.failFile}"
    584         outp.puts "then"
    585         outp.puts "    (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
    586         outp.puts "    " + plan.failCommand
    587         outp.puts "elif grep -i -q failed! #{outputFilename}"
    588         outp.puts "then"
    589         outp.puts "    (echo Detected failures: && cat #{outputFilename}) | " + redirectAndPrefixCommand(plan.name)
    590         outp.puts "    " + plan.failCommand
    591         outp.puts "else"
    592         outp.puts "    " + plan.successCommand
    593         outp.puts "fi"
    594     }
    595 end
    596 
    597 # Error handler for tests that report error by saying "failed!", and are expected to
    598 # fail. This is used by Mozilla tests.
    599 def mozillaFailErrorHandler
    600     Proc.new {
    601         | outp, plan |
    602         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
    603 
    604         outp.puts "if test -e #{plan.failFile}"
    605         outp.puts "then"
    606         outp.puts "    " + plan.successCommand
    607         outp.puts "elif grep -i -q failed! #{outputFilename}"
    608         outp.puts "then"
    609         outp.puts "    " + plan.successCommand
    610         outp.puts "else"
    611         outp.puts "    (echo NOTICE: You made this test pass, but it was expected to fail) | " + redirectAndPrefixCommand(plan.name)
    612         outp.puts "    " + plan.failCommand
    613         outp.puts "fi"
    614     }
    615 end
    616 
    617 # Error handler for tests that report error by saying "failed!", and are expected to have
    618 # an exit code of 3.
    619 def mozillaExit3ErrorHandler
    620     Proc.new {
    621         | outp, plan |
    622         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
    623 
    624         outp.puts "if test -e #{plan.failFile}"
    625         outp.puts "then"
    626         outp.puts "    if [ `cat #{plan.failFile}` -eq 3 ]"
    627         outp.puts "    then"
    628         outp.puts "        if grep -i -q failed! #{outputFilename}"
    629         outp.puts "        then"
    630         outp.puts "            (echo Detected failures: && cat #{outputFilename}) | " + redirectAndPrefixCommand(plan.name)
    631         outp.puts "            " + plan.failCommand
    632         outp.puts "        else"
    633         outp.puts "            " + plan.successCommand
    634         outp.puts "        fi"
    635         outp.puts "    else"
    636         outp.puts "        (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
    637         outp.puts "        " + plan.failCommand
    638         outp.puts "    fi"
    639         outp.puts "else"
    640         outp.puts "    (cat #{outputFilename} && echo ERROR: Test expected to fail, but returned successfully) | " + redirectAndPrefixCommand(plan.name)
    641         outp.puts "    " + plan.failCommand
    642         outp.puts "fi"
    643     }
    644 end
    645 
    646 # Error handler for tests that report success by saying "Passed" or error by saying "FAILED".
    647 # This is used by Chakra tests.
    648 def chakraPassFailErrorHandler
    649     Proc.new {
    650         | outp, plan |
    651         outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s)
    652 
    653         outp.puts "if test -e #{plan.failFile}"
    654         outp.puts "then"
    655         outp.puts "    (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)
    656         outp.puts "    " + plan.failCommand
    657         outp.puts "elif grep -i -q FAILED #{outputFilename}"
    658         outp.puts "then"
    659         outp.puts "    (echo Detected failures: && cat #{outputFilename}) | " + redirectAndPrefixCommand(plan.name)
    660         outp.puts "    " + plan.failCommand
    661         outp.puts "else"
    662         outp.puts "    " + plan.successCommand
    663         outp.puts "fi"
    664     }
    665 end
    666 
    667476$runCommandOptions = {}
    668 
    669 class Plan
    670     attr_reader :directory, :arguments, :family, :name, :outputHandler, :errorHandler
    671     attr_accessor :index
    672    
    673     def initialize(directory, arguments, family, name, outputHandler, errorHandler)
    674         @directory = directory
    675         @arguments = arguments
    676         @family = family
    677         @name = name
    678         @outputHandler = outputHandler
    679         @errorHandler = errorHandler
    680         @isSlow = !!$runCommandOptions[:isSlow]
    681     end
    682    
    683     def shellCommand
    684         # It's important to remember that the test is actually run in a subshell, so if we change directory
    685         # in the subshell when we return we will be in our original directory. This is nice because we don't
    686         # have to bend over backwards to do things relative to the root.
    687         script = "(cd ../#{Shellwords.shellescape(@directory.to_s)} && ("
    688         $envVars.each { |var| script += "export " << var << "; " }
    689         script += "\"$@\" " + escapeAll(@arguments) + "))"
    690         return script
    691     end
    692    
    693     def reproScriptCommand
    694         # We have to find our way back to the .runner directory since that's where all of the relative
    695         # paths assume they start out from.
    696         script = "CURRENT_DIR=\"$( cd \"$( dirname \"${BASH_SOURCE[0]}\" )\" && pwd )\"\n"
    697         script += "cd $CURRENT_DIR\n"
    698         Pathname.new(@name).dirname.each_filename {
    699             | pathComponent |
    700             script += "cd ..\n"
    701         }
    702         script += "cd .runner\n"
    703 
    704         script += "export DYLD_FRAMEWORK_PATH=$(cd #{$testingFrameworkPath.dirname}; pwd)\n"
    705         script += "export JSCTEST_timeout=#{Shellwords.shellescape(ENV['JSCTEST_timeout'])}\n"
    706         $envVars.each { |var| script += "export " << var << "\n" }
    707         script += "#{shellCommand} || exit 1"
    708         "echo #{Shellwords.shellescape(script)} > #{Shellwords.shellescape((Pathname.new("..") + @name).to_s)}"
    709     end
    710    
    711     def failCommand
    712         "echo FAIL: #{Shellwords.shellescape(@name)} ; touch #{failFile} ; " + reproScriptCommand
    713     end
    714    
    715     def successCommand
    716         if $progressMeter or $verbosity >= 2
    717             "rm -f #{failFile} ; echo PASS: #{Shellwords.shellescape(@name)}"
    718         else
    719             "rm -f #{failFile}"
    720         end
    721     end
    722    
    723     def failFile
    724         "test_fail_#{@index}"
    725     end
    726    
    727     def writeRunScript(filename)
    728         File.open(filename, "w") {
    729             | outp |
    730             outp.puts "echo Running #{Shellwords.shellescape(@name)}"
    731             cmd  = "(" + shellCommand + " || (echo $? > #{failFile})) 2>&1 "
    732             cmd += @outputHandler.call(@name)
    733             if $verbosity >= 3
    734                 outp.puts "echo #{Shellwords.shellescape(cmd)}"
    735             end
    736             outp.puts cmd
    737             @errorHandler.call(outp, self)
    738         }
    739     end
    740 end
    741477
    742478$uniqueFilenameCounter = 0
    … …  
    18961632end
    18971633
    1898 def prepareShellTestRunner
    1899     FileUtils.cp SCRIPTS_PATH + "jsc-stress-test-helpers" + "shell-runner.sh", $runnerDir + "runscript"
    1900 end
    1901 
    1902 def prepareMakeTestRunner
    1903     # The goals of our parallel test runner are scalability and simplicity. The
    1904     # simplicity part is particularly important. We don't want to have to have
    1905     # a full-time contributor just philosophising about parallel testing.
    1906     #
    1907     # As such, we just pass off all of the hard work to 'make'. This creates a
    1908     # dummy directory ("$outputDir/.runner") in which we create a dummy
    1909     # Makefile. The Makefile has an 'all' rule that depends on all of the tests.
    1910     # That is, for each test we know we will run, there is a rule in the
    1911     # Makefile and 'all' depends on it. Running 'make -j <whatever>' on this
    1912     # Makefile results in 'make' doing all of the hard work:
    1913     #
    1914     # - Load balancing just works. Most systems have a great load balancer in
    1915     #   'make'. If your system doesn't then just install a real 'make'.
    1916     #
    1917     # - Interruptions just work. For example Ctrl-C handling in 'make' is
    1918     #   exactly right. You don't have to worry about zombie processes.
    1919     #
    1920     # We then do some tricks to make failure detection work and to make this
    1921     # totally sound. If a test fails, we don't want the whole 'make' job to
    1922     # stop. We also don't have any facility for makefile-escaping of path names.
    1923     # We do have such a thing for shell-escaping, though. We fix both problems
    1924     # by having the actual work for each of the test rules be done in a shell
    1925     # script on the side. There is one such script per test. The script responds
    1926     # to failure by printing something on the console and then touching a
    1927     # failure file for that test, but then still returns 0. This makes 'make'
    1928     # continue past that failure and complete all the tests anyway.
    1929     #
    1930     # In the end, this script collects all of the failures by searching for
    1931     # files in the .runner directory whose name matches /^test_fail_/, where
    1932     # the thing after the 'fail_' is the test index. Those are the files that
    1933     # would be created by the test scripts if they detect failure. We're
    1934     # basically using the filesystem as a concurrent database of test failures.
    1935     # Even if two tests fail at the same time, since they're touching different
    1936     # files we won't miss any failures.
    1937     runIndices = []
    1938     $runlist.each {
    1939         | plan |
    1940         runIndices << plan.index
    1941     }
    1942    
    1943     File.open($runnerDir + "Makefile", "w") {
    1944         | outp |
    1945         outp.puts("all: " + runIndices.map{|v| "test_done_#{v}"}.join(' '))
    1946         runIndices.each {
    1947             | index |
    1948             plan = $runlist[index]
    1949             outp.puts "test_done_#{index}:"
    1950             outp.puts "\tsh test_script_#{plan.index}"
    1951         }
    1952     }
    1953 end
    1954 
    19551634def cleanRunnerDirectory
    19561635    raise unless $bundle
    … …  
    20781757
    20791758def runTestRunner
    2080     case $testRunnerType
    2081     when :shell
    2082         testRunnerCommand = "sh runscript"
    2083     when :make
    2084         testRunnerCommand = "make -j #{$numChildProcesses.to_s} -s -f Makefile"
    2085     else
    2086         raise "Unknown test runner type: #{$testRunnerType.to_s}"
    2087     end
    2088 
    20891759    if $remote
    20901760        if !$remoteDirectory
Note: See TracChangeset for help on using the changeset viewer.