Changeset 277318 in webkit
- Timestamp:
- May 11, 2021, 1:34:07 AM (5 years ago)
- Location:
- trunk/Tools
- Files:
-
- 6 edited
-
ChangeLog (modified) (1 diff)
-
Scripts/run-javascriptcore-tests (modified) (1 diff)
-
Scripts/run-jsc-stress-tests (modified) (12 diffs)
-
Scripts/webkitruby/jsc-stress-test-writer-default.rb (modified) (14 diffs)
-
Scripts/webkitruby/jsc-stress-test-writer-playstation.rb (modified) (5 diffs)
-
Scripts/webkitruby/jsc-stress-test-writer-ruby.rb (modified) (4 diffs)
Legend:
- Unmodified
- Added
- Removed
-
trunk/Tools/ChangeLog
r277317 r277318 1 2021-05-11 Angelos Oikonomopoulos <angelos@igalia.com> 2 3 [JSC] detect infrastructure failure for remote stress tests 4 https://bugs.webkit.org/show_bug.cgi?id=222601 5 6 Reviewed by Mark Lam. 7 8 run-jsc-stress-tests currently detects failures by the absence of 9 a failure file (that is generated by each failing test). This is 10 fragile to begin with, as it assumes that tests that fail to run 11 (e.g. because of an error in the runner script) are successful by 12 default. 13 14 However, the main motivation for this patch is to make execution 15 more robust when using remote hosts. Currently, 16 --gnu-parallel-runner will transparently reschedule jobs on a 17 different host when a remote host goes away. But detectFailures 18 expects to be able to connect to all hosts and fetch the failure 19 files, which fails if a remote host is still down when the run 20 finishes. 21 22 Instead, this patch changes the runners to always generate a status 23 file with the exit code. detectFailures then fetches all status 24 files from all hosts that are live on exit. Tests that failed to 25 run are explicitly accounted for as 'noreport' and are set to 26 ERROR in the final report. 27 28 * Scripts/run-javascriptcore-tests: 29 (runJSCStressTests): 30 * Scripts/run-jsc-stress-tests: 31 * Scripts/webkitruby/jsc-stress-test-writer-default.rb: 32 1 33 2021-05-10 Chris Dumez <cdumez@apple.com> 2 34 -
trunk/Tools/Scripts/run-javascriptcore-tests
r275814 r277318 933 933 print "\n"; 934 934 935 my @jscStressNoResultList = readAllLines($jscStressResultsDir . "/noresult"); 936 my $numJSCStressNoResultTests = @jscStressNoResultList; 937 938 if ($numJSCStressNoResultTests) { 939 $isTestFailed = 1; 940 } 941 foreach my $testNoResult (@jscStressNoResultList) { 942 $reportData{$testNoResult} = {actual => "ERROR"}; 943 } 944 935 945 print "Results for JSC stress tests:\n"; 936 946 printThingsFound($numJSCStressFailures, "failure", "failures", "found"); 937 print " OK.\n" if $numJSCStressFailures == 0; 947 printThingsFound($numJSCStressNoResultTests, "test", "tests", "failed to complete"); 948 print " OK.\n" if $numJSCStressFailures == 0 and $numJSCStressNoResultTests == 0; 938 949 939 950 print "\n"; -
trunk/Tools/Scripts/run-jsc-stress-tests
r276646 r277318 58 58 59 59 HELPERS_PATH = SCRIPTS_PATH + "jsc-stress-test-helpers" 60 STATUS_FILE_PREFIX = "test_status_" 61 STATUS_FILE_PASS = "P" 62 STATUS_FILE_FAIL = "F" 60 63 61 64 begin … … 136 139 $artifact_exec_wrapper = nil 137 140 $numChildProcessesSetByUser = false 141 $runUniqueId = Random.new.bytes(16).unpack("H*")[0] 138 142 139 143 def usage … … 538 542 end 539 543 540 $numFailures = 0541 $numPasses = 0542 543 544 # We force all tests to use a smaller (1.5M) stack so that stack overflow tests can run faster. 544 545 BASE_OPTIONS = ["--useFTLJIT=false", "--useFunctionDotArguments=true", "--validateExceptionChecks=true", "--useDollarVM=true", "--maxPerThreadStackUsage=1572864"] … … 1824 1825 outp.puts plan.name 1825 1826 } 1826 $numFailures += 11827 1827 end 1828 1828 … … 1832 1832 outp.puts plan.name 1833 1833 } 1834 $numPasses += 1 1834 end 1835 1836 def appendNoResult(plan) 1837 File.open($outputDir + "noresult", "a") { 1838 | outp | 1839 outp.puts plan.name 1840 } 1835 1841 end 1836 1842 … … 2040 2046 end 2041 2047 2042 def sshRead(cmd, remoteIndex=0 )2048 def sshRead(cmd, remoteIndex=0, options={}) 2043 2049 raise unless $remote 2044 2050 … … 2053 2059 } 2054 2060 } 2055 raise "#{$?}" unless $?.success? 2061 raise "#{$?}" unless $?.success? or options[:ignoreFailure] 2056 2062 result 2057 2063 end … … 2208 2214 end 2209 2215 2210 def detectFailures2211 raise if $bundle2212 failures = []2216 def getStatusMap 2217 name_re = /^[.]\/#{STATUS_FILE_PREFIX}(\d+)$/ 2218 map = {} 2213 2219 if $remote 2214 2220 $remoteHosts.each_with_index { 2215 2221 | host, remoteIndex | 2216 output = sshRead("cd #{host.remoteDirectory}/#{$outputDir.basename}/.runner && find . -maxdepth 1 -name \" test_fail_*\"", remoteIndex)2222 output = sshRead("cd #{host.remoteDirectory}/#{$outputDir.basename}/.runner && find . -maxdepth 1 -name \"#{STATUS_FILE_PREFIX}*\" -exec sh -c \"printf \\\"%s \\\" {}; cat {}\" \\;", remoteIndex, :ignoreFailure => true) 2217 2223 output.split(/\n/).each { 2218 2224 | line | 2219 next unless line =~ /test_fail_/ 2220 failures << $~.post_match.to_i 2225 name, run_id, _, result = line.split(' ') 2226 md = name_re.match(name) 2227 if md.nil? 2228 $stderr.puts("Could not parse name in `#{line}`") 2229 exit(1) 2230 end 2231 if run_id != $runUniqueId 2232 # This may conceivably happen if a remote goes 2233 # away in the middle of a run and comes back 2234 # online in the middle of a different run. 2235 $stderr.puts("Ignoring stale status file for #{name} (ID #{run_id} but current ID is #{$runUniqueId})") 2236 next 2237 end 2238 index = md[1].to_i 2239 if map.has_key?(index) 2240 $stderr.puts("Duplicate state file for #{index}") 2241 # One scenario in which this could happen: 2242 # Test T runs on remote host A and 2243 # 1. the status file reaches A's disk 2244 # 2. somehow the gnu parallel runner is not made aware of the test's completion (packet loss?) 2245 # 3. A machine crashes 2246 # 4. gnu parallel re-schedules the test to run on remote host B, where it runs to completion 2247 # 5. B comes back online before the end of the run 2248 # 6. we collect the status files from all remotes and end up with two status files for T. 2249 prev = map[index] 2250 # map[index] holds 2251 # - a number, if all results codes we've observed for a test are the same 2252 # - an array, if they diverge. 2253 if prev.is_a?(Array) 2254 prev.push(result) 2255 elsif prev != result 2256 # If the two results differ, keep them 2257 # both. This is simply a way to make note of 2258 # the divergence (for later reporting). 2259 map[index] = [prev, result] 2260 else 2261 # Got the same result, no need to do anything. 2262 end 2263 else 2264 map[index] = result 2265 end 2221 2266 } 2222 2267 } … … 2224 2269 Dir.foreach($runnerDir) { 2225 2270 | filename | 2226 next unless filename =~ /test_fail_/ 2227 failures << $~.post_match.to_i 2271 md = name_re.match("./#{filename}") 2272 next unless md 2273 File.open("#{$runnerDir}/#{filename}", "r") { |f| 2274 runId, _, result = f.read.chomp.split(' ') 2275 if runId != $runUniqueId 2276 # We clean the dir before a starting a run. 2277 raise "Can't happen" 2278 end 2279 map[md[1].to_i] = result 2280 } 2228 2281 } 2229 2282 end 2230 2231 failureSet = {} 2232 2233 failures.each { 2234 | failure | 2235 appendFailure($runlist[failure]) 2236 failureSet[failure] = true 2237 } 2238 2283 map 2284 end 2285 2286 def detectFailures 2287 raise if $bundle 2288 noresult = 0 2289 statusMap = getStatusMap 2239 2290 familyMap = {} 2291 2240 2292 $runlist.each_with_index { 2241 2293 | plan, index | … … 2243 2295 familyMap[plan.family] = [] 2244 2296 end 2245 if failureSet[index]2246 append Result(plan, false)2247 familyMap[plan.family] << {:result => "FAIL", :plan => plan};2297 if not statusMap.has_key?(index) or statusMap[index].is_a?(Array) 2298 appendNoResult(plan) 2299 noresult += 1 2248 2300 next 2301 end 2302 result = nil 2303 if statusMap[index] == STATUS_FILE_PASS 2304 appendPass(plan) 2305 result = "PASS" 2249 2306 else 2250 append Result(plan, true)2251 familyMap[plan.family] << {:result => "PASS", :plan => plan};2307 appendFailure(plan) 2308 result = "FAIL" 2252 2309 end 2253 appendPass(plan) 2254 } 2310 appendResult(plan, statusMap[index] == STATUS_FILE_PASS) 2311 familyMap[plan.family] << {:result => result, :plan => plan } 2312 } 2313 2314 if noresult > 0 2315 $stderr.puts("Could not get the exit status for #{noresult} tests") 2316 # We can't change our exit code, as run-javascriptcore-tests 2317 # expects 0 even when there are failures. 2318 end 2255 2319 2256 2320 File.open($outputDir + "resultsByFamily", "w") { … … 2264 2328 outp.puts 2265 2329 end 2266 2330 2267 2331 outp.print "#{familyName}:" 2268 2332 … … 2302 2366 clean($outputDir + "failed") 2303 2367 clean($outputDir + "passed") 2368 clean($outputDir + "noresult") 2304 2369 clean($outputDir + "results") 2305 2370 clean($outputDir + "resultsByFamily") -
trunk/Tools/Scripts/webkitruby/jsc-stress-test-writer-default.rb
r275814 r277318 38 38 Proc.new { 39 39 | name | 40 " | " +pipeAndPrefixCommand((Pathname("..") + (name + ".out")).to_s, name)40 pipeAndPrefixCommand((Pathname("..") + (name + ".out")).to_s, name) 41 41 } 42 42 end … … 46 46 Proc.new { 47 47 | name | 48 " | cat > " + Shellwords.shellescape((Pathname("..") + (name + ".out")).to_s) 49 } 48 "cat > " + Shellwords.shellescape((Pathname("..") + (name + ".out")).to_s) 49 } 50 end 51 52 def getAndTestExitCode(plan, condition) 53 <<-EOF 54 if test "$exitCode" #{condition} 55 EOF 50 56 end 51 57 … … 55 61 Proc.new { 56 62 | outp, plan | 57 outp.puts "if test -e #{plan.failFile}"58 outp.puts "then" 59 outp.puts " (echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)63 outp.puts getAndTestExitCode(plan, "-ne 0") 64 outp.puts "then" 65 outp.puts " (echo ERROR: Unexpected exit code: $exitCode) | " + redirectAndPrefixCommand(plan.name) 60 66 outp.puts " " + plan.failCommand 61 67 outp.puts "else" … … 69 75 Proc.new { 70 76 | outp, plan | 71 outp.puts "if test -e #{plan.failFile}"77 outp.puts getAndTestExitCode(plan, "-ne 0") 72 78 outp.puts "then" 73 79 outp.puts " " + plan.successCommand … … 85 91 | outp, plan | 86 92 outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s) 87 88 outp.puts "if test -e #{plan.failFile}"89 outp.puts "then" 90 outp.puts " (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)93 94 outp.puts getAndTestExitCode(plan, "-ne 0") 95 outp.puts "then" 96 outp.puts " (cat #{outputFilename} && echo ERROR: Unexpected exit code: $exitCode) | " + redirectAndPrefixCommand(plan.name) 91 97 outp.puts " " + plan.failCommand 92 98 outp.puts "else" … … 102 108 outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s) 103 109 diffFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".diff")).to_s) 104 105 outp.puts "if test -e #{plan.failFile}"106 outp.puts "then" 107 outp.puts " (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)110 111 outp.puts getAndTestExitCode(plan, "-ne 0") 112 outp.puts "then" 113 outp.puts " (cat #{outputFilename} && echo ERROR: Unexpected exit code: $exitCode) | " + redirectAndPrefixCommand(plan.name) 108 114 outp.puts " " + plan.failCommand 109 115 outp.puts "elif test -e ../#{Shellwords.shellescape(expectedFilename)}" … … 131 137 outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s) 132 138 133 outp.puts "if test -e #{plan.failFile}"134 outp.puts "then" 135 outp.puts " (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)139 outp.puts getAndTestExitCode(plan, "-ne 0") 140 outp.puts "then" 141 outp.puts " (cat #{outputFilename} && echo ERROR: Unexpected exit code: $exitCode) | " + redirectAndPrefixCommand(plan.name) 136 142 outp.puts " " + plan.failCommand 137 143 outp.puts "elif grep -i -q failed! #{outputFilename}" … … 152 158 outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s) 153 159 154 outp.puts "if test -e #{plan.failFile}"160 outp.puts getAndTestExitCode(plan, "-ne 0") 155 161 outp.puts "then" 156 162 outp.puts " " + plan.successCommand … … 172 178 outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s) 173 179 174 outp.puts "if test -e #{plan.failFile}"175 outp.puts "then" 176 outp.puts " if [ `cat #{plan.failFile}`-eq 3 ]"180 outp.puts getAndTestExitCode(plan, "-ne 0") 181 outp.puts "then" 182 outp.puts " if [ \"$exitCode\" -eq 3 ]" 177 183 outp.puts " then" 178 184 outp.puts " if grep -i -q failed! #{outputFilename}" … … 184 190 outp.puts " fi" 185 191 outp.puts " else" 186 outp.puts " (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)192 outp.puts " (cat #{outputFilename} && echo ERROR: Unexpected exit code: $exitCode) | " + redirectAndPrefixCommand(plan.name) 187 193 outp.puts " " + plan.failCommand 188 194 outp.puts " fi" … … 201 207 outputFilename = Shellwords.shellescape((Pathname("..") + (plan.name + ".out")).to_s) 202 208 203 outp.puts "if test -e #{plan.failFile}"204 outp.puts "then" 205 outp.puts " (cat #{outputFilename} && echo ERROR: Unexpected exit code: `cat #{plan.failFile}`) | " + redirectAndPrefixCommand(plan.name)209 outp.puts getAndTestExitCode(plan, "-ne 0") 210 outp.puts "then" 211 outp.puts " (cat #{outputFilename} && echo ERROR: Unexpected exit code: $exitCode) | " + redirectAndPrefixCommand(plan.name) 206 212 outp.puts " " + plan.failCommand 207 213 outp.puts "elif grep -i -q FAILED #{outputFilename}" … … 263 269 "echo #{Shellwords.shellescape(script)} > #{Shellwords.shellescape((Pathname.new("..") + @name).to_s)}" 264 270 end 265 271 272 def statusCommand(status) 273 "echo #{$runUniqueId} $exitCode #{status} > #{statusFile}" 274 end 275 266 276 def failCommand 267 " echo FAIL: #{Shellwords.shellescape(@name)} ; touch #{failFile}; " + reproScriptCommand277 "#{statusCommand(STATUS_FILE_FAIL)}; echo FAIL: #{Shellwords.shellescape(@name)}; " + reproScriptCommand 268 278 end 269 279 270 280 def successCommand 281 command = "" 271 282 executionTimeMessage = "" 272 283 if $reportExecutionTime … … 274 285 end 275 286 if $progressMeter or $reportExecutionTime or $verbosity >= 2 276 "rm -f #{failFile} ; echo PASS: #{Shellwords.shellescape(@name)}#{executionTimeMessage}" 277 else 278 "rm -f #{failFile}" 287 command = "echo PASS: #{Shellwords.shellescape(@name)}#{executionTimeMessage}" 279 288 end 280 end 281 282 def failFile 283 "test_fail_#{@index}" 289 "#{statusCommand(STATUS_FILE_PASS)}; #{command}" 290 end 291 292 def statusFile 293 "#{STATUS_FILE_PREFIX}#{@index}" 284 294 end 285 295 … … 291 301 end 292 302 outp.puts "echo Running #{Shellwords.shellescape(@name)}" 293 cmd = "(" + shellCommand + " || (echo $? > #{failFile})) 2>&1 " 294 cmd += @outputHandler.call(@name) 303 # 304 # +--------------------------------------------------------------------+ 305 # | +-----------------------------------------------+ | 306 # | | +--------------+ +-------------------+ | | 307 # | | | cmd 1 ----> 1|---> |0 --> outH 1 ---> 4|-> 4|---------------> 1| 308 # | | | 2 / | +-------------------+ | +-----------+ | 309 # | | |echo $? 0 -> 3|---------------------------> 1|-> |0 read xs | | 310 # | | +--------------+ | | exit $xs | | 311 # | | | +-----------+ | 312 # | +-----------------------------------------------+ | 313 # +--------------------------------------------------------------------+ 314 # From the top down (i.e. reading from the outer expression inwards): 315 # 316 # - Redirect FD 4 to our stdout 317 # 318 # - Build a pipe of two command sequences. The 319 # right-hand-side sequence reads a number from stdin and 320 # exits with it. Since it's the last command in the 321 # pipeline, this will be the value of $? after the 322 # pipeline completes. 323 # 324 # - In the left-hand-side sequence, redirect FD 3 to FD 1. 325 # 326 # - Build a pipe of two commands 327 # - run shellCommand, writing its exit code to FD 3. 328 # - run the outputHandler, with its stdin coming from 329 # the pipe, redirecting its output to FD 4. The 330 # outputHandler needs to be in a command sequence 331 # (i.e. in { cmd; ...}) as it may do its own 332 # redirections. 333 # 334 # We do all this 335 # - to avoid having to use a temporary file for the exit code 336 # - to keep within the bounds of POSIX sh (i.e. can't use 337 # PIPESTATUS) 338 cmd = "{ { { { #{shellCommand} 2>&1; echo $? >&3; } | { #{outputHandler.call(@name)} ;} >&4; } 3>&1; } | { read xs; exit $xs; } } 4>&1\nexitCode=$?\n" 295 339 if $verbosity >= 3 296 340 outp.puts "echo #{Shellwords.shellescape(cmd)}" -
trunk/Tools/Scripts/webkitruby/jsc-stress-test-writer-playstation.rb
r262991 r277318 306 306 end 307 307 308 def statusCommand(status_code) 309 # May be called in th rescue block, so status is not 310 # guaranteed to be set; if it isn't, set the exit code to 311 # something that's clearly invalid. 312 <<-END_STATUS_COMMAND 313 File.open("#{statusFile}", "w") { |f| 314 f.puts("#{$runUniqueId} \#{status.nil? ? 999999999 : status.exitstatus} #{status_code}") 315 } 316 END_STATUS_COMMAND 317 end 318 308 319 def failCommand 309 320 <<-END_FAIL_COMMAND 310 321 print "FAIL: #{Shellwords.shellescape(@name)}\n" 311 FileUtils.touch("#{failFile}")322 #{statusCommand(STATUS_FILE_FAIL)} 312 323 #{reproScriptCommand} 313 324 END_FAIL_COMMAND … … 317 328 if $progressMeter or $verbosity >= 2 318 329 <<-END_VERBOSE_SUCCESS_COMMAND 319 File.unlink("#{failFile}") if File.exists?("#{failFile}")320 330 print "PASS: #{Shellwords.shellescape(@name)}\n" 331 #{statusCommand(STATUS_FILE_PASS)} 321 332 END_VERBOSE_SUCCESS_COMMAND 322 333 else 323 " File.unlink(\"#{failFile}\") if File.exists?(\"#{failFile}\")\n"334 "#{statusCommand(STATUS_FILE_PASS)}\n" 324 335 end 325 336 end 326 337 327 def failFile 328 "test_fail_#{@index}" 329 end 330 331 def statusWrite 332 <<-END_STATUS_WRITE 333 if !success 334 File.open("#{failFile}", "w") do |code_file| 335 code_file.puts status 336 end 337 end 338 END_STATUS_WRITE 338 def statusFile 339 "#{STATUS_FILE_PREFIX}#{@index}" 339 340 end 340 341 … … 358 359 checkScript: filename, 359 360 args: @arguments, 360 failFile: "#{failFile}"361 361 }) 362 362 } … … 372 372 cmd = shellCommand 373 373 374 cmd += statusWrite375 376 374 cmd += @outputHandler.call(@name) 377 375 … … 383 381 outp.puts "rescue RuntimeError => e" 384 382 outp.puts " print \"FAIL: #{Shellwords.shellescape(@name)}\\n\"" 385 outp.puts " FileUtils.touch(\"#{failFile}\")"383 outp.puts " #{statusCommand(STATUS_FILE_FAIL)}" 386 384 outp.puts "end" 387 385 } -
trunk/Tools/Scripts/webkitruby/jsc-stress-test-writer-ruby.rb
r271427 r277318 336 336 end 337 337 338 def statusCommand(status_code) 339 # May be called in th rescue block, so status is not 340 # guaranteed to be set; if it isn't, set the exit code to 341 # something that's clearly invalid. 342 <<-END_STATUS_COMMAND 343 File.open("#{statusFile}", "w") { |f| 344 f.puts("#{$runUniqueId} \#{status.nil? ? 999999999 : status.exitstatus} #{status_code}") 345 } 346 END_STATUS_COMMAND 347 end 348 338 349 def failCommand 339 350 <<-END_FAIL_COMMAND 340 351 print "FAIL: #{Shellwords.shellescape(@name)}\n" 341 FileUtils.touch("#{failFile}")352 #{statusCommand(STATUS_FILE_FAIL)} 342 353 #{reproScriptCommand} 343 354 END_FAIL_COMMAND … … 347 358 if $progressMeter or $verbosity >= 2 348 359 <<-END_VERBOSE_SUCCESS_COMMAND 349 File.unlink("#{failFile}") if File.exists?("#{failFile}")350 360 print "PASS: #{Shellwords.shellescape(@name)}\n" 361 #{statusCommand(STATUS_FILE_PASS)} 351 362 END_VERBOSE_SUCCESS_COMMAND 352 363 else 353 " File.unlink(\"#{failFile}\") if File.exists?(\"#{failFile}\")\n"364 "#{statusCommand(STATUS_FILE_PASS)}\n" 354 365 end 355 366 end 356 367 357 def failFile 358 "test_fail_#{@index}" 359 end 360 361 def statusWrite 362 <<-END_STATUS_WRITE 363 if !success(status) 364 File.open("#{failFile}", "w") do |code_file| 365 code_file.puts status.exitstatus 366 end 367 end 368 END_STATUS_WRITE 369 end 370 368 def statusFile 369 "#{STATUS_FILE_PREFIX}#{@index}" 370 end 371 371 372 def writeRunScript(filename) 372 373 File.open(filename, "w") { … … 383 384 cmd = shellCommand 384 385 385 cmd += statusWrite386 387 386 cmd += @outputHandler.call(@name) 388 387 … … 394 393 outp.puts "rescue" 395 394 outp.puts " print \"FAIL: #{Shellwords.shellescape(@name)}\\n\"" 396 outp.puts " FileUtils.touch(\"#{failFile}\")"395 outp.puts " #{statusCommand(STATUS_FILE_FAIL)}" 397 396 outp.puts "end" 398 397 }
Note:
See TracChangeset
for help on using the changeset viewer.
