Changeset 232972 in webkit

Timestamp:

Jun 19, 2018 11:59:51 AM (6 years ago)

Author:

commit-queue@webkit.org

Message:

Test262-Runner: Improve files queue to optimize CPU usage/balancing
​https://bugs.webkit.org/show_bug.cgi?id=186443

Patch by Leo Balter <Leo Balter> on 2018-06-19
Reviewed by Michael Saboff.

This patch creates a queue manager for to keep the child process open while the parent process feed each child with test files to run.

• Scripts/test262/Runner.pm:

(processCLI):
(main):

Location:

trunk/Tools

Files:

• ChangeLog (modified) (1 diff)
• Scripts/test262/Runner.pm (modified) (7 diffs)

trunk/Tools/ChangeLog

@@ +1 @@
+2018-06-19  Leo Balter  <leonardo.balter@gmail.com>
+
+        Test262-Runner: Improve files queue to optimize CPU usage/balancing
+        https://bugs.webkit.org/show_bug.cgi?id=186443
+
+        Reviewed by Michael Saboff.
+
+        This patch creates a queue manager for to keep the child process open while the parent process feed each child with test files to run.
+        * Scripts/test262/Runner.pm:
+        (processCLI):
+        (main):
+
 2018-06-19  Wenson Hsieh  <wenson_hsieh@apple.com>
 

trunk/Tools/Scripts/test262/Runner.pm

@@ -43 +43 @@
 use Config;
 use Time::HiRes qw(time);
+use IO::Handle;
+use IO::Select;
 
 my $Bin;
@@ -75 +77 @@
 
 # Commandline settings
-my $max_process;
+my $maxProcesses;
 my @cliTestDirs;
 my $verbose;
@@ -127 +129 @@
         't|t262=s' => \$test262Dir,
         'o|test-only=s@' => \@cliTestDirs,
-        'p|child-processes=i' => \$max_process,
+        'p|child-processes=i' => \$maxProcesses,
         'h|help' => \$help,
         'release' => \$release,
@@ -256 +258 @@
     }
 
-    $max_process ||= getProcesses();
+    $maxProcesses ||= getProcesses();
 
     print "\nSettings:\n"
         . "Test262 Dir: " . abs2rel($test262Dir) . "\n"
         . "JSC: " . abs2rel($JSC) . "\n"
-        . "Child Processes: $max_process\n";
+        . "Child Processes: $maxProcesses\n";
 
     print "Test timeout: $timeout\n" if $timeout;
@@ -278 +280 @@
     print "---\n\n";
 }
-
 
 sub main {
@@ -314 +315 @@
     }
 
-    # If we are processing many files, fork process
-    if (scalar @files > $max_process * 5) {
+    my $pm = Parallel::ForkManager->new($maxProcesses);
+    my $select = IO::Select->new();
+
+    my @children;
+    my @parents;
+    my $activeChildren = 0;
+
+    my @resultsfhs;
+
+    # Open each process
+    PROCESSES:
+    foreach (0..$maxProcesses-1) {
+        my $i = $_;
 
         # Make temporary files to record results
-        my @resultsfhs;
-        for (my $i = 0; $i <= $max_process-1; $i++) {
-            my ($fh, $filename) = getTempFile();
-            $resultsfhs[$i] = $fh;
-        }
-
-        my $pm = Parallel::ForkManager->new($max_process);
-        my $filesperprocess = int(scalar @files / $max_process);
-
-        FILES:
-        for (my $i = 0; $i <= $max_process-1; $i++) {
-            $pm->start and next FILES; # do the fork
-            srand(time ^ $$); # Creates a new seed for each fork
-
-            my $first = $filesperprocess * $i;
-            my $last = $i == $max_process-1 ? scalar @files : $filesperprocess * ($i+1);
-
-            for (my $j = $first; $j < $last; $j++) {
-                processFile($files[$j], $resultsfhs[$i]);
-            };
-
-            $pm->finish; # do the exit in the child process
-        };
-
-        $pm->wait_all_children;
-
-        # Read results from file into @results and close
-        for (my $i = 0; $i <= $max_process-1; $i++) {
-            seek($resultsfhs[$i], 0, 0);
-            push @results, LoadFile($resultsfhs[$i]);
-            close $resultsfhs[$i];
-        }
-    }
-    # Otherwising, running sequentially is fine
-    else {
-        my ($resfh, $resfilename) = getTempFile();
-        foreach my $file (@files) {
-            processFile($file, $resfh);
-        };
-        seek($resfh, 0, 0);
-        @results = LoadFile($resfh);
-        close $resfh;
+        my ($fh, $filename) = getTempFile();
+        $resultsfhs[$i] = $fh;
+
+        socketpair($children[$i], $parents[$i], 1, 1, 0);
+        my $child = $children[$i];
+        my $parent = $parents[$i];
+        $child->autoflush(1);
+        $parent->autoflush(1);
+
+        # seeds each child with a file;
+
+        my $pid = $pm->start;
+        if ($pid) { # parent
+            $select->add($child);
+            # each child starts with a file;
+            my $file = shift @files;
+            chomp $file;
+            if ($file) {
+                print $child "$file\n";
+                $activeChildren++;
+            }
+
+            next PROCESSES;
+        }
+
+        # children will start here
+        srand(time ^ $$); # Creates a new seed for each fork
+        CHILD:
+        while (<$parent>) {
+            my $file = $_;
+            chomp $file;
+            if ($file eq 'END') {
+                last;
+            }
+
+            processFile($file, $resultsfhs[$i]);
+            print $parent "signal\n";
+        }
+
+        $child->close();
+        $pm->finish;
+    }
+
+    my @ready;
+    FILES:
+    while ($activeChildren and @ready = $select->can_read($timeout)) {
+        foreach (@ready) {
+            my $readyChild = $_;
+            my $childMsg = <$readyChild>;
+            chomp $childMsg;
+            $activeChildren--;
+            my $file = shift @files;
+            if ($file) {
+                chomp $file;
+                print $readyChild "$file\n";
+                $activeChildren++;
+            } elsif (!$activeChildren) {
+                last FILES;
+            }
+        }
+    }
+
+    foreach (@children) {
+        print $_ "END\n";
+    }
+
+    foreach (@parents) {
+        $_->close();
+    }
+
+    my $count = 0;
+    for my $parent (@parents) {
+        my $child = $children[$count];
+        print $child "END\n";
+        $parent->close();
+        $count++;
+    }
+
+    $pm->wait_all_children;
+
+    # Read results from file into @results and close
+    foreach (0..$maxProcesses-1) {
+        my $i = $_;
+        seek($resultsfhs[$i], 0, 0);
+        push @results, LoadFile($resultsfhs[$i]);
+        close $resultsfhs[$i];
     }
 
@@ -1036 +1092 @@
 =item B<--child-processes, -p>
 
-Specify number of child processes.
+Specify the number of child processes.
 
 =item B<--t262, -t>