Changeset 225565 in webkit


Ignore:
Timestamp:
Dec 5, 2017 7:54:51 PM (6 years ago)
Author:
commit-queue@webkit.org
Message:

Fast binding generation by batching several files at once
​https://bugs.webkit.org/show_bug.cgi?id=179958

Patch by Basuke Suzuki <Basuke Suzuki> on 2017-12-05
Reviewed by Alex Christensen.

Passing several files to the sub process at once and reducing the total cost of
process generation. Especially Windows port has great speed improvement because
its implementation of fork/exec is terrible.

  • bindings/scripts/generate-bindings-all.pl:

(spawnGenerateBindingsIfNeeded):

  • bindings/scripts/generate-bindings.pl:

(generateBindings):

Location:
trunk/Source/WebCore
Files:
3 edited

Legend:

Unmodified
Added
Removed

  • trunk/Source/WebCore/ChangeLog

    r225564 r225565  
     12017-12-05  Basuke Suzuki  <Basuke.Suzuki@sony.com>
     2
     3        Fast binding generation by batching several files at once
     4        https://bugs.webkit.org/show_bug.cgi?id=179958
     5
     6        Reviewed by Alex Christensen.
     7
     8        Passing several files to the sub process at once and reducing the total cost of
     9        process generation. Especially Windows port has great speed improvement because
     10        its implementation of fork/exec is terrible.
     11
     12        * bindings/scripts/generate-bindings-all.pl:
     13        (spawnGenerateBindingsIfNeeded):
     14        * bindings/scripts/generate-bindings.pl:
     15        (generateBindings):
     16
    1172017-12-05  David Quesada  <david_quesada@apple.com>
    218

  • trunk/Source/WebCore/bindings/scripts/generate-bindings-all.pl

    r215166 r225565  
    164164    return if $abort;
    165165    return unless @idlFilesToUpdate;
    166     my $file = shift @idlFilesToUpdate;
    167     $currentCount++;
    168     my $basename = basename($file);
    169     printProgress("[$currentCount/$totalCount] $basename");
    170     my $pid = spawnCommand($perl, @args, $file);
     166    my $batchCount = 30;
     167    # my $batchCount = int(($totalCount - $currentCount) / $numOfJobs) || 1;
     168    my @files = splice(@idlFilesToUpdate, 0, $batchCount);
     169    for (@files) {
     170        $currentCount++;
     171        my $basename = basename($_);
     172        printProgress("[$currentCount/$totalCount] $basename");
     173    }
     174    my $pid = spawnCommand($perl, @args, @files);
    171175    $abort = 1 unless defined $pid;
    172176}

  • trunk/Source/WebCore/bindings/scripts/generate-bindings.pl

    r223047 r225565  
    7272           'idlAttributesFile=s' => \$idlAttributesFile);
    7373
    74 my $targetIdlFile = $ARGV[0];
    75 
    76 die('Must specify input file.') unless defined($targetIdlFile);
     74die('Must specify input file.') unless @ARGV;
    7775die('Must specify generator') unless defined($generator);
    7876die('Must specify output directory.') unless defined($outputDirectory);
    … …  
    8280    $outputHeadersDirectory = $outputDirectory;
    8381}
    84 $targetIdlFile = Cwd::realpath($targetIdlFile);
    85 if ($verbose) {
    86     print "$generator: $targetIdlFile\n";
    87 }
    88 my $targetInterfaceName = fileparse($targetIdlFile, ".idl");
    89 
    90 my $idlFound = 0;
    91 my @supplementedIdlFiles;
    92 if ($supplementalDependencyFile) {
    93     # The format of a supplemental dependency file:
    94     #
    95     # DOMWindow.idl P.idl Q.idl R.idl
    96     # Document.idl S.idl
    97     # Event.idl
    98     # ...
    99     #
    100     # The above indicates that DOMWindow.idl is supplemented by P.idl, Q.idl and R.idl,
    101     # Document.idl is supplemented by S.idl, and Event.idl is supplemented by no IDLs.
    102     # The IDL that supplements another IDL (e.g. P.idl) never appears in the dependency file.
    103     open FH, "< $supplementalDependencyFile" or die "Cannot open $supplementalDependencyFile\n";
    104     while (my $line = <FH>) {
    105         my ($idlFile, @followingIdlFiles) = split(/\s+/, $line);
    106         if ($idlFile and fileparse($idlFile) eq fileparse($targetIdlFile)) {
    107             $idlFound = 1;
    108             @supplementedIdlFiles = sort @followingIdlFiles;
    109         }
    110     }
    111     close FH;
    112 
    113     # $additionalIdlFiles is list of IDL files which should not be included in
    114     # DerivedSources*.cpp (i.e. they are not described in the supplemental
    115     # dependency file) but should generate .h and .cpp files.
    116     if (!$idlFound and $additionalIdlFiles) {
    117         my @idlFiles = shellwords($additionalIdlFiles);
    118         $idlFound = grep { $_ and fileparse($_) eq fileparse($targetIdlFile) } @idlFiles;
    119     }
    120 
    121     if (!$idlFound) {
    122         my $codeGen = CodeGenerator->new(\@idlDirectories, $generator, $outputDirectory, $outputHeadersDirectory, $preprocessor, $writeDependencies, $verbose);
    123 
    124         # We generate empty .h and .cpp files just to tell build scripts that .h and .cpp files are created.
    125         generateEmptyHeaderAndCpp($codeGen->FileNamePrefix(), $targetInterfaceName, $outputHeadersDirectory, $outputDirectory);
    126         exit 0;
    127     }
    128 }
    129 
    130 my $input;
     82
     83generateBindings($_) for (@ARGV);
     84
     85sub generateBindings
    13186{
    132     local $INPUT_RECORD_SEPARATOR;
    133     open(JSON, "<", $idlAttributesFile) or die "Couldn't open $idlAttributesFile: $!";
    134     $input = <JSON>;
    135     close(JSON);
    136 }
    137 
    138 my $jsonDecoder = JSON::PP->new->utf8;
    139 my $jsonHashRef = $jsonDecoder->decode($input);
    140 my $idlAttributes = $jsonHashRef->{attributes};
    141 
    142 # Parse the target IDL file.
    143 my $targetParser = IDLParser->new(!$verbose);
    144 my $targetDocument = $targetParser->Parse($targetIdlFile, $defines, $preprocessor, $idlAttributes);
    145 
    146 foreach my $idlFile (@supplementedIdlFiles) {
    147     next if $idlFile eq $targetIdlFile;
    148 
    149     my $interfaceName = fileparse($idlFile, ".idl");
    150     my $parser = IDLParser->new(!$verbose);
    151     my $document = $parser->Parse($idlFile, $defines, $preprocessor, $idlAttributes);
    152 
    153     foreach my $interface (@{$document->interfaces}) {
    154         if (!$interface->isPartial || $interface->type->name eq $targetInterfaceName) {
    155             my $targetDataNode;
    156             my @targetGlobalContexts;
    157             foreach my $interface (@{$targetDocument->interfaces}) {
    158                 if ($interface->type->name eq $targetInterfaceName) {
    159                     $targetDataNode = $interface;
    160                     my $exposedAttribute = $targetDataNode->extendedAttributes->{"Exposed"} || "Window";
    161                     $exposedAttribute = substr($exposedAttribute, 1, -1) if substr($exposedAttribute, 0, 1) eq "(";
    162                     @targetGlobalContexts = split(",", $exposedAttribute);
    163                     last;
    164                 }
     87    my ($targetIdlFile) = @_;
     88
     89    $targetIdlFile = Cwd::realpath($targetIdlFile);
     90    if ($verbose) {
     91        print "$generator: $targetIdlFile\n";
     92    }
     93    my $targetInterfaceName = fileparse($targetIdlFile, ".idl");
     94
     95    my $idlFound = 0;
     96    my @supplementedIdlFiles;
     97    if ($supplementalDependencyFile) {
     98        # The format of a supplemental dependency file:
     99        #
     100        # DOMWindow.idl P.idl Q.idl R.idl
     101        # Document.idl S.idl
     102        # Event.idl
     103        # ...
     104        #
     105        # The above indicates that DOMWindow.idl is supplemented by P.idl, Q.idl and R.idl,
     106        # Document.idl is supplemented by S.idl, and Event.idl is supplemented by no IDLs.
     107        # The IDL that supplements another IDL (e.g. P.idl) never appears in the dependency file.
     108        open FH, "< $supplementalDependencyFile" or die "Cannot open $supplementalDependencyFile\n";
     109        while (my $line = <FH>) {
     110            my ($idlFile, @followingIdlFiles) = split(/\s+/, $line);
     111            if ($idlFile and fileparse($idlFile) eq fileparse($targetIdlFile)) {
     112                $idlFound = 1;
     113                @supplementedIdlFiles = sort @followingIdlFiles;
    165114            }
    166             die "Not found an interface ${targetInterfaceName} in ${targetInterfaceName}.idl." unless defined $targetDataNode;
    167 
    168             # Support for attributes of partial interfaces.
    169             foreach my $attribute (@{$interface->attributes}) {
    170                 next unless shouldPropertyBeExposed($attribute, \@targetGlobalContexts);
    171 
    172                 # Record that this attribute is implemented by $interfaceName.
    173                 $attribute->extendedAttributes->{"ImplementedBy"} = $interfaceName if $interface->isPartial && !$attribute->extendedAttributes->{Reflect};
    174 
    175                 # Add interface-wide extended attributes to each attribute.
    176                 foreach my $extendedAttributeName (keys %{$interface->extendedAttributes}) {
    177                     $attribute->extendedAttributes->{$extendedAttributeName} = $interface->extendedAttributes->{$extendedAttributeName};
    178                 }
    179                 push(@{$targetDataNode->attributes}, $attribute);
     115        }
     116        close FH;
     117
     118        # $additionalIdlFiles is list of IDL files which should not be included in
     119        # DerivedSources*.cpp (i.e. they are not described in the supplemental
     120        # dependency file) but should generate .h and .cpp files.
     121        if (!$idlFound and $additionalIdlFiles) {
     122            my @idlFiles = shellwords($additionalIdlFiles);
     123            $idlFound = grep { $_ and fileparse($_) eq fileparse($targetIdlFile) } @idlFiles;
     124        }
     125
     126        if (!$idlFound) {
     127            my $codeGen = CodeGenerator->new(\@idlDirectories, $generator, $outputDirectory, $outputHeadersDirectory, $preprocessor, $writeDependencies, $verbose);
     128
     129            # We generate empty .h and .cpp files just to tell build scripts that .h and .cpp files are created.
     130            generateEmptyHeaderAndCpp($codeGen->FileNamePrefix(), $targetInterfaceName, $outputHeadersDirectory, $outputDirectory);
     131            return;
     132        }
     133    }
     134
     135    my $input;
     136    {
     137        local $INPUT_RECORD_SEPARATOR;
     138        open(JSON, "<", $idlAttributesFile) or die "Couldn't open $idlAttributesFile: $!";
     139        $input = <JSON>;
     140        close(JSON);
     141    }
     142
     143    my $jsonDecoder = JSON::PP->new->utf8;
     144    my $jsonHashRef = $jsonDecoder->decode($input);
     145    my $idlAttributes = $jsonHashRef->{attributes};
     146
     147    # Parse the target IDL file.
     148    my $targetParser = IDLParser->new(!$verbose);
     149    my $targetDocument = $targetParser->Parse($targetIdlFile, $defines, $preprocessor, $idlAttributes);
     150
     151    foreach my $idlFile (@supplementedIdlFiles) {
     152        next if $idlFile eq $targetIdlFile;
     153
     154        my $interfaceName = fileparse($idlFile, ".idl");
     155        my $parser = IDLParser->new(!$verbose);
     156        my $document = $parser->Parse($idlFile, $defines, $preprocessor, $idlAttributes);
     157
     158        foreach my $interface (@{$document->interfaces}) {
     159            if (!$interface->isPartial || $interface->type->name eq $targetInterfaceName) {
     160                my $targetDataNode;
     161                my @targetGlobalContexts;
     162                foreach my $interface (@{$targetDocument->interfaces}) {
     163                    if ($interface->type->name eq $targetInterfaceName) {
     164                        $targetDataNode = $interface;
     165                        my $exposedAttribute = $targetDataNode->extendedAttributes->{"Exposed"} || "Window";
     166                        $exposedAttribute = substr($exposedAttribute, 1, -1) if substr($exposedAttribute, 0, 1) eq "(";
     167                        @targetGlobalContexts = split(",", $exposedAttribute);
     168                        last;
     169                    }
     170                }
     171                die "Not found an interface ${targetInterfaceName} in ${targetInterfaceName}.idl." unless defined $targetDataNode;
     172
     173                # Support for attributes of partial interfaces.
     174                foreach my $attribute (@{$interface->attributes}) {
     175                    next unless shouldPropertyBeExposed($attribute, \@targetGlobalContexts);
     176
     177                    # Record that this attribute is implemented by $interfaceName.
     178                    $attribute->extendedAttributes->{"ImplementedBy"} = $interfaceName if $interface->isPartial && !$attribute->extendedAttributes->{Reflect};
     179
     180                    # Add interface-wide extended attributes to each attribute.
     181                    foreach my $extendedAttributeName (keys %{$interface->extendedAttributes}) {
     182                        $attribute->extendedAttributes->{$extendedAttributeName} = $interface->extendedAttributes->{$extendedAttributeName};
     183                    }
     184                    push(@{$targetDataNode->attributes}, $attribute);
     185                }
     186
     187                # Support for methods of partial interfaces.
     188                foreach my $operation (@{$interface->operations}) {
     189                    next unless shouldPropertyBeExposed($operation, \@targetGlobalContexts);
     190
     191                    # Record that this method is implemented by $interfaceName.
     192                    $operation->extendedAttributes->{"ImplementedBy"} = $interfaceName if $interface->isPartial;
     193
     194                    # Add interface-wide extended attributes to each method.
     195                    foreach my $extendedAttributeName (keys %{$interface->extendedAttributes}) {
     196                        $operation->extendedAttributes->{$extendedAttributeName} = $interface->extendedAttributes->{$extendedAttributeName};
     197                    }
     198                    push(@{$targetDataNode->operations}, $operation);
     199                }
     200
     201                # Support for constants of partial interfaces.
     202                foreach my $constant (@{$interface->constants}) {
     203                    next unless shouldPropertyBeExposed($constant, \@targetGlobalContexts);
     204
     205                    # Record that this constant is implemented by $interfaceName.
     206                    $constant->extendedAttributes->{"ImplementedBy"} = $interfaceName if $interface->isPartial;
     207
     208                    # Add interface-wide extended attributes to each constant.
     209                    foreach my $extendedAttributeName (keys %{$interface->extendedAttributes}) {
     210                        $constant->extendedAttributes->{$extendedAttributeName} = $interface->extendedAttributes->{$extendedAttributeName};
     211                    }
     212                    push(@{$targetDataNode->constants}, $constant);
     213                }
     214            } else {
     215                die "$idlFile is not a supplemental dependency of $targetIdlFile. There maybe a bug in the supplemental dependency generator (preprocess-idls.pl).\n";
    180216            }
    181 
    182             # Support for methods of partial interfaces.
    183             foreach my $operation (@{$interface->operations}) {
    184                 next unless shouldPropertyBeExposed($operation, \@targetGlobalContexts);
    185 
    186                 # Record that this method is implemented by $interfaceName.
    187                 $operation->extendedAttributes->{"ImplementedBy"} = $interfaceName if $interface->isPartial;
    188 
    189                 # Add interface-wide extended attributes to each method.
    190                 foreach my $extendedAttributeName (keys %{$interface->extendedAttributes}) {
    191                     $operation->extendedAttributes->{$extendedAttributeName} = $interface->extendedAttributes->{$extendedAttributeName};
    192                 }
    193                 push(@{$targetDataNode->operations}, $operation);
    194             }
    195 
    196             # Support for constants of partial interfaces.
    197             foreach my $constant (@{$interface->constants}) {
    198                 next unless shouldPropertyBeExposed($constant, \@targetGlobalContexts);
    199 
    200                 # Record that this constant is implemented by $interfaceName.
    201                 $constant->extendedAttributes->{"ImplementedBy"} = $interfaceName if $interface->isPartial;
    202 
    203                 # Add interface-wide extended attributes to each constant.
    204                 foreach my $extendedAttributeName (keys %{$interface->extendedAttributes}) {
    205                     $constant->extendedAttributes->{$extendedAttributeName} = $interface->extendedAttributes->{$extendedAttributeName};
    206                 }
    207                 push(@{$targetDataNode->constants}, $constant);
    208             }
    209         } else {
    210             die "$idlFile is not a supplemental dependency of $targetIdlFile. There maybe a bug in the supplemental dependency generator (preprocess-idls.pl).\n";
    211         }
    212     }
    213 }
    214 
    215 # Generate desired output for the target IDL file.
    216 my $codeGen = CodeGenerator->new(\@idlDirectories, $generator, $outputDirectory, $outputHeadersDirectory, $preprocessor, $writeDependencies, $verbose, $targetIdlFile, $idlAttributes);
    217 $codeGen->ProcessDocument($targetDocument, $defines);
     217        }
     218    }
     219
     220    # Generate desired output for the target IDL file.
     221    my $codeGen = CodeGenerator->new(\@idlDirectories, $generator, $outputDirectory, $outputHeadersDirectory, $preprocessor, $writeDependencies, $verbose, $targetIdlFile, $idlAttributes);
     222    $codeGen->ProcessDocument($targetDocument, $defines);
     223}
    218224
    219225# Attributes / Operations / Constants of supplemental interfaces can have an [Exposed=XX] attribute which restricts
Note: See TracChangeset for help on using the changeset viewer.