Changeset 181170 in webkit


Ignore:
Timestamp:
Mar 6, 2015 10:47:28 AM (9 years ago)
Author:
mitz@apple.com
Message:

Expose some of the functionality of extract-localizable-strings.pl as a module
​https://bugs.webkit.org/show_bug.cgi?id=142038

Reviewed by Sam Weinig.

  • LocalizableStrings.pm: Copied from Source/WebCore/extract-localizable-strings.pl.

(setTreatWarningsAsErrors): Added this setter for the variable moved here.
(sawError): Added a getter.
(emitError): Added. Emits the error message and sets $sawError.
(unescapeHexSequence): Moved from extract-localizable-strings.pl and renamed to start with
a lowercase letter.
(keyCollisionCount): Added this getter.
(localizedCount): Added this getter.
(HandleUIString): Moved from extract-localizable-strings.pl and made it increment the
localized string count.
(writeStringsFile): Moved code from extract-localizable-strings.pl into this new subroutine.
(verifyStringsFile): Ditto.

  • WebCore.xcodeproj/project.pbxproj: Added LocalizableStrings.pm to the Copy Scripts build

phase.

  • extract-localizable-strings.pl:

(emitWarning): Moved to the module.
(UnescapeHexSequence): Ditto.
(HandleUIString): Ditto.

Location:
trunk/Source/WebCore
Files:
1 added
3 edited

Legend:

Unmodified
Added
Removed

  • trunk/Source/WebCore/ChangeLog

    r181169 r181170  
     12015-03-06  Dan Bernstein  <mitz@apple.com>
     2
     3        Expose some of the functionality of extract-localizable-strings.pl as a module
     4        https://bugs.webkit.org/show_bug.cgi?id=142038
     5
     6        Reviewed by Sam Weinig.
     7
     8        * LocalizableStrings.pm: Copied from Source/WebCore/extract-localizable-strings.pl.
     9        (setTreatWarningsAsErrors): Added this setter for the variable moved here.
     10        (sawError): Added a getter.
     11        (emitError): Added. Emits the error message and sets $sawError.
     12        (unescapeHexSequence): Moved from extract-localizable-strings.pl and renamed to start with
     13        a lowercase letter.
     14        (keyCollisionCount): Added this getter.
     15        (localizedCount): Added this getter.
     16        (HandleUIString): Moved from extract-localizable-strings.pl and made it increment the
     17        localized string count.
     18        (writeStringsFile): Moved code from extract-localizable-strings.pl into this new subroutine.
     19        (verifyStringsFile): Ditto.
     20
     21        * WebCore.xcodeproj/project.pbxproj: Added LocalizableStrings.pm to the Copy Scripts build
     22        phase.
     23
     24        * extract-localizable-strings.pl:
     25        (emitWarning): Moved to the module.
     26        (UnescapeHexSequence): Ditto.
     27        (HandleUIString): Ditto.
     28
    1292015-03-06  Darin Adler  <darin@apple.com>
    230

  • trunk/Source/WebCore/WebCore.xcodeproj/project.pbxproj

    r181152 r181170  
    13701370                37C28A6810F659CC008C7813 /* TypesettingFeatures.h in Headers */ = {isa = PBXBuildFile; fileRef = 37C28A6710F659CC008C7813 /* TypesettingFeatures.h */; settings = {ATTRIBUTES = (Private, ); }; };
    13711371                37C61F0112095C87007A3C67 /* AtomicStringKeyedMRUCache.h in Headers */ = {isa = PBXBuildFile; fileRef = 37C61F0012095C87007A3C67 /* AtomicStringKeyedMRUCache.h */; };
     1372                37D456FD1A9A50D8003330A1 /* LocalizableStrings.pm in Copy Scripts */ = {isa = PBXBuildFile; fileRef = 37D456FB1A9A50B6003330A1 /* LocalizableStrings.pm */; };
    13721373                37DDCD9413844FD50008B793 /* MIMEHeader.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 37DDCD9213844FD50008B793 /* MIMEHeader.cpp */; };
    13731374                37DDCD9513844FD50008B793 /* MIMEHeader.h in Headers */ = {isa = PBXBuildFile; fileRef = 37DDCD9313844FD50008B793 /* MIMEHeader.h */; };
    … …  
    69596960                        dstSubfolderSpec = 1;
    69606961                        files = (
     6962                                37D456FD1A9A50D8003330A1 /* LocalizableStrings.pm in Copy Scripts */,
    69616963                                3717D7E817ECC591003C276D /* extract-localizable-strings.pl in Copy Scripts */,
    69626964                        );
    … …  
    84628464                37C28A6710F659CC008C7813 /* TypesettingFeatures.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = TypesettingFeatures.h; sourceTree = "<group>"; };
    84638465                37C61F0012095C87007A3C67 /* AtomicStringKeyedMRUCache.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = AtomicStringKeyedMRUCache.h; sourceTree = "<group>"; };
     8466                37D456FB1A9A50B6003330A1 /* LocalizableStrings.pm */ = {isa = PBXFileReference; lastKnownFileType = text.script.perl; path = LocalizableStrings.pm; sourceTree = "<group>"; };
    84648467                37DDCD9213844FD50008B793 /* MIMEHeader.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = MIMEHeader.cpp; sourceTree = "<group>"; };
    84658468                37DDCD9313844FD50008B793 /* MIMEHeader.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = MIMEHeader.h; sourceTree = "<group>"; };
    … …  
    1563115634                        children = (
    1563215635                                3717D7E517ECC3A6003C276D /* extract-localizable-strings.pl */,
     15636                                37D456FB1A9A50B6003330A1 /* LocalizableStrings.pm */,
    1563315637                        );
    1563415638                        name = Scripts;

  • trunk/Source/WebCore/extract-localizable-strings.pl

    r179970 r181170  
    4646use File::Compare;
    4747use File::Copy;
     48use FindBin;
    4849use Getopt::Long;
     50use lib $FindBin::Bin;
     51use LocalizableStrings;
    4952no warnings 'deprecated';
    50 
    51 sub UnescapeHexSequence($);
    5253
    5354my %isDebugMacro = ( ASSERT_WITH_MESSAGE => 1, LOG_ERROR => 1, ERROR => 1, NSURL_ERROR => 1, FATAL => 1, LOG => 1, LOG_WARNING => 1, UI_STRING_LOCALIZE_LATER => 1, UI_STRING_LOCALIZE_LATER_KEY => 1, LPCTSTR_UI_STRING_LOCALIZE_LATER => 1, UNLOCALIZED_STRING => 1, UNLOCALIZED_LPCTSTR => 1, dprintf => 1, NSException => 1, NSLog => 1, printf => 1 );
    … …  
    6768GetOptions(%options);
    6869
     70setTreatWarningsAsErrors($treatWarningsAsErrors);
     71
    6972@ARGV >= 2 or die "Usage: extract-localizable-strings [--verify] [--treat-warnings-as-errors] [--exceptions <exceptions file>] <file to update> [--skip directory | directory]...\nDid you mean to run update-webkit-localizable-strings instead?\n";
    7073
    … …  
    8588}
    8689
    87 my $sawError = 0;
    88 
    89 my $localizedCount = 0;
    90 my $keyCollisionCount = 0;
    9190my $notLocalizedCount = 0;
    9291my $NSLocalizeCount = 0;
    … …  
    9493my %exception;
    9594my %usedException;
    96 
    97 sub emitWarning($$$)
    98 {
    99     my ($file, $line, $message) = @_;
    100     my $prefix = $treatWarningsAsErrors ? "" : "warning: ";
    101     print "$file:$line: $prefix$message\n";
    102     $sawError = 1 if $treatWarningsAsErrors;
    103 }
    10495
    10596if (defined $exceptionsFile && open EXCEPTIONS, $exceptionsFile) {
    … …  
    166157            if ($token eq "\"") {
    167158                if ($expected and $expected ne "a quoted string") {
    168                     print "$file:$.: found a quoted string but expected $expected\n";
    169                     $sawError = 1;
     159                    emitError($file, $., "found a quoted string but expected $expected");
    170160                    $expected = "";
    171161                }
    … …  
    178168                    }
    179169                } else {
    180                     print "$file:$.: mismatched quotes\n";
    181                     $sawError = 1;
     170                    emitError($file, $., "mismatched quotes");
    182171                    $_ = "";
    183172                }
    … …  
    228217
    229218            if ($token =~ /^NSLocalized/ && $token !~ /NSLocalizedDescriptionKey/ && $token !~ /NSLocalizedStringFromTableInBundle/ && $token !~ /NSLocalizedFileSizeDescription/ && $token !~ /NSLocalizedDescriptionKey/ && $token !~ /NSLocalizedRecoverySuggestionErrorKey/) {
    230                 print "$file:$.: found a use of an NSLocalized macro ($token); not supported\n";
     219                emitError($file, $., "found a use of an NSLocalized macro ($token); not supported");
    231220                $nestingLevel = 0 if !defined $nestingLevel;
    232                 $sawError = 1;
    233221                $NSLocalizeCount++;
    234222            } elsif ($token eq "/*") {
    … …  
    241229            } elsif ($token eq "'") {
    242230                if (!s-([^\\]|\\.)'--) { #' <-- that single quote makes the Project Builder editor less confused
    243                     print "$file:$.: mismatched single quote\n";
    244                     $sawError = 1;
     231                    emitError($file, $., "mismatched single quote");
    245232                    $_ = "";
    246233                }
    247234            } else {
    248235                if ($expected and $expected ne $token) {
    249                     print "$file:$.: found $token but expected $expected\n";
    250                     $sawError = 1;
     236                    emitError($file, $., "found $token but expected $expected");
    251237                    $expected = "";
    252238                }
    … …  
    270256                        $macro = "";
    271257                        $expected = "";
    272                         $localizedCount++;
    273258                    }
    274259                } elsif ($isDebugMacro{$token}) {
    … …  
    283268   
    284269    if ($expected) {
    285         print "$file: reached end of file but expected $expected\n";
    286         $sawError = 1;
     270        emitError($file, 0, "reached end of file but expected $expected");
    287271    }
    288272   
    … …  
    290274}
    291275
    292 # Unescapes C language hexadecimal escape sequences.
    293 sub UnescapeHexSequence($)
    294 {
    295     my ($originalStr) = @_;
    296 
    297     my $escapedStr = $originalStr;
    298     my $unescapedStr = "";
    299 
    300     for (;;) {
    301         if ($escapedStr =~ s-^\\x([[:xdigit:]]+)--) {
    302             if (256 <= hex($1)) {
    303                 print "Hexadecimal escape sequence out of range: \\x$1\n";
    304                 return undef;
    305             }
    306             $unescapedStr .= pack("H*", $1);
    307         } elsif ($escapedStr =~ s-^(.)--) {
    308             $unescapedStr .= $1;
    309         } else {
    310             return $unescapedStr;
    311         }
    312     }
    313 }
    314 
    315 my %stringByKey;
    316 my %commentByKey;
    317 my %fileByKey;
    318 my %lineByKey;
    319 
    320 sub HandleUIString
    321 {
    322     my ($string, $key, $comment, $file, $line) = @_;
    323 
    324     my $bad = 0;
    325     $string = UnescapeHexSequence($string);
    326     if (!defined($string)) {
    327         print "$file:$line: string has an illegal hexadecimal escape sequence\n";
    328         $bad = 1;
    329     }
    330     $key = UnescapeHexSequence($key);
    331     if (!defined($key)) {
    332         print "$file:$line: key has an illegal hexadecimal escape sequence\n";
    333         $bad = 1;
    334     }
    335     $comment = UnescapeHexSequence($comment);
    336     if (!defined($comment)) {
    337         print "$file:$line: comment has an illegal hexadecimal escape sequence\n";
    338         $bad = 1;
    339     }
    340     if (grep { $_ == 0xFFFD } unpack "U*", $string) {
    341         print "$file:$line: string for translation has illegal UTF-8 -- most likely a problem with the Text Encoding of the source file\n";
    342         $bad = 1;
    343     }
    344     if ($string ne $key && grep { $_ == 0xFFFD } unpack "U*", $key) {
    345         print "$file:$line: key has illegal UTF-8 -- most likely a problem with the Text Encoding of the source file\n";
    346         $bad = 1;
    347     }
    348     if (grep { $_ == 0xFFFD } unpack "U*", $comment) {
    349         print "$file:$line: comment for translation has illegal UTF-8 -- most likely a problem with the Text Encoding of the source file\n";
    350         $bad = 1;
    351     }
    352     if ($bad) {
    353         $sawError = 1;
    354         return;
    355     }
    356    
    357     if ($stringByKey{$key} && $stringByKey{$key} ne $string) {
    358         emitWarning($file, $line, "encountered the same key, \"$key\", twice, with different strings");
    359         emitWarning($fileByKey{$key}, $lineByKey{$key}, "previous occurrence");
    360         $keyCollisionCount++;
    361         return;
    362     }
    363     if ($commentByKey{$key} && $commentByKey{$key} ne $comment) {
    364         emitWarning($file, $line, "encountered the same key, \"$key\", twice, with different comments");
    365         emitWarning($fileByKey{$key}, $lineByKey{$key}, "previous occurrence");
    366         $keyCollisionCount++;
    367         return;
    368     }
    369 
    370     $fileByKey{$key} = $file;
    371     $lineByKey{$key} = $line;
    372     $stringByKey{$key} = $string;
    373     $commentByKey{$key} = $comment;
    374 }
    375 
    376 print "\n" if $sawError || $notLocalizedCount || $NSLocalizeCount;
     276print "\n" if sawError() || $notLocalizedCount || $NSLocalizeCount;
    377277
    378278my @unusedExceptions = sort grep { !$usedException{$_} } keys %exception;
    … …  
    384284}
    385285
    386 print "$localizedCount localizable strings\n" if $localizedCount;
    387 print "$keyCollisionCount key collisions\n" if $keyCollisionCount;
     286print localizedCount() . " localizable strings\n" if localizedCount();
     287print keyCollisionCount() . " key collisions\n" if keyCollisionCount();
    388288print "$notLocalizedCount strings not marked for localization\n" if $notLocalizedCount;
    389289print "$NSLocalizeCount uses of NSLocalize\n" if $NSLocalizeCount;
    390290print scalar(@unusedExceptions), " unused exceptions\n" if @unusedExceptions;
    391291
    392 if ($sawError) {
     292if (sawError()) {
    393293    print "\nErrors encountered. Exiting without writing to $fileToUpdate.\n";
    394294    exit 1;
    395295}
    396296
    397 my $localizedStrings = "";
    398 
    399 for my $key (sort keys %commentByKey) {
    400     $localizedStrings .= "/* $commentByKey{$key} */\n\"$key\" = \"$stringByKey{$key}\";\n\n";
    401 }
    402 
    403297if (-e "$fileToUpdate") {
    404298    if (!$verify) {
    405         # Write out the strings file as UTF-8
    406299        my $temporaryFile = "$fileToUpdate.updated";
    407         open STRINGS, ">", $temporaryFile or die;
    408         print STRINGS $localizedStrings;
    409         close STRINGS;
     300        writeStringsFile($temporaryFile);
    410301
    411302        # Avoid updating the target file's modification time if the contents have not changed.
    … …  
    416307        }
    417308    } else {
    418         open STRINGS, $fileToUpdate or die;
    419 
    420         my $lastComment;
    421         my $line;
    422 
    423         while (<STRINGS>) {
    424             chomp;
    425 
    426             next if (/^\s*$/);
    427 
    428             if (/^\/\* (.*) \*\/$/) {
    429                 $lastComment = $1;
    430             } elsif (/^"((?:[^\\]|\\[^"])*)"\s*=\s*"((?:[^\\]|\\[^"])*)";$/) #
    431             {
    432                 my $string = delete $stringByKey{$1};
    433                 if (!defined $string) {
    434                     print "$fileToUpdate:$.: unused key \"$1\"\n";
    435                     $sawError = 1;
    436                 } else {
    437                     if (!($string eq $2)) {
    438                         print "$fileToUpdate:$.: unexpected value \"$2\" for key \"$1\"\n";
    439                         print "$fileByKey{$1}:$lineByKey{$1}: expected value \"$string\" defined here\n";
    440                         $sawError = 1;
    441                     }
    442                     if (!($lastComment eq $commentByKey{$1})) {
    443                         print "$fileToUpdate:$.: unexpected comment /* $lastComment */ for key \"$1\"\n";
    444                         print "$fileByKey{$1}:$lineByKey{$1}: expected comment /* $commentByKey{$1} */ defined here\n";
    445                         $sawError = 1;
    446                     }
    447                 }
    448             } else {
    449                 print "$fileToUpdate:$.: line with unexpected format: $_\n";
    450                 $sawError = 1;
    451             }
    452         }
    453 
    454         for my $missing (keys %stringByKey) {
    455             print "$fileByKey{$missing}:$lineByKey{$missing}: missing key \"$missing\"\n";
    456             $sawError = 1;
    457         }
    458 
    459         if ($sawError) {
    460             print "\n$fileToUpdate:0: file is not up to date.\n";
    461             exit 1;
    462         }
     309        verifyStringsFile($fileToUpdate);
    463310    }
    464311} else {
Note: See TracChangeset for help on using the changeset viewer.