Changeset 142497 in webkit
- Timestamp:
- Feb 11, 2013 12:41:49 PM (11 years ago)
- Location:
- trunk/Source/WebCore
- Files:
-
- 1 added
- 14 edited
Legend:
- Unmodified
- Added
- Removed
-
trunk/Source/WebCore/ChangeLog
r142492 r142497 1 2013-02-11 Eric Seidel <eric@webkit.org> 2 3 Make WebVTTTokenizer stop inheriting from MarkupTokenizerBase 4 https://bugs.webkit.org/show_bug.cgi?id=109411 5 6 Reviewed by Adam Barth. 7 8 Moved InputStreamPreprocessor into its own header file so it can be 9 used by both WebVTTTokenizer and HTMLTokenizer. 10 11 Also split out kEndOfFileMarker from InputStreamPreprocessor<T> so that 12 it can be used w/o a specific instantiation of the template class. 13 This also made it possible to fix three old fixmes about wanting to share 14 that constant. 15 16 Again, separating WebVTT code from Markup* base classes made it simpler 17 at the cost of a little copy/paste code. WebVTT tokenization is remarkably 18 simple compared to HTML. 19 20 This will make it immediately possible to pull MarkupTokenizerBase up into 21 HTMLTokenizer and further simplify the code. 22 23 * GNUmakefile.list.am: 24 * Target.pri: 25 * WebCore.gypi: 26 * WebCore.vcproj/WebCore.vcproj: 27 * WebCore.vcxproj/WebCore.vcxproj: 28 * WebCore.vcxproj/WebCore.vcxproj.filters: 29 * WebCore.xcodeproj/project.pbxproj: 30 * html/parser/BackgroundHTMLParser.cpp: 31 (WebCore::BackgroundHTMLParser::markEndOfFile): 32 * html/parser/HTMLInputStream.h: 33 (WebCore::HTMLInputStream::markEndOfFile): 34 * html/parser/HTMLTokenizer.cpp: 35 (WebCore::HTMLTokenizer::nextToken): 36 * html/parser/InputStreamPreprocessor.h: Added. 37 (WebCore): 38 (InputStreamPreprocessor): 39 (WebCore::InputStreamPreprocessor::InputStreamPreprocessor): 40 (WebCore::InputStreamPreprocessor::nextInputCharacter): 41 (WebCore::InputStreamPreprocessor::peek): 42 (WebCore::InputStreamPreprocessor::advance): 43 (WebCore::InputStreamPreprocessor::skipNextNewLine): 44 (WebCore::InputStreamPreprocessor::reset): 45 (WebCore::InputStreamPreprocessor::shouldTreatNullAsEndOfFileMarker): 46 * html/track/WebVTTTokenizer.cpp: 47 (WebCore::WebVTTTokenizer::WebVTTTokenizer): 48 (WebCore::WebVTTTokenizer::nextToken): 49 * html/track/WebVTTTokenizer.h: 50 (WebVTTTokenizer): 51 (WebCore::WebVTTTokenizer::haveBufferedCharacterToken): 52 (WebCore::WebVTTTokenizer::bufferCharacter): 53 (WebCore::WebVTTTokenizer::emitAndResumeIn): 54 (WebCore::WebVTTTokenizer::emitEndOfFile): 55 (WebCore::WebVTTTokenizer::shouldSkipNullCharacters): 56 * xml/parser/MarkupTokenizerBase.h: 57 (MarkupTokenizerBase): 58 (WebCore::MarkupTokenizerBase::bufferCharacter): 59 1 60 2013-02-11 Adam Barth <abarth@webkit.org> 2 61 -
trunk/Source/WebCore/GNUmakefile.list.am
r142460 r142497 3619 3619 Source/WebCore/html/parser/HTMLViewSourceParser.cpp \ 3620 3620 Source/WebCore/html/parser/HTMLViewSourceParser.h \ 3621 Source/WebCore/html/parser/InputStreamPreprocessor.h \ 3621 3622 Source/WebCore/html/parser/NestingLevelIncrementer.h \ 3622 3623 Source/WebCore/html/parser/TextDocumentParser.cpp \ -
trunk/Source/WebCore/Target.pri
r142460 r142497 1936 1936 html/parser/HTMLTreeBuilder.h \ 1937 1937 html/parser/HTMLViewSourceParser.h \ 1938 html/parser/InputStreamPreprocessor.h \ 1938 1939 html/parser/XSSAuditor.h \ 1939 1940 html/parser/XSSAuditorDelegate.h \ -
trunk/Source/WebCore/WebCore.gypi
r142474 r142497 3428 3428 'html/parser/HTMLViewSourceParser.cpp', 3429 3429 'html/parser/HTMLViewSourceParser.h', 3430 'html/parser/InputStreamPreprocessor.h', 3430 3431 'html/parser/NestingLevelIncrementer.h', 3431 3432 'html/parser/TextDocumentParser.cpp', -
trunk/Source/WebCore/WebCore.vcproj/WebCore.vcproj
r142474 r142497 65011 65011 </File> 65012 65012 <File 65013 RelativePath="..\html\parser\InputStreamPreprocessor.h" 65014 > 65015 </File> 65016 <File 65013 65017 RelativePath="..\html\parser\NestingLevelIncrementer.h" 65014 65018 > -
trunk/Source/WebCore/WebCore.vcxproj/WebCore.vcxproj
r142460 r142497 7954 7954 <ClInclude Include="..\html\parser\HTMLTreeBuilder.h" /> 7955 7955 <ClInclude Include="..\html\parser\HTMLViewSourceParser.h" /> 7956 <ClInclude Include="..\html\parser\InputStreamPreprocessor.h" /> 7956 7957 <ClInclude Include="..\html\parser\NestingLevelIncrementer.h" /> 7957 7958 <ClInclude Include="..\html\parser\TextDocumentParser.h" /> -
trunk/Source/WebCore/WebCore.vcxproj/WebCore.vcxproj.filters
r142460 r142497 12613 12613 <Filter>html\parser</Filter> 12614 12614 </ClInclude> 12615 <ClInclude Include="..\html\parser\InputStreamPreprocessor.h"> 12616 <Filter>html\parser</Filter> 12617 </ClInclude> 12615 12618 <ClInclude Include="..\html\parser\NestingLevelIncrementer.h"> 12616 12619 <Filter>html\parser</Filter> -
trunk/Source/WebCore/WebCore.xcodeproj/project.pbxproj
r142460 r142497 1562 1562 510D4A37103165EE0049EA54 /* SocketStreamHandleBase.h in Headers */ = {isa = PBXBuildFile; fileRef = 510D4A31103165EE0049EA54 /* SocketStreamHandleBase.h */; }; 1563 1563 510D4A38103165EE0049EA54 /* SocketStreamHandleClient.h in Headers */ = {isa = PBXBuildFile; fileRef = 510D4A32103165EE0049EA54 /* SocketStreamHandleClient.h */; }; 1564 5112935F3D54B4B52FAF973F /* InspectorHeapProfilerAgent.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 511293613D6DB4B52FAF973F /* InspectorHeapProfilerAgent.cpp */; }; 1565 511293603D60B4B52FAF973F /* InspectorHeapProfilerAgent.h in Headers */ = {isa = PBXBuildFile; fileRef = 511293623D85B4B52FAF973F /* InspectorHeapProfilerAgent.h */; }; 1564 1566 511F23170DC160DA004F0032 /* StorageTask.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 511F23130DC160DA004F0032 /* StorageTask.cpp */; }; 1565 1567 511F23180DC160DA004F0032 /* StorageTask.h in Headers */ = {isa = PBXBuildFile; fileRef = 511F23140DC160DA004F0032 /* StorageTask.h */; }; … … 3816 3818 9F0D6B2E121BFEBA006C0288 /* InspectorProfilerAgent.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 9F0D6B2C121BFEBA006C0288 /* InspectorProfilerAgent.cpp */; }; 3817 3819 9F0D6B2F121BFEBA006C0288 /* InspectorProfilerAgent.h in Headers */ = {isa = PBXBuildFile; fileRef = 9F0D6B2D121BFEBA006C0288 /* InspectorProfilerAgent.h */; }; 3818 5112935F3D54B4B52FAF973F /* InspectorHeapProfilerAgent.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 511293613D6DB4B52FAF973F /* InspectorHeapProfilerAgent.cpp */; };3819 511293603D60B4B52FAF973F /* InspectorHeapProfilerAgent.h in Headers */ = {isa = PBXBuildFile; fileRef = 511293623D85B4B52FAF973F /* InspectorHeapProfilerAgent.h */; };3820 3820 9F3B947E12241758005304E7 /* ScriptHeapSnapshot.h in Headers */ = {isa = PBXBuildFile; fileRef = 9F3B947D12241758005304E7 /* ScriptHeapSnapshot.h */; }; 3821 3821 9F6FC1961122E82A00E80196 /* ScriptDebugServer.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 9F6FC1941122E82A00E80196 /* ScriptDebugServer.cpp */; }; … … 8977 8977 510D4A31103165EE0049EA54 /* SocketStreamHandleBase.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = SocketStreamHandleBase.h; sourceTree = "<group>"; }; 8978 8978 510D4A32103165EE0049EA54 /* SocketStreamHandleClient.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = SocketStreamHandleClient.h; sourceTree = "<group>"; }; 8979 511293613D6DB4B52FAF973F /* InspectorHeapProfilerAgent.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = InspectorHeapProfilerAgent.cpp; sourceTree = "<group>"; }; 8980 511293623D85B4B52FAF973F /* InspectorHeapProfilerAgent.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = InspectorHeapProfilerAgent.h; sourceTree = "<group>"; }; 8979 8981 511F23130DC160DA004F0032 /* StorageTask.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = StorageTask.cpp; sourceTree = "<group>"; }; 8980 8982 511F23140DC160DA004F0032 /* StorageTask.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = StorageTask.h; sourceTree = "<group>"; }; … … 11283 11285 9F0D6B2C121BFEBA006C0288 /* InspectorProfilerAgent.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = InspectorProfilerAgent.cpp; sourceTree = "<group>"; }; 11284 11286 9F0D6B2D121BFEBA006C0288 /* InspectorProfilerAgent.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = InspectorProfilerAgent.h; sourceTree = "<group>"; }; 11285 511293613D6DB4B52FAF973F /* InspectorHeapProfilerAgent.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = InspectorHeapProfilerAgent.cpp; sourceTree = "<group>"; };11286 511293623D85B4B52FAF973F /* InspectorHeapProfilerAgent.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = InspectorHeapProfilerAgent.h; sourceTree = "<group>"; };11287 11287 9F3B947D12241758005304E7 /* ScriptHeapSnapshot.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = ScriptHeapSnapshot.h; sourceTree = "<group>"; }; 11288 11288 9F6FC1941122E82A00E80196 /* ScriptDebugServer.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = ScriptDebugServer.cpp; sourceTree = "<group>"; }; … … 11718 11718 A8748D6612CC3763001FBA41 /* ImageOrientation.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = ImageOrientation.h; sourceTree = "<group>"; }; 11719 11719 A8748D7412CC3F89001FBA41 /* ImageOrientation.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = ImageOrientation.cpp; sourceTree = "<group>"; }; 11720 A8805AF116C96AEA000E9D98 /* InputStreamPreprocessor.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = InputStreamPreprocessor.h; path = parser/InputStreamPreprocessor.h; sourceTree = "<group>"; }; 11720 11721 A882DA1F1593846A000115ED /* CSSToStyleMap.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = CSSToStyleMap.cpp; sourceTree = "<group>"; }; 11721 11722 A882DA221593848D000115ED /* CSSToStyleMap.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = CSSToStyleMap.h; sourceTree = "<group>"; }; … … 14865 14866 isa = PBXGroup; 14866 14867 children = ( 14867 CDA98DCD1601515C00FEA3B1 /* EncryptedMedia */,14868 14868 A83B79100CCB001B000B0825 /* Core */, 14869 14869 A83B790E0CCAFF97000B0825 /* CSS */, 14870 CDA98DCD1601515C00FEA3B1 /* EncryptedMedia */, 14870 14871 A83B79120CCB003F000B0825 /* Events */, 14871 14872 89878576122CA1DA003AABDA /* FileAPI */, … … 15570 15571 A7BE7EDD14C9175A0014489D /* MallocStatistics.h */, 15571 15572 A7BE7EDE14C9175A0014489D /* MallocStatistics.idl */, 15573 CDC26B3C160A62B00026757B /* MockCDM.cpp */, 15574 CDC26B3D160A62B00026757B /* MockCDM.h */, 15572 15575 EB081CD81696084400553730 /* TypeConversions.h */, 15573 15576 EB081CD91696084400553730 /* TypeConversions.idl */, 15574 CDC26B3C160A62B00026757B /* MockCDM.cpp */,15575 CDC26B3D160A62B00026757B /* MockCDM.h */,15576 15577 ); 15577 15578 path = testing; … … 18749 18750 977B3860122883E900B81FF8 /* HTMLViewSourceParser.cpp */, 18750 18751 977B3861122883E900B81FF8 /* HTMLViewSourceParser.h */, 18752 A8805AF116C96AEA000E9D98 /* InputStreamPreprocessor.h */, 18751 18753 8A309C9E123950BE00CB9204 /* NestingLevelIncrementer.h */, 18752 18754 97BC84811236FD93000C6161 /* TextDocumentParser.cpp */, … … 21539 21541 C5BAC16F14E30E4700008837 /* PlatformPasteboard.h */, 21540 21542 BCEC01D60C274EB4009F4EC9 /* PlatformScreen.h */, 21543 2527CC9516BF95DD009DDAC0 /* PlatformSpeechSynthesisUtterance.cpp */, 21544 2527CC9116BF8BA1009DDAC0 /* PlatformSpeechSynthesisUtterance.h */, 21545 2527CC9316BF92EC009DDAC0 /* PlatformSpeechSynthesisVoice.cpp */, 21546 2527CC9216BF90B4009DDAC0 /* PlatformSpeechSynthesisVoice.h */, 21541 21547 297BE3D916C043D8003316BD /* PlatformSpeechSynthesizer.cpp */, 21542 21548 29E4D8DF16B0940F00C84704 /* PlatformSpeechSynthesizer.h */, 21543 2527CC9316BF92EC009DDAC0 /* PlatformSpeechSynthesisVoice.cpp */,21544 2527CC9216BF90B4009DDAC0 /* PlatformSpeechSynthesisVoice.h */,21545 2527CC9516BF95DD009DDAC0 /* PlatformSpeechSynthesisUtterance.cpp */,21546 2527CC9116BF8BA1009DDAC0 /* PlatformSpeechSynthesisUtterance.h */,21547 21549 1AD8F81A11CAB9E900E93E54 /* PlatformStrategies.cpp */, 21548 21550 1AD8F81911CAB9E900E93E54 /* PlatformStrategies.h */, … … 21634 21636 isa = PBXGroup; 21635 21637 children = ( 21638 CDA98E091603CD5900FEA3B1 /* CDM.cpp */, 21639 CDA98E0A1603CD5900FEA3B1 /* CDM.h */, 21636 21640 CDA98DD516025BED00FEA3B1 /* MediaKeyMessageEvent.cpp */, 21637 21641 CDA98DD616025BED00FEA3B1 /* MediaKeyMessageEvent.h */, … … 21646 21650 CDA98DC816014F2C00FEA3B1 /* MediaKeySession.h */, 21647 21651 CDA98DC916014F4000FEA3B1 /* MediaKeySession.idl */, 21648 CDA98E091603CD5900FEA3B1 /* CDM.cpp */,21649 CDA98E0A1603CD5900FEA3B1 /* CDM.h */,21650 21652 CDA98E0C1603CF3C00FEA3B1 /* Modules/encryptedmedia/CDMPrivate.h */, 21651 21653 ); … … 23088 23090 A740B59514C935AB00A77FA4 /* JSMallocStatistics.h in Headers */, 23089 23091 EBF5121D1696496C0056BD25 /* JSTypeConversions.h in Headers */, 23092 CDC26B41160A8CCE0026757B /* MockCDM.h in Headers */, 23090 23093 41815C1F138319830057AAA4 /* WebCoreTestSupport.h in Headers */, 23091 CDC26B41160A8CCE0026757B /* MockCDM.h in Headers */,23092 23094 ); 23093 23095 runOnlyForDeploymentPostprocessing = 0; … … 26823 26825 A740B59714C935AF00A77FA4 /* JSMallocStatistics.cpp in Sources */, 26824 26826 EBF5121C1696496C0056BD25 /* JSTypeConversions.cpp in Sources */, 26827 CDC26B40160A8CC60026757B /* MockCDM.cpp in Sources */, 26825 26828 41815C1E138319830057AAA4 /* WebCoreTestSupport.cpp in Sources */, 26826 CDC26B40160A8CC60026757B /* MockCDM.cpp in Sources */,26827 26829 ); 26828 26830 runOnlyForDeploymentPostprocessing = 0; … … 26981 26983 079D086C162F21F900DB8658 /* CaptionUserPreferencesMac.mm in Sources */, 26982 26984 6550B69D099DF0270090D781 /* CDATASection.cpp in Sources */, 26985 CDA98E0B1603CD6000FEA3B1 /* CDM.cpp in Sources */, 26983 26986 97BC69DC1505F076001B74AC /* ChangeVersionWrapper.cpp in Sources */, 26984 26987 FD315FFE12B0267600C1A359 /* ChannelMergerNode.cpp in Sources */, … … 28292 28295 FD23A12513F5FA5900F67001 /* JSMediaElementAudioSourceNode.cpp in Sources */, 28293 28296 E44614180CD6826900FADA75 /* JSMediaError.cpp in Sources */, 28297 CDA98DA31601464100FEA3B1 /* JSMediaKeyError.cpp in Sources */, 28294 28298 CDB859FA160D494900E5B07F /* JSMediaKeyEvent.cpp in Sources */, 28299 CDA98DDF16026A3700FEA3B1 /* JSMediaKeyMessageEvent.cpp in Sources */, 28300 CDA98DE016026A3700FEA3B1 /* JSMediaKeyNeededEvent.cpp in Sources */, 28301 CDA98E0416039E1A00FEA3B1 /* JSMediaKeys.cpp in Sources */, 28302 CDA98E0616039E1F00FEA3B1 /* JSMediaKeySession.cpp in Sources */, 28295 28303 BC3C39B60C0D3D8D005F4D7A /* JSMediaList.cpp in Sources */, 28296 28304 1464E06C135EC10600FDB00A /* JSMediaListCustom.cpp in Sources */, … … 28711 28719 07A6D1EB1491137700051D0C /* MediaFragmentURIParser.cpp in Sources */, 28712 28720 CDB859F7160D48A400E5B07F /* MediaKeyEvent.cpp in Sources */, 28721 CDA98DD816025BEF00FEA3B1 /* MediaKeyMessageEvent.cpp in Sources */, 28722 CD1B4A65160786AE00282DF9 /* MediaKeyNeededEvent.cpp in Sources */, 28723 CDA98E0D1603FE4A00FEA3B1 /* MediaKeys.cpp in Sources */, 28724 CDA98E0E1603FE5800FEA3B1 /* MediaKeySession.cpp in Sources */, 28713 28725 A8EA80090A19516E00A8EF5F /* MediaList.cpp in Sources */, 28714 28726 E44613E30CD6819F00FADA75 /* MediaPlayer.cpp in Sources */, … … 28856 28868 C598902E14E9B0F800E8D18B /* PlatformPasteboardMac.mm in Sources */, 28857 28869 BC94D1080C274F88006BC617 /* PlatformScreenMac.mm in Sources */, 28870 2934940B16C02C0A00901318 /* PlatformSpeechSynthesisUtterance.cpp in Sources */, 28858 28871 2934940A16C02C0700901318 /* PlatformSpeechSynthesisVoice.cpp in Sources */, 28859 29 34940B16C02C0A00901318 /* PlatformSpeechSynthesisUtterance.cpp in Sources */,28872 297BE3DA16C043D8003316BD /* PlatformSpeechSynthesizer.cpp in Sources */, 28860 28873 297BE3D816C03CCE003316BD /* PlatformSpeechSynthesizerMac.mm in Sources */, 28861 297BE3DA16C043D8003316BD /* PlatformSpeechSynthesizer.cpp in Sources */,28862 28874 1AD8F81C11CAB9E900E93E54 /* PlatformStrategies.cpp in Sources */, 28863 28875 A9C6E4F30D745E48006442E9 /* PluginData.cpp in Sources */, … … 29672 29684 977E2E0E12F0FC9C00C13380 /* XSSAuditorDelegate.cpp in Sources */, 29673 29685 FD537352137B651800008DCE /* ZeroPole.cpp in Sources */, 29674 CDA98DA31601464100FEA3B1 /* JSMediaKeyError.cpp in Sources */,29675 CDA98DD816025BEF00FEA3B1 /* MediaKeyMessageEvent.cpp in Sources */,29676 CDA98DDF16026A3700FEA3B1 /* JSMediaKeyMessageEvent.cpp in Sources */,29677 CDA98DE016026A3700FEA3B1 /* JSMediaKeyNeededEvent.cpp in Sources */,29678 CDA98E0416039E1A00FEA3B1 /* JSMediaKeys.cpp in Sources */,29679 CDA98E0616039E1F00FEA3B1 /* JSMediaKeySession.cpp in Sources */,29680 CDA98E0B1603CD6000FEA3B1 /* CDM.cpp in Sources */,29681 CDA98E0D1603FE4A00FEA3B1 /* MediaKeys.cpp in Sources */,29682 CDA98E0E1603FE5800FEA3B1 /* MediaKeySession.cpp in Sources */,29683 CD1B4A65160786AE00282DF9 /* MediaKeyNeededEvent.cpp in Sources */,29684 29686 ); 29685 29687 runOnlyForDeploymentPostprocessing = 0; -
trunk/Source/WebCore/html/parser/BackgroundHTMLParser.cpp
r142363 r142497 107 107 void BackgroundHTMLParser::markEndOfFile() 108 108 { 109 // FIXME: This should use InputStreamPreprocessor::endOfFileMarker110 // once InputStreamPreprocessor is split off into its own header.111 const LChar endOfFileMarker = 0;112 113 109 ASSERT(!m_input.current().isClosed()); 114 m_input.append(String(& endOfFileMarker, 1));110 m_input.append(String(&kEndOfFileMarker, 1)); 115 111 m_input.close(); 116 112 } -
trunk/Source/WebCore/html/parser/HTMLInputStream.h
r124679 r142497 27 27 #define HTMLInputStream_h 28 28 29 #include "InputStreamPreprocessor.h" 29 30 #include "SegmentedString.h" 30 31 … … 73 74 void markEndOfFile() 74 75 { 75 // FIXME: This should use InputStreamPreprocessor::endOfFileMarker 76 // once InputStreamPreprocessor is split off into its own header. 77 static const LChar endOfFileMarker = 0; 78 m_last->append(SegmentedString(String(&endOfFileMarker, 1))); 76 m_last->append(SegmentedString(String(&kEndOfFileMarker, 1))); 79 77 m_last->close(); 80 78 } -
trunk/Source/WebCore/html/parser/HTMLTokenizer.cpp
r142480 r142497 254 254 } 255 255 HTML_ADVANCE_TO(TagOpenState); 256 } else if (cc == InputStreamPreprocessor::endOfFileMarker)256 } else if (cc == kEndOfFileMarker) 257 257 return emitEndOfFile(source); 258 258 else { … … 275 275 else if (cc == '<') 276 276 HTML_ADVANCE_TO(RCDATALessThanSignState); 277 else if (cc == InputStreamPreprocessor::endOfFileMarker)277 else if (cc == kEndOfFileMarker) 278 278 return emitEndOfFile(source); 279 279 else { … … 294 294 if (cc == '<') 295 295 HTML_ADVANCE_TO(RAWTEXTLessThanSignState); 296 else if (cc == InputStreamPreprocessor::endOfFileMarker)296 else if (cc == kEndOfFileMarker) 297 297 return emitEndOfFile(source); 298 298 else { … … 306 306 if (cc == '<') 307 307 HTML_ADVANCE_TO(ScriptDataLessThanSignState); 308 else if (cc == InputStreamPreprocessor::endOfFileMarker)308 else if (cc == kEndOfFileMarker) 309 309 return emitEndOfFile(source); 310 310 else { … … 316 316 317 317 HTML_BEGIN_STATE(PLAINTEXTState) { 318 if (cc == InputStreamPreprocessor::endOfFileMarker)318 if (cc == kEndOfFileMarker) 319 319 return emitEndOfFile(source); 320 else 321 bufferCharacter(cc); 320 bufferCharacter(cc); 322 321 HTML_ADVANCE_TO(PLAINTEXTState); 323 322 } … … 361 360 parseError(); 362 361 HTML_ADVANCE_TO(DataState); 363 } else if (cc == InputStreamPreprocessor::endOfFileMarker) {362 } else if (cc == kEndOfFileMarker) { 364 363 parseError(); 365 364 bufferCharacter('<'); … … 385 384 m_token->appendToName(toLowerCase(cc)); 386 385 HTML_ADVANCE_TO(TagNameState); 387 } else if (cc == InputStreamPreprocessor::endOfFileMarker) {386 } else if (cc == kEndOfFileMarker) { 388 387 parseError(); 389 388 HTML_RECONSUME_IN(DataState); … … 618 617 } else if (cc == '<') 619 618 HTML_ADVANCE_TO(ScriptDataEscapedLessThanSignState); 620 else if (cc == InputStreamPreprocessor::endOfFileMarker) {619 else if (cc == kEndOfFileMarker) { 621 620 parseError(); 622 621 HTML_RECONSUME_IN(DataState); … … 634 633 } else if (cc == '<') 635 634 HTML_ADVANCE_TO(ScriptDataEscapedLessThanSignState); 636 else if (cc == InputStreamPreprocessor::endOfFileMarker) {635 else if (cc == kEndOfFileMarker) { 637 636 parseError(); 638 637 HTML_RECONSUME_IN(DataState); … … 653 652 bufferCharacter(cc); 654 653 HTML_ADVANCE_TO(ScriptDataState); 655 } else if (cc == InputStreamPreprocessor::endOfFileMarker) {654 } else if (cc == kEndOfFileMarker) { 656 655 parseError(); 657 656 HTML_RECONSUME_IN(DataState); … … 767 766 bufferCharacter(cc); 768 767 HTML_ADVANCE_TO(ScriptDataDoubleEscapedLessThanSignState); 769 } else if (cc == InputStreamPreprocessor::endOfFileMarker) {768 } else if (cc == kEndOfFileMarker) { 770 769 parseError(); 771 770 HTML_RECONSUME_IN(DataState); … … 784 783 bufferCharacter(cc); 785 784 HTML_ADVANCE_TO(ScriptDataDoubleEscapedLessThanSignState); 786 } else if (cc == InputStreamPreprocessor::endOfFileMarker) {785 } else if (cc == kEndOfFileMarker) { 787 786 parseError(); 788 787 HTML_RECONSUME_IN(DataState); … … 804 803 bufferCharacter(cc); 805 804 HTML_ADVANCE_TO(ScriptDataState); 806 } else if (cc == InputStreamPreprocessor::endOfFileMarker) {805 } else if (cc == kEndOfFileMarker) { 807 806 parseError(); 808 807 HTML_RECONSUME_IN(DataState); … … 858 857 m_token->appendToAttributeName(toLowerCase(cc)); 859 858 HTML_ADVANCE_TO(AttributeNameState); 860 } else if (cc == InputStreamPreprocessor::endOfFileMarker) {859 } else if (cc == kEndOfFileMarker) { 861 860 parseError(); 862 861 HTML_RECONSUME_IN(DataState); … … 891 890 m_token->appendToAttributeName(toLowerCase(cc)); 892 891 HTML_ADVANCE_TO(AttributeNameState); 893 } else if (cc == InputStreamPreprocessor::endOfFileMarker) {892 } else if (cc == kEndOfFileMarker) { 894 893 parseError(); 895 894 m_token->endAttributeName(source.numberOfCharactersConsumed()); … … 920 919 m_token->appendToAttributeName(toLowerCase(cc)); 921 920 HTML_ADVANCE_TO(AttributeNameState); 922 } else if (cc == InputStreamPreprocessor::endOfFileMarker) {921 } else if (cc == kEndOfFileMarker) { 923 922 parseError(); 924 923 HTML_RECONSUME_IN(DataState); … … 949 948 parseError(); 950 949 return emitAndResumeIn(source, HTMLTokenizerState::DataState); 951 } else if (cc == InputStreamPreprocessor::endOfFileMarker) {950 } else if (cc == kEndOfFileMarker) { 952 951 parseError(); 953 952 HTML_RECONSUME_IN(DataState); … … 969 968 m_additionalAllowedCharacter = '"'; 970 969 HTML_ADVANCE_TO(CharacterReferenceInAttributeValueState); 971 } else if (cc == InputStreamPreprocessor::endOfFileMarker) {970 } else if (cc == kEndOfFileMarker) { 972 971 parseError(); 973 972 m_token->endAttributeValue(source.numberOfCharactersConsumed()); … … 987 986 m_additionalAllowedCharacter = '\''; 988 987 HTML_ADVANCE_TO(CharacterReferenceInAttributeValueState); 989 } else if (cc == InputStreamPreprocessor::endOfFileMarker) {988 } else if (cc == kEndOfFileMarker) { 990 989 parseError(); 991 990 m_token->endAttributeValue(source.numberOfCharactersConsumed()); … … 1008 1007 m_token->endAttributeValue(source.numberOfCharactersConsumed()); 1009 1008 return emitAndResumeIn(source, HTMLTokenizerState::DataState); 1010 } else if (cc == InputStreamPreprocessor::endOfFileMarker) {1009 } else if (cc == kEndOfFileMarker) { 1011 1010 parseError(); 1012 1011 m_token->endAttributeValue(source.numberOfCharactersConsumed()); … … 1058 1057 else if (m_options.usePreHTML5ParserQuirks && cc == '<') 1059 1058 return emitAndReconsumeIn(source, HTMLTokenizerState::DataState); 1060 else if (cc == InputStreamPreprocessor::endOfFileMarker) {1059 else if (cc == kEndOfFileMarker) { 1061 1060 parseError(); 1062 1061 HTML_RECONSUME_IN(DataState); … … 1072 1071 m_token->setSelfClosing(); 1073 1072 return emitAndResumeIn(source, HTMLTokenizerState::DataState); 1074 } else if (cc == InputStreamPreprocessor::endOfFileMarker) {1073 } else if (cc == kEndOfFileMarker) { 1075 1074 parseError(); 1076 1075 HTML_RECONSUME_IN(DataState); … … 1091 1090 if (cc == '>') 1092 1091 return emitAndResumeIn(source, HTMLTokenizerState::DataState); 1093 else if (cc == InputStreamPreprocessor::endOfFileMarker)1092 else if (cc == kEndOfFileMarker) 1094 1093 return emitAndReconsumeIn(source, HTMLTokenizerState::DataState); 1095 1094 else { … … 1139 1138 parseError(); 1140 1139 return emitAndResumeIn(source, HTMLTokenizerState::DataState); 1141 } else if (cc == InputStreamPreprocessor::endOfFileMarker) {1140 } else if (cc == kEndOfFileMarker) { 1142 1141 parseError(); 1143 1142 return emitAndReconsumeIn(source, HTMLTokenizerState::DataState); … … 1155 1154 parseError(); 1156 1155 return emitAndResumeIn(source, HTMLTokenizerState::DataState); 1157 } else if (cc == InputStreamPreprocessor::endOfFileMarker) {1156 } else if (cc == kEndOfFileMarker) { 1158 1157 parseError(); 1159 1158 return emitAndReconsumeIn(source, HTMLTokenizerState::DataState); … … 1169 1168 if (cc == '-') 1170 1169 HTML_ADVANCE_TO(CommentEndDashState); 1171 else if (cc == InputStreamPreprocessor::endOfFileMarker) {1170 else if (cc == kEndOfFileMarker) { 1172 1171 parseError(); 1173 1172 return emitAndReconsumeIn(source, HTMLTokenizerState::DataState); … … 1182 1181 if (cc == '-') 1183 1182 HTML_ADVANCE_TO(CommentEndState); 1184 else if (cc == InputStreamPreprocessor::endOfFileMarker) {1183 else if (cc == kEndOfFileMarker) { 1185 1184 parseError(); 1186 1185 return emitAndReconsumeIn(source, HTMLTokenizerState::DataState); … … 1203 1202 m_token->appendToComment('-'); 1204 1203 HTML_ADVANCE_TO(CommentEndState); 1205 } else if (cc == InputStreamPreprocessor::endOfFileMarker) {1204 } else if (cc == kEndOfFileMarker) { 1206 1205 parseError(); 1207 1206 return emitAndReconsumeIn(source, HTMLTokenizerState::DataState); … … 1224 1223 } else if (cc == '>') 1225 1224 return emitAndResumeIn(source, HTMLTokenizerState::DataState); 1226 else if (cc == InputStreamPreprocessor::endOfFileMarker) {1225 else if (cc == kEndOfFileMarker) { 1227 1226 parseError(); 1228 1227 return emitAndReconsumeIn(source, HTMLTokenizerState::DataState); … … 1240 1239 if (isTokenizerWhitespace(cc)) 1241 1240 HTML_ADVANCE_TO(BeforeDOCTYPENameState); 1242 else if (cc == InputStreamPreprocessor::endOfFileMarker) {1241 else if (cc == kEndOfFileMarker) { 1243 1242 parseError(); 1244 1243 m_token->beginDOCTYPE(); … … 1263 1262 m_token->setForceQuirks(); 1264 1263 return emitAndResumeIn(source, HTMLTokenizerState::DataState); 1265 } else if (cc == InputStreamPreprocessor::endOfFileMarker) {1264 } else if (cc == kEndOfFileMarker) { 1266 1265 parseError(); 1267 1266 m_token->beginDOCTYPE(); … … 1283 1282 m_token->appendToName(toLowerCase(cc)); 1284 1283 HTML_ADVANCE_TO(DOCTYPENameState); 1285 } else if (cc == InputStreamPreprocessor::endOfFileMarker) {1284 } else if (cc == kEndOfFileMarker) { 1286 1285 parseError(); 1287 1286 m_token->setForceQuirks(); … … 1299 1298 if (cc == '>') 1300 1299 return emitAndResumeIn(source, HTMLTokenizerState::DataState); 1301 else if (cc == InputStreamPreprocessor::endOfFileMarker) {1300 else if (cc == kEndOfFileMarker) { 1302 1301 parseError(); 1303 1302 m_token->setForceQuirks(); … … 1343 1342 m_token->setForceQuirks(); 1344 1343 return emitAndResumeIn(source, HTMLTokenizerState::DataState); 1345 } else if (cc == InputStreamPreprocessor::endOfFileMarker) {1344 } else if (cc == kEndOfFileMarker) { 1346 1345 parseError(); 1347 1346 m_token->setForceQuirks(); … … 1368 1367 m_token->setForceQuirks(); 1369 1368 return emitAndResumeIn(source, HTMLTokenizerState::DataState); 1370 } else if (cc == InputStreamPreprocessor::endOfFileMarker) {1369 } else if (cc == kEndOfFileMarker) { 1371 1370 parseError(); 1372 1371 m_token->setForceQuirks(); … … 1387 1386 m_token->setForceQuirks(); 1388 1387 return emitAndResumeIn(source, HTMLTokenizerState::DataState); 1389 } else if (cc == InputStreamPreprocessor::endOfFileMarker) {1388 } else if (cc == kEndOfFileMarker) { 1390 1389 parseError(); 1391 1390 m_token->setForceQuirks(); … … 1405 1404 m_token->setForceQuirks(); 1406 1405 return emitAndResumeIn(source, HTMLTokenizerState::DataState); 1407 } else if (cc == InputStreamPreprocessor::endOfFileMarker) {1406 } else if (cc == kEndOfFileMarker) { 1408 1407 parseError(); 1409 1408 m_token->setForceQuirks(); … … 1429 1428 m_token->setSystemIdentifierToEmptyString(); 1430 1429 HTML_ADVANCE_TO(DOCTYPESystemIdentifierSingleQuotedState); 1431 } else if (cc == InputStreamPreprocessor::endOfFileMarker) {1430 } else if (cc == kEndOfFileMarker) { 1432 1431 parseError(); 1433 1432 m_token->setForceQuirks(); … … 1452 1451 m_token->setSystemIdentifierToEmptyString(); 1453 1452 HTML_ADVANCE_TO(DOCTYPESystemIdentifierSingleQuotedState); 1454 } else if (cc == InputStreamPreprocessor::endOfFileMarker) {1453 } else if (cc == kEndOfFileMarker) { 1455 1454 parseError(); 1456 1455 m_token->setForceQuirks(); … … 1479 1478 m_token->setForceQuirks(); 1480 1479 return emitAndResumeIn(source, HTMLTokenizerState::DataState); 1481 } else if (cc == InputStreamPreprocessor::endOfFileMarker) {1480 } else if (cc == kEndOfFileMarker) { 1482 1481 parseError(); 1483 1482 m_token->setForceQuirks(); … … 1504 1503 m_token->setForceQuirks(); 1505 1504 return emitAndResumeIn(source, HTMLTokenizerState::DataState); 1506 } else if (cc == InputStreamPreprocessor::endOfFileMarker) {1505 } else if (cc == kEndOfFileMarker) { 1507 1506 parseError(); 1508 1507 m_token->setForceQuirks(); … … 1523 1522 m_token->setForceQuirks(); 1524 1523 return emitAndResumeIn(source, HTMLTokenizerState::DataState); 1525 } else if (cc == InputStreamPreprocessor::endOfFileMarker) {1524 } else if (cc == kEndOfFileMarker) { 1526 1525 parseError(); 1527 1526 m_token->setForceQuirks(); … … 1541 1540 m_token->setForceQuirks(); 1542 1541 return emitAndResumeIn(source, HTMLTokenizerState::DataState); 1543 } else if (cc == InputStreamPreprocessor::endOfFileMarker) {1542 } else if (cc == kEndOfFileMarker) { 1544 1543 parseError(); 1545 1544 m_token->setForceQuirks(); … … 1557 1556 else if (cc == '>') 1558 1557 return emitAndResumeIn(source, HTMLTokenizerState::DataState); 1559 else if (cc == InputStreamPreprocessor::endOfFileMarker) {1558 else if (cc == kEndOfFileMarker) { 1560 1559 parseError(); 1561 1560 m_token->setForceQuirks(); … … 1571 1570 if (cc == '>') 1572 1571 return emitAndResumeIn(source, HTMLTokenizerState::DataState); 1573 else if (cc == InputStreamPreprocessor::endOfFileMarker)1572 else if (cc == kEndOfFileMarker) 1574 1573 return emitAndReconsumeIn(source, HTMLTokenizerState::DataState); 1575 1574 HTML_ADVANCE_TO(BogusDOCTYPEState); … … 1580 1579 if (cc == ']') 1581 1580 HTML_ADVANCE_TO(CDATASectionRightSquareBracketState); 1582 else if (cc == InputStreamPreprocessor::endOfFileMarker)1581 else if (cc == kEndOfFileMarker) 1583 1582 HTML_RECONSUME_IN(DataState); 1584 1583 else { -
trunk/Source/WebCore/html/track/WebVTTTokenizer.cpp
r133995 r142497 43 43 44 44 WebVTTTokenizer::WebVTTTokenizer() 45 : m_inputStreamPreprocessor(this) 45 46 { 46 47 reset(); 47 }48 49 template<>50 inline bool MarkupTokenizerBase<WebVTTToken, WebVTTTokenizerState>::shouldSkipNullCharacters() const51 {52 return true;53 48 } 54 49 … … 96 91 else 97 92 return emitAndResumeIn(source, WebVTTTokenizerState::TagState); 98 } else if (cc == InputStreamPreprocessor::endOfFileMarker)93 } else if (cc == kEndOfFileMarker) 99 94 return emitEndOfFile(source); 100 95 else { … … 122 117 m_buffer.append(static_cast<LChar>(cc)); 123 118 WEBVTT_ADVANCE_TO(EscapeState); 124 } else if (cc == InputStreamPreprocessor::endOfFileMarker) {119 } else if (cc == kEndOfFileMarker) { 125 120 m_token->appendToCharacter(m_buffer); 126 121 return emitEndOfFile(source); … … 146 141 m_token->beginTimestampTag(cc); 147 142 WEBVTT_ADVANCE_TO(TimestampTagState); 148 } else if (cc == '>' || cc == InputStreamPreprocessor::endOfFileMarker) {143 } else if (cc == '>' || cc == kEndOfFileMarker) { 149 144 m_token->beginEmptyStartTag(); 150 145 return emitAndResumeIn(source, WebVTTTokenizerState::DataState); … … 161 156 else if (cc == '.') 162 157 WEBVTT_ADVANCE_TO(StartTagClassState); 163 else if (cc == '>' || cc == InputStreamPreprocessor::endOfFileMarker)158 else if (cc == '>' || cc == kEndOfFileMarker) 164 159 return emitAndResumeIn(source, WebVTTTokenizerState::DataState); 165 160 else { … … 177 172 m_token->addNewClass(); 178 173 WEBVTT_ADVANCE_TO(StartTagClassState); 179 } else if (cc == '>' || cc == InputStreamPreprocessor::endOfFileMarker) {174 } else if (cc == '>' || cc == kEndOfFileMarker) { 180 175 m_token->addNewClass(); 181 176 return emitAndResumeIn(source, WebVTTTokenizerState::DataState); … … 189 184 190 185 WEBVTT_BEGIN_STATE(StartTagAnnotationState) { 191 if (cc == '>' || cc == InputStreamPreprocessor::endOfFileMarker) {186 if (cc == '>' || cc == kEndOfFileMarker) { 192 187 m_token->addNewAnnotation(); 193 188 return emitAndResumeIn(source, WebVTTTokenizerState::DataState); … … 199 194 200 195 WEBVTT_BEGIN_STATE(EndTagOpenState) { 201 if (cc == '>' || cc == InputStreamPreprocessor::endOfFileMarker) {196 if (cc == '>' || cc == kEndOfFileMarker) { 202 197 m_token->beginEndTag('\0'); 203 198 return emitAndResumeIn(source, WebVTTTokenizerState::DataState); … … 209 204 210 205 WEBVTT_BEGIN_STATE(EndTagState) { 211 if (cc == '>' || cc == InputStreamPreprocessor::endOfFileMarker)206 if (cc == '>' || cc == kEndOfFileMarker) 212 207 return emitAndResumeIn(source, WebVTTTokenizerState::DataState); 213 208 m_token->appendToName(cc); … … 217 212 218 213 WEBVTT_BEGIN_STATE(TimestampTagState) { 219 if (cc == '>' || cc == InputStreamPreprocessor::endOfFileMarker)214 if (cc == '>' || cc == kEndOfFileMarker) 220 215 return emitAndResumeIn(source, WebVTTTokenizerState::DataState); 221 216 m_token->appendToTimestamp(cc); -
trunk/Source/WebCore/html/track/WebVTTTokenizer.h
r131311 r142497 34 34 #if ENABLE(VIDEO_TRACK) 35 35 36 #include "MarkupTokenizerBase.h" 37 #include "SegmentedString.h" 36 #include "InputStreamPreprocessor.h" 38 37 #include "WebVTTToken.h" 38 #include <wtf/PassOwnPtr.h> 39 39 40 40 namespace WebCore { … … 55 55 }; 56 56 57 class WebVTTTokenizer : MarkupTokenizerBase<WebVTTToken, WebVTTTokenizerState>{57 class WebVTTTokenizer { 58 58 WTF_MAKE_NONCOPYABLE(WebVTTTokenizer); 59 59 WTF_MAKE_FAST_ALLOCATED; … … 61 61 static PassOwnPtr<WebVTTTokenizer> create() { return adoptPtr(new WebVTTTokenizer); } 62 62 63 typedef WebVTTTokenizerState State; 64 63 65 void reset(); 64 66 65 67 bool nextToken(SegmentedString&, WebVTTToken&); 66 68 69 inline bool haveBufferedCharacterToken() 70 { 71 return m_token->type() == WebVTTToken::Type::Character; 72 } 73 74 inline void bufferCharacter(UChar character) 75 { 76 ASSERT(character != kEndOfFileMarker); 77 m_token->ensureIsCharacterToken(); 78 m_token->appendToCharacter(character); 79 } 80 81 inline bool emitAndResumeIn(SegmentedString& source, State::State state) 82 { 83 m_state = state; 84 source.advanceAndUpdateLineNumber(); 85 return true; 86 } 87 88 inline bool emitEndOfFile(SegmentedString& source) 89 { 90 if (haveBufferedCharacterToken()) 91 return true; 92 m_state = State::DataState; 93 source.advanceAndUpdateLineNumber(); 94 m_token->clear(); 95 m_token->makeEndOfFile(); 96 return true; 97 } 98 99 bool shouldSkipNullCharacters() const { return true; } 100 67 101 private: 68 102 WebVTTTokenizer(); 69 103 104 // m_token is owned by the caller. If nextToken is not on the stack, 105 // this member might be pointing to unallocated memory. 106 WebVTTToken* m_token; 107 WebVTTTokenizerState::State m_state; 108 70 109 Vector<LChar, 32> m_buffer; 110 111 // ://www.whatwg.org/specs/web-apps/current-work/#preprocessing-the-input-stream 112 InputStreamPreprocessor<WebVTTTokenizer> m_inputStreamPreprocessor; 71 113 }; 72 114 -
trunk/Source/WebCore/xml/parser/MarkupTokenizerBase.h
r139497 r142497 29 29 #define MarkupTokenizerBase_h 30 30 31 #include "SegmentedString.h" 32 #include <wtf/Noncopyable.h> 31 #include "InputStreamPreprocessor.h" 33 32 #include <wtf/PassOwnPtr.h> 34 33 #include <wtf/Vector.h> … … 56 55 57 56 protected: 58 // http://www.whatwg.org/specs/web-apps/current-work/#preprocessing-the-input-stream59 class InputStreamPreprocessor {60 WTF_MAKE_NONCOPYABLE(InputStreamPreprocessor);61 public:62 InputStreamPreprocessor(MarkupTokenizerBase<Token, State>* tokenizer)63 : m_tokenizer(tokenizer)64 {65 reset();66 }67 68 ALWAYS_INLINE UChar nextInputCharacter() const { return m_nextInputCharacter; }69 70 // Returns whether we succeeded in peeking at the next character.71 // The only way we can fail to peek is if there are no more72 // characters in |source| (after collapsing \r\n, etc).73 ALWAYS_INLINE bool peek(SegmentedString& source)74 {75 PeekAgain:76 m_nextInputCharacter = source.currentChar();77 78 // Every branch in this function is expensive, so we have a79 // fast-reject branch for characters that don't require special80 // handling. Please run the parser benchmark whenever you touch81 // this function. It's very hot.82 static const UChar specialCharacterMask = '\n' | '\r' | '\0';83 if (m_nextInputCharacter & ~specialCharacterMask) {84 m_skipNextNewLine = false;85 return true;86 }87 88 if (m_nextInputCharacter == '\n' && m_skipNextNewLine) {89 m_skipNextNewLine = false;90 source.advancePastNewlineAndUpdateLineNumber();91 if (source.isEmpty())92 return false;93 m_nextInputCharacter = source.currentChar();94 }95 if (m_nextInputCharacter == '\r') {96 m_nextInputCharacter = '\n';97 m_skipNextNewLine = true;98 } else {99 m_skipNextNewLine = false;100 // FIXME: The spec indicates that the surrogate pair range as well as101 // a number of specific character values are parse errors and should be replaced102 // by the replacement character. We suspect this is a problem with the spec as doing103 // that filtering breaks surrogate pair handling and causes us not to match Minefield.104 if (m_nextInputCharacter == '\0' && !shouldTreatNullAsEndOfFileMarker(source)) {105 if (m_tokenizer->shouldSkipNullCharacters()) {106 source.advancePastNonNewline();107 if (source.isEmpty())108 return false;109 goto PeekAgain;110 }111 m_nextInputCharacter = 0xFFFD;112 }113 }114 return true;115 }116 117 // Returns whether there are more characters in |source| after advancing.118 ALWAYS_INLINE bool advance(SegmentedString& source)119 {120 source.advanceAndUpdateLineNumber();121 if (source.isEmpty())122 return false;123 return peek(source);124 }125 126 bool skipNextNewLine() const { return m_skipNextNewLine; }127 128 void reset(bool skipNextNewLine = false)129 {130 m_nextInputCharacter = '\0';131 m_skipNextNewLine = skipNextNewLine;132 }133 134 static const UChar endOfFileMarker = 0;135 136 private:137 bool shouldTreatNullAsEndOfFileMarker(SegmentedString& source) const138 {139 return source.isClosed() && source.length() == 1;140 }141 142 MarkupTokenizerBase<Token, State>* m_tokenizer;143 144 // http://www.whatwg.org/specs/web-apps/current-work/#next-input-character145 UChar m_nextInputCharacter;146 bool m_skipNextNewLine;147 };148 149 57 MarkupTokenizerBase() : m_inputStreamPreprocessor(this) { reset(); } 150 58 151 59 inline void bufferCharacter(UChar character) 152 60 { 153 ASSERT(character != InputStreamPreprocessor::endOfFileMarker);61 ASSERT(character != kEndOfFileMarker); 154 62 m_token->ensureIsCharacterToken(); 155 63 m_token->appendToCharacter(character); … … 207 115 208 116 // http://www.whatwg.org/specs/web-apps/current-work/#preprocessing-the-input-stream 209 InputStreamPreprocessor m_inputStreamPreprocessor;117 InputStreamPreprocessor<MarkupTokenizerBase<Token, State> > m_inputStreamPreprocessor; 210 118 }; 211 119
Note: See TracChangeset
for help on using the changeset viewer.