Changeset 121858 in webkit

Timestamp:

Jul 4, 2012 8:31:53 AM (12 years ago)

Author:

kseo@webkit.org

Message:

Stop tracking line number in tokenizer
​https://bugs.webkit.org/show_bug.cgi?id=90544

Reviewed by Adam Barth.

Because SegmentedString knows how to track the current line and column,
a tokenizer does not need to keep track of the current line by itself.
No behavior change, so no new tests.

• html/parser/HTMLDocumentParser.cpp:

(WebCore::HTMLDocumentParser::pumpTokenizer):
(WebCore::HTMLDocumentParser::lineNumber):
(WebCore::HTMLDocumentParser::textPosition):

• html/parser/HTMLTokenizer.cpp:

(WebCore::HTMLTokenizer::reset):
(WebCore::HTMLTokenizer::flushBufferedEndTag):
(WebCore):
(WebCore::HTMLTokenizer::nextToken):

• html/parser/HTMLTreeBuilder.cpp:

(WebCore::HTMLTreeBuilder::processScriptStartTag):

• html/track/WebVTTTokenizer.cpp:

(WebCore::WebVTTTokenizer::reset):
(WebCore::WebVTTTokenizer::nextToken):

• platform/text/SegmentedString.cpp:

(WebCore::SegmentedString::advanceAndUpdateLineNumberSlowCase):

• platform/text/SegmentedString.h:

(WebCore::SegmentedString::advancePastNewlineAndUpdateLineNumber):
(WebCore::SegmentedString::advanceAndUpdateLineNumber):
(SegmentedString):

• xml/parser/MarkupTokenizerBase.h:

(WebCore::MarkupTokenizerBase::InputStreamPreprocessor::peek):
(WebCore::MarkupTokenizerBase::InputStreamPreprocessor::advance):
(WebCore::MarkupTokenizerBase::emitAndResumeIn):
(WebCore::MarkupTokenizerBase::emitEndOfFile):
(WebCore::MarkupTokenizerBase::reset):
(MarkupTokenizerBase):

• xml/parser/MarkupTokenizerInlineMethods.h:

(WebCore):

• xml/parser/XMLTokenizer.cpp:

(WebCore::XMLTokenizer::nextToken):

Location:

trunk/Source/WebCore

Files:

• ChangeLog (modified) (1 diff)
• html/parser/HTMLDocumentParser.cpp (modified) (4 diffs)
• html/parser/HTMLTokenizer.cpp (modified) (4 diffs)
• html/parser/HTMLTreeBuilder.cpp (modified) (1 diff)
• html/track/WebVTTTokenizer.cpp (modified) (2 diffs)
• platform/text/SegmentedString.cpp (modified) (2 diffs)
• platform/text/SegmentedString.h (modified) (5 diffs)
• xml/parser/MarkupTokenizerBase.h (modified) (8 diffs)
• xml/parser/MarkupTokenizerInlineMethods.h (modified) (2 diffs)
• xml/parser/XMLTokenizer.cpp (modified) (1 diff)

trunk/Source/WebCore/ChangeLog

@@ +1 @@
+2012-07-04  Kwang Yul Seo  <skyul@company100.net>
+
+        Stop tracking line number in tokenizer
+        https://bugs.webkit.org/show_bug.cgi?id=90544
+
+        Reviewed by Adam Barth.
+
+        Because SegmentedString knows how to track the current line and column,
+        a tokenizer does not need to keep track of the current line by itself.
+        No behavior change, so no new tests.
+
+        * html/parser/HTMLDocumentParser.cpp:
+        (WebCore::HTMLDocumentParser::pumpTokenizer):
+        (WebCore::HTMLDocumentParser::lineNumber):
+        (WebCore::HTMLDocumentParser::textPosition):
+        * html/parser/HTMLTokenizer.cpp:
+        (WebCore::HTMLTokenizer::reset):
+        (WebCore::HTMLTokenizer::flushBufferedEndTag):
+        (WebCore):
+        (WebCore::HTMLTokenizer::nextToken):
+        * html/parser/HTMLTreeBuilder.cpp:
+        (WebCore::HTMLTreeBuilder::processScriptStartTag):
+        * html/track/WebVTTTokenizer.cpp:
+        (WebCore::WebVTTTokenizer::reset):
+        (WebCore::WebVTTTokenizer::nextToken):
+        * platform/text/SegmentedString.cpp:
+        (WebCore::SegmentedString::advanceAndUpdateLineNumberSlowCase):
+        * platform/text/SegmentedString.h:
+        (WebCore::SegmentedString::advancePastNewlineAndUpdateLineNumber):
+        (WebCore::SegmentedString::advanceAndUpdateLineNumber):
+        (SegmentedString):
+        * xml/parser/MarkupTokenizerBase.h:
+        (WebCore::MarkupTokenizerBase::InputStreamPreprocessor::peek):
+        (WebCore::MarkupTokenizerBase::InputStreamPreprocessor::advance):
+        (WebCore::MarkupTokenizerBase::emitAndResumeIn):
+        (WebCore::MarkupTokenizerBase::emitEndOfFile):
+        (WebCore::MarkupTokenizerBase::reset):
+        (MarkupTokenizerBase):
+        * xml/parser/MarkupTokenizerInlineMethods.h:
+        (WebCore):
+        * xml/parser/XMLTokenizer.cpp:
+        (WebCore::XMLTokenizer::nextToken):
+
 2012-07-04  Ryuan Choi  <ryuan.choi@samsung.com>
 

trunk/Source/WebCore/html/parser/HTMLDocumentParser.cpp

@@ -259 +259 @@
     // end up parsing the whole buffer in this pump.  We should pass how
     // much we parsed as part of didWriteHTML instead of willWriteHTML.
-    InspectorInstrumentationCookie cookie = InspectorInstrumentation::willWriteHTML(document(), m_input.current().length(), m_tokenizer->lineNumber().zeroBasedInt());
+    InspectorInstrumentationCookie cookie = InspectorInstrumentation::willWriteHTML(document(), m_input.current().length(), m_input.current().currentLine().zeroBasedInt());
 
     while (canTakeNextToken(mode, session) && !session.needsYield) {
@@ -299 +299 @@
     }
 
-    InspectorInstrumentation::didWriteHTML(cookie, m_tokenizer->lineNumber().zeroBasedInt());
+    InspectorInstrumentation::didWriteHTML(cookie, m_input.current().currentLine().zeroBasedInt());
 }
 
@@ -449 +449 @@
 OrdinalNumber HTMLDocumentParser::lineNumber() const
 {
-    return m_tokenizer->lineNumber();
+    return m_input.current().currentLine();
 }
 
@@ -457 +457 @@
     OrdinalNumber line = currentString.currentLine();
     OrdinalNumber column = currentString.currentColumn();
-    ASSERT(m_tokenizer->lineNumber() == line);
 
     return TextPosition(line, column);

trunk/Source/WebCore/html/parser/HTMLTokenizer.cpp

@@ -135 +135 @@
     m_state = HTMLTokenizerState::DataState;
     m_token = 0;
-    m_lineNumber = 0;
     m_forceNullCharacterReplacement = false;
     m_shouldAllowCDATA = false;
@@ -161 +160 @@
 {
     ASSERT(m_token->type() == HTMLTokenTypes::Character || m_token->type() == HTMLTokenTypes::Uninitialized);
-    source.advance(m_lineNumber);
+    source.advanceAndUpdateLineNumber();
     if (m_token->type() == HTMLTokenTypes::Character)
         return true;
@@ -176 +175 @@
             return true;                                                   \
         if (source.isEmpty()                                               \
-            || !m_inputStreamPreprocessor.peek(source, m_lineNumber))      \
+            || !m_inputStreamPreprocessor.peek(source))                    \
             return haveBufferedCharacterToken();                           \
         cc = m_inputStreamPreprocessor.nextInputCharacter();               \
@@ -208 +207 @@
     }
 
-    if (source.isEmpty() || !m_inputStreamPreprocessor.peek(source, m_lineNumber))
+    if (source.isEmpty() || !m_inputStreamPreprocessor.peek(source))
         return haveBufferedCharacterToken();
     UChar cc = m_inputStreamPreprocessor.nextInputCharacter();

trunk/Source/WebCore/html/parser/HTMLTreeBuilder.cpp

@@ -2647 +2647 @@
     TextPosition position = m_parser->textPosition();
 
-    ASSERT(position.m_line == m_parser->tokenizer()->lineNumber());
-
     m_lastScriptElementStartPosition = position;
 

trunk/Source/WebCore/html/track/WebVTTTokenizer.cpp

@@ -66 +66 @@
     m_state = WebVTTTokenizerState::DataState;
     m_token = 0;
-    m_lineNumber = 0;
     m_buffer.clear();
 }
@@ -77 +76 @@
     m_token = &token;
 
-    if (source.isEmpty() || !m_inputStreamPreprocessor.peek(source, m_lineNumber))
+    if (source.isEmpty() || !m_inputStreamPreprocessor.peek(source))
         return haveBufferedCharacterToken();
 

trunk/Source/WebCore/platform/text/SegmentedString.cpp

@@ -223 +223 @@
 }
 
-void SegmentedString::advanceSlowCase(int& lineNumber)
+void SegmentedString::advanceAndUpdateLineNumberSlowCase()
 {
     if (m_pushedChar1) {
@@ -230 +230 @@
     } else if (m_currentString.m_current) {
         if (*m_currentString.m_current++ == '\n' && m_currentString.doNotExcludeLineNumbers()) {
-            ++lineNumber;
             ++m_currentLine;
             // Plus 1 because numberOfCharactersConsumed value hasn't incremented yet; it does with m_length decrement below.

trunk/Source/WebCore/platform/text/SegmentedString.h

@@ -158 +158 @@
     }
 
-    void advancePastNewline(int& lineNumber)
+    void advancePastNewlineAndUpdateLineNumber()
     {
         ASSERT(*current() == '\n');
         if (!m_pushedChar1 && m_currentString.m_length > 1) {
             int newLineFlag = m_currentString.doNotExcludeLineNumbers();
-            lineNumber += newLineFlag;
             m_currentLine += newLineFlag;
             if (newLineFlag)
@@ -171 +170 @@
             return;
         }
-        advanceSlowCase(lineNumber);
+        advanceAndUpdateLineNumberSlowCase();
     }
     
@@ -185 +184 @@
     }
     
-    void advance(int& lineNumber)
+    void advanceAndUpdateLineNumber()
     {
         if (!m_pushedChar1 && m_currentString.m_length > 1) {
             int newLineFlag = (*m_currentString.m_current == '\n') & m_currentString.doNotExcludeLineNumbers();
-            lineNumber += newLineFlag;
             m_currentLine += newLineFlag;
             if (newLineFlag)
@@ -197 +195 @@
             return;
         }
-        advanceSlowCase(lineNumber);
+        advanceAndUpdateLineNumberSlowCase();
     }
 
@@ -235 +233 @@
 
     void advanceSlowCase();
-    void advanceSlowCase(int& lineNumber);
+    void advanceAndUpdateLineNumberSlowCase();
     void advanceSubstring();
     const UChar* current() const { return m_currentChar; }

trunk/Source/WebCore/xml/parser/MarkupTokenizerBase.h

@@ -46 +46 @@
     virtual ~MarkupTokenizerBase() { }
 
-    OrdinalNumber lineNumber() const { return OrdinalNumber::fromZeroBasedInt(m_lineNumber); }
-
     typename State::State state() const { return m_state; }
     void setState(typename State::State state) { m_state = state; }
@@ -74 +72 @@
         // The only way we can fail to peek is if there are no more
         // characters in |source| (after collapsing \r\n, etc).
-        ALWAYS_INLINE bool peek(SegmentedString& source, int& lineNumber)
+        ALWAYS_INLINE bool peek(SegmentedString& source)
         {
         PeekAgain:
@@ -91 +89 @@
             if (m_nextInputCharacter == '\n' && m_skipNextNewLine) {
                 m_skipNextNewLine = false;
-                source.advancePastNewline(lineNumber);
+                source.advancePastNewlineAndUpdateLineNumber();
                 if (source.isEmpty())
                     return false;
@@ -119 +117 @@
 
         // Returns whether there are more characters in |source| after advancing.
-        bool advance(SegmentedString& source, int& lineNumber)
-        {
-            source.advance(lineNumber);
+        bool advance(SegmentedString& source)
+        {
+            source.advanceAndUpdateLineNumber();
             if (source.isEmpty())
                 return false;
-            return peek(source, lineNumber);
+            return peek(source);
         }
 
@@ -157 +155 @@
     {
         m_state = state;
-        source.advance(m_lineNumber);
+        source.advanceAndUpdateLineNumber();
         return true;
     }
@@ -173 +171 @@
             return true;
         m_state = State::DataState;
-        source.advance(m_lineNumber);
+        source.advanceAndUpdateLineNumber();
         m_token->clear();
         m_token->makeEndOfFile();
@@ -183 +181 @@
         m_state = State::DataState;
         m_token = 0;
-        m_lineNumber = 0;
     }
 
@@ -196 +193 @@
     // this member might be pointing to unallocated memory.
     Token* m_token;
-    int m_lineNumber;
 
     bool m_forceNullCharacterReplacement;

trunk/Source/WebCore/xml/parser/MarkupTokenizerInlineMethods.h

@@ -72 +72 @@
     do {                                                                   \
         m_state = prefix::stateName;                                       \
-        if (!m_inputStreamPreprocessor.advance(source, m_lineNumber))      \
+        if (!m_inputStreamPreprocessor.advance(source))                    \
             return haveBufferedCharacterToken();                           \
         cc = m_inputStreamPreprocessor.nextInputCharacter();               \
@@ -85 +85 @@
     do {                                                                   \
         m_state = prefix::stateName;                                       \
-        if (source.isEmpty() || !m_inputStreamPreprocessor.peek(source, m_lineNumber)) \
+        if (source.isEmpty() || !m_inputStreamPreprocessor.peek(source))   \
             return haveBufferedCharacterToken();                           \
         cc = m_inputStreamPreprocessor.nextInputCharacter();               \

trunk/Source/WebCore/xml/parser/XMLTokenizer.cpp

@@ -226 +226 @@
     m_token = &token;
 
-    if (source.isEmpty() || !m_inputStreamPreprocessor.peek(source, m_lineNumber))
+    if (source.isEmpty() || !m_inputStreamPreprocessor.peek(source))
         return haveBufferedCharacterToken();
     UChar cc = m_inputStreamPreprocessor.nextInputCharacter();