Changeset 140610 in webkit

Timestamp:

Jan 23, 2013 4:26:32 PM (11 years ago)

Author:

commit-queue@webkit.org

Message:

WebKit should support decoding multi-byte entities in XML content
​https://bugs.webkit.org/show_bug.cgi?id=107459

Patch by Martin Robinson <​mrobinson@igalia.com> on 2013-01-23
Reviewed by Adam Barth.

Source/WebCore:

Test: fast/parser/entities-in-xhtml.xhtml

• html/parser/HTMLEntityParser.cpp:

(WebCore::appendUChar32ToUCharArray): Added this helper function. Later patches
may try to move this code to somewhere that it can be shared more easily.
(WebCore::decodeNamedEntityToUCharArray): Modify this function to work on a UChar
array four elements long, so that multi-byte and multi-character entities can be resolved.

• html/parser/HTMLEntityParser.h: Updated function declaratoin.
• xml/parser/XMLDocumentParserLibxml2.cpp:

(WebCore): Modify the statically allocated entity string memory area to accommodate
up to two UTF-8 characters. Each UTF-8 character can be 4 bytes, so this brings the
total size to 9 bytes.
(WebCore::getXHTMLEntity): Use the new entity decoding API.

• xml/parser/XMLDocumentParserQt.cpp:

(WebCore::EntityResolver::resolveUndeclaredEntity): Ditto.
(WebCore::XMLDocumentParser::parse): Ditto.

LayoutTests:

Add a simple test for decoding some entities that resolve to multiple
bytes in XML.

• fast/parser/entities-in-xhtml.xhtml: Added a few multi-bye entities.
• fast/parser/entities-in-xhtml-expected.txt: Updated.

Location:

trunk

Files:

• LayoutTests/ChangeLog (modified) (1 diff)
• LayoutTests/fast/parser/entities-in-xhtml-expected.txt (modified) (1 diff)
• LayoutTests/fast/parser/entities-in-xhtml.xhtml (modified) (1 diff)
• Source/WebCore/ChangeLog (modified) (1 diff)
• Source/WebCore/html/parser/HTMLEntityParser.cpp (modified) (2 diffs)
• Source/WebCore/html/parser/HTMLEntityParser.h (modified) (1 diff)
• Source/WebCore/xml/parser/XMLDocumentParserLibxml2.cpp (modified) (3 diffs)
• Source/WebCore/xml/parser/XMLDocumentParserQt.cpp (modified) (2 diffs)

trunk/LayoutTests/ChangeLog

@@ +1 @@
+2013-01-23  Martin Robinson  <mrobinson@igalia.com>
+
+        WebKit should support decoding multi-byte entities in XML content
+        https://bugs.webkit.org/show_bug.cgi?id=107459
+
+        Reviewed by Adam Barth.
+
+        Add a simple test for decoding some entities that resolve to multiple
+        bytes in XML.
+
+        * fast/parser/entities-in-xhtml.xhtml: Added a few multi-bye entities.
+        * fast/parser/entities-in-xhtml-expected.txt: Updated.
+
 2013-01-23  Filip Pizlo  <fpizlo@apple.com>
 

trunk/LayoutTests/fast/parser/entities-in-xhtml-expected.txt

@@ -267 +267 @@
 9829    2665    hearts  ♥       ♥       ♥
 9830    2666    diams   ♦       ♦       ♦
+Multi-byte entities from HTML5
+
+decimal hexadecimal     entity name     &#nnn;  &#xhhh; &entity;
+8882    22B2    vltri   ⊲       ⊲       ⊲
+8834 + 8402     2282 + 20D2     sub     ⊂⃒      ⊂⃒      ⊂⃒
+8804 + 8402     2264 + 20D2     sub     ≤⃒      ≤⃒      ≤⃒

trunk/LayoutTests/fast/parser/entities-in-xhtml.xhtml

@@ -333 +333 @@
 <tr><td>9829</td><td>2665</td><td>hearts</td><td>&#9829;</td><td>&#x2665;</td><td>&hearts;</td></tr>
 <tr><td>9830</td><td>2666</td><td>diams</td><td>&#9830;</td><td>&#x2666;</td><td>&diams;</td></tr>
+
+</tbody>
+</table>
+
+<h2>Multi-byte entities from HTML5</h2>
+<table border="1" summary="this table lists decimal value, hexadecimal value,
+entity name and respective entity references of XHTML entities.">
+<thead>
+<tr>
+<th>decimal</th>
+<th>hexadecimal</th>
+<th>entity name</th>
+<th>&amp;#nnn;</th>
+<th>&amp;#xhhh;</th>
+<th>&amp;entity;</th>
+</tr>
+</thead>
+<tbody>
+<tr><td>8882</td><td>22B2</td><td>vltri</td><td>&#8882;</td><td>&#x22B2;</td><td>&vltri;</td></tr>
+<tr><td>8834 + 8402</td><td>2282 + 20D2</td><td>sub</td><td>&#8834;&#8402;</td><td>&#x2282;&#x20D2;</td><td>&vnsub;</td></tr>
+<tr><td>8804 + 8402</td><td>2264 + 20D2</td><td>sub</td><td>&#8804;&#8402;</td><td>&#x2264;&#x20D2;</td><td>&nvle;</td></tr>
 </tbody>
 </table>

trunk/Source/WebCore/ChangeLog

@@ +1 @@
+2013-01-23  Martin Robinson  <mrobinson@igalia.com>
+
+        WebKit should support decoding multi-byte entities in XML content
+        https://bugs.webkit.org/show_bug.cgi?id=107459
+
+        Reviewed by Adam Barth.
+
+        Test: fast/parser/entities-in-xhtml.xhtml
+
+        * html/parser/HTMLEntityParser.cpp:
+        (WebCore::appendUChar32ToUCharArray): Added this helper function. Later patches
+        may try to move this code to somewhere that it can be shared more easily.
+        (WebCore::decodeNamedEntityToUCharArray): Modify this function to work on a UChar
+        array four elements long, so that multi-byte and multi-character entities can be resolved.
+        * html/parser/HTMLEntityParser.h: Updated function declaratoin.
+        * xml/parser/XMLDocumentParserLibxml2.cpp:
+        (WebCore): Modify the statically allocated entity string memory area to accommodate
+        up to two UTF-8 characters. Each UTF-8 character can be 4 bytes, so this brings the
+        total size to 9 bytes.
+        (WebCore::getXHTMLEntity): Use the new entity decoding API.
+        * xml/parser/XMLDocumentParserQt.cpp:
+        (WebCore::EntityResolver::resolveUndeclaredEntity): Ditto.
+        (WebCore::XMLDocumentParser::parse): Ditto.
+
 2013-01-23  Eric Seidel  <eric@webkit.org>
 

trunk/Source/WebCore/html/parser/HTMLEntityParser.cpp

@@ -141 +141 @@
 }
 
-UChar decodeNamedEntity(const char* name)
+static size_t appendUChar32ToUCharArray(UChar32 value, UChar* result)
+{
+    if (U_IS_BMP(value)) {
+        UChar character = static_cast<UChar>(value);
+        ASSERT(character == value);
+        result[0] = character;
+        return 1;
+    }
+
+    result[0] = U16_LEAD(value);
+    result[1] = U16_TRAIL(value);
+    return 2;
+}
+
+size_t decodeNamedEntityToUCharArray(const char* name, UChar result[4])
 {
     HTMLEntitySearch search;
@@ -153 +167 @@
         return 0;
 
-    UChar32 firstValue = search.mostRecentMatch()->firstValue;
-    if (U16_LENGTH(firstValue) != 1 || search.mostRecentMatch()->secondValue) {
-        // FIXME: Callers need to move off this API. Not all entities can be
-        // represented in a single UChar!
-        return 0;
-    }
-    return static_cast<UChar>(firstValue);
+    size_t numberOfCodePoints = appendUChar32ToUCharArray(search.mostRecentMatch()->firstValue, result);
+    if (!search.mostRecentMatch()->secondValue)
+        return numberOfCodePoints;
+    return numberOfCodePoints + appendUChar32ToUCharArray(search.mostRecentMatch()->secondValue, result + numberOfCodePoints);
 }
 

trunk/Source/WebCore/html/parser/HTMLEntityParser.h

@@ -35 +35 @@
 
 // Used by the XML parser.  Not suitable for use in HTML parsing.  Use consumeHTMLEntity instead.
-// FIXME: Move the XML parser to an entity decoding function works for non-BMP characters!
-UChar decodeNamedEntity(const char*);
+size_t decodeNamedEntityToUCharArray(const char*, UChar result[4]);
 
 }

trunk/Source/WebCore/xml/parser/XMLDocumentParserLibxml2.cpp

@@ -63 +63 @@
 #include <wtf/UnusedParam.h>
 #include <wtf/Vector.h>
+#include <wtf/unicode/UTF8.h>
 
 #if ENABLE(XSLT)
@@ -1161 +1162 @@
 // a hack to avoid malloc/free. Using a global variable like this could cause trouble
 // if libxml implementation details were to change
-static xmlChar sharedXHTMLEntityResult[5] = {0, 0, 0, 0, 0};
+static xmlChar sharedXHTMLEntityResult[9] = {0, 0, 0, 0, 0, 0, 0, 0, 0};
 
 static xmlEntityPtr sharedXHTMLEntity()
@@ -1175 +1176 @@
 }
 
+static size_t convertUTF16EntityToUTF8(const UChar* utf16Entity, size_t numberOfCodeUnits, char* target, size_t targetSize)
+{
+    const char* originalTarget = target;
+    WTF::Unicode::ConversionResult conversionResult = WTF::Unicode::convertUTF16ToUTF8(&utf16Entity,
+        utf16Entity + numberOfCodeUnits, &target, target + targetSize);
+    if (conversionResult != WTF::Unicode::conversionOK)
+        return 0;
+
+    // Even though we must pass the length, libxml expects the entity string to be null terminated.
+    ASSERT(target > originalTarget + 1);
+    *target = '\0';
+    return target - originalTarget;
+}
+
 static xmlEntityPtr getXHTMLEntity(const xmlChar* name)
 {
-    UChar c = decodeNamedEntity(reinterpret_cast<const char*>(name));
-    if (!c)
+    UChar utf16DecodedEntity[4];
+    size_t numberOfCodeUnits = decodeNamedEntityToUCharArray(reinterpret_cast<const char*>(name), utf16DecodedEntity);
+    if (!numberOfCodeUnits)
         return 0;
 
-    CString value = String(&c, 1).utf8();
-    ASSERT(value.length() < 5);
+    ASSERT(numberOfCodeUnits <= 4);
+    size_t entityLengthInUTF8 = convertUTF16EntityToUTF8(utf16DecodedEntity, numberOfCodeUnits,
+        reinterpret_cast<char*>(sharedXHTMLEntityResult), WTF_ARRAY_LENGTH(sharedXHTMLEntityResult));
+    if (!entityLengthInUTF8)
+        return 0;
+
     xmlEntityPtr entity = sharedXHTMLEntity();
-    entity->length = value.length();
+    entity->length = entityLengthInUTF8;
     entity->name = name;
-    memcpy(sharedXHTMLEntityResult, value.data(), entity->length + 1);
-
     return entity;
 }

trunk/Source/WebCore/xml/parser/XMLDocumentParserQt.cpp

@@ -68 +68 @@
 };
 
+static QString decodeNamedEntity(const QString& entityName)
+{
+    UChar utf16DecodedEntity[4];
+    size_t numberOfCodePoints = decodeNamedEntityToUCharArray(entityName.toUtf8().constData(), utf16DecodedEntity);
+    return QString::fromUTF16(utf16DecodedEntity, numberOfCodePoints);
+}
+
 QString EntityResolver::resolveUndeclaredEntity(const QString &name)
 {
-    UChar c = decodeNamedEntity(name.toUtf8().constData());
-    return QString(c);
+    return decodeNamedEntity(name);
 }
 
@@ -395 +401 @@
             if (isXHTMLDocument()) {
                 QString entity = m_stream.name().toString();
-                UChar c = decodeNamedEntity(entity.toUtf8().constData());
                 if (!m_leafTextNode)
                     enterText();
                 ExceptionCode ec = 0;
-                String str(&c, 1);
                 // qDebug()<<" ------- adding entity "<<str;
-                m_leafTextNode->appendData(str, ec);
+                m_leafTextNode->appendData(decodeNamedEntity(entity), ec);
             }
             break;