Changeset 24052 in webkit

Timestamp:

Jul 6, 2007 3:00:45 AM (17 years ago)

Author:

bdash

Message:

2007-07-06 Jungshik Shin <​jungshik.shin@gmail.com>

Reviewed by Alexey.

• Add UTF-32 encoding support ​http://bugs.webkit.org/show_bug.cgi?id=13415

Test:

• fast/encoding/utf-32-big-endian-bom.html
• fast/encoding/utf-32-big-endian-nobom.xml
• fast/encoding/utf-32-little-endian-bom.html
• fast/encoding/utf-32-little-endian-nobom.xml

• loader/TextResourceDecoder.cpp: (WebCore::TextResourceDecoder::checkForBOM): (WebCore::TextResourceDecoder::checkForHeadCharset):
• platform/TextDecoder.cpp: (WebCore::TextDecoder::checkForBOM):
• platform/TextDecoder.h:
• platform/TextEncoding.cpp: (WebCore::UTF32BigEndianEncoding): (WebCore::UTF32LittleEndianEncoding):
• platform/TextEncoding.h:

2007-07-06 Jungshik Shin <​jungshik.shin@gmail.com>

Reviewed by Alexey.

• test for ​http://bugs.webkit.org/show_bug.cgi?id=13415

• fast/encoding/utf-32-big-endian-bom-expected.txt: Added.
• fast/encoding/utf-32-big-endian-bom.html: Added.
• fast/encoding/utf-32-big-endian-nobom-expected.txt: Added.
• fast/encoding/utf-32-big-endian-nobom.xml: Added.
• fast/encoding/utf-32-little-endian-bom-expected.txt: Added.
• fast/encoding/utf-32-little-endian-bom.html: Added.
• fast/encoding/utf-32-little-endian-nobom-expected.txt: Added.
• fast/encoding/utf-32-little-endian-nobom.xml: Added.

Location:

trunk

Files:

• LayoutTests/ChangeLog (modified) (1 diff)
• LayoutTests/fast/encoding/utf-32-big-endian-bom-expected.txt (added)
• LayoutTests/fast/encoding/utf-32-big-endian-bom.html (added)
• LayoutTests/fast/encoding/utf-32-big-endian-nobom-expected.txt (added)
• LayoutTests/fast/encoding/utf-32-big-endian-nobom.xml (added)
• LayoutTests/fast/encoding/utf-32-little-endian-bom-expected.txt (added)
• LayoutTests/fast/encoding/utf-32-little-endian-bom.html (added)
• LayoutTests/fast/encoding/utf-32-little-endian-nobom-expected.txt (added)
• LayoutTests/fast/encoding/utf-32-little-endian-nobom.xml (added)
• WebCore/ChangeLog (modified) (1 diff)
• WebCore/loader/TextResourceDecoder.cpp (modified) (4 diffs)
• WebCore/platform/TextDecoder.cpp (modified) (1 diff)
• WebCore/platform/TextDecoder.h (modified) (1 diff)
• WebCore/platform/TextEncoding.cpp (modified) (1 diff)
• WebCore/platform/TextEncoding.h (modified) (1 diff)

trunk/LayoutTests/ChangeLog

@@ +1 @@
+2007-07-06  Jungshik Shin <jungshik.shin@gmail.com>
+
+        Reviewed by Alexey.
+
+       - test for http://bugs.webkit.org/show_bug.cgi?id=13415
+
+        * fast/encoding/utf-32-big-endian-bom-expected.txt: Added.
+        * fast/encoding/utf-32-big-endian-bom.html: Added.
+        * fast/encoding/utf-32-big-endian-nobom-expected.txt: Added.
+        * fast/encoding/utf-32-big-endian-nobom.xml: Added.
+        * fast/encoding/utf-32-little-endian-bom-expected.txt: Added.
+        * fast/encoding/utf-32-little-endian-bom.html: Added.
+        * fast/encoding/utf-32-little-endian-nobom-expected.txt: Added.
+        * fast/encoding/utf-32-little-endian-nobom.xml: Added.
+
 2007-07-06  Rob Buis  <buis@kde.org>
 

trunk/WebCore/ChangeLog

@@ +1 @@
+2007-07-06  Jungshik Shin  <jungshik.shin@gmail.com>
+
+        Reviewed by Alexey.
+
+        - Add UTF-32 encoding support 
+          http://bugs.webkit.org/show_bug.cgi?id=13415
+
+        Test:
+         - fast/encoding/utf-32-big-endian-bom.html
+         - fast/encoding/utf-32-big-endian-nobom.xml
+         - fast/encoding/utf-32-little-endian-bom.html
+         - fast/encoding/utf-32-little-endian-nobom.xml
+
+        * loader/TextResourceDecoder.cpp:
+        (WebCore::TextResourceDecoder::checkForBOM):
+        (WebCore::TextResourceDecoder::checkForHeadCharset):
+        * platform/TextDecoder.cpp:
+        (WebCore::TextDecoder::checkForBOM):
+        * platform/TextDecoder.h:
+        * platform/TextEncoding.cpp:
+        (WebCore::UTF32BigEndianEncoding):
+        (WebCore::UTF32LittleEndianEncoding):
+        * platform/TextEncoding.h:
+
 2007-07-06  Holger Hans Peter Freyther  <zecke@selfish.org>
 

trunk/WebCore/loader/TextResourceDecoder.cpp

@@ -347 +347 @@
 void TextResourceDecoder::checkForBOM(const char* data, size_t len)
 {
-    // Check for UTF-16 or UTF-8 BOM mark at the beginning, which is a sure sign of a Unicode encoding.
+    // Check for UTF-16/32 or UTF-8 BOM mark at the beginning, which is a sure sign of a Unicode encoding.
 
     if (m_source == UserChosenEncoding) {
@@ -357 +357 @@
     // Check if we have enough data.
     size_t bufferLength = m_buffer.size();
-    if (bufferLength + len < 3)
+    if (bufferLength + len < 4)
         return;
 
     m_checkedForBOM = true;
 
-    // Extract the first three bytes.
+    // Extract the first four bytes.
     // Handle the case where some of bytes are already in the buffer.
     // The last byte is always guaranteed to not be in the buffer.
@@ -368 +368 @@
     unsigned char c1 = bufferLength >= 1 ? m_buffer[0] : *udata++;
     unsigned char c2 = bufferLength >= 2 ? m_buffer[1] : *udata++;
-    ASSERT(bufferLength < 3);
-    unsigned char c3 = *udata;
+    unsigned char c3 = bufferLength >= 3 ? m_buffer[2] : *udata++;
+    ASSERT(bufferLength < 4);
+    unsigned char c4 = *udata;
 
     // Check for the BOM.
-    if (c1 == 0xFE && c2 == 0xFF)
-        setEncoding(UTF16BigEndianEncoding(), AutoDetectedEncoding);
-    else if (c1 == 0xFF && c2 == 0xFE)
-        setEncoding(UTF16LittleEndianEncoding(), AutoDetectedEncoding);
+    if (c1 == 0xFF && c2 == 0xFE) {
+        if (c3 !=0 || c4 != 0)
+            setEncoding(UTF16LittleEndianEncoding(), AutoDetectedEncoding);
+        else 
+            setEncoding(UTF32LittleEndianEncoding(), AutoDetectedEncoding);
+    }
     else if (c1 == 0xEF && c2 == 0xBB && c3 == 0xBF)
         setEncoding(UTF8Encoding(), AutoDetectedEncoding);
+    else if (c1 == 0xFE && c2 == 0xFF)
+        setEncoding(UTF16BigEndianEncoding(), AutoDetectedEncoding);
+    else if (c1 == 0 && c2 == 0 && c3 == 0xFE && c4 == 0xFF)
+        setEncoding(UTF32BigEndianEncoding(), AutoDetectedEncoding);
 }
 
@@ -520 +527 @@
             } else if (ptr[0] == 0 && ptr[1] == '?' && ptr[2] == 0 && ptr[3] == 'x' && ptr[4] == 0 && ptr[5] == 'm' && ptr[6] == 0 && ptr[7] == 'l') {
                 // UTF-16 without BOM
-                setEncoding(((ptr - m_buffer.data()) % 2) ? "UTF-16LE" : "UTF-16BE", AutoDetectedEncoding);
+                setEncoding(((ptr - m_buffer.data()) % 2) ? UTF16LittleEndianEncoding() : UTF16BigEndianEncoding(), AutoDetectedEncoding);
+                return true;
+            } else if (ptr[0] == 0 && ptr[1] == 0 && ptr[2] == 0 && ptr[3] == '?' && ptr[4] == 0 && ptr[5] == 0 && ptr[6] == 0 && ptr[7] == 'x') {
+                // UTF-32 without BOM
+                setEncoding(((ptr - m_buffer.data()) % 4) ? UTF32LittleEndianEncoding() : UTF32BigEndianEncoding(), AutoDetectedEncoding);
                 return true;
             }

trunk/WebCore/platform/TextDecoder.cpp

@@ -58 +58 @@
     unsigned char c1 = buf1Len ? (--buf1Len, *buf1++) : buf2Len ? (--buf2Len, *buf2++) : 0;
     unsigned char c2 = buf1Len ? (--buf1Len, *buf1++) : buf2Len ? (--buf2Len, *buf2++) : 0;
-    unsigned char c3 = buf2Len ? (--buf2Len, *buf2++) : 0;
+    unsigned char c3 = buf1Len ? (--buf1Len, *buf1++) : buf2Len ? (--buf2Len, *buf2++) : 0;
+    unsigned char c4 = buf2Len ? (--buf2Len, *buf2++) : 0;
 
     const TextEncoding* encodingConsideringBOM = &m_encoding;
-    if (c1 == 0xFF && c2 == 0xFE)
-        encodingConsideringBOM = &UTF16LittleEndianEncoding();
+    bool foundBOM = true;
+    if (c1 == 0xFF && c2 == 0xFE) {
+        if (c3 != 0 || c4 != 0) 
+            encodingConsideringBOM = &UTF16LittleEndianEncoding();
+        else if (numBufferedBytes + length > sizeof(m_bufferedBytes))
+            encodingConsideringBOM = &UTF32LittleEndianEncoding();
+        else
+            foundBOM = false;
+    }
+    else if (c1 == 0xEF && c2 == 0xBB && c3 == 0xBF)
+        encodingConsideringBOM = &UTF8Encoding();
     else if (c1 == 0xFE && c2 == 0xFF)
         encodingConsideringBOM = &UTF16BigEndianEncoding();
-    else if (c1 == 0xEF && c2 == 0xBB && c3 == 0xBF)
-        encodingConsideringBOM = &UTF8Encoding();
-    else if (numBufferedBytes + length <= sizeof(m_bufferedBytes) && !flush) {
+    else if (c1 == 0 && c2 == 0 && c3 == 0xFE && c4 == 0xFF)
+        encodingConsideringBOM = &UTF32BigEndianEncoding();
+    else
+        foundBOM = false;
+    if (!foundBOM && numBufferedBytes + length <= sizeof(m_bufferedBytes) && !flush) {
         // Continue to look for the BOM.
         memcpy(&m_bufferedBytes[numBufferedBytes], data, length);

trunk/WebCore/platform/TextDecoder.h

@@ -57 +57 @@
         bool m_checkedForBOM;
         unsigned char m_numBufferedBytes;
-        unsigned char m_bufferedBytes[2];
+        unsigned char m_bufferedBytes[3];
     };
 

trunk/WebCore/platform/TextEncoding.cpp

@@ -186 +186 @@
 }
 
+const TextEncoding& UTF32BigEndianEncoding()
+{
+    static TextEncoding globalUTF32BigEndianEncoding("UTF-32BE");
+    return globalUTF32BigEndianEncoding;
+}
+
+const TextEncoding& UTF32LittleEndianEncoding()
+{
+    static TextEncoding globalUTF32LittleEndianEncoding("UTF-32LE");
+    return globalUTF32LittleEndianEncoding;
+}
+
+
 const TextEncoding& UTF8Encoding()
 {

trunk/WebCore/platform/TextEncoding.h

@@ -61 +61 @@
     const TextEncoding& UTF16BigEndianEncoding();
     const TextEncoding& UTF16LittleEndianEncoding();
+    const TextEncoding& UTF32BigEndianEncoding();
+    const TextEncoding& UTF32LittleEndianEncoding();
     const TextEncoding& UTF8Encoding();
     const TextEncoding& WindowsLatin1Encoding();