Changeset 279609 in webkit


Ignore:
Timestamp:
Jul 6, 2021 12:37:30 PM (13 months ago)
Author:
Wenson Hsieh
Message:

[Live Text] Extraneous spaces when copying or translating Chinese and Japanese text in WebKit
​https://bugs.webkit.org/show_bug.cgi?id=227674
rdar://79669807

Reviewed by Tim Horton.

Source/WebCore:

See WebKit/ChangeLog for more details.

Test: fast/images/text-recognition/image-overlay-text-without-leading-whitespace.html

  • html/HTMLElement.cpp:

(WebCore::HTMLElement::updateWithTextRecognitionResult):

For each recognized word in text, only append a trailing newline in the text content of the text container
element if hasLeadingWhitespace is set. Drive-by fix: additionally remove logic to append a line break when
computing and applying CSS transforms to text container elements.

  • platform/TextRecognitionResult.h:

(WebCore::TextRecognitionWordData::TextRecognitionWordData):
(WebCore::TextRecognitionWordData::encode const):
(WebCore::TextRecognitionWordData::decode):

Add a per-word hasLeadingWhitespace flag.

  • testing/Internals.cpp:

(WebCore::makeDataForLine):

  • testing/Internals.h:
  • testing/Internals.idl:

Source/WebKit:

When injecting Live Text into images, we currently assume that all pieces of recognized text from VisionKit are
separated by spaces. Upon selecting Japanese or Chinese text, this causes text selections to include extraneous
spaces between words.

To fix this, we add a bit for each TextRecognitionWordData, to indicate whether that word should be followed by
whitespace in the image overlay. When converting VKWKTextInfo into TextRecognitionWordData, we then set this bit
to true only if the word is the first on a line of text that is preceded by another line, or there exists
whitespace between the word and the previous word on the same line.

  • Platform/cocoa/TextRecognitionUtilities.mm:

(WebKit::makeTextRecognitionResult):

LayoutTests:

Add a layout test to verify that no extraneous spaces are inserted when injecting recognized text into images
without leading whitespace.

  • fast/images/text-recognition/image-overlay-text-without-leading-whitespace-expected.txt: Added.
  • fast/images/text-recognition/image-overlay-text-without-leading-whitespace.html: Added.
Location:
trunk
Files:
2 added
9 edited

Legend:

Unmodified
Added
Removed

  • trunk/LayoutTests/ChangeLog

    r279602 r279609  
     12021-07-06  Wenson Hsieh  <wenson_hsieh@apple.com>
     2
     3        [Live Text] Extraneous spaces when copying or translating Chinese and Japanese text in WebKit
     4        https://bugs.webkit.org/show_bug.cgi?id=227674
     5        rdar://79669807
     6
     7        Reviewed by Tim Horton.
     8
     9        Add a layout test to verify that no extraneous spaces are inserted when injecting recognized text into images
     10        without leading whitespace.
     11
     12        * fast/images/text-recognition/image-overlay-text-without-leading-whitespace-expected.txt: Added.
     13        * fast/images/text-recognition/image-overlay-text-without-leading-whitespace.html: Added.
     14
    1152021-07-06  Youenn Fablet  <youenn@apple.com>
    216

  • trunk/Source/WebCore/ChangeLog

    r279602 r279609  
     12021-07-06  Wenson Hsieh  <wenson_hsieh@apple.com>
     2
     3        [Live Text] Extraneous spaces when copying or translating Chinese and Japanese text in WebKit
     4        https://bugs.webkit.org/show_bug.cgi?id=227674
     5        rdar://79669807
     6
     7        Reviewed by Tim Horton.
     8
     9        See WebKit/ChangeLog for more details.
     10
     11        Test: fast/images/text-recognition/image-overlay-text-without-leading-whitespace.html
     12
     13        * html/HTMLElement.cpp:
     14        (WebCore::HTMLElement::updateWithTextRecognitionResult):
     15
     16        For each recognized word in text, only append a trailing newline in the text content of the text container
     17        element if `hasLeadingWhitespace` is set. Drive-by fix: additionally remove logic to append a line break when
     18        computing and applying CSS transforms to text container elements.
     19
     20        * platform/TextRecognitionResult.h:
     21        (WebCore::TextRecognitionWordData::TextRecognitionWordData):
     22        (WebCore::TextRecognitionWordData::encode const):
     23        (WebCore::TextRecognitionWordData::decode):
     24
     25        Add a per-word `hasLeadingWhitespace` flag.
     26
     27        * testing/Internals.cpp:
     28        (WebCore::makeDataForLine):
     29        * testing/Internals.h:
     30        * testing/Internals.idl:
     31
    1322021-07-06  Youenn Fablet  <youenn@apple.com>
    233

  • trunk/Source/WebCore/html/HTMLElement.cpp

    r279406 r279609  
    13481348    struct TextRecognitionLineElements {
    13491349        Ref<HTMLDivElement> line;
    1350         Vector<Ref<HTMLDivElement>> children;
     1350        Vector<Ref<HTMLElement>> children;
    13511351    };
    13521352
    … …  
    14361436                textContainer->classList().add(imageOverlayTextClass);
    14371437                lineContainer->appendChild(textContainer);
    1438                 textContainer->appendChild(Text::create(document(), makeString('\n', child.text)));
     1438                textContainer->appendChild(Text::create(document(), child.hasLeadingWhitespace ? makeString('\n', child.text) : child.text));
    14391439                lineElements.children.uncheckedAppend(WTFMove(textContainer));
    14401440            }
    … …  
    15681568        }
    15691569
    1570         lineContainer->appendChild(HTMLBRElement::create(document()));
    1571 
    15721570        if (document().isImageDocument())
    15731571            lineContainer->setInlineStyleProperty(CSSPropertyCursor, CSSValueText);

  • trunk/Source/WebCore/platform/TextRecognitionResult.h

    r278575 r279609  
    3939
    4040struct TextRecognitionWordData {
    41     TextRecognitionWordData(const String& theText, FloatQuad&& quad)
     41    TextRecognitionWordData(const String& theText, FloatQuad&& quad, bool leadingWhitespace)
    4242        : text(theText)
    4343        , normalizedQuad(WTFMove(quad))
     44        , hasLeadingWhitespace(leadingWhitespace)
    4445    {
    4546    }
    … …  
    4748    String text;
    4849    FloatQuad normalizedQuad;
     50    bool hasLeadingWhitespace { true };
    4951
    5052    template<class Encoder> void encode(Encoder&) const;
    … …  
    5658    encoder << text;
    5759    encoder << normalizedQuad;
     60    encoder << hasLeadingWhitespace;
    5861}
    5962
    … …  
    7073        return std::nullopt;
    7174
    72     return {{ WTFMove(*text), WTFMove(*normalizedQuad) }};
     75    std::optional<bool> hasLeadingWhitespace;
     76    decoder >> hasLeadingWhitespace;
     77    if (!hasLeadingWhitespace)
     78        return std::nullopt;
     79
     80    return {{ WTFMove(*text), WTFMove(*normalizedQuad), *hasLeadingWhitespace }};
    7381}
    7482

  • trunk/Source/WebCore/testing/Internals.cpp

    r279550 r279609  
    56775677        getQuad<Internals::ImageOverlayLine>(line),
    56785678        line.children.map([](auto& textChild) -> TextRecognitionWordData {
    5679             return { textChild.text, getQuad<Internals::ImageOverlayText>(textChild) };
     5679            return { textChild.text, getQuad<Internals::ImageOverlayText>(textChild), textChild.hasLeadingWhitespace };
    56805680        })
    56815681    };

  • trunk/Source/WebCore/testing/Internals.h

    r279550 r279609  
    884884        RefPtr<DOMPointReadOnly> bottomRight;
    885885        RefPtr<DOMPointReadOnly> bottomLeft;
     886        bool hasLeadingWhitespace { true };
    886887
    887888        ~ImageOverlayText();

  • trunk/Source/WebCore/testing/Internals.idl

    r279550 r279609  
    267267    required DOMPointReadOnly bottomRight;
    268268    required DOMPointReadOnly bottomLeft;
     269    boolean hasLeadingWhitespace = true;
    269270};
    270271

  • trunk/Source/WebKit/ChangeLog

    r279607 r279609  
     12021-07-06  Wenson Hsieh  <wenson_hsieh@apple.com>
     2
     3        [Live Text] Extraneous spaces when copying or translating Chinese and Japanese text in WebKit
     4        https://bugs.webkit.org/show_bug.cgi?id=227674
     5        rdar://79669807
     6
     7        Reviewed by Tim Horton.
     8
     9        When injecting Live Text into images, we currently assume that all pieces of recognized text from VisionKit are
     10        separated by spaces. Upon selecting Japanese or Chinese text, this causes text selections to include extraneous
     11        spaces between words.
     12
     13        To fix this, we add a bit for each TextRecognitionWordData, to indicate whether that word should be followed by
     14        whitespace in the image overlay. When converting VKWKTextInfo into TextRecognitionWordData, we then set this bit
     15        to `true` only if the word is the first on a line of text that is preceded by another line, or there exists
     16        whitespace between the word and the previous word on the same line.
     17
     18        * Platform/cocoa/TextRecognitionUtilities.mm:
     19        (WebKit::makeTextRecognitionResult):
     20
    1212021-07-06  Alex Christensen  <achristensen@webkit.org>
    222

  • trunk/Source/WebKit/Platform/cocoa/TextRecognitionUtilities.mm

    r278804 r279609  
    4848@end
    4949
    50 @interface VKWKTextInfo (Staging_77086105)
    51 @property (nonatomic, readonly) NSArray<VKWKTextInfo *> *children;
    52 @end
    53 
    5450namespace WebKit {
    5551using namespace WebCore;
    … …  
    7470    TextRecognitionResult result;
    7571    result.lines.reserveInitialCapacity(allLines.count);
    76     for (VKWKTextInfo *line in allLines) {
     72
     73    bool isFirstLine = true;
     74    for (VKWKLineInfo *line in allLines) {
    7775        Vector<TextRecognitionWordData> children;
    78         if ([line respondsToSelector:@selector(children)]) {
    79             NSArray<VKWKTextInfo *> *vkChildren = line.children;
    80             children.reserveInitialCapacity(vkChildren.count);
    81             for (VKWKTextInfo *child in vkChildren)
    82                 children.uncheckedAppend({ child.string, floatQuad(child.quad) });
    83         } else
    84             children.append({ line.string, floatQuad(line.quad) });
     76        NSArray<VKWKTextInfo *> *vkChildren = line.children;
     77        children.reserveInitialCapacity(vkChildren.count);
     78
     79        String lineText = line.string;
     80        unsigned searchLocation = 0;
     81        for (VKWKTextInfo *child in vkChildren) {
     82            if (searchLocation >= lineText.length()) {
     83                ASSERT_NOT_REACHED();
     84                continue;
     85            }
     86
     87            String childText = child.string;
     88            auto matchLocation = lineText.find(childText, searchLocation);
     89            if (matchLocation == notFound) {
     90                ASSERT_NOT_REACHED();
     91                continue;
     92            }
     93
     94            bool hasLeadingWhitespace = ([&] {
     95                if (matchLocation == searchLocation)
     96                    return !isFirstLine && !searchLocation;
     97
     98                auto textBeforeMatch = lineText.substring(searchLocation, matchLocation - searchLocation);
     99                return !textBeforeMatch.isEmpty() && isSpaceOrNewline(textBeforeMatch[0]);
     100            })();
     101
     102            searchLocation = matchLocation + childText.length();
     103            children.uncheckedAppend({ WTFMove(childText), floatQuad(child.quad), hasLeadingWhitespace });
     104        }
    85105        result.lines.uncheckedAppend({ floatQuad(line.quad), WTFMove(children) });
     106        isFirstLine = false;
    86107    }
    87108
Note: See TracChangeset for help on using the changeset viewer.