Changeset 287863 in webkit

Timestamp:

Jan 10, 2022, 3:48:40 PM (5 years ago)

Author:

Wenson Hsieh

Message:

Add support for a -shouldWrap property when injecting Live Text
​https://bugs.webkit.org/show_bug.cgi?id=235035
rdar://85139146

Reviewed by Tim Horton.

Source/WebCore:

Add support for a new property on VKWKLineInfo to indicate whether or not the line should wrap. If set to true,
we will avoid adding a br element at the end of the div.image-overlay-line that corresponds to the line of
recognized text in the image overlay; currently, we always add this trailing br.

Test: fast/images/text-recognition/image-overlay-line-wrapping.html

• dom/ImageOverlay.cpp:

(WebCore::ImageOverlay::updateSubtree):

Consult the flag when converting each TextRecognitionLineData into shadow DOM elements.

• platform/TextRecognitionResult.h:

Add the new boolean flag to TextRecognitionLineData, and add encoding/decoding support.

(WebCore::TextRecognitionLineData::TextRecognitionLineData):
(WebCore::TextRecognitionLineData::encode const):
(WebCore::TextRecognitionLineData::decode):

• testing/Internals.cpp:

(WebCore::makeDataForLine):

Add a way to specify a value for shouldWrap, when using internals to inject image overlay content in layout
and API tests.

• testing/Internals.h:
• testing/Internals.idl:

Source/WebCore/PAL:

Add a forward staging declaration for the new SPI property.

• pal/spi/cocoa/VisionKitCoreSPI.h:

Source/WebKit:

Consult the new SPI property (behind a runtime selector check), and pass the boolean flag through the
TextRecognitionResult.

• Platform/cocoa/TextRecognitionUtilities.mm:

(WebKit::makeTextRecognitionResult):

LayoutTests:

Add a layout test to exercise the changes. See WebCore/ChangeLog for more details.

• fast/images/text-recognition/image-overlay-line-wrapping-expected.txt: Added.
• fast/images/text-recognition/image-overlay-line-wrapping.html: Added.

Location:

trunk

Files:

• LayoutTests/ChangeLog (modified) (1 diff)
• LayoutTests/fast/images/text-recognition/image-overlay-line-wrapping-expected.txt (added)
• LayoutTests/fast/images/text-recognition/image-overlay-line-wrapping.html (added)
• Source/WebCore/ChangeLog (modified) (1 diff)
• Source/WebCore/PAL/ChangeLog (modified) (1 diff)
• Source/WebCore/PAL/pal/spi/cocoa/VisionKitCoreSPI.h (modified) (1 diff)
• Source/WebCore/dom/ImageOverlay.cpp (modified) (5 diffs)
• Source/WebCore/platform/TextRecognitionResult.h (modified) (4 diffs)
• Source/WebCore/testing/Internals.cpp (modified) (1 diff)
• Source/WebCore/testing/Internals.h (modified) (1 diff)
• Source/WebCore/testing/Internals.idl (modified) (1 diff)
• Source/WebKit/ChangeLog (modified) (1 diff)
• Source/WebKit/Platform/cocoa/TextRecognitionUtilities.mm (modified) (1 diff)

trunk/LayoutTests/ChangeLog

@@ +1 @@
+2022-01-10  Wenson Hsieh  <wenson_hsieh@apple.com>
+
+        Add support for a -shouldWrap property when injecting Live Text
+        https://bugs.webkit.org/show_bug.cgi?id=235035
+        rdar://85139146
+
+        Reviewed by Tim Horton.
+
+        Add a layout test to exercise the changes. See WebCore/ChangeLog for more details.
+
+        * fast/images/text-recognition/image-overlay-line-wrapping-expected.txt: Added.
+        * fast/images/text-recognition/image-overlay-line-wrapping.html: Added.
+
 2022-01-10  Chris Dumez  <cdumez@apple.com>
 

trunk/Source/WebCore/ChangeLog

@@ +1 @@
+2022-01-10  Wenson Hsieh  <wenson_hsieh@apple.com>
+
+        Add support for a -shouldWrap property when injecting Live Text
+        https://bugs.webkit.org/show_bug.cgi?id=235035
+        rdar://85139146
+
+        Reviewed by Tim Horton.
+
+        Add support for a new property on VKWKLineInfo to indicate whether or not the line should wrap. If set to true,
+        we will avoid adding a `br` element at the end of the `div.image-overlay-line` that corresponds to the line of
+        recognized text in the image overlay; currently, we always add this trailing `br`.
+
+        Test: fast/images/text-recognition/image-overlay-line-wrapping.html
+
+        * dom/ImageOverlay.cpp:
+        (WebCore::ImageOverlay::updateSubtree):
+
+        Consult the flag when converting each TextRecognitionLineData into shadow DOM elements.
+
+        * platform/TextRecognitionResult.h:
+
+        Add the new boolean flag to TextRecognitionLineData, and add encoding/decoding support.
+
+        (WebCore::TextRecognitionLineData::TextRecognitionLineData):
+        (WebCore::TextRecognitionLineData::encode const):
+        (WebCore::TextRecognitionLineData::decode):
+        * testing/Internals.cpp:
+        (WebCore::makeDataForLine):
+
+        Add a way to specify a value for `shouldWrap`, when using `internals` to inject image overlay content in layout
+        and API tests.
+
+        * testing/Internals.h:
+        * testing/Internals.idl:
+
 2022-01-10  Eric Carlson  <eric.carlson@apple.com>
 

trunk/Source/WebCore/PAL/ChangeLog

@@ +1 @@
+2022-01-10  Wenson Hsieh  <wenson_hsieh@apple.com>
+
+        Add support for a -shouldWrap property when injecting Live Text
+        https://bugs.webkit.org/show_bug.cgi?id=235035
+        rdar://85139146
+
+        Reviewed by Tim Horton.
+
+        Add a forward staging declaration for the new SPI property.
+
+        * pal/spi/cocoa/VisionKitCoreSPI.h:
+
 2022-01-10  Alex Christensen  <achristensen@webkit.org>
 

trunk/Source/WebCore/PAL/pal/spi/cocoa/VisionKitCoreSPI.h

@@ -147 +147 @@
 #endif
 
+@interface VKWKLineInfo (Staging_85139101)
+@property (nonatomic, readonly) BOOL shouldWrap;
+@end
+
 #if USE(APPLE_INTERNAL_SDK)
 #include <WebKitAdditions/VisionKitCoreSPIAdditions.h>

trunk/Source/WebCore/dom/ImageOverlay.cpp

@@ -193 +193 @@
     Ref<HTMLDivElement> line;
     Vector<Ref<HTMLElement>> children;
+    RefPtr<HTMLBRElement> lineBreak;
 };
 
@@ -254 +255 @@
 
             ASSERT(classes.contains(imageOverlayLineClass()));
-            LineElements lineElements { childElement, { } };
+            Vector<Ref<HTMLElement>> lineChildren;
             for (auto& text : childrenOfType<HTMLDivElement>(childElement))
-                lineElements.children.append(text);
-            elements.lines.append(WTFMove(lineElements));
+                lineChildren.append(text);
+            elements.lines.append({ childElement, WTFMove(lineChildren), childrenOfType<HTMLBRElement>(childElement).first() });
         }
 
@@ -271 +272 @@
 
             for (size_t lineIndex = 0; lineIndex < result.lines.size(); ++lineIndex) {
-                auto& childResults = result.lines[lineIndex].children;
-                auto& childTextElements = elements.lines[lineIndex].children;
+                auto& lineResult = result.lines[lineIndex];
+                auto& childResults = lineResult.children;
+
+                auto& lineElements = elements.lines[lineIndex];
+                auto& childTextElements = lineElements.children;
+                if (lineResult.shouldWrap != !lineElements.lineBreak)
+                    return false;
+
                 if (childResults.size() != childTextElements.size())
                     return false;
@@ -318 +325 @@
             lineContainer->classList().add(imageOverlayLineClass());
             rootContainer->appendChild(lineContainer);
-            LineElements lineElements { lineContainer, { } };
+            LineElements lineElements { lineContainer, { }, { } };
             lineElements.children.reserveInitialCapacity(line.children.size());
             for (size_t childIndex = 0; childIndex < line.children.size(); ++childIndex) {
@@ -329 +336 @@
             }
 
-            lineContainer->appendChild(HTMLBRElement::create(document.get()));
+            if (!line.shouldWrap) {
+                lineElements.lineBreak = HTMLBRElement::create(document.get());
+                lineContainer->appendChild(*lineElements.lineBreak);
+            }
+
             elements.lines.uncheckedAppend(WTFMove(lineElements));
         }

trunk/Source/WebCore/platform/TextRecognitionResult.h

@@ -82 +82 @@
 
 struct TextRecognitionLineData {
-    TextRecognitionLineData(FloatQuad&& quad, Vector<TextRecognitionWordData>&& theChildren)
+    TextRecognitionLineData(FloatQuad&& quad, Vector<TextRecognitionWordData>&& theChildren, bool wrap)
         : normalizedQuad(WTFMove(quad))
         , children(WTFMove(theChildren))
+        , shouldWrap(wrap)
     {
     }
@@ -90 +91 @@
     FloatQuad normalizedQuad;
     Vector<TextRecognitionWordData> children;
+    bool shouldWrap { false };
 
     template<class Encoder> void encode(Encoder&) const;
@@ -115 +117 @@
     encoder << normalizedQuad;
     encoder << children;
+    encoder << shouldWrap;
 }
 
@@ -129 +132 @@
         return std::nullopt;
 
-    return {{ WTFMove(*normalizedQuad), WTFMove(*children) }};
+    std::optional<bool> shouldWrap;
+    decoder >> shouldWrap;
+    if (!shouldWrap)
+        return std::nullopt;
+
+    return { { WTFMove(*normalizedQuad), WTFMove(*children), *shouldWrap } };
 }
 

trunk/Source/WebCore/testing/Internals.cpp

@@ -5770 +5770 @@
         line.children.map([](auto& textChild) -> TextRecognitionWordData {
             return { textChild.text, getQuad<Internals::ImageOverlayText>(textChild), textChild.hasLeadingWhitespace };
-        })
+        }),
+        line.shouldWrap
     };
 }

trunk/Source/WebCore/testing/Internals.h

@@ -923 +923 @@
         RefPtr<DOMPointReadOnly> bottomLeft;
         Vector<ImageOverlayText> children;
+        bool shouldWrap { false };
 
         ~ImageOverlayLine();

trunk/Source/WebCore/testing/Internals.idl

@@ -293 +293 @@
     required DOMPointReadOnly bottomLeft;
     sequence<ImageOverlayText> children;
+    boolean shouldWrap = false;
 };
 

trunk/Source/WebKit/ChangeLog

@@ +1 @@
+2022-01-10  Wenson Hsieh  <wenson_hsieh@apple.com>
+
+        Add support for a -shouldWrap property when injecting Live Text
+        https://bugs.webkit.org/show_bug.cgi?id=235035
+        rdar://85139146
+
+        Reviewed by Tim Horton.
+
+        Consult the new SPI property (behind a runtime selector check), and pass the boolean flag through the
+        TextRecognitionResult.
+
+        * Platform/cocoa/TextRecognitionUtilities.mm:
+        (WebKit::makeTextRecognitionResult):
+
 2022-01-10  Alex Christensen  <achristensen@webkit.org>
 

trunk/Source/WebKit/Platform/cocoa/TextRecognitionUtilities.mm

@@ -115 +115 @@
             children.uncheckedAppend({ WTFMove(childText), floatQuad(child.quad), hasLeadingWhitespace });
         }
-        result.lines.uncheckedAppend({ floatQuad(line.quad), WTFMove(children) });
+        result.lines.uncheckedAppend({
+            floatQuad(line.quad),
+            WTFMove(children),
+            [line respondsToSelector:@selector(shouldWrap)] && [line shouldWrap]
+        });
         isFirstLine = false;
     }