Changeset 282740 in webkit

Timestamp:

Sep 19, 2021, 5:10:37 PM (4 years ago)

Author:

Wenson Hsieh

Message:

Teach WebKit::createShareableBitmap to snapshot video elements
​https://bugs.webkit.org/show_bug.cgi?id=230468

Reviewed by Tim Horton.

Source/WebCore:

Now that createShareableBitmap may return images for video elements, we need to ensure that we explicitly
avoid allowing Live Text in video elements, since doing so will (1) break built-in platform media controls,
which also share the same shadow root, and (2) lead to confusing results when playing or seeking in videos,
since the recognized text falls out of sync with the video frame.

• html/HTMLVideoElement.h:
• page/EventHandler.cpp:

(WebCore::EventHandler::textRecognitionCandidateElement const):

For the above reasons, we refactor logic to check if the currently hovered node is a candidate for text
recognition, such that the video element check is consolidated all in one place.

(WebCore::EventHandler::updateMouseEventTargetNode):
(WebCore::EventHandler::textRecognitionHoverTimerFired):

• page/EventHandler.h:
• rendering/RenderVideo.h:

Source/WebKit:

createShareableBitmap currently only returns a non-null image bitmap for image renderers that have cached
images; notably, this omits video elements. For use in future patches, we should allow this helper function to
handle video elements by snapshotting the current frame of the video.

• WebProcess/WebCoreSupport/ShareableBitmapUtilities.cpp:

(WebKit::createShareableBitmap):

Location:

trunk/Source

Files:

• WebCore/ChangeLog (modified) (1 diff)
• WebCore/html/HTMLVideoElement.h (modified) (1 diff)
• WebCore/page/EventHandler.cpp (modified) (5 diffs)
• WebCore/page/EventHandler.h (modified) (1 diff)
• WebCore/rendering/RenderVideo.h (modified) (1 diff)
• WebKit/ChangeLog (modified) (1 diff)
• WebKit/WebProcess/WebCoreSupport/ShareableBitmapUtilities.cpp (modified) (2 diffs)

trunk/Source/WebCore/ChangeLog

@@ +1 @@
+2021-09-19  Wenson Hsieh  <wenson_hsieh@apple.com>
+
+        Teach `WebKit::createShareableBitmap` to snapshot video elements
+        https://bugs.webkit.org/show_bug.cgi?id=230468
+
+        Reviewed by Tim Horton.
+
+        Now that `createShareableBitmap` may return images for video elements, we need to ensure that we explicitly
+        avoid allowing Live Text in video elements, since doing so will (1) break built-in platform media controls,
+        which also share the same shadow root, and (2) lead to confusing results when playing or seeking in videos,
+        since the recognized text falls out of sync with the video frame.
+
+        * html/HTMLVideoElement.h:
+        * page/EventHandler.cpp:
+        (WebCore::EventHandler::textRecognitionCandidateElement const):
+
+        For the above reasons, we refactor logic to check if the currently hovered node is a candidate for text
+        recognition, such that the video element check is consolidated all in one place.
+
+        (WebCore::EventHandler::updateMouseEventTargetNode):
+        (WebCore::EventHandler::textRecognitionHoverTimerFired):
+        * page/EventHandler.h:
+        * rendering/RenderVideo.h:
+
 2021-09-19  Alan Bujtas  <zalan@apple.com>
 

trunk/Source/WebCore/html/HTMLVideoElement.h

@@ -78 +78 @@
     void paintCurrentFrameInContext(GraphicsContext&, const FloatRect&);
 
-    RefPtr<NativeImage> nativeImageForCurrentTime();
+    WEBCORE_EXPORT RefPtr<NativeImage> nativeImageForCurrentTime();
 
     WEBCORE_EXPORT bool shouldDisplayPosterImage() const;

trunk/Source/WebCore/page/EventHandler.cpp

@@ -64 +64 @@
 #include "HTMLInputElement.h"
 #include "HTMLNames.h"
+#include "HTMLVideoElement.h"
 #include "HitTestRequest.h"
 #include "HitTestResult.h"
@@ -84 +85 @@
 #include "Range.h"
 #include "RenderFrameSet.h"
+#include "RenderImage.h"
 #include "RenderLayer.h"
 #include "RenderLayerScrollableArea.h"
@@ -2513 +2515 @@
 }
 
+RefPtr<Element> EventHandler::textRecognitionCandidateElement() const
+{
+    RefPtr shadowHost = m_elementUnderMouse ? m_elementUnderMouse->shadowHost() : nullptr;
+    if (!shadowHost)
+        return nullptr;
+
+    auto renderer = shadowHost->renderer();
+    if (!is<RenderImage>(renderer))
+        return nullptr;
+
+    if (is<HTMLVideoElement>(*shadowHost))
+        return nullptr;
+
+    return shadowHost;
+}
+
 void EventHandler::updateMouseEventTargetNode(const AtomString& eventType, Node* targetNode, const PlatformMouseEvent& platformMouseEvent, FireMouseOverOut fireMouseOverOut)
 {
@@ -2532 +2550 @@
 #if ENABLE(IMAGE_ANALYSIS)
     if (m_frame.settings().preferInlineTextSelectionInImages()) {
-        if (!m_elementUnderMouse || !is<RenderImage>(m_elementUnderMouse->renderer()))
+        if (!textRecognitionCandidateElement())
             m_textRecognitionHoverTimer.stop();
         else if (!platformMouseEvent.movementDelta().isZero())
@@ -3411 +3429 @@
 void EventHandler::textRecognitionHoverTimerFired()
 {
-    if (!m_elementUnderMouse || !is<RenderImage>(m_elementUnderMouse->renderer()))
+    auto element = this->textRecognitionCandidateElement();
+    if (!element)
         return;
 
     if (auto* page = m_frame.page())
-        page->chrome().client().requestTextRecognition(*m_elementUnderMouse);
+        page->chrome().client().requestTextRecognition(*element);
 }
 

trunk/Source/WebCore/page/EventHandler.h

@@ -364 +364 @@
 #endif
 
+    RefPtr<Element> textRecognitionCandidateElement() const;
+
     bool eventActivatedView(const PlatformMouseEvent&) const;
     bool updateSelectionForMouseDownDispatchingSelectStart(Node*, const VisibleSelection&, TextGranularity);

trunk/Source/WebCore/rendering/RenderVideo.h

@@ -39 +39 @@
     virtual ~RenderVideo();
 
-    HTMLVideoElement& videoElement() const;
+    WEBCORE_EXPORT HTMLVideoElement& videoElement() const;
 
     WEBCORE_EXPORT IntRect videoBox() const;

trunk/Source/WebKit/ChangeLog

@@ +1 @@
+2021-09-19  Wenson Hsieh  <wenson_hsieh@apple.com>
+
+        Teach `WebKit::createShareableBitmap` to snapshot video elements
+        https://bugs.webkit.org/show_bug.cgi?id=230468
+
+        Reviewed by Tim Horton.
+
+        `createShareableBitmap` currently only returns a non-null image bitmap for image renderers that have cached
+        images; notably, this omits video elements. For use in future patches, we should allow this helper function to
+        handle video elements by snapshotting the current frame of the video.
+
+        * WebProcess/WebCoreSupport/ShareableBitmapUtilities.cpp:
+        (WebKit::createShareableBitmap):
+
 2021-09-17  Alex Christensen  <achristensen@webkit.org>
 

trunk/Source/WebKit/WebProcess/WebCoreSupport/ShareableBitmapUtilities.cpp

@@ -37 +37 @@
 #include <WebCore/PlatformScreen.h>
 #include <WebCore/RenderImage.h>
+#include <WebCore/RenderVideo.h>
 
 namespace WebKit {
@@ -71 +72 @@
     }
 
+    if (is<RenderVideo>(renderImage)) {
+        auto& renderVideo = downcast<RenderVideo>(renderImage);
+        Ref video = renderVideo.videoElement();
+        auto image = video->nativeImageForCurrentTime();
+        if (!image)
+            return { };
+
+        auto imageSize = image->size();
+        if (imageSize.isEmpty() || imageSize.width() <= 1 || imageSize.height() <= 1)
+            return { };
+
+        auto bitmap = ShareableBitmap::createShareable(imageSize, { WTFMove(colorSpaceForBitmap) });
+        if (!bitmap)
+            return { };
+
+        auto context = bitmap->createGraphicsContext();
+        if (!context)
+            return { };
+
+        context->drawNativeImage(*image, imageSize, FloatRect { { }, imageSize }, FloatRect { { }, imageSize });
+        return bitmap;
+    }
+
     auto* cachedImage = renderImage.cachedImage();
     if (!cachedImage || cachedImage->errorOccurred())