Changeset 288025 in webkit

Timestamp:

Jan 14, 2022 12:41:15 PM (6 months ago)

Author:

commit-queue@webkit.org

Message:

gl.texImage2D upload of getUserMedia streams via <video> element fails
​https://bugs.webkit.org/show_bug.cgi?id=230617
<rdar://problem/83407577>

Patch by Kimmo Kinnunen <​kkinnunen@apple.com> on 2022-01-14
Reviewed by Youenn Fablet.

Source/WebCore:

Fix MSE camera to WebGL texture uploads.
Partially revert r280963 for Cocoa MediaPlayer implementatations
that do not have nativeImageForCurrentTime / pixelBufferForCurrentTime.
Turns out MSE does not have these implemented, so currently fall back
to the painting path.

Test: fast/mediastream/getUserMedia-to-canvas.html

• html/canvas/WebGLRenderingContextBase.cpp:

(WebCore::WebGLRenderingContextBase::videoFrameToImage):

LayoutTests:

• fast/mediastream/getUserMedia-to-canvas-expected.txt: Added.
• fast/mediastream/getUserMedia-to-canvas.html: Added.

Add a test to test getting video frame to 2DContext and
WebGL canvas elements.

• webrtc/routines.js:

Add a function to assert that ImageData contains
the simulated mock camera image. This is useful
in verifying that the image is exactly as
expected. The test tests only the default orientation.
Later changes will update the test to address the
camera rotation.

Location:

trunk

Files:

• LayoutTests/ChangeLog (modified) (1 diff)
• LayoutTests/fast/mediastream/getUserMedia-to-canvas-expected.txt (added)
• LayoutTests/fast/mediastream/getUserMedia-to-canvas.html (added)
• LayoutTests/webrtc/routines.js (modified) (1 diff)
• Source/WebCore/ChangeLog (modified) (1 diff)
• Source/WebCore/html/canvas/WebGLRenderingContextBase.cpp (modified) (2 diffs)

trunk/LayoutTests/ChangeLog

@@ +1 @@
+2022-01-14  Kimmo Kinnunen  <kkinnunen@apple.com>
+
+        gl.texImage2D upload of getUserMedia streams via <video> element fails
+        https://bugs.webkit.org/show_bug.cgi?id=230617
+        <rdar://problem/83407577>
+
+        Reviewed by Youenn Fablet.
+
+        * fast/mediastream/getUserMedia-to-canvas-expected.txt: Added.
+        * fast/mediastream/getUserMedia-to-canvas.html: Added.
+        Add a test to test getting video frame to 2DContext and
+        WebGL canvas elements.
+
+        * webrtc/routines.js:
+        Add a function to assert that ImageData contains
+        the simulated mock camera image. This is useful
+        in verifying that the image is exactly as
+        expected. The test tests only the default orientation.
+        Later changes will update the test to address the
+        camera rotation.
+
 2022-01-14  Chris Dumez  <cdumez@apple.com>
 

trunk/LayoutTests/webrtc/routines.js

@@ -298 +298 @@
     }).join('\r\n');
 }
+
+// Returns Uint8Array[4] of RGBA color.
+// p: [x, y] of 0..1 range.
+function getImageDataPixel(imageData, p)
+{
+    let xi = Math.floor(p[0] * imageData.width);
+    let yi = Math.floor(p[1] * imageData.height);
+    let i = (yi * imageData.width + xi) * 4;
+    return imageData.data.slice(i, i + 4);
+}
+
+// Asserts that ImageData instance contains mock camera image rendered by MiniBrowser and WebKitTestRunner.
+// Obtain full camera image of size `width`:
+//  await navigator.mediaDevices.getUserMedia({ video: { width: { exact: width } } });
+function assertImageDataContainsMockCameraImage(imageData)
+{
+    const white = [ 255, 255, 255, 255 ];
+    const yellow = [ 255, 255, 0, 255 ];
+    const cyan = [ 0, 255, 255, 255 ];
+    const lightGreen = [ 0, 128, 0, 255 ];
+
+    let err = 3;
+    assert_array_approx_equals(getImageDataPixel(imageData, [ 0.04, 0.7 ]), white, err, "white rect not found");
+    assert_array_approx_equals(getImageDataPixel(imageData, [ 0.08, 0.7 ]), yellow, err, "yellow rect not found");
+    assert_array_approx_equals(getImageDataPixel(imageData, [ 0.12, 0.7 ]), cyan, err, "cyan rect not found");
+    assert_array_approx_equals(getImageDataPixel(imageData, [ 0.16, 0.7 ]), lightGreen, err, "light green rect not found");
+}

trunk/Source/WebCore/ChangeLog

@@ +1 @@
+2022-01-14  Kimmo Kinnunen  <kkinnunen@apple.com>
+
+        gl.texImage2D upload of getUserMedia streams via <video> element fails
+        https://bugs.webkit.org/show_bug.cgi?id=230617
+        <rdar://problem/83407577>
+
+        Reviewed by Youenn Fablet.
+
+        Fix MSE camera to WebGL texture uploads.
+        Partially revert r280963 for Cocoa MediaPlayer implementatations
+        that do not have nativeImageForCurrentTime / pixelBufferForCurrentTime.
+        Turns out MSE does not have these implemented, so currently fall back
+        to the painting path.
+
+        Test: fast/mediastream/getUserMedia-to-canvas.html
+
+        * html/canvas/WebGLRenderingContextBase.cpp:
+        (WebCore::WebGLRenderingContextBase::videoFrameToImage):
+
 2022-01-14  Chris Dumez  <cdumez@apple.com>
 

trunk/Source/WebCore/html/canvas/WebGLRenderingContextBase.cpp

@@ -5889 +5889 @@
 RefPtr<Image> WebGLRenderingContextBase::videoFrameToImage(HTMLVideoElement* video, BackingStoreCopy backingStoreCopy, const char* functionName)
 {
+    ImageBuffer* imageBuffer = nullptr;
     // FIXME: When texImage2D is passed an HTMLVideoElement, implementations
     // interoperably use the native RGB color values of the video frame (e.g.
@@ -5903 +5904 @@
     // Currently we might be missing an image due to MSE not being able to provide the first requested frame.
     // https://bugs.webkit.org/show_bug.cgi?id=228997
-    if (!nativeImage)
-        return nullptr;
-    IntSize imageSize = nativeImage->size();
-    if (imageSize.isEmpty()) {
-        synthesizeGLError(GraphicsContextGL::INVALID_VALUE, functionName, "video visible size is empty");
-        return nullptr;
-    }
-    FloatRect imageRect { { }, imageSize };
-    ImageBuffer* imageBuffer = m_generatedImageCache.imageBuffer(imageSize, nativeImage->colorSpace(), CompositeOperator::Copy);
+    if (nativeImage) {
+        IntSize imageSize = nativeImage->size();
+        if (imageSize.isEmpty()) {
+            synthesizeGLError(GraphicsContextGL::INVALID_VALUE, functionName, "video visible size is empty");
+            return nullptr;
+        }
+        FloatRect imageRect { { }, imageSize };
+        ImageBuffer* imageBuffer = m_generatedImageCache.imageBuffer(imageSize, nativeImage->colorSpace(), CompositeOperator::Copy);
+        if (!imageBuffer) {
+            synthesizeGLError(GraphicsContextGL::OUT_OF_MEMORY, functionName, "out of memory");
+            return nullptr;
+        }
+        imageBuffer->context().drawNativeImage(*nativeImage, imageRect.size(), imageRect, imageRect, CompositeOperator::Copy);
+    }
+#endif
     if (!imageBuffer) {
-        synthesizeGLError(GraphicsContextGL::OUT_OF_MEMORY, functionName, "out of memory");
-        return nullptr;
-    }
-    imageBuffer->context().drawNativeImage(*nativeImage, imageRect.size(), imageRect, imageRect, CompositeOperator::Copy);
-#else
-    // This is a legacy code path that produces incompatible texture size when the
-    // video visible size is different to the natural size. This should be removed
-    // once all platforms implement nativeImageForCurrentTime().
-    IntSize videoSize { static_cast<int>(video->videoWidth()), static_cast<int>(video->videoHeight()) };
-    auto colorSpace = video->colorSpace();
-    if (!colorSpace)
-        colorSpace = DestinationColorSpace::SRGB();
-    ImageBuffer* imageBuffer = m_generatedImageCache.imageBuffer(videoSize, *colorSpace);
-    if (!imageBuffer) {
-        synthesizeGLError(GraphicsContextGL::OUT_OF_MEMORY, functionName, "out of memory");
-        return nullptr;
-    }
-    video->paintCurrentFrameInContext(imageBuffer->context(), { { }, videoSize });
-#endif
+        // This is a legacy code path that produces incompatible texture size when the
+        // video visible size is different to the natural size. This should be removed
+        // once all platforms implement nativeImageForCurrentTime().
+        IntSize videoSize { static_cast<int>(video->videoWidth()), static_cast<int>(video->videoHeight()) };
+        auto colorSpace = video->colorSpace();
+        if (!colorSpace)
+            colorSpace = DestinationColorSpace::SRGB();
+        imageBuffer = m_generatedImageCache.imageBuffer(videoSize, *colorSpace);
+        if (!imageBuffer) {
+            synthesizeGLError(GraphicsContextGL::OUT_OF_MEMORY, functionName, "out of memory");
+            return nullptr;
+        }
+        video->paintCurrentFrameInContext(imageBuffer->context(), { { }, videoSize });
+    }
     RefPtr<Image> image = imageBuffer->copyImage(backingStoreCopy);
     if (!image) {