⚠ Archived content — this site is no longer maintained.   Current WebKit documentation is at docs.webkit.org.

Changeset 288504 in webkit


Ignore:
Timestamp:
Jan 24, 2022, 5:55:14 PM (5 years ago)
Author:
Russell Epstein
Message:

Cherry-pick r288025. rdar://problem/83407577

gl.texImage2D upload of getUserMedia streams via <video> element fails
​https://bugs.webkit.org/show_bug.cgi?id=230617
<rdar://problem/83407577>

Patch by Kimmo Kinnunen <​kkinnunen@apple.com> on 2022-01-14
Reviewed by Youenn Fablet.

Source/WebCore:

Fix MSE camera to WebGL texture uploads.
Partially revert r280963 for Cocoa MediaPlayer implementatations
that do not have nativeImageForCurrentTime / pixelBufferForCurrentTime.
Turns out MSE does not have these implemented, so currently fall back
to the painting path.

Test: fast/mediastream/getUserMedia-to-canvas.html

  • html/canvas/WebGLRenderingContextBase.cpp: (WebCore::WebGLRenderingContextBase::videoFrameToImage):

LayoutTests:

  • fast/mediastream/getUserMedia-to-canvas-expected.txt: Added.
  • fast/mediastream/getUserMedia-to-canvas.html: Added. Add a test to test getting video frame to 2DContext and WebGL canvas elements.
  • webrtc/routines.js: Add a function to assert that ImageData contains the simulated mock camera image. This is useful in verifying that the image is exactly as expected. The test tests only the default orientation. Later changes will update the test to address the camera rotation.

git-svn-id: ​https://svn.webkit.org/repository/webkit/trunk@288025 268f45cc-cd09-0410-ab3c-d52691b4dbfc

Location:
branches/safari-613-branch
Files:
2 added
4 edited

Legend:

Unmodified
Added
Removed

  • branches/safari-613-branch/LayoutTests/ChangeLog

    r288503 r288504  
     12022-01-24  Alan Coon  <alancoon@apple.com>
     2
     3        Cherry-pick r288025. rdar://problem/83407577
     4
     5    gl.texImage2D upload of getUserMedia streams via <video> element fails
     6    https://bugs.webkit.org/show_bug.cgi?id=230617
     7    <rdar://problem/83407577>
     8   
     9    Patch by Kimmo Kinnunen <kkinnunen@apple.com> on 2022-01-14
     10    Reviewed by Youenn Fablet.
     11   
     12    Source/WebCore:
     13   
     14    Fix MSE camera to WebGL texture uploads.
     15    Partially revert r280963 for Cocoa MediaPlayer implementatations
     16    that do not have nativeImageForCurrentTime / pixelBufferForCurrentTime.
     17    Turns out MSE does not have these implemented, so currently fall back
     18    to the painting path.
     19   
     20    Test: fast/mediastream/getUserMedia-to-canvas.html
     21   
     22    * html/canvas/WebGLRenderingContextBase.cpp:
     23    (WebCore::WebGLRenderingContextBase::videoFrameToImage):
     24   
     25    LayoutTests:
     26   
     27    * fast/mediastream/getUserMedia-to-canvas-expected.txt: Added.
     28    * fast/mediastream/getUserMedia-to-canvas.html: Added.
     29    Add a test to test getting video frame to 2DContext and
     30    WebGL canvas elements.
     31   
     32    * webrtc/routines.js:
     33    Add a function to assert that ImageData contains
     34    the simulated mock camera image. This is useful
     35    in verifying that the image is exactly as
     36    expected. The test tests only the default orientation.
     37    Later changes will update the test to address the
     38    camera rotation.
     39   
     40    git-svn-id: https://svn.webkit.org/repository/webkit/trunk@288025 268f45cc-cd09-0410-ab3c-d52691b4dbfc
     41
     42    2022-01-14  Kimmo Kinnunen  <kkinnunen@apple.com>
     43
     44            gl.texImage2D upload of getUserMedia streams via <video> element fails
     45            https://bugs.webkit.org/show_bug.cgi?id=230617
     46            <rdar://problem/83407577>
     47
     48            Reviewed by Youenn Fablet.
     49
     50            * fast/mediastream/getUserMedia-to-canvas-expected.txt: Added.
     51            * fast/mediastream/getUserMedia-to-canvas.html: Added.
     52            Add a test to test getting video frame to 2DContext and
     53            WebGL canvas elements.
     54
     55            * webrtc/routines.js:
     56            Add a function to assert that ImageData contains
     57            the simulated mock camera image. This is useful
     58            in verifying that the image is exactly as
     59            expected. The test tests only the default orientation.
     60            Later changes will update the test to address the
     61            camera rotation.
     62
    1632022-01-24  Alan Coon  <alancoon@apple.com>
    264

  • branches/safari-613-branch/LayoutTests/webrtc/routines.js

    r267504 r288504  
    298298    }).join('\r\n');
    299299}
     300
     301// Returns Uint8Array[4] of RGBA color.
     302// p: [x, y] of 0..1 range.
     303function getImageDataPixel(imageData, p)
     304{
     305    let xi = Math.floor(p[0] * imageData.width);
     306    let yi = Math.floor(p[1] * imageData.height);
     307    let i = (yi * imageData.width + xi) * 4;
     308    return imageData.data.slice(i, i + 4);
     309}
     310
     311// Asserts that ImageData instance contains mock camera image rendered by MiniBrowser and WebKitTestRunner.
     312// Obtain full camera image of size `width`:
     313//  await navigator.mediaDevices.getUserMedia({ video: { width: { exact: width } } });
     314function assertImageDataContainsMockCameraImage(imageData)
     315{
     316    const white = [ 255, 255, 255, 255 ];
     317    const yellow = [ 255, 255, 0, 255 ];
     318    const cyan = [ 0, 255, 255, 255 ];
     319    const lightGreen = [ 0, 128, 0, 255 ];
     320
     321    let err = 3;
     322    assert_array_approx_equals(getImageDataPixel(imageData, [ 0.04, 0.7 ]), white, err, "white rect not found");
     323    assert_array_approx_equals(getImageDataPixel(imageData, [ 0.08, 0.7 ]), yellow, err, "yellow rect not found");
     324    assert_array_approx_equals(getImageDataPixel(imageData, [ 0.12, 0.7 ]), cyan, err, "cyan rect not found");
     325    assert_array_approx_equals(getImageDataPixel(imageData, [ 0.16, 0.7 ]), lightGreen, err, "light green rect not found");
     326}

  • branches/safari-613-branch/Source/WebCore/ChangeLog

    r288503 r288504  
     12022-01-24  Alan Coon  <alancoon@apple.com>
     2
     3        Cherry-pick r288025. rdar://problem/83407577
     4
     5    gl.texImage2D upload of getUserMedia streams via <video> element fails
     6    https://bugs.webkit.org/show_bug.cgi?id=230617
     7    <rdar://problem/83407577>
     8   
     9    Patch by Kimmo Kinnunen <kkinnunen@apple.com> on 2022-01-14
     10    Reviewed by Youenn Fablet.
     11   
     12    Source/WebCore:
     13   
     14    Fix MSE camera to WebGL texture uploads.
     15    Partially revert r280963 for Cocoa MediaPlayer implementatations
     16    that do not have nativeImageForCurrentTime / pixelBufferForCurrentTime.
     17    Turns out MSE does not have these implemented, so currently fall back
     18    to the painting path.
     19   
     20    Test: fast/mediastream/getUserMedia-to-canvas.html
     21   
     22    * html/canvas/WebGLRenderingContextBase.cpp:
     23    (WebCore::WebGLRenderingContextBase::videoFrameToImage):
     24   
     25    LayoutTests:
     26   
     27    * fast/mediastream/getUserMedia-to-canvas-expected.txt: Added.
     28    * fast/mediastream/getUserMedia-to-canvas.html: Added.
     29    Add a test to test getting video frame to 2DContext and
     30    WebGL canvas elements.
     31   
     32    * webrtc/routines.js:
     33    Add a function to assert that ImageData contains
     34    the simulated mock camera image. This is useful
     35    in verifying that the image is exactly as
     36    expected. The test tests only the default orientation.
     37    Later changes will update the test to address the
     38    camera rotation.
     39   
     40    git-svn-id: https://svn.webkit.org/repository/webkit/trunk@288025 268f45cc-cd09-0410-ab3c-d52691b4dbfc
     41
     42    2022-01-14  Kimmo Kinnunen  <kkinnunen@apple.com>
     43
     44            gl.texImage2D upload of getUserMedia streams via <video> element fails
     45            https://bugs.webkit.org/show_bug.cgi?id=230617
     46            <rdar://problem/83407577>
     47
     48            Reviewed by Youenn Fablet.
     49
     50            Fix MSE camera to WebGL texture uploads.
     51            Partially revert r280963 for Cocoa MediaPlayer implementatations
     52            that do not have nativeImageForCurrentTime / pixelBufferForCurrentTime.
     53            Turns out MSE does not have these implemented, so currently fall back
     54            to the painting path.
     55
     56            Test: fast/mediastream/getUserMedia-to-canvas.html
     57
     58            * html/canvas/WebGLRenderingContextBase.cpp:
     59            (WebCore::WebGLRenderingContextBase::videoFrameToImage):
     60
    1612022-01-24  Alan Coon  <alancoon@apple.com>
    262

  • branches/safari-613-branch/Source/WebCore/html/canvas/WebGLRenderingContextBase.cpp

    r286943 r288504  
    59105910RefPtr<Image> WebGLRenderingContextBase::videoFrameToImage(HTMLVideoElement* video, BackingStoreCopy backingStoreCopy, const char* functionName)
    59115911{
     5912    ImageBuffer* imageBuffer = nullptr;
    59125913    // FIXME: When texImage2D is passed an HTMLVideoElement, implementations
    59135914    // interoperably use the native RGB color values of the video frame (e.g.
    … …  
    59245925    // Currently we might be missing an image due to MSE not being able to provide the first requested frame.
    59255926    // https://bugs.webkit.org/show_bug.cgi?id=228997
    5926     if (!nativeImage)
    5927         return nullptr;
    5928     IntSize imageSize = nativeImage->size();
    5929     if (imageSize.isEmpty()) {
    5930         synthesizeGLError(GraphicsContextGL::INVALID_VALUE, functionName, "video visible size is empty");
    5931         return nullptr;
    5932     }
    5933     FloatRect imageRect { { }, imageSize };
    5934     ImageBuffer* imageBuffer = m_generatedImageCache.imageBuffer(imageSize, nativeImage->colorSpace(), CompositeOperator::Copy);
     5927    if (nativeImage) {
     5928        IntSize imageSize = nativeImage->size();
     5929        if (imageSize.isEmpty()) {
     5930            synthesizeGLError(GraphicsContextGL::INVALID_VALUE, functionName, "video visible size is empty");
     5931            return nullptr;
     5932        }
     5933        FloatRect imageRect { { }, imageSize };
     5934        ImageBuffer* imageBuffer = m_generatedImageCache.imageBuffer(imageSize, nativeImage->colorSpace(), CompositeOperator::Copy);
     5935        if (!imageBuffer) {
     5936            synthesizeGLError(GraphicsContextGL::OUT_OF_MEMORY, functionName, "out of memory");
     5937            return nullptr;
     5938        }
     5939        imageBuffer->context().drawNativeImage(*nativeImage, imageRect.size(), imageRect, imageRect, CompositeOperator::Copy);
     5940    }
     5941#endif
    59355942    if (!imageBuffer) {
    5936         synthesizeGLError(GraphicsContextGL::OUT_OF_MEMORY, functionName, "out of memory");
    5937         return nullptr;
    5938     }
    5939     imageBuffer->context().drawNativeImage(*nativeImage, imageRect.size(), imageRect, imageRect, CompositeOperator::Copy);
    5940 #else
    5941     // This is a legacy code path that produces incompatible texture size when the
    5942     // video visible size is different to the natural size. This should be removed
    5943     // once all platforms implement nativeImageForCurrentTime().
    5944     IntSize videoSize { static_cast<int>(video->videoWidth()), static_cast<int>(video->videoHeight()) };
    5945     auto colorSpace = video->colorSpace();
    5946     if (!colorSpace)
    5947         colorSpace = DestinationColorSpace::SRGB();
    5948     ImageBuffer* imageBuffer = m_generatedImageCache.imageBuffer(videoSize, *colorSpace);
    5949     if (!imageBuffer) {
    5950         synthesizeGLError(GraphicsContextGL::OUT_OF_MEMORY, functionName, "out of memory");
    5951         return nullptr;
    5952     }
    5953     video->paintCurrentFrameInContext(imageBuffer->context(), { { }, videoSize });
    5954 #endif
     5943        // This is a legacy code path that produces incompatible texture size when the
     5944        // video visible size is different to the natural size. This should be removed
     5945        // once all platforms implement nativeImageForCurrentTime().
     5946        IntSize videoSize { static_cast<int>(video->videoWidth()), static_cast<int>(video->videoHeight()) };
     5947        auto colorSpace = video->colorSpace();
     5948        if (!colorSpace)
     5949            colorSpace = DestinationColorSpace::SRGB();
     5950        imageBuffer = m_generatedImageCache.imageBuffer(videoSize, *colorSpace);
     5951        if (!imageBuffer) {
     5952            synthesizeGLError(GraphicsContextGL::OUT_OF_MEMORY, functionName, "out of memory");
     5953            return nullptr;
     5954        }
     5955        video->paintCurrentFrameInContext(imageBuffer->context(), { { }, videoSize });
     5956    }
    59555957    RefPtr<Image> image = imageBuffer->copyImage(backingStoreCopy);
    59565958    if (!image) {
Note: See TracChangeset for help on using the changeset viewer.