Changeset 239319 in webkit


Ignore:
Timestamp:
Dec 17, 2018 9:31:22 PM (5 years ago)
Author:
eric.carlson@apple.com
Message:

[MediaStream] A stream's first video frame should be rendered
​https://bugs.webkit.org/show_bug.cgi?id=192629
<rdar://problem/46664353>

Reviewed by Youenn Fablet.

Source/WebCore:

Test: fast/mediastream/media-stream-renders-first-frame.html

  • platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.h:
  • platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.mm:

(WebCore::MediaPlayerPrivateMediaStreamAVFObjC::enqueueVideoSample):
(WebCore::MediaPlayerPrivateMediaStreamAVFObjC::ensureLayers):
(WebCore::MediaPlayerPrivateMediaStreamAVFObjC::currentDisplayMode const):
(WebCore::MediaPlayerPrivateMediaStreamAVFObjC::updateDisplayMode):
(WebCore::MediaPlayerPrivateMediaStreamAVFObjC::play):
(WebCore::MediaPlayerPrivateMediaStreamAVFObjC::currentReadyState):
(WebCore::MediaPlayerPrivateMediaStreamAVFObjC::characteristicsChanged):
(WebCore::MediaPlayerPrivateMediaStreamAVFObjC::checkSelectedVideoTrack):
(WebCore::MediaPlayerPrivateMediaStreamAVFObjC::paintCurrentFrameInContext):

  • platform/mediastream/RealtimeMediaSource.cpp:

(WebCore::RealtimeMediaSource::size const):

  • platform/mediastream/mac/AVVideoCaptureSource.mm:

(WebCore::AVVideoCaptureSource::processNewFrame):

  • platform/mediastream/mac/RealtimeIncomingVideoSourceCocoa.mm:

(WebCore::RealtimeIncomingVideoSourceCocoa::processNewSample):

LayoutTests:

  • fast/mediastream/MediaStream-video-element-displays-buffer.html: Updated.
  • fast/mediastream/MediaStream-video-element-video-tracks-disabled-then-enabled-expected.txt: Ditto.
  • fast/mediastream/MediaStream-video-element-video-tracks-disabled-then-enabled.html: Ditto.
  • fast/mediastream/media-stream-renders-first-frame-expected.txt: Added.
  • fast/mediastream/media-stream-renders-first-frame.html: Added.
  • http/tests/media/media-stream/getusermedia-with-canvas-expected.txt: Removed.
  • http/tests/media/media-stream/getusermedia-with-canvas.html: Removed.
Location:
trunk
Files:
2 added
2 deleted
12 edited

Legend:

Unmodified
Added
Removed

  • trunk/LayoutTests/ChangeLog

    r239314 r239319  
     12018-12-17  Eric Carlson  <eric.carlson@apple.com>
     2
     3        [MediaStream] A stream's first video frame should be rendered
     4        https://bugs.webkit.org/show_bug.cgi?id=192629
     5        <rdar://problem/46664353>
     6
     7        Reviewed by Youenn Fablet.
     8
     9        * fast/mediastream/MediaStream-video-element-displays-buffer.html: Updated.
     10        * fast/mediastream/MediaStream-video-element-video-tracks-disabled-then-enabled-expected.txt: Ditto.
     11        * fast/mediastream/MediaStream-video-element-video-tracks-disabled-then-enabled.html: Ditto.
     12        * fast/mediastream/media-stream-renders-first-frame-expected.txt: Added.
     13        * fast/mediastream/media-stream-renders-first-frame.html: Added.
     14        * http/tests/media/media-stream/getusermedia-with-canvas-expected.txt: Removed.
     15        * http/tests/media/media-stream/getusermedia-with-canvas.html: Removed.
     16
    1172018-12-17  Wenson Hsieh  <wenson_hsieh@apple.com>
    218

  • trunk/LayoutTests/fast/mediastream/MediaStream-video-element-displays-buffer.html

    r214951 r239319  
    33<head>
    44    <script src="../../resources/js-test-pre.js"></script>
    5     <script src="./resources/getUserMedia-helper.js"></script>
    65</head>
    76<body onload="start()">
    … …  
    8685        debug(`<br> === checking pixels from ${!currentTest ? "front" : "back"} camera ===`);
    8786        let constraints = {video : !currentTest ? true : {facingMode: "environment"}};
    88         getUserMedia("allow", constraints, setupVideoElement);
     87        navigator.mediaDevices.getUserMedia(constraints).then(setupVideoElement);
    8988    }
    9089   
    … …  
    9291    {
    9392        description("Tests that the stream displays captured buffers to the video element.");
     93        if (window.testRunner)
     94            testRunner.setUserMediaPermission(true);
    9495
    9596        videos = Array.from(document.getElementsByTagName('video'));

  • trunk/LayoutTests/fast/mediastream/MediaStream-video-element-video-tracks-disabled-then-enabled-expected.txt

    r214951 r239319  
    88
    99 === beginning round of pixel tests ===
    10 PASS pixel was black.
     10PASS pixel was white.
    1111
    1212 === all video tracks disabled ===
    1313PASS pixel was black.
    1414
    15  === video track reenabled, should NOT render current frame ===
    16 PASS pixel was black.
     15 === video track reenabled, should render current frame ===
     16PASS pixel was white.
    1717
    1818 ===== play video =====

  • trunk/LayoutTests/fast/mediastream/MediaStream-video-element-video-tracks-disabled-then-enabled.html

    r213880 r239319  
    33<head>
    44    <script src="../../resources/js-test-pre.js"></script>
    5     <script src="./resources/getUserMedia-helper.js"></script>
    65</head>
    76<body onload="start()">
    … …  
    1514    let mediaStream;
    1615    let video;
    17     let havePlayed = false;
    1816   
    1917    let buffer;
    … …  
    3634    function canvasShouldBeBlack()
    3735    {
    38         return !(mediaStream.getVideoTracks()[0].enabled && havePlayed);
     36        return !mediaStream.getVideoTracks()[0].enabled;
    3937    }
    4038   
    … …  
    6260            debug('<br> ===== play video =====');
    6361            evalAndLog('video.play()');
    64             havePlayed = true;
    6562            beginTestRound();
    6663        } else {
    … …  
    7471    {
    7572        mediaStream.getVideoTracks()[0].enabled = true;
    76         debug(`<br> === video track reenabled, should${havePlayed ? "" : " NOT"} render current frame ===`);
     73        debug(`<br> === video track reenabled, should render current frame ===`);
    7774
    7875        // The video is not guaranteed to render non-black frames before the canvas is drawn to and the pixels are checked.
    … …  
    124121    {
    125122        description("Tests that re-enabling a video MediaStreamTrack when all tracks were previously disabled causes captured media to display.");
     123        if (window.testRunner)
     124            testRunner.setUserMediaPermission(true);
    126125
    127126        video = document.querySelector('video');
    128127        video.addEventListener('canplay', canplay);
    129128
    130         getUserMedia("allow", {video:true}, setupVideoElementWithStream);
     129        navigator.mediaDevices.getUserMedia({ video : true })
     130            .then((stream) => {
     131                mediaStream = stream;
     132                testPassed('mediaDevices.getUserMedia generated a stream successfully.');
     133                evalAndLog('video.srcObject = mediaStream');
     134            });
    131135    }
    132136

  • trunk/LayoutTests/platform/gtk/TestExpectations

    r239156 r239319  
    611611webkit.org/b/79203 fast/mediastream/MediaStream-video-element-displays-buffer.html [ Failure ]
    612612webkit.org/b/79203 fast/mediastream/RTCPeerConnection-stats.html [ Timeout Crash ]
    613 webkit.org/b/79203 http/tests/media/media-stream/getusermedia-with-canvas.html [ Timeout ]
    614613webkit.org/b/79203 imported/w3c/web-platform-tests/mediacapture-streams/MediaStream-MediaElement-preload-none.https.html [ Failure Pass ]
    615614webkit.org/b/79203 imported/w3c/web-platform-tests/mediacapture-streams/MediaStreamTrack-end-manual.https.html [ Failure ]

  • trunk/Source/WebCore/ChangeLog

    r239316 r239319  
     12018-12-17  Eric Carlson  <eric.carlson@apple.com>
     2
     3        [MediaStream] A stream's first video frame should be rendered
     4        https://bugs.webkit.org/show_bug.cgi?id=192629
     5        <rdar://problem/46664353>
     6
     7        Reviewed by Youenn Fablet.
     8
     9        Test: fast/mediastream/media-stream-renders-first-frame.html
     10
     11        * platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.h:
     12        * platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.mm:
     13        (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::enqueueVideoSample):
     14        (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::ensureLayers):
     15        (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::currentDisplayMode const):
     16        (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::updateDisplayMode):
     17        (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::play):
     18        (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::currentReadyState):
     19        (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::characteristicsChanged):
     20        (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::checkSelectedVideoTrack):
     21        (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::paintCurrentFrameInContext):
     22        * platform/mediastream/RealtimeMediaSource.cpp:
     23        (WebCore::RealtimeMediaSource::size const):
     24        * platform/mediastream/mac/AVVideoCaptureSource.mm:
     25        (WebCore::AVVideoCaptureSource::processNewFrame):
     26        * platform/mediastream/mac/RealtimeIncomingVideoSourceCocoa.mm:
     27        (WebCore::RealtimeIncomingVideoSourceCocoa::processNewSample):
     28
    1292018-12-17  Justin Michaud  <justin_michaud@apple.com>
    230

  • trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.h

    r232613 r239319  
    190190        None,
    191191        PaintItBlack,
     192        WaitingForFirstImage,
    192193        PausedImage,
    193194        LivePreview,
    … …  
    229230
    230231    void applicationDidBecomeActive() final;
     232
     233    bool hideBackgroundLayer() const { return (!m_activeVideoTrack || m_waitingForFirstImage) && m_displayMode != PaintItBlack; }
    231234
    232235    MediaPlayer* m_player { nullptr };
    … …  
    275278    bool m_hasEverEnqueuedVideoFrame { false };
    276279    bool m_pendingSelectedTrackCheck { false };
    277     bool m_shouldDisplayFirstVideoFrame { false };
    278280    bool m_transformIsValid { false };
    279281    bool m_visible { false };
    280282    bool m_haveSeenMetadata { false };
     283    bool m_waitingForFirstImage { false };
    281284};
    282285   

  • trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.mm

    r237266 r239319  
    166166    if ((CALayer *)object == _parent->backgroundLayer()) {
    167167        if ([keyPath isEqualToString:@"bounds"]) {
     168            if (!_parent)
     169                return;
     170
     171            if (isMainThread()) {
     172                _parent->backgroundLayerBoundsChanged();
     173                return;
     174            }
     175
    168176            callOnMainThread([protectedSelf = RetainPtr<WebAVSampleBufferStatusChangeListener>(self)] {
    169177                if (!protectedSelf->_parent)
    … …  
    364372    }
    365373
    366     if (m_displayMode != LivePreview || (m_displayMode == PausedImage && m_imagePainter.mediaSample))
     374    if (m_displayMode != LivePreview && !m_waitingForFirstImage)
    367375        return;
    368376
    … …  
    392400
    393401    enqueueCorrectedVideoSample(sample);
     402    if (m_waitingForFirstImage) {
     403        m_waitingForFirstImage = false;
     404        updateDisplayMode();
     405    }
    394406}
    395407
    … …  
    482494
    483495    m_backgroundLayer = adoptNS([[CALayer alloc] init]);
     496    m_backgroundLayer.get().hidden = hideBackgroundLayer();
     497
    484498    m_backgroundLayer.get().backgroundColor = cachedCGColor(Color::black);
    485499    m_backgroundLayer.get().needsDisplayOnBoundsChange = YES;
     500
     501    auto size = snappedIntRect(m_player->client().mediaPlayerContentBoxRect()).size();
     502    m_backgroundLayer.get().bounds = CGRectMake(0, 0, size.width(), size.height());
    486503
    487504    [m_statusChangeListener beginObservingLayers];
    … …  
    497514    updateDisplayLayer();
    498515
    499     m_videoFullscreenLayerManager->setVideoLayer(m_backgroundLayer.get(), snappedIntRect(m_player->client().mediaPlayerContentBoxRect()).size());
     516    m_videoFullscreenLayerManager->setVideoLayer(m_backgroundLayer.get(), size);
    500517}
    501518
    … …  
    602619    }
    603620
     621    if (m_waitingForFirstImage)
     622        return WaitingForFirstImage;
     623
    604624    if (playing() && !m_ended) {
    605625        if (!m_mediaStreamPrivate->isProducingData())
    … …  
    624644    m_displayMode = displayMode;
    625645
    626     if (m_sampleBufferDisplayLayer) {
    627         runWithoutAnimations([this] {
    628             m_sampleBufferDisplayLayer.get().hidden = m_displayMode < PausedImage;
     646    auto hidden = m_displayMode < PausedImage;
     647    if (m_sampleBufferDisplayLayer && m_sampleBufferDisplayLayer.get().hidden != hidden) {
     648        runWithoutAnimations([this, hidden] {
     649            m_sampleBufferDisplayLayer.get().hidden = hidden;
     650        });
     651    }
     652    hidden = hideBackgroundLayer();
     653    if (m_backgroundLayer && m_backgroundLayer.get().hidden != hidden) {
     654        runWithoutAnimations([this, hidden] {
     655            m_backgroundLayer.get().hidden = hidden;
    629656        });
    630657    }
    … …  
    647674        track->play();
    648675
    649     m_shouldDisplayFirstVideoFrame = true;
    650676    updateDisplayMode();
    651677
    … …  
    766792
    767793        if (track == m_mediaStreamPrivate->activeVideoTrack() && !m_imagePainter.mediaSample) {
    768             if (!m_haveSeenMetadata)
     794            if (!m_haveSeenMetadata || m_waitingForFirstImage)
    769795                return MediaPlayer::ReadyState::HaveNothing;
    770796            allTracksAreLive = false;
    … …  
    772798    }
    773799
    774     if (!allTracksAreLive && !m_haveSeenMetadata)
     800    if (m_waitingForFirstImage || (!allTracksAreLive && !m_haveSeenMetadata))
    775801        return MediaPlayer::ReadyState::HaveMetadata;
    776802
    … …  
    829855        m_intrinsicSize = intrinsicSize;
    830856        sizeChanged = true;
     857        if (m_playbackState == PlaybackState::None)
     858            m_playbackState = PlaybackState::Paused;
    831859    }
    832860
    … …  
    9801008        }
    9811009
    982         if (oldVideoTrack != m_activeVideoTrack)
     1010        if (oldVideoTrack != m_activeVideoTrack) {
    9831011            m_imagePainter.reset();
     1012            if (m_displayMode == None)
     1013                m_waitingForFirstImage = true;
     1014        }
    9841015        ensureLayers();
    9851016        m_sampleBufferDisplayLayer.get().hidden = hideVideoLayer || m_displayMode < PausedImage;
     1017        m_backgroundLayer.get().hidden = hideBackgroundLayer();
     1018
    9861019        m_pendingSelectedTrackCheck = false;
    9871020        updateDisplayMode();
    … …  
    10761109
    10771110    GraphicsContextStateSaver stateSaver(context);
    1078     if (m_displayMode == PaintItBlack || !m_imagePainter.cgImage || !m_imagePainter.mediaSample) {
     1111    if (m_displayMode == PaintItBlack) {
    10791112        context.fillRect(IntRect(IntPoint(), IntSize(destRect.width(), destRect.height())), Color::black);
    10801113        return;
    10811114    }
     1115
     1116    if (!m_imagePainter.cgImage || !m_imagePainter.mediaSample)
     1117        return;
    10821118
    10831119    auto image = m_imagePainter.cgImage.get();
    … …  
    11651201void MediaPlayerPrivateMediaStreamAVFObjC::backgroundLayerBoundsChanged()
    11661202{
    1167     scheduleDeferredTask([this] {
    1168         runWithoutAnimations([this] {
    1169             updateDisplayLayer();
    1170         });
     1203    runWithoutAnimations([this] {
     1204        updateDisplayLayer();
    11711205    });
    11721206}

  • trunk/Source/WebCore/platform/mediastream/RealtimeMediaSource.cpp

    r239163 r239319  
    875875
    876876    if (size.isEmpty() && !m_intrinsicSize.isEmpty()) {
    877         if (size.width())
     877        if (size.isZero())
     878            size = m_intrinsicSize;
     879        else if (size.width())
    878880            size.setHeight(size.width() * (m_intrinsicSize.height() / static_cast<double>(m_intrinsicSize.width())));
    879881        else if (size.height())

  • trunk/Source/WebCore/platform/mediastream/mac/AVVideoCaptureSource.h

    r238145 r239319  
    127127    double m_pendingFrameRate;
    128128    InterruptionReason m_interruption { InterruptionReason::None };
     129    int m_framesToDropAtStartup { 0 };
    129130    bool m_isRunning { false };
    130131};

  • trunk/Source/WebCore/platform/mediastream/mac/AVVideoCaptureSource.mm

    r239065 r239319  
    525525
    526526    m_buffer = &sample.get();
     527    setIntrinsicSize(expandedIntSize(sample->presentationSize()));
    527528    dispatchMediaSampleToObservers(WTFMove(sample));
    528529}
    … …  
    530531void AVVideoCaptureSource::captureOutputDidOutputSampleBufferFromConnection(AVCaptureOutputType*, CMSampleBufferRef sampleBuffer, AVCaptureConnectionType* captureConnection)
    531532{
     533    if (m_framesToDropAtStartup && m_framesToDropAtStartup--)
     534        return;
     535
    532536    auto sample = MediaSampleAVFObjC::create(sampleBuffer, m_sampleRotation, [captureConnection isVideoMirrored]);
    533537    scheduleDeferredTask([this, sample = WTFMove(sample)] () mutable {
    … …  
    543547
    544548        m_isRunning = state;
     549        if (m_isRunning)
     550            m_framesToDropAtStartup = 4;
     551
    545552        notifyMutedChange(!m_isRunning);
    546553    });
    … …  
    574581
    575582        CMVideoDimensions dimensions = CMVideoFormatDescriptionGetDimensions(format.formatDescription);
    576         IntSize size = {dimensions.width, dimensions.height};
     583        IntSize size = { dimensions.width, dimensions.height };
    577584        auto index = presets.findMatching([&size](auto& preset) {
    578585            return size == preset->size;

  • trunk/Source/WebCore/platform/mediastream/mac/RealtimeIncomingVideoSourceCocoa.mm

    r237266 r239319  
    230230    auto size = this->size();
    231231    if (WTF::safeCast<int>(width) != size.width() || WTF::safeCast<int>(height) != size.height())
    232         setSize(IntSize(width, height));
     232        setIntrinsicSize(IntSize(width, height));
    233233
    234234    videoSampleAvailable(MediaSampleAVFObjC::create(sample, rotation));
Note: See TracChangeset for help on using the changeset viewer.