Changeset 262189 in webkit


Ignore:
Timestamp:
May 27, 2020 2:38:28 AM (4 years ago)
Author:
youenn@apple.com
Message:

Video freezes when attaching a local MediaStream to multiple elements
https://bugs.webkit.org/show_bug.cgi?id=194802
<rdar://problem/63613107>

Reviewed by Eric Carlson.

AVSampleBufferDisplayLayer sometimes does not update the rendering when the same local source is rendered several times.
To workaround this, we set kCMSampleAttachmentKey_DisplayImmediately to true, which fixes the issue as per my testing.
We clone the sample buffer before setting this property as it might not be thread safe to modify the attachments of a sample
that might also be encoded.
We implement this at LocalSampleBufferDisplayLayer level and enable this for local capture sources only.

Manually tested.

  • platform/graphics/avfoundation/SampleBufferDisplayLayer.h:

(WebCore::SampleBufferDisplayLayer::setRenderPolicy):

  • platform/graphics/avfoundation/objc/LocalSampleBufferDisplayLayer.h:

(WebCore::LocalSampleBufferDisplayLayer::setRenderPolicy):

  • platform/graphics/avfoundation/objc/LocalSampleBufferDisplayLayer.mm:

(WebCore::LocalSampleBufferDisplayLayer::enqueueSample):
(WebCore::LocalSampleBufferDisplayLayer::removeOldSamplesFromPendingQueue):

  • platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.mm:

(WebCore::MediaPlayerPrivateMediaStreamAVFObjC::ensureLayers):
(WebCore::MediaPlayerPrivateMediaStreamAVFObjC::checkSelectedVideoTrack):

  • platform/graphics/avfoundation/objc/MediaSampleAVFObjC.h:
  • platform/graphics/avfoundation/objc/MediaSampleAVFObjC.mm:

(WebCore::setSampleBufferAsDisplayImmediately):
(WebCore::MediaSampleAVFObjC::setAsDisplayImmediately):
(WebCore::MediaSampleAVFObjC::cloneSampleBuffer):

Location:
trunk/Source/WebCore
Files:
7 edited

Legend:

Unmodified
Added
Removed
  • trunk/Source/WebCore/ChangeLog

    r262188 r262189  
     12020-05-27  Youenn Fablet  <youenn@apple.com>
     2
     3        Video freezes when attaching a local MediaStream to multiple elements
     4        https://bugs.webkit.org/show_bug.cgi?id=194802
     5        <rdar://problem/63613107>
     6
     7        Reviewed by Eric Carlson.
     8
     9        AVSampleBufferDisplayLayer sometimes does not update the rendering when the same local source is rendered several times.
     10        To workaround this, we set kCMSampleAttachmentKey_DisplayImmediately to true, which fixes the issue as per my testing.
     11        We clone the sample buffer before setting this property as it might not be thread safe to modify the attachments of a sample
     12        that might also be encoded.
     13        We implement this at LocalSampleBufferDisplayLayer level and enable this for local capture sources only.
     14
     15        Manually tested.
     16
     17        * platform/graphics/avfoundation/SampleBufferDisplayLayer.h:
     18        (WebCore::SampleBufferDisplayLayer::setRenderPolicy):
     19        * platform/graphics/avfoundation/objc/LocalSampleBufferDisplayLayer.h:
     20        (WebCore::LocalSampleBufferDisplayLayer::setRenderPolicy):
     21        * platform/graphics/avfoundation/objc/LocalSampleBufferDisplayLayer.mm:
     22        (WebCore::LocalSampleBufferDisplayLayer::enqueueSample):
     23        (WebCore::LocalSampleBufferDisplayLayer::removeOldSamplesFromPendingQueue):
     24        * platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.mm:
     25        (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::ensureLayers):
     26        (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::checkSelectedVideoTrack):
     27        * platform/graphics/avfoundation/objc/MediaSampleAVFObjC.h:
     28        * platform/graphics/avfoundation/objc/MediaSampleAVFObjC.mm:
     29        (WebCore::setSampleBufferAsDisplayImmediately):
     30        (WebCore::MediaSampleAVFObjC::setAsDisplayImmediately):
     31        (WebCore::MediaSampleAVFObjC::cloneSampleBuffer):
     32
    1332020-05-19  Sergio Villar Senin  <svillar@igalia.com>
    234
  • trunk/Source/WebCore/platform/graphics/avfoundation/SampleBufferDisplayLayer.h

    r258208 r262189  
    7070    virtual PlatformLayer* rootLayer() = 0;
    7171
     72    enum class RenderPolicy { TimingInfo, Immediately };
     73    virtual void setRenderPolicy(RenderPolicy) { };
     74
    7275protected:
    7376    explicit SampleBufferDisplayLayer(Client&);
  • trunk/Source/WebCore/platform/graphics/avfoundation/objc/LocalSampleBufferDisplayLayer.h

    r258208 r262189  
    7373    void enqueueSample(MediaSample&) final;
    7474    void clearEnqueuedSamples() final;
     75    void setRenderPolicy(RenderPolicy) final;
    7576
    7677private:
     
    8384    RetainPtr<AVSampleBufferDisplayLayer> m_sampleBufferDisplayLayer;
    8485    RetainPtr<PlatformLayer> m_rootLayer;
     86    RenderPolicy m_renderPolicy { RenderPolicy::TimingInfo };
    8587
    8688    using PendingSampleQueue = Deque<Ref<MediaSample>>;
     
    8890};
    8991
     92inline void LocalSampleBufferDisplayLayer::setRenderPolicy(RenderPolicy renderPolicy)
     93{
     94    m_renderPolicy = renderPolicy;
     95}
     96
    9097}
    9198
  • trunk/Source/WebCore/platform/graphics/avfoundation/objc/LocalSampleBufferDisplayLayer.mm

    r258208 r262189  
    3131#import "Color.h"
    3232#import "IntSize.h"
    33 #import "MediaSample.h"
     33#import "Logging.h"
     34#import "MediaSampleAVFObjC.h"
    3435
    3536#import <AVFoundation/AVSampleBufferDisplayLayer.h>
     
    296297    }
    297298
    298     [m_sampleBufferDisplayLayer enqueueSampleBuffer:sample.platformSample().sample.cmSampleBuffer];
     299    auto sampleToEnqueue = sample.platformSample().sample.cmSampleBuffer;
     300    RetainPtr<CMSampleBufferRef> newSampleBuffer;
     301
     302    // If needed, we set the sample buffer to kCMSampleAttachmentKey_DisplayImmediately as a workaround to rdar://problem/49274083.
     303    // We clone the sample buffer as modifying the attachments of a sample buffer used elsewhere (encoding e.g.) may not be thread safe.
     304    if (m_renderPolicy == RenderPolicy::Immediately) {
     305        newSampleBuffer = MediaSampleAVFObjC::cloneSampleBufferAndSetAsDisplayImmediately(sampleToEnqueue);
     306        sampleToEnqueue = newSampleBuffer.get();
     307    }
     308
     309    [m_sampleBufferDisplayLayer enqueueSampleBuffer:sampleToEnqueue];
    299310}
    300311
     
    303314    if (m_pendingVideoSampleQueue.isEmpty() || !m_client)
    304315        return;
     316
     317    if (m_renderPolicy == RenderPolicy::Immediately) {
     318        m_pendingVideoSampleQueue.clear();
     319        return;
     320    }
    305321
    306322    auto decodeTime = m_pendingVideoSampleQueue.first()->decodeTime();
  • trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.mm

    r262114 r262189  
    380380        return;
    381381
     382    if (activeVideoTrack->source().isCaptureSource())
     383        m_sampleBufferDisplayLayer->setRenderPolicy(SampleBufferDisplayLayer::RenderPolicy::Immediately);
     384
    382385    auto size = snappedIntRect(m_player->playerContentBoxRect()).size();
    383386    m_sampleBufferDisplayLayer->initialize(hideRootLayer(), size, [this, weakThis = makeWeakPtr(this), size](auto didSucceed) {
     
    868871            if (oldVideoTrack)
    869872                oldVideoTrack->streamTrack().source().removeVideoSampleObserver(*this);
    870             if (m_activeVideoTrack)
     873            if (m_activeVideoTrack) {
     874                if (m_sampleBufferDisplayLayer && m_activeVideoTrack->streamTrack().source().isCaptureSource())
     875                    m_sampleBufferDisplayLayer->setRenderPolicy(SampleBufferDisplayLayer::RenderPolicy::Immediately);
    871876                m_activeVideoTrack->streamTrack().source().addVideoSampleObserver(*this);
     877            }
    872878        }
    873879    });
  • trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaSampleAVFObjC.h

    r258846 r262189  
    4141
    4242    WEBCORE_EXPORT static void setAsDisplayImmediately(MediaSample&);
     43    static RetainPtr<CMSampleBufferRef> cloneSampleBufferAndSetAsDisplayImmediately(CMSampleBufferRef);
    4344
    4445    RefPtr<JSC::Uint8ClampedArray> getRGBAImageData() const override;
  • trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaSampleAVFObjC.mm

    r261460 r262189  
    312312}
    313313
    314 void MediaSampleAVFObjC::setAsDisplayImmediately(MediaSample& sample)
    315 {
    316     CFArrayRef attachmentsArray = CMSampleBufferGetSampleAttachmentsArray(sample.platformSample().sample.cmSampleBuffer, true);
     314static inline void setSampleBufferAsDisplayImmediately(CMSampleBufferRef sampleBuffer)
     315{
     316    CFArrayRef attachmentsArray = CMSampleBufferGetSampleAttachmentsArray(sampleBuffer, true);
    317317    for (CFIndex i = 0; i < CFArrayGetCount(attachmentsArray); ++i) {
    318318        CFMutableDictionaryRef attachments = checked_cf_cast<CFMutableDictionaryRef>(CFArrayGetValueAtIndex(attachmentsArray, i));
    319319        CFDictionarySetValue(attachments, kCMSampleAttachmentKey_DisplayImmediately, kCFBooleanTrue);
    320320    }
     321}
     322
     323void MediaSampleAVFObjC::setAsDisplayImmediately(MediaSample& sample)
     324{
     325    setSampleBufferAsDisplayImmediately(sample.platformSample().sample.cmSampleBuffer);
    321326}
    322327
     
    391396}
    392397
    393 }
     398RetainPtr<CMSampleBufferRef> MediaSampleAVFObjC::cloneSampleBufferAndSetAsDisplayImmediately(CMSampleBufferRef sample)
     399{
     400    auto pixelBuffer = static_cast<CVImageBufferRef>(PAL::CMSampleBufferGetImageBuffer(sample));
     401    if (!pixelBuffer)
     402        return nullptr;
     403
     404    CMVideoFormatDescriptionRef formatDescription = nullptr;
     405    auto status = CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, pixelBuffer, &formatDescription);
     406    if (status)
     407        return nullptr;
     408    auto retainedFormatDescription = adoptCF(formatDescription);
     409
     410    CMItemCount itemCount = 0;
     411    status = CMSampleBufferGetSampleTimingInfoArray(sample, 0, nullptr, &itemCount);
     412    if (status)
     413        return nullptr;
     414
     415    Vector<CMSampleTimingInfo> timingInfoArray;
     416    timingInfoArray.grow(itemCount);
     417    status = CMSampleBufferGetSampleTimingInfoArray(sample, itemCount, timingInfoArray.data(), nullptr);
     418    if (status)
     419        return nullptr;
     420
     421    CMSampleBufferRef newSampleBuffer;
     422    status = CMSampleBufferCreateReadyWithImageBuffer(kCFAllocatorDefault, pixelBuffer, formatDescription, timingInfoArray.data(), &newSampleBuffer);
     423    if (status)
     424        return nullptr;
     425
     426    setSampleBufferAsDisplayImmediately(newSampleBuffer);
     427
     428    return adoptCF(newSampleBuffer);
     429}
     430
     431}
Note: See TracChangeset for help on using the changeset viewer.