Changeset 261908 in webkit

Timestamp:

May 20, 2020 12:54:05 AM (4 years ago)

Author:

youenn@apple.com

Message:

Allow calling VideoSampleObserver::videoSampleAvailable from a background thread
​https://bugs.webkit.org/show_bug.cgi?id=212024

Reviewed by Eric Carlson.

Allow RealtimeMediaSource::videoSampleAvailable to be called on a background thread, typically the capture thread.
Make WebRTC remote sources and mock capture sources do that.

RealtimeMediaSource is then updating its intrinsic size from the generation thread while updating its size in the main thread.
The size() getter can be called from both threads.

Existing consumers do the following:

• media player will hop to the main thread.
• media recorder will do processing from the background thread.
• WebRTC sender will do processing from the background thread, except when sending black frames where this will still be done on the main thread.

This is ok as we ensure either we send black frames on the main thread (and we do not observe the source) or we observe the source to send.

Follow-ups will migrate the real capture sources as well as migrating media player processing out of the main thread.
Covered by existing tests.

• platform/MediaSample.h:
• platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.mm:

(WebCore::MediaPlayerPrivateMediaStreamAVFObjC::MediaPlayerPrivateMediaStreamAVFObjC):
(WebCore::MediaPlayerPrivateMediaStreamAVFObjC::videoSampleAvailable):

• platform/mediastream/RealtimeMediaSource.cpp:

(WebCore::RealtimeMediaSource::videoSampleAvailable):
(WebCore::RealtimeMediaSource::setIntrinsicSize):

• platform/mediastream/RealtimeMediaSource.h:
• platform/mediastream/mac/MockRealtimeVideoSourceMac.h:
• platform/mediastream/mac/MockRealtimeVideoSourceMac.mm:

(WebCore::MockRealtimeVideoSourceMac::MockRealtimeVideoSourceMac):
(WebCore::MockRealtimeVideoSourceMac::updateSampleBuffer):

• platform/mediastream/mac/RealtimeIncomingVideoSourceCocoa.h:
• platform/mediastream/mac/RealtimeIncomingVideoSourceCocoa.mm:

(WebCore::RealtimeIncomingVideoSourceCocoa::OnFrame):
(WebCore::RealtimeIncomingVideoSourceCocoa::processNewSample): Deleted.

Location:

trunk/Source/WebCore

Files:

• ChangeLog (modified) (1 diff)
• platform/MediaSample.h (modified) (2 diffs)
• platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.h (modified) (1 diff)
• platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.mm (modified) (3 diffs)
• platform/mediastream/RealtimeMediaSource.cpp (modified) (2 diffs)
• platform/mediastream/RealtimeMediaSource.h (modified) (2 diffs)
• platform/mediastream/mac/MockRealtimeVideoSourceMac.h (modified) (2 diffs)
• platform/mediastream/mac/MockRealtimeVideoSourceMac.mm (modified) (2 diffs)
• platform/mediastream/mac/RealtimeIncomingVideoSourceCocoa.h (modified) (1 diff)
• platform/mediastream/mac/RealtimeIncomingVideoSourceCocoa.mm (modified) (1 diff)

trunk/Source/WebCore/ChangeLog

@@ +1 @@
+2020-05-20  Youenn Fablet  <youenn@apple.com>
+
+        Allow calling VideoSampleObserver::videoSampleAvailable from a background thread
+        https://bugs.webkit.org/show_bug.cgi?id=212024
+
+        Reviewed by Eric Carlson.
+
+        Allow RealtimeMediaSource::videoSampleAvailable to be called on a background thread, typically the capture thread.
+        Make WebRTC remote sources and mock capture sources do that.
+
+        RealtimeMediaSource is then updating its intrinsic size from the generation thread while updating its size in the main thread.
+        The size() getter can be called from both threads.
+
+        Existing consumers do the following:
+        - media player will hop to the main thread.
+        - media recorder will do processing from the background thread.
+        - WebRTC sender will do processing from the background thread, except when sending black frames where this will still be done on the main thread.
+        This is ok as we ensure either we send black frames on the main thread (and we do not observe the source) or we observe the source to send.
+
+        Follow-ups will migrate the real capture sources as well as migrating media player processing out of the main thread.
+        Covered by existing tests.
+
+        * platform/MediaSample.h:
+        * platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.mm:
+        (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::MediaPlayerPrivateMediaStreamAVFObjC):
+        (WebCore::MediaPlayerPrivateMediaStreamAVFObjC::videoSampleAvailable):
+        * platform/mediastream/RealtimeMediaSource.cpp:
+        (WebCore::RealtimeMediaSource::videoSampleAvailable):
+        (WebCore::RealtimeMediaSource::setIntrinsicSize):
+        * platform/mediastream/RealtimeMediaSource.h:
+        * platform/mediastream/mac/MockRealtimeVideoSourceMac.h:
+        * platform/mediastream/mac/MockRealtimeVideoSourceMac.mm:
+        (WebCore::MockRealtimeVideoSourceMac::MockRealtimeVideoSourceMac):
+        (WebCore::MockRealtimeVideoSourceMac::updateSampleBuffer):
+        * platform/mediastream/mac/RealtimeIncomingVideoSourceCocoa.h:
+        * platform/mediastream/mac/RealtimeIncomingVideoSourceCocoa.mm:
+        (WebCore::RealtimeIncomingVideoSourceCocoa::OnFrame):
+        (WebCore::RealtimeIncomingVideoSourceCocoa::processNewSample): Deleted.
+
 2020-05-20  Oriol Brufau  <obrufau@igalia.com>
 

trunk/Source/WebCore/platform/MediaSample.h

@@ -31 +31 @@
 #include <wtf/EnumTraits.h>
 #include <wtf/MediaTime.h>
-#include <wtf/RefCounted.h>
+#include <wtf/ThreadSafeRefCounted.h>
 #include <wtf/text/AtomString.h>
 
@@ -55 +55 @@
 };
 
-class MediaSample : public RefCounted<MediaSample> {
+class MediaSample : public ThreadSafeRefCounted<MediaSample> {
 public:
     virtual ~MediaSample() = default;

trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.h

@@ -140 +140 @@
     MediaTime calculateTimelineOffset(const MediaSample&, double);
     
-    void enqueueVideoSample(MediaSample&);
     void enqueueCorrectedVideoSample(MediaSample&);
     void requestNotificationWhenReadyForVideoData();

trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaStreamAVFObjC.mm

@@ -142 +142 @@
 {
     INFO_LOG(LOGIDENTIFIER);
+    // MediaPlayerPrivateMediaStreamAVFObjC::videoSampleAvailable expects a weak pointer to be created in the constructor.
     m_boundsChangeListener = adoptNS([[WebRootSampleBufferBoundsChangeListener alloc] initWithCallback:[this, weakThis = makeWeakPtr(this)] {
         if (!weakThis)
@@ -282 +283 @@
 }
 
-void MediaPlayerPrivateMediaStreamAVFObjC::enqueueVideoSample(MediaSample& sample)
-{
+void MediaPlayerPrivateMediaStreamAVFObjC::videoSampleAvailable(MediaSample& sample)
+{
+    if (!isMainThread()) {
+        callOnMainThread([weakThis = makeWeakPtr(this), sample = makeRef(sample)]() mutable {
+            if (weakThis)
+                weakThis->videoSampleAvailable(sample.get());
+        });
+        return;
+    }
+
+    if (!m_activeVideoTrack)
+        return;
+
     if (!m_imagePainter.mediaSample || m_displayMode != PausedImage) {
         m_imagePainter.mediaSample = &sample;
@@ -734 +746 @@
 {
     updateTracks();
-}
-
-void MediaPlayerPrivateMediaStreamAVFObjC::videoSampleAvailable(MediaSample& mediaSample)
-{
-    if (streamTime().toDouble() < 0)
-        return;
-
-    enqueueVideoSample(mediaSample);
 }
 

trunk/Source/WebCore/platform/mediastream/RealtimeMediaSource.cpp

@@ -188 +188 @@
 void RealtimeMediaSource::videoSampleAvailable(MediaSample& mediaSample)
 {
-    // FIXME: Migrate RealtimeMediaSource clients to non main thread processing.
-    ASSERT(isMainThread());
 #if !RELEASE_LOG_DISABLED
     ++m_frameCount;
@@ -967 +965 @@
     m_intrinsicSize = size;
 
-    if (currentSize != this->size())
-        notifySettingsDidChangeObservers({ RealtimeMediaSourceSettings::Flag::Width, RealtimeMediaSourceSettings::Flag::Height });
+    if (currentSize != this->size()) {
+        scheduleDeferredTask([this] {
+            notifySettingsDidChangeObservers({ RealtimeMediaSourceSettings::Flag::Width, RealtimeMediaSourceSettings::Flag::Height });
+        });
+    }
 }
 

trunk/Source/WebCore/platform/mediastream/RealtimeMediaSource.h

@@ -103 +103 @@
         virtual ~VideoSampleObserver() = default;
 
-        virtual void videoSampleAvailable(MediaSample&) { }
+        // May be called on a background thread.
+        virtual void videoSampleAvailable(MediaSample&) = 0;
     };
 
@@ -279 +280 @@
     HashSet<VideoSampleObserver*> m_videoSampleObservers;
 
+    // Set on the main thread from constraints.
     IntSize m_size;
+    // Set on sample generation thread.
     IntSize m_intrinsicSize;
     double m_frameRate { 30 };

trunk/Source/WebCore/platform/mediastream/mac/MockRealtimeVideoSourceMac.h

@@ -35 +35 @@
 #include "MockRealtimeVideoSource.h"
 #include "PixelBufferConformerCV.h"
+#include <wtf/WorkQueue.h>
 
 typedef struct __CVBuffer *CVBufferRef;
@@ -61 +62 @@
     std::unique_ptr<ImageTransferSessionVT> m_imageTransferSession;
     IntSize m_presetSize;
+    Ref<WorkQueue> m_workQueue;
 };
 

trunk/Source/WebCore/platform/mediastream/mac/MockRealtimeVideoSourceMac.mm

@@ -79 +79 @@
 MockRealtimeVideoSourceMac::MockRealtimeVideoSourceMac(String&& deviceID, String&& name, String&& hashSalt)
     : MockRealtimeVideoSource(WTFMove(deviceID), WTFMove(name), WTFMove(hashSalt))
+    , m_workQueue(WorkQueue::create("MockRealtimeVideoSource Render Queue", WorkQueue::Type::Serial, WorkQueue::QOS::UserInteractive))
 {
 }
@@ -96 +97 @@
         return;
 
-    dispatchMediaSampleToObservers(*sampleBuffer);
+    m_workQueue->dispatch([this, protectedThis = makeRef(*this), sampleBuffer = WTFMove(sampleBuffer)]() mutable {
+        dispatchMediaSampleToObservers(*sampleBuffer);
+    });
 }
 

trunk/Source/WebCore/platform/mediastream/mac/RealtimeIncomingVideoSourceCocoa.h

@@ -44 +44 @@
 private:
     RealtimeIncomingVideoSourceCocoa(rtc::scoped_refptr<webrtc::VideoTrackInterface>&&, String&&);
-    void processNewSample(CMSampleBufferRef, unsigned, unsigned, MediaSample::VideoRotation);
     RetainPtr<CVPixelBufferRef> pixelBufferFromVideoFrame(const webrtc::VideoFrame&);
     CVPixelBufferPoolRef pixelBufferPool(size_t width, size_t height);

trunk/Source/WebCore/platform/mediastream/mac/RealtimeIncomingVideoSourceCocoa.mm

@@ -194 +194 @@
     }
 
-    callOnMainThread([protectedThis = makeRef(*this), sample = WTFMove(sample), width, height, rotation] {
-        protectedThis->processNewSample(sample.get(), width, height, rotation);
-    });
-}
-
-void RealtimeIncomingVideoSourceCocoa::processNewSample(CMSampleBufferRef sample, unsigned width, unsigned height, MediaSample::VideoRotation rotation)
-{
-    auto size = this->size();
-    if (WTF::safeCast<int>(width) != size.width() || WTF::safeCast<int>(height) != size.height())
-        setIntrinsicSize(IntSize(width, height));
-
-    videoSampleAvailable(MediaSampleAVFObjC::create(sample, rotation));
+    setIntrinsicSize(IntSize(width, height));
+    videoSampleAvailable(MediaSampleAVFObjC::create(sample.get(), rotation));
 }