Changeset 258314 in webkit

Timestamp:

Mar 11, 2020 9:57:31 PM (4 years ago)

Author:

jer.noble@apple.com

Message:

Adopt AVSampleBufferVideoOutput
​https://bugs.webkit.org/show_bug.cgi?id=208951

Reviewed by Eric Carlson.

Source/WebCore:

Adopt AVSampleBufferVideoOutput, used for extracting decoded samples from an AVSampleBufferDisplayLayer.

Rather than use a separate decompression session when we need to, e.g., paint samples into WebGL, we will
add a AVSBVO to the AVSBDL and use it to extract CVPixelBuffers when asked.

• platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaSourceAVFObjC.h:
• platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaSourceAVFObjC.mm:

(WebCore::MediaPlayerPrivateMediaSourceAVFObjC::updateLastPixelBuffer):
(WebCore::MediaPlayerPrivateMediaSourceAVFObjC::acceleratedRenderingStateChanged):
(WebCore::MediaPlayerPrivateMediaSourceAVFObjC::ensureLayer):
(WebCore::MediaPlayerPrivateMediaSourceAVFObjC::isVideoOutputAvailable const):

Source/WebCore/PAL:

• pal/cocoa/AVFoundationSoftLink.h:
• pal/cocoa/AVFoundationSoftLink.mm:
• pal/spi/cocoa/AVFoundationSPI.h:

Source/WTF:

• wtf/PlatformHave.h:

Location:

trunk/Source

Files:

• WTF/ChangeLog (modified) (1 diff)
• WTF/wtf/PlatformHave.h (modified) (1 diff)
• WebCore/ChangeLog (modified) (1 diff)
• WebCore/PAL/ChangeLog (modified) (1 diff)
• WebCore/PAL/pal/cocoa/AVFoundationSoftLink.h (modified) (1 diff)
• WebCore/PAL/pal/cocoa/AVFoundationSoftLink.mm (modified) (1 diff)
• WebCore/PAL/pal/spi/cocoa/AVFoundationSPI.h (modified) (1 diff)
• WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaSourceAVFObjC.h (modified) (3 diffs)
• WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaSourceAVFObjC.mm (modified) (4 diffs)

trunk/Source/WTF/ChangeLog

@@ +1 @@
+2020-03-11  Jer Noble  <jer.noble@apple.com>
+
+        Adopt AVSampleBufferVideoOutput
+        https://bugs.webkit.org/show_bug.cgi?id=208951
+
+        Reviewed by Eric Carlson.
+
+        * wtf/PlatformHave.h:
+
 2020-03-11  Jer Noble  <jer.noble@apple.com>
 

trunk/Source/WTF/wtf/PlatformHave.h

@@ -601 +601 @@
 #if (PLATFORM(MAC) && __MAC_OS_X_VERSION_MIN_REQUIRED >= 101600) || (PLATFORM(IOS_FAMILY) && __IPHONE_OS_VERSION_MIN_REQUIRED >= 140000)
 #define HAVE_WEBP 1
+#define HAVE_AVSAMPLEBUFFERVIDEOOUTPUT 1
 #endif
 

trunk/Source/WebCore/ChangeLog

@@ +1 @@
+2020-03-11  Jer Noble  <jer.noble@apple.com>
+
+        Adopt AVSampleBufferVideoOutput
+        https://bugs.webkit.org/show_bug.cgi?id=208951
+
+        Reviewed by Eric Carlson.
+
+        Adopt AVSampleBufferVideoOutput, used for extracting decoded samples from an AVSampleBufferDisplayLayer.
+
+        Rather than use a separate decompression session when we need to, e.g., paint samples into WebGL, we will
+        add a AVSBVO to the AVSBDL and use it to extract CVPixelBuffers when asked.
+
+        * platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaSourceAVFObjC.h:
+        * platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaSourceAVFObjC.mm:
+        (WebCore::MediaPlayerPrivateMediaSourceAVFObjC::updateLastPixelBuffer):
+        (WebCore::MediaPlayerPrivateMediaSourceAVFObjC::acceleratedRenderingStateChanged):
+        (WebCore::MediaPlayerPrivateMediaSourceAVFObjC::ensureLayer):
+        (WebCore::MediaPlayerPrivateMediaSourceAVFObjC::isVideoOutputAvailable const):
+
 2020-03-11  Andres Gonzalez  <andresg_22@apple.com>
 

trunk/Source/WebCore/PAL/ChangeLog

@@ +1 @@
+2020-03-11  Jer Noble  <jer.noble@apple.com>
+
+        Adopt AVSampleBufferVideoOutput
+        https://bugs.webkit.org/show_bug.cgi?id=208951
+
+        Reviewed by Eric Carlson.
+
+        * pal/cocoa/AVFoundationSoftLink.h:
+        * pal/cocoa/AVFoundationSoftLink.mm:
+        * pal/spi/cocoa/AVFoundationSPI.h:
+
 2020-03-11  Jer Noble  <jer.noble@apple.com>
 

trunk/Source/WebCore/PAL/pal/cocoa/AVFoundationSoftLink.h

@@ -63 +63 @@
 SOFT_LINK_CLASS_FOR_HEADER(PAL, AVSampleBufferDisplayLayer)
 SOFT_LINK_CLASS_FOR_HEADER(PAL, AVSampleBufferRenderSynchronizer)
+SOFT_LINK_CLASS_FOR_HEADER(PAL, AVSampleBufferVideoOutput)
 SOFT_LINK_CLASS_FOR_HEADER(PAL, AVStreamDataParser)
 SOFT_LINK_CLASS_FOR_HEADER(PAL, AVURLAsset)

trunk/Source/WebCore/PAL/pal/cocoa/AVFoundationSoftLink.mm

@@ -90 +90 @@
 SOFT_LINK_CLASS_FOR_SOURCE_OPTIONAL_WITH_EXPORT(PAL, AVFoundation, AVSampleBufferDisplayLayer, PAL_EXPORT)
 SOFT_LINK_CLASS_FOR_SOURCE_OPTIONAL_WITH_EXPORT(PAL, AVFoundation, AVSampleBufferRenderSynchronizer, PAL_EXPORT)
+SOFT_LINK_CLASS_FOR_SOURCE_OPTIONAL_WITH_EXPORT(PAL, AVFoundation, AVSampleBufferVideoOutput, PAL_EXPORT)
 SOFT_LINK_CLASS_FOR_SOURCE_OPTIONAL_WITH_EXPORT(PAL, AVFoundation, AVStreamDataParser, PAL_EXPORT)
 

trunk/Source/WebCore/PAL/pal/spi/cocoa/AVFoundationSPI.h

@@ -356 +356 @@
 @end
 #endif
+
+#if HAVE(AVSAMPLEBUFFERVIDEOOUTPUT)
+#if USE(APPLE_INTERNAL_SDK)
+#include <AVFoundation/AVSampleBufferVideoOutput.h>
+#else
+@interface AVSampleBufferVideoOutput : NSObject
+- (CVPixelBufferRef)copyPixelBufferForSourceTime:(CMTime)sourceTime sourceTimeForDisplay:(nullable CMTime *)outSourceTimeForDisplay;
+@end
+
+@interface AVSampleBufferDisplayLayer (VideoOutput)
+@property (nonatomic, nullable) AVSampleBufferVideoOutput *output;
+@end
+#endif
+#endif

trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaSourceAVFObjC.h

@@ -42 +42 @@
 OBJC_CLASS AVSampleBufferDisplayLayer;
 OBJC_CLASS AVSampleBufferRenderSynchronizer;
+OBJC_CLASS AVSampleBufferVideoOutput;
 OBJC_CLASS AVStreamSession;
 
@@ -261 +262 @@
 
     bool shouldBePlaying() const;
+
+    bool isVideoOutputAvailable() const;
 
     friend class MediaSourcePrivateAVFObjC;
@@ -283 +286 @@
     RetainPtr<AVAsset> m_asset;
     RetainPtr<AVSampleBufferDisplayLayer> m_sampleBufferDisplayLayer;
+#if HAVE(AVSAMPLEBUFFERVIDEOOUTPUT)
+    RetainPtr<AVSampleBufferVideoOutput> m_videoOutput;
+#endif
 
     struct AudioRendererProperties {

trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaSourceAVFObjC.mm

@@ -586 +586 @@
 bool MediaPlayerPrivateMediaSourceAVFObjC::updateLastPixelBuffer()
 {
+#if HAVE(AVSAMPLEBUFFERVIDEOOUTPUT)
+    if (m_videoOutput) {
+        CMTime outputTime;
+        if (auto pixelBuffer = [m_videoOutput copyPixelBufferForSourceTime:toCMTime(currentMediaTime()) sourceTimeForDisplay:&outputTime]) {
+            INFO_LOG(LOGIDENTIFIER, "new pixelbuffer found for time ", toMediaTime(outputTime));
+            m_lastPixelBuffer = WTFMove(pixelBuffer);
+            return true;
+        }
+    }
+#endif
+
     if (m_sampleBufferDisplayLayer || !m_decompressionSession)
         return false;
@@ -671 +682 @@
 void MediaPlayerPrivateMediaSourceAVFObjC::acceleratedRenderingStateChanged()
 {
-    if (!m_hasBeenAskedToPaintGL) {
+    if (!m_hasBeenAskedToPaintGL || isVideoOutputAvailable()) {
         destroyDecompressionSession();
         ensureLayer();
@@ -756 +767 @@
 #endif
 
+#if HAVE(AVSAMPLEBUFFERVIDEOOUTPUT)
+    ASSERT(!m_videoOutput);
+    if (isVideoOutputAvailable()) {
+        m_videoOutput = adoptNS([PAL::allocAVSampleBufferVideoOutputInstance() init]);
+        ASSERT(m_videoOutput);
+        [m_sampleBufferDisplayLayer setOutput:m_videoOutput.get()];
+    }
+#endif
+
     ASSERT(m_sampleBufferDisplayLayer);
     if (!m_sampleBufferDisplayLayer) {
@@ -821 +841 @@
 {
     return m_playing && !seeking() && allRenderersHaveAvailableSamples() && m_readyState >= MediaPlayer::ReadyState::HaveFutureData;
+}
+
+bool MediaPlayerPrivateMediaSourceAVFObjC::isVideoOutputAvailable() const
+{
+#if HAVE(AVSAMPLEBUFFERVIDEOOUTPUT)
+    return PAL::getAVSampleBufferVideoOutputClass;
+#else
+    return false;
+#endif
 }