Changeset 277424 in webkit

Timestamp:

May 12, 2021, 10:37:17 PM (5 years ago)

Author:

Peng Liu

Message:

[iPad] SourceBufferPrivateAVFObjC should not report an error to the web page when the video playback is interrupted
​https://bugs.webkit.org/show_bug.cgi?id=225620

Reviewed by Jer Noble.

If SourceBufferPrivateAVFObjC reports an error to a web page when
AVSampleBufferDisplayLayer reports AVErrorOperationInterrupted (the playback
was interrupted), the web page will likely destroy the video player (and teardown
the video element). That behavior will lead to an empty picture-in-picture window
if the video was playing in picture-in-picture.

With this patch, SourceBufferPrivateAVFObjC will not report an error to the
web page in case of playback interruption. Instead, it takes a note that
the playback was interrupted, so that when we try to resume the playback later,
it will flush the AVSampleBufferDisplayLayer to recover the playback state.

In addition, we need to enqueue an IDR frame first before we enqueue more
samples after flushing the AVSampleBufferDisplayLayer. That is guaranteed
by SourceBufferPrivate::reenqueSamples().

• platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaSourceAVFObjC.mm:

(WebCore::MediaPlayerPrivateMediaSourceAVFObjC::playInternal):

• platform/graphics/avfoundation/objc/MediaSourcePrivateAVFObjC.h:
• platform/graphics/avfoundation/objc/MediaSourcePrivateAVFObjC.mm:

(WebCore::MediaSourcePrivateAVFObjC::flushActiveSourceBuffersIfNeeded):

• platform/graphics/avfoundation/objc/SourceBufferPrivateAVFObjC.h:
• platform/graphics/avfoundation/objc/SourceBufferPrivateAVFObjC.mm:

(WebCore::SourceBufferPrivateAVFObjC::flushIfNeeded):
(WebCore::SourceBufferPrivateAVFObjC::layerDidReceiveError):
(WebCore::SourceBufferPrivateAVFObjC::isReadyForMoreSamples):

Location:

trunk/Source/WebCore

Files:

• ChangeLog (modified) (1 diff)
• platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaSourceAVFObjC.mm (modified) (1 diff)
• platform/graphics/avfoundation/objc/MediaSourcePrivateAVFObjC.h (modified) (1 diff)
• platform/graphics/avfoundation/objc/MediaSourcePrivateAVFObjC.mm (modified) (1 diff)
• platform/graphics/avfoundation/objc/SourceBufferPrivateAVFObjC.h (modified) (2 diffs)
• platform/graphics/avfoundation/objc/SourceBufferPrivateAVFObjC.mm (modified) (3 diffs)

trunk/Source/WebCore/ChangeLog

@@ +1 @@
+2021-05-12  Peng Liu  <peng.liu6@apple.com>
+
+        [iPad] SourceBufferPrivateAVFObjC should not report an error to the web page when the video playback is interrupted
+        https://bugs.webkit.org/show_bug.cgi?id=225620
+
+        Reviewed by Jer Noble.
+
+        If `SourceBufferPrivateAVFObjC` reports an error to a web page when
+        `AVSampleBufferDisplayLayer` reports `AVErrorOperationInterrupted` (the playback
+        was interrupted), the web page will likely destroy the video player (and teardown
+        the video element). That behavior will lead to an empty picture-in-picture window
+        if the video was playing in picture-in-picture.
+
+        With this patch, `SourceBufferPrivateAVFObjC` will not report an error to the
+        web page in case of playback interruption. Instead, it takes a note that
+        the playback was interrupted, so that when we try to resume the playback later,
+        it will flush the `AVSampleBufferDisplayLayer` to recover the playback state.
+
+        In addition, we need to enqueue an IDR frame first before we enqueue more
+        samples after flushing the `AVSampleBufferDisplayLayer`. That is guaranteed
+        by `SourceBufferPrivate::reenqueSamples()`.
+
+        * platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaSourceAVFObjC.mm:
+        (WebCore::MediaPlayerPrivateMediaSourceAVFObjC::playInternal):
+        * platform/graphics/avfoundation/objc/MediaSourcePrivateAVFObjC.h:
+        * platform/graphics/avfoundation/objc/MediaSourcePrivateAVFObjC.mm:
+        (WebCore::MediaSourcePrivateAVFObjC::flushActiveSourceBuffersIfNeeded):
+        * platform/graphics/avfoundation/objc/SourceBufferPrivateAVFObjC.h:
+        * platform/graphics/avfoundation/objc/SourceBufferPrivateAVFObjC.mm:
+        (WebCore::SourceBufferPrivateAVFObjC::flushIfNeeded):
+        (WebCore::SourceBufferPrivateAVFObjC::layerDidReceiveError):
+        (WebCore::SourceBufferPrivateAVFObjC::isReadyForMoreSamples):
+
 2021-05-12  Chris Dumez  <cdumez@apple.com>
 

trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaSourceAVFObjC.mm

@@ -323 +323 @@
 
     ALWAYS_LOG(LOGIDENTIFIER);
+#if PLATFORM(IOS_FAMILY)
+    m_mediaSourcePrivate->flushActiveSourceBuffersIfNeeded();
+#endif
     m_playing = true;
     if (shouldBePlaying())

trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaSourcePrivateAVFObjC.h

@@ -94 +94 @@
     void setDecompressionSession(WebCoreDecompressionSession*);
 
+#if PLATFORM(IOS_FAMILY)
+    void flushActiveSourceBuffersIfNeeded();
+#endif
+
 #if ENABLE(ENCRYPTED_MEDIA)
     void cdmInstanceAttached(CDMInstance&);

trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaSourcePrivateAVFObjC.mm

@@ -283 +283 @@
 }
 
+#if PLATFORM(IOS_FAMILY)
+void MediaSourcePrivateAVFObjC::flushActiveSourceBuffersIfNeeded()
+{
+    for (auto* sourceBuffer : m_activeSourceBuffers)
+        sourceBuffer->flushIfNeeded();
+}
+#endif
+
 #if ENABLE(ENCRYPTED_MEDIA)
 void MediaSourcePrivateAVFObjC::cdmInstanceAttached(CDMInstance& instance)

trunk/Source/WebCore/platform/graphics/avfoundation/objc/SourceBufferPrivateAVFObjC.h

@@ -110 +110 @@
 
     void flush();
+#if PLATFORM(IOS_FAMILY)
+    void flushIfNeeded();
+#endif
 
     void registerForErrorNotifications(SourceBufferPrivateAVFObjCErrorClient*);
@@ -198 +201 @@
     ALLOW_NEW_API_WITHOUT_GUARDS_END
     RetainPtr<WebAVSampleBufferErrorListener> m_errorListener;
+#if PLATFORM(IOS_FAMILY)
+    bool m_displayLayerWasInterrupted { false };
+#endif
     RetainPtr<NSError> m_hdcpError;
     Box<BinarySemaphore> m_hasSessionSemaphore;

trunk/Source/WebCore/platform/graphics/avfoundation/objc/SourceBufferPrivateAVFObjC.mm

@@ -929 +929 @@
 }
 
+#if PLATFORM(IOS_FAMILY)
+void SourceBufferPrivateAVFObjC::flushIfNeeded()
+{
+    if (!m_displayLayerWasInterrupted)
+        return;
+
+    m_displayLayerWasInterrupted = false;
+    if (m_videoTracks.size())
+        flushVideo();
+
+    // We initiatively enqueue samples instead of waiting for the
+    // media data requests from m_decompressionSession and m_displayLayer.
+    // In addition, we need to enqueue a sync sample (IDR video frame) first.
+    if (m_decompressionSession)
+        m_decompressionSession->stopRequestingMediaData();
+    [m_displayLayer stopRequestingMediaData];
+
+    reenqueSamples(AtomString::number(m_enabledVideoTrackID));
+}
+#endif
+
 void SourceBufferPrivateAVFObjC::registerForErrorNotifications(SourceBufferPrivateAVFObjCErrorClient* client)
 {
@@ -944 +965 @@
 {
     ERROR_LOG(LOGIDENTIFIER, [[error description] UTF8String]);
+
+#if PLATFORM(IOS_FAMILY)
+    if ([layer status] == AVQueuedSampleBufferRenderingStatusFailed && [[error domain] isEqualToString:@"AVFoundationErrorDomain"] && [error code] == AVErrorOperationInterrupted) {
+        m_displayLayerWasInterrupted = true;
+        return;
+    }
+#endif
 
     // FIXME(142246): Remove the following once <rdar://problem/20027434> is resolved.
@@ -1175 +1203 @@
     auto trackID = parseIntegerAllowingTrailingJunk<uint64_t>(trackIDString).valueOr(0);
     if (trackID == m_enabledVideoTrackID) {
+#if PLATFORM(IOS_FAMILY)
+        if (m_displayLayerWasInterrupted)
+            return false;
+#endif
+
         if (m_decompressionSession)
             return m_decompressionSession->isReadyForMoreMediaData();