Changeset 275514 in webkit

Timestamp:

Apr 6, 2021 6:53:24 AM (3 years ago)

Author:

commit-queue@webkit.org

Message:

[GStreamer][WebRTC] Audio is not played from an audio element when the srcObject object has unstarted video tracks
​https://bugs.webkit.org/show_bug.cgi?id=209163

Patch by Philippe Normand <​pnormand@igalia.com> on 2021-04-06
Reviewed by Xabier Rodriguez-Calvar.

Source/WebCore:

Ensure no MediaStream (active) video tracks can be added in a pipeline representing an <audio> element.

• platform/graphics/gstreamer/MediaPlayerPrivateGStreamer.cpp:

(WebCore::MediaPlayerPrivateGStreamer::sourceSetup):
(WebCore::MediaPlayerPrivateGStreamer::hasFirstSampleReachedSink const):
(WebCore::MediaPlayerPrivateGStreamer::videoSinkCapsChanged):
(WebCore::MediaPlayerPrivateGStreamer::playbackPosition const):
(WebCore::MediaPlayerPrivateGStreamer::updateTracks):

• platform/graphics/gstreamer/MediaPlayerPrivateGStreamer.h:
• platform/mediastream/gstreamer/GStreamerMediaStreamSource.cpp:

(webkitMediaStreamSrcPostStreamCollection):
(webkitMediaStreamSrcSetStream):

• platform/mediastream/gstreamer/GStreamerMediaStreamSource.h:

LayoutTests:

• platform/glib/TestExpectations: Unflag now-passing test.

Location:

trunk

Files:

• LayoutTests/ChangeLog (modified) (1 diff)
• LayoutTests/platform/glib/TestExpectations (modified) (1 diff)
• Source/WebCore/ChangeLog (modified) (1 diff)
• Source/WebCore/platform/graphics/gstreamer/MediaPlayerPrivateGStreamer.cpp (modified) (7 diffs)
• Source/WebCore/platform/graphics/gstreamer/MediaPlayerPrivateGStreamer.h (modified) (1 diff)
• Source/WebCore/platform/mediastream/gstreamer/GStreamerMediaStreamSource.cpp (modified) (3 diffs)
• Source/WebCore/platform/mediastream/gstreamer/GStreamerMediaStreamSource.h (modified) (1 diff)

trunk/LayoutTests/ChangeLog

@@ +1 @@
+2021-04-06  Philippe Normand  <pnormand@igalia.com>
+
+        [GStreamer][WebRTC] Audio is not played from an audio element when the srcObject object has unstarted video tracks
+        https://bugs.webkit.org/show_bug.cgi?id=209163
+
+        Reviewed by Xabier Rodriguez-Calvar.
+
+        * platform/glib/TestExpectations: Unflag now-passing test.
+
 2021-04-06  Keith Miller  <keith_miller@apple.com>
 

trunk/LayoutTests/platform/glib/TestExpectations

@@ -1070 +1070 @@
 
 webkit.org/b/208125 webrtc/peerconnection-new-candidate-page-cache.html [ Failure Timeout ]
-
-webkit.org/b/209163 webrtc/audio-video-element-playing.html [ Crash Failure Pass ]
 
 webkit.org/b/216538 webrtc/captureCanvas-webrtc-software-h264-baseline.html [ Slow Failure ]

trunk/Source/WebCore/ChangeLog

@@ +1 @@
+2021-04-06  Philippe Normand  <pnormand@igalia.com>
+
+        [GStreamer][WebRTC] Audio is not played from an audio element when the srcObject object has unstarted video tracks
+        https://bugs.webkit.org/show_bug.cgi?id=209163
+
+        Reviewed by Xabier Rodriguez-Calvar.
+
+        Ensure no MediaStream (active) video tracks can be added in a pipeline representing an <audio> element.
+
+        * platform/graphics/gstreamer/MediaPlayerPrivateGStreamer.cpp:
+        (WebCore::MediaPlayerPrivateGStreamer::sourceSetup):
+        (WebCore::MediaPlayerPrivateGStreamer::hasFirstSampleReachedSink const):
+        (WebCore::MediaPlayerPrivateGStreamer::videoSinkCapsChanged):
+        (WebCore::MediaPlayerPrivateGStreamer::playbackPosition const):
+        (WebCore::MediaPlayerPrivateGStreamer::updateTracks):
+        * platform/graphics/gstreamer/MediaPlayerPrivateGStreamer.h:
+        * platform/mediastream/gstreamer/GStreamerMediaStreamSource.cpp:
+        (webkitMediaStreamSrcPostStreamCollection):
+        (webkitMediaStreamSrcSetStream):
+        * platform/mediastream/gstreamer/GStreamerMediaStreamSource.h:
+
 2021-04-06  Philippe Normand  <pnormand@igalia.com>
 

trunk/Source/WebCore/platform/graphics/gstreamer/MediaPlayerPrivateGStreamer.cpp

@@ -873 +873 @@
         auto stream = m_streamPrivate.get();
         ASSERT(stream);
-        webkitMediaStreamSrcSetStream(WEBKIT_MEDIA_STREAM_SRC(sourceElement), stream);
+        webkitMediaStreamSrcSetStream(WEBKIT_MEDIA_STREAM_SRC(sourceElement), stream, m_player->isVideoPlayer());
 #endif
     }
@@ -1034 +1034 @@
     m_player->mediaEngineUpdated();
 }
+bool MediaPlayerPrivateGStreamer::hasFirstVideoSampleReachedSink() const
+{
+    auto sampleLocker = holdLock(m_sampleMutex);
+    return !!m_sample;
+}
 
 void MediaPlayerPrivateGStreamer::videoSinkCapsChanged(GstPad* videoSinkPad)
@@ -1046 +1051 @@
     GST_DEBUG_OBJECT(videoSinkPad, "Received new caps: %" GST_PTR_FORMAT, caps.get());
 
-    bool hasFirstSampleReachedSink;
-    // This actually lacks contention since both notify::caps and triggerRepaint() are both run in the same streaming thread.
-    {
-        auto sampleLocker = holdLock(m_sampleMutex);
-        hasFirstSampleReachedSink = !!m_sample;
-    }
-
-    if (!hasFirstSampleReachedSink) {
+    if (!hasFirstVideoSampleReachedSink()) {
         // We want to wait for the sink to receive the first buffer before emitting dimensions, since only by then we
         // are guaranteed that any potential tag event with a rotation has been handled.
@@ -1342 +1340 @@
 {
     GST_TRACE_OBJECT(pipeline(), "isEndReached: %s, seeking: %s, seekTime: %s", boolForPrinting(m_isEndReached), boolForPrinting(m_isSeeking), m_seekTime.toString().utf8().data());
+
+#if ENABLE(MEDIA_STREAM)
+    if (m_streamPrivate && m_player->isVideoPlayer() && !hasFirstVideoSampleReachedSink())
+        return MediaTime::zeroTime();
+#endif
+
     if (m_isSeeking)
         return m_seekTime;
@@ -1485 +1489 @@
 
 #define CREATE_TRACK(type, Type) G_STMT_START {                         \
-        m_has##Type = true;                                             \
         if (!useMediaSource) {                                          \
             RefPtr<Type##TrackPrivateGStreamer> track = Type##TrackPrivateGStreamer::create(makeWeakPtr(*this), type##TrackIndex, stream); \
@@ -1509 +1512 @@
         if (type & GST_STREAM_TYPE_AUDIO)
             CREATE_TRACK(audio, Audio);
-        else if (type & GST_STREAM_TYPE_VIDEO)
+        else if (type & GST_STREAM_TYPE_VIDEO && m_player->isVideoPlayer())
             CREATE_TRACK(video, Video);
         else if (type & GST_STREAM_TYPE_TEXT && !useMediaSource) {
@@ -1519 +1522 @@
     }
 
+    m_hasAudio = !m_audioTracks.isEmpty();
+    m_hasVideo = !m_videoTracks.isEmpty();
+
     if (oldHasVideo != m_hasVideo || oldHasAudio != m_hasAudio)
         m_player->characteristicChanged();
 
-    if (m_hasVideo)
+    if (!oldHasVideo && m_hasVideo)
         m_player->sizeChanged();
 

trunk/Source/WebCore/platform/graphics/gstreamer/MediaPlayerPrivateGStreamer.h

@@ -449 +449 @@
     void videoSinkCapsChanged(GstPad*);
     void updateVideoSizeAndOrientationFromCaps(const GstCaps*);
+    bool hasFirstVideoSampleReachedSink() const;
 
 #if ENABLE(ENCRYPTED_MEDIA)

trunk/Source/WebCore/platform/mediastream/gstreamer/GStreamerMediaStreamSource.cpp

@@ -575 +575 @@
 
     GST_OBJECT_LOCK(self);
-    String upstreamId;
-    if (priv->stream)
-        upstreamId = priv->stream->id();
-    else
-        upstreamId = createCanonicalUUIDString();
-
+    if (priv->stream && (!priv->stream->active() || !priv->stream->hasTracks())) {
+        GST_OBJECT_UNLOCK(self);
+        return;
+    }
+
+    auto upstreamId = priv->stream ? priv->stream->id() : createCanonicalUUIDString();
     priv->streamCollection = adoptGRef(gst_stream_collection_new(upstreamId.ascii().data()));
-    for (auto& track : priv->tracks)
+    for (auto& track : priv->tracks) {
+        if (!track->isActive())
+            continue;
         gst_stream_collection_add_stream(priv->streamCollection.get(), webkitMediaStreamNew(track.get()));
+    }
 
     GST_OBJECT_UNLOCK(self);
@@ -616 +619 @@
 }
 
-void webkitMediaStreamSrcSetStream(WebKitMediaStreamSrc* self, MediaStreamPrivate* stream)
+void webkitMediaStreamSrcSetStream(WebKitMediaStreamSrc* self, MediaStreamPrivate* stream, bool isVideoPlayer)
 {
     ASSERT(WEBKIT_IS_MEDIA_STREAM_SRC(self));
@@ -625 +628 @@
     auto tracks = stream->tracks();
     bool onlyTrack = tracks.size() == 1;
-    for (auto& track : tracks)
+    for (auto& track : tracks) {
+        if (!isVideoPlayer && track->type() == RealtimeMediaSource::Type::Video)
+            continue;
         webkitMediaStreamSrcAddTrack(self, track.get(), onlyTrack);
-
+    }
     webkitMediaStreamSrcPostStreamCollection(self);
     gst_element_no_more_pads(GST_ELEMENT_CAST(self));

trunk/Source/WebCore/platform/mediastream/gstreamer/GStreamerMediaStreamSource.h

@@ -55 +55 @@
 };
 
-void webkitMediaStreamSrcSetStream(WebKitMediaStreamSrc*, WebCore::MediaStreamPrivate*);
+void webkitMediaStreamSrcSetStream(WebKitMediaStreamSrc*, WebCore::MediaStreamPrivate*, bool isVideoPlayer);
 void webkitMediaStreamSrcAddTrack(WebKitMediaStreamSrc*, WebCore::MediaStreamTrackPrivate*, bool onlyTrack);
 GstElement* webkitMediaStreamSrcNew();