Changeset 208676 in webkit

Timestamp:

Nov 14, 2016 1:56:06 AM (7 years ago)

Author:

Philippe Normand

Message:

[GStreamer][OWR] poor video rendering in apprtc
​https://bugs.webkit.org/show_bug.cgi?id=164585

Reviewed by Xabier Rodriguez-Calvar.

The apprtc service uses 3 video elements in total, one for local, one
for remote and one called preview. During a call only remote and
preview are displayed, preview being linked to the same mediastream as
local. The consequence is that 2 OWR video renderers of the same
source are created. When gst-gl is enabled this isn't a problem but
when it is disabled a performance issue appears and the webkit video
sink starts dropping frames.

The solution is to have the video renderer shared between the 2
media players in this scenario.

• platform/graphics/gstreamer/MediaPlayerPrivateGStreamerBase.h:

(WebCore::MediaPlayerPrivateGStreamerBase::videoSink): Add video sink getter.

• platform/graphics/gstreamer/MediaPlayerPrivateGStreamerOwr.cpp:

(WebCore::MediaPlayerPrivateGStreamerOwr::load): Make sure the m_streamPrivate is
set before creating the video sink.
(WebCore::MediaPlayerPrivateGStreamerOwr::createVideoSink): Re-use video renderer
and sink if they have previously been created for another media player.

• platform/mediastream/MediaStreamPrivate.h: Store GStreamer sink and renderer so

they can be potentially used by multiple media players.
(WebCore::MediaStreamPrivate::setVideoRenderer):
(WebCore::MediaStreamPrivate::getVideoSinkElement):
(WebCore::MediaStreamPrivate::getVideoRenderer):

Location:

trunk/Source/WebCore

Files:

• ChangeLog (modified) (1 diff)
• platform/graphics/gstreamer/MediaPlayerPrivateGStreamerBase.h (modified) (1 diff)
• platform/graphics/gstreamer/MediaPlayerPrivateGStreamerOwr.cpp (modified) (3 diffs)
• platform/mediastream/MediaStreamPrivate.h (modified) (2 diffs)

trunk/Source/WebCore/ChangeLog

@@ +1 @@
+2016-11-10  Philippe Normand  <pnormand@igalia.com>
+
+        [GStreamer][OWR] poor video rendering in apprtc
+        https://bugs.webkit.org/show_bug.cgi?id=164585
+
+        Reviewed by Xabier Rodriguez-Calvar.
+
+        The apprtc service uses 3 video elements in total, one for local, one
+        for remote and one called preview. During a call only remote and
+        preview are displayed, preview being linked to the same mediastream as
+        local. The consequence is that 2 OWR video renderers of the same
+        source are created. When gst-gl is enabled this isn't a problem but
+        when it is disabled a performance issue appears and the webkit video
+        sink starts dropping frames.
+
+        The solution is to have the video renderer shared between the 2
+        media players in this scenario.
+
+        * platform/graphics/gstreamer/MediaPlayerPrivateGStreamerBase.h:
+        (WebCore::MediaPlayerPrivateGStreamerBase::videoSink): Add video sink getter.
+        * platform/graphics/gstreamer/MediaPlayerPrivateGStreamerOwr.cpp:
+        (WebCore::MediaPlayerPrivateGStreamerOwr::load): Make sure the m_streamPrivate is
+        set before creating the video sink.
+        (WebCore::MediaPlayerPrivateGStreamerOwr::createVideoSink): Re-use video renderer
+        and sink if they have previously been created for another media player.
+        * platform/mediastream/MediaStreamPrivate.h: Store GStreamer sink and renderer so
+        they can be potentially used by multiple media players.
+        (WebCore::MediaStreamPrivate::setVideoRenderer):
+        (WebCore::MediaStreamPrivate::getVideoSinkElement):
+        (WebCore::MediaStreamPrivate::getVideoRenderer):
+
 2016-11-13  Fujii Hironori  <Hironori.Fujii@sony.com>
 

trunk/Source/WebCore/platform/graphics/gstreamer/MediaPlayerPrivateGStreamerBase.h

@@ -157 +157 @@
 #endif
 
+    GstElement* videoSink() const { return m_videoSink.get(); }
+
     void setStreamVolumeElement(GstStreamVolume*);
     virtual GstElement* createAudioSink() { return 0; }

trunk/Source/WebCore/platform/graphics/gstreamer/MediaPlayerPrivateGStreamerOwr.cpp

@@ -143 +143 @@
         return;
 
-    if (streamPrivate.hasVideo() && !m_videoSink)
-        createVideoSink();
-
-    if (streamPrivate.hasAudio() && !m_audioSink)
-        createGSTAudioSinkBin();
-
-    GST_DEBUG("Loading MediaStreamPrivate %p video: %s, audio: %s", &streamPrivate, streamPrivate.hasVideo() ? "yes":"no", streamPrivate.hasAudio() ? "yes":"no");
-
     m_streamPrivate = &streamPrivate;
     if (!m_streamPrivate->active()) {
@@ -156 +148 @@
         return;
     }
+
+    if (streamPrivate.hasVideo() && !m_videoSink)
+        createVideoSink();
+
+    if (streamPrivate.hasAudio() && !m_audioSink)
+        createGSTAudioSinkBin();
+
+    GST_DEBUG("Loading MediaStreamPrivate %p video: %s, audio: %s", &streamPrivate, streamPrivate.hasVideo() ? "yes":"no", streamPrivate.hasAudio() ? "yes":"no");
 
     m_readyState = MediaPlayer::HaveNothing;
@@ -341 +341 @@
 GstElement* MediaPlayerPrivateGStreamerOwr::createVideoSink()
 {
+    GstElement* sink;
 #if USE(GSTREAMER_GL)
     // No need to create glupload and glcolorconvert here because they are
     // already created by the video renderer.
-    GstElement* sink = MediaPlayerPrivateGStreamerBase::createGLAppSink();
+    // FIXME: This should probably return a RefPtr. See https://bugs.webkit.org/show_bug.cgi?id=164709.
+    sink = MediaPlayerPrivateGStreamerBase::createGLAppSink();
     m_videoSink = sink;
 #else
-    GstElement* sink = gst_bin_new(nullptr);
-    GstElement* gldownload = gst_element_factory_make("gldownload", nullptr);
-    GstElement* videoconvert = gst_element_factory_make("videoconvert", nullptr);
-    GstElement* webkitSink = MediaPlayerPrivateGStreamerBase::createVideoSink();
-    gst_bin_add_many(GST_BIN(sink), gldownload, videoconvert, webkitSink, nullptr);
-    gst_element_link_many(gldownload, videoconvert, webkitSink, nullptr);
-    GRefPtr<GstPad> pad = gst_element_get_static_pad(gldownload, "sink");
-    gst_element_add_pad(sink, gst_ghost_pad_new("sink", pad.get()));
+    if (m_streamPrivate->getVideoRenderer()) {
+        m_videoRenderer = m_streamPrivate->getVideoRenderer();
+        m_videoSink = m_streamPrivate->getVideoSinkElement();
+        g_signal_connect_swapped(m_videoSink.get(), "repaint-requested", G_CALLBACK(MediaPlayerPrivateGStreamerBase::repaintCallback), this);
+        g_object_get(m_videoRenderer.get(), "sink", &sink, nullptr);
+    } else {
+        GstElement* gldownload = gst_element_factory_make("gldownload", nullptr);
+        GstElement* videoconvert = gst_element_factory_make("videoconvert", nullptr);
+        GstElement* webkitSink = MediaPlayerPrivateGStreamerBase::createVideoSink();
+        sink = gst_bin_new(nullptr);
+        gst_bin_add_many(GST_BIN(sink), gldownload, videoconvert, webkitSink, nullptr);
+        gst_element_link_many(gldownload, videoconvert, webkitSink, nullptr);
+        GRefPtr<GstPad> pad = adoptGRef(gst_element_get_static_pad(gldownload, "sink"));
+        gst_element_add_pad(sink, gst_ghost_pad_new("sink", pad.get()));
+    }
 #endif
-
-    m_videoRenderer = adoptGRef(owr_gst_video_renderer_new(sink));
+    if (!m_videoRenderer) {
+        m_videoRenderer = adoptGRef(owr_gst_video_renderer_new(sink));
 #if USE(GSTREAMER_GL)
-    owr_video_renderer_set_request_context_callback(OWR_VIDEO_RENDERER(m_videoRenderer.get()), (OwrVideoRendererRequestContextCallback) MediaPlayerPrivateGStreamerBase::requestGLContext, this, nullptr);
+        owr_video_renderer_set_request_context_callback(OWR_VIDEO_RENDERER(m_videoRenderer.get()), (OwrVideoRendererRequestContextCallback) MediaPlayerPrivateGStreamerBase::requestGLContext, this, nullptr);
 #endif
+        m_streamPrivate->setVideoRenderer(m_videoRenderer.get(), videoSink());
+    }
     return sink;
 }

trunk/Source/WebCore/platform/mediastream/MediaStreamPrivate.h

@@ -47 +47 @@
 #include <wtf/Vector.h>
 #include <wtf/WeakPtr.h>
+
+#if USE(GSTREAMER)
+#include "GRefPtrGStreamer.h"
+#include <owr/owr_gst_video_renderer.h>
+#endif
 
 namespace WebCore {
@@ -101 +106 @@
     WeakPtr<MediaStreamPrivate> createWeakPtr() { return m_weakPtrFactory.createWeakPtr(); }
 
+#if USE(GSTREAMER)
+    void setVideoRenderer(OwrGstVideoRenderer* renderer, GstElement* sink) { m_gstVideoRenderer = renderer; m_gstVideoSinkElement = sink; }
+    GRefPtr<GstElement> getVideoSinkElement() const { return m_gstVideoSinkElement; }
+    GRefPtr<OwrGstVideoRenderer> getVideoRenderer() const { return m_gstVideoRenderer; }
+
+private:
+    GRefPtr<GstElement> m_gstVideoSinkElement;
+    GRefPtr<OwrGstVideoRenderer> m_gstVideoRenderer;
+#endif
+
 private:
     MediaStreamPrivate(const String&, const MediaStreamTrackPrivateVector&);