Changeset 269849 in webkit


Ignore:
Timestamp:
Nov 16, 2020 4:46:38 AM (3 years ago)
Author:
commit-queue@webkit.org
Message:

[GStreamer] Clean-up Audio{Data,StreamDescription} implementations
https://bugs.webkit.org/show_bug.cgi?id=218957

Patch by Philippe Normand <pnormand@igalia.com> on 2020-11-16
Reviewed by Xabier Rodriguez-Calvar.

Refactor the GStreamerAudioData and GStreamerStreamDescription implementations in order to
avoid un-necessary copies. The call-sites were adapted accordingly. Some usage of the latter
class was removed because it was simpler to use the GstAudioInfo API directly.

  • Modules/webaudio/MediaStreamAudioSourceGStreamer.cpp:

(WebCore::MediaStreamAudioSource::consumeAudio):

  • platform/audio/gstreamer/GStreamerAudioData.h:
  • platform/audio/gstreamer/GStreamerAudioStreamDescription.h:
  • platform/mediastream/gstreamer/GStreamerAudioCaptureSource.cpp:

(WebCore::GStreamerAudioCaptureSource::newSampleCallback):

  • platform/mediastream/gstreamer/GStreamerMediaStreamSource.cpp:

(webkitMediaStreamSrcPushAudioSample):

  • platform/mediastream/gstreamer/MockRealtimeAudioSourceGStreamer.cpp:

(WebCore::MockRealtimeAudioSourceGStreamer::render):

  • platform/mediastream/gstreamer/RealtimeIncomingAudioSourceLibWebRTC.cpp:

(WebCore::RealtimeIncomingAudioSourceLibWebRTC::OnData):

  • platform/mediastream/gstreamer/RealtimeOutgoingAudioSourceLibWebRTC.cpp:

(WebCore::libwebrtcAudioFormat):
(WebCore::RealtimeOutgoingAudioSourceLibWebRTC::audioSamplesAvailable):
(WebCore::RealtimeOutgoingAudioSourceLibWebRTC::pullAudioData):

  • platform/mediastream/gstreamer/RealtimeOutgoingAudioSourceLibWebRTC.h:
Location:
trunk/Source/WebCore
Files:
10 edited

Legend:

Unmodified
Added
Removed
  • trunk/Source/WebCore/ChangeLog

    r269848 r269849  
     12020-11-16  Philippe Normand  <pnormand@igalia.com>
     2
     3        [GStreamer] Clean-up Audio{Data,StreamDescription} implementations
     4        https://bugs.webkit.org/show_bug.cgi?id=218957
     5
     6        Reviewed by Xabier Rodriguez-Calvar.
     7
     8        Refactor the GStreamerAudioData and GStreamerStreamDescription implementations in order to
     9        avoid un-necessary copies. The call-sites were adapted accordingly. Some usage of the latter
     10        class was removed because it was simpler to use the GstAudioInfo API directly.
     11
     12        * Modules/webaudio/MediaStreamAudioSourceGStreamer.cpp:
     13        (WebCore::MediaStreamAudioSource::consumeAudio):
     14        * platform/audio/gstreamer/GStreamerAudioData.h:
     15        * platform/audio/gstreamer/GStreamerAudioStreamDescription.h:
     16        * platform/mediastream/gstreamer/GStreamerAudioCaptureSource.cpp:
     17        (WebCore::GStreamerAudioCaptureSource::newSampleCallback):
     18        * platform/mediastream/gstreamer/GStreamerMediaStreamSource.cpp:
     19        (webkitMediaStreamSrcPushAudioSample):
     20        * platform/mediastream/gstreamer/MockRealtimeAudioSourceGStreamer.cpp:
     21        (WebCore::MockRealtimeAudioSourceGStreamer::render):
     22        * platform/mediastream/gstreamer/RealtimeIncomingAudioSourceLibWebRTC.cpp:
     23        (WebCore::RealtimeIncomingAudioSourceLibWebRTC::OnData):
     24        * platform/mediastream/gstreamer/RealtimeOutgoingAudioSourceLibWebRTC.cpp:
     25        (WebCore::libwebrtcAudioFormat):
     26        (WebCore::RealtimeOutgoingAudioSourceLibWebRTC::audioSamplesAvailable):
     27        (WebCore::RealtimeOutgoingAudioSourceLibWebRTC::pullAudioData):
     28        * platform/mediastream/gstreamer/RealtimeOutgoingAudioSourceLibWebRTC.h:
     29
    1302020-11-16  Andres Gonzalez  <andresg_22@apple.com>
    231
  • trunk/Source/WebCore/Modules/webaudio/MediaStreamAudioSourceGStreamer.cpp

    r269827 r269849  
    5353    }
    5454
    55     auto mediaTime = MediaTime((m_numberOfFrames * G_USEC_PER_SEC) / m_currentSettings.sampleRate(), G_USEC_PER_SEC);
     55    MediaTime mediaTime((m_numberOfFrames * G_USEC_PER_SEC) / m_currentSettings.sampleRate(), G_USEC_PER_SEC);
    5656    m_numberOfFrames += numberOfFrames;
    5757
     
    6565    copyBusData(bus, buffer.get(), muted());
    6666    auto sample = adoptGRef(gst_sample_new(buffer.get(), caps.get(), nullptr, nullptr));
    67     m_audioBuffer = makeUnique<GStreamerAudioData>(WTFMove(sample), info);
    68 
    69     GStreamerAudioStreamDescription description(info);
    70     audioSamplesAvailable(mediaTime, *m_audioBuffer, description, numberOfFrames);
     67    GStreamerAudioData audioBuffer(WTFMove(sample), info);
     68    GStreamerAudioStreamDescription description(&info);
     69    audioSamplesAvailable(mediaTime, audioBuffer, description, numberOfFrames);
    7170}
    7271
  • trunk/Source/WebCore/platform/audio/gstreamer/GStreamerAudioData.h

    r269827 r269849  
    3333class GStreamerAudioData final : public PlatformAudioData {
    3434public:
    35     GStreamerAudioData(GRefPtr<GstSample>&& sample, GstAudioInfo info)
     35    GStreamerAudioData(GRefPtr<GstSample>&& sample, GstAudioInfo&& info)
     36        : m_sample(WTFMove(sample))
     37        , m_audioInfo(WTFMove(info))
     38    {
     39    }
     40
     41    GStreamerAudioData(GRefPtr<GstSample>&& sample, const GstAudioInfo& info)
    3642        : m_sample(WTFMove(sample))
    3743        , m_audioInfo(info)
     
    4652
    4753    void setSample(GRefPtr<GstSample>&& sample) { m_sample = WTFMove(sample); }
    48     GstSample* getSample() { return m_sample.get(); }
    49     GstAudioInfo getAudioInfo() { return m_audioInfo; }
     54    const GRefPtr<GstSample>& getSample() const { return m_sample; }
     55    const GstAudioInfo& getAudioInfo() const { return m_audioInfo; }
    5056    uint32_t channelCount() const { return GST_AUDIO_INFO_CHANNELS(&m_audioInfo); }
    5157
  • trunk/Source/WebCore/platform/audio/gstreamer/GStreamerAudioStreamDescription.h

    r269827 r269849  
    3131class GStreamerAudioStreamDescription final: public AudioStreamDescription {
    3232public:
    33     GStreamerAudioStreamDescription(GstAudioInfo info)
     33    GStreamerAudioStreamDescription(GstAudioInfo&& info)
     34        : m_info(WTFMove(info))
     35        , m_caps(adoptGRef(gst_audio_info_to_caps(&m_info)))
     36    {
     37    }
     38
     39    GStreamerAudioStreamDescription(const GstAudioInfo& info)
    3440        : m_info(info)
    3541        , m_caps(adoptGRef(gst_audio_info_to_caps(&m_info)))
     
    3743    }
    3844
    39     GStreamerAudioStreamDescription(GstAudioInfo *info)
     45    GStreamerAudioStreamDescription(GstAudioInfo* info)
    4046        : m_info(*info)
    4147        , m_caps(adoptGRef(gst_audio_info_to_caps(&m_info)))
     
    94100    bool operator!=(const GStreamerAudioStreamDescription& other) { return !operator == (other); }
    95101
    96     GstCaps* caps() { return m_caps.get(); }
    97     GstAudioInfo* getInfo() { return &m_info; }
     102    const GRefPtr<GstCaps>& caps() const { return m_caps; }
     103    const GstAudioInfo& getInfo() const { return m_info; }
    98104
    99105private:
  • trunk/Source/WebCore/platform/mediastream/gstreamer/GStreamerAudioCaptureSource.cpp

    r267934 r269849  
    125125
    126126    // FIXME - figure out a way to avoid copying (on write) the data.
    127     GstBuffer* buf = gst_sample_get_buffer(sample.get());
    128     auto frames(std::unique_ptr<GStreamerAudioData>(new GStreamerAudioData(WTFMove(sample))));
    129     auto streamDesc(std::unique_ptr<GStreamerAudioStreamDescription>(new GStreamerAudioStreamDescription(frames->getAudioInfo())));
     127    auto* buffer = gst_sample_get_buffer(sample.get());
     128    GStreamerAudioData frames(WTFMove(sample));
     129    GStreamerAudioStreamDescription description(frames.getAudioInfo());
    130130
    131131    source->audioSamplesAvailable(
    132         MediaTime(GST_TIME_AS_USECONDS(GST_BUFFER_PTS(buf)), G_USEC_PER_SEC),
    133         *frames, *streamDesc, gst_buffer_get_size(buf) / frames->getAudioInfo().bpf);
     132        MediaTime(GST_TIME_AS_USECONDS(GST_BUFFER_PTS(buffer)), G_USEC_PER_SEC),
     133        frames, description, gst_buffer_get_size(buffer) / description.getInfo().bpf);
    134134
    135135    return GST_FLOW_OK;
  • trunk/Source/WebCore/platform/mediastream/gstreamer/GStreamerMediaStreamSource.cpp

    r264648 r269849  
    4141
    4242static void webkitMediaStreamSrcPushVideoSample(WebKitMediaStreamSrc*, GstSample*);
    43 static void webkitMediaStreamSrcPushAudioSample(WebKitMediaStreamSrc*, GstSample*);
     43static void webkitMediaStreamSrcPushAudioSample(WebKitMediaStreamSrc*, const GRefPtr<GstSample>&);
    4444static void webkitMediaStreamSrcTrackEnded(WebKitMediaStreamSrc*, MediaStreamTrackPrivate&);
    4545static void webkitMediaStreamSrcRemoveTrackByType(WebKitMediaStreamSrc*, RealtimeMediaSource::Type);
     
    592592}
    593593
    594 static void webkitMediaStreamSrcPushAudioSample(WebKitMediaStreamSrc* self, GstSample* sample)
     594static void webkitMediaStreamSrcPushAudioSample(WebKitMediaStreamSrc* self, const GRefPtr<GstSample>& sample)
    595595{
    596596    if (self->priv->audioSrc)
    597         self->priv->audioSrc->pushSample(sample);
     597        self->priv->audioSrc->pushSample(sample.get());
    598598}
    599599
  • trunk/Source/WebCore/platform/mediastream/gstreamer/MockRealtimeAudioSourceGStreamer.cpp

    r267934 r269849  
    8282
    8383        ASSERT(m_streamFormat);
    84         GstAudioInfo* info = m_streamFormat->getInfo();
     84        const auto& info = m_streamFormat->getInfo();
    8585        GRefPtr<GstBuffer> buffer = adoptGRef(gst_buffer_new_allocate(nullptr, bipBopCount * m_streamFormat->bytesPerFrame(), nullptr));
    8686        {
     
    8888
    8989            if (muted())
    90                 gst_audio_format_fill_silence(info->finfo, map.data(), map.size());
     90                gst_audio_format_fill_silence(info.finfo, map.data(), map.size());
    9191            else {
    9292                memcpy(map.data(), &m_bipBopBuffer[bipBopStart], sizeof(float) * bipBopCount);
     
    9999        frameCount = std::min(totalFrameCount, m_maximiumFrameCount);
    100100
    101         GRefPtr<GstCaps> caps = adoptGRef(gst_audio_info_to_caps(info));
     101        auto caps = adoptGRef(gst_audio_info_to_caps(&info));
    102102        auto sample = adoptGRef(gst_sample_new(buffer.get(), caps.get(), nullptr, nullptr));
    103         auto data(std::unique_ptr<GStreamerAudioData>(new GStreamerAudioData(WTFMove(sample), *info)));
    104         auto mediaTime = MediaTime((m_samplesRendered * G_USEC_PER_SEC) / sampleRate(), G_USEC_PER_SEC);
    105         audioSamplesAvailable(mediaTime, *data.get(), *m_streamFormat, bipBopCount);
     103        GStreamerAudioData data(WTFMove(sample), info);
     104        MediaTime mediaTime((m_samplesRendered * G_USEC_PER_SEC) / sampleRate(), G_USEC_PER_SEC);
     105        audioSamplesAvailable(mediaTime, data, *m_streamFormat, bipBopCount);
    106106    }
    107107}
  • trunk/Source/WebCore/platform/mediastream/gstreamer/RealtimeIncomingAudioSourceLibWebRTC.cpp

    r243163 r269849  
    7373
    7474    auto buffer = adoptGRef(gst_buffer_new_wrapped(bufferData, bufferSize));
    75     GRefPtr<GstCaps> caps = adoptGRef(gst_audio_info_to_caps(&info));
     75    auto caps = adoptGRef(gst_audio_info_to_caps(&info));
    7676    auto sample = adoptGRef(gst_sample_new(buffer.get(), caps.get(), nullptr, nullptr));
    77     auto data(std::unique_ptr<GStreamerAudioData>(new GStreamerAudioData(WTFMove(sample), info)));
    78 
    79     auto mediaTime = MediaTime((m_numberOfFrames * G_USEC_PER_SEC) / sampleRate, G_USEC_PER_SEC);
    80     audioSamplesAvailable(mediaTime, *data.get(), GStreamerAudioStreamDescription(info), numberOfFrames);
     77    GStreamerAudioData data(WTFMove(sample), info);
     78    MediaTime mediaTime((m_numberOfFrames * G_USEC_PER_SEC) / sampleRate, G_USEC_PER_SEC);
     79    audioSamplesAvailable(mediaTime, data, GStreamerAudioStreamDescription(info), numberOfFrames);
    8180
    8281    m_numberOfFrames += numberOfFrames;
  • trunk/Source/WebCore/platform/mediastream/gstreamer/RealtimeOutgoingAudioSourceLibWebRTC.cpp

    r269827 r269849  
    2323#include "RealtimeOutgoingAudioSourceLibWebRTC.h"
    2424
     25#include "GStreamerAudioData.h"
     26#include "GStreamerAudioStreamDescription.h"
    2527#include "LibWebRTCAudioFormat.h"
    2628#include "LibWebRTCProvider.h"
    2729#include "NotImplemented.h"
    28 #include "gstreamer/GStreamerAudioData.h"
    2930
    3031namespace WebCore {
     
    4748}
    4849
    49 static inline std::unique_ptr<GStreamerAudioStreamDescription> libwebrtcAudioFormat(int sampleRate,
    50     size_t channelCount)
     50static inline GstAudioInfo libwebrtcAudioFormat(int sampleRate, size_t channelCount)
    5151{
    5252    GstAudioFormat format = gst_audio_format_build_integer(
     
    6060    size_t libWebRTCChannelCount = channelCount >= 2 ? 2 : channelCount;
    6161    gst_audio_info_set_format(&info, format, sampleRate, libWebRTCChannelCount, nullptr);
    62 
    63     return std::unique_ptr<GStreamerAudioStreamDescription>(new GStreamerAudioStreamDescription(info));
     62    return info;
    6463}
    6564
    66 void RealtimeOutgoingAudioSourceLibWebRTC::audioSamplesAvailable(const MediaTime&,
    67     const PlatformAudioData& audioData, const AudioStreamDescription& streamDescription,
    68     size_t /* sampleCount */)
     65void RealtimeOutgoingAudioSourceLibWebRTC::audioSamplesAvailable(const MediaTime&, const PlatformAudioData& audioData, const AudioStreamDescription& streamDescription, size_t /* sampleCount */)
    6966{
    7067    auto data = static_cast<const GStreamerAudioData&>(audioData);
    7168    auto desc = static_cast<const GStreamerAudioStreamDescription&>(streamDescription);
    7269
    73     if (m_sampleConverter && !gst_audio_info_is_equal(m_inputStreamDescription->getInfo(), desc.getInfo())) {
     70    if (m_sampleConverter && !gst_audio_info_is_equal(&m_inputStreamDescription, &desc.getInfo())) {
    7471        GST_ERROR_OBJECT(this, "FIXME - Audio format renegotiation is not possible yet!");
    7572        m_sampleConverter = nullptr;
     
    7774
    7875    if (!m_sampleConverter) {
    79         m_inputStreamDescription = std::unique_ptr<GStreamerAudioStreamDescription>(new GStreamerAudioStreamDescription(desc.getInfo()));
    80         m_outputStreamDescription = libwebrtcAudioFormat(LibWebRTCAudioFormat::sampleRate, streamDescription.numberOfChannels());
    81         m_sampleConverter.reset(gst_audio_converter_new(GST_AUDIO_CONVERTER_FLAG_IN_WRITABLE,
    82             m_inputStreamDescription->getInfo(),
    83             m_outputStreamDescription->getInfo(),
    84             nullptr));
     76        m_inputStreamDescription = desc.getInfo();
     77        m_outputStreamDescription = libwebrtcAudioFormat(LibWebRTCAudioFormat::sampleRate, desc.numberOfChannels());
     78        m_sampleConverter.reset(gst_audio_converter_new(GST_AUDIO_CONVERTER_FLAG_IN_WRITABLE, &m_inputStreamDescription,
     79            &m_outputStreamDescription, nullptr));
    8580    }
    8681
    8782    {
    8883        LockHolder locker(m_adapterMutex);
    89         auto* buffer = gst_sample_get_buffer(data.getSample());
     84        const auto& sample = data.getSample();
     85        auto* buffer = gst_sample_get_buffer(sample.get());
    9086        gst_adapter_push(m_adapter.get(), gst_buffer_ref(buffer));
    9187    }
     
    9793void RealtimeOutgoingAudioSourceLibWebRTC::pullAudioData()
    9894{
    99     if (!m_inputStreamDescription || !m_outputStreamDescription) {
     95    if (!GST_AUDIO_INFO_IS_VALID(&m_inputStreamDescription) || !GST_AUDIO_INFO_IS_VALID(&m_outputStreamDescription)) {
    10096        GST_INFO("No stream description set yet.");
    101 
    10297        return;
    10398    }
    10499
    105100    size_t outChunkSampleCount = LibWebRTCAudioFormat::chunkSampleCount;
    106     size_t outBufferSize = outChunkSampleCount * m_outputStreamDescription->getInfo()->bpf;
     101    size_t outBufferSize = outChunkSampleCount * m_outputStreamDescription.bpf;
    107102
    108103    LockHolder locker(m_adapterMutex);
    109104    size_t inChunkSampleCount = gst_audio_converter_get_in_frames(m_sampleConverter.get(), outChunkSampleCount);
    110     size_t inBufferSize = inChunkSampleCount * m_inputStreamDescription->getInfo()->bpf;
     105    size_t inBufferSize = inChunkSampleCount * m_inputStreamDescription.bpf;
    111106
    112107    while (gst_adapter_available(m_adapter.get()) > inBufferSize) {
     
    114109        m_audioBuffer.grow(outBufferSize);
    115110        if (isSilenced())
    116             gst_audio_format_fill_silence(m_outputStreamDescription->getInfo()->finfo, m_audioBuffer.data(), outBufferSize);
     111            gst_audio_format_fill_silence(m_outputStreamDescription.finfo, m_audioBuffer.data(), outBufferSize);
    117112        else {
    118113            GstMappedBuffer inMap(inBuffer.get(), GST_MAP_READ);
     
    127122        }
    128123
    129         sendAudioFrames(m_audioBuffer.data(), LibWebRTCAudioFormat::sampleSize, static_cast<int>(m_outputStreamDescription->sampleRate()),
    130             static_cast<int>(m_outputStreamDescription->numberOfChannels()), outChunkSampleCount);
     124        sendAudioFrames(m_audioBuffer.data(), LibWebRTCAudioFormat::sampleSize, GST_AUDIO_INFO_RATE(&m_outputStreamDescription),
     125            GST_AUDIO_INFO_CHANNELS(&m_outputStreamDescription), outChunkSampleCount);
    131126    }
    132127}
  • trunk/Source/WebCore/platform/mediastream/gstreamer/RealtimeOutgoingAudioSourceLibWebRTC.h

    r248278 r269849  
    2222#if USE(LIBWEBRTC)
    2323
    24 #include "GStreamerAudioStreamDescription.h"
    2524#include "GStreamerCommon.h"
    2625#include "RealtimeOutgoingAudioSource.h"
     
    5049
    5150    GUniquePtr<GstAudioConverter> m_sampleConverter;
    52     std::unique_ptr<GStreamerAudioStreamDescription> m_inputStreamDescription;
    53     std::unique_ptr<GStreamerAudioStreamDescription> m_outputStreamDescription;
     51    GstAudioInfo m_inputStreamDescription;
     52    GstAudioInfo m_outputStreamDescription;
    5453
    5554    Lock m_adapterMutex;
Note: See TracChangeset for help on using the changeset viewer.