Changeset 271575 in webkit


Ignore:
Timestamp:
Jan 18, 2021 12:40:55 AM (18 months ago)
Author:
youenn@apple.com
Message:

WebRTC live Opus audio stream stutters
​https://bugs.webkit.org/show_bug.cgi?id=220599
<rdar://problem/73190139>

Reviewed by Darin Adler.

Source/WebCore:

Before the patch, the MediaStreamTrack was owning its audio source provider.
Two MediaStreamAudioSourceNode reading the same track would read on the same provider buffer.
Instead, create a provider for each node.

Covered by updated test.

  • Modules/mediastream/MediaStreamTrack.cpp:

(WebCore::MediaStreamTrack::createAudioSourceProvider):

  • Modules/mediastream/MediaStreamTrack.h:
  • Modules/webaudio/MediaStreamAudioSourceNode.cpp:

(WebCore::MediaStreamAudioSourceNode::create):
(WebCore::MediaStreamAudioSourceNode::MediaStreamAudioSourceNode):
(WebCore::MediaStreamAudioSourceNode::~MediaStreamAudioSourceNode):
(WebCore::MediaStreamAudioSourceNode::provideInput):

  • Modules/webaudio/MediaStreamAudioSourceNode.h:
  • platform/mediastream/MediaStreamTrackPrivate.cpp:

(WebCore::MediaStreamTrackPrivate::createAudioSourceProvider):

  • platform/mediastream/MediaStreamTrackPrivate.h:

LayoutTests:

Add media stream source nodes to ensure that concurrent reading is not an issue anymore.

  • fast/mediastream/mock-media-source-webaudio.html:
Location:
trunk
Files:
9 edited

Legend:

Unmodified
Added
Removed

  • trunk/LayoutTests/ChangeLog

    r271554 r271575  
     12021-01-18  Youenn Fablet  <youenn@apple.com>
     2
     3        WebRTC live Opus audio stream stutters
     4        https://bugs.webkit.org/show_bug.cgi?id=220599
     5        <rdar://problem/73190139>
     6
     7        Reviewed by Darin Adler.
     8
     9        Add media stream source nodes to ensure that concurrent reading is not an issue anymore.
     10
     11        * fast/mediastream/mock-media-source-webaudio.html:
     12
    1132021-01-15  Rob Buis  <rbuis@igalia.com>
    214

  • trunk/LayoutTests/fast/mediastream/mock-media-source-webaudio.html

    r267504 r271575  
    3232            analyser.connect(gain);
    3333            gain.connect(context.destination);
     34
     35            const secondSource = context.createMediaStreamSource(stream);
     36            const sourceGain = new GainNode(context, { gain : 0 });
     37            secondSource.connect(gain);
     38            sourceGain.connect(context.destination);
    3439
    3540            function analyse() {

  • trunk/Source/WebCore/ChangeLog

    r271572 r271575  
     12021-01-18  Youenn Fablet  <youenn@apple.com>
     2
     3        WebRTC live Opus audio stream stutters
     4        https://bugs.webkit.org/show_bug.cgi?id=220599
     5        <rdar://problem/73190139>
     6
     7        Reviewed by Darin Adler.
     8
     9        Before the patch, the MediaStreamTrack was owning its audio source provider.
     10        Two MediaStreamAudioSourceNode reading the same track would read on the same provider buffer.
     11        Instead, create a provider for each node.
     12
     13        Covered by updated test.
     14
     15        * Modules/mediastream/MediaStreamTrack.cpp:
     16        (WebCore::MediaStreamTrack::createAudioSourceProvider):
     17        * Modules/mediastream/MediaStreamTrack.h:
     18        * Modules/webaudio/MediaStreamAudioSourceNode.cpp:
     19        (WebCore::MediaStreamAudioSourceNode::create):
     20        (WebCore::MediaStreamAudioSourceNode::MediaStreamAudioSourceNode):
     21        (WebCore::MediaStreamAudioSourceNode::~MediaStreamAudioSourceNode):
     22        (WebCore::MediaStreamAudioSourceNode::provideInput):
     23        * Modules/webaudio/MediaStreamAudioSourceNode.h:
     24        * platform/mediastream/MediaStreamTrackPrivate.cpp:
     25        (WebCore::MediaStreamTrackPrivate::createAudioSourceProvider):
     26        * platform/mediastream/MediaStreamTrackPrivate.h:
     27
    1282021-01-17  Kimmo Kinnunen  <kkinnunen@apple.com>
    229

  • trunk/Source/WebCore/Modules/mediastream/MediaStreamTrack.cpp

    r271488 r271575  
    5151#include "ScriptExecutionContext.h"
    5252#include "Settings.h"
     53#include "WebAudioSourceProvider.h"
    5354#include <wtf/CompletionHandler.h>
    5455#include <wtf/IsoMallocInlines.h>
    … …  
    623624}
    624625
    625 AudioSourceProvider* MediaStreamTrack::audioSourceProvider()
    626 {
    627     return m_private->audioSourceProvider();
     626RefPtr<WebAudioSourceProvider> MediaStreamTrack::createAudioSourceProvider()
     627{
     628    return m_private->createAudioSourceProvider();
    628629}
    629630

  • trunk/Source/WebCore/Modules/mediastream/MediaStreamTrack.h

    r271440 r271575  
    140140    MediaStreamTrackPrivate& privateTrack() { return m_private.get(); }
    141141
    142     AudioSourceProvider* audioSourceProvider();
     142    RefPtr<WebAudioSourceProvider> createAudioSourceProvider();
    143143
    144144    MediaProducer::MediaStateFlags mediaState() const;

  • trunk/Source/WebCore/Modules/webaudio/MediaStreamAudioSourceNode.cpp

    r270468 r271575  
    3535#include "Logging.h"
    3636#include "MediaStreamAudioSourceOptions.h"
     37#include "WebAudioSourceProvider.h"
    3738#include <wtf/IsoMallocInlines.h>
    3839#include <wtf/Locker.h>
    … …  
    5051        return Exception { InvalidStateError, "Media stream has no audio tracks"_s };
    5152
    52     MediaStreamTrack* providerTrack = nullptr;
     53    RefPtr<WebAudioSourceProvider> provider;
    5354    for (auto& track : audioTracks) {
    54         if (track->audioSourceProvider()) {
    55             providerTrack = track.get();
     55        provider = track->createAudioSourceProvider();
     56        if (provider)
    5657            break;
    57         }
    5858    }
    59     if (!providerTrack)
     59    if (!provider)
    6060        return Exception { InvalidStateError, "Could not find an audio track with an audio source provider"_s };
    6161
    62     auto node = adoptRef(*new MediaStreamAudioSourceNode(context, *options.mediaStream, *providerTrack));
     62    auto node = adoptRef(*new MediaStreamAudioSourceNode(context, *options.mediaStream, provider.releaseNonNull()));
    6363    node->setFormat(2, context.sampleRate());
    6464
    … …  
    6969}
    7070
    71 MediaStreamAudioSourceNode::MediaStreamAudioSourceNode(BaseAudioContext& context, MediaStream& mediaStream, MediaStreamTrack& audioTrack)
     71MediaStreamAudioSourceNode::MediaStreamAudioSourceNode(BaseAudioContext& context, MediaStream& mediaStream, Ref<WebAudioSourceProvider>&& provider)
    7272    : AudioNode(context, NodeTypeMediaStreamAudioSource)
    7373    , m_mediaStream(mediaStream)
    74     , m_audioTrack(audioTrack)
     74    , m_provider(provider)
    7575{
    76     AudioSourceProvider* audioSourceProvider = m_audioTrack->audioSourceProvider();
    77     ASSERT(audioSourceProvider);
    78 
    79     audioSourceProvider->setClient(this);
     76    m_provider->setClient(this);
    8077   
    8178    // Default to stereo. This could change depending on the format of the MediaStream's audio track.
    … …  
    8784MediaStreamAudioSourceNode::~MediaStreamAudioSourceNode()
    8885{
    89     AudioSourceProvider* audioSourceProvider = m_audioTrack->audioSourceProvider();
    90     ASSERT(audioSourceProvider);
    91     audioSourceProvider->setClient(nullptr);
     86    m_provider->setClient(nullptr);
    9287    uninitialize();
    9388}
    … …  
    133128void MediaStreamAudioSourceNode::provideInput(AudioBus* bus, size_t framesToProcess)
    134129{
    135     if (auto* provider = m_audioTrack->audioSourceProvider())
    136         provider->provideInput(bus, framesToProcess);
    137     else
    138         bus->zero();
     130    m_provider->provideInput(bus, framesToProcess);
    139131}
    140132

  • trunk/Source/WebCore/Modules/webaudio/MediaStreamAudioSourceNode.h

    r270468 r271575  
    3838struct MediaStreamAudioSourceOptions;
    3939class MultiChannelResampler;
     40class WebAudioSourceProvider;
    4041
    4142class MediaStreamAudioSourceNode final : public AudioNode, public AudioSourceProviderClient {
    … …  
    4445    static ExceptionOr<Ref<MediaStreamAudioSourceNode>> create(BaseAudioContext&, MediaStreamAudioSourceOptions&&);
    4546
    46     virtual ~MediaStreamAudioSourceNode();
     47    ~MediaStreamAudioSourceNode();
    4748
    4849    MediaStream* mediaStream() { return &m_mediaStream.get(); }
    4950
     51private:
     52    MediaStreamAudioSourceNode(BaseAudioContext&, MediaStream&, Ref<WebAudioSourceProvider>&&);
     53
    5054    // AudioNode
    51     void process(size_t framesToProcess) override;
    52 
     55    void process(size_t framesToProcess) final;
    5356    // AudioSourceProviderClient
    54     void setFormat(size_t numberOfChannels, float sampleRate) override;
    55 
    56 private:
    57     MediaStreamAudioSourceNode(BaseAudioContext&, MediaStream&, MediaStreamTrack&);
     57    void setFormat(size_t numberOfChannels, float sampleRate) final;
    5858
    5959    void provideInput(AudioBus*, size_t framesToProcess);
    … …  
    6767
    6868    Ref<MediaStream> m_mediaStream;
    69     Ref<MediaStreamTrack> m_audioTrack;
     69    Ref<WebAudioSourceProvider> m_provider;
    7070    std::unique_ptr<MultiChannelResampler> m_multiChannelResampler;
    7171

  • trunk/Source/WebCore/platform/mediastream/MediaStreamTrackPrivate.cpp

    r268521 r271575  
    185185}
    186186
    187 AudioSourceProvider* MediaStreamTrackPrivate::audioSourceProvider()
     187RefPtr<WebAudioSourceProvider> MediaStreamTrackPrivate::createAudioSourceProvider()
    188188{
    189189#if PLATFORM(COCOA)
    190     if (!m_audioSourceProvider)
    191         m_audioSourceProvider = MediaStreamTrackAudioSourceProviderCocoa::create(*this);
     190    return MediaStreamTrackAudioSourceProviderCocoa::create(*this);
    192191#elif USE(LIBWEBRTC) && USE(GSTREAMER)
    193     if (!m_audioSourceProvider)
    194         m_audioSourceProvider = AudioSourceProviderGStreamer::create(*this);
    195 #endif
    196     return m_audioSourceProvider.get();
     192    return AudioSourceProviderGStreamer::create(*this);
     193#else
     194    return nullptr;
     195#endif
    197196}
    198197

  • trunk/Source/WebCore/platform/mediastream/MediaStreamTrackPrivate.h

    r261553 r271575  
    3737namespace WebCore {
    3838
    39 class AudioSourceProvider;
    4039class GraphicsContext;
    4140class MediaSample;
    … …  
    110109    void applyConstraints(const MediaConstraints&, RealtimeMediaSource::ApplyConstraintsHandler&&);
    111110
    112     AudioSourceProvider* audioSourceProvider();
     111    RefPtr<WebAudioSourceProvider> createAudioSourceProvider();
    113112
    114113    void paintCurrentFrameInContext(GraphicsContext&, const FloatRect&);
    … …  
    154153    bool m_hasStartedProducingData { false };
    155154    HintValue m_contentHint { HintValue::Empty };
    156     RefPtr<WebAudioSourceProvider> m_audioSourceProvider;
    157155    Ref<const Logger> m_logger;
    158156#if !RELEASE_LOG_DISABLED
Note: See TracChangeset for help on using the changeset viewer.