Changeset 278905 in webkit

Timestamp:

Jun 15, 2021, 2:49:42 PM (4 years ago)

Author:

Alan Coon

Message:

Apply patch. rdar://problem/79355311

Location:

branches/safari-611-branch/Source/WebCore

Files:

• ChangeLog (modified) (1 diff)
• platform/audio/cocoa/AudioSampleDataSource.h (modified) (3 diffs)
• platform/audio/cocoa/AudioSampleDataSource.mm (modified) (3 diffs)
• platform/mediastream/libwebrtc/LibWebRTCAudioModule.cpp (modified) (2 diffs)
• platform/mediastream/libwebrtc/LibWebRTCAudioModule.h (modified) (1 diff)
• platform/mediastream/mac/AudioMediaStreamTrackRendererCocoa.cpp (modified) (3 diffs)
• platform/mediastream/mac/MediaStreamTrackAudioSourceProviderCocoa.cpp (modified) (2 diffs)
• platform/mediastream/mac/WebAudioSourceProviderCocoa.h (modified) (2 diffs)
• platform/mediastream/mac/WebAudioSourceProviderCocoa.mm (modified) (1 diff)

branches/safari-611-branch/Source/WebCore/ChangeLog

@@ +1 @@
+2021-06-15  Alan Coon  <alancoon@apple.com>
+
+        Apply patch. rdar://problem/79355311
+
+    2021-06-15  Youenn Fablet  <youenn@apple.com>
+
+            Reduce crackling at start of playing a live audio track
+            https://bugs.webkit.org/show_bug.cgi?id=218898
+            <rdar://problem/71625010>
+
+            Reviewed by Eric Carlson.
+
+            We often hear crackles at the beginning of playing a MediaStreamTrack.
+            This is due to starting at a point where there is not enough data, so we start rendering and quickly run short of data.
+            Our heuristic to start was not good since we were not accounting to LibWebRTC audio module which sends audio samples by batch of 5 samples.
+            So we would have needed to at least have 6 samples to be sure to not run out of data.
+
+            What this patch does:
+            - Add an extra parameter to AudioSampleDataSource so that we do not start until we have at least been pushed a given number of sample count.
+            - Change LibWebRTAudioModule to send audio samples by batch of 3 (hence 30 ms) instead of 5
+            - Set this new parameter to 2 for local tracks and 4 for remote tracks
+            - Add a new boolean to AudioSampleDataSource to know whether we start the track for the first time or not. If we start it, use the new parameter to buffer enough data.
+              Otherwise, use current heuristic.
+
+            We also reduce the AudioSampleDataSource buffer to 0.5 seconds instead of 2 seconds, since 2 seconds is too much for real time audio.
+
+            Manually tested.
+
+            * platform/audio/cocoa/AudioSampleDataSource.h:
+            * platform/audio/cocoa/AudioSampleDataSource.mm:
+            (WebCore::AudioSampleDataSource::create):
+            (WebCore::AudioSampleDataSource::AudioSampleDataSource):
+            (WebCore::AudioSampleDataSource::pullSamplesInternal):
+            * platform/mediastream/libwebrtc/LibWebRTCAudioModule.cpp:
+            (WebCore::LibWebRTCAudioModule::pollFromSource):
+            * platform/mediastream/libwebrtc/LibWebRTCAudioModule.h:
+            * platform/mediastream/mac/AudioMediaStreamTrackRendererCocoa.cpp:
+            (WebCore::pollSamplesCount):
+            (WebCore::AudioMediaStreamTrackRendererCocoa::pushSamples):
+            * platform/mediastream/mac/MediaStreamTrackAudioSourceProviderCocoa.cpp:
+            (WebCore::MediaStreamTrackAudioSourceProviderCocoa::MediaStreamTrackAudioSourceProviderCocoa):
+            * platform/mediastream/mac/WebAudioSourceProviderCocoa.h:
+            * platform/mediastream/mac/WebAudioSourceProviderCocoa.mm:
+            (WebCore::WebAudioSourceProviderCocoa::prepare):
+
 2021-06-15  Alan Coon  <alancoon@apple.com>
 

branches/safari-611-branch/Source/WebCore/platform/audio/cocoa/AudioSampleDataSource.h

@@ -48 +48 @@
     {
 public:
-    static Ref<AudioSampleDataSource> create(size_t, WTF::LoggerHelper&);
+    static Ref<AudioSampleDataSource> create(size_t, WTF::LoggerHelper&, size_t waitToStartForPushCount = 2);
 
     ~AudioSampleDataSource();
@@ -81 +81 @@
 
 private:
-    AudioSampleDataSource(size_t, LoggerHelper&);
+    AudioSampleDataSource(size_t, LoggerHelper&, size_t waitToStartForPushCount);
 
     OSStatus setupConverter();
@@ -99 +99 @@
 
     uint64_t m_lastPushedSampleCount { 0 };
+    size_t m_waitToStartForPushCount { 2 };
     MediaTime m_expectedNextPushedSampleTime { MediaTime::invalidTime() };
+    bool m_isFirstPull { true };
 
     MediaTime m_inputSampleOffset;

branches/safari-611-branch/Source/WebCore/platform/audio/cocoa/AudioSampleDataSource.mm

@@ -45 +45 @@
 using namespace JSC;
 
-Ref<AudioSampleDataSource> AudioSampleDataSource::create(size_t maximumSampleCount, LoggerHelper& loggerHelper)
-{
-    return adoptRef(*new AudioSampleDataSource(maximumSampleCount, loggerHelper));
-}
-
-AudioSampleDataSource::AudioSampleDataSource(size_t maximumSampleCount, LoggerHelper& loggerHelper)
-    : m_inputSampleOffset(MediaTime::invalidTime())
+Ref<AudioSampleDataSource> AudioSampleDataSource::create(size_t maximumSampleCount, LoggerHelper& loggerHelper, size_t waitToStartForPushCount)
+{
+    return adoptRef(*new AudioSampleDataSource(maximumSampleCount, loggerHelper, waitToStartForPushCount));
+}
+
+AudioSampleDataSource::AudioSampleDataSource(size_t maximumSampleCount, LoggerHelper& loggerHelper, size_t waitToStartForPushCount)
+    : m_waitToStartForPushCount(waitToStartForPushCount)
+    , m_inputSampleOffset(MediaTime::invalidTime())
     , m_maximumSampleCount(maximumSampleCount)
 #if !RELEASE_LOG_DISABLED
@@ -231 +232 @@
     if (m_shouldComputeOutputSampleOffset) {
         uint64_t buffered = endFrame - startFrame;
-        if (buffered < sampleCount * 2 || (m_endFrameWhenNotEnoughData && m_endFrameWhenNotEnoughData == endFrame)) {
-            if (mode != AudioSampleDataSource::Mix)
-                AudioSampleBufferList::zeroABL(buffer, byteCount);
-            return false;
+        if (m_isFirstPull) {
+            if (buffered >= m_waitToStartForPushCount * m_lastPushedSampleCount) {
+                m_outputSampleOffset = startFrame - timeStamp;
+                m_shouldComputeOutputSampleOffset = false;
+                m_endFrameWhenNotEnoughData = 0;
+            } else {
+                // We wait for one chunk of value before starting to play.
+                if (mode != AudioSampleDataSource::Mix)
+                    AudioSampleBufferList::zeroABL(buffer, byteCount);
+                return false;
+            }
+        } else {
+            if (buffered < sampleCount * 2 || (m_endFrameWhenNotEnoughData && m_endFrameWhenNotEnoughData == endFrame)) {
+                if (mode != AudioSampleDataSource::Mix)
+                    AudioSampleBufferList::zeroABL(buffer, byteCount);
+                return false;
+            }
+
+            m_shouldComputeOutputSampleOffset = false;
+            m_endFrameWhenNotEnoughData = 0;
+
+            m_outputSampleOffset = (endFrame - sampleCount) - timeStamp;
+            m_outputSampleOffset -= computeOffsetDelay(m_outputDescription->sampleRate(), m_lastPushedSampleCount);
+            dispatch_async(dispatch_get_main_queue(), [logIdentifier = LOGIDENTIFIER, outputSampleOffset = m_outputSampleOffset, this, protectedThis = makeRefPtr(*this)] {
+                ALWAYS_LOG(logIdentifier, "setting new offset to ", outputSampleOffset);
+            });
         }
-
-        m_shouldComputeOutputSampleOffset = false;
-        m_endFrameWhenNotEnoughData = 0;
-
-        m_outputSampleOffset = (endFrame - sampleCount) - timeStamp;
-        m_outputSampleOffset -= computeOffsetDelay(m_outputDescription->sampleRate(), m_lastPushedSampleCount);
-        dispatch_async(dispatch_get_main_queue(), [logIdentifier = LOGIDENTIFIER, outputSampleOffset = m_outputSampleOffset, this, protectedThis = makeRefPtr(*this)] {
-            ALWAYS_LOG(logIdentifier, "setting new offset to ", outputSampleOffset);
-        });
     }
 
@@ -272 +286 @@
     }
 
+    m_isFirstPull = false;
+
     if (mode == Copy) {
         m_ringBuffer->fetch(&buffer, sampleCount, timeStamp, CARingBuffer::Copy);

branches/safari-611-branch/Source/WebCore/platform/mediastream/libwebrtc/LibWebRTCAudioModule.cpp

@@ -72 +72 @@
 // libwebrtc uses 10ms frames.
 const unsigned frameLengthMs = 1000 * LibWebRTCAudioFormat::chunkSampleCount / LibWebRTCAudioFormat::sampleRate;
-const unsigned pollSamples = 5;
-const unsigned pollInterval = 5 * frameLengthMs;
+const unsigned pollInterval = LibWebRTCAudioModule::PollSamplesCount * frameLengthMs;
 const unsigned channels = 2;
 
@@ -102 +101 @@
         return;
 
-    for (unsigned i = 0; i < pollSamples; i++) {
+    for (unsigned i = 0; i < PollSamplesCount; i++) {
         int64_t elapsedTime = -1;
         int64_t ntpTime = -1;

branches/safari-611-branch/Source/WebCore/platform/mediastream/libwebrtc/LibWebRTCAudioModule.h

@@ -45 +45 @@
 public:
     LibWebRTCAudioModule();
+
+    static constexpr unsigned PollSamplesCount = 3;
 
 private:

branches/safari-611-branch/Source/WebCore/platform/mediastream/mac/AudioMediaStreamTrackRendererCocoa.cpp

@@ -32 +32 @@
 #include "AudioSampleDataSource.h"
 #include "CAAudioStreamDescription.h"
+#include "LibWebRTCAudioModule.h"
 
 namespace WebCore {
@@ -75 +76 @@
 }
 
+static unsigned pollSamplesCount()
+{
+#if USE(LIBWEBRTC)
+    return LibWebRTCAudioModule::PollSamplesCount + 1;
+#else
+    return 2;
+#endif
+}
+
 void AudioMediaStreamTrackRendererCocoa::pushSamples(const MediaTime& sampleTime, const PlatformAudioData& audioData, const AudioStreamDescription& description, size_t sampleCount)
 {
@@ -80 +90 @@
     ASSERT(description.platformDescription().type == PlatformDescription::CAAudioStreamBasicType);
     if (!m_dataSource || m_shouldReset || !m_dataSource->inputDescription() || *m_dataSource->inputDescription() != description) {
-        auto dataSource = AudioSampleDataSource::create(description.sampleRate() * 2, *this);
+        // FIXME: For non libwebrtc sources, we can probably reduce poll samples count to 2.
+        
+        auto dataSource = AudioSampleDataSource::create(description.sampleRate() * 0.5, *this, pollSamplesCount());
 
         if (dataSource->setInputFormat(toCAAudioStreamDescription(description))) {

branches/safari-611-branch/Source/WebCore/platform/mediastream/mac/MediaStreamTrackAudioSourceProviderCocoa.cpp

@@ -29 +29 @@
 #if ENABLE(WEB_AUDIO) && ENABLE(MEDIA_STREAM)
 
+#import "LibWebRTCAudioModule.h"
+
 namespace WebCore {
 
@@ -40 +42 @@
     , m_source(source.source())
 {
+#if USE(LIBWEBRTC)
+    if (m_source->isIncomingAudioSource())
+        setPollSamplesCount(LibWebRTCAudioModule::PollSamplesCount + 1);
+#endif
 }
 

branches/safari-611-branch/Source/WebCore/platform/mediastream/mac/WebAudioSourceProviderCocoa.h

@@ -61 +61 @@
     void receivedNewAudioSamples(const PlatformAudioData&, const AudioStreamDescription&, size_t);
 
+    void setPollSamplesCount(size_t);
+
 private:
     virtual void hasNewClient(AudioSourceProviderClient*) = 0;
@@ -81 +83 @@
     RefPtr<AudioSampleDataSource> m_dataSource;
 
+    size_t m_pollSamplesCount { 3 };
     uint64_t m_writeCount { 0 };
     uint64_t m_readCount { 0 };
 };
 
+inline void WebAudioSourceProviderCocoa::setPollSamplesCount(size_t count)
+{
+    m_pollSamplesCount = count;
+}
+
 }
 

branches/safari-611-branch/Source/WebCore/platform/mediastream/mac/WebAudioSourceProviderCocoa.mm

@@ -121 +121 @@
 
     if (!m_dataSource)
-        m_dataSource = AudioSampleDataSource::create(kRingBufferDuration * sampleRate, loggerHelper());
+        m_dataSource = AudioSampleDataSource::create(kRingBufferDuration * sampleRate, loggerHelper(), m_pollSamplesCount);
     m_dataSource->setInputFormat(m_inputDescription.value());
     m_dataSource->setOutputFormat(m_outputDescription.value());