Changeset 261626 in webkit

Timestamp:

May 13, 2020 11:02:09 AM (4 years ago)

Author:

youenn@apple.com

Message:

Allow WebAudioBufferList to dynamically change its number of frames
​https://bugs.webkit.org/show_bug.cgi?id=211720

Reviewed by Eric Carlson.

Source/WebCore:

We sometimes create WebAudioBufferList on the stack which triggers allocation of several vectors.
Instead of doing that for every audio sample chunk, we should allocate the WebAudioBufferList and resize it as necessary.
For that purpose, we introduce WebAudioBufferList::updateWithNumberOfFrames and use it in two places:

• When creating an audio track into WebAudio.
• When receiving audio chunks from another process.

Covered by existing tests.

• Modules/webaudio/MediaStreamAudioSource.cpp:

(WebCore::MediaStreamAudioSource::~MediaStreamAudioSource):

• Modules/webaudio/MediaStreamAudioSource.h:
• Modules/webaudio/MediaStreamAudioSourceCocoa.cpp:

(WebCore::streamDescription):
(WebCore::MediaStreamAudioSource::consumeAudio):

• platform/audio/cocoa/WebAudioBufferList.cpp:

(WebCore::WebAudioBufferList::WebAudioBufferList):
(WebCore::WebAudioBufferList::updateWithNumberOfFrames):
(WebCore::WebAudioBufferList::channelCount const):

• platform/audio/cocoa/WebAudioBufferList.h:
• platform/mediastream/mac/MockAudioSharedUnit.mm:

(WebCore::MockAudioSharedUnit::reconfigure):
Drive-by fix, we do not need to give the size in bytes but the size in samples.

Source/WebKit:

• WebProcess/cocoa/RemoteCaptureSampleManager.cpp:

(WebKit::RemoteCaptureSampleManager::RemoteAudio::setStorage):
(WebKit::RemoteCaptureSampleManager::RemoteAudio::audioSamplesAvailable):

• WebProcess/cocoa/RemoteCaptureSampleManager.h:

Location:

trunk/Source

Files:

• WebCore/ChangeLog (modified) (1 diff)
• WebCore/Modules/webaudio/MediaStreamAudioSource.cpp (modified) (2 diffs)
• WebCore/Modules/webaudio/MediaStreamAudioSource.h (modified) (3 diffs)
• WebCore/Modules/webaudio/MediaStreamAudioSourceCocoa.cpp (modified) (2 diffs)
• WebCore/platform/audio/cocoa/WebAudioBufferList.cpp (modified) (4 diffs)
• WebCore/platform/audio/cocoa/WebAudioBufferList.h (modified) (3 diffs)
• WebCore/platform/mediastream/mac/MockAudioSharedUnit.mm (modified) (1 diff)
• WebKit/ChangeLog (modified) (1 diff)
• WebKit/WebProcess/cocoa/RemoteCaptureSampleManager.cpp (modified) (1 diff)
• WebKit/WebProcess/cocoa/RemoteCaptureSampleManager.h (modified) (2 diffs)

trunk/Source/WebCore/ChangeLog

@@ +1 @@
+2020-05-13  Youenn Fablet  <youenn@apple.com>
+
+        Allow WebAudioBufferList to dynamically change its number of frames
+        https://bugs.webkit.org/show_bug.cgi?id=211720
+
+        Reviewed by Eric Carlson.
+
+        We sometimes create WebAudioBufferList on the stack which triggers allocation of several vectors.
+        Instead of doing that for every audio sample chunk, we should allocate the WebAudioBufferList and resize it as necessary.
+        For that purpose, we introduce WebAudioBufferList::updateWithNumberOfFrames and use it in two places:
+        - When creating an audio track into WebAudio.
+        - When receiving audio chunks from another process.
+        Covered by existing tests.
+
+        * Modules/webaudio/MediaStreamAudioSource.cpp:
+        (WebCore::MediaStreamAudioSource::~MediaStreamAudioSource):
+        * Modules/webaudio/MediaStreamAudioSource.h:
+        * Modules/webaudio/MediaStreamAudioSourceCocoa.cpp:
+        (WebCore::streamDescription):
+        (WebCore::MediaStreamAudioSource::consumeAudio):
+        * platform/audio/cocoa/WebAudioBufferList.cpp:
+        (WebCore::WebAudioBufferList::WebAudioBufferList):
+        (WebCore::WebAudioBufferList::updateWithNumberOfFrames):
+        (WebCore::WebAudioBufferList::channelCount const):
+        * platform/audio/cocoa/WebAudioBufferList.h:
+        * platform/mediastream/mac/MockAudioSharedUnit.mm:
+        (WebCore::MockAudioSharedUnit::reconfigure):
+        Drive-by fix, we do not need to give the size in bytes but the size in samples.
+
 2020-05-13  Andres Gonzalez  <andresg_22@apple.com>
 

trunk/Source/WebCore/Modules/webaudio/MediaStreamAudioSource.cpp

@@ -30 +30 @@
 
 #include "NotImplemented.h"
-#include <wtf/UUID.h>
+#include "PlatformAudioData.h"
 
 namespace WebCore {
@@ -39 +39 @@
     m_currentSettings.setSampleRate(sampleRate);
 }
+
+MediaStreamAudioSource::~MediaStreamAudioSource() = default;
 
 const RealtimeMediaSourceCapabilities& MediaStreamAudioSource::capabilities()

trunk/Source/WebCore/Modules/webaudio/MediaStreamAudioSource.h

@@ -36 +36 @@
 
 class AudioBus;
+class PlatformAudioData;
 class RealtimeMediaSourceCapabilities;
 
@@ -42 +43 @@
     static Ref<MediaStreamAudioSource> create(float sampleRate) { return adoptRef(*new MediaStreamAudioSource { sampleRate }); }
 
-    ~MediaStreamAudioSource() = default;
+    ~MediaStreamAudioSource();
 
     const RealtimeMediaSourceCapabilities& capabilities() final;
@@ -59 +60 @@
     String m_deviceId;
     RealtimeMediaSourceSettings m_currentSettings;
+    std::unique_ptr<PlatformAudioData> m_audioBuffer;
 #if USE(AVFOUNDATION)
     size_t m_numberOfFrames { 0 };

trunk/Source/WebCore/Modules/webaudio/MediaStreamAudioSourceCocoa.cpp

@@ -42 +42 @@
 namespace WebCore {
 
-static inline AudioStreamBasicDescription streamDescription(size_t sampleRate, size_t channelCount)
+static inline CAAudioStreamDescription streamDescription(size_t sampleRate, size_t channelCount)
 {
     bool isFloat = true;
@@ -76 +76 @@
     m_numberOfFrames += numberOfFrames;
 
-    AudioStreamBasicDescription newDescription = streamDescription(m_currentSettings.sampleRate(), bus.numberOfChannels());
+    auto* audioBuffer = m_audioBuffer ? &downcast<WebAudioBufferList>(*m_audioBuffer) : nullptr;
 
-    // FIXME: We should do the memory allocation once in MediaStreamAudioSource and resize it according numberOfFrames.
-    WebAudioBufferList audioBufferList { CAAudioStreamDescription(newDescription), WTF::safeCast<uint32_t>(numberOfFrames) };
+    auto description = streamDescription(m_currentSettings.sampleRate(), bus.numberOfChannels());
+    if (!audioBuffer || audioBuffer->channelCount() != bus.numberOfChannels()) {
+        m_audioBuffer = makeUnique<WebAudioBufferList>(description, WTF::safeCast<uint32_t>(numberOfFrames));
+        audioBuffer = &downcast<WebAudioBufferList>(*m_audioBuffer);
+    } else
+        audioBuffer->setSampleCount(numberOfFrames);
 
     for (size_t cptr = 0; cptr < bus.numberOfChannels(); ++cptr)
-        copyChannelData(*bus.channel(cptr), *audioBufferList.buffer(cptr), numberOfFrames, muted());
+        copyChannelData(*bus.channel(cptr), *audioBuffer->buffer(cptr), numberOfFrames, muted());
 
-    audioSamplesAvailable(mediaTime, audioBufferList, CAAudioStreamDescription(newDescription), numberOfFrames);
+    audioSamplesAvailable(mediaTime, *m_audioBuffer, description, numberOfFrames);
 }
 

trunk/Source/WebCore/platform/audio/cocoa/WebAudioBufferList.cpp

@@ -34 +34 @@
 
 WebAudioBufferList::WebAudioBufferList(const CAAudioStreamDescription& format)
+    : m_bytesPerFrame(format.bytesPerFrame())
+    , m_channelCount(format.numberOfInterleavedChannels())
 {
     // AudioBufferList is a variable-length struct, so create on the heap with a generic new() operator
     // with a custom size, and initialize the struct manually.
     uint32_t bufferCount = format.numberOfChannelStreams();
-    uint32_t channelCount = format.numberOfInterleavedChannels();
 
     uint64_t bufferListSize = offsetof(AudioBufferList, mBuffers) + (sizeof(AudioBuffer) * std::max(1U, bufferCount));
@@ -48 +49 @@
     m_canonicalList->mNumberBuffers = bufferCount;
     for (uint32_t buffer = 0; buffer < bufferCount; ++buffer)
-        m_canonicalList->mBuffers[buffer].mNumberChannels = channelCount;
+        m_canonicalList->mBuffers[buffer].mNumberChannels = m_channelCount;
 
     reset();
@@ -56 +57 @@
     : WebAudioBufferList(format)
 {
-    if (!sampleCount)
+    setSampleCount(sampleCount);
+}
+
+void WebAudioBufferList::setSampleCount(uint32_t sampleCount)
+{
+    if (!sampleCount || m_sampleCount == sampleCount)
         return;
 
-    uint32_t bufferCount = format.numberOfChannelStreams();
-    uint32_t channelCount = format.numberOfInterleavedChannels();
-
-    size_t bytesPerBuffer = sampleCount * channelCount * format.bytesPerFrame();
-    m_flatBuffer.reserveInitialCapacity(bufferCount * bytesPerBuffer);
-    auto data = m_flatBuffer.data();
+    m_sampleCount = sampleCount;
+    size_t bytesPerBuffer = m_sampleCount * m_channelCount * m_bytesPerFrame;
+    m_flatBuffer.resize(m_canonicalList->mNumberBuffers * bytesPerBuffer);
+    auto* data = m_flatBuffer.data();
 
     for (uint32_t buffer = 0; buffer < m_canonicalList->mNumberBuffers; ++buffer) {
@@ -78 +82 @@
     : WebAudioBufferList(format)
 {
-
     if (!sampleBuffer)
         return;

trunk/Source/WebCore/platform/audio/cocoa/WebAudioBufferList.h

@@ -47 +47 @@
 
     void reset();
+    WEBCORE_EXPORT void setSampleCount(uint32_t);
 
     AudioBufferList* list() const { return m_list.get(); }
@@ -52 +53 @@
 
     uint32_t bufferCount() const;
+    uint32_t channelCount() const { return m_channelCount; }
     AudioBuffer* buffer(uint32_t index) const;
     WTF::IteratorRange<AudioBuffer*> buffers() const;
@@ -59 +61 @@
 
     size_t m_listBufferSize { 0 };
+    uint32_t m_bytesPerFrame { 0 };
+    uint32_t m_channelCount { 0 };
+    uint32_t m_sampleCount { 0 };
     std::unique_ptr<AudioBufferList> m_canonicalList;
     std::unique_ptr<AudioBufferList> m_list;

trunk/Source/WebCore/platform/mediastream/mac/MockAudioSharedUnit.mm

@@ -214 +214 @@
     FillOutASBDForLPCM(m_streamFormat, rate, channelCount, bitsPerByte * bytesPerFloat, bitsPerByte * bytesPerFloat, isFloat, isBigEndian, isNonInterleaved);
 
-    m_audioBufferList = makeUnique<WebAudioBufferList>(m_streamFormat, m_streamFormat.mBytesPerFrame * m_maximiumFrameCount);
+    m_audioBufferList = makeUnique<WebAudioBufferList>(m_streamFormat, m_maximiumFrameCount);
 
     CMFormatDescriptionRef formatDescription;

trunk/Source/WebKit/ChangeLog

@@ +1 @@
+2020-05-13  Youenn Fablet  <youenn@apple.com>
+
+        Allow WebAudioBufferList to dynamically change its number of frames
+        https://bugs.webkit.org/show_bug.cgi?id=211720
+
+        Reviewed by Eric Carlson.
+
+        * WebProcess/cocoa/RemoteCaptureSampleManager.cpp:
+        (WebKit::RemoteCaptureSampleManager::RemoteAudio::setStorage):
+        (WebKit::RemoteCaptureSampleManager::RemoteAudio::audioSamplesAvailable):
+        * WebProcess/cocoa/RemoteCaptureSampleManager.h:
+
 2020-05-13  Per Arne Vollan  <pvollan@apple.com>
 

trunk/Source/WebKit/WebProcess/cocoa/RemoteCaptureSampleManager.cpp

@@ -139 +139 @@
     storage.setReadOnly(true);
     m_ringBuffer->allocate(description, numberOfFrames);
+    m_buffer = makeUnique<WebAudioBufferList>(description, numberOfFrames);
 }
 
 void RemoteCaptureSampleManager::RemoteAudio::audioSamplesAvailable(MediaTime time, uint64_t numberOfFrames, uint64_t startFrame, uint64_t endFrame)
 {
-    // FIXME: We should allocate this buffer once and resize it as needed.
-    WebAudioBufferList audioData(m_description, numberOfFrames);
+    m_buffer->setSampleCount(numberOfFrames);
 
     m_ringBuffer->setCurrentFrameBounds(startFrame, endFrame);
-    m_ringBuffer->fetch(audioData.list(), numberOfFrames, time.timeValue());
+    m_ringBuffer->fetch(m_buffer->list(), numberOfFrames, time.timeValue());
 
-    m_source->remoteAudioSamplesAvailable(time, audioData, m_description, numberOfFrames);
+    m_source->remoteAudioSamplesAvailable(time, *m_buffer, m_description, numberOfFrames);
 }
 

trunk/Source/WebKit/WebProcess/cocoa/RemoteCaptureSampleManager.h

@@ -34 +34 @@
 #include <WebCore/CAAudioStreamDescription.h>
 #include <WebCore/CARingBuffer.h>
+#include <WebCore/WebAudioBufferList.h>
 #include <wtf/HashMap.h>
 #include <wtf/WorkQueue.h>
@@ -72 +73 @@
         WebCore::CAAudioStreamDescription m_description;
         std::unique_ptr<WebCore::CARingBuffer> m_ringBuffer;
+        std::unique_ptr<WebCore::WebAudioBufferList> m_buffer;
     };