Changeset 212144 in webkit


Ignore:
Timestamp:
Feb 10, 2017 1:29:32 PM (7 years ago)
Author:
commit-queue@webkit.org
Message:

[WebRTC] Implement Outgoing libwebrtc audio source support
​https://bugs.webkit.org/show_bug.cgi?id=168118

Patch by Youenn Fablet <​youenn@apple.com> on 2017-02-10
Reviewed by Eric Carlson.

No new automated tests as we need the audio rendering to work to test the whole loop.

Using an AudioSampleDataSource to convert the captured data to libwebrtc expected format.
Capturing and pushing data happens in the capture thread.
Pulling of converted data happens in libwebrtc thread.

Introducing LibWebRTCAudioFormat.h to centralize libwbebrtc expected audio format.

  • WebCore.xcodeproj/project.pbxproj:
  • platform/audio/WebAudioBufferList.cpp:

(WebCore::WebAudioBufferList::WebAudioBufferList): Missing initialization leads to assertion failure.

  • platform/audio/mac/AudioSampleBufferList.cpp:

(WebCore::AudioSampleBufferList::copyFrom): In case of interleaved channels, there is one buffer but two channels.

  • platform/audio/mac/AudioSampleDataSource.cpp:

(WebCore::AudioSampleDataSource::pushSamples):
(WebCore::AudioSampleDataSource::pullAvalaibleSamplesAsChunks):

  • platform/audio/mac/AudioSampleDataSource.h:
  • platform/mediastream/libwebrtc/LibWebRTCAudioFormat.h: Added.
  • platform/mediastream/mac/RealtimeOutgoingAudioSource.cpp:

(WebCore::libwebrtcAudioFormat):
(WebCore::RealtimeOutgoingAudioSource::RealtimeOutgoingAudioSource):
(WebCore::RealtimeOutgoingAudioSource::sourceMutedChanged):
(WebCore::RealtimeOutgoingAudioSource::audioSamplesAvailable):
(WebCore::RealtimeOutgoingAudioSource::pullAudioData):

  • platform/mediastream/mac/RealtimeOutgoingAudioSource.h:
Location:
trunk/Source/WebCore
Files:
1 added
8 edited

Legend:

Unmodified
Added
Removed

  • trunk/Source/WebCore/ChangeLog

    r212140 r212144  
     12017-02-10  Youenn Fablet  <youenn@apple.com>
     2
     3        [WebRTC] Implement Outgoing libwebrtc audio source support
     4        https://bugs.webkit.org/show_bug.cgi?id=168118
     5
     6        Reviewed by Eric Carlson.
     7
     8        No new automated tests as we need the audio rendering to work to test the whole loop.
     9
     10        Using an AudioSampleDataSource to convert the captured data to libwebrtc expected format.
     11        Capturing and pushing data happens in the capture thread.
     12        Pulling of converted data happens in libwebrtc thread.
     13
     14        Introducing LibWebRTCAudioFormat.h to centralize libwbebrtc expected audio format.
     15
     16        * WebCore.xcodeproj/project.pbxproj:
     17        * platform/audio/WebAudioBufferList.cpp:
     18        (WebCore::WebAudioBufferList::WebAudioBufferList): Missing initialization leads to assertion failure.
     19        * platform/audio/mac/AudioSampleBufferList.cpp:
     20        (WebCore::AudioSampleBufferList::copyFrom): In case of interleaved channels, there is one buffer but two channels.
     21        * platform/audio/mac/AudioSampleDataSource.cpp:
     22        (WebCore::AudioSampleDataSource::pushSamples):
     23        (WebCore::AudioSampleDataSource::pullAvalaibleSamplesAsChunks):
     24        * platform/audio/mac/AudioSampleDataSource.h:
     25        * platform/mediastream/libwebrtc/LibWebRTCAudioFormat.h: Added.
     26        * platform/mediastream/mac/RealtimeOutgoingAudioSource.cpp:
     27        (WebCore::libwebrtcAudioFormat):
     28        (WebCore::RealtimeOutgoingAudioSource::RealtimeOutgoingAudioSource):
     29        (WebCore::RealtimeOutgoingAudioSource::sourceMutedChanged):
     30        (WebCore::RealtimeOutgoingAudioSource::audioSamplesAvailable):
     31        (WebCore::RealtimeOutgoingAudioSource::pullAudioData):
     32        * platform/mediastream/mac/RealtimeOutgoingAudioSource.h:
     33
    1342017-02-10  Ryosuke Niwa  <rniwa@webkit.org>
    235

  • trunk/Source/WebCore/WebCore.xcodeproj/project.pbxproj

    r211962 r212144  
    88778877                41D015C80F4B5C71004A662F /* ContentType.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = ContentType.h; sourceTree = "<group>"; };
    88788878                41D015C90F4B5C71004A662F /* ContentType.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = ContentType.cpp; sourceTree = "<group>"; };
     8879                41D51BB21E4E2E8100131A5B /* LibWebRTCAudioFormat.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = LibWebRTCAudioFormat.h; path = libwebrtc/LibWebRTCAudioFormat.h; sourceTree = "<group>"; };
    88798880                41E1B1CA0FF5986900576B3B /* AbstractWorker.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = AbstractWorker.cpp; sourceTree = "<group>"; };
    88808881                41E1B1CB0FF5986900576B3B /* AbstractWorker.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = AbstractWorker.h; sourceTree = "<group>"; };
    … …  
    1684616847                        isa = PBXGroup;
    1684716848                        children = (
     16849                                41D51BB21E4E2E8100131A5B /* LibWebRTCAudioFormat.h */,
    1684816850                                415080341E3F00AA0051D75D /* LibWebRTCAudioModule.cpp */,
    1684916851                                415080351E3F00AA0051D75D /* LibWebRTCAudioModule.h */,

  • trunk/Source/WebCore/platform/audio/WebAudioBufferList.cpp

    r211959 r212144  
    6565    for (uint32_t buffer = 0; buffer < m_list->mNumberBuffers; ++buffer) {
    6666        m_list->mBuffers[buffer].mData = data;
     67        m_list->mBuffers[buffer].mDataByteSize = bytesPerBuffer;
    6768        data += bytesPerBuffer;
    6869    }

  • trunk/Source/WebCore/platform/audio/mac/AudioSampleBufferList.cpp

    r211978 r212144  
    261261    AudioConverterGetProperty(converter, kAudioConverterCurrentOutputStreamDescription, &propertyDataSize, &outputFormat);
    262262
    263     ASSERT(outputFormat.mChannelsPerFrame == m_bufferList->bufferCount());
     263    ASSERT(CAAudioStreamDescription(outputFormat).numberOfChannelStreams() == m_bufferList->bufferCount());
    264264    for (uint32_t i = 0; i < m_bufferList->bufferCount(); ++i) {
    265265        ASSERT(m_bufferList->buffer(i)->mData);

  • trunk/Source/WebCore/platform/audio/mac/AudioSampleDataSource.cpp

    r211978 r212144  
    202202    std::unique_lock<Lock> lock(m_lock, std::try_to_lock);
    203203    ASSERT(is<WebAudioBufferList>(audioData));
    204     pushSamplesInternal(downcast<WebAudioBufferList>(audioData), sampleTime, sampleCount);
     204    pushSamplesInternal(*downcast<WebAudioBufferList>(audioData).list(), sampleTime, sampleCount);
    205205}
    206206
    … …  
    287287}
    288288
     289bool AudioSampleDataSource::pullAvalaibleSamplesAsChunks(AudioBufferList& buffer, size_t sampleCountPerChunk, uint64_t timeStamp, Function<void()>&& consumeFilledBuffer)
     290{
     291    std::unique_lock<Lock> lock(m_lock, std::try_to_lock);
     292    if (!lock.owns_lock() || !m_ringBuffer)
     293        return false;
     294
     295    ASSERT(buffer.mNumberBuffers == m_ringBuffer->channelCount());
     296    if (buffer.mNumberBuffers != m_ringBuffer->channelCount())
     297        return false;
     298
     299    uint64_t startFrame = 0;
     300    uint64_t endFrame = 0;
     301    m_ringBuffer->getCurrentFrameBounds(startFrame, endFrame);
     302    if (timeStamp < startFrame)
     303        return false;
     304
     305    startFrame = timeStamp;
     306    while (endFrame - startFrame >= sampleCountPerChunk) {
     307        if (m_ringBuffer->fetch(&buffer, sampleCountPerChunk, startFrame, CARingBuffer::Copy))
     308            return false;
     309        consumeFilledBuffer();
     310        startFrame += sampleCountPerChunk;
     311    }
     312    return true;
     313}
     314
    289315bool AudioSampleDataSource::pullSamples(AudioBufferList& buffer, size_t sampleCount, uint64_t timeStamp, double hostTime, PullMode mode)
    290316{

  • trunk/Source/WebCore/platform/audio/mac/AudioSampleDataSource.h

    r211978 r212144  
    6060    bool pullSamples(AudioBufferList&, size_t, uint64_t, double, PullMode);
    6161
     62    bool pullAvalaibleSamplesAsChunks(AudioBufferList&, size_t frameCount, uint64_t timeStamp, Function<void()>&&);
     63
    6264    void setPaused(bool);
    6365

  • trunk/Source/WebCore/platform/mediastream/mac/RealtimeOutgoingAudioSource.cpp

    r211978 r212144  
    3232#if USE(LIBWEBRTC)
    3333
    34 #include "NotImplemented.h"
     34#include "CAAudioStreamDescription.h"
     35#include "LibWebRTCAudioFormat.h"
     36#include "LibWebRTCUtils.h"
    3537
    3638namespace WebCore {
    3739
    38 void RealtimeOutgoingAudioSource::audioSamplesAvailable(const MediaTime&, const PlatformAudioData&, const AudioStreamDescription&, size_t)
     40static inline AudioStreamBasicDescription libwebrtcAudioFormat(size_t channelCount)
    3941{
    40     notImplemented();
     42    AudioStreamBasicDescription streamFormat;
     43    FillOutASBDForLPCM(streamFormat, LibWebRTCAudioFormat::sampleRate, channelCount, LibWebRTCAudioFormat::sampleSize, LibWebRTCAudioFormat::sampleSize, LibWebRTCAudioFormat::isFloat, LibWebRTCAudioFormat::isBigEndian, LibWebRTCAudioFormat::isNonInterleaved);
     44    return streamFormat;
     45}
     46
     47RealtimeOutgoingAudioSource::RealtimeOutgoingAudioSource(Ref<RealtimeMediaSource>&& audioSource)
     48    : m_audioSource(WTFMove(audioSource))
     49    , m_sampleConverter(AudioSampleDataSource::create(LibWebRTCAudioFormat::sampleRate * 2))
     50{
     51    m_audioSource->addObserver(*this);
     52}
     53
     54void RealtimeOutgoingAudioSource::sourceMutedChanged()
     55{
     56    m_isMuted = m_audioSource->muted();
     57}
     58
     59void RealtimeOutgoingAudioSource::audioSamplesAvailable(const MediaTime& time, const PlatformAudioData& audioData, const AudioStreamDescription& streamDescription, size_t sampleCount)
     60{
     61    if (m_inputStreamDescription != streamDescription) {
     62        m_inputStreamDescription = toCAAudioStreamDescription(streamDescription);
     63        auto status  = m_sampleConverter->setInputFormat(m_inputStreamDescription);
     64        ASSERT_UNUSED(status, !status);
     65
     66        status = m_sampleConverter->setOutputFormat(libwebrtcAudioFormat(streamDescription.numberOfChannels()));
     67        ASSERT(!status);
     68    }
     69    m_sampleConverter->pushSamples(time, audioData, sampleCount);
     70
     71    callOnWebRTCSignalingThread([protectedThis = makeRef(*this)] {
     72        protectedThis->pullAudioData();
     73    });
     74}
     75
     76void RealtimeOutgoingAudioSource::pullAudioData()
     77{
     78    size_t bufferSize = LibWebRTCAudioFormat::chunkSampleCount * LibWebRTCAudioFormat::sampleByteSize * m_inputStreamDescription.numberOfChannels();
     79    m_audioBuffer.reserveCapacity(bufferSize);
     80
     81    AudioBufferList bufferList;
     82    bufferList.mNumberBuffers = 1;
     83    bufferList.mBuffers[0].mNumberChannels = m_inputStreamDescription.numberOfChannels();
     84    bufferList.mBuffers[0].mDataByteSize = bufferSize;
     85    bufferList.mBuffers[0].mData = m_audioBuffer.data();
     86
     87    m_sampleConverter->pullAvalaibleSamplesAsChunks(bufferList, LibWebRTCAudioFormat::chunkSampleCount, m_startFrame, [this] {
     88        m_startFrame += LibWebRTCAudioFormat::chunkSampleCount;
     89        if (m_isMuted)
     90            return;
     91        for (auto sink : m_sinks)
     92            sink->OnData(m_audioBuffer.data(), LibWebRTCAudioFormat::sampleSize, LibWebRTCAudioFormat::sampleRate, m_inputStreamDescription.numberOfChannels(), LibWebRTCAudioFormat::chunkSampleCount);
     93    });
    4194}
    4295

  • trunk/Source/WebCore/platform/mediastream/mac/RealtimeOutgoingAudioSource.h

    r211978 r212144  
    3131#if USE(LIBWEBRTC)
    3232
     33#include "AudioSampleDataSource.h"
    3334#include "LibWebRTCMacros.h"
    3435#include "RealtimeMediaSource.h"
    … …  
    5051
    5152private:
    52     explicit RealtimeOutgoingAudioSource(Ref<RealtimeMediaSource>&& audioSource) : m_audioSource(WTFMove(audioSource)) { m_audioSource->addObserver(*this); }
     53    explicit RealtimeOutgoingAudioSource(Ref<RealtimeMediaSource>&&);
    5354
    5455    virtual void AddSink(webrtc::AudioTrackSinkInterface* sink) { m_sinks.append(sink); }
    … …  
    6465    // RealtimeMediaSource::Observer API
    6566    void sourceStopped() final { }
    66     void sourceMutedChanged() final { }
     67    void sourceMutedChanged() final;
    6768    void sourceSettingsChanged() final { }
    6869    bool preventSourceFromStopping() final { return false; }
    6970    void audioSamplesAvailable(const MediaTime&, const PlatformAudioData&, const AudioStreamDescription&, size_t) final;
    7071
    71     void convertAndSendMonoSamples();
    72     void convertAndSendStereoSamples();
     72    void pullAudioData();
    7373
    7474    Vector<webrtc::AudioTrackSinkInterface*> m_sinks;
    7575    Ref<RealtimeMediaSource> m_audioSource;
    7676    rtc::scoped_refptr<webrtc::AudioTrackInterface> m_track;
     77    Ref<AudioSampleDataSource> m_sampleConverter;
     78    CAAudioStreamDescription m_inputStreamDescription;
     79
     80    Vector<uint16_t> m_audioBuffer;
     81    uint64_t m_startFrame { 0 };
     82    bool m_isMuted { false };
    7783};
    7884
Note: See TracChangeset for help on using the changeset viewer.