Changeset 284523 in webkit


Ignore:
Timestamp:
Oct 20, 2021 1:13:27 AM (9 months ago)
Author:
youenn@apple.com
Message:

WebRTC p2p call, VP9-SVC only low layer decoded on receiver side
​https://bugs.webkit.org/show_bug.cgi?id=231071
<rdar://problem/83763291>

Reviewed by Eric Carlson.

Source/ThirdParty/libwebrtc:

In case VP9 SVC is used, fallback to software decoder for now.

  • Source/webrtc/sdk/WebKit/WebKitDecoder.mm:

Source/WebCore:

Add Internals API to check whether VP9 VTB is used or not.

Test: webrtc/vp9-svc.html

  • Modules/mediastream/PeerConnectionBackend.h:
  • Modules/mediastream/RTCPeerConnection.cpp:
  • Modules/mediastream/RTCPeerConnection.h:
  • Modules/mediastream/libwebrtc/LibWebRTCMediaEndpoint.cpp:
  • Modules/mediastream/libwebrtc/LibWebRTCMediaEndpoint.h:
  • Modules/mediastream/libwebrtc/LibWebRTCPeerConnectionBackend.cpp:
  • Modules/mediastream/libwebrtc/LibWebRTCPeerConnectionBackend.h:
  • Modules/mediastream/libwebrtc/LibWebRTCStatsCollector.cpp:
  • Modules/mediastream/libwebrtc/LibWebRTCStatsCollector.h:
  • testing/Internals.cpp:
  • testing/Internals.h:
  • testing/Internals.idl:

LayoutTests:

  • webrtc/vp9-svc-expected.txt: Added.
  • webrtc/vp9-svc.html: Added.
Location:
trunk
Files:
2 added
16 edited

Legend:

Unmodified
Added
Removed

  • trunk/LayoutTests/ChangeLog

    r284522 r284523  
     12021-10-20  Youenn Fablet  <youenn@apple.com>
     2
     3        WebRTC p2p call, VP9-SVC only low layer decoded on receiver side
     4        https://bugs.webkit.org/show_bug.cgi?id=231071
     5        <rdar://problem/83763291>
     6
     7        Reviewed by Eric Carlson.
     8
     9        * webrtc/vp9-svc-expected.txt: Added.
     10        * webrtc/vp9-svc.html: Added.
     11
    1122021-10-20  John Cunningham  <johncunningham@apple.com>
    213

  • trunk/Source/ThirdParty/libwebrtc/ChangeLog

    r284433 r284523  
     12021-10-20  Youenn Fablet  <youenn@apple.com>
     2
     3        WebRTC p2p call, VP9-SVC only low layer decoded on receiver side
     4        https://bugs.webkit.org/show_bug.cgi?id=231071
     5        <rdar://problem/83763291>
     6
     7        Reviewed by Eric Carlson.
     8
     9        In case VP9 SVC is used, fallback to software decoder for now.
     10
     11        * Source/webrtc/sdk/WebKit/WebKitDecoder.mm:
     12
    1132021-10-18  Cameron McCormack  <heycam@apple.com>
    214

  • trunk/Source/ThirdParty/libwebrtc/Source/webrtc/sdk/WebKit/WebKitDecoder.mm

    r275034 r284523  
    3232#import "api/video_codecs/video_decoder.h"
    3333#import "api/video_codecs/video_decoder_factory.h"
     34#import "api/video_codecs/video_decoder_software_fallback_wrapper.h"
    3435#import "modules/video_coding/codecs/h264/include/h264.h"
    3536#import "modules/video_coding/include/video_error_codes.h"
    … …  
    125126class RemoteVideoDecoder final : public webrtc::VideoDecoder {
    126127public:
    127     explicit RemoteVideoDecoder(WebKitVideoDecoder);
     128    RemoteVideoDecoder(WebKitVideoDecoder, bool isVP9);
    128129    ~RemoteVideoDecoder();
     130
     131    bool isVP9() const { return m_isVP9; }
    129132
    130133private:
    … …  
    136139
    137140    WebKitVideoDecoder m_internalDecoder;
     141    bool m_isVP9 { false };
    138142};
    139143
    … …  
    159163}
    160164
    161 RemoteVideoDecoder::RemoteVideoDecoder(WebKitVideoDecoder internalDecoder)
     165RemoteVideoDecoder::RemoteVideoDecoder(WebKitVideoDecoder internalDecoder, bool isVP9)
    162166    : m_internalDecoder(internalDecoder)
     167    , m_isVP9(isVP9)
    163168{
    164169}
    … …  
    194199        encodedHeight = rtc::dchecked_cast<uint16_t>(input_image._encodedHeight);
    195200    }
     201
     202    // VP9 VTB does not support SVC
     203    if (m_isVP9 && input_image._frameType == VideoFrameType::kVideoFrameKey && input_image.SpatialIndex() && *input_image.SpatialIndex() > 0)
     204        return WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE;
     205
    196206    return videoDecoderCallbacks().decodeCallback(m_internalDecoder, input_image.Timestamp(), input_image.data(), input_image.size(), encodedWidth, encodedHeight);
    197207}
    … …  
    227237        return m_internalFactory->CreateVideoDecoder(format);
    228238
    229     return std::make_unique<RemoteVideoDecoder>(identifier);
     239    auto decoder = std::make_unique<RemoteVideoDecoder>(identifier, format.name == "VP9");
     240    if (!decoder->isVP9())
     241        return decoder;
     242    return webrtc::CreateVideoDecoderSoftwareFallbackWrapper(m_internalFactory->CreateVideoDecoder(format), std::move(decoder));
    230243}
    231244

  • trunk/Source/WebCore/ChangeLog

    r284522 r284523  
     12021-10-20  Youenn Fablet  <youenn@apple.com>
     2
     3        WebRTC p2p call, VP9-SVC only low layer decoded on receiver side
     4        https://bugs.webkit.org/show_bug.cgi?id=231071
     5        <rdar://problem/83763291>
     6
     7        Reviewed by Eric Carlson.
     8
     9        Add Internals API to check whether VP9 VTB is used or not.
     10
     11        Test: webrtc/vp9-svc.html
     12
     13        * Modules/mediastream/PeerConnectionBackend.h:
     14        * Modules/mediastream/RTCPeerConnection.cpp:
     15        * Modules/mediastream/RTCPeerConnection.h:
     16        * Modules/mediastream/libwebrtc/LibWebRTCMediaEndpoint.cpp:
     17        * Modules/mediastream/libwebrtc/LibWebRTCMediaEndpoint.h:
     18        * Modules/mediastream/libwebrtc/LibWebRTCPeerConnectionBackend.cpp:
     19        * Modules/mediastream/libwebrtc/LibWebRTCPeerConnectionBackend.h:
     20        * Modules/mediastream/libwebrtc/LibWebRTCStatsCollector.cpp:
     21        * Modules/mediastream/libwebrtc/LibWebRTCStatsCollector.h:
     22        * testing/Internals.cpp:
     23        * testing/Internals.h:
     24        * testing/Internals.idl:
     25
    1262021-10-20  John Cunningham  <johncunningham@apple.com>
    227

  • trunk/Source/WebCore/Modules/mediastream/PeerConnectionBackend.h

    r282825 r284523  
    110110    virtual bool setConfiguration(MediaEndpointConfiguration&&) = 0;
    111111
     112    virtual void gatherDecoderImplementationName(Function<void(String&&)>&&) = 0;
    112113    virtual void getStats(Ref<DeferredPromise>&&) = 0;
    113114    virtual void getStats(RTCRtpSender&, Ref<DeferredPromise>&&) = 0;

  • trunk/Source/WebCore/Modules/mediastream/RTCPeerConnection.cpp

    r284095 r284523  
    524524}
    525525
     526void RTCPeerConnection::gatherDecoderImplementationName(Function<void(String&&)>&& callback)
     527{
     528    m_backend->gatherDecoderImplementationName(WTFMove(callback));
     529}
     530
    526531ExceptionOr<Ref<RTCDataChannel>> RTCPeerConnection::createDataChannel(String&& label, RTCDataChannelInit&& options)
    527532{

  • trunk/Source/WebCore/Modules/mediastream/RTCPeerConnection.h

    r284075 r284523  
    160160    // 8.2 Statistics API
    161161    void getStats(MediaStreamTrack*, Ref<DeferredPromise>&&);
     162    // Used for testing
     163    WEBCORE_EXPORT void gatherDecoderImplementationName(Function<void(String&&)>&&);
    162164
    163165    // EventTarget

  • trunk/Source/WebCore/Modules/mediastream/libwebrtc/LibWebRTCMediaEndpoint.cpp

    r283315 r284523  
    256256rtc::scoped_refptr<LibWebRTCStatsCollector> LibWebRTCMediaEndpoint::createStatsCollector(Ref<DeferredPromise>&& promise)
    257257{
    258     return LibWebRTCStatsCollector::create([promise = WTFMove(promise), protectedThis = Ref { *this }](auto&& report) mutable {
     258    return LibWebRTCStatsCollector::create([promise = WTFMove(promise), protectedThis = Ref { *this }](auto&& rtcReport) mutable {
    259259        ASSERT(isMainThread());
    260         if (protectedThis->isStopped() || !report)
    261             return;
    262 
    263         promise->resolve<IDLInterface<RTCStatsReport>>(report.releaseNonNull());
    264     });
     260        if (protectedThis->isStopped())
     261            return;
     262
     263        promise->resolve<IDLInterface<RTCStatsReport>>(LibWebRTCStatsCollector::createReport(rtcReport));
     264    });
     265}
     266
     267void LibWebRTCMediaEndpoint::gatherDecoderImplementationName(Function<void(String&&)>&& callback)
     268{
     269    if (!m_backend) {
     270        callback({ });
     271        return;
     272    }
     273    auto collector = LibWebRTCStatsCollector::create([callback = WTFMove(callback)](auto&& rtcReport) mutable {
     274        ASSERT(isMainThread());
     275        if (rtcReport) {
     276            for (const auto& rtcStats : *rtcReport) {
     277                if (rtcStats.type() == webrtc::RTCInboundRTPStreamStats::kType) {
     278                    auto& inboundRTPStats = static_cast<const webrtc::RTCInboundRTPStreamStats&>(rtcStats);
     279                    if (inboundRTPStats.decoder_implementation.is_defined()) {
     280                        callback(fromStdString(*inboundRTPStats.decoder_implementation));
     281                        return;
     282                    }
     283                }
     284            }
     285        }
     286        callback({ });
     287    });
     288    m_backend->GetStats(WTFMove(collector));
    265289}
    266290

  • trunk/Source/WebCore/Modules/mediastream/libwebrtc/LibWebRTCMediaEndpoint.h

    r282494 r284523  
    8686    void doCreateOffer(const RTCOfferOptions&);
    8787    void doCreateAnswer();
     88    void gatherDecoderImplementationName(Function<void(String&&)>&&);
    8889    void getStats(Ref<DeferredPromise>&&);
    8990    void getStats(webrtc::RtpReceiverInterface&, Ref<DeferredPromise>&&);

  • trunk/Source/WebCore/Modules/mediastream/libwebrtc/LibWebRTCPeerConnectionBackend.cpp

    r282860 r284523  
    200200}
    201201
     202void LibWebRTCPeerConnectionBackend::gatherDecoderImplementationName(Function<void(String&&)>&& callback)
     203{
     204    m_endpoint->gatherDecoderImplementationName(WTFMove(callback));
     205
     206}
     207
    202208void LibWebRTCPeerConnectionBackend::getStats(Ref<DeferredPromise>&& promise)
    203209{

  • trunk/Source/WebCore/Modules/mediastream/libwebrtc/LibWebRTCPeerConnectionBackend.h

    r282217 r284523  
    6868    void restartIce() final;
    6969    bool setConfiguration(MediaEndpointConfiguration&&) final;
     70    void gatherDecoderImplementationName(Function<void(String&&)>&&) final;
    7071    void getStats(Ref<DeferredPromise>&&) final;
    7172    void getStats(RTCRtpSender&, Ref<DeferredPromise>&&) final;

  • trunk/Source/WebCore/Modules/mediastream/libwebrtc/LibWebRTCStatsCollector.cpp

    r278701 r284523  
    635635{
    636636    callOnMainThread([this, protectedThis = rtc::scoped_refptr<LibWebRTCStatsCollector>(this), rtcReport]() {
    637         m_callback(RTCStatsReport::create([rtcReport](auto& mapAdapter) {
    638             if (rtcReport)
    639                 initializeRTCStatsReportBackingMap(mapAdapter, *rtcReport);
    640         }));
     637        m_callback(rtcReport);
    641638    });
    642639}
    643640
     641Ref<RTCStatsReport> LibWebRTCStatsCollector::createReport(const rtc::scoped_refptr<const webrtc::RTCStatsReport>& rtcReport)
     642{
     643    return RTCStatsReport::create([rtcReport](auto& mapAdapter) {
     644        if (rtcReport)
     645            initializeRTCStatsReportBackingMap(mapAdapter, *rtcReport);
     646    });
     647}
     648
    644649}; // namespace WTF
    645650

  • trunk/Source/WebCore/Modules/mediastream/libwebrtc/LibWebRTCStatsCollector.h

    r267855 r284523  
    4848class LibWebRTCStatsCollector : public webrtc::RTCStatsCollectorCallback {
    4949public:
    50     using CollectorCallback = CompletionHandler<void(RefPtr<RTCStatsReport>&&)>;
     50    using CollectorCallback = CompletionHandler<void(const rtc::scoped_refptr<const webrtc::RTCStatsReport>&)>;
    5151    static rtc::scoped_refptr<LibWebRTCStatsCollector> create(CollectorCallback&& callback) { return new rtc::RefCountedObject<LibWebRTCStatsCollector>(WTFMove(callback)); }
     52
     53    static Ref<RTCStatsReport> createReport(const rtc::scoped_refptr<const webrtc::RTCStatsReport>&);
    5254
    5355    explicit LibWebRTCStatsCollector(CollectorCallback&&);

  • trunk/Source/WebCore/testing/Internals.cpp

    r284518 r284523  
    16181618}
    16191619
     1620bool Internals::isSupportingVP9VTB() const
     1621{
     1622#if USE(LIBWEBRTC)
     1623    if (auto* page = contextDocument()->page())
     1624        return page->libWebRTCProvider().isSupportingVP9VTB();
     1625#endif
     1626    return false;
     1627}
     1628
     1629void Internals::isVP9VTBDeccoderUsed(RTCPeerConnection& connection, DOMPromiseDeferred<IDLBoolean>&& promise)
     1630{
     1631    connection.gatherDecoderImplementationName([promise = WTFMove(promise)](auto&& name) mutable {
     1632        promise.resolve(name.contains("VideoToolBox"));
     1633    });
     1634}
     1635
    16201636void Internals::setSFrameCounter(RTCRtpSFrameTransform& transform, const String& counter)
    16211637{

  • trunk/Source/WebCore/testing/Internals.h

    r284518 r284523  
    640640    void setWebRTCVP9Support(bool supportVP9Profile0, bool supportVP9Profile2);
    641641    void setWebRTCVP9VTBSupport(bool);
     642    bool isSupportingVP9VTB() const;
     643    void isVP9VTBDeccoderUsed(RTCPeerConnection&, DOMPromiseDeferred<IDLBoolean>&&);
     644
    642645    void setSFrameCounter(RTCRtpSFrameTransform&, const String&);
    643646    uint64_t sframeCounter(const RTCRtpSFrameTransform&);

  • trunk/Source/WebCore/testing/Internals.idl

    r284518 r284523  
    869869    [Conditional=WEB_RTC] undefined setWebRTCVP9Support(boolean supportVP9Profile0, boolean supportVP9Profile2);
    870870    [Conditional=WEB_RTC] undefined setWebRTCVP9VTBSupport(boolean allowed);
     871    [Conditional=WEB_RTC] boolean isSupportingVP9VTB();
     872    [Conditional=WEB_RTC] Promise<boolean> isVP9VTBDeccoderUsed(RTCPeerConnection connection);
    871873    [Conditional=WEB_RTC] undefined setSFrameCounter(RTCRtpSFrameTransform transform, DOMString counter);
    872874    [Conditional=WEB_RTC] unsigned long long sframeCounter(RTCRtpSFrameTransform transform);
Note: See TracChangeset for help on using the changeset viewer.