Changeset 270641 in webkit


Ignore:
Timestamp:
Dec 10, 2020 11:48:38 AM (20 months ago)
Author:
youenn@apple.com
Message:

Add a SFrame VP8 experimental compatibility mode
​https://bugs.webkit.org/show_bug.cgi?id=219674

Reviewed by Eric Carlson.

Source/WebCore:

Similarly to H264, prefix the SFrame content with some data to ensure key frames or resolution can be detected
without revealing the actual content.

Covered by updated test.

  • Modules/mediastream/RTCRtpSFrameTransform.idl:
  • Modules/mediastream/RTCRtpSFrameTransformer.cpp:

(WebCore::RTCRtpSFrameTransformer::decryptFrame):
(WebCore::RTCRtpSFrameTransformer::encryptFrame):

  • Modules/mediastream/RTCRtpSFrameTransformer.h:
  • Modules/mediastream/SFrameUtils.cpp: Renamed from Source/WebCore/Modules/mediastream/H264Utils.cpp.

(WebCore::computeH264PrefixOffset):
(WebCore::computeH264PrefixBuffer):
(WebCore::isVP8KeyFrame):
(WebCore::computeVP8PrefixOffset):
(WebCore::computeVP8PrefixBuffer):

  • Modules/mediastream/SFrameUtils.h: Renamed from Source/WebCore/Modules/mediastream/H264Utils.h.
  • Sources.txt:
  • WebCore.xcodeproj/project.pbxproj:

LayoutTests:

  • webrtc/video-sframe-expected.txt:
  • webrtc/video-sframe.html:
Location:
trunk
Files:
10 edited
2 moved

Legend:

Unmodified
Added
Removed

  • trunk/LayoutTests/ChangeLog

    r270638 r270641  
     12020-12-10  Youenn Fablet  <youenn@apple.com>
     2
     3        Add a SFrame VP8 experimental compatibility mode
     4        https://bugs.webkit.org/show_bug.cgi?id=219674
     5
     6        Reviewed by Eric Carlson.
     7
     8        * webrtc/video-sframe-expected.txt:
     9        * webrtc/video-sframe.html:
     10
    1112020-12-10  Alex Christensen  <achristensen@webkit.org>
    212

  • trunk/LayoutTests/webrtc/video-sframe-expected.txt

    r270506 r270641  
    33PASS Video exchange with SFrame - H264 baseline
    44PASS Video exchange with SFrame - H264 high
     5PASS Video exchange with SFrame - VP8
    56

  • trunk/LayoutTests/webrtc/video-sframe.html

    r270620 r270641  
    1111        <script src ="routines.js"></script>
    1212        <script>
    13 async function doSFrameTest(selectCodec)
     13async function doSFrameTest(codec, shouldClose)
    1414{
     15    let pc1, pc2;
    1516    let sender, receiver;
    1617    const localStream = await navigator.mediaDevices.getUserMedia({video: true});
    … …  
    1819    const stream = await new Promise((resolve, reject) => {
    1920        createConnections((firstConnection) => {
     21            pc1 = firstConnection;
    2022            sender = firstConnection.addTrack(localStream.getTracks()[0], localStream);
     23            firstConnection.getTransceivers()[0].setCodecPreferences([codec]);
    2124            let transform = new SFrameTransform({compatibilityMode: "H264"});
    2225            transform.setEncryptionKey(key);
    2326            sender.transform = transform;
    2427        }, (secondConnection) => {
     28            pc2 = secondConnection;
    2529            secondConnection.ontrack = (trackEvent) => {
    2630                let transform = new SFrameTransform({compatibilityMode: "H264"});
    … …  
    3034                resolve(trackEvent.streams[0]);
    3135            };
    32         }, { observeOffer : (offer) => {
    33             offer.sdp = selectCodec(offer.sdp);
    34             return offer;
    35         }
    3636        });
    3737        setTimeout(() => reject("Test timed out"), 5000);
    … …  
    5151        assert_not_equals(internals.sframeCounter(sender.transform), senderCounter, "sender");
    5252    }
     53    if (shouldClose) {
     54        pc1.close();
     55        pc2.close();
     56    }
    5357}
    5458
    5559promise_test((test) => {
    56     return doSFrameTest(setH264BaselineCodec);
     60    return doSFrameTest({clockRate: 90000, mimeType: "video/H264", sdpFmtpLine: "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f"}, "H264", true);
    5761}, "Video exchange with SFrame - H264 baseline");
    5862
    5963promise_test((test) => {
    60     return doSFrameTest(setH264HighCodec);
     64    return doSFrameTest({clockRate: 90000, mimeType: "video/H264", sdpFmtpLine: "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=640c1f"}, "H264", true);
    6165}, "Video exchange with SFrame - H264 high");
     66
     67promise_test((test) => {
     68    return doSFrameTest({mimeType: "video/VP8", clockRate: 90000}, "VP8", false);
     69}, "Video exchange with SFrame - VP8");
    6270        </script>
    6371    </body>

  • trunk/Source/WebCore/ChangeLog

    r270640 r270641  
     12020-12-10  Youenn Fablet  <youenn@apple.com>
     2
     3        Add a SFrame VP8 experimental compatibility mode
     4        https://bugs.webkit.org/show_bug.cgi?id=219674
     5
     6        Reviewed by Eric Carlson.
     7
     8        Similarly to H264, prefix the SFrame content with some data to ensure key frames or resolution can be detected
     9        without revealing the actual content.
     10
     11        Covered by updated test.
     12
     13        * Modules/mediastream/RTCRtpSFrameTransform.idl:
     14        * Modules/mediastream/RTCRtpSFrameTransformer.cpp:
     15        (WebCore::RTCRtpSFrameTransformer::decryptFrame):
     16        (WebCore::RTCRtpSFrameTransformer::encryptFrame):
     17        * Modules/mediastream/RTCRtpSFrameTransformer.h:
     18        * Modules/mediastream/SFrameUtils.cpp: Renamed from Source/WebCore/Modules/mediastream/H264Utils.cpp.
     19        (WebCore::computeH264PrefixOffset):
     20        (WebCore::computeH264PrefixBuffer):
     21        (WebCore::isVP8KeyFrame):
     22        (WebCore::computeVP8PrefixOffset):
     23        (WebCore::computeVP8PrefixBuffer):
     24        * Modules/mediastream/SFrameUtils.h: Renamed from Source/WebCore/Modules/mediastream/H264Utils.h.
     25        * Sources.txt:
     26        * WebCore.xcodeproj/project.pbxproj:
     27
    1282020-12-10  Don Olmstead  <don.olmstead@sony.com>
    229

  • trunk/Source/WebCore/Modules/mediastream/RTCRtpSFrameTransform.idl

    r270620 r270641  
    3131enum RTCRtpSFrameTransformCompatibilityMode {
    3232    "none",
    33     "H264"
     33    "H264",
     34    "VP8"
    3435};
    3536

  • trunk/Source/WebCore/Modules/mediastream/RTCRtpSFrameTransformer.cpp

    r270532 r270641  
    2929#if ENABLE(WEB_RTC)
    3030
    31 #include "H264Utils.h"
     31#include "SFrameUtils.h"
    3232
    3333namespace WebCore {
    … …  
    197197{
    198198    Vector<uint8_t> buffer;
    199     if (m_compatibilityMode == CompatibilityMode::H264) {
    200         auto offset = computePrefixOffset(frameData, frameSize);
     199    switch (m_compatibilityMode) {
     200    case CompatibilityMode::H264: {
     201        auto offset = computeH264PrefixOffset(frameData, frameSize);
    201202        frameData += offset;
    202203        frameSize -= offset;
    … …  
    206207            frameSize = buffer.size();
    207208        }
     209        break;
     210    }
     211    case CompatibilityMode::VP8: {
     212        auto offset = computeVP8PrefixOffset(frameData, frameSize);
     213        frameData += offset;
     214        frameSize -= offset;
     215        break;
     216    }
     217    case CompatibilityMode::None:
     218        break;
    208219    }
    209220
    … …  
    260271
    261272    Vector<uint8_t> transformedData;
    262     H264PrefixBuffer prefixBuffer;
    263     if (m_compatibilityMode == CompatibilityMode::H264)
    264         prefixBuffer = computePrefixBuffer(frameData, frameSize);
     273    SFrameCompatibilityPrefixBuffer prefixBuffer;
     274    switch (m_compatibilityMode) {
     275    case CompatibilityMode::H264:
     276        prefixBuffer = computeH264PrefixBuffer(frameData, frameSize);
     277        break;
     278    case CompatibilityMode::VP8:
     279        prefixBuffer = computeVP8PrefixBuffer(frameData, frameSize);
     280        break;
     281    case CompatibilityMode::None:
     282        break;
     283    }
    265284
    266285    auto locker = holdLock(m_keyLock);

  • trunk/Source/WebCore/Modules/mediastream/RTCRtpSFrameTransformer.h

    r270532 r270641  
    3939class RTCRtpSFrameTransformer : public ThreadSafeRefCounted<RTCRtpSFrameTransformer, WTF::DestructionThread::Main> {
    4040public:
    41     enum class CompatibilityMode { None, H264 };
     41    enum class CompatibilityMode { None, H264, VP8 };
    4242
    4343    WEBCORE_EXPORT static Ref<RTCRtpSFrameTransformer> create(CompatibilityMode = CompatibilityMode::None);

  • trunk/Source/WebCore/Modules/mediastream/SFrameUtils.cpp

    r270640 r270641  
    2525
    2626#include "config.h"
    27 #include "H264Utils.h"
     27#include "SFrameUtils.h"
    2828
    2929#if ENABLE(WEB_RTC)
    … …  
    6161}
    6262
    63 size_t computePrefixOffset(const uint8_t* frameData, size_t frameSize)
     63size_t computeH264PrefixOffset(const uint8_t* frameData, size_t frameSize)
    6464{
    6565    size_t offset = 0;
    … …  
    105105}
    106106
    107 H264PrefixBuffer computePrefixBuffer(const uint8_t* frameData, size_t frameSize)
     107SFrameCompatibilityPrefixBuffer computeH264PrefixBuffer(const uint8_t* frameData, size_t frameSize)
    108108{
    109109    // Delta and key prefixes assume SPS/PPS with IDs equal to 0 have been transmitted.
    … …  
    115115    // We assume a key frame starts with SPS, then PPS. Otherwise we wrap it as a delta frame.
    116116    if (!isSPSNALU(frameData[4]))
    117         return H264PrefixBuffer { prefixDeltaFrame, sizeof(prefixDeltaFrame), { } };
     117        return SFrameCompatibilityPrefixBuffer { prefixDeltaFrame, sizeof(prefixDeltaFrame), { } };
    118118
    119119    // Search for PPS
    … …  
    125125    });
    126126    if (!spsPpsLength)
    127         return H264PrefixBuffer { prefixDeltaFrame, sizeof(prefixDeltaFrame), { } };
     127        return SFrameCompatibilityPrefixBuffer { prefixDeltaFrame, sizeof(prefixDeltaFrame), { } };
    128128
    129129    // Search for next NALU to compute the real spsPpsLength, including the next 00 00 00 01.
    … …  
    182182}
    183183
     184static inline bool isVP8KeyFrame(const uint8_t* frame, size_t size)
     185{
     186    ASSERT_UNUSED(size, size);
     187    return !(*frame & 0x01);
     188}
     189
     190size_t computeVP8PrefixOffset(const uint8_t* frame, size_t size)
     191{
     192    return isVP8KeyFrame(frame, size) ? 10 : 3;
     193}
     194
     195SFrameCompatibilityPrefixBuffer computeVP8PrefixBuffer(const uint8_t* frame, size_t size)
     196{
     197    Vector<uint8_t> prefix;
     198    prefix.append(frame, isVP8KeyFrame(frame, size) ? 10 : 3);
     199    return { prefix.data(), prefix.size(), WTFMove(prefix) };
     200}
     201
    184202} // namespace WebCore
    185203

  • trunk/Source/WebCore/Modules/mediastream/SFrameUtils.h

    r270640 r270641  
    3232namespace WebCore {
    3333
    34 struct H264PrefixBuffer {
     34struct SFrameCompatibilityPrefixBuffer {
    3535    const uint8_t* data { nullptr };
    3636    size_t size { 0 };
    … …  
    3838};
    3939
    40 size_t computePrefixOffset(const uint8_t*, size_t);
    41 H264PrefixBuffer computePrefixBuffer(const uint8_t*, size_t);
     40size_t computeH264PrefixOffset(const uint8_t*, size_t);
     41SFrameCompatibilityPrefixBuffer computeH264PrefixBuffer(const uint8_t*, size_t);
    4242
    4343WEBCORE_EXPORT bool needsRbspUnescaping(const uint8_t*, size_t);
    … …  
    4545WEBCORE_EXPORT void toRbsp(Vector<uint8_t>&, size_t);
    4646
     47size_t computeVP8PrefixOffset(const uint8_t*, size_t);
     48SFrameCompatibilityPrefixBuffer computeVP8PrefixBuffer(const uint8_t*, size_t);
     49
    4750} // namespace WebCore
    4851

  • trunk/Source/WebCore/Sources.txt

    r270587 r270641  
    144144Modules/mediasource/VideoPlaybackQuality.cpp
    145145Modules/mediastream/CanvasCaptureMediaStreamTrack.cpp
    146 Modules/mediastream/H264Utils.cpp
    147146Modules/mediastream/MediaDeviceInfo.cpp
    148147Modules/mediastream/MediaDevices.cpp
    … …  
    178177Modules/mediastream/RTCSessionDescription.cpp
    179178Modules/mediastream/RTCTrackEvent.cpp
     179Modules/mediastream/SFrameUtils.cpp
    180180Modules/mediastream/UserMediaController.cpp
    181181Modules/mediastream/UserMediaRequest.cpp

  • trunk/Source/WebCore/WebCore.xcodeproj/project.pbxproj

    r270587 r270641  
    11351135                41B2A6261EF1BF6D002B9D7A /* WebAudioSourceProvider.h in Headers */ = {isa = PBXBuildFile; fileRef = 41B2A6251EF1BF60002B9D7A /* WebAudioSourceProvider.h */; settings = {ATTRIBUTES = (Private, ); }; };
    11361136                41B8776223DE1045003638B8 /* RealtimeMediaSourceIdentifier.h in Headers */ = {isa = PBXBuildFile; fileRef = 41B8776023DE1042003638B8 /* RealtimeMediaSourceIdentifier.h */; settings = {ATTRIBUTES = (Private, ); }; };
    1137                 41BBA866257E373B00AC7F6D /* H264Utils.h in Headers */ = {isa = PBXBuildFile; fileRef = 41209E24257A2FBB00120ACA /* H264Utils.h */; settings = {ATTRIBUTES = (Private, ); }; };
     1137                41BBA866257E373B00AC7F6D /* SFrameUtils.h in Headers */ = {isa = PBXBuildFile; fileRef = 41209E24257A2FBB00120ACA /* SFrameUtils.h */; settings = {ATTRIBUTES = (Private, ); }; };
    11381138                41BF204922BA7BE80004F812 /* RealtimeVideoSource.h in Headers */ = {isa = PBXBuildFile; fileRef = 41BF204022B947160004F812 /* RealtimeVideoSource.h */; settings = {ATTRIBUTES = (Private, ); }; };
    11391139                41C760B10EDE03D300C1655F /* ScriptState.h in Headers */ = {isa = PBXBuildFile; fileRef = 41C760B00EDE03D300C1655F /* ScriptState.h */; settings = {ATTRIBUTES = (Private, ); }; };
    … …  
    75917591                41189EF71AD8232800B93F64 /* ReadableByteStreamController.idl */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = ReadableByteStreamController.idl; sourceTree = "<group>"; };
    75927592                41189EF71AD8232800B95672 /* ReadableStreamBYOBRequest.idl */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = ReadableStreamBYOBRequest.idl; sourceTree = "<group>"; };
    7593                 41209E24257A2FBB00120ACA /* H264Utils.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = H264Utils.h; sourceTree = "<group>"; };
    7594                 41209E26257A2FBB00120ACA /* H264Utils.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = H264Utils.cpp; sourceTree = "<group>"; };
     7593                41209E24257A2FBB00120ACA /* SFrameUtils.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = SFrameUtils.h; sourceTree = "<group>"; };
     7594                41209E26257A2FBB00120ACA /* SFrameUtils.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = SFrameUtils.cpp; sourceTree = "<group>"; };
    75957595                41209E91216D593C00A73A12 /* RTCCertificate.idl */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = RTCCertificate.idl; sourceTree = "<group>"; };
    75967596                41209E93216D5C7900A73A12 /* RTCCertificate.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = RTCCertificate.h; sourceTree = "<group>"; };
    … …  
    1709217092                                93A806111E03B51C008A1F26 /* DoubleRange.h */,
    1709317093                                93A806121E03B51C008A1F26 /* DoubleRange.idl */,
    17094                                 41209E26257A2FBB00120ACA /* H264Utils.cpp */,
    17095                                 41209E24257A2FBB00120ACA /* H264Utils.h */,
    1709617094                                93A806131E03B51C008A1F26 /* LongRange.h */,
    1709717095                                93A806141E03B51C008A1F26 /* LongRange.idl */,
    … …  
    1727017268                                5E2C43651BCEE3720001E2BC /* RTCTrackEvent.h */,
    1727117269                                5E2C43661BCEE3720001E2BC /* RTCTrackEvent.idl */,
     17270                                41209E26257A2FBB00120ACA /* SFrameUtils.cpp */,
     17271                                41209E24257A2FBB00120ACA /* SFrameUtils.h */,
    1727217272                                07221B8D17CEC32700848E51 /* UserMediaClient.h */,
    1727317273                                07221B8E17CEC32700848E51 /* UserMediaController.cpp */,
    … …  
    3217432174                                A12A1050166444FD008FA311 /* GridTrackSize.h in Headers */,
    3217532175                                E12DE7181E4B74A600F9ACCF /* GridTrackSizingAlgorithm.h in Headers */,
    32176                                 41BBA866257E373B00AC7F6D /* H264Utils.h in Headers */,
    3217732176                                26EA89A71B4F2B75008C5FD2 /* HashableActionList.h in Headers */,
    3217832177                                8482B7461198C35400BFB005 /* HashChangeEvent.h in Headers */,
    … …  
    3450434503                                7CD58DFB1F9565A500112791 /* Settings.h in Headers */,
    3450534504                                7CC9722E1F93E3360004D4CF /* SettingsBase.h in Headers */,
     34505                                41BBA866257E373B00AC7F6D /* SFrameUtils.h in Headers */,
    3450634506                                0F3DD45012F5EA1B000D9190 /* ShadowBlur.h in Headers */,
    3450734507                                BC5EB8C40E82031B00B25965 /* ShadowData.h in Headers */,

  • trunk/Tools/TestWebKitAPI/Tests/WebCore/RTCRtpSFrameTransformerTests.cpp

    r270506 r270641  
    2828#if ENABLE(WEB_RTC)
    2929
    30 #include <WebCore/H264Utils.h>
    3130#include <WebCore/RTCRtpSFrameTransformer.h>
     31#include <WebCore/SFrameUtils.h>
    3232#include <wtf/Vector.h>
    3333
Note: See TracChangeset for help on using the changeset viewer.