Changeset 278158 in webkit

Timestamp:

May 27, 2021 3:19:17 AM (14 months ago)

Author:

youenn@apple.com

Message:

Support H264 profiles in MediaRecorder
​https://bugs.webkit.org/show_bug.cgi?id=226219
<rdar://78027944>

Reviewed by Eric Carlson.

Source/WebCore:

Use mimeType option given in MediaRecorder options to detect which H264 profile to use.
Implement this by parsing the mime type codec parameter and use the avc1.WXYZ string to compute the profile.
By default, use baseline profile, which is not VideoToolbox profile but has wider decoding support.
If profile specified by the application fails, we downgrade to baseline.

Test: http/wpt/mediarecorder/MediaRecorder-video-h264-profiles.html

• Modules/mediarecorder/MediaRecorderProvider.cpp:

(WebCore::MediaRecorderProvider::isSupported):

• platform/mediarecorder/cocoa/MediaRecorderPrivateWriterCocoa.h:
• platform/mediarecorder/cocoa/MediaRecorderPrivateWriterCocoa.mm:

(WebCore::MediaRecorderPrivateWriter::create):
(WebCore::MediaRecorderPrivateWriter::initialize):
(WebCore::MediaRecorderPrivateWriter::setOptions): Deleted.

• platform/mediarecorder/cocoa/VideoSampleBufferCompressor.h:
• platform/mediarecorder/cocoa/VideoSampleBufferCompressor.mm:

(WebCore::VideoSampleBufferCompressor::create):
(WebCore::VideoSampleBufferCompressor::VideoSampleBufferCompressor):
(WebCore::VideoSampleBufferCompressor::vtProfileLevel const):
(WebCore::VideoSampleBufferCompressor::initCompressionSession):

LayoutTests:

• http/wpt/mediarecorder/MediaRecorder-video-h264-profiles-expected.txt: Added.
• http/wpt/mediarecorder/MediaRecorder-video-h264-profiles.html: Added.

Location:

trunk

Files:

• LayoutTests/ChangeLog (modified) (1 diff)
• LayoutTests/http/wpt/mediarecorder/MediaRecorder-video-h264-profiles-expected.txt (added)
• LayoutTests/http/wpt/mediarecorder/MediaRecorder-video-h264-profiles.html (added)
• Source/WebCore/ChangeLog (modified) (1 diff)
• Source/WebCore/Modules/mediarecorder/MediaRecorderProvider.cpp (modified) (1 diff)
• Source/WebCore/platform/mediarecorder/cocoa/MediaRecorderPrivateWriterCocoa.h (modified) (1 diff)
• Source/WebCore/platform/mediarecorder/cocoa/MediaRecorderPrivateWriterCocoa.mm (modified) (3 diffs)
• Source/WebCore/platform/mediarecorder/cocoa/VideoSampleBufferCompressor.h (modified) (4 diffs)
• Source/WebCore/platform/mediarecorder/cocoa/VideoSampleBufferCompressor.mm (modified) (5 diffs)

trunk/LayoutTests/ChangeLog

@@ +1 @@
+2021-05-27  Youenn Fablet  <youenn@apple.com>
+
+        Support H264 profiles in MediaRecorder
+        https://bugs.webkit.org/show_bug.cgi?id=226219
+        <rdar://78027944>
+
+        Reviewed by Eric Carlson.
+
+        * http/wpt/mediarecorder/MediaRecorder-video-h264-profiles-expected.txt: Added.
+        * http/wpt/mediarecorder/MediaRecorder-video-h264-profiles.html: Added.
+
 2021-05-27  Kimmo Kinnunen  <kkinnunen@apple.com>
 

trunk/Source/WebCore/ChangeLog

@@ +1 @@
+2021-05-27  Youenn Fablet  <youenn@apple.com>
+
+        Support H264 profiles in MediaRecorder
+        https://bugs.webkit.org/show_bug.cgi?id=226219
+        <rdar://78027944>
+
+        Reviewed by Eric Carlson.
+
+        Use mimeType option given in MediaRecorder options to detect which H264 profile to use.
+        Implement this by parsing the mime type codec parameter and use the avc1.WXYZ string to compute the profile.
+        By default, use baseline profile, which is not VideoToolbox profile but has wider decoding support.
+        If profile specified by the application fails, we downgrade to baseline.
+
+        Test: http/wpt/mediarecorder/MediaRecorder-video-h264-profiles.html
+
+        * Modules/mediarecorder/MediaRecorderProvider.cpp:
+        (WebCore::MediaRecorderProvider::isSupported):
+        * platform/mediarecorder/cocoa/MediaRecorderPrivateWriterCocoa.h:
+        * platform/mediarecorder/cocoa/MediaRecorderPrivateWriterCocoa.mm:
+        (WebCore::MediaRecorderPrivateWriter::create):
+        (WebCore::MediaRecorderPrivateWriter::initialize):
+        (WebCore::MediaRecorderPrivateWriter::setOptions): Deleted.
+        * platform/mediarecorder/cocoa/VideoSampleBufferCompressor.h:
+        * platform/mediarecorder/cocoa/VideoSampleBufferCompressor.mm:
+        (WebCore::VideoSampleBufferCompressor::create):
+        (WebCore::VideoSampleBufferCompressor::VideoSampleBufferCompressor):
+        (WebCore::VideoSampleBufferCompressor::vtProfileLevel const):
+        (WebCore::VideoSampleBufferCompressor::initCompressionSession):
+
 2021-05-27  Carlos Garcia Campos  <cgarcia@igalia.com>
 

trunk/Source/WebCore/Modules/mediarecorder/MediaRecorderProvider.cpp

@@ -59 +59 @@
     for (auto& item : mimeType.codecs()) {
         auto codec = StringView(item).stripLeadingAndTrailingMatchedCharacters(isHTMLSpace<UChar>);
-        // FIXME: We should further validate paramters.
+        // FIXME: We should further validate parameters.
         if (!startsWithLettersIgnoringASCIICase(codec, "avc1") && !startsWithLettersIgnoringASCIICase(codec, "mp4a"))
             return false;

trunk/Source/WebCore/platform/mediarecorder/cocoa/MediaRecorderPrivateWriterCocoa.h

@@ -86 +86 @@
     void clear();
 
-    bool initialize();
-    void setOptions(const MediaRecorderPrivateOptions&);
+    bool initialize(const MediaRecorderPrivateOptions&);
 
     static void compressedVideoOutputBufferCallback(void*, CMBufferQueueTriggerToken);

trunk/Source/WebCore/platform/mediarecorder/cocoa/MediaRecorderPrivateWriterCocoa.mm

@@ -99 +99 @@
 {
     auto writer = adoptRef(*new MediaRecorderPrivateWriter(hasAudio, hasVideo));
-    if (!writer->initialize())
+    if (!writer->initialize(options))
         return nullptr;
-    writer->setOptions(options);
     return writer;
 }
@@ -132 +131 @@
 }
 
-bool MediaRecorderPrivateWriter::initialize()
+bool MediaRecorderPrivateWriter::initialize(const MediaRecorderPrivateOptions& options)
 {
     NSError *error = nil;
@@ -150 +149 @@
         if (!m_audioCompressor)
             return false;
+        if (options.audioBitsPerSecond)
+            m_audioCompressor->setBitsPerSecond(*options.audioBitsPerSecond);
     }
     if (m_hasVideo) {
-        m_videoCompressor = VideoSampleBufferCompressor::create(kCMVideoCodecType_H264, compressedVideoOutputBufferCallback, this);
+        m_videoCompressor = VideoSampleBufferCompressor::create(options.mimeType, compressedVideoOutputBufferCallback, this);
         if (!m_videoCompressor)
             return false;
-    }
+        if (options.videoBitsPerSecond)
+            m_videoCompressor->setBitsPerSecond(*options.videoBitsPerSecond);
+    }
+
     return true;
-}
-
-void MediaRecorderPrivateWriter::setOptions(const MediaRecorderPrivateOptions& options)
-{
-    if (options.audioBitsPerSecond && m_audioCompressor)
-        m_audioCompressor->setBitsPerSecond(*options.audioBitsPerSecond);
-    if (options.videoBitsPerSecond && m_videoCompressor)
-        m_videoCompressor->setBitsPerSecond(*options.videoBitsPerSecond);
 }
 

trunk/Source/WebCore/platform/mediarecorder/cocoa/VideoSampleBufferCompressor.h

@@ -39 +39 @@
     WTF_MAKE_FAST_ALLOCATED;
 public:
-    static std::unique_ptr<VideoSampleBufferCompressor> create(CMVideoCodecType, CMBufferQueueTriggerCallback, void* callbackObject);
+    static std::unique_ptr<VideoSampleBufferCompressor> create(String mimeType, CMBufferQueueTriggerCallback, void* callbackObject);
     ~VideoSampleBufferCompressor();
 
@@ -51 +51 @@
 
 private:
-    explicit VideoSampleBufferCompressor(CMVideoCodecType);
+    enum class Profile { Baseline, Main, High };
+    VideoSampleBufferCompressor(CMVideoCodecType, Profile);
 
     bool initialize(CMBufferQueueTriggerCallback, void* callbackObject);
@@ -57 +58 @@
     void processSampleBuffer(CMSampleBufferRef);
     bool initCompressionSession(CMVideoFormatDescriptionRef);
+    CFStringRef vtProfileLevel() const;
 
     static void videoCompressionCallback(void *refCon, void*, OSStatus, VTEncodeInfoFlags, CMSampleBufferRef);
@@ -70 +72 @@
     unsigned m_expectedFrameRate { 30 };
     Optional<unsigned> m_outputBitRate;
+    Profile m_profile;
 };
 

trunk/Source/WebCore/platform/mediarecorder/cocoa/VideoSampleBufferCompressor.mm

@@ -28 +28 @@
 #if ENABLE(MEDIA_STREAM) && USE(AVFOUNDATION)
 
+#import "ContentType.h"
 #import "Logging.h"
 #import <CoreMedia/CoreMedia.h>
@@ -40 +41 @@
 using namespace PAL;
 
-std::unique_ptr<VideoSampleBufferCompressor> VideoSampleBufferCompressor::create(CMVideoCodecType outputCodecType, CMBufferQueueTriggerCallback callback, void* callbackObject)
-{
-    auto compressor = std::unique_ptr<VideoSampleBufferCompressor>(new VideoSampleBufferCompressor(outputCodecType));
+std::unique_ptr<VideoSampleBufferCompressor> VideoSampleBufferCompressor::create(String mimeType, CMBufferQueueTriggerCallback callback, void* callbackObject)
+{
+    auto profile = Profile::Baseline;
+    for (auto codec : ContentType(mimeType).codecs()) {
+        if (startsWithLettersIgnoringASCIICase(codec, "avc1.") && codec.length() >= 11) {
+            if (codec[5] == '6' && codec[6] == '4')
+                profile = Profile::High;
+            else if (codec[5] == '4' && (codec[6] == 'd' || codec[6] == 'D'))
+                profile = Profile::Main;
+            break;
+        }
+    }
+
+    auto compressor = std::unique_ptr<VideoSampleBufferCompressor>(new VideoSampleBufferCompressor(kCMVideoCodecType_H264, profile));
     if (!compressor->initialize(callback, callbackObject))
         return nullptr;
@@ -48 +60 @@
 }
 
-VideoSampleBufferCompressor::VideoSampleBufferCompressor(CMVideoCodecType outputCodecType)
+VideoSampleBufferCompressor::VideoSampleBufferCompressor(CMVideoCodecType outputCodecType, Profile profile)
     : m_serialDispatchQueue { WorkQueue::create("com.apple.VideoSampleBufferCompressor") }
     , m_outputCodecType { outputCodecType }
+    , m_profile { profile }
 {
 }
@@ -112 +125 @@
     OSStatus status = VTSessionSetProperty(vtSession, key, cfValue.get());
     return status;
+}
+
+CFStringRef VideoSampleBufferCompressor::vtProfileLevel() const
+{
+    switch (m_profile) {
+    case Profile::Baseline:
+        return kVTProfileLevel_H264_Baseline_AutoLevel;
+    case Profile::High:
+        return kVTProfileLevel_H264_High_AutoLevel;
+    case Profile::Main:
+        return kVTProfileLevel_H264_Main_AutoLevel;
+    }
 }
 
@@ -138 +163 @@
     RELEASE_LOG_ERROR_IF(error, MediaStream, "VideoSampleBufferCompressor VTSessionSetProperty kVTCompressionPropertyKey_ExpectedFrameRate failed with %d", error);
 
+    error = VTSessionSetProperty(m_vtSession.get(), kVTCompressionPropertyKey_ProfileLevel, vtProfileLevel());
+    if (error) {
+        RELEASE_LOG_ERROR(MediaStream, "VideoSampleBufferCompressor VTSessionSetProperty kVTCompressionPropertyKey_ProfileLevel failed with %d for profile %d", error, m_profile);
+        if (m_profile != Profile::Baseline) {
+            error = VTSessionSetProperty(m_vtSession.get(), kVTCompressionPropertyKey_ProfileLevel, kVTProfileLevel_H264_Baseline_AutoLevel);
+            RELEASE_LOG_ERROR_IF(error, MediaStream, "VideoSampleBufferCompressor VTSessionSetProperty kVTCompressionPropertyKey_ProfileLevel failed with %d for default profile", error);
+        }
+    }
+
     if (m_outputBitRate) {
         error = setCompressionSessionProperty(m_vtSession.get(), kVTCompressionPropertyKey_AverageBitRate, *m_outputBitRate);