Changeset 278272 in webkit

Timestamp:

May 31, 2021 12:28:32 AM (14 months ago)

Author:

youenn@apple.com

Message:

Use tighter bitrate allocation rules for WebRTC H264 software encoder
​https://bugs.webkit.org/show_bug.cgi?id=226319
<rdar://73150695>

Reviewed by Eric Carlson.

Software H264 encoder is sometimes overshooting target bitrate in which case WebRTC backend will start dropping frames.
The encoder might then think it is on target and will not try to increase compression.
This makes it possible to be locked in a very low frame rate but high quality image situation.
It is often better to preserve frame rate and lower quality, the application could always lower frame rate if desired.

To do so, we detect whether the encoder is using software code path or not.
If so, we compute the actual frame rate and compare it with the expected frame rate.
If the actual frame rate is twice smaller or even below, we enter in a low frame rate mode.
Otherwise, we are in a regular frame rate mode where we apply the normal bitrate rules.
In the low frame rate mode, we divide the target bitrate by 3 so as to quickly recover frame rate.

This works well in situations where motion increases from time to time.
It is still not perfect for instance in case the video is muted and gets unmuted or when the scene is completely still and suddenly large motion happens.
In those cases, frame rate is recovered after a minute or so according my testing.

• Source/webrtc/sdk/objc/components/video_codec/RTCVideoEncoderH264.mm:

(-[RTCVideoEncoderH264 initWithCodecInfo:]):
(-[RTCVideoEncoderH264 encode:codecSpecificInfo:frameTypes:]):
(-[RTCVideoEncoderH264 resetCompressionSessionWithPixelFormat:]):
(-[RTCVideoEncoderH264 setEncoderBitrateBps:frameRate:]):
(-[RTCVideoEncoderH264 updateBitRateAccordingActualFrameRate]):

Location:

trunk/Source/ThirdParty/libwebrtc

Files:

• ChangeLog (modified) (1 diff)
• Source/webrtc/sdk/objc/components/video_codec/RTCVideoEncoderH264.mm (modified) (10 diffs)

trunk/Source/ThirdParty/libwebrtc/ChangeLog

@@ +1 @@
+2021-05-31  Youenn Fablet  <youenn@apple.com>
+
+        Use tighter bitrate allocation rules for WebRTC H264 software encoder
+        https://bugs.webkit.org/show_bug.cgi?id=226319
+        <rdar://73150695>
+
+        Reviewed by Eric Carlson.
+
+        Software H264 encoder is sometimes overshooting target bitrate in which case WebRTC backend will start dropping frames.
+        The encoder might then think it is on target and will not try to increase compression.
+        This makes it possible to be locked in a very low frame rate but high quality image situation.
+        It is often better to preserve frame rate and lower quality, the application could always lower frame rate if desired.
+
+        To do so, we detect whether the encoder is using software code path or not.
+        If so, we compute the actual frame rate and compare it with the expected frame rate.
+        If the actual frame rate is twice smaller or even below, we enter in a low frame rate mode.
+        Otherwise, we are in a regular frame rate mode where we apply the normal bitrate rules.
+        In the low frame rate mode, we divide the target bitrate by 3 so as to quickly recover frame rate.
+
+        This works well in situations where motion increases from time to time.
+        It is still not perfect for instance in case the video is muted and gets unmuted or when the scene is completely still and suddenly large motion happens.
+        In those cases, frame rate is recovered after a minute or so according my testing.
+
+        * Source/webrtc/sdk/objc/components/video_codec/RTCVideoEncoderH264.mm:
+        (-[RTCVideoEncoderH264 initWithCodecInfo:]):
+        (-[RTCVideoEncoderH264 encode:codecSpecificInfo:frameTypes:]):
+        (-[RTCVideoEncoderH264 resetCompressionSessionWithPixelFormat:]):
+        (-[RTCVideoEncoderH264 setEncoderBitrateBps:frameRate:]):
+        (-[RTCVideoEncoderH264 updateBitRateAccordingActualFrameRate]):
+
 2021-05-20  Youenn Fablet  <youenn@apple.com>
 

trunk/Source/ThirdParty/libwebrtc/Source/webrtc/sdk/objc/components/video_codec/RTCVideoEncoderH264.mm

@@ -36 +36 @@
 #include "rtc_base/buffer.h"
 #include "rtc_base/logging.h"
+#include "rtc_base/system/arch.h"
 #include "rtc_base/time_utils.h"
 #include "sdk/objc/components/video_codec/nalu_rewriter.h"
@@ -61 +62 @@
                rotation:(RTCVideoRotation)rotation
      isKeyFrameRequired:(bool)isKeyFrameRequired;
-
+- (void)updateBitRateAccordingActualFrameRate;
 @end
 
@@ -76 +77 @@
 
 const OSType kNV12PixelFormat = kCVPixelFormatType_420YpCbCr8BiPlanarFullRange;
+
+const int kBelowFrameRateThreshold = 2;
+const int kBelowFrameRateBitRateDecrease = 3;
 
 // Struct that we pass to the encoder per frame to encode. We receive it again
@@ -350 +354 @@
   bool _isKeyFrameRequired;
   bool _isH264LowLatencyEncoderEnabled;
+  bool _isUsingSoftwareEncoder;
+  bool _isBelowExpectedFrameRate;
+  uint32_t _frameCount;
+  int64_t _lastFrameRateEstimationTime;
 }
 
@@ -378 +386 @@
   _isKeyFrameRequired = false;
   _isH264LowLatencyEncoderEnabled = true;
-
+  _isUsingSoftwareEncoder = false;
+  _isBelowExpectedFrameRate = false;
+  _frameCount = 0;
+  _lastFrameRateEstimationTime = 0;
   return self;
 }
@@ -442 +453 @@
   if (_disableEncoding) {
     return WEBRTC_VIDEO_CODEC_ERROR;
+  }
+  if (_isUsingSoftwareEncoder) {
+    [self updateBitRateAccordingActualFrameRate];
   }
 
@@ -752 +766 @@
                                &_vcpCompressionSession);
   }
+#elif defined(WEBRTC_MAC) && !defined(WEBRTC_IOS) && !(ENABLE_VCP_FOR_H264_BASELINE) && defined(WEBRTC_ARCH_X86_FAMILY)
+  CFBooleanRef hwaccl_enabled = nullptr;
+  if (status == noErr) {
+    auto result = VTSessionCopyProperty(_vtCompressionSession, kVTCompressionPropertyKey_UsingHardwareAcceleratedVideoEncoder, nullptr, &hwaccl_enabled);
+    _isUsingSoftwareEncoder = result == noErr ? !CFBooleanGetValue(hwaccl_enabled) : _width <= 480 && _height <= 480;
+  }
 #else
   // Provided encoder should be good enough.
@@ -840 +860 @@
 - (void)setEncoderBitrateBps:(uint32_t)bitrateBps frameRate:(uint32_t)frameRate {
   if ([self hasCompressionSession]) {
-    SetVTSessionProperty(_vtCompressionSession, _vcpCompressionSession, kVTCompressionPropertyKey_AverageBitRate, bitrateBps);
+    auto actualTarget = _isBelowExpectedFrameRate ? bitrateBps / kBelowFrameRateBitRateDecrease : bitrateBps;
+    SetVTSessionProperty(_vtCompressionSession, _vcpCompressionSession, kVTCompressionPropertyKey_AverageBitRate, actualTarget);
 
     // With zero |_maxAllowedFrameRate|, we fall back to automatic frame rate detection.
@@ -850 +871 @@
     // TODO(tkchin): Add a helper method to set array value.
     int64_t dataLimitBytesPerSecondValue =
-        static_cast<int64_t>(bitrateBps * kLimitToAverageBitRateFactor / 8);
+        static_cast<int64_t>(_isBelowExpectedFrameRate ? actualTarget / 8 : actualTarget * kLimitToAverageBitRateFactor / 8);
     CFNumberRef bytesPerSecond =
         CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt64Type, &dataLimitBytesPerSecondValue);
@@ -874 +895 @@
     _encoderFrameRate = frameRate;
   }
+}
+
+- (void)updateBitRateAccordingActualFrameRate {
+  _frameCount++;
+  auto currentTime = rtc::TimeMillis();
+  if (!_lastFrameRateEstimationTime) {
+    _lastFrameRateEstimationTime = currentTime;
+    return;
+  }
+
+  auto timeDifference = currentTime - _lastFrameRateEstimationTime;
+
+  // Let's not check too often.
+  if (timeDifference < 1000) {
+    return;
+  }
+
+  auto frameRate = _frameCount * 1000 / timeDifference;
+  _lastFrameRateEstimationTime = currentTime;
+  _frameCount = 0;
+
+  bool isBelow = frameRate * kBelowFrameRateThreshold < _encoderFrameRate;
+  if (isBelow == _isBelowExpectedFrameRate) {
+    return;
+  }
+
+  _isBelowExpectedFrameRate = isBelow;
+  [self setEncoderBitrateBps:_encoderBitrateBps frameRate:_encoderFrameRate];
 }