Changeset 258846 in webkit

Timestamp:

Mar 23, 2020 8:38:54 AM (4 years ago)

Author:

jer.noble@apple.com

Message:

[MSE] Handle the case where AVStreamDataParser packages sync and non-sync samples together in a CMSampleBufferRef.
​https://bugs.webkit.org/show_bug.cgi?id=209365
<rdar://problem/60625209>

Reviewed by Eric Carlson.

AVStreamDataParser will package together muliple samples into a single CMSampleBufferRef for efficiency's sake. When
this occurs, it may include sync and non-sync samples together into the same CMSampleBufferRef, which is problematic
as we consider a CMSampleBufferRef to be "sync" only when every sample inside the buffer is also sync.

To handle this scenario, when receiving a CMSampleBufferRef from AVStreamDataParser, first check whether that buffer
is "homogeneous", meaning every sample within the buffer has the same effective MediaSample flags. Then, if the buffer
is not homogenous, break the buffer into muliple homogenious CMSampleBufferRefs. Then, each of those resulting buffers
is passed up to SourceBuffer as a MediaSample individually.

• platform/graphics/avfoundation/objc/MediaSampleAVFObjC.h:
• platform/graphics/avfoundation/objc/MediaSampleAVFObjC.mm:

(WebCore::isCMSampleBufferAttachmentRandomAccess):
(WebCore::isCMSampleBufferRandomAccess):
(WebCore::isCMSampleBufferAttachmentNonDisplaying):
(WebCore::isCMSampleBufferNonDisplaying):
(WebCore::MediaSampleAVFObjC::flags const):
(WebCore::MediaSampleAVFObjC::isHomogeneous const):
(WebCore::MediaSampleAVFObjC::divideIntoHomogeneousSamples):

• platform/graphics/avfoundation/objc/SourceBufferPrivateAVFObjC.mm:

(WebCore::SourceBufferPrivateAVFObjC::processCodedFrame):

Location:

trunk/Source/WebCore

Files:

• ChangeLog (modified) (1 diff)
• platform/graphics/avfoundation/objc/MediaSampleAVFObjC.h (modified) (1 diff)
• platform/graphics/avfoundation/objc/MediaSampleAVFObjC.mm (modified) (6 diffs)
• platform/graphics/avfoundation/objc/SourceBufferPrivateAVFObjC.mm (modified) (1 diff)

trunk/Source/WebCore/ChangeLog

@@ +1 @@
+2020-03-23  Jer Noble  <jer.noble@apple.com>
+
+        [MSE] Handle the case where AVStreamDataParser packages sync and non-sync samples together in a CMSampleBufferRef.
+        https://bugs.webkit.org/show_bug.cgi?id=209365
+        <rdar://problem/60625209>
+
+        Reviewed by Eric Carlson.
+
+        AVStreamDataParser will package together muliple samples into a single CMSampleBufferRef for efficiency's sake. When
+        this occurs, it may include sync and non-sync samples together into the same CMSampleBufferRef, which is problematic
+        as we consider a CMSampleBufferRef to be "sync" only when every sample inside the buffer is also sync.
+
+        To handle this scenario, when receiving a CMSampleBufferRef from AVStreamDataParser, first check whether that buffer
+        is "homogeneous", meaning every sample within the buffer has the same effective MediaSample flags. Then, if the buffer
+        is not homogenous, break the buffer into muliple homogenious CMSampleBufferRefs. Then, each of those resulting buffers
+        is passed up to SourceBuffer as a MediaSample individually.
+
+        * platform/graphics/avfoundation/objc/MediaSampleAVFObjC.h:
+        * platform/graphics/avfoundation/objc/MediaSampleAVFObjC.mm:
+        (WebCore::isCMSampleBufferAttachmentRandomAccess):
+        (WebCore::isCMSampleBufferRandomAccess):
+        (WebCore::isCMSampleBufferAttachmentNonDisplaying):
+        (WebCore::isCMSampleBufferNonDisplaying):
+        (WebCore::MediaSampleAVFObjC::flags const):
+        (WebCore::MediaSampleAVFObjC::isHomogeneous const):
+        (WebCore::MediaSampleAVFObjC::divideIntoHomogeneousSamples):
+        * platform/graphics/avfoundation/objc/SourceBufferPrivateAVFObjC.mm:
+        (WebCore::SourceBufferPrivateAVFObjC::processCodedFrame):
+
 2020-03-23  Alicia Boya García  <aboya@igalia.com>
 

trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaSampleAVFObjC.h

@@ -69 +69 @@
     CMSampleBufferRef sampleBuffer() const { return m_sample.get(); }
 
+    bool isHomogeneous() const;
+    Vector<Ref<MediaSampleAVFObjC>> divideIntoHomogeneousSamples();
+
 protected:
     MediaSampleAVFObjC(RetainPtr<CMSampleBufferRef>&& sample)

trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaSampleAVFObjC.mm

@@ -116 +116 @@
 }
 
+static bool isCMSampleBufferAttachmentRandomAccess(CFDictionaryRef attachmentDict)
+{
+    return !CFDictionaryContainsKey(attachmentDict, kCMSampleAttachmentKey_NotSync);
+}
+
 static bool isCMSampleBufferRandomAccess(CMSampleBufferRef sample)
 {
@@ -123 +128 @@
     
     for (CFIndex i = 0, count = CFArrayGetCount(attachments); i < count; ++i) {
-        CFDictionaryRef attachmentDict = checked_cf_cast<CFDictionaryRef>(CFArrayGetValueAtIndex(attachments, i));
-        if (CFDictionaryContainsKey(attachmentDict, kCMSampleAttachmentKey_NotSync))
+        if (!isCMSampleBufferAttachmentRandomAccess(checked_cf_cast<CFDictionaryRef>(CFArrayGetValueAtIndex(attachments, i))))
             return false;
     }
     return true;
+}
+
+static bool isCMSampleBufferAttachmentNonDisplaying(CFDictionaryRef attachmentDict)
+{
+    return CFDictionaryContainsKey(attachmentDict, kCMSampleAttachmentKey_DoNotDisplay);
 }
 
@@ -137 +146 @@
     
     for (CFIndex i = 0; i < CFArrayGetCount(attachments); ++i) {
-        CFDictionaryRef attachmentDict = checked_cf_cast<CFDictionaryRef>(CFArrayGetValueAtIndex(attachments, i));
-        if (CFDictionaryContainsKey(attachmentDict, kCMSampleAttachmentKey_DoNotDisplay))
+        if (isCMSampleBufferAttachmentNonDisplaying(checked_cf_cast<CFDictionaryRef>(CFArrayGetValueAtIndex(attachments, i))))
             return true;
     }
@@ -148 +156 @@
 {
     int returnValue = MediaSample::None;
-    
+
     if (isCMSampleBufferRandomAccess(m_sample.get()))
         returnValue |= MediaSample::IsSync;
@@ -154 +162 @@
     if (isCMSampleBufferNonDisplaying(m_sample.get()))
         returnValue |= MediaSample::IsNonDisplaying;
-    
+
     return SampleFlags(returnValue);
 }
@@ -317 +325 @@
 }
 
-}
+bool MediaSampleAVFObjC::isHomogeneous() const
+{
+    CFArrayRef attachmentsArray = CMSampleBufferGetSampleAttachmentsArray(m_sample.get(), true);
+    if (!attachmentsArray)
+        return true;
+
+    auto count = CFArrayGetCount(attachmentsArray);
+    if (count <= 1)
+        return true;
+
+    CFDictionaryRef firstAttachment = checked_cf_cast<CFDictionaryRef>(CFArrayGetValueAtIndex(attachmentsArray, 0));
+    bool isSync = isCMSampleBufferAttachmentRandomAccess(firstAttachment);
+    bool isNonDisplaying = isCMSampleBufferAttachmentNonDisplaying(firstAttachment);
+
+    for (CFIndex i = 1; i < count; ++i) {
+        auto attachmentDict = checked_cf_cast<CFDictionaryRef>(CFArrayGetValueAtIndex(attachmentsArray, i));
+        if (isSync != isCMSampleBufferAttachmentRandomAccess(attachmentDict))
+            return false;
+
+        if (isNonDisplaying != isCMSampleBufferAttachmentNonDisplaying(attachmentDict))
+            return false;
+    };
+
+    return true;
+}
+
+Vector<Ref<MediaSampleAVFObjC>> MediaSampleAVFObjC::divideIntoHomogeneousSamples()
+{
+    using SampleVector = Vector<Ref<MediaSampleAVFObjC>>;
+
+    CFArrayRef attachmentsArray = CMSampleBufferGetSampleAttachmentsArray(m_sample.get(), true);
+    if (!attachmentsArray)
+        return SampleVector::from(makeRef(*this));
+
+    auto count = CFArrayGetCount(attachmentsArray);
+    if (count <= 1)
+        return SampleVector::from(makeRef(*this));
+
+    CFDictionaryRef firstAttachment = checked_cf_cast<CFDictionaryRef>(CFArrayGetValueAtIndex(attachmentsArray, 0));
+    bool isSync = isCMSampleBufferAttachmentRandomAccess(firstAttachment);
+    bool isNonDisplaying = isCMSampleBufferAttachmentNonDisplaying(firstAttachment);
+    Vector<CFRange> ranges;
+    CFIndex currentRangeStart = 0;
+    CFIndex currentRangeLength = 1;
+
+    for (CFIndex i = 1; i < count; ++i, ++currentRangeLength) {
+        CFDictionaryRef attachmentDict = checked_cf_cast<CFDictionaryRef>(CFArrayGetValueAtIndex(attachmentsArray, i));
+        if (isSync == isCMSampleBufferAttachmentRandomAccess(attachmentDict) && isNonDisplaying == isCMSampleBufferAttachmentNonDisplaying(attachmentDict))
+            continue;
+
+        ranges.append(CFRangeMake(currentRangeStart, currentRangeLength));
+        currentRangeStart = i;
+        currentRangeLength = 0;
+    }
+    ranges.append(CFRangeMake(currentRangeStart, currentRangeLength));
+
+    if (ranges.size() == 1)
+        return SampleVector::from(makeRef(*this));
+
+    SampleVector samples;
+    samples.reserveInitialCapacity(ranges.size());
+    for (auto& range : ranges) {
+        CMSampleBufferRef rawSample = nullptr;
+        if (CMSampleBufferCopySampleBufferForRange(kCFAllocatorDefault, m_sample.get(), range, &rawSample) != noErr || !rawSample)
+            return { };
+        samples.uncheckedAppend(MediaSampleAVFObjC::create(adoptCF(rawSample).get(), m_id));
+    }
+    return samples;
+}
+
+}

trunk/Source/WebCore/platform/graphics/avfoundation/objc/SourceBufferPrivateAVFObjC.mm

@@ -599 +599 @@
         return false;
 
-    if (m_client) {
-        Ref<MediaSample> mediaSample = MediaSampleAVFObjC::create(sampleBuffer, trackID);
-        DEBUG_LOG(LOGIDENTIFIER, mediaSample.get());
+    if (!m_client)
+        return true;
+
+    auto mediaSample = MediaSampleAVFObjC::create(sampleBuffer, trackID);
+
+    if (mediaSample->isHomogeneous()) {
+        DEBUG_LOG(LOGIDENTIFIER, mediaSample->toJSONString());
         m_client->sourceBufferPrivateDidReceiveSample(mediaSample);
+        return true;
+    }
+
+    auto mediaSamples = mediaSample->divideIntoHomogeneousSamples();
+    for (auto& sample : mediaSamples) {
+        DEBUG_LOG(LOGIDENTIFIER, sample->toJSONString());
+        m_client->sourceBufferPrivateDidReceiveSample(sample);
     }