Changeset 273461 in webkit

Timestamp:

Feb 24, 2021 4:57:53 PM (17 months ago)

Author:

commit-queue@webkit.org

Message:

[MSE] Media segment is incorrectly dropped when using negative timestampOffset or when source buffer appendWindow is set.
​https://bugs.webkit.org/show_bug.cgi?id=222260

Patch by Jean-Yves Avenard <Jean-Yves Avenard> on 2021-02-24
Reviewed by Eric Carlson.

Source/WebCore:

Test: media/media-source/media-source-timestampoffset-trim.html

CoreMedia packs multiple audio frames together into a single CMSampleBuffer,
this allows for faster processing and easier insertion into the track buffer tree.
However, per mediasoure spec [1], a frame is to be dropped according to
its start time and duration. So if only the beginning of the MediaSample
was to be dropped, we would have incorrectly dropped the lot.
We now split the MediaSample if it is going to be dropped to ensure that
all usable content is inserted into the track buffer.
Audio splicing isn't done yet, but this gets us closer to it.

[1] ​https://w3c.github.io/media-source/#sourcebuffer-coded-frame-processing

• platform/graphics/SourceBufferPrivate.cpp:

(WebCore::SourceBufferPrivate::didReceiveSample):

• platform/graphics/SourceBufferPrivate.h:
• platform/graphics/avfoundation/objc/SourceBufferPrivateAVFObjC.mm:

(WebCore::SourceBufferPrivateAVFObjC::didParseInitializationData):
(WebCore::SourceBufferPrivateAVFObjC::didProvideMediaDataForTrackId):

• platform/graphics/gstreamer/mse/SourceBufferPrivateGStreamer.cpp:

(WebCore::SourceBufferPrivateGStreamer::didReceiveSample):

• platform/graphics/gstreamer/mse/SourceBufferPrivateGStreamer.h:

LayoutTests:

• media/media-source/media-source-timestampoffset-trim.html:

Location:

trunk

Files:

• LayoutTests/ChangeLog (modified) (1 diff)
• LayoutTests/media/media-source/media-source-timestampoffset-trim-expected.txt (added)
• LayoutTests/media/media-source/media-source-timestampoffset-trim.html (added)
• Source/WebCore/ChangeLog (modified) (1 diff)
• Source/WebCore/platform/graphics/SourceBufferPrivate.cpp (modified) (11 diffs)
• Source/WebCore/platform/graphics/SourceBufferPrivate.h (modified) (1 diff)
• Source/WebCore/platform/graphics/avfoundation/objc/SourceBufferPrivateAVFObjC.mm (modified) (2 diffs)
• Source/WebCore/platform/graphics/gstreamer/mse/SourceBufferPrivateGStreamer.cpp (modified) (1 diff)
• Source/WebCore/platform/graphics/gstreamer/mse/SourceBufferPrivateGStreamer.h (modified) (1 diff)

trunk/LayoutTests/ChangeLog

@@ +1 @@
+2021-02-24  Jean-Yves Avenard  <jya@apple.com>
+
+        [MSE] Media segment is incorrectly dropped when using negative timestampOffset or when source buffer appendWindow is set.
+        https://bugs.webkit.org/show_bug.cgi?id=222260
+
+        Reviewed by Eric Carlson.
+
+        * media/media-source/media-source-timestampoffset-trim.html:
+
 2021-02-24  Chris Dumez  <cdumez@apple.com>
 

trunk/Source/WebCore/ChangeLog

@@ +1 @@
+2021-02-24  Jean-Yves Avenard  <jya@apple.com>
+
+        [MSE] Media segment is incorrectly dropped when using negative timestampOffset or when source buffer appendWindow is set.
+        https://bugs.webkit.org/show_bug.cgi?id=222260
+
+        Reviewed by Eric Carlson.
+
+        Test: media/media-source/media-source-timestampoffset-trim.html
+
+        CoreMedia packs multiple audio frames together into a single CMSampleBuffer,
+        this allows for faster processing and easier insertion into the track buffer tree.
+        However, per mediasoure spec [1], a frame is to be dropped according to
+        its start time and duration. So if only the beginning of the MediaSample
+        was to be dropped, we would have incorrectly dropped the lot.
+        We now split the MediaSample if it is going to be dropped to ensure that
+        all usable content is inserted into the track buffer.
+        Audio splicing isn't done yet, but this gets us closer to it.
+
+        [1] https://w3c.github.io/media-source/#sourcebuffer-coded-frame-processing
+
+        * platform/graphics/SourceBufferPrivate.cpp:
+        (WebCore::SourceBufferPrivate::didReceiveSample):
+        * platform/graphics/SourceBufferPrivate.h:
+        * platform/graphics/avfoundation/objc/SourceBufferPrivateAVFObjC.mm:
+        (WebCore::SourceBufferPrivateAVFObjC::didParseInitializationData):
+        (WebCore::SourceBufferPrivateAVFObjC::didProvideMediaDataForTrackId):
+        * platform/graphics/gstreamer/mse/SourceBufferPrivateGStreamer.cpp:
+        (WebCore::SourceBufferPrivateGStreamer::didReceiveSample):
+        * platform/graphics/gstreamer/mse/SourceBufferPrivateGStreamer.h:
+
 2021-02-24  Chris Dumez  <cdumez@apple.com>
 

trunk/Source/WebCore/platform/graphics/SourceBufferPrivate.cpp

@@ -817 +817 @@
 }
 
-void SourceBufferPrivate::didReceiveSample(MediaSample& sample)
+void SourceBufferPrivate::didReceiveSample(Ref<MediaSample>&& originalSample)
 {
     if (!m_isAttached)
@@ -843 +843 @@
     // 1. For each coded frame in the media segment run the following steps:
     // 1.1. Loop Top
+
+    Ref<MediaSample> sample = WTFMove(originalSample);
+
     do {
         MediaTime presentationTimestamp;
@@ -851 +854 @@
         // 1.2 Let frame duration be a double precision floating point representation of the coded frame's
         // duration in seconds.
-        MediaTime frameDuration = sample.duration();
+        MediaTime frameDuration = sample->duration();
 
         if (m_shouldGenerateTimestamps) {
@@ -866 +869 @@
             // 1. Let presentation timestamp be a double precision floating point representation of
             // the coded frame's presentation timestamp in seconds.
-            presentationTimestamp = sample.presentationTime();
+            presentationTimestamp = sample->presentationTime();
 
             // 2. Let decode timestamp be a double precision floating point representation of the coded frame's
             // decode timestamp in seconds.
-            decodeTimestamp = sample.decodeTime();
+            decodeTimestamp = sample->decodeTime();
         }
 
@@ -896 +899 @@
         // NOTE: this is out-of-order, but we need TrackBuffer to be able to cache the results of timestamp offset rounding
         // 1.5 Let track buffer equal the track buffer that the coded frame will be added to.
-        AtomString trackID = sample.trackID();
+        AtomString trackID = sample->trackID();
         auto it = m_trackBufferMap.find(trackID);
         if (it == m_trackBufferMap.end()) {
@@ -976 +979 @@
         if (m_appendMode == SourceBufferAppendMode::Sequence) {
             // Use the generated timestamps instead of the sample's timestamps.
-            sample.setTimestamps(presentationTimestamp, decodeTimestamp);
+            sample->setTimestamps(presentationTimestamp, decodeTimestamp);
         } else if (trackBuffer.roundedTimestampOffset) {
             // Reflect the timestamp offset into the sample.
-            sample.offsetTimestampsBy(trackBuffer.roundedTimestampOffset);
-        }
-
-        DEBUG_LOG(LOGIDENTIFIER, sample);
+            sample->offsetTimestampsBy(trackBuffer.roundedTimestampOffset);
+        }
+
+        DEBUG_LOG(LOGIDENTIFIER, sample.get());
 
         // 1.7 Let frame end timestamp equal the sum of presentation timestamp and frame duration.
@@ -994 +997 @@
         // the next coded frame.
         if (presentationTimestamp < m_appendWindowStart || frameEndTimestamp > m_appendWindowEnd) {
+            // 1.8 Note.
+            // Some implementations MAY choose to collect some of these coded frames with presentation
+            // timestamp less than appendWindowStart and use them to generate a splice at the first coded
+            // frame that has a presentation timestamp greater than or equal to appendWindowStart even if
+            // that frame is not a random access point. Supporting this requires multiple decoders or
+            // faster than real-time decoding so for now this behavior will not be a normative
+            // requirement.
+            // 1.9 Note.
+            // Some implementations MAY choose to collect coded frames with presentation timestamp less
+            // than appendWindowEnd and frame end timestamp greater than appendWindowEnd and use them to
+            // generate a splice across the portion of the collected coded frames within the append
+            // window at time of collection, and the beginning portion of later processed frames which
+            // only partially overlap the end of the collected coded frames. Supporting this requires
+            // multiple decoders or faster than real-time decoding so for now this behavior will not be a
+            // normative requirement. In conjunction with collecting coded frames that span
+            // appendWindowStart, implementations MAY thus support gapless audio splicing.
+            // Audio MediaSamples are typically made of packed audio samples. Trim sample to make it fit within the appendWindow.
+            if (sample->isDivisable()) {
+                std::pair<RefPtr<MediaSample>, RefPtr<MediaSample>> replacementSamples = sample->divide(m_appendWindowStart);
+                if (replacementSamples.second) {
+                    replacementSamples = replacementSamples.second->divide(m_appendWindowEnd);
+                    if (replacementSamples.first) {
+                        sample = replacementSamples.first.releaseNonNull();
+                        if (m_appendMode != SourceBufferAppendMode::Sequence && trackBuffer.roundedTimestampOffset)
+                            sample->offsetTimestampsBy(-trackBuffer.roundedTimestampOffset);
+                        continue;
+                    }
+                }
+            }
             trackBuffer.needRandomAccessFlag = true;
             m_client->sourceBufferPrivateDidDropSample();
@@ -1014 +1046 @@
             // 1.11.1 If the coded frame is not a random access point, then drop the coded frame and jump
             // to the top of the loop to start processing the next coded frame.
-            if (!sample.isSync()) {
+            if (!sample->isSync()) {
                 m_client->sourceBufferPrivateDidDropSample();
                 return;
@@ -1082 +1114 @@
         // the how the MSE specification should handlie this secnario.
         do {
-            if (!sample.isSync())
+            if (!sample->isSync())
                 break;
 
-            DecodeOrderSampleMap::KeyType decodeKey(sample.decodeTime(), sample.presentationTime());
+            DecodeOrderSampleMap::KeyType decodeKey(sample->decodeTime(), sample->presentationTime());
             auto nextSampleInDecodeOrder = trackBuffer.samples.decodeOrder().findSampleAfterDecodeKey(decodeKey);
             if (nextSampleInDecodeOrder == trackBuffer.samples.decodeOrder().end())
@@ -1152 +1184 @@
             // timestamp < presentationTime, but whose decode timestamp >= decodeTime. These will eventually cause
             // a decode error if left in place, so remove these samples as well.
-            DecodeOrderSampleMap::KeyType decodeKey(sample.decodeTime(), sample.presentationTime());
+            DecodeOrderSampleMap::KeyType decodeKey(sample->decodeTime(), sample->presentationTime());
             auto samplesWithHigherDecodeTimes = trackBuffer.samples.decodeOrder().findSamplesBetweenDecodeKeys(decodeKey, erasedSamples.decodeOrder().begin()->first);
             if (samplesWithHigherDecodeTimes.first != samplesWithHigherDecodeTimes.second)
@@ -1198 +1230 @@
         // If the frame is after the discontinuity boundary, the enqueueing algorithm will hold it there until samples
         // with earlier timestamps are enqueued. The decode queue is not FIFO, but rather an ordered map.
-        DecodeOrderSampleMap::KeyType decodeKey(sample.decodeTime(), sample.presentationTime());
+        DecodeOrderSampleMap::KeyType decodeKey(sample->decodeTime(), sample->presentationTime());
         if (trackBuffer.lastEnqueuedDecodeKey.first.isInvalid() || decodeKey > trackBuffer.lastEnqueuedDecodeKey) {
-            trackBuffer.decodeQueue.insert(DecodeOrderSampleMap::MapType::value_type(decodeKey, &sample));
-
-            if (trackBuffer.minimumEnqueuedPresentationTime.isValid() && sample.presentationTime() < trackBuffer.minimumEnqueuedPresentationTime)
+            trackBuffer.decodeQueue.insert(DecodeOrderSampleMap::MapType::value_type(decodeKey, &sample.get()));
+
+            if (trackBuffer.minimumEnqueuedPresentationTime.isValid() && sample->presentationTime() < trackBuffer.minimumEnqueuedPresentationTime)
                 trackBuffer.needsMinimumUpcomingPresentationTimeUpdating = true;
         }

trunk/Source/WebCore/platform/graphics/SourceBufferPrivate.h

@@ -162 +162 @@
     void reenqueSamples(const AtomString& trackID);
     WEBCORE_EXPORT void didReceiveInitializationSegment(SourceBufferPrivateClient::InitializationSegment&&, CompletionHandler<void()>&&);
-    WEBCORE_EXPORT void didReceiveSample(MediaSample&);
+    WEBCORE_EXPORT void didReceiveSample(Ref<MediaSample>&&);
     void provideMediaData(const AtomString& trackID);
     uint64_t totalTrackBufferSizeInBytes() const;

trunk/Source/WebCore/platform/graphics/avfoundation/objc/SourceBufferPrivateAVFObjC.mm

@@ -410 +410 @@
                 if (trackId == m_enabledVideoTrackID || m_audioRenderers.contains(trackId)) {
                     DEBUG_LOG(LOGIDENTIFIER, mediaSample.get());
-                    didReceiveSample(mediaSample);
+                    didReceiveSample(WTFMove(mediaSample));
                 }
             }
@@ -451 +451 @@
 
     DEBUG_LOG(LOGIDENTIFIER, mediaSample.get());
-    didReceiveSample(mediaSample);
+    didReceiveSample(WTFMove(mediaSample));
 }
 

trunk/Source/WebCore/platform/graphics/gstreamer/mse/SourceBufferPrivateGStreamer.cpp

@@ -187 +187 @@
 }
 
-void SourceBufferPrivateGStreamer::didReceiveSample(MediaSample& sample)
-{
-    SourceBufferPrivate::didReceiveSample(sample);
+void SourceBufferPrivateGStreamer::didReceiveSample(Ref<MediaSample>&& sample)
+{
+    SourceBufferPrivate::didReceiveSample(WTFMove(sample));
 }
 

trunk/Source/WebCore/platform/graphics/gstreamer/mse/SourceBufferPrivateGStreamer.h

@@ -74 +74 @@
 
     void didReceiveInitializationSegment(SourceBufferPrivateClient::InitializationSegment&&, CompletionHandler<void()>&&);
-    void didReceiveSample(MediaSample&);
+    void didReceiveSample(Ref<MediaSample>&&);
     void didReceiveAllPendingSamples();
     void appendParsingFailed();