Changeset 221098 in webkit


Ignore:
Timestamp:
Aug 23, 2017 1:16:51 PM (7 years ago)
Author:
jer.noble@apple.com
Message:

Track VideoPlaybackQuality metrics when using WebCoreDecompressionSession.
​https://bugs.webkit.org/show_bug.cgi?id=175835
<rdar://problem/34022234>

Reviewed by Eric Carlson.

Source/WebCore:

Test: platform/mac/media/media-source/videoplaybackquality-decompressionsession.html

Track the total number of frames decoded, dropped, & corrupted, as well as the total
delay imposed by decoding in the WebCoreDecompressionSession.

Drive-by fix: implement frame dropping by skipping frames whose presentation times are
before the video's current time and which aren't depended upon by other frames.

  • platform/cf/CoreMediaSoftLink.cpp:
  • platform/cf/CoreMediaSoftLink.h:
  • platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaSourceAVFObjC.mm:

(WebCore::MediaPlayerPrivateMediaSourceAVFObjC::videoPlaybackQualityMetrics):

  • platform/graphics/cocoa/WebCoreDecompressionSession.h:

(WebCore::WebCoreDecompressionSession::totalVideoFrames):
(WebCore::WebCoreDecompressionSession::droppedVideoFrames):
(WebCore::WebCoreDecompressionSession::corruptedVideoFrames):
(WebCore::WebCoreDecompressionSession::totalFrameDelay):

  • platform/graphics/cocoa/WebCoreDecompressionSession.mm:

(WebCore::WebCoreDecompressionSession::shouldDecodeSample):
(WebCore::WebCoreDecompressionSession::decodeSample):
(WebCore::WebCoreDecompressionSession::handleDecompressionOutput):

LayoutTests:

  • platform/mac/media/media-source/videoplaybackquality-decompressionsession-expected.txt: Added.
  • platform/mac/media/media-source/videoplaybackquality-decompressionsession.html: Added.
Location:
trunk
Files:
3 added
8 edited

Legend:

Unmodified
Added
Removed

  • trunk/LayoutTests/ChangeLog

    r221096 r221098  
     12017-08-23  Jer Noble  <jer.noble@apple.com>
     2
     3        Track VideoPlaybackQuality metrics when using WebCoreDecompressionSession.
     4        https://bugs.webkit.org/show_bug.cgi?id=175835
     5        <rdar://problem/34022234>
     6
     7        Reviewed by Eric Carlson.
     8
     9        * platform/mac/media/media-source/videoplaybackquality-decompressionsession-expected.txt: Added.
     10        * platform/mac/media/media-source/videoplaybackquality-decompressionsession.html: Added.
     11
    1122017-08-23  Matt Lewis  <jlewis3@apple.com>
    213

  • trunk/LayoutTests/media/media-source/media-source-loader.js

    r178486 r221098  
    1 function MediaSourceLoader(url)
     1function MediaSourceLoader(url, prefix)
    22{
    33    this._url = url;
     4    this._prefix = prefix;
    45    setTimeout(this.loadManifest.bind(this));
    56
    … …  
    4142    {
    4243        var request = new XMLHttpRequest();
    43         request.open('GET', this._manifest.url, true);
     44        var url = (this._prefix ? this._prefix : '') + this._manifest.url
     45        request.open('GET', url, true);
    4446        request.responseType = 'arraybuffer';
    4547        request.onload = this.loadMediaDataSucceeded.bind(this);

  • trunk/Source/WebCore/ChangeLog

    r221094 r221098  
     12017-08-23  Jer Noble  <jer.noble@apple.com>
     2
     3        Track VideoPlaybackQuality metrics when using WebCoreDecompressionSession.
     4        https://bugs.webkit.org/show_bug.cgi?id=175835
     5        <rdar://problem/34022234>
     6
     7        Reviewed by Eric Carlson.
     8
     9        Test: platform/mac/media/media-source/videoplaybackquality-decompressionsession.html
     10
     11        Track the total number of frames decoded, dropped, & corrupted, as well as the total
     12        delay imposed by decoding in the WebCoreDecompressionSession.
     13
     14        Drive-by fix: implement frame dropping by skipping frames whose presentation times are
     15        before the video's current time and which aren't depended upon by other frames.
     16
     17        * platform/cf/CoreMediaSoftLink.cpp:
     18        * platform/cf/CoreMediaSoftLink.h:
     19        * platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaSourceAVFObjC.mm:
     20        (WebCore::MediaPlayerPrivateMediaSourceAVFObjC::videoPlaybackQualityMetrics):
     21        * platform/graphics/cocoa/WebCoreDecompressionSession.h:
     22        (WebCore::WebCoreDecompressionSession::totalVideoFrames):
     23        (WebCore::WebCoreDecompressionSession::droppedVideoFrames):
     24        (WebCore::WebCoreDecompressionSession::corruptedVideoFrames):
     25        (WebCore::WebCoreDecompressionSession::totalFrameDelay):
     26        * platform/graphics/cocoa/WebCoreDecompressionSession.mm:
     27        (WebCore::WebCoreDecompressionSession::shouldDecodeSample):
     28        (WebCore::WebCoreDecompressionSession::decodeSample):
     29        (WebCore::WebCoreDecompressionSession::handleDecompressionOutput):
     30
    1312017-08-23  Andy Estes  <aestes@apple.com>
    232

  • trunk/Source/WebCore/platform/cf/CoreMediaSoftLink.cpp

    r220243 r221098  
    4949SOFT_LINK_FUNCTION_FOR_SOURCE(WebCore, CoreMedia, CMTimeMake, CMTime, (int64_t value, int32_t timescale), (value, timescale))
    5050SOFT_LINK_FUNCTION_FOR_SOURCE(WebCore, CoreMedia, CMTimeMakeWithSeconds, CMTime, (Float64 seconds, int32_t preferredTimeScale), (seconds, preferredTimeScale))
     51SOFT_LINK_FUNCTION_FOR_SOURCE(WebCore, CoreMedia, CMTimeSubtract, CMTime, (CMTime minuend, CMTime subtrahend), (minuend, subtrahend))
    5152SOFT_LINK_FUNCTION_FOR_SOURCE(WebCore, CoreMedia, CMTimeRangeGetEnd, CMTime, (CMTimeRange range), (range))
    5253SOFT_LINK_FUNCTION_FOR_SOURCE(WebCore, CoreMedia, CMTimeRangeMake, CMTimeRange, (CMTime start, CMTime duration), (start, duration))
    … …  
    156157SOFT_LINK_FUNCTION_FOR_SOURCE(WebCore, CoreMedia, CMTimeMinimum, CMTime, (CMTime time1, CMTime time2), (time1, time2))
    157158SOFT_LINK_FUNCTION_FOR_SOURCE(WebCore, CoreMedia, CMTimeRangeContainsTime, Boolean, (CMTimeRange range, CMTime time), (range, time))
    158 SOFT_LINK_FUNCTION_FOR_SOURCE(WebCore, CoreMedia, CMTimeSubtract, CMTime, (CMTime minuend, CMTime subtrahend), (minuend, subtrahend))
    159159
    160160SOFT_LINK_CONSTANT_FOR_SOURCE(WebCore, CoreMedia, kCMTimeIndefinite, CMTime)

  • trunk/Source/WebCore/platform/cf/CoreMediaSoftLink.h

    r220243 r221098  
    6060SOFT_LINK_FUNCTION_FOR_HEADER(WebCore, CoreMedia, CMTimeMakeWithSeconds, CMTime, (Float64 seconds, int32_t preferredTimeScale), (seconds, preferredTimeScale))
    6161#define CMTimeMakeWithSeconds softLink_CoreMedia_CMTimeMakeWithSeconds
     62SOFT_LINK_FUNCTION_FOR_HEADER(WebCore, CoreMedia, CMTimeSubtract, CMTime, (CMTime minuend, CMTime subtrahend), (minuend, subtrahend))
     63#define CMTimeSubtract softLink_CoreMedia_CMTimeSubtract
    6264SOFT_LINK_FUNCTION_FOR_HEADER(WebCore, CoreMedia, CMTimeRangeGetEnd, CMTime, (CMTimeRange range), (range))
    6365#define CMTimeRangeGetEnd softLink_CoreMedia_CMTimeRangeGetEnd
    … …  
    264266SOFT_LINK_FUNCTION_FOR_HEADER(WebCore, CoreMedia, CMTimeRangeContainsTime, Boolean, (CMTimeRange range, CMTime time), (range, time))
    265267#define CMTimeRangeContainsTime softLink_CoreMedia_CMTimeRangeContainsTime
    266 SOFT_LINK_FUNCTION_FOR_HEADER(WebCore, CoreMedia, CMTimeSubtract, CMTime, (CMTime minuend, CMTime subtrahend), (minuend, subtrahend))
    267 #define CMTimeSubtract softLink_CoreMedia_CMTimeSubtract
    268268
    269269SOFT_LINK_CONSTANT_FOR_HEADER(WebCore, CoreMedia, kCMTimeIndefinite, CMTime)

  • trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaSourceAVFObjC.mm

    r221046 r221098  
    690690std::optional<PlatformVideoPlaybackQualityMetrics> MediaPlayerPrivateMediaSourceAVFObjC::videoPlaybackQualityMetrics()
    691691{
     692    if (m_decompressionSession) {
     693        return PlatformVideoPlaybackQualityMetrics(
     694            m_decompressionSession->totalVideoFrames(),
     695            m_decompressionSession->droppedVideoFrames(),
     696            m_decompressionSession->corruptedVideoFrames(),
     697            m_decompressionSession->totalFrameDelay().toDouble()
     698        );
     699    }
    692700
    693701    auto metrics = [m_sampleBufferDisplayLayer videoPerformanceMetrics];

  • trunk/Source/WebCore/platform/graphics/cocoa/WebCoreDecompressionSession.h

    r220186 r221098  
    7171    void flush();
    7272
     73    unsigned long totalVideoFrames() { return m_totalVideoFrames; }
     74    unsigned long droppedVideoFrames() { return m_droppedVideoFrames; }
     75    unsigned long corruptedVideoFrames() { return m_corruptedVideoFrames; }
     76    MediaTime totalFrameDelay() { return m_totalFrameDelay; }
     77
    7378private:
    7479    WebCoreDecompressionSession();
    … …  
    8085    void resetAutomaticDequeueTimer();
    8186    void automaticDequeue();
     87    bool shouldDecodeSample(CMSampleBufferRef, bool displaying);
    8288
    8389    static void decompressionOutputCallback(void* decompressionOutputRefCon, void* sourceFrameRefCon, OSStatus, VTDecodeInfoFlags, CVImageBufferRef, CMTime presentationTimeStamp, CMTime presentationDuration);
    … …  
    107113    bool m_invalidated { false };
    108114    int m_framesBeingDecoded { 0 };
     115    unsigned long m_totalVideoFrames { 0 };
     116    unsigned long m_droppedVideoFrames { 0 };
     117    unsigned long m_corruptedVideoFrames { 0 };
     118    MediaTime m_totalFrameDelay;
    109119};
    110120

  • trunk/Source/WebCore/platform/graphics/cocoa/WebCoreDecompressionSession.mm

    r220538 r221098  
    181181}
    182182
     183bool WebCoreDecompressionSession::shouldDecodeSample(CMSampleBufferRef sample, bool displaying)
     184{
     185    if (!displaying)
     186        return true;
     187
     188    if (!m_timebase)
     189        return true;
     190
     191    auto currentTime = CMTimebaseGetTime(m_timebase.get());
     192    auto presentationTimeStamp = CMSampleBufferGetPresentationTimeStamp(sample);
     193    if (CMTimeCompare(presentationTimeStamp, currentTime) >= 0)
     194        return true;
     195
     196    CFArrayRef attachments = CMSampleBufferGetSampleAttachmentsArray(sample, false);
     197    if (!attachments)
     198        return true;
     199
     200    for (CFIndex index = 0, count = CFArrayGetCount(attachments); index < count; ++index) {
     201        CFDictionaryRef attachmentDict = (CFDictionaryRef)CFArrayGetValueAtIndex(attachments, index);
     202        CFBooleanRef dependedOn = (CFBooleanRef)CFDictionaryGetValue(attachmentDict, kCMSampleAttachmentKey_IsDependedOnByOthers);
     203        if (dependedOn && !CFBooleanGetValue(dependedOn))
     204            return false;
     205    }
     206
     207    return true;
     208}
     209
    183210void WebCoreDecompressionSession::decodeSample(CMSampleBufferRef sample, bool displaying)
    184211{
    … …  
    213240        flags |= kVTDecodeFrame_DoNotOutputFrame;
    214241
     242    if (!shouldDecodeSample(sample, displaying)) {
     243        ++m_totalVideoFrames;
     244        ++m_droppedVideoFrames;
     245        return;
     246    }
     247
    215248    VTDecompressionSessionDecodeFrame(m_decompressionSession.get(), sample, flags, reinterpret_cast<void*>(displaying), nullptr);
    216249}
    … …  
    225258void WebCoreDecompressionSession::handleDecompressionOutput(bool displaying, OSStatus status, VTDecodeInfoFlags infoFlags, CVImageBufferRef rawImageBuffer, CMTime presentationTimeStamp, CMTime presentationDuration)
    226259{
    227     UNUSED_PARAM(status);
    228     UNUSED_PARAM(infoFlags);
     260    ++m_totalVideoFrames;
     261    if (infoFlags & kVTDecodeInfo_FrameDropped)
     262        ++m_droppedVideoFrames;
    229263
    230264    CMVideoFormatDescriptionRef rawImageBufferDescription = nullptr;
    231     if (noErr != CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, rawImageBuffer, &rawImageBufferDescription))
    232         return;
     265    if (status != noErr || noErr != CMVideoFormatDescriptionCreateForImageBuffer(kCFAllocatorDefault, rawImageBuffer, &rawImageBufferDescription)) {
     266        ++m_corruptedVideoFrames;
     267        return;
     268    }
    233269    RetainPtr<CMVideoFormatDescriptionRef> imageBufferDescription = adoptCF(rawImageBufferDescription);
    234270
    … …  
    240276
    241277    CMSampleBufferRef rawImageSampleBuffer = nullptr;
    242     if (noErr != CMSampleBufferCreateReadyWithImageBuffer(kCFAllocatorDefault, rawImageBuffer, imageBufferDescription.get(), &imageBufferTiming, &rawImageSampleBuffer))
    243         return;
    244     RefPtr<WebCoreDecompressionSession> protectedThis { this };
    245     RetainPtr<CMSampleBufferRef> imageSampleBuffer = adoptCF(rawImageSampleBuffer);
    246     dispatch_async(m_enqueingQueue.get(), [protectedThis, imageSampleBuffer, displaying] {
     278    if (noErr != CMSampleBufferCreateReadyWithImageBuffer(kCFAllocatorDefault, rawImageBuffer, imageBufferDescription.get(), &imageBufferTiming, &rawImageSampleBuffer)) {
     279        ++m_corruptedVideoFrames;
     280        return;
     281    }
     282
     283    auto currentTime = CMTimebaseGetTime(m_timebase.get());
     284    if (m_timebase && CMTimeCompare(presentationTimeStamp, currentTime) < 0)
     285        m_totalFrameDelay += toMediaTime(CMTimeSubtract(currentTime, presentationTimeStamp));
     286
     287    dispatch_async(m_enqueingQueue.get(), [protectedThis = makeRefPtr(this), status, imageSampleBuffer = adoptCF(rawImageSampleBuffer), infoFlags, displaying] {
     288        UNUSED_PARAM(infoFlags);
    247289        protectedThis->enqueueDecodedSample(imageSampleBuffer.get(), displaying);
    248290    });
Note: See TracChangeset for help on using the changeset viewer.