Changeset 283585 in webkit


Ignore:
Timestamp:
Oct 5, 2021 4:43:27 PM (10 months ago)
Author:
Jean-Yves Avenard
Message:

createImageBitmap using a HLS video as source always return a black image.
​https://bugs.webkit.org/show_bug.cgi?id=231225
rdar://83884031

Source/WebCore:

When playing HLS content, [AVURLAsset tracks] return an empty array.
We need to instead retrieve it from the AVPlayerItem object. The method
paintWithVideoOutput would have bailed out early as a consequence.
So we refactor the code a little to retrieve the tracks where they can be found

Reviewed by Eric Carlson.

Test: http/tests/media/video-hls-copy-into-canvas.html

  • platform/graphics/avfoundation/objc/MediaPlayerPrivateAVFoundationObjC.h:
  • platform/graphics/avfoundation/objc/MediaPlayerPrivateAVFoundationObjC.mm:

(WebCore::MediaPlayerPrivateAVFoundationObjC::createAVPlayerItem):
(WebCore::MediaPlayerPrivateAVFoundationObjC::tracksChanged):
(WebCore::MediaPlayerPrivateAVFoundationObjC::updateRotationSession):
(WebCore::MediaPlayerPrivateAVFoundationObjC::audioSourceProvider):
(WebCore::MediaPlayerPrivateAVFoundationObjC::updateLastImage):
(WebCore::MediaPlayerPrivateAVFoundationObjC::paintWithVideoOutput):
(WebCore::MediaPlayerPrivateAVFoundationObjC::firstEnabledTrack const):
(WebCore::MediaPlayerPrivateAVFoundationObjC::firstEnabledAudibleTrack const):
(WebCore::MediaPlayerPrivateAVFoundationObjC::firstEnabledVisibleTrack const):

LayoutTests:

Reviewed by Eric Carlson.

  • http/tests/media/video-hls-copy-into-canvas-expected.txt: Added.
  • http/tests/media/video-hls-copy-into-canvas.html: Added.
Location:
trunk
Files:
2 added
4 edited

Legend:

Unmodified
Added
Removed

  • trunk/LayoutTests/ChangeLog

    r283584 r283585  
     12021-10-05  Jean-Yves Avenard  <jya@apple.com>
     2
     3        createImageBitmap using a HLS video as source always return a black image.
     4        https://bugs.webkit.org/show_bug.cgi?id=231225
     5        rdar://83884031
     6
     7        Reviewed by Eric Carlson.
     8
     9        * http/tests/media/video-hls-copy-into-canvas-expected.txt: Added.
     10        * http/tests/media/video-hls-copy-into-canvas.html: Added.
     11
    1122021-10-05  Ayumi Kojima  <ayumi_kojima@apple.com>
    213

  • trunk/Source/WebCore/ChangeLog

    r283583 r283585  
     12021-10-05  Jean-Yves Avenard  <jya@apple.com>
     2
     3        createImageBitmap using a HLS video as source always return a black image.
     4        https://bugs.webkit.org/show_bug.cgi?id=231225
     5        rdar://83884031
     6
     7        When playing HLS content, [AVURLAsset tracks] return an empty array.
     8        We need to instead retrieve it from the AVPlayerItem object. The method
     9        paintWithVideoOutput would have bailed out early as a consequence.
     10        So we refactor the code a little to retrieve the tracks where they can be found
     11
     12        Reviewed by Eric Carlson.
     13
     14        Test: http/tests/media/video-hls-copy-into-canvas.html
     15
     16        * platform/graphics/avfoundation/objc/MediaPlayerPrivateAVFoundationObjC.h:
     17        * platform/graphics/avfoundation/objc/MediaPlayerPrivateAVFoundationObjC.mm:
     18        (WebCore::MediaPlayerPrivateAVFoundationObjC::createAVPlayerItem):
     19        (WebCore::MediaPlayerPrivateAVFoundationObjC::tracksChanged):
     20        (WebCore::MediaPlayerPrivateAVFoundationObjC::updateRotationSession):
     21        (WebCore::MediaPlayerPrivateAVFoundationObjC::audioSourceProvider):
     22        (WebCore::MediaPlayerPrivateAVFoundationObjC::updateLastImage):
     23        (WebCore::MediaPlayerPrivateAVFoundationObjC::paintWithVideoOutput):
     24        (WebCore::MediaPlayerPrivateAVFoundationObjC::firstEnabledTrack const):
     25        (WebCore::MediaPlayerPrivateAVFoundationObjC::firstEnabledAudibleTrack const):
     26        (WebCore::MediaPlayerPrivateAVFoundationObjC::firstEnabledVisibleTrack const):
     27
    1282021-10-05  Jean-Yves Avenard  <jya@apple.com>
    229

  • trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateAVFoundationObjC.h

    r283583 r283585  
    3333
    3434OBJC_CLASS AVAssetImageGenerator;
     35OBJC_CLASS AVAssetTrack;
    3536OBJC_CLASS AVAssetResourceLoadingRequest;
    3637OBJC_CLASS AVMediaSelectionGroup;
    … …  
    5152typedef struct CGImage *CGImageRef;
    5253typedef struct __CVBuffer *CVPixelBufferRef;
     54typedef NSString *AVMediaCharacteristic;
    5355
    5456namespace WebCore {
    … …  
    276278    AVMediaSelectionGroup *safeMediaSelectionGroupForVisualMedia();
    277279
    278     NSArray *safeAVAssetTracksForAudibleMedia();
    279     NSArray *safeAVAssetTracksForVisualMedia();
     280    AVAssetTrack* firstEnabledAudibleTrack() const;
     281    AVAssetTrack* firstEnabledVisibleTrack() const;
     282    AVAssetTrack* firstEnabledTrack(AVMediaCharacteristic) const;
    280283
    281284#if ENABLE(DATACUE_VALUE)

  • trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateAVFoundationObjC.mm

    r283583 r283585  
    248248static NSArray *assetTrackMetadataKeyNames();
    249249static NSArray *playerKVOProperties();
    250 static AVAssetTrack* firstEnabledTrack(NSArray* tracks);
    251250
    252251static dispatch_queue_t globalLoaderDelegateQueue()
    … …  
    11691168    if (m_provider) {
    11701169        m_provider->setPlayerItem(m_avPlayerItem.get());
    1171         m_provider->setAudioTrack(firstEnabledTrack(safeAVAssetTracksForAudibleMedia()));
     1170        m_provider->setAudioTrack(firstEnabledAudibleTrack());
    11721171    }
    11731172#endif
    … …  
    20872086}
    20882087
    2089 static AVAssetTrack* firstEnabledTrack(NSArray* tracks)
    2090 {
    2091     NSUInteger index = [tracks indexOfObjectPassingTest:^(id obj, NSUInteger, BOOL *) {
    2092         return [static_cast<AVAssetTrack*>(obj) isEnabled];
    2093     }];
    2094     if (index == NSNotFound)
    2095         return nil;
    2096     return [tracks objectAtIndex:index];
    2097 }
    2098 
    20992088void MediaPlayerPrivateAVFoundationObjC::metadataLoaded()
    21002089{
    … …  
    21422131        // We don't have a player item yet, so check with the asset because some assets support inspection
    21432132        // prior to becoming ready to play.
    2144         AVAssetTrack* firstEnabledVideoTrack = firstEnabledTrack(safeAVAssetTracksForVisualMedia());
     2133        auto* firstEnabledVideoTrack = firstEnabledVisibleTrack();
    21452134        setHasVideo(firstEnabledVideoTrack);
    2146         setHasAudio(firstEnabledTrack(safeAVAssetTracksForAudibleMedia()));
     2135        setHasAudio(firstEnabledAudibleTrack());
    21472136        auto size = firstEnabledVideoTrack ? FloatSize(CGSizeApplyAffineTransform([firstEnabledVideoTrack naturalSize], [firstEnabledVideoTrack preferredTransform])) : FloatSize();
    21482137        // For videos with rotation tag set, the transformation above might return a CGSize instance with negative width or height.
    … …  
    22152204#if ENABLE(WEB_AUDIO) && USE(MEDIATOOLBOX)
    22162205    if (m_provider)
    2217         m_provider->setAudioTrack(firstEnabledTrack(safeAVAssetTracksForAudibleMedia()));
     2206        m_provider->setAudioTrack(firstEnabledAudibleTrack());
    22182207#endif
    22192208
    … …  
    22282217    AffineTransform finalTransform = m_avAsset.get().preferredTransform;
    22292218    FloatSize naturalSize;
    2230     if (auto* firstEnabledVideoTrack = firstEnabledTrack(safeAVAssetTracksForVisualMedia())) {
     2219    if (auto* firstEnabledVideoTrack = firstEnabledVisibleTrack()) {
    22312220        naturalSize = FloatSize(firstEnabledVideoTrack.naturalSize);
    22322221        finalTransform *= firstEnabledVideoTrack.preferredTransform;
    … …  
    24182407    if (!m_provider) {
    24192408        m_provider = AudioSourceProviderAVFObjC::create(m_avPlayerItem.get());
    2420         m_provider->setAudioTrack(firstEnabledTrack(safeAVAssetTracksForAudibleMedia()));
     2409        m_provider->setAudioTrack(firstEnabledAudibleTrack());
    24212410    }
    24222411    return m_provider.get();
    … …  
    25462535        return;
    25472536
     2537    auto* firstEnabledVideoTrack = firstEnabledVisibleTrack();
     2538    if (!firstEnabledVideoTrack)
     2539        return;
     2540
    25482541    if (type == UpdateType::UpdateSynchronously && !m_lastImage && !videoOutputHasAvailableFrame())
    25492542        waitForVideoOutputMediaDataWillChange();
    … …  
    25712564    updateLastImage(UpdateType::UpdateSynchronously);
    25722565    if (!m_lastImage)
    2573         return;
    2574 
    2575     AVAssetTrack* firstEnabledVideoTrack = firstEnabledTrack(safeAVAssetTracksForVisualMedia());
    2576     if (!firstEnabledVideoTrack)
    25772566        return;
    25782567
    … …  
    27512740#endif
    27522741
    2753 NSArray* MediaPlayerPrivateAVFoundationObjC::safeAVAssetTracksForAudibleMedia()
    2754 {
     2742AVAssetTrack* MediaPlayerPrivateAVFoundationObjC::firstEnabledTrack(AVMediaCharacteristic characteristic) const
     2743{
     2744    if (m_avPlayerItem) {
     2745        for (AVPlayerItemTrack* track in [m_avPlayerItem tracks]) {
     2746            if (!track.enabled)
     2747                continue;
     2748            if (!track.assetTrack)
     2749                continue;
     2750            if ([track.assetTrack hasMediaCharacteristic:characteristic])
     2751                return track.assetTrack;
     2752        }
     2753    }
    27552754    if (!m_avAsset)
    27562755        return nil;
    … …  
    27592758        return nil;
    27602759
    2761     return [m_avAsset tracksWithMediaCharacteristic:AVMediaCharacteristicAudible];
    2762 }
    2763 
    2764 NSArray* MediaPlayerPrivateAVFoundationObjC::safeAVAssetTracksForVisualMedia()
    2765 {
    2766     if (!m_avAsset)
    2767         return nil;
    2768 
    2769     if ([m_avAsset.get() statusOfValueForKey:@"tracks" error:NULL] != AVKeyValueStatusLoaded)
    2770         return nil;
    2771 
    2772     return [m_avAsset tracksWithMediaCharacteristic:AVMediaCharacteristicVisual];
     2760    return [] (NSArray* tracks) -> AVAssetTrack* {
     2761        NSUInteger index = [tracks indexOfObjectPassingTest:^(id obj, NSUInteger, BOOL *) {
     2762            return [static_cast<AVAssetTrack*>(obj) isEnabled];
     2763        }];
     2764        if (index == NSNotFound)
     2765            return nil;
     2766        return [tracks objectAtIndex:index];
     2767    }([m_avAsset tracksWithMediaCharacteristic:characteristic]);
     2768}
     2769
     2770AVAssetTrack* MediaPlayerPrivateAVFoundationObjC::firstEnabledAudibleTrack() const
     2771{
     2772    return firstEnabledTrack(AVMediaCharacteristicAudible);
     2773}
     2774
     2775AVAssetTrack* MediaPlayerPrivateAVFoundationObjC::firstEnabledVisibleTrack() const
     2776{
     2777    return firstEnabledTrack(AVMediaCharacteristicVisual);
    27732778}
    27742779
Note: See TracChangeset for help on using the changeset viewer.