Changeset 247165 in webkit


Ignore:
Timestamp:
Jul 5, 2019 11:54:14 AM (5 years ago)
Author:
youenn@apple.com
Message:

[iOS] Local capture MediaStreamTrack does not render in portrait mode
​https://bugs.webkit.org/show_bug.cgi?id=199519
<rdar://problem/52689720>

Reviewed by Eric Carlson.

Source/WebCore:

RealtimeVideoSource was badly computing its size in case of rotation.
Whenever its underlying source is notifying of settings change,
compute the size and transpose it only in left/right case.

Update mock video source to cover that case.
Covered by updated test.

  • platform/mediastream/RealtimeVideoSource.cpp:

(WebCore::RealtimeVideoSource::sourceSettingsChanged):

  • platform/mediastream/mac/MockRealtimeVideoSourceMac.h:
  • platform/mediastream/mac/MockRealtimeVideoSourceMac.mm:

(WebCore::MockRealtimeVideoSourceMac::updateSampleBuffer):

  • platform/mock/MockRealtimeVideoSource.cpp:

(WebCore::MockRealtimeVideoSource::settings):
(WebCore::MockRealtimeVideoSource::orientationChanged):
(WebCore::MockRealtimeVideoSource::monitorOrientation):

  • platform/mock/MockRealtimeVideoSource.h:

LayoutTests:

  • webrtc/video-rotation-expected.txt:
  • webrtc/video-rotation.html:
Location:
trunk
Files:
9 edited

Legend:

Unmodified
Added
Removed

  • trunk/LayoutTests/ChangeLog

    r247164 r247165  
     12019-07-05  Youenn Fablet  <youenn@apple.com>
     2
     3        [iOS] Local capture MediaStreamTrack does not render in portrait mode
     4        https://bugs.webkit.org/show_bug.cgi?id=199519
     5        <rdar://problem/52689720>
     6
     7        Reviewed by Eric Carlson.
     8
     9        * webrtc/video-rotation-expected.txt:
     10        * webrtc/video-rotation.html:
     11
    1122019-07-05  Ryan Haddad  <ryanhaddad@apple.com>
    213

  • trunk/LayoutTests/webrtc/video-rotation-expected.txt

    r245033 r247165  
    1  
     1   
    22
    33PASS Setting video exchange
    44PASS Track is enabled, video should not be black
    5 PASS Track is enabled and rotated, video should not be black and should change size
     5PASS Track is enabled and rotated, local video should not be black and should change size
     6PASS Track is enabled and rotated, remote video should not be black and should change size
    67PASS Track is enabled and rotated again, video should not be black and should change size
    78

  • trunk/LayoutTests/webrtc/video-rotation.html

    r245033 r247165  
    88    </head>
    99    <body>
    10         <video id="video" autoplay playsInline width="320" height="240"></video>
     10        <video id="localVideo" autoplay playsInline width="320" height="240"></video>
     11        <video id="remoteVideo" autoplay playsInline width="320" height="240"></video>
     12        <canvas id="canvas0" width="320" height="240"></canvas>
    1113        <canvas id="canvas1" width="320" height="240"></canvas>
    1214        <canvas id="canvas2" width="320" height="240"></canvas>
    … …  
    1416        <script src ="routines.js"></script>
    1517        <script>
    16 function isVideoBlack(id)
     18function isVideoBlack(video, canvasId)
    1719{
    18     var canvas = document.getElementById(id);
     20    var canvas = document.getElementById(canvasId);
    1921    canvas.width = video.videoWidth;
    2022    canvas.height = video.videoHeight;
    … …  
    3133}
    3234
    33 function pollVideoBlackCheck(expected, id, resolve)
     35function pollVideoBlackCheck(expected, video, canvasId, resolve)
    3436{
    35     if (isVideoBlack(id) === expected) {
     37    if (isVideoBlack(video, canvasId) === expected) {
    3638        resolve();
    3739        return;
    3840    }
    3941
    40     setTimeout(() => pollVideoBlackCheck(expected, id, resolve), 100);
     42    setTimeout(() => pollVideoBlackCheck(expected, video, canvasId, resolve), 100);
    4143}
    4244
    43 function checkVideoBlack(expected, id)
     45function checkVideoBlack(expected, video, canvasId)
    4446{
    4547    return new Promise((resolve, reject) => {
    46         pollVideoBlackCheck(expected, id, resolve);
    47         setTimeout(() => reject("checkVideoBlack timed out for '" + id + "', expected " + expected), 5000);
     48        pollVideoBlackCheck(expected, video, canvasId, resolve);
     49        setTimeout(() => reject("checkVideoBlack timed out for '" + canvasId + "', expected " + expected), 5000);
    4850    });
    4951}
    … …  
    5557
    5658    return navigator.mediaDevices.getUserMedia({video: {width: 320, height: 240, facingMode: "environment"}}).then((localStream) => {
     59        localVideo.srcObject = localStream;
    5760        return new Promise((resolve, reject) => {
    5861            track = localStream.getVideoTracks()[0];
    … …  
    7073        });
    7174    }).then((remoteStream) => {
    72         video.srcObject = remoteStream;
    73         return video.play();
     75        remoteVideo.srcObject = remoteStream;
     76        return remoteVideo.play();
    7477    });
    7578}, "Setting video exchange");
    7679
    77 promise_test((test) => {
    78    return checkVideoBlack(false, "canvas1");
     80promise_test(async (test) => {
     81   await checkVideoBlack(false, remoteVideo, "canvas0");
     82   await waitForVideoSize(remoteVideo, 320, 240);
    7983}, "Track is enabled, video should not be black");
     84
     85promise_test(async (test) => {
     86    await checkVideoBlack(false, localVideo, "canvas0");
     87    await waitForVideoSize(localVideo, 320, 240);
     88
     89    if (window.internals)
     90        window.internals.setCameraMediaStreamTrackOrientation(track, 90);
     91
     92    await checkVideoBlack(false, localVideo, "canvas1");
     93    await waitForVideoSize(localVideo, 240, 320);
     94}, "Track is enabled and rotated, local video should not be black and should change size");
    8095
    8196promise_test((test) => {
    … …  
    8398        window.internals.setCameraMediaStreamTrackOrientation(track, 90);
    8499
    85     return checkVideoBlack(false, "canvas1").then(() => {
    86         return waitForVideoSize(video, 240, 320);
     100    return checkVideoBlack(false, remoteVideo, "canvas2").then(() => {
     101        return waitForVideoSize(remoteVideo, 240, 320);
    87102    });
    88 }, "Track is enabled and rotated, video should not be black and should change size");
     103}, "Track is enabled and rotated, remote video should not be black and should change size");
    89104
    90105promise_test((test) => {
    … …  
    92107        window.internals.setCameraMediaStreamTrackOrientation(track, 180);
    93108
    94     return checkVideoBlack(false, "canvas1").then(() => {
    95         return waitForVideoSize(video, 320, 240);
     109    return checkVideoBlack(false, remoteVideo, "canvas3").then(() => {
     110        return waitForVideoSize(remoteVideo, 320, 240);
    96111    });
    97112}, "Track is enabled and rotated again, video should not be black and should change size");

  • trunk/Source/WebCore/ChangeLog

    r247164 r247165  
     12019-07-05  Youenn Fablet  <youenn@apple.com>
     2
     3        [iOS] Local capture MediaStreamTrack does not render in portrait mode
     4        https://bugs.webkit.org/show_bug.cgi?id=199519
     5        <rdar://problem/52689720>
     6
     7        Reviewed by Eric Carlson.
     8
     9        RealtimeVideoSource was badly computing its size in case of rotation.
     10        Whenever its underlying source is notifying of settings change,
     11        compute the size and transpose it only in left/right case.
     12
     13        Update mock video source to cover that case.
     14        Covered by updated test.
     15
     16        * platform/mediastream/RealtimeVideoSource.cpp:
     17        (WebCore::RealtimeVideoSource::sourceSettingsChanged):
     18        * platform/mediastream/mac/MockRealtimeVideoSourceMac.h:
     19        * platform/mediastream/mac/MockRealtimeVideoSourceMac.mm:
     20        (WebCore::MockRealtimeVideoSourceMac::updateSampleBuffer):
     21        * platform/mock/MockRealtimeVideoSource.cpp:
     22        (WebCore::MockRealtimeVideoSource::settings):
     23        (WebCore::MockRealtimeVideoSource::orientationChanged):
     24        (WebCore::MockRealtimeVideoSource::monitorOrientation):
     25        * platform/mock/MockRealtimeVideoSource.h:
     26
    1272019-07-05  Ryan Haddad  <ryanhaddad@apple.com>
    228

  • trunk/Source/WebCore/platform/mediastream/RealtimeVideoSource.cpp

    r246644 r247165  
    9393{
    9494    auto rotation = m_source->sampleRotation();
    95     if (rotation == MediaSample::VideoRotation::Left || rotation == MediaSample::VideoRotation::Right) {
    96         auto size = this->size();
     95    auto size = this->size();
     96    if (size.isEmpty())
     97        size = m_source->size();
     98    if (rotation == MediaSample::VideoRotation::Left || rotation == MediaSample::VideoRotation::Right)
    9799        size = size.transposedSize();
    98         m_currentSettings.setWidth(size.width());
    99         m_currentSettings.setHeight(size.height());
    100     }
     100    m_currentSettings.setWidth(size.width());
     101    m_currentSettings.setHeight(size.height());
    101102
    102103    forEachObserver([&](auto& observer) {

  • trunk/Source/WebCore/platform/mediastream/mac/MockRealtimeVideoSourceMac.h

    r246644 r247165  
    3434
    3535#include "MockRealtimeVideoSource.h"
    36 #include "OrientationNotifier.h"
    3736#include "PixelBufferConformerCV.h"
    3837
    … …  
    4645class ImageTransferSessionVT;
    4746
    48 class MockRealtimeVideoSourceMac final : public MockRealtimeVideoSource, private OrientationNotifier::Observer {
     47class MockRealtimeVideoSourceMac final : public MockRealtimeVideoSource {
    4948public:
    5049    virtual ~MockRealtimeVideoSourceMac() = default;
    … …  
    5857    bool canResizeVideoFrames() const final { return true; }
    5958
    60     void orientationChanged(int orientation) final;
    61     void monitorOrientation(OrientationNotifier&) final;
    62     MediaSample::VideoRotation sampleRotation() const final { return m_deviceOrientation; }
    63 
    64     MediaSample::VideoRotation m_deviceOrientation { MediaSample::VideoRotation::None };
    6559    std::unique_ptr<ImageTransferSessionVT> m_imageTransferSession;
    6660    IntSize m_presetSize;

  • trunk/Source/WebCore/platform/mediastream/mac/MockRealtimeVideoSourceMac.mm

    r246644 r247165  
    8686
    8787    auto sampleTime = MediaTime::createWithDouble((elapsedTime() + 100_ms).seconds());
    88     auto sampleBuffer = m_imageTransferSession->createMediaSample(imageBuffer->copyImage()->nativeImage().get(), sampleTime, size(), m_deviceOrientation);
     88    auto sampleBuffer = m_imageTransferSession->createMediaSample(imageBuffer->copyImage()->nativeImage().get(), sampleTime, size(), sampleRotation());
    8989    if (!sampleBuffer)
    9090        return;
    9191
    92     // We use m_deviceOrientation to emulate sensor orientation
    9392    dispatchMediaSampleToObservers(*sampleBuffer);
    94 }
    95 
    96 void MockRealtimeVideoSourceMac::orientationChanged(int orientation)
    97 {
    98     // FIXME: Do something with m_deviceOrientation. See bug 169822.
    99     switch (orientation) {
    100     case 0:
    101         m_deviceOrientation = MediaSample::VideoRotation::None;
    102         break;
    103     case 90:
    104         m_deviceOrientation = MediaSample::VideoRotation::Right;
    105         break;
    106     case -90:
    107         m_deviceOrientation = MediaSample::VideoRotation::Left;
    108         break;
    109     case 180:
    110         m_deviceOrientation = MediaSample::VideoRotation::UpsideDown;
    111         break;
    112     default:
    113         return;
    114     }
    115 }
    116 
    117 void MockRealtimeVideoSourceMac::monitorOrientation(OrientationNotifier& notifier)
    118 {
    119     notifier.addObserver(*this);
    120     orientationChanged(notifier.orientation());
    12193}
    12294

  • trunk/Source/WebCore/platform/mock/MockRealtimeVideoSource.cpp

    r246644 r247165  
    158158    }
    159159    settings.setFrameRate(frameRate());
    160     auto& size = this->size();
     160    auto size = this->size();
     161    if (mockCamera()) {
     162        if (m_deviceOrientation == MediaSample::VideoRotation::Left || m_deviceOrientation == MediaSample::VideoRotation::Right)
     163            size = size.transposedSize();
     164    }
    161165    settings.setWidth(size.width());
    162166    settings.setHeight(size.height());
    … …  
    473477}
    474478
     479void MockRealtimeVideoSource::orientationChanged(int orientation)
     480{
     481    auto deviceOrientation = m_deviceOrientation;
     482    switch (orientation) {
     483    case 0:
     484        m_deviceOrientation = MediaSample::VideoRotation::None;
     485        break;
     486    case 90:
     487        m_deviceOrientation = MediaSample::VideoRotation::Right;
     488        break;
     489    case -90:
     490        m_deviceOrientation = MediaSample::VideoRotation::Left;
     491        break;
     492    case 180:
     493        m_deviceOrientation = MediaSample::VideoRotation::UpsideDown;
     494        break;
     495    default:
     496        return;
     497    }
     498
     499    if (deviceOrientation == m_deviceOrientation)
     500        return;
     501
     502    notifySettingsDidChangeObservers({ RealtimeMediaSourceSettings::Flag::Width, RealtimeMediaSourceSettings::Flag::Height });
     503}
     504
     505void MockRealtimeVideoSource::monitorOrientation(OrientationNotifier& notifier)
     506{
     507    if (!mockCamera())
     508        return;
     509
     510    notifier.addObserver(*this);
     511    orientationChanged(notifier.orientation());
     512}
     513
    475514} // namespace WebCore
    476515

  • trunk/Source/WebCore/platform/mock/MockRealtimeVideoSource.h

    r246644 r247165  
    3737#include "ImageBuffer.h"
    3838#include "MockMediaDevice.h"
     39#include "OrientationNotifier.h"
    3940#include "RealtimeMediaSourceFactory.h"
    4041#include "RealtimeVideoCaptureSource.h"
    … …  
    4647class GraphicsContext;
    4748
    48 class MockRealtimeVideoSource : public RealtimeVideoCaptureSource {
     49class MockRealtimeVideoSource : public RealtimeVideoCaptureSource, private OrientationNotifier::Observer {
    4950public:
    5051
    … …  
    6162    Seconds elapsedTime();
    6263    void settingsDidChange(OptionSet<RealtimeMediaSourceSettings::Flag>) override;
     64    MediaSample::VideoRotation sampleRotation() const final { return m_deviceOrientation; }
    6365
    6466private:
    … …  
    7779    void generatePresets() final;
    7880
     81    // OrientationNotifier::Observer
     82    void orientationChanged(int orientation) final;
     83    void monitorOrientation(OrientationNotifier&) final;
     84
    7985    void drawAnimation(GraphicsContext&);
    8086    void drawText(GraphicsContext&);
    … …  
    8490    void startCaptureTimer();
    8591
    86     void delaySamples(Seconds) override;
     92    void delaySamples(Seconds) final;
    8793
    8894    bool mockCamera() const { return WTF::holds_alternative<MockCameraProperties>(m_device.properties); }
    … …  
    113119    MockMediaDevice m_device;
    114120    RefPtr<VideoPreset> m_preset;
     121    MediaSample::VideoRotation m_deviceOrientation { MediaSample::VideoRotation::None };
    115122};
    116123
Note: See TracChangeset for help on using the changeset viewer.