Changeset 249428 in webkit


Ignore:
Timestamp:
Sep 3, 2019 7:14:57 AM (5 years ago)
Author:
commit-queue@webkit.org
Message:

[GStreamer] Add support to copy YUV video textures into platform textures
​https://bugs.webkit.org/show_bug.cgi?id=200914

Patch by Chris Lord <Chris Lord> on 2019-09-03
Reviewed by Xabier Rodriguez-Calvar and Miguel Gomez.

Enable YUV (including planar and semi-planar) video texture to platform
texture copy in VideoTextureCopierGStreamer.

No new tests, not changing behavior.

  • platform/graphics/gstreamer/MediaPlayerPrivateGStreamerBase.cpp:

(WebCore::GstVideoFrameHolder::platformLayerBuffer):
(WebCore::MediaPlayerPrivateGStreamerBase::pushTextureToCompositor):
(WebCore::MediaPlayerPrivateGStreamerBase::copyVideoTextureToPlatformTexture):
(WebCore::MediaPlayerPrivateGStreamerBase::nativeImageForCurrentTime):

  • platform/graphics/gstreamer/VideoTextureCopierGStreamer.cpp:

(WebCore::VideoTextureCopierGStreamer::VideoTextureCopierGStreamer):
(WebCore::VideoTextureCopierGStreamer::copyVideoTextureToPlatformTexture):

  • platform/graphics/gstreamer/VideoTextureCopierGStreamer.h:
  • platform/graphics/texmap/TextureMapperPlatformLayerBuffer.h:

(WebCore::TextureMapperPlatformLayerBuffer::textureVariant):

Location:
trunk/Source/WebCore
Files:
5 edited

Legend:

Unmodified
Added
Removed

  • trunk/Source/WebCore/ChangeLog

    r249427 r249428  
     12019-09-03  Chris Lord  <clord@igalia.com>
     2
     3        [GStreamer] Add support to copy YUV video textures into platform textures
     4        https://bugs.webkit.org/show_bug.cgi?id=200914
     5
     6        Reviewed by Xabier Rodriguez-Calvar and Miguel Gomez.
     7
     8        Enable YUV (including planar and semi-planar) video texture to platform
     9        texture copy in VideoTextureCopierGStreamer.
     10
     11        No new tests, not changing behavior.
     12
     13        * platform/graphics/gstreamer/MediaPlayerPrivateGStreamerBase.cpp:
     14        (WebCore::GstVideoFrameHolder::platformLayerBuffer):
     15        (WebCore::MediaPlayerPrivateGStreamerBase::pushTextureToCompositor):
     16        (WebCore::MediaPlayerPrivateGStreamerBase::copyVideoTextureToPlatformTexture):
     17        (WebCore::MediaPlayerPrivateGStreamerBase::nativeImageForCurrentTime):
     18        * platform/graphics/gstreamer/VideoTextureCopierGStreamer.cpp:
     19        (WebCore::VideoTextureCopierGStreamer::VideoTextureCopierGStreamer):
     20        (WebCore::VideoTextureCopierGStreamer::copyVideoTextureToPlatformTexture):
     21        * platform/graphics/gstreamer/VideoTextureCopierGStreamer.h:
     22        * platform/graphics/texmap/TextureMapperPlatformLayerBuffer.h:
     23        (WebCore::TextureMapperPlatformLayerBuffer::textureVariant):
     24
    1252019-09-03  Zan Dobersek  <zdobersek@igalia.com>  and  Chris Lord  <clord@igalia.com>
    226

  • trunk/Source/WebCore/platform/graphics/gstreamer/MediaPlayerPrivateGStreamerBase.cpp

    r249427 r249428  
    217217    }
    218218
     219    std::unique_ptr<TextureMapperPlatformLayerBuffer> platformLayerBuffer()
     220    {
     221        if (!m_hasMappedTextures)
     222            return nullptr;
     223
     224        using Buffer = TextureMapperPlatformLayerBuffer;
     225
     226        if ((GST_VIDEO_INFO_IS_RGB(&m_videoFrame.info) && GST_VIDEO_INFO_N_PLANES(&m_videoFrame.info) == 1))
     227            return makeUnique<Buffer>(Buffer::TextureVariant { Buffer::RGBTexture { *static_cast<GLuint*>(m_videoFrame.data[0]) } }, m_size, m_flags, GraphicsContext3D::RGBA);
     228
     229        if (GST_VIDEO_INFO_IS_YUV(&m_videoFrame.info)) {
     230            if (GST_VIDEO_INFO_N_COMPONENTS(&m_videoFrame.info) < 3 || GST_VIDEO_INFO_N_PLANES(&m_videoFrame.info) > 3)
     231                return nullptr;
     232
     233            unsigned numberOfPlanes = GST_VIDEO_INFO_N_PLANES(&m_videoFrame.info);
     234            std::array<GLuint, 3> planes;
     235            std::array<unsigned, 3> yuvPlane;
     236            std::array<unsigned, 3> yuvPlaneOffset;
     237            for (unsigned i = 0; i < numberOfPlanes; ++i)
     238                planes[i] = *static_cast<GLuint*>(m_videoFrame.data[i]);
     239            for (unsigned i = 0; i < 3; ++i) {
     240                yuvPlane[i] = GST_VIDEO_INFO_COMP_PLANE(&m_videoFrame.info, i);
     241                yuvPlaneOffset[i] = GST_VIDEO_INFO_COMP_POFFSET(&m_videoFrame.info, i);
     242            }
     243
     244            std::array<GLfloat, 9> yuvToRgb;
     245            if (gst_video_colorimetry_matches(&GST_VIDEO_INFO_COLORIMETRY(&m_videoFrame.info), GST_VIDEO_COLORIMETRY_BT709)) {
     246                yuvToRgb = {
     247                    1.164f,  0.0f,    1.787f,
     248                    1.164f, -0.213f, -0.531f,
     249                    1.164f,  2.112f,  0.0f
     250                };
     251            } else {
     252                // Default to bt601. This is the same behaviour as GStreamer's glcolorconvert element.
     253                yuvToRgb = {
     254                    1.164f,  0.0f,    1.596f,
     255                    1.164f, -0.391f, -0.813f,
     256                    1.164f,  2.018f,  0.0f
     257                };
     258            }
     259
     260            return makeUnique<Buffer>( Buffer::TextureVariant { Buffer::YUVTexture { numberOfPlanes, planes, yuvPlane, yuvPlaneOffset, yuvToRgb } }, m_size, m_flags, GraphicsContext3D::RGBA);
     261        }
     262
     263        return nullptr;
     264    }
     265
    219266private:
    220267    GstBuffer* m_buffer;
    … …  
    733780            std::unique_ptr<GstVideoFrameHolder> frameHolder = makeUnique<GstVideoFrameHolder>(m_sample.get(), m_textureMapperFlags, !m_usingFallbackVideoSink);
    734781
    735             using Buffer = TextureMapperPlatformLayerBuffer;
    736             std::unique_ptr<Buffer> layerBuffer;
     782            std::unique_ptr<TextureMapperPlatformLayerBuffer> layerBuffer;
    737783            if (frameHolder->hasMappedTextures()) {
    738                 auto& videoFrame = frameHolder->videoFrame();
    739                 [&] {
    740                     if ((GST_VIDEO_INFO_IS_RGB(&videoFrame.info) && GST_VIDEO_INFO_N_PLANES(&videoFrame.info) == 1)) {
    741                         layerBuffer = makeUnique<Buffer>(
    742                             Buffer::TextureVariant { Buffer::RGBTexture { *static_cast<GLuint*>(videoFrame.data[0]) } },
    743                             frameHolder->size(), frameHolder->flags(), GraphicsContext3D::RGBA);
    744                         return;
    745                     }
    746 
    747                     if (GST_VIDEO_INFO_IS_YUV(&videoFrame.info)) {
    748                         if (GST_VIDEO_INFO_N_COMPONENTS(&videoFrame.info) < 3 || GST_VIDEO_INFO_N_PLANES(&videoFrame.info) > 3)
    749                             return;
    750 
    751                         unsigned numberOfPlanes = GST_VIDEO_INFO_N_PLANES(&videoFrame.info);
    752                         std::array<GLuint, 3> planes;
    753                         std::array<unsigned, 3> yuvPlane;
    754                         std::array<unsigned, 3> yuvPlaneOffset;
    755                         for (unsigned i = 0; i < numberOfPlanes; ++i)
    756                             planes[i] = *static_cast<GLuint*>(videoFrame.data[i]);
    757                         for (unsigned i = 0; i < 3; ++i) {
    758                             yuvPlane[i] = GST_VIDEO_INFO_COMP_PLANE(&videoFrame.info, i);
    759                             yuvPlaneOffset[i] = GST_VIDEO_INFO_COMP_POFFSET(&videoFrame.info, i);
    760                         }
    761 
    762                         std::array<GLfloat, 9> yuvToRgb;
    763                         if (gst_video_colorimetry_matches(&GST_VIDEO_INFO_COLORIMETRY(&videoFrame.info), GST_VIDEO_COLORIMETRY_BT709)) {
    764                             yuvToRgb = {
    765                                 1.164f,  0.0f,    1.787f,
    766                                 1.164f, -0.213f, -0.531f,
    767                                 1.164f,  2.112f,  0.0f
    768                             };
    769                         } else {
    770                             // Default to bt601. This is the same behaviour as GStreamer's glcolorconvert element.
    771                             yuvToRgb = {
    772                                 1.164f,  0.0f,    1.596f,
    773                                 1.164f, -0.391f, -0.813f,
    774                                 1.164f,  2.018f,  0.0f
    775                             };
    776                         }
    777 
    778                         layerBuffer = makeUnique<Buffer>(Buffer::TextureVariant { Buffer::YUVTexture { numberOfPlanes, planes, yuvPlane, yuvPlaneOffset, yuvToRgb } },
    779                             frameHolder->size(), frameHolder->flags(), GraphicsContext3D::RGBA);
    780                     }
    781                 }();
    782 
     784                layerBuffer = frameHolder->platformLayerBuffer();
    783785                if (!layerBuffer)
    784786                    return;
    785 
    786787                layerBuffer->setUnmanagedBufferDataHolder(WTFMove(frameHolder));
    787788            } else {
    … …  
    994995    std::unique_ptr<GstVideoFrameHolder> frameHolder = makeUnique<GstVideoFrameHolder>(m_sample.get(), m_textureMapperFlags, true);
    995996
    996     auto textureID = frameHolder->textureID();
    997     if (!textureID)
     997    std::unique_ptr<TextureMapperPlatformLayerBuffer> layerBuffer = frameHolder->platformLayerBuffer();
     998    if (!layerBuffer)
    998999        return false;
    9991000
    … …  
    10051006        m_videoTextureCopier = makeUnique<VideoTextureCopierGStreamer>(TEXTURE_COPIER_COLOR_CONVERT_FLAG);
    10061007
    1007     return m_videoTextureCopier->copyVideoTextureToPlatformTexture(textureID, size, outputTexture, outputTarget, level, internalFormat, format, type, flipY, m_videoSourceOrientation);
     1008    frameHolder->waitForCPUSync();
     1009
     1010    return m_videoTextureCopier->copyVideoTextureToPlatformTexture(*layerBuffer.get(), size, outputTexture, outputTarget, level, internalFormat, format, type, flipY, m_videoSourceOrientation);
    10081011}
    10091012
    … …  
    10211024    std::unique_ptr<GstVideoFrameHolder> frameHolder = makeUnique<GstVideoFrameHolder>(m_sample.get(), m_textureMapperFlags, true);
    10221025
    1023     auto textureID = frameHolder->textureID();
    1024     if (!textureID)
     1026    std::unique_ptr<TextureMapperPlatformLayerBuffer> layerBuffer = frameHolder->platformLayerBuffer();
     1027    if (!layerBuffer)
    10251028        return nullptr;
    10261029
    … …  
    10351038        m_videoTextureCopier = makeUnique<VideoTextureCopierGStreamer>(TEXTURE_COPIER_COLOR_CONVERT_FLAG);
    10361039
    1037     if (!m_videoTextureCopier->copyVideoTextureToPlatformTexture(textureID, size, 0, GraphicsContext3D::TEXTURE_2D, 0, GraphicsContext3D::RGBA, GraphicsContext3D::RGBA, GraphicsContext3D::UNSIGNED_BYTE, false, m_videoSourceOrientation))
     1040    frameHolder->waitForCPUSync();
     1041
     1042    if (!m_videoTextureCopier->copyVideoTextureToPlatformTexture(*layerBuffer.get(), size, 0, GraphicsContext3D::TEXTURE_2D, 0, GraphicsContext3D::RGBA, GraphicsContext3D::RGBA, GraphicsContext3D::UNSIGNED_BYTE, false, m_videoSourceOrientation))
    10381043        return nullptr;
    10391044

  • trunk/Source/WebCore/platform/graphics/gstreamer/VideoTextureCopierGStreamer.cpp

    r249427 r249428  
    3535    ASSERT(previousContext);
    3636    PlatformDisplay::sharedDisplayForCompositing().sharingGLContext()->makeContextCurrent();
    37 
    38     m_shaderProgram = TextureMapperShaderProgram::create(TextureMapperShaderProgram::TextureRGB);
    3937
    4038    glGenFramebuffers(1, &m_framebuffer);
    … …  
    138136}
    139137
    140 bool VideoTextureCopierGStreamer::copyVideoTextureToPlatformTexture(GLuint inputTexture, IntSize& frameSize, GLuint outputTexture, GLenum outputTarget, GLint level, GLenum internalFormat, GLenum format, GLenum type, bool flipY, ImageOrientation sourceOrientation)
    141 {
    142     if (!m_shaderProgram || !m_framebuffer || !m_vbo || frameSize.isEmpty())
     138bool VideoTextureCopierGStreamer::copyVideoTextureToPlatformTexture(TextureMapperPlatformLayerBuffer& inputTexture, IntSize& frameSize, GLuint outputTexture, GLenum outputTarget, GLint level, GLenum internalFormat, GLenum format, GLenum type, bool flipY, ImageOrientation sourceOrientation)
     139{
     140    if (!m_framebuffer || !m_vbo || frameSize.isEmpty())
    143141        return false;
    144142
    … …  
    158156    ASSERT(previousContext);
    159157    PlatformDisplay::sharedDisplayForCompositing().sharingGLContext()->makeContextCurrent();
     158
     159    // Determine what shader program to use and create it if necessary.
     160    using Buffer = TextureMapperPlatformLayerBuffer;
     161    TextureMapperShaderProgram::Options options;
     162    WTF::switchOn(inputTexture.textureVariant(),
     163        [&](const Buffer::RGBTexture&) { options = TextureMapperShaderProgram::TextureRGB; },
     164        [&](const Buffer::YUVTexture& texture) {
     165            switch (texture.numberOfPlanes) {
     166            case 1:
     167                ASSERT(texture.yuvPlane[0] == texture.yuvPlane[1] && texture.yuvPlane[1] == texture.yuvPlane[2]);
     168                ASSERT(texture.yuvPlaneOffset[0] == 2 && texture.yuvPlaneOffset[1] == 1 && !texture.yuvPlaneOffset[2]);
     169                options = TextureMapperShaderProgram::TexturePackedYUV;
     170                break;
     171            case 2:
     172                ASSERT(!texture.yuvPlaneOffset[0]);
     173                options = texture.yuvPlaneOffset[1] ? TextureMapperShaderProgram::TextureNV21 : TextureMapperShaderProgram::TextureNV12;
     174                break;
     175            case 3:
     176                ASSERT(!texture.yuvPlaneOffset[0] && !texture.yuvPlaneOffset[1] && !texture.yuvPlaneOffset[2]);
     177                options = TextureMapperShaderProgram::TextureYUV;
     178                break;
     179            }
     180        });
     181
     182    if (options != m_shaderOptions) {
     183        m_shaderProgram = TextureMapperShaderProgram::create(options);
     184        m_shaderOptions = options;
     185    }
     186    if (!m_shaderProgram) {
     187        previousContext->makeContextCurrent();
     188        return false;
     189    }
    160190
    161191    // Save previous bound framebuffer, texture and viewport.
    … …  
    182212    glFramebufferTexture2D(GL_FRAMEBUFFER, GL_COLOR_ATTACHMENT0, GL_TEXTURE_2D, outputTexture, 0);
    183213
    184     // Set proper wrap parameter to the source texture.
    185     glBindTexture(GL_TEXTURE_2D, inputTexture);
    186     glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
    187     glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
    188     glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
    189 
    190214    // Set the viewport.
    191215    glViewport(0, 0, m_size.width(), m_size.height());
    … …  
    193217    // Set program parameters.
    194218    glUseProgram(m_shaderProgram->programID());
    195     glUniform1i(m_shaderProgram->samplerLocation(), 0);
     219
     220    WTF::switchOn(inputTexture.textureVariant(),
     221        [&](const Buffer::RGBTexture& texture) {
     222            glUniform1i(m_shaderProgram->samplerLocation(), 0);
     223
     224            glActiveTexture(GL_TEXTURE0);
     225            glBindTexture(GL_TEXTURE_2D, texture.id);
     226            glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
     227            glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
     228            glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
     229        },
     230        [&](const Buffer::YUVTexture& texture) {
     231            switch (texture.numberOfPlanes) {
     232            case 1:
     233                glUniform1i(m_shaderProgram->samplerLocation(), texture.yuvPlane[0]);
     234                break;
     235            case 2:
     236                glUniform1i(m_shaderProgram->samplerYLocation(), texture.yuvPlane[0]);
     237                glUniform1i(m_shaderProgram->samplerULocation(), texture.yuvPlane[1]);
     238                break;
     239            case 3:
     240                glUniform1i(m_shaderProgram->samplerYLocation(), texture.yuvPlane[0]);
     241                glUniform1i(m_shaderProgram->samplerULocation(), texture.yuvPlane[1]);
     242                glUniform1i(m_shaderProgram->samplerVLocation(), texture.yuvPlane[2]);
     243                break;
     244            }
     245            glUniformMatrix3fv(m_shaderProgram->yuvToRgbLocation(), 1, GL_FALSE, static_cast<const GLfloat *>(&texture.yuvToRgbMatrix[0]));
     246
     247            for (int i = texture.numberOfPlanes - 1; i >= 0; --i) {
     248                glActiveTexture(GL_TEXTURE0 + i);
     249                glBindTexture(GL_TEXTURE_2D, texture.planes[i]);
     250                glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
     251                glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
     252                glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
     253            }
     254        });
     255
    196256    m_shaderProgram->setMatrix(m_shaderProgram->modelViewMatrixLocation(), m_modelViewMatrix);
    197257    m_shaderProgram->setMatrix(m_shaderProgram->projectionMatrixLocation(), m_projectionMatrix);

  • trunk/Source/WebCore/platform/graphics/gstreamer/VideoTextureCopierGStreamer.h

    r248946 r249428  
    2525#include "ImageOrientation.h"
    2626#include "TextureMapperGLHeaders.h"
     27#include "TextureMapperPlatformLayerBuffer.h"
    2728#include "TransformationMatrix.h"
    2829#include <wtf/RefPtr.h>
    … …  
    4546    ~VideoTextureCopierGStreamer();
    4647
    47     bool copyVideoTextureToPlatformTexture(GLuint inputTexture, IntSize& frameSize, GLuint outputTexture, GLenum outputTarget, GLint level, GLenum internalFormat, GLenum format, GLenum type, bool flipY, ImageOrientation sourceOrientation);
     48    bool copyVideoTextureToPlatformTexture(TextureMapperPlatformLayerBuffer& inputTexture, IntSize& frameSize, GLuint outputTexture, GLenum outputTarget, GLint level, GLenum internalFormat, GLenum format, GLenum type, bool flipY, ImageOrientation sourceOrientation);
    4849    void updateColorConversionMatrix(ColorConversion);
    4950    void updateTextureSpaceMatrix();
    … …  
    5354private:
    5455    RefPtr<TextureMapperShaderProgram> m_shaderProgram;
     56    unsigned m_shaderOptions { 0 };
    5557    GLuint m_framebuffer { 0 };
    5658    GLuint m_vbo { 0 };

  • trunk/Source/WebCore/platform/graphics/texmap/TextureMapperPlatformLayerBuffer.h

    r249427 r249428  
    9393    void setHolePunchClient(std::unique_ptr<HolePunchClient>&& client) { m_holePunchClient = WTFMove(client); }
    9494
     95    const TextureVariant& textureVariant() { return m_variant; }
     96
    9597private:
    9698
Note: See TracChangeset for help on using the changeset viewer.