Changeset 223280 in webkit
- Timestamp:
- Oct 13, 2017 9:32:38 AM (6 years ago)
- Location:
- trunk/Source/WebCore
- Files:
-
- 11 edited
Legend:
- Unmodified
- Added
- Removed
-
trunk/Source/WebCore/ChangeLog
r223279 r223280 1 2017-10-13 Jer Noble <jer.noble@apple.com> 2 3 Performance: do pixel conformance and texturing in a single step. 4 https://bugs.webkit.org/show_bug.cgi?id=178219 5 <rdar://problem/34937237> 6 7 Reviewed by Dean Jackson. 8 9 No new tests; performance improvements should have no behavior change. 10 11 Rather than asking the VTDecompressionSession to conform the output CVPixelBuffer into a 12 pixel format compatible with OpenGL (& ES), don't constrain the output at all, and only do a 13 conformance step if the output is not already compatible with OpenGL. This eliminates one 14 copy (in hardware) operation. 15 16 Move the TextureCacheCV object into VideoTextureCopierCV; it will be conditionally used to 17 create the texture if the pixel buffer is compatible. 18 19 Refactor copyVideoTextureToPlatformTexture(CVOpenGLTextureRef) in VideoTextureCopierCV. The 20 new entry point, copyImageToPlatformTexture(), will attempt to use the texture cache first, 21 and call a new common copyVideoTextureToPlatformTexture(Platform3DObject) with the result. 22 23 The new copyImageToPlatformTexture() will pull planar YUV frames into two textures, and combine 24 the two with a color transfer function when drawing to the output texture. 25 26 * platform/graphics/GraphicsContext3D.h: 27 * platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaSourceAVFObjC.mm: 28 (WebCore::MediaPlayerPrivateMediaSourceAVFObjC::copyVideoTextureToPlatformTexture): 29 * platform/graphics/cocoa/GraphicsContext3DCocoa.mm: 30 (WebCore::GraphicsContext3D::texImageIOSurface2D): 31 * platform/graphics/cocoa/WebCoreDecompressionSession.mm: 32 (WebCore::WebCoreDecompressionSession::ensureDecompressionSessionForSample): 33 * platform/graphics/cv/TextureCacheCV.h: 34 * platform/graphics/cv/TextureCacheCV.mm: 35 (WebCore::TextureCacheCV::textureFromImage): 36 * platform/graphics/cv/VideoTextureCopierCV.cpp: 37 (WebCore::pixelRangeFromPixelFormat): 38 (WebCore::transferFunctionFromString): 39 (WebCore::YCbCrToRGBMatrixForRangeAndTransferFunction): 40 (WebCore::VideoTextureCopierCV::~VideoTextureCopierCV): 41 (WebCore::VideoTextureCopierCV::initializeUVContextObjects): 42 (WebCore::VideoTextureCopierCV::copyImageToPlatformTexture): 43 (WebCore::VideoTextureCopierCV::copyVideoTextureToPlatformTexture): 44 * platform/graphics/cv/VideoTextureCopierCV.h: 45 1 46 2017-10-13 Romain Bellessort <romain.bellessort@crf.canon.fr> 2 47 -
trunk/Source/WebCore/platform/cocoa/CoreVideoSoftLink.cpp
r222197 r223280 53 53 SOFT_LINK_CONSTANT_FOR_SOURCE(WebCore, CoreVideo, kCVPixelBufferIOSurfacePropertiesKey, CFStringRef) 54 54 SOFT_LINK_CONSTANT_FOR_SOURCE(WebCore, CoreVideo, kCVPixelBufferPoolMinimumBufferCountKey, CFStringRef) 55 SOFT_LINK_CONSTANT_FOR_SOURCE(WebCore, CoreVideo, kCVImageBufferYCbCrMatrixKey, CFStringRef) 56 SOFT_LINK_CONSTANT_FOR_SOURCE(WebCore, CoreVideo, kCVImageBufferYCbCrMatrix_ITU_R_709_2, CFStringRef) 57 SOFT_LINK_CONSTANT_FOR_SOURCE(WebCore, CoreVideo, kCVImageBufferYCbCrMatrix_ITU_R_601_4, CFStringRef) 58 SOFT_LINK_CONSTANT_FOR_SOURCE(WebCore, CoreVideo, kCVImageBufferYCbCrMatrix_SMPTE_240M_1995, CFStringRef) 59 SOFT_LINK_CONSTANT_FOR_SOURCE(WebCore, CoreVideo, kCVImageBufferYCbCrMatrix_DCI_P3, CFStringRef) 60 SOFT_LINK_CONSTANT_FOR_SOURCE(WebCore, CoreVideo, kCVImageBufferYCbCrMatrix_P3_D65, CFStringRef) 61 SOFT_LINK_CONSTANT_FOR_SOURCE(WebCore, CoreVideo, kCVImageBufferYCbCrMatrix_ITU_R_2020, CFStringRef) 55 62 56 63 #if PLATFORM(IOS) -
trunk/Source/WebCore/platform/cocoa/CoreVideoSoftLink.h
r222197 r223280 71 71 SOFT_LINK_CONSTANT_FOR_HEADER(WebCore, CoreVideo, kCVPixelBufferPoolMinimumBufferCountKey, CFStringRef) 72 72 #define kCVPixelBufferPoolMinimumBufferCountKey get_CoreVideo_kCVPixelBufferPoolMinimumBufferCountKey() 73 SOFT_LINK_CONSTANT_FOR_HEADER(WebCore, CoreVideo, kCVImageBufferYCbCrMatrixKey, CFStringRef) 74 #define kCVImageBufferYCbCrMatrixKey get_CoreVideo_kCVImageBufferYCbCrMatrixKey() 75 SOFT_LINK_CONSTANT_FOR_HEADER(WebCore, CoreVideo, kCVImageBufferYCbCrMatrix_ITU_R_709_2, CFStringRef) 76 #define kCVImageBufferYCbCrMatrix_ITU_R_709_2 get_CoreVideo_kCVImageBufferYCbCrMatrix_ITU_R_709_2() 77 SOFT_LINK_CONSTANT_FOR_HEADER(WebCore, CoreVideo, kCVImageBufferYCbCrMatrix_ITU_R_601_4, CFStringRef) 78 #define kCVImageBufferYCbCrMatrix_ITU_R_601_4 get_CoreVideo_kCVImageBufferYCbCrMatrix_ITU_R_601_4() 79 SOFT_LINK_CONSTANT_FOR_HEADER(WebCore, CoreVideo, kCVImageBufferYCbCrMatrix_SMPTE_240M_1995, CFStringRef) 80 #define kCVImageBufferYCbCrMatrix_SMPTE_240M_1995 get_CoreVideo_kCVImageBufferYCbCrMatrix_SMPTE_240M_1995() 81 SOFT_LINK_CONSTANT_FOR_HEADER(WebCore, CoreVideo, kCVImageBufferYCbCrMatrix_DCI_P3, CFStringRef) 82 #define kCVImageBufferYCbCrMatrix_DCI_P3 get_CoreVideo_kCVImageBufferYCbCrMatrix_DCI_P3() 83 SOFT_LINK_CONSTANT_FOR_HEADER(WebCore, CoreVideo, kCVImageBufferYCbCrMatrix_P3_D65, CFStringRef) 84 #define kCVImageBufferYCbCrMatrix_P3_D65 get_CoreVideo_kCVImageBufferYCbCrMatrix_P3_D65() 85 SOFT_LINK_CONSTANT_FOR_HEADER(WebCore, CoreVideo, kCVImageBufferYCbCrMatrix_ITU_R_2020, CFStringRef) 86 #define kCVImageBufferYCbCrMatrix_ITU_R_2020 get_CoreVideo_kCVImageBufferYCbCrMatrix_ITU_R_2020() 73 87 74 88 #if PLATFORM(IOS) -
trunk/Source/WebCore/platform/graphics/GraphicsContext3D.h
r222961 r223280 60 60 OBJC_CLASS CALayer; 61 61 OBJC_CLASS WebGLLayer; 62 typedef struct __IOSurface* IOSurfaceRef; 62 63 #elif PLATFORM(GTK) || PLATFORM(WIN_CAIRO) || PLATFORM(WPE) 63 64 typedef unsigned int GLuint; … … 1150 1151 #if PLATFORM(COCOA) 1151 1152 void endPaint(); 1153 bool texImageIOSurface2D(GC3Denum target, GC3Denum internalFormat, GC3Dsizei width, GC3Dsizei height, GC3Denum format, GC3Denum type, IOSurfaceRef, GC3Duint plane); 1152 1154 #endif 1153 1155 -
trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaPlayerPrivateMediaSourceAVFObjC.mm
r222995 r223280 609 609 if (!m_lastPixelBuffer) 610 610 return false; 611 612 if (!m_textureCache) {613 m_textureCache = TextureCacheCV::create(*context);614 if (!m_textureCache)615 return false;616 }617 618 m_lastTexture = m_textureCache->textureFromImage(m_lastPixelBuffer.get(), outputTarget, level, internalFormat, format, type);619 611 } 620 612 … … 625 617 m_videoTextureCopier = std::make_unique<VideoTextureCopierCV>(*context); 626 618 627 return m_videoTextureCopier->copy VideoTextureToPlatformTexture(m_lastTexture.get(), width, height, outputTexture, outputTarget, level, internalFormat, format, type, premultiplyAlpha, flipY);619 return m_videoTextureCopier->copyImageToPlatformTexture(m_lastPixelBuffer.get(), width, height, outputTexture, outputTarget, level, internalFormat, format, type, premultiplyAlpha, flipY); 628 620 } 629 621 -
trunk/Source/WebCore/platform/graphics/cocoa/GraphicsContext3DCocoa.mm
r222961 r223280 51 51 #import <OpenGLES/EAGL.h> 52 52 #import <OpenGLES/EAGLDrawable.h> 53 #import <OpenGLES/EAGLIOSurface.h> 53 54 #import <OpenGLES/ES2/glext.h> 54 55 #import <QuartzCore/QuartzCore.h> … … 673 674 } 674 675 676 bool GraphicsContext3D::texImageIOSurface2D(GC3Denum target, GC3Denum internalFormat, GC3Dsizei width, GC3Dsizei height, GC3Denum format, GC3Denum type, IOSurfaceRef surface, GC3Duint plane) 677 { 678 #if PLATFORM(MAC) 679 return kCGLNoError == CGLTexImageIOSurface2D(platformGraphicsContext3D(), target, internalFormat, width, height, format, type, surface, plane); 680 #elif PLATFORM(IOS) && !PLATFORM(IOS_SIMULATOR) 681 return [platformGraphicsContext3D() texImageIOSurface:surface target:target internalFormat:internalFormat width:width height:height format:format type:type plane:plane]; 682 #else 683 return false; 684 #endif 685 } 686 675 687 #if PLATFORM(MAC) 676 688 void GraphicsContext3D::allocateIOSurfaceBackingStore(IntSize size) -
trunk/Source/WebCore/platform/graphics/cocoa/WebCoreDecompressionSession.mm
r223212 r223280 220 220 NSDictionary *attributes; 221 221 if (m_mode == OpenGL) { 222 #if PLATFORM(IOS) 223 attributes = @{(NSString *)kCVPixelBufferIOSurfaceOpenGLESFBOCompatibilityKey: @YES}; 224 #else 225 attributes = @{(NSString *)kCVPixelBufferIOSurfaceOpenGLFBOCompatibilityKey: @YES}; 226 #endif 222 attributes = nil; 227 223 } else { 228 224 ASSERT(m_mode == RGB); -
trunk/Source/WebCore/platform/graphics/cv/TextureCacheCV.h
r197375 r223280 32 32 #include <wtf/WeakPtr.h> 33 33 34 typedef struct __CVBuffer* CVImageBufferRef; 34 typedef struct __CVBuffer* CVImageBufferRef; 35 typedef CVImageBufferRef CVPixelBufferRef; 35 36 typedef CVImageBufferRef CVOpenGLTextureRef; 36 37 typedef CVImageBufferRef CVOpenGLESTextureRef; … … 56 57 TextureCacheCV(GraphicsContext3D&, RetainPtr<TextureCacheType>&&); 57 58 58 RetainPtr<TextureType> textureFromImage(CV ImageBufferRef, GC3Denum outputTarget, GC3Dint level, GC3Denum internalFormat, GC3Denum format, GC3Denum type);59 RetainPtr<TextureType> textureFromImage(CVPixelBufferRef, GC3Denum outputTarget, GC3Dint level, GC3Denum internalFormat, GC3Denum format, GC3Denum type); 59 60 GraphicsContext3D& context() { return m_context.get(); } 60 61 -
trunk/Source/WebCore/platform/graphics/cv/TextureCacheCV.mm
r222422 r223280 54 54 } 55 55 56 RetainPtr<TextureCacheCV::TextureType> TextureCacheCV::textureFromImage(CV ImageBufferRef image, GC3Denum outputTarget, GC3Dint level, GC3Denum internalFormat, GC3Denum format, GC3Denum type)56 RetainPtr<TextureCacheCV::TextureType> TextureCacheCV::textureFromImage(CVPixelBufferRef image, GC3Denum outputTarget, GC3Dint level, GC3Denum internalFormat, GC3Denum format, GC3Denum type) 57 57 { 58 58 TextureType bareVideoTexture = nullptr; -
trunk/Source/WebCore/platform/graphics/cv/VideoTextureCopierCV.cpp
r222198 r223280 27 27 #include "VideoTextureCopierCV.h" 28 28 29 #include "FourCC.h" 29 30 #include "Logging.h" 31 #include "TextureCacheCV.h" 30 32 #include <wtf/NeverDestroyed.h> 31 33 #include <wtf/text/StringBuilder.h> … … 38 40 39 41 namespace WebCore { 42 43 enum class PixelRange { 44 Unknown, 45 Video, 46 Full, 47 }; 48 49 enum class TransferFunction { 50 Unknown, 51 kITU_R_709_2, 52 kITU_R_601_4, 53 kSMPTE_240M_1995, 54 kDCI_P3, 55 kP3_D65, 56 kITU_R_2020, 57 }; 58 59 #if PLATFORM(MAC) && __MAC_OS_X_VERSION_MIN_REQUIRED < 101300 60 enum { 61 kCVPixelFormatType_ARGB2101010LEPacked = 'l10r', 62 kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange = 'x420', 63 kCVPixelFormatType_422YpCbCr10BiPlanarVideoRange = 'x422', 64 kCVPixelFormatType_444YpCbCr10BiPlanarVideoRange = 'x444', 65 kCVPixelFormatType_420YpCbCr10BiPlanarFullRange = 'xf20', 66 kCVPixelFormatType_422YpCbCr10BiPlanarFullRange = 'xf22', 67 kCVPixelFormatType_444YpCbCr10BiPlanarFullRange = 'xf44', 68 }; 69 #endif 70 71 static PixelRange pixelRangeFromPixelFormat(OSType pixelFormat) 72 { 73 switch (pixelFormat) { 74 case kCVPixelFormatType_4444AYpCbCr8: 75 case kCVPixelFormatType_4444AYpCbCr16: 76 case kCVPixelFormatType_422YpCbCr_4A_8BiPlanar: 77 case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange: 78 case kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange: 79 case kCVPixelFormatType_422YpCbCr10BiPlanarVideoRange: 80 case kCVPixelFormatType_444YpCbCr10BiPlanarVideoRange: 81 return PixelRange::Video; 82 case kCVPixelFormatType_420YpCbCr8PlanarFullRange: 83 case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange: 84 case kCVPixelFormatType_422YpCbCr8FullRange: 85 case kCVPixelFormatType_ARGB2101010LEPacked: 86 case kCVPixelFormatType_420YpCbCr10BiPlanarFullRange: 87 case kCVPixelFormatType_422YpCbCr10BiPlanarFullRange: 88 case kCVPixelFormatType_444YpCbCr10BiPlanarFullRange: 89 return PixelRange::Full; 90 default: 91 return PixelRange::Unknown; 92 } 93 } 94 95 static TransferFunction transferFunctionFromString(CFStringRef string) 96 { 97 if (string == kCVImageBufferYCbCrMatrix_ITU_R_709_2) 98 return TransferFunction::kITU_R_709_2; 99 if (string == kCVImageBufferYCbCrMatrix_ITU_R_601_4) 100 return TransferFunction::kITU_R_601_4; 101 if (string == kCVImageBufferYCbCrMatrix_SMPTE_240M_1995) 102 return TransferFunction::kSMPTE_240M_1995; 103 if (string == kCVImageBufferYCbCrMatrix_DCI_P3) 104 return TransferFunction::kDCI_P3; 105 if (string == kCVImageBufferYCbCrMatrix_P3_D65) 106 return TransferFunction::kP3_D65; 107 if (string == kCVImageBufferYCbCrMatrix_ITU_R_2020) 108 return TransferFunction::kITU_R_2020; 109 return TransferFunction::Unknown; 110 } 111 112 static const Vector<GLfloat> YCbCrToRGBMatrixForRangeAndTransferFunction(PixelRange range, TransferFunction transferFunction) 113 { 114 using MapKey = std::pair<PixelRange, TransferFunction>; 115 using MatrixMap = std::map<MapKey, Vector<GLfloat>>; 116 117 static NeverDestroyed<MatrixMap> matrices; 118 static dispatch_once_t onceToken; 119 dispatch_once(&onceToken, ^{ 120 // Matrices are derived from the components in the ITU R.601 rev 4 specification 121 // https://www.itu.int/rec/R-REC-BT.601 122 matrices.get().emplace(MapKey(PixelRange::Video, TransferFunction::kITU_R_601_4), Vector<GLfloat>({ 123 1.164383562f, 0.0f, 1.596026786f, 124 1.164383562f, -0.3917622901f, -0.8129676472f, 125 1.164383562f, 2.017232143f, 0.0f, 126 })); 127 matrices.get().emplace(MapKey({PixelRange::Full, TransferFunction::kITU_R_601_4}), Vector<GLfloat>({ 128 1.000000000f, 0.0f, 1.4075196850f, 129 1.000000000f, -0.3454911535f, -0.7169478464f, 130 1.000000000f, 1.7789763780f, 0.0f, 131 })); 132 // Matrices are derived from the components in the ITU R.709 rev 2 specification 133 // https://www.itu.int/rec/R-REC-BT.709-2-199510-S 134 matrices.get().emplace(MapKey({PixelRange::Video, TransferFunction::kITU_R_709_2}), Vector<GLfloat>({ 135 1.164383562f, 0.0f, 1.792741071f, 136 1.164383562f, -0.2132486143f, -0.5329093286f, 137 1.164383562f, 2.112401786f, 0.0f, 138 })); 139 matrices.get().emplace(MapKey({PixelRange::Full, TransferFunction::kITU_R_709_2}), Vector<GLfloat>({ 140 1.000000000f, 0.0f, 1.5810000000f, 141 1.000000000f, -0.1880617701f, -0.4699672819f, 142 1.000000000f, 1.8629055118f, 0.0f, 143 })); 144 }); 145 146 // We should never be asked to handle a Pixel Format whose range value is unknown. 147 ASSERT(range != PixelRange::Unknown); 148 if (range == PixelRange::Unknown) 149 range = PixelRange::Full; 150 151 auto iterator = matrices.get().find({range, transferFunction}); 152 153 // Assume unknown transfer functions are r.601: 154 if (iterator == matrices.get().end()) 155 iterator = matrices.get().find({range, TransferFunction::kITU_R_601_4}); 156 157 ASSERT(iterator != matrices.get().end()); 158 return iterator->second; 159 } 40 160 41 161 VideoTextureCopierCV::VideoTextureCopierCV(GraphicsContext3D& context) … … 51 171 if (m_program) 52 172 m_context->deleteProgram(m_program); 173 if (m_yuvVertexBuffer) 174 m_context->deleteProgram(m_yuvVertexBuffer); 175 if (m_yuvProgram) 176 m_context->deleteProgram(m_yuvProgram); 53 177 m_context->deleteFramebuffer(m_framebuffer); 54 178 } … … 259 383 } 260 384 261 bool VideoTextureCopierCV::copyVideoTextureToPlatformTexture(TextureType inputVideoTexture, size_t width, size_t height, Platform3DObject outputTexture, GC3Denum outputTarget, GC3Dint level, GC3Denum internalFormat, GC3Denum format, GC3Denum type, bool premultiplyAlpha, bool flipY) 262 { 263 if (!inputVideoTexture) 385 bool VideoTextureCopierCV::initializeUVContextObjects() 386 { 387 String vertexShaderSource = ASCIILiteral( 388 "attribute vec2 a_position;\n" 389 "uniform vec2 u_yTextureSize;\n" 390 "uniform vec2 u_uvTextureSize;\n" 391 "uniform int u_flipY;\n" 392 "varying vec2 v_yTextureCoordinate;\n" 393 "varying vec2 v_uvTextureCoordinate;\n" 394 "void main() {\n" 395 " gl_Position = vec4(a_position, 0, 1.0);\n" 396 " if (u_flipY == 1) {\n" 397 " gl_Position.y = -gl_Position.y;\n" 398 " }\n" 399 " vec2 normalizedPosition = a_position * .5 + .5;\n" 400 #if PLATFORM(IOS) 401 " v_yTextureCoordinate = normalizedPosition;\n" 402 " v_uvTextureCoordinate = normalizedPosition;\n" 403 #else 404 " v_yTextureCoordinate = normalizedPosition * u_yTextureSize;\n" 405 " v_uvTextureCoordinate = normalizedPosition * u_uvTextureSize;\n" 406 #endif 407 "}\n" 408 ); 409 410 Platform3DObject vertexShader = m_context->createShader(GraphicsContext3D::VERTEX_SHADER); 411 m_context->shaderSource(vertexShader, vertexShaderSource); 412 m_context->compileShaderDirect(vertexShader); 413 414 GC3Dint status = 0; 415 m_context->getShaderiv(vertexShader, GraphicsContext3D::COMPILE_STATUS, &status); 416 if (!status) { 417 LOG(WebGL, "VideoTextureCopierCV::initializeUVContextObjects(%p) - Vertex shader failed to compile.", this); 418 m_context->deleteShader(vertexShader); 419 return false; 420 } 421 422 String fragmentShaderSource = ASCIILiteral( 423 #if PLATFORM(IOS) 424 "precision mediump float;\n" 425 "#define SAMPLERTYPE sampler2D\n" 426 "#define TEXTUREFUNC texture2D\n" 427 #else 428 "#define SAMPLERTYPE sampler2DRect\n" 429 "#define TEXTUREFUNC texture2DRect\n" 430 #endif 431 "uniform SAMPLERTYPE u_yTexture;\n" 432 "uniform SAMPLERTYPE u_uvTexture;\n" 433 "uniform mat3 u_colorMatrix;\n" 434 "varying vec2 v_yTextureCoordinate;\n" 435 "varying vec2 v_uvTextureCoordinate;\n" 436 "void main() {\n" 437 " vec3 yuv;\n" 438 " yuv.r = TEXTUREFUNC(u_yTexture, v_yTextureCoordinate).r;\n" 439 " yuv.gb = TEXTUREFUNC(u_uvTexture, v_uvTextureCoordinate).rg - vec2(0.5, 0.5);\n" 440 " gl_FragColor = vec4(yuv * u_colorMatrix, 1);\n" 441 "}\n" 442 ); 443 444 Platform3DObject fragmentShader = m_context->createShader(GraphicsContext3D::FRAGMENT_SHADER); 445 m_context->shaderSource(fragmentShader, fragmentShaderSource); 446 m_context->compileShaderDirect(fragmentShader); 447 448 m_context->getShaderiv(fragmentShader, GraphicsContext3D::COMPILE_STATUS, &status); 449 if (!status) { 450 LOG(WebGL, "VideoTextureCopierCV::initializeUVContextObjects(%p) - Fragment shader failed to compile.", this); 451 m_context->deleteShader(vertexShader); 452 m_context->deleteShader(fragmentShader); 453 return false; 454 } 455 456 m_yuvProgram = m_context->createProgram(); 457 m_context->attachShader(m_yuvProgram, vertexShader); 458 m_context->attachShader(m_yuvProgram, fragmentShader); 459 m_context->linkProgram(m_yuvProgram); 460 461 m_context->getProgramiv(m_yuvProgram, GraphicsContext3D::LINK_STATUS, &status); 462 if (!status) { 463 LOG(WebGL, "VideoTextureCopierCV::initializeUVContextObjects(%p) - Program failed to link.", this); 464 m_context->deleteShader(vertexShader); 465 m_context->deleteShader(fragmentShader); 466 m_context->deleteProgram(m_yuvProgram); 467 m_yuvProgram = 0; 468 return false; 469 } 470 471 m_yTextureUniformLocation = m_context->getUniformLocation(m_yuvProgram, ASCIILiteral("u_yTexture")); 472 m_uvTextureUniformLocation = m_context->getUniformLocation(m_yuvProgram, ASCIILiteral("u_uvTexture")); 473 m_colorMatrixUniformLocation = m_context->getUniformLocation(m_yuvProgram, ASCIILiteral("u_colorMatrix")); 474 m_yuvFlipYUniformLocation = m_context->getUniformLocation(m_yuvProgram, ASCIILiteral("u_flipY")); 475 m_yTextureSizeUniformLocation = m_context->getUniformLocation(m_yuvProgram, ASCIILiteral("u_yTextureSize")); 476 m_uvTextureSizeUniformLocation = m_context->getUniformLocation(m_yuvProgram, ASCIILiteral("u_uvTextureSize")); 477 m_yuvPositionAttributeLocation = m_context->getAttribLocationDirect(m_yuvProgram, ASCIILiteral("a_position")); 478 479 m_context->detachShader(m_yuvProgram, vertexShader); 480 m_context->detachShader(m_yuvProgram, fragmentShader); 481 m_context->deleteShader(vertexShader); 482 m_context->deleteShader(fragmentShader); 483 484 m_yuvVertexBuffer = m_context->createBuffer(); 485 float vertices[12] = { -1, -1, 1, -1, 1, 1, 1, 1, -1, 1, -1, -1 }; 486 487 m_context->bindBuffer(GraphicsContext3D::ARRAY_BUFFER, m_yuvVertexBuffer); 488 m_context->bufferData(GraphicsContext3D::ARRAY_BUFFER, sizeof(vertices), vertices, GraphicsContext3D::STATIC_DRAW); 489 490 return true; 491 } 492 493 bool VideoTextureCopierCV::copyImageToPlatformTexture(CVPixelBufferRef image, size_t width, size_t height, Platform3DObject outputTexture, GC3Denum outputTarget, GC3Dint level, GC3Denum internalFormat, GC3Denum format, GC3Denum type, bool premultiplyAlpha, bool flipY) 494 { 495 if (!m_textureCache) { 496 m_textureCache = TextureCacheCV::create(m_context); 497 if (!m_textureCache) 498 return false; 499 } 500 501 if (auto texture = m_textureCache->textureFromImage(image, outputTarget, level, internalFormat, format, type)) 502 return copyVideoTextureToPlatformTexture(texture.get(), width, height, outputTexture, outputTarget, level, internalFormat, format, type, premultiplyAlpha, flipY); 503 504 // FIXME: This currently only supports '420v' and '420f' pixel formats. Investigate supporting more pixel formats. 505 OSType pixelFormat = CVPixelBufferGetPixelFormatType(image); 506 if (pixelFormat != kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange && pixelFormat != kCVPixelFormatType_420YpCbCr8BiPlanarFullRange) { 507 LOG(WebGL, "VideoTextureCopierCV::copyVideoTextureToPlatformTexture(%p) - Asked to copy an unsupported pixel format ('%s').", this, FourCC(pixelFormat).toString().utf8().data()); 508 return false; 509 } 510 511 IOSurfaceRef surface = CVPixelBufferGetIOSurface(image); 512 if (!surface) 264 513 return false; 265 514 266 515 GC3DStateSaver stateSaver(m_context.get()); 267 516 268 if (!m_ program) {269 if (!initialize ContextObjects()) {517 if (!m_yuvProgram) { 518 if (!initializeUVContextObjects()) { 270 519 LOG(WebGL, "VideoTextureCopierCV::copyVideoTextureToPlatformTexture(%p) - Unable to initialize OpenGL context objects.", this); 271 520 return false; … … 273 522 } 274 523 275 stateSaver.saveVertexAttribState(m_positionAttributeLocation); 524 stateSaver.saveVertexAttribState(m_yuvPositionAttributeLocation); 525 526 m_context->bindFramebuffer(GraphicsContext3D::FRAMEBUFFER, m_framebuffer); 527 528 // Allocate memory for the output texture. 529 m_context->bindTexture(GraphicsContext3D::TEXTURE_2D, outputTexture); 530 m_context->texParameteri(GraphicsContext3D::TEXTURE_2D, GraphicsContext3D::TEXTURE_MAG_FILTER, GraphicsContext3D::LINEAR); 531 m_context->texParameteri(GraphicsContext3D::TEXTURE_2D, GraphicsContext3D::TEXTURE_MIN_FILTER, GraphicsContext3D::LINEAR); 532 m_context->texParameteri(GraphicsContext3D::TEXTURE_2D, GraphicsContext3D::TEXTURE_WRAP_S, GraphicsContext3D::CLAMP_TO_EDGE); 533 m_context->texParameteri(GraphicsContext3D::TEXTURE_2D, GraphicsContext3D::TEXTURE_WRAP_T, GraphicsContext3D::CLAMP_TO_EDGE); 534 m_context->texImage2DDirect(GraphicsContext3D::TEXTURE_2D, level, internalFormat, width, height, 0, format, type, nullptr); 535 536 m_context->framebufferTexture2D(GraphicsContext3D::FRAMEBUFFER, GraphicsContext3D::COLOR_ATTACHMENT0, GraphicsContext3D::TEXTURE_2D, outputTexture, level); 537 GC3Denum status = m_context->checkFramebufferStatus(GraphicsContext3D::FRAMEBUFFER); 538 if (status != GraphicsContext3D::FRAMEBUFFER_COMPLETE) { 539 LOG(WebGL, "VideoTextureCopierCV::copyVideoTextureToPlatformTexture(%p) - Unable to create framebuffer for outputTexture.", this); 540 return false; 541 } 542 543 m_context->useProgram(m_yuvProgram); 544 m_context->viewport(0, 0, width, height); 545 546 // Bind and set up the textures for the video source. 547 auto yPlaneWidth = IOSurfaceGetWidthOfPlane(surface, 0); 548 auto yPlaneHeight = IOSurfaceGetHeightOfPlane(surface, 0); 549 auto uvPlaneWidth = IOSurfaceGetWidthOfPlane(surface, 1); 550 auto uvPlaneHeight = IOSurfaceGetHeightOfPlane(surface, 1); 551 552 #if PLATFORM(IOS) 553 GC3Denum videoTextureTarget = GraphicsContext3D::TEXTURE_2D; 554 #else 555 GC3Denum videoTextureTarget = GL_TEXTURE_RECTANGLE_ARB; 556 #endif 557 auto uvTexture = m_context->createTexture(); 558 m_context->activeTexture(GraphicsContext3D::TEXTURE1); 559 m_context->bindTexture(videoTextureTarget, uvTexture); 560 m_context->texParameteri(videoTextureTarget, GraphicsContext3D::TEXTURE_MAG_FILTER, GraphicsContext3D::LINEAR); 561 m_context->texParameteri(videoTextureTarget, GraphicsContext3D::TEXTURE_MIN_FILTER, GraphicsContext3D::LINEAR); 562 m_context->texParameteri(videoTextureTarget, GraphicsContext3D::TEXTURE_WRAP_S, GraphicsContext3D::CLAMP_TO_EDGE); 563 m_context->texParameteri(videoTextureTarget, GraphicsContext3D::TEXTURE_WRAP_T, GraphicsContext3D::CLAMP_TO_EDGE); 564 if (!m_context->texImageIOSurface2D(videoTextureTarget, GL_RG, uvPlaneWidth, uvPlaneHeight, GL_RG, GL_UNSIGNED_BYTE, surface, 1)) { 565 m_context->deleteTexture(uvTexture); 566 return false; 567 } 568 569 auto yTexture = m_context->createTexture(); 570 m_context->activeTexture(GraphicsContext3D::TEXTURE0); 571 m_context->bindTexture(videoTextureTarget, yTexture); 572 m_context->texParameteri(videoTextureTarget, GraphicsContext3D::TEXTURE_MAG_FILTER, GraphicsContext3D::LINEAR); 573 m_context->texParameteri(videoTextureTarget, GraphicsContext3D::TEXTURE_MIN_FILTER, GraphicsContext3D::LINEAR); 574 m_context->texParameteri(videoTextureTarget, GraphicsContext3D::TEXTURE_WRAP_S, GraphicsContext3D::CLAMP_TO_EDGE); 575 m_context->texParameteri(videoTextureTarget, GraphicsContext3D::TEXTURE_WRAP_T, GraphicsContext3D::CLAMP_TO_EDGE); 576 if (!m_context->texImageIOSurface2D(videoTextureTarget, GL_LUMINANCE, yPlaneWidth, yPlaneHeight, GL_LUMINANCE, GL_UNSIGNED_BYTE, surface, 0)) { 577 m_context->deleteTexture(yTexture); 578 m_context->deleteTexture(uvTexture); 579 return false; 580 } 581 582 // Configure the drawing parameters. 583 m_context->uniform1i(m_yTextureUniformLocation, 0); 584 m_context->uniform1i(m_uvTextureUniformLocation, 1); 585 m_context->uniform1i(m_yuvFlipYUniformLocation, flipY); 586 m_context->uniform2f(m_yTextureSizeUniformLocation, yPlaneWidth, yPlaneHeight); 587 m_context->uniform2f(m_uvTextureSizeUniformLocation, uvPlaneWidth, uvPlaneHeight); 588 589 auto range = pixelRangeFromPixelFormat(pixelFormat); 590 auto transferFunction = transferFunctionFromString((CFStringRef)CVBufferGetAttachment(image, kCVImageBufferYCbCrMatrixKey, nil)); 591 auto& colorMatrix = YCbCrToRGBMatrixForRangeAndTransferFunction(range, transferFunction); 592 m_context->uniformMatrix3fv(m_colorMatrixUniformLocation, 1, GL_FALSE, colorMatrix.data()); 593 594 // Do the actual drawing. 595 m_context->bindBuffer(GraphicsContext3D::ARRAY_BUFFER, m_yuvVertexBuffer); 596 m_context->enableVertexAttribArray(m_yuvPositionAttributeLocation); 597 m_context->vertexAttribPointer(m_yuvPositionAttributeLocation, 2, GraphicsContext3D::FLOAT, false, 0, 0); 598 m_context->drawArrays(GraphicsContext3D::TRIANGLES, 0, 6); 599 600 // Clean-up. 601 m_context->deleteTexture(yTexture); 602 m_context->deleteTexture(uvTexture); 603 m_context->bindTexture(videoTextureTarget, 0); 604 605 return true; 606 } 607 608 bool VideoTextureCopierCV::copyVideoTextureToPlatformTexture(TextureType inputVideoTexture, size_t width, size_t height, Platform3DObject outputTexture, GC3Denum outputTarget, GC3Dint level, GC3Denum internalFormat, GC3Denum format, GC3Denum type, bool premultiplyAlpha, bool flipY) 609 { 610 if (!inputVideoTexture) 611 return false; 276 612 277 613 GLfloat lowerLeft[2] = { 0, 0 }; … … 289 625 #endif 290 626 627 if (lowerLeft[1] < upperRight[1]) 628 flipY = !flipY; 629 630 return copyVideoTextureToPlatformTexture(videoTextureName, videoTextureTarget, width, height, outputTexture, outputTarget, level, internalFormat, format, type, premultiplyAlpha, flipY); 631 } 632 633 bool VideoTextureCopierCV::copyVideoTextureToPlatformTexture(Platform3DObject videoTextureName, GC3Denum videoTextureTarget, size_t width, size_t height, Platform3DObject outputTexture, GC3Denum outputTarget, GC3Dint level, GC3Denum internalFormat, GC3Denum format, GC3Denum type, bool premultiplyAlpha, bool flipY) 634 { 291 635 LOG(WebGL, "VideoTextureCopierCV::copyVideoTextureToPlatformTexture(%p) - internalFormat: %s, format: %s, type: %s flipY: %s, premultiplyAlpha: %s", this, enumToStringMap()[internalFormat], enumToStringMap()[format], enumToStringMap()[type], flipY ? "true" : "false", premultiplyAlpha ? "true" : "false"); 636 637 GC3DStateSaver stateSaver(m_context.get()); 638 639 if (!m_program) { 640 if (!initializeContextObjects()) { 641 LOG(WebGL, "VideoTextureCopierCV::copyVideoTextureToPlatformTexture(%p) - Unable to initialize OpenGL context objects.", this); 642 return false; 643 } 644 } 645 646 stateSaver.saveVertexAttribState(m_positionAttributeLocation); 292 647 293 648 m_context->bindFramebuffer(GraphicsContext3D::FRAMEBUFFER, m_framebuffer); … … 327 682 #endif 328 683 329 if (lowerLeft[1] < upperRight[1])330 flipY = !flipY;331 332 684 m_context->uniform1i(m_flipYUniformLocation, flipY); 333 685 m_context->uniform1i(m_premultiplyUniformLocation, premultiplyAlpha); -
trunk/Source/WebCore/platform/graphics/cv/VideoTextureCopierCV.h
r222198 r223280 30 30 31 31 typedef struct __CVBuffer* CVImageBufferRef; 32 typedef struct __CVBuffer* CVPixelBufferRef; 32 33 typedef CVImageBufferRef CVOpenGLTextureRef; 33 34 typedef CVImageBufferRef CVOpenGLESTextureRef; 34 35 35 36 namespace WebCore { 37 38 class TextureCacheCV; 36 39 37 40 class VideoTextureCopierCV { … … 46 49 #endif 47 50 51 bool copyImageToPlatformTexture(CVPixelBufferRef, size_t width, size_t height, Platform3DObject outputTexture, GC3Denum outputTarget, GC3Dint level, GC3Denum internalFormat, GC3Denum format, GC3Denum type, bool premultiplyAlpha, bool flipY); 48 52 bool copyVideoTextureToPlatformTexture(TextureType, size_t width, size_t height, Platform3DObject outputTexture, GC3Denum outputTarget, GC3Dint level, GC3Denum internalFormat, GC3Denum format, GC3Denum type, bool premultiplyAlpha, bool flipY); 49 53 … … 51 55 52 56 private: 57 bool copyVideoTextureToPlatformTexture(Platform3DObject inputTexture, GC3Denum inputTarget, size_t width, size_t height, Platform3DObject outputTexture, GC3Denum outputTarget, GC3Dint level, GC3Denum internalFormat, GC3Denum format, GC3Denum type, bool premultiplyAlpha, bool flipY); 58 53 59 class GC3DStateSaver { 54 60 public: … … 76 82 77 83 bool initializeContextObjects(); 84 bool initializeUVContextObjects(); 78 85 79 86 Ref<GraphicsContext3D> m_context; 87 std::unique_ptr<TextureCacheCV> m_textureCache; 80 88 Platform3DObject m_framebuffer { 0 }; 81 89 Platform3DObject m_program { 0 }; … … 86 94 GC3Dint m_premultiplyUniformLocation { -1 }; 87 95 GC3Dint m_positionAttributeLocation { -1 }; 96 Platform3DObject m_yuvProgram { 0 }; 97 Platform3DObject m_yuvVertexBuffer { 0 }; 98 GC3Dint m_yTextureUniformLocation { -1 }; 99 GC3Dint m_uvTextureUniformLocation { -1 }; 100 GC3Dint m_yuvFlipYUniformLocation { -1 }; 101 GC3Dint m_colorMatrixUniformLocation { -1 }; 102 GC3Dint m_yuvPositionAttributeLocation { -1 }; 103 GC3Dint m_yTextureSizeUniformLocation { -1 }; 104 GC3Dint m_uvTextureSizeUniformLocation { -1 }; 88 105 }; 89 106
Note: See TracChangeset
for help on using the changeset viewer.