Changeset 214806 in webkit
- Timestamp:
- Apr 3, 2017 9:59:41 AM (7 years ago)
- Location:
- trunk
- Files:
-
- 2 added
- 25 edited
Legend:
- Unmodified
- Added
- Removed
-
trunk/LayoutTests/ChangeLog
r214787 r214806 1 2017-04-03 Youenn Fablet <youenn@apple.com> 2 3 captureStream is getting black frames with webgl canvas 4 https://bugs.webkit.org/show_bug.cgi?id=170325 5 6 Reviewed by Dean Jackson. 7 8 * fast/mediastream/captureStream/canvas3d-expected.txt: Added. 9 * fast/mediastream/captureStream/canvas3d.html: Added. 10 1 11 2017-04-03 Per Arne Vollan <pvollan@apple.com> 2 12 -
trunk/Source/WebCore/ChangeLog
r214787 r214806 1 2017-04-03 Youenn Fablet <youenn@apple.com> 2 3 captureStream is getting black frames with webgl canvas 4 https://bugs.webkit.org/show_bug.cgi?id=170325 5 6 Reviewed by Dean Jackson. 7 8 Test: fast/mediastream/captureStream/canvas3d.html 9 10 Changing the webgl context to save buffers in case the canvas is captured. 11 Adding a canvas changed notification in case of clear. 12 In the future, we might want to change this notification and do it when endPaint or similar is called. 13 14 Adding an Internals API to grab the RGBA equivalent of the next track frame. 15 For that purpose, adding a bunch of WEBCORE_EXPORT. 16 17 * Modules/mediastream/CanvasCaptureMediaStreamTrack.cpp: 18 (WebCore::CanvasCaptureMediaStreamTrack::Source::Source): Adding constraints support so that track settings 19 getter actually transmits the width and height of the source. 20 (WebCore::CanvasCaptureMediaStreamTrack::Source::canvasChanged): ensuring webgl canvas context keep their drawing buffer. 21 * Modules/mediastream/MediaStreamTrack.h: 22 * bindings/js/JSDOMGuardedObject.h: 23 * bindings/js/JSDOMPromise.h: 24 (WebCore::DeferredPromise::resolve): 25 (WebCore::DeferredPromise::reject): 26 * dom/ActiveDOMCallback.h: 27 * html/HTMLCanvasElement.cpp: 28 (WebCore::HTMLCanvasElement::captureStream): 29 * html/ImageData.h: 30 * html/ImageData.idl: 31 * html/canvas/WebGLRenderingContext.cpp: 32 (WebCore::WebGLRenderingContext::clear): ensuring canvas observers get notified in case of clear calls. 33 * html/canvas/WebGLRenderingContextBase.h: 34 (WebCore::WebGLRenderingContextBase::preserveDrawingBuffer): Added to allow canvas capture to update this property. 35 * platform/MediaSample.h: 36 (WebCore::MediaSample::getRGBAImageData): Added for internals API. 37 * platform/graphics/avfoundation/MediaSampleAVFObjC.h: 38 * platform/graphics/avfoundation/objc/MediaSampleAVFObjC.mm: 39 (WebCore::MediaSampleAVFObjC::getRGBAImageData): 40 * platform/graphics/cv/PixelBufferConformerCV.cpp: 41 (WebCore::PixelBufferConformerCV::convert): Helper routine for getRGBAImageData. 42 * platform/graphics/cv/PixelBufferConformerCV.h: 43 * platform/mediastream/RealtimeMediaSourceSettings.h: 44 (WebCore::RealtimeMediaSourceSettings::setSupportedConstraints): 45 (WebCore::RealtimeMediaSourceSettings::setSupportedConstraits): Deleted. 46 * platform/mediastream/mac/AVMediaCaptureSource.mm: 47 (WebCore::AVMediaCaptureSource::initializeSettings): 48 * platform/mediastream/openwebrtc/RealtimeAudioSourceOwr.h: 49 * platform/mediastream/openwebrtc/RealtimeVideoSourceOwr.h: 50 * platform/mock/MockRealtimeMediaSource.cpp: 51 (WebCore::MockRealtimeMediaSource::initializeSettings): 52 * testing/Internals.cpp: 53 (WebCore::Internals::grabNextMediaStreamTrackFrame): 54 (WebCore::Internals::videoSampleAvailable): 55 * testing/Internals.h: 56 * testing/Internals.idl: 57 1 58 2017-04-03 Per Arne Vollan <pvollan@apple.com> 2 59 -
trunk/Source/WebCore/Modules/mediastream/CanvasCaptureMediaStreamTrack.cpp
r213972 r214806 27 27 28 28 #include "GraphicsContext.h" 29 #include "WebGLRenderingContextBase.h" 29 30 30 31 #if ENABLE(MEDIA_STREAM) … … 68 69 m_settings.setWidth(canvas.width()); 69 70 m_settings.setHeight(canvas.height()); 71 RealtimeMediaSourceSupportedConstraints constraints; 72 constraints.setSupportsWidth(true); 73 constraints.setSupportsHeight(true); 74 m_settings.setSupportedConstraints(constraints); 70 75 } 71 76 … … 125 130 ASSERT_UNUSED(canvas, m_canvas == &canvas); 126 131 132 // FIXME: We need to preserve drawing buffer as we are currently grabbing frames asynchronously. 133 // We should instead add an anchor point for both 2d and 3d contexts where canvas will actually paint. 134 // And call canvas observers from that point. 135 if (canvas.renderingContext() && canvas.renderingContext()->isWebGL()) { 136 auto& context = static_cast<WebGLRenderingContextBase&>(*canvas.renderingContext()); 137 if (!context.isPreservingDrawingBuffer()) { 138 canvas.document().addConsoleMessage(MessageSource::JS, MessageLevel::Warning, ASCIILiteral("Turning drawing buffer preservation for the WebGL canvas being captured")); 139 context.setPreserveDrawingBuffer(true); 140 } 141 } 142 127 143 // FIXME: We should try to generate the frame at the time the screen is being updated. 128 144 if (m_canvasChangedTimer.isActive()) -
trunk/Source/WebCore/Modules/mediastream/MediaStreamTrack.h
r214042 r214806 90 90 String groupId; 91 91 }; 92 TrackSettings getSettings() const;92 WEBCORE_EXPORT TrackSettings getSettings() const; 93 93 94 94 struct TrackCapabilities { -
trunk/Source/WebCore/bindings/js/JSDOMGuardedObject.h
r213108 r214806 34 34 namespace WebCore { 35 35 36 class DOMGuardedObject : public RefCounted<DOMGuardedObject>, public ActiveDOMCallback {36 class WEBCORE_EXPORT DOMGuardedObject : public RefCounted<DOMGuardedObject>, public ActiveDOMCallback { 37 37 public: 38 38 ~DOMGuardedObject(); -
trunk/Source/WebCore/bindings/js/JSDOMPromise.h
r213108 r214806 89 89 void reject(std::nullptr_t); 90 90 void reject(Exception&&); 91 void reject(ExceptionCode, const String& = { });91 WEBCORE_EXPORT void reject(ExceptionCode, const String& = { }); 92 92 void reject(const JSC::PrivateName&); 93 93 … … 123 123 JSC::JSPromiseDeferred* deferred() const { return guarded(); } 124 124 125 void callFunction(JSC::ExecState&, JSC::JSValue function, JSC::JSValue resolution);125 WEBCORE_EXPORT void callFunction(JSC::ExecState&, JSC::JSValue function, JSC::JSValue resolution); 126 126 void resolve(JSC::ExecState& state, JSC::JSValue resolution) { callFunction(state, deferred()->resolve(), resolution); } 127 127 void reject(JSC::ExecState& state, JSC::JSValue resolution) { callFunction(state, deferred()->reject(), resolution); } … … 213 213 void fulfillPromiseWithArrayBuffer(Ref<DeferredPromise>&&, ArrayBuffer*); 214 214 void fulfillPromiseWithArrayBuffer(Ref<DeferredPromise>&&, const void*, size_t); 215 void rejectPromiseWithExceptionIfAny(JSC::ExecState&, JSDOMGlobalObject&, JSC::JSPromiseDeferred&);215 WEBCORE_EXPORT void rejectPromiseWithExceptionIfAny(JSC::ExecState&, JSDOMGlobalObject&, JSC::JSPromiseDeferred&); 216 216 JSC::EncodedJSValue createRejectedPromiseWithTypeError(JSC::ExecState&, const String&); 217 217 -
trunk/Source/WebCore/dom/ActiveDOMCallback.h
r210293 r214806 47 47 virtual ~ActiveDOMCallback(); 48 48 49 bool canInvokeCallback() const;49 WEBCORE_EXPORT bool canInvokeCallback() const; 50 50 }; 51 51 -
trunk/Source/WebCore/html/HTMLCanvasElement.cpp
r214017 r214806 604 604 if (frameRequestRate && frameRequestRate.value() < 0) 605 605 return Exception(NOT_SUPPORTED_ERR, ASCIILiteral("frameRequestRate is negative")); 606 606 607 607 auto track = CanvasCaptureMediaStreamTrack::create(context, *this, WTFMove(frameRequestRate)); 608 608 auto stream = MediaStream::create(context); -
trunk/Source/WebCore/html/ImageData.h
r208659 r214806 35 35 namespace WebCore { 36 36 37 class ImageData : public RefCounted<ImageData> {37 class WEBCORE_EXPORT ImageData : public RefCounted<ImageData> { 38 38 public: 39 39 static ExceptionOr<Ref<ImageData>> create(unsigned sw, unsigned sh); -
trunk/Source/WebCore/html/ImageData.idl
r209005 r214806 32 32 ConstructorMayThrowException, 33 33 CustomToJSObject, 34 ExportMacro=WEBCORE_EXPORT, 34 35 Exposed=(Window,Worker), 35 36 ImplementationLacksVTable, -
trunk/Source/WebCore/html/canvas/WebGLRenderingContext.cpp
r214086 r214806 471 471 if (!clearIfComposited(mask)) 472 472 m_context->clear(mask); 473 markContextChanged ();473 markContextChangedAndNotifyCanvasObserver(); 474 474 } 475 475 -
trunk/Source/WebCore/html/canvas/WebGLRenderingContextBase.h
r214017 r214806 208 208 long long getVertexAttribOffset(GC3Duint index, GC3Denum pname); 209 209 210 bool isPreservingDrawingBuffer() const { return m_attributes.preserveDrawingBuffer; } 211 void setPreserveDrawingBuffer(bool value) { m_attributes.preserveDrawingBuffer = value; } 212 210 213 virtual void hint(GC3Denum target, GC3Denum mode) = 0; 211 214 GC3Dboolean isBuffer(WebGLBuffer*); -
trunk/Source/WebCore/platform/MediaSample.h
r214120 r214806 28 28 29 29 #include "FloatSize.h" 30 #include <runtime/TypedArrays.h> 30 31 #include <wtf/MediaTime.h> 31 32 #include <wtf/RefCounted.h> … … 70 71 virtual Ref<MediaSample> createNonDisplayingCopy() const = 0; 71 72 73 virtual RefPtr<JSC::Uint8ClampedArray> getRGBAImageData() const { return nullptr; } 74 72 75 enum SampleFlags { 73 76 None = 0, -
trunk/Source/WebCore/platform/graphics/avfoundation/MediaSampleAVFObjC.h
r214120 r214806 41 41 static RefPtr<MediaSampleAVFObjC> createImageSample(Ref<JSC::Uint8ClampedArray>&&, unsigned long width, unsigned long height); 42 42 static RefPtr<MediaSampleAVFObjC> createImageSample(Vector<uint8_t>&&, unsigned long width, unsigned long height); 43 44 RefPtr<JSC::Uint8ClampedArray> getRGBAImageData() const final; 43 45 44 46 private: -
trunk/Source/WebCore/platform/graphics/avfoundation/objc/MediaSampleAVFObjC.mm
r213598 r214806 27 27 #import "MediaSampleAVFObjC.h" 28 28 29 #import "PixelBufferConformerCV.h" 30 #import <runtime/TypedArrayInlines.h> 29 31 #import <wtf/PrintStream.h> 30 32 … … 266 268 } 267 269 268 } 270 RefPtr<JSC::Uint8ClampedArray> MediaSampleAVFObjC::getRGBAImageData() const 271 { 272 const OSType imageFormat = kCVPixelFormatType_32RGBA; 273 RetainPtr<CFNumberRef> imageFormatNumber = adoptCF(CFNumberCreate(nullptr, kCFNumberIntType, &imageFormat)); 274 275 RetainPtr<CFMutableDictionaryRef> conformerOptions = adoptCF(CFDictionaryCreateMutable(0, 1, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks)); 276 CFDictionarySetValue(conformerOptions.get(), kCVPixelBufferPixelFormatTypeKey, imageFormatNumber.get()); 277 PixelBufferConformerCV pixelBufferConformer(conformerOptions.get()); 278 279 auto pixelBuffer = static_cast<CVPixelBufferRef>(CMSampleBufferGetImageBuffer(m_sample.get())); 280 auto rgbaPixelBuffer = pixelBufferConformer.convert(pixelBuffer); 281 auto status = CVPixelBufferLockBaseAddress(rgbaPixelBuffer.get(), kCVPixelBufferLock_ReadOnly); 282 ASSERT(status == noErr); 283 284 void* data = CVPixelBufferGetBaseAddressOfPlane(rgbaPixelBuffer.get(), 0); 285 size_t byteLength = CVPixelBufferGetHeight(pixelBuffer) * CVPixelBufferGetWidth(pixelBuffer) * 4; 286 auto result = JSC::Uint8ClampedArray::create(JSC::ArrayBuffer::create(data, byteLength), 0, byteLength); 287 288 status = CVPixelBufferUnlockBaseAddress(rgbaPixelBuffer.get(), kCVPixelBufferLock_ReadOnly); 289 ASSERT(status == noErr); 290 291 return result; 292 } 293 294 } -
trunk/Source/WebCore/platform/graphics/cv/PixelBufferConformerCV.cpp
r197375 r214806 73 73 } 74 74 75 RetainPtr<CVPixelBufferRef> PixelBufferConformerCV::convert(CVPixelBufferRef rawBuffer) 76 { 77 #if USE(VIDEOTOOLBOX) 78 RetainPtr<CVPixelBufferRef> buffer { rawBuffer }; 79 80 if (!VTPixelBufferConformerIsConformantPixelBuffer(m_pixelConformer.get(), buffer.get())) { 81 CVPixelBufferRef outputBuffer = nullptr; 82 OSStatus status = VTPixelBufferConformerCopyConformedPixelBuffer(m_pixelConformer.get(), buffer.get(), false, &outputBuffer); 83 if (status != noErr || !outputBuffer) 84 return nullptr; 85 return adoptCF(outputBuffer); 86 } 87 #else 88 UNUSED_PARAM(rawBuffer); 89 #endif 90 return nullptr; 91 } 92 75 93 RetainPtr<CGImageRef> PixelBufferConformerCV::createImageFromPixelBuffer(CVPixelBufferRef rawBuffer) 76 94 { -
trunk/Source/WebCore/platform/graphics/cv/PixelBufferConformerCV.h
r197375 r214806 38 38 public: 39 39 PixelBufferConformerCV(CFDictionaryRef attributes); 40 RetainPtr<CVPixelBufferRef> convert(CVPixelBufferRef); 40 41 RetainPtr<CGImageRef> createImageFromPixelBuffer(CVPixelBufferRef); 41 42 -
trunk/Source/WebCore/platform/mediastream/RealtimeMediaSourceSettings.h
r213936 r214806 93 93 void setGroupId(const AtomicString& groupId) { m_groupId = groupId; } 94 94 95 void setSupportedConstrai ts(const RealtimeMediaSourceSupportedConstraints& supportedConstraints) { m_supportedConstraints = supportedConstraints; }95 void setSupportedConstraints(const RealtimeMediaSourceSupportedConstraints& supportedConstraints) { m_supportedConstraints = supportedConstraints; } 96 96 97 97 template<class Encoder> void encode(Encoder&) const; -
trunk/Source/WebCore/platform/mediastream/mac/AVMediaCaptureSource.mm
r213880 r214806 205 205 { 206 206 if (m_currentSettings.deviceId().isEmpty()) 207 m_currentSettings.setSupportedConstrai ts(supportedConstraints());207 m_currentSettings.setSupportedConstraints(supportedConstraints()); 208 208 209 209 m_currentSettings.setDeviceId(id()); -
trunk/Source/WebCore/platform/mediastream/openwebrtc/RealtimeAudioSourceOwr.h
r210499 r214806 60 60 void initializeSettings() final { 61 61 if (m_currentSettings.deviceId().isEmpty()) 62 m_currentSettings.setSupportedConstrai ts(supportedConstraints());62 m_currentSettings.setSupportedConstraints(supportedConstraints()); 63 63 64 64 m_currentSettings.setDeviceId(id()); -
trunk/Source/WebCore/platform/mediastream/openwebrtc/RealtimeVideoSourceOwr.h
r210499 r214806 60 60 void initializeSettings() final { 61 61 if (m_currentSettings.deviceId().isEmpty()) 62 m_currentSettings.setSupportedConstrai ts(supportedConstraints());62 m_currentSettings.setSupportedConstraints(supportedConstraints()); 63 63 64 64 m_currentSettings.setDeviceId(id()); -
trunk/Source/WebCore/platform/mock/MockRealtimeMediaSource.cpp
r214178 r214806 99 99 { 100 100 if (m_currentSettings.deviceId().isEmpty()) { 101 m_currentSettings.setSupportedConstrai ts(supportedConstraints());101 m_currentSettings.setSupportedConstraints(supportedConstraints()); 102 102 m_currentSettings.setDeviceId(id()); 103 103 } -
trunk/Source/WebCore/testing/Internals.cpp
r214713 r214806 86 86 #include "IntRect.h" 87 87 #include "InternalSettings.h" 88 #include "JSImageData.h" 88 89 #include "Language.h" 89 90 #include "LibWebRTCProvider.h" … … 3896 3897 m_track->source().addObserver(*this); 3897 3898 } 3899 3900 void Internals::grabNextMediaStreamTrackFrame(TrackFramePromise&& promise) 3901 { 3902 m_nextTrackFramePromise = WTFMove(promise); 3903 } 3904 3905 void Internals::videoSampleAvailable(MediaSample& sample) 3906 { 3907 m_trackVideoSampleCount++; 3908 if (!m_nextTrackFramePromise) 3909 return; 3910 3911 auto videoSettings = m_track->getSettings(); 3912 if (!videoSettings.width || !videoSettings.height) 3913 return; 3914 3915 auto rgba = sample.getRGBAImageData(); 3916 if (!rgba) 3917 return; 3918 auto imageData = ImageData::create(rgba.releaseNonNull(), *videoSettings.width, *videoSettings.height); 3919 if (!imageData.hasException()) 3920 m_nextTrackFramePromise->resolve(imageData.releaseReturnValue().releaseNonNull()); 3921 else 3922 m_nextTrackFramePromise->reject(imageData.exception().code()); 3923 m_nextTrackFramePromise = std::nullopt; 3924 } 3925 3898 3926 #endif 3899 3927 -
trunk/Source/WebCore/testing/Internals.h
r214713 r214806 30 30 #include "ContextDestructionObserver.h" 31 31 #include "ExceptionOr.h" 32 #include "JSDOMPromise.h" 32 33 #include "PageConsoleClient.h" 33 34 #include "RealtimeMediaSource.h" 34 35 #include <runtime/Float32Array.h> 36 #include <wtf/Optional.h> 35 37 36 38 #if ENABLE(MEDIA_SESSION) … … 55 57 class HTMLMediaElement; 56 58 class HTMLSelectElement; 59 class ImageData; 57 60 class InspectorStubFrontend; 58 61 class InternalSettings; … … 563 566 unsigned long trackVideoSampleCount() const { return m_trackVideoSampleCount; } 564 567 void observeMediaStreamTrack(MediaStreamTrack&); 568 using TrackFramePromise = DOMPromise<IDLInterface<ImageData>>; 569 void grabNextMediaStreamTrackFrame(TrackFramePromise&&); 565 570 #endif 566 571 … … 574 579 // RealtimeMediaSource::Observer API 575 580 #if ENABLE(MEDIA_STREAM) 576 void videoSampleAvailable(MediaSample&) final { m_trackVideoSampleCount++; }581 void videoSampleAvailable(MediaSample&) final; 577 582 void audioSamplesAvailable(const MediaTime&, const PlatformAudioData&, const AudioStreamDescription&, size_t) final { m_trackAudioSampleCount++; } 578 583 … … 580 585 unsigned long m_trackAudioSampleCount { 0 }; 581 586 RefPtr<MediaStreamTrack> m_track; 587 std::optional<TrackFramePromise> m_nextTrackFramePromise; 582 588 #endif 583 589 -
trunk/Source/WebCore/testing/Internals.idl
r214713 r214806 527 527 528 528 [Conditional=MEDIA_STREAM] void observeMediaStreamTrack(MediaStreamTrack track); 529 [Conditional=MEDIA_STREAM] Promise<ImageData> grabNextMediaStreamTrackFrame(); 529 530 [Conditional=MEDIA_STREAM] readonly attribute unsigned long trackAudioSampleCount; 530 531 [Conditional=MEDIA_STREAM] readonly attribute unsigned long trackVideoSampleCount;
Note: See TracChangeset
for help on using the changeset viewer.