Changeset 271154 in webkit
- Timestamp:
- Jan 5, 2021 10:16:02 AM (19 months ago)
- Location:
- trunk
- Files:
-
- 1 added
- 18 edited
-
Source/WebCore/ChangeLog (modified) (1 diff)
-
Source/WebCore/Modules/speech/SpeechRecognitionCaptureSource.cpp (modified) (1 diff)
-
Source/WebCore/Modules/speech/SpeechRecognitionCaptureSource.h (modified) (1 diff)
-
Source/WebCore/Modules/speech/SpeechRecognitionCaptureSourceImpl.cpp (modified) (1 diff)
-
Source/WebCore/Modules/speech/SpeechRecognitionCaptureSourceImpl.h (modified) (1 diff)
-
Source/WebCore/Modules/speech/SpeechRecognizer.h (modified) (1 diff)
-
Source/WebKit/ChangeLog (modified) (1 diff)
-
Source/WebKit/UIProcess/SpeechRecognitionPermissionManager.h (modified) (1 diff)
-
Source/WebKit/UIProcess/SpeechRecognitionServer.cpp (modified) (3 diffs)
-
Source/WebKit/UIProcess/SpeechRecognitionServer.h (modified) (1 diff)
-
Source/WebKit/UIProcess/UserMediaProcessManager.cpp (modified) (2 diffs)
-
Source/WebKit/UIProcess/UserMediaProcessManager.h (modified) (2 diffs)
-
Source/WebKit/UIProcess/WebPageProxy.cpp (modified) (2 diffs)
-
Source/WebKit/UIProcess/WebProcessProxy.cpp (modified) (1 diff)
-
Source/WebKit/UIProcess/WebProcessProxy.h (modified) (2 diffs)
-
Tools/ChangeLog (modified) (1 diff)
-
Tools/TestWebKitAPI/TestWebKitAPI.xcodeproj/project.pbxproj (modified) (4 diffs)
-
Tools/TestWebKitAPI/Tests/WebKitCocoa/SpeechRecognition.mm (modified) (4 diffs)
-
Tools/TestWebKitAPI/Tests/WebKitCocoa/speechrecognition-basic.html (added)
Legend:
- Unmodified
- Added
- Removed
-
trunk/Source/WebCore/ChangeLog
r271153 r271154 1 2021-01-05 Sihui Liu <sihui_liu@appe.com> 2 3 Fail speech recognition when page is muted for audio capture 4 https://bugs.webkit.org/show_bug.cgi?id=220133 5 <rdar://problem/72745232> 6 7 Reviewed by Youenn Fablet. 8 9 API test: WebKit2.SpeechRecognitionErrorWhenStartingAudioCaptureOnDifferentPage 10 11 * Modules/speech/SpeechRecognitionCaptureSource.cpp: 12 (WebCore::SpeechRecognitionCaptureSource::mute): 13 * Modules/speech/SpeechRecognitionCaptureSource.h: 14 * Modules/speech/SpeechRecognitionCaptureSourceImpl.cpp: 15 (WebCore::SpeechRecognitionCaptureSourceImpl::mute): 16 * Modules/speech/SpeechRecognitionCaptureSourceImpl.h: 17 * Modules/speech/SpeechRecognizer.h: 18 (WebCore::SpeechRecognizer::source): 19 1 20 2021-01-05 Alex Christensen <achristensen@webkit.org> 2 21 -
trunk/Source/WebCore/Modules/speech/SpeechRecognitionCaptureSource.cpp
r270574 r271154 35 35 namespace WebCore { 36 36 37 void SpeechRecognitionCaptureSource::mute() 38 { 39 #if ENABLE(MEDIA_STREAM) 40 m_impl->mute(); 41 #endif 42 } 43 37 44 #if ENABLE(MEDIA_STREAM) 38 45 -
trunk/Source/WebCore/Modules/speech/SpeechRecognitionCaptureSource.h
r270574 r271154 45 45 SpeechRecognitionCaptureSource() = default; 46 46 ~SpeechRecognitionCaptureSource() = default; 47 WEBCORE_EXPORT void mute(); 48 49 #if ENABLE(MEDIA_STREAM) 47 50 using DataCallback = Function<void(const WTF::MediaTime&, const PlatformAudioData&, const AudioStreamDescription&, size_t)>; 48 51 using StateUpdateCallback = Function<void(const SpeechRecognitionUpdate&)>; 49 50 #if ENABLE(MEDIA_STREAM)51 52 SpeechRecognitionCaptureSource(SpeechRecognitionConnectionClientIdentifier, DataCallback&&, StateUpdateCallback&&, Ref<RealtimeMediaSource>&&); 52 53 WEBCORE_EXPORT static Optional<WebCore::CaptureDevice> findCaptureDevice(); -
trunk/Source/WebCore/Modules/speech/SpeechRecognitionCaptureSourceImpl.cpp
r270158 r271154 147 147 } 148 148 149 void SpeechRecognitionCaptureSourceImpl::mute() 150 { 151 m_source->setMuted(true); 152 } 153 149 154 } // namespace WebCore 150 155 -
trunk/Source/WebCore/Modules/speech/SpeechRecognitionCaptureSourceImpl.h
r270158 r271154 55 55 SpeechRecognitionCaptureSourceImpl(SpeechRecognitionConnectionClientIdentifier, DataCallback&&, StateUpdateCallback&&, Ref<RealtimeMediaSource>&&); 56 56 ~SpeechRecognitionCaptureSourceImpl(); 57 void mute(); 57 58 58 59 private: -
trunk/Source/WebCore/Modules/speech/SpeechRecognizer.h
r270772 r271154 53 53 54 54 Optional<SpeechRecognitionConnectionClientIdentifier> currentClientIdentifier() const { return m_clientIdentifier; } 55 SpeechRecognitionCaptureSource* source() { return m_source.get(); } 55 56 56 57 private: -
trunk/Source/WebKit/ChangeLog
r271153 r271154 1 2021-01-05 Sihui Liu <sihui_liu@appe.com> 2 3 Fail speech recognition when page is muted for audio capture 4 https://bugs.webkit.org/show_bug.cgi?id=220133 5 <rdar://problem/72745232> 6 7 Reviewed by Youenn Fablet. 8 9 We currently only allow one page to capture media at a time and we did this by muting (stop capture in) other 10 pages. To make speech recognition work with this behavior, two changes are made: 11 1. when page is muted, mute audio capture source used for speech recognition on the page. This will 12 ultimately fail recognition. 13 2. when speech recognition is about to start, make sure other pages are muted for capture. 14 15 * UIProcess/SpeechRecognitionPermissionManager.h: 16 (WebKit::SpeechRecognitionPermissionManager::page): 17 * UIProcess/SpeechRecognitionServer.cpp: 18 (WebKit::SpeechRecognitionServer::handleRequest): 19 (WebKit::SpeechRecognitionServer::mute): 20 * UIProcess/SpeechRecognitionServer.h: 21 * UIProcess/UserMediaProcessManager.cpp: 22 (WebKit::UserMediaProcessManager::muteCaptureMediaStreamsExceptIn): Deleted. 23 * UIProcess/UserMediaProcessManager.h: 24 * UIProcess/WebPageProxy.cpp: 25 (WebKit::WebPageProxy::activateMediaStreamCaptureInPage): 26 (WebKit::WebPageProxy::setMuted): 27 * UIProcess/WebProcessProxy.cpp: 28 (WebKit::WebProcessProxy::muteCaptureInPagesExcept): 29 (WebKit::WebProcessProxy::pageMutedStateChanged): 30 * UIProcess/WebProcessProxy.h: 31 1 32 2021-01-05 Alex Christensen <achristensen@webkit.org> 2 33 -
trunk/Source/WebKit/UIProcess/SpeechRecognitionPermissionManager.h
r271031 r271154 43 43 44 44 void decideByDefaultAction(const WebCore::SecurityOrigin&, CompletionHandler<void(bool)>&&); 45 WebPageProxy& page() { return m_page; } 45 46 46 47 private: -
trunk/Source/WebKit/UIProcess/SpeechRecognitionServer.cpp
r271031 r271154 28 28 29 29 #include "SpeechRecognitionPermissionRequest.h" 30 #include "UserMediaProcessManager.h" 30 31 #include "WebProcessProxy.h" 31 32 #include "WebSpeechRecognitionConnectionMessages.h" … … 125 126 } 126 127 128 WebProcessProxy::muteCaptureInPagesExcept(m_identifier); 129 127 130 bool mockDeviceCapturesEnabled = m_checkIfMockSpeechRecognitionEnabled(); 128 131 m_recognizer->start(clientIdentifier, sourceOrError.source(), mockDeviceCapturesEnabled, request.lang(), request.continuous(), request.interimResults(), request.maxAlternatives()); … … 191 194 } 192 195 196 void SpeechRecognitionServer::mute() 197 { 198 if (!m_recognizer) 199 return; 200 201 if (auto* source = m_recognizer->source()) 202 source->mute(); 203 } 204 193 205 } // namespace WebKit 194 206 -
trunk/Source/WebKit/UIProcess/SpeechRecognitionServer.h
r271031 r271154 64 64 void abort(WebCore::SpeechRecognitionConnectionClientIdentifier); 65 65 void invalidate(WebCore::SpeechRecognitionConnectionClientIdentifier); 66 void mute(); 66 67 67 68 private: -
trunk/Source/WebKit/UIProcess/UserMediaProcessManager.cpp
r264521 r271154 24 24 #include "Logging.h" 25 25 #include "MediaDeviceSandboxExtensions.h" 26 #include "SpeechRecognitionPermissionManager.h" 26 27 #include "WebPageProxy.h" 27 28 #include "WebProcessMessages.h" … … 57 58 } 58 59 59 void UserMediaProcessManager::muteCaptureMediaStreamsExceptIn(WebPageProxy& pageStartingCapture)60 {61 #if PLATFORM(COCOA)62 UserMediaPermissionRequestManagerProxy::forEach([&pageStartingCapture](auto& proxy) {63 if (&proxy.page() != &pageStartingCapture)64 proxy.page().setMediaStreamCaptureMuted(true);65 });66 #else67 UNUSED_PARAM(pageStartingCapture);68 #endif69 }70 71 60 #if ENABLE(SANDBOX_EXTENSIONS) 72 61 static bool needsAppleCameraService() -
trunk/Source/WebKit/UIProcess/UserMediaProcessManager.h
r245335 r271154 23 23 #include "UserMediaPermissionRequestManagerProxy.h" 24 24 #include <WebCore/CaptureDevice.h> 25 #include <WebCore/PageIdentifier.h> 25 26 #include <wtf/RunLoop.h> 26 27 … … 37 38 38 39 bool willCreateMediaStream(UserMediaPermissionRequestManagerProxy&, bool withAudio, bool withVideo); 39 void muteCaptureMediaStreamsExceptIn(WebPageProxy&);40 40 41 41 void revokeSandboxExtensionsIfNeeded(WebProcessProxy&); -
trunk/Source/WebKit/UIProcess/WebPageProxy.cpp
r271059 r271154 2388 2388 { 2389 2389 #if ENABLE(MEDIA_STREAM) 2390 UserMediaProcessManager::singleton().muteCaptureMediaStreamsExceptIn(*this);2390 WebProcessProxy::muteCaptureInPagesExcept(m_webPageID); 2391 2391 #endif 2392 2392 setMuted(m_mutedState & ~WebCore::MediaProducer::MediaStreamCaptureIsMuted); … … 5955 5955 bool hasMutedCaptureStreams = m_mediaState & WebCore::MediaProducer::MutedCaptureMask; 5956 5956 if (hasMutedCaptureStreams && !(state & WebCore::MediaProducer::MediaStreamCaptureIsMuted)) 5957 UserMediaProcessManager::singleton().muteCaptureMediaStreamsExceptIn(*this); 5958 #endif 5957 WebProcessProxy::muteCaptureInPagesExcept(m_webPageID); 5958 #endif 5959 5960 m_process->pageMutedStateChanged(m_webPageID, state); 5959 5961 5960 5962 send(Messages::WebPage::SetMuted(state)); -
trunk/Source/WebKit/UIProcess/WebProcessProxy.cpp
r271031 r271154 1765 1765 } 1766 1766 1767 #endif 1767 void WebProcessProxy::muteCaptureInPagesExcept(WebCore::PageIdentifier pageID) 1768 { 1769 #if PLATFORM(COCOA) 1770 for (auto* page : globalPageMap().values()) { 1771 if (page->webPageID() != pageID) 1772 page->setMediaStreamCaptureMuted(true); 1773 } 1774 #else 1775 UNUSED_PARAM(pageID); 1776 #endif 1777 } 1778 1779 #endif 1780 1781 void WebProcessProxy::pageMutedStateChanged(WebCore::PageIdentifier identifier, WebCore::MediaProducer::MutedStateFlags flags) 1782 { 1783 bool mutedForCapture = flags & MediaProducer::AudioAndVideoCaptureIsMuted; 1784 if (!mutedForCapture) 1785 return; 1786 1787 if (auto speechRecognitionServer = m_speechRecognitionServerMap.get(identifier)) 1788 speechRecognitionServer->mute(); 1789 } 1768 1790 1769 1791 #if PLATFORM(WATCHOS) -
trunk/Source/WebKit/UIProcess/WebProcessProxy.h
r270574 r271154 46 46 #include "WebProcessProxyMessagesReplies.h" 47 47 #include <WebCore/FrameIdentifier.h> 48 #include <WebCore/MediaProducer.h> 48 49 #include <WebCore/PageIdentifier.h> 49 50 #include <WebCore/ProcessIdentifier.h> … … 399 400 void setIgnoreInvalidMessageForTesting(); 400 401 #endif 401 402 402 403 #if ENABLE(MEDIA_STREAM) 404 static void muteCaptureInPagesExcept(WebCore::PageIdentifier); 403 405 SpeechRecognitionRemoteRealtimeMediaSourceManager& ensureSpeechRecognitionRemoteRealtimeMediaSourceManager(); 404 406 #endif 407 void pageMutedStateChanged(WebCore::PageIdentifier, WebCore::MediaProducer::MutedStateFlags); 405 408 406 409 protected: -
trunk/Tools/ChangeLog
r271151 r271154 1 2021-01-05 Sihui Liu <sihui_liu@appe.com> 2 3 Fail speech recognition when page is muted for audio capture 4 https://bugs.webkit.org/show_bug.cgi?id=220133 5 <rdar://problem/72745232> 6 7 Reviewed by Youenn Fablet. 8 9 * TestWebKitAPI/TestWebKitAPI.xcodeproj/project.pbxproj: 10 * TestWebKitAPI/Tests/WebKitCocoa/SpeechRecognition.mm: 11 (-[SpeechRecognitionPermissionUIDelegate _webView:requestMediaCaptureAuthorization:decisionHandler:]): 12 (-[SpeechRecognitionPermissionUIDelegate _webView:checkUserMediaPermissionForURL:mainFrameURL:frameIdentifier:decisionHandler:]): 13 (TestWebKitAPI::TEST): 14 * TestWebKitAPI/Tests/WebKitCocoa/speechrecognition-basic.html: Added. 15 1 16 2021-01-05 Alexey Proskuryakov <ap@apple.com> 2 17 -
trunk/Tools/TestWebKitAPI/TestWebKitAPI.xcodeproj/project.pbxproj
r270638 r271154 810 810 935786CD20F6A2910000CDFC /* IndexedDB.sqlite3 in Copy Resources */ = {isa = PBXBuildFile; fileRef = 934FA5C720F69FEE0040DC1B /* IndexedDB.sqlite3 */; }; 811 811 935786CE20F6A2A10000CDFC /* IndexedDB.sqlite3-shm in Copy Resources */ = {isa = PBXBuildFile; fileRef = 934FA5C620F69FED0040DC1B /* IndexedDB.sqlite3-shm */; }; 812 9360270625A3CF7600367670 /* speechrecognition-basic.html in Copy Resources */ = {isa = PBXBuildFile; fileRef = 9360270525A3B28E00367670 /* speechrecognition-basic.html */; }; 812 813 9361002914DC95A70061379D /* lots-of-iframes.html in Copy Resources */ = {isa = PBXBuildFile; fileRef = 9361002814DC957B0061379D /* lots-of-iframes.html */; }; 813 814 93625D271CD9741C006DC1F1 /* large-video-without-audio.html in Copy Resources */ = {isa = PBXBuildFile; fileRef = 93625D261CD973AF006DC1F1 /* large-video-without-audio.html */; }; … … 1620 1621 C01A23F21266156700C9ED55 /* spacebar-scrolling.html in Copy Resources */, 1621 1622 F4CFCDDA249FC9E400527482 /* SpaceOnly.otf in Copy Resources */, 1623 9360270625A3CF7600367670 /* speechrecognition-basic.html in Copy Resources */, 1622 1624 9342589E255B6A120059EEDD /* speechrecognition-user-permission-persistence.html in Copy Resources */, 1623 1625 E194E1BD177E53C7009C4D4E /* StopLoadingFromDidReceiveResponse.html in Copy Resources */, … … 2423 2425 934FA5C720F69FEE0040DC1B /* IndexedDB.sqlite3 */ = {isa = PBXFileReference; lastKnownFileType = file; path = IndexedDB.sqlite3; sourceTree = "<group>"; }; 2424 2426 93575C551D30366E000D604D /* focus-inputs.html */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.html; path = "focus-inputs.html"; sourceTree = "<group>"; }; 2427 9360270525A3B28E00367670 /* speechrecognition-basic.html */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.html; path = "speechrecognition-basic.html"; sourceTree = "<group>"; }; 2425 2428 9361002814DC957B0061379D /* lots-of-iframes.html */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.html; path = "lots-of-iframes.html"; sourceTree = "<group>"; }; 2426 2429 93625D261CD973AF006DC1F1 /* large-video-without-audio.html */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.html; path = "large-video-without-audio.html"; sourceTree = "<group>"; }; … … 3930 3933 F4F405BB1D4C0CF8007A9707 /* skinny-autoplaying-video-with-audio.html */, 3931 3934 F4CFCDD8249FC9D900527482 /* SpaceOnly.otf */, 3935 9360270525A3B28E00367670 /* speechrecognition-basic.html */, 3932 3936 9342589D255B66A00059EEDD /* speechrecognition-user-permission-persistence.html */, 3933 3937 515BE16E1D4288FF00DD7C68 /* StoreBlobToBeDeleted.html */, -
trunk/Tools/TestWebKitAPI/Tests/WebKitCocoa/SpeechRecognition.mm
r269810 r271154 31 31 #import <WebKit/WKUIDelegatePrivate.h> 32 32 #import <WebKit/WKWebView.h> 33 #import <WebKit/WKWebViewConfiguration .h>33 #import <WebKit/WKWebViewConfigurationPrivate.h> 34 34 #import <wtf/RetainPtr.h> 35 35 … … 41 41 @interface SpeechRecognitionPermissionUIDelegate : NSObject<WKUIDelegatePrivate> 42 42 - (void)_webView:(WKWebView *)webView requestSpeechRecognitionPermissionForOrigin:(WKSecurityOrigin *)origin decisionHandler:(void (^)(BOOL))decisionHandler; 43 - (void)_webView:(WKWebView *)webView requestMediaCaptureAuthorization: (_WKCaptureDevices)devices decisionHandler:(void (^)(BOOL))decisionHandler; 44 - (void)_webView:(WKWebView *)webView checkUserMediaPermissionForURL:(NSURL *)url mainFrameURL:(NSURL *)mainFrameURL frameIdentifier:(NSUInteger)frameIdentifier decisionHandler:(void (^)(NSString *salt, BOOL authorized))decisionHandler; 43 45 @end 44 46 … … 48 50 permissionRequested = true; 49 51 decisionHandler(shouldGrantPermissionRequest); 52 } 53 54 - (void)_webView:(WKWebView *)webView requestMediaCaptureAuthorization: (_WKCaptureDevices)devices decisionHandler:(void (^)(BOOL))decisionHandler 55 { 56 decisionHandler(YES); 57 } 58 59 - (void)_webView:(WKWebView *)webView checkUserMediaPermissionForURL:(NSURL *)url mainFrameURL:(NSURL *)mainFrameURL frameIdentifier:(NSUInteger)frameIdentifier decisionHandler:(void (^)(NSString *salt, BOOL authorized))decisionHandler 60 { 61 decisionHandler(@"0x9876543210", YES); 50 62 } 51 63 @end … … 104 116 } 105 117 118 TEST(WebKit2, SpeechRecognitionErrorWhenStartingAudioCaptureOnDifferentPage) 119 { 120 shouldGrantPermissionRequest = true; 121 122 auto configuration = adoptNS([[WKWebViewConfiguration alloc] init]); 123 auto handler = adoptNS([[SpeechRecognitionMessageHandler alloc] init]); 124 [[configuration userContentController] addScriptMessageHandler:handler.get() name:@"testHandler"]; 125 configuration.get()._mediaCaptureEnabled = YES; 126 auto preferences = [configuration preferences]; 127 preferences._mockCaptureDevicesEnabled = YES; 128 preferences._speechRecognitionEnabled = YES; 129 preferences._mediaCaptureRequiresSecureConnection = NO; 130 auto delegate = adoptNS([[SpeechRecognitionPermissionUIDelegate alloc] init]); 131 auto firstWebView = adoptNS([[TestWKWebView alloc] initWithFrame:CGRectMake(0, 0, 800, 600) configuration:configuration.get()]); 132 [firstWebView setUIDelegate:delegate.get()]; 133 auto secondWebView = adoptNS([[TestWKWebView alloc] initWithFrame:CGRectMake(0, 0, 800, 600) configuration:configuration.get()]); 134 [secondWebView setUIDelegate:delegate.get()]; 135 136 // First page starts recognition successfully. 137 receivedScriptMessage = false; 138 [firstWebView synchronouslyLoadTestPageNamed:@"speechrecognition-basic"]; 139 [firstWebView stringByEvaluatingJavaScript:@"start()"]; 140 TestWebKitAPI::Util::run(&receivedScriptMessage); 141 EXPECT_WK_STREQ(@"Start", [lastScriptMessage body]); 142 143 // First page is muted when second page starts recognition. 144 // Load html string instead of test page to make sure message only comes from one page. 145 receivedScriptMessage = false; 146 [secondWebView synchronouslyLoadHTMLString:@"<script>speechRecognition = new webkitSpeechRecognition(); speechRecognition.start();</script>" baseURL:[NSURL URLWithString:@"https://webkit.org"]]; 147 TestWebKitAPI::Util::run(&receivedScriptMessage); 148 EXPECT_WK_STREQ(@"Error: audio-capture - Source is muted", [lastScriptMessage body]); 149 150 // First page restarts recognition successfully. 151 receivedScriptMessage = false; 152 [firstWebView stringByEvaluatingJavaScript:@"start()"]; 153 TestWebKitAPI::Util::run(&receivedScriptMessage); 154 EXPECT_WK_STREQ(@"Start", [lastScriptMessage body]); 155 156 // First page is muted when second page starts media recorder. 157 receivedScriptMessage = false; 158 [secondWebView synchronouslyLoadTestPageNamed:@"speechrecognition-basic"]; 159 [secondWebView stringByEvaluatingJavaScript:@"startRecorder()"]; 160 TestWebKitAPI::Util::run(&receivedScriptMessage); 161 EXPECT_WK_STREQ(@"Error: audio-capture - Source is muted", [lastScriptMessage body]); 162 163 // Second page is muted when first page starts recognition. 164 receivedScriptMessage = false; 165 [firstWebView synchronouslyLoadHTMLString:@"<script>speechRecognition = new webkitSpeechRecognition(); speechRecognition.start();</script>" baseURL:[NSURL URLWithString:@"https://webkit.org"]]; 166 TestWebKitAPI::Util::run(&receivedScriptMessage); 167 EXPECT_WK_STREQ(@"Recorder Mute", [lastScriptMessage body]); 168 } 169 106 170 } // namespace TestWebKitAPI
Note: See TracChangeset
for help on using the changeset viewer.