Changeset 271205 in webkit

Timestamp:

Jan 6, 2021 11:22:03 AM (19 months ago)

Author:

commit-queue@webkit.org

Message:

Stop speech recognition if page becomes invisible
​https://bugs.webkit.org/show_bug.cgi?id=220073
<rdar://problem/72710704>

Patch by Sihui Liu <​sihui_liu@appe.com> on 2021-01-06
Reviewed by Youenn Fablet.

Source/WebKit:

We should not allow speech recognition on page invisible to user, because page may get content of captured audio
without user's notice. To do this, we should:

1. deny speech recognition request on invisibile page
2. abort ongoing speech recognition if page becomes invisible

API test: WebKit2.SpeechRecognitionPageBecomesInvisible

• UIProcess/SpeechRecognitionPermissionManager.cpp:

(WebKit::SpeechRecognitionPermissionManager::continueProcessingRequest):

• UIProcess/SpeechRecognitionServer.cpp:

(WebKit::SpeechRecognitionServer::abortForPageIsBecomingInvisible):

• UIProcess/SpeechRecognitionServer.h:
• UIProcess/WebPageProxy.cpp:

(WebKit::WebPageProxy::dispatchActivityStateChange):

• UIProcess/WebProcessProxy.cpp:

(WebKit::WebProcessProxy::pageIsBecomingInvisible):

• UIProcess/WebProcessProxy.h:

Tools:

• TestWebKitAPI/Tests/WebKitCocoa/SpeechRecognition.mm: updated

WebKit2.SpeechRecognitionErrorWhenStartingAudioCaptureOnDifferentPage for updated behavior.
(TestWebKitAPI::TEST):

• TestWebKitAPI/cocoa/TestWKWebView.mm:

(-[TestWKWebView _setUpTestWindow:]): set the origin of host window to be the origin of frame of TestWKWebView.

Location:

trunk

Files:

• Source/WebKit/ChangeLog (modified) (1 diff)
• Source/WebKit/UIProcess/SpeechRecognitionPermissionManager.cpp (modified) (1 diff)
• Source/WebKit/UIProcess/SpeechRecognitionServer.cpp (modified) (1 diff)
• Source/WebKit/UIProcess/SpeechRecognitionServer.h (modified) (1 diff)
• Source/WebKit/UIProcess/WebPageProxy.cpp (modified) (1 diff)
• Source/WebKit/UIProcess/WebProcessProxy.cpp (modified) (1 diff)
• Source/WebKit/UIProcess/WebProcessProxy.h (modified) (1 diff)
• Tools/ChangeLog (modified) (1 diff)
• Tools/TestWebKitAPI/Tests/WebKitCocoa/SpeechRecognition.mm (modified) (2 diffs)
• Tools/TestWebKitAPI/cocoa/TestWKWebView.mm (modified) (1 diff)

trunk/Source/WebKit/ChangeLog

@@ +1 @@
+2021-01-06  Sihui Liu  <sihui_liu@appe.com>
+
+        Stop speech recognition if page becomes invisible
+        https://bugs.webkit.org/show_bug.cgi?id=220073
+        <rdar://problem/72710704>
+
+        Reviewed by Youenn Fablet.
+
+        We should not allow speech recognition on page invisible to user, because page may get content of captured audio
+        without user's notice. To do this, we should:
+        1. deny speech recognition request on invisibile page
+        2. abort ongoing speech recognition if page becomes invisible
+
+        API test: WebKit2.SpeechRecognitionPageBecomesInvisible
+
+        * UIProcess/SpeechRecognitionPermissionManager.cpp:
+        (WebKit::SpeechRecognitionPermissionManager::continueProcessingRequest):
+        * UIProcess/SpeechRecognitionServer.cpp:
+        (WebKit::SpeechRecognitionServer::abortForPageIsBecomingInvisible):
+        * UIProcess/SpeechRecognitionServer.h:
+        * UIProcess/WebPageProxy.cpp:
+        (WebKit::WebPageProxy::dispatchActivityStateChange):
+        * UIProcess/WebProcessProxy.cpp:
+        (WebKit::WebProcessProxy::pageIsBecomingInvisible):
+        * UIProcess/WebProcessProxy.h:
+
 2021-01-06  Alex Christensen  <achristensen@webkit.org>
 

trunk/Source/WebKit/UIProcess/SpeechRecognitionPermissionManager.cpp

@@ -162 +162 @@
     ASSERT(m_userPermissionCheck == CheckResult::Granted);
 
+    if (!m_page.isViewVisible()) {
+        completeCurrentRequest(SpeechRecognitionPermissionDecision::Deny);
+        return;
+    }
+
     completeCurrentRequest(SpeechRecognitionPermissionDecision::Grant);
 }

trunk/Source/WebKit/UIProcess/SpeechRecognitionServer.cpp

@@ -169 +169 @@
 }
 
+void SpeechRecognitionServer::abortForPageIsBecomingInvisible()
+{
+    if (!m_recognizer)
+        return;
+
+    auto currentClientIdentifier = m_recognizer->currentClientIdentifier();
+    if (!currentClientIdentifier)
+        return;
+
+    auto error = WebCore::SpeechRecognitionError { WebCore::SpeechRecognitionErrorType::Aborted, "Page is no longer visible"_s };
+    sendUpdate(*currentClientIdentifier, WebCore::SpeechRecognitionUpdateType::Error, error);
+    m_recognizer->reset();
+}
+
 void SpeechRecognitionServer::sendUpdate(WebCore::SpeechRecognitionConnectionClientIdentifier clientIdentifier, WebCore::SpeechRecognitionUpdateType type, Optional<WebCore::SpeechRecognitionError> error, Optional<Vector<WebCore::SpeechRecognitionResultData>> result)
 {

trunk/Source/WebKit/UIProcess/SpeechRecognitionServer.h

@@ -65 +65 @@
     void invalidate(WebCore::SpeechRecognitionConnectionClientIdentifier);
     void mute();
+    void abortForPageIsBecomingInvisible();
 
 private:

trunk/Source/WebKit/UIProcess/WebPageProxy.cpp

@@ -1996 +1996 @@
         updateCurrentModifierState();
 
-    if ((m_potentiallyChangedActivityStateFlags & ActivityState::IsVisible) && isViewVisible())
-        viewIsBecomingVisible();
+    if ((m_potentiallyChangedActivityStateFlags & ActivityState::IsVisible)) {
+        if (isViewVisible())
+            viewIsBecomingVisible();
+        else
+            m_process->pageIsBecomingInvisible(m_webPageID);
+    }
 
     bool isNowInWindow = (changed & ActivityState::IsInWindow) && isInWindow();

trunk/Source/WebKit/UIProcess/WebProcessProxy.cpp

@@ -1789 +1789 @@
 }
 
+void WebProcessProxy::pageIsBecomingInvisible(WebCore::PageIdentifier identifier)
+{
+    if (auto server = m_speechRecognitionServerMap.get(identifier))
+        server->abortForPageIsBecomingInvisible();
+}
+
 #if PLATFORM(WATCHOS)
 

trunk/Source/WebKit/UIProcess/WebProcessProxy.h

@@ -406 +406 @@
 #endif
     void pageMutedStateChanged(WebCore::PageIdentifier, WebCore::MediaProducer::MutedStateFlags);
+    void pageIsBecomingInvisible(WebCore::PageIdentifier);
 
 protected:

trunk/Tools/ChangeLog

@@ +1 @@
+2021-01-06  Sihui Liu  <sihui_liu@appe.com>
+
+        Stop speech recognition if page becomes invisible
+        https://bugs.webkit.org/show_bug.cgi?id=220073
+        <rdar://problem/72710704>
+
+        Reviewed by Youenn Fablet.
+
+        * TestWebKitAPI/Tests/WebKitCocoa/SpeechRecognition.mm: updated 
+        WebKit2.SpeechRecognitionErrorWhenStartingAudioCaptureOnDifferentPage for updated behavior.
+        (TestWebKitAPI::TEST):
+        * TestWebKitAPI/cocoa/TestWKWebView.mm:
+        (-[TestWKWebView _setUpTestWindow:]): set the origin of host window to be the origin of frame of TestWKWebView.
+
 2021-01-06  Jonathan Bedard <jbedard@apple.com>
 

trunk/Tools/TestWebKitAPI/Tests/WebKitCocoa/SpeechRecognition.mm

@@ -129 +129 @@
     preferences._mediaCaptureRequiresSecureConnection = NO;
     auto delegate = adoptNS([[SpeechRecognitionPermissionUIDelegate alloc] init]);
-    auto firstWebView = adoptNS([[TestWKWebView alloc] initWithFrame:CGRectMake(0, 0, 800, 600) configuration:configuration.get()]);
+    auto firstWebView = adoptNS([[TestWKWebView alloc] initWithFrame:CGRectMake(0, 0, 100, 100) configuration:configuration.get()]);
     [firstWebView setUIDelegate:delegate.get()];
-    auto secondWebView = adoptNS([[TestWKWebView alloc] initWithFrame:CGRectMake(0, 0, 800, 600) configuration:configuration.get()]);
+    auto secondWebView = adoptNS([[TestWKWebView alloc] initWithFrame:CGRectMake(100, 0, 100, 100) configuration:configuration.get()]);
     [secondWebView setUIDelegate:delegate.get()];
 
@@ -168 +168 @@
 }
 
+// FIXME: enable this test on iOS when https://webkit.org/b/175204 is fixed.
+#if PLATFORM(MAC)
+
+TEST(WebKit2, SpeechRecognitionPageBecomesInvisible)
+{
+    auto configuration = adoptNS([[WKWebViewConfiguration alloc] init]);
+    auto handler = adoptNS([[SpeechRecognitionMessageHandler alloc] init]);
+    [[configuration userContentController] addScriptMessageHandler:handler.get() name:@"testHandler"];
+    auto preferences = [configuration preferences];
+    preferences._mockCaptureDevicesEnabled = YES;
+    preferences._speechRecognitionEnabled = YES;
+    auto webView = adoptNS([[TestWKWebView alloc] initWithFrame:CGRectMake(0, 0, 800, 600) configuration:configuration.get()]);
+    auto delegate = adoptNS([[SpeechRecognitionPermissionUIDelegate alloc] init]);
+    [webView setUIDelegate:delegate.get()];
+
+    // Page is visible.
+    shouldGrantPermissionRequest = true;
+    receivedScriptMessage = false;
+    [webView synchronouslyLoadTestPageNamed:@"speechrecognition-basic"];
+    [webView stringByEvaluatingJavaScript:@"start()"];
+    TestWebKitAPI::Util::run(&receivedScriptMessage);
+    EXPECT_WK_STREQ(@"Start", [lastScriptMessage body]);
+
+    // Hide page.
+    receivedScriptMessage = false;
+#if PLATFORM(MAC)
+    [webView.get().window setIsVisible:NO];
+#else
+    webView.get().window.hidden = YES;
+#endif
+    TestWebKitAPI::Util::run(&receivedScriptMessage);
+    EXPECT_WK_STREQ(@"Error: aborted - Page is no longer visible", [lastScriptMessage body]);
+
+    // Page is invisible.
+    receivedScriptMessage = false;
+    [webView evaluateJavaScript:@"start()" completionHandler:nil];
+    TestWebKitAPI::Util::run(&receivedScriptMessage);
+    EXPECT_WK_STREQ(@"Error: not-allowed - Permission check failed", [lastScriptMessage body]);
+}
+
+#endif
+
 } // namespace TestWebKitAPI

trunk/Tools/TestWebKitAPI/cocoa/TestWKWebView.mm

@@ -463 +463 @@
 #if PLATFORM(MAC)
     _hostWindow = adoptNS([[TestWKWebViewHostWindow alloc] initWithWebView:self contentRect:frame styleMask:(NSWindowStyleMaskBorderless | NSWindowStyleMaskMiniaturizable) backing:NSBackingStoreBuffered defer:NO]);
-    [_hostWindow setFrameOrigin:NSMakePoint(0, 0)];
+    [_hostWindow setFrameOrigin:frame.origin];
     [_hostWindow setIsVisible:YES];
     [_hostWindow contentView].wantsLayer = YES;