Changeset 266993 in webkit

Timestamp:

Sep 13, 2020 8:43:58 AM (4 years ago)

Author:

Wenson Hsieh

Message:

Add a key to the text manipulation userInfo dictionary indicating whether the translated item is on-screen
​https://bugs.webkit.org/show_bug.cgi?id=216452
<rdar://problem/68785397>

Reviewed by Darin Adler.

Source/WebCore:

For debugging purposes, and also to provide a hint as to what text should be prioritized when translating
web pages, WebKit clients have requested a new field in the userInfo metadata dictionary associated with each
token that indicates whether or not a translation (i.e. text manipulation) token represents an element that is
currently on-screen. See below for more details.

Test: TextManipulation.StartTextManipulationExtractsUserInfo

• editing/TextManipulationController.cpp:

(WebCore::tokenInfo):

Set the flag by checking whether or not the absolute bounding rect intersects with the visible content rect of
the enclosing frame. Note that since subframe content is currently never extracted for translation, we don't
need logic yet to recursively check that parent iframe elements are visible.

• editing/TextManipulationController.h:

Add a new bool flag in ManipulationTokenInfo.

(WebCore::TextManipulationController::ManipulationTokenInfo::encode const):
(WebCore::TextManipulationController::ManipulationTokenInfo::decode):

Source/WebKit:

Add _WKTextManipulationTokenUserInfoVisibilityKey and set its value to the value of the isVisible member in
ManipulationTokenInfo. See WebCore ChangeLog for more details.

• UIProcess/API/Cocoa/WKWebView.mm:

(createUserInfo):

• UIProcess/API/Cocoa/_WKTextManipulationToken.h:
• UIProcess/API/Cocoa/_WKTextManipulationToken.mm:

Tools:

Adjust an existing test so that it adds a fourth text paragraph with 2000px of top margin, and also
programmatically scrolls after loading the page so that only this last paragraph is visible. We expect the
metadata to indicate that none of the other tokens except this last one has a value of YES for
_WKTextManipulationTokenUserInfoVisibilityKey.

• TestWebKitAPI/Tests/WebKitCocoa/TextManipulation.mm:

(TestWebKitAPI::TEST):

Location:

trunk

Files:

• Source/WebCore/ChangeLog (modified) (1 diff)
• Source/WebCore/editing/TextManipulationController.cpp (modified) (1 diff)
• Source/WebCore/editing/TextManipulationController.h (modified) (3 diffs)
• Source/WebKit/ChangeLog (modified) (1 diff)
• Source/WebKit/UIProcess/API/Cocoa/WKWebView.mm (modified) (1 diff)
• Source/WebKit/UIProcess/API/Cocoa/_WKTextManipulationToken.h (modified) (1 diff)
• Source/WebKit/UIProcess/API/Cocoa/_WKTextManipulationToken.mm (modified) (1 diff)
• Tools/ChangeLog (modified) (1 diff)
• Tools/TestWebKitAPI/Tests/WebKitCocoa/TextManipulation.mm (modified) (5 diffs)

trunk/Source/WebCore/ChangeLog

@@ +1 @@
+2020-09-13  Wenson Hsieh  <wenson_hsieh@apple.com>
+
+        Add a key to the text manipulation userInfo dictionary indicating whether the translated item is on-screen
+        https://bugs.webkit.org/show_bug.cgi?id=216452
+        <rdar://problem/68785397>
+
+        Reviewed by Darin Adler.
+
+        For debugging purposes, and also to provide a hint as to what text should be prioritized when translating
+        web pages, WebKit clients have requested a new field in the userInfo metadata dictionary associated with each
+        token that indicates whether or not a translation (i.e. text manipulation) token represents an element that is
+        currently on-screen. See below for more details.
+
+        Test: TextManipulation.StartTextManipulationExtractsUserInfo
+
+        * editing/TextManipulationController.cpp:
+        (WebCore::tokenInfo):
+
+        Set the flag by checking whether or not the absolute bounding rect intersects with the visible content rect of
+        the enclosing frame. Note that since subframe content is currently never extracted for translation, we don't
+        need logic yet to recursively check that parent iframe elements are visible.
+
+        * editing/TextManipulationController.h:
+
+        Add a new `bool` flag in `ManipulationTokenInfo`.
+
+        (WebCore::TextManipulationController::ManipulationTokenInfo::encode const):
+        (WebCore::TextManipulationController::ManipulationTokenInfo::decode):
+
 2020-09-13  Philippe Normand  <pnormand@igalia.com>
 

trunk/Source/WebCore/editing/TextManipulationController.cpp

@@ -298 +298 @@
         if (element->hasAttributeWithoutSynchronization(HTMLNames::roleAttr))
             result.roleAttribute = element->attributeWithoutSynchronization(HTMLNames::roleAttr);
+        if (auto frame = makeRefPtr(node->document().frame()); frame && frame->view() && element->renderer()) {
+            // FIXME: This doesn't account for overflow clip.
+            auto elementRect = element->renderer()->absoluteAnchorRect();
+            auto visibleContentRect = frame->view()->visibleContentRect();
+            result.isVisible = visibleContentRect.intersects(enclosingIntRect(elementRect));
+        }
     }
     return result;

trunk/Source/WebCore/editing/TextManipulationController.h

@@ -53 +53 @@
         String roleAttribute;
         URL documentURL;
+        bool isVisible { false };
 
         template<class Encoder> void encode(Encoder&) const;
@@ -203 +204 @@
     encoder << roleAttribute;
     encoder << documentURL;
+    encoder << isVisible;
 }
 
@@ -216 +218 @@
 
     if (!decoder.decode(result.documentURL))
+        return WTF::nullopt;
+
+    if (!decoder.decode(result.isVisible))
         return WTF::nullopt;
 

trunk/Source/WebKit/ChangeLog

@@ +1 @@
+2020-09-13  Wenson Hsieh  <wenson_hsieh@apple.com>
+
+        Add a key to the text manipulation userInfo dictionary indicating whether the translated item is on-screen
+        https://bugs.webkit.org/show_bug.cgi?id=216452
+        <rdar://problem/68785397>
+
+        Reviewed by Darin Adler.
+
+        Add `_WKTextManipulationTokenUserInfoVisibilityKey` and set its value to the value of the `isVisible` member in
+        `ManipulationTokenInfo`. See WebCore ChangeLog for more details.
+
+        * UIProcess/API/Cocoa/WKWebView.mm:
+        (createUserInfo):
+        * UIProcess/API/Cocoa/_WKTextManipulationToken.h:
+        * UIProcess/API/Cocoa/_WKTextManipulationToken.mm:
+
 2020-09-13  Pablo Saavedra  <psaavedra@igalia.com>
 

trunk/Source/WebKit/UIProcess/API/Cocoa/WKWebView.mm

@@ -1623 +1623 @@
     if (!info->roleAttribute.isNull())
         [result setObject:(NSString *)info->roleAttribute forKey:_WKTextManipulationTokenUserInfoRoleAttributeKey];
+    [result setObject:@(info->isVisible) forKey:_WKTextManipulationTokenUserInfoVisibilityKey];
 
     return result;

trunk/Source/WebKit/UIProcess/API/Cocoa/_WKTextManipulationToken.h

@@ -32 +32 @@
 WK_EXTERN NSString * const _WKTextManipulationTokenUserInfoTagNameKey WK_API_AVAILABLE(macos(WK_MAC_TBA), ios(WK_IOS_TBA));
 WK_EXTERN NSString * const _WKTextManipulationTokenUserInfoRoleAttributeKey WK_API_AVAILABLE(macos(WK_MAC_TBA), ios(WK_IOS_TBA));
+WK_EXTERN NSString * const _WKTextManipulationTokenUserInfoVisibilityKey WK_API_AVAILABLE(macos(WK_MAC_TBA), ios(WK_IOS_TBA));
 
 WK_CLASS_AVAILABLE(macos(WK_MAC_TBA), ios(WK_IOS_TBA))

trunk/Source/WebKit/UIProcess/API/Cocoa/_WKTextManipulationToken.mm

@@ -32 +32 @@
 NSString * const _WKTextManipulationTokenUserInfoTagNameKey = @"_WKTextManipulationTokenUserInfoTagNameKey";
 NSString * const _WKTextManipulationTokenUserInfoRoleAttributeKey = @"_WKTextManipulationTokenUserInfoRoleAttributeKey";
+NSString * const _WKTextManipulationTokenUserInfoVisibilityKey = @"_WKTextManipulationTokenUserInfoVisibilityKey";
 
 @implementation _WKTextManipulationToken {

trunk/Tools/ChangeLog

@@ +1 @@
+2020-09-13  Wenson Hsieh  <wenson_hsieh@apple.com>
+
+        Add a key to the text manipulation userInfo dictionary indicating whether the translated item is on-screen
+        https://bugs.webkit.org/show_bug.cgi?id=216452
+        <rdar://problem/68785397>
+
+        Reviewed by Darin Adler.
+
+        Adjust an existing test so that it adds a fourth text paragraph with 2000px of top margin, and also
+        programmatically scrolls after loading the page so that only this last paragraph is visible. We expect the
+        metadata to indicate that none of the other tokens except this last one has a value of `YES` for
+        `_WKTextManipulationTokenUserInfoVisibilityKey`.
+
+        * TestWebKitAPI/Tests/WebKitCocoa/TextManipulation.mm:
+        (TestWebKitAPI::TEST):
+
 2020-09-12  Darin Adler  <darin@apple.com>
 

trunk/Tools/TestWebKitAPI/Tests/WebKitCocoa/TextManipulation.mm

@@ -802 +802 @@
         "    <div role='button'>Second</div>"
         "    <span>Third</span>"
+        "    <div style='margin-top: 2000px;'>Fourth</div>"
+        "    <script>scrollTo(0, 2000);</script>"
         "</body>"];
 
+    [webView waitForNextPresentationUpdate];
+
     done = false;
     [webView _startTextManipulationsWithConfiguration:nil completion:^{
@@ -811 +815 @@
 
     auto items = [delegate items];
-    EXPECT_EQ(items.count, 4UL);
+    EXPECT_EQ(items.count, 5UL);
     EXPECT_EQ(items[0].tokens.count, 1UL);
     EXPECT_EQ(items[1].tokens.count, 1UL);
     EXPECT_EQ(items[2].tokens.count, 1UL);
     EXPECT_EQ(items[3].tokens.count, 1UL);
+    EXPECT_EQ(items[4].tokens.count, 1UL);
     EXPECT_WK_STREQ("This is a test", items[0].tokens[0].content);
     EXPECT_WK_STREQ("First", items[1].tokens[0].content);
     EXPECT_WK_STREQ("Second", items[2].tokens[0].content);
     EXPECT_WK_STREQ("Third", items[3].tokens[0].content);
+    EXPECT_WK_STREQ("Fourth", items[4].tokens[0].content);
     {
         auto userInfo = items[0].tokens[0].userInfo;
         EXPECT_WK_STREQ("TestWebKitAPI.resources", [(NSURL *)userInfo[_WKTextManipulationTokenUserInfoDocumentURLKey] lastPathComponent]);
         EXPECT_WK_STREQ("TITLE", (NSString *)userInfo[_WKTextManipulationTokenUserInfoTagNameKey]);
+        EXPECT_FALSE([userInfo[_WKTextManipulationTokenUserInfoVisibilityKey] boolValue]);
     }
     {
@@ -829 +836 @@
         EXPECT_WK_STREQ("TestWebKitAPI.resources", [(NSURL *)userInfo[_WKTextManipulationTokenUserInfoDocumentURLKey] lastPathComponent]);
         EXPECT_WK_STREQ("P", (NSString *)userInfo[_WKTextManipulationTokenUserInfoTagNameKey]);
+        EXPECT_FALSE([userInfo[_WKTextManipulationTokenUserInfoVisibilityKey] boolValue]);
     }
     {
@@ -835 +843 @@
         EXPECT_WK_STREQ("DIV", (NSString *)userInfo[_WKTextManipulationTokenUserInfoTagNameKey]);
         EXPECT_WK_STREQ("button", (NSString *)userInfo[_WKTextManipulationTokenUserInfoRoleAttributeKey]);
+        EXPECT_FALSE([userInfo[_WKTextManipulationTokenUserInfoVisibilityKey] boolValue]);
     }
     {
@@ -840 +849 @@
         EXPECT_WK_STREQ("TestWebKitAPI.resources", [(NSURL *)userInfo[_WKTextManipulationTokenUserInfoDocumentURLKey] lastPathComponent]);
         EXPECT_WK_STREQ("SPAN", (NSString *)userInfo[_WKTextManipulationTokenUserInfoTagNameKey]);
+        EXPECT_FALSE([userInfo[_WKTextManipulationTokenUserInfoVisibilityKey] boolValue]);
+    }
+    {
+        auto userInfo = items[4].tokens[0].userInfo;
+        EXPECT_WK_STREQ("TestWebKitAPI.resources", [(NSURL *)userInfo[_WKTextManipulationTokenUserInfoDocumentURLKey] lastPathComponent]);
+        EXPECT_WK_STREQ("DIV", (NSString *)userInfo[_WKTextManipulationTokenUserInfoTagNameKey]);
+        EXPECT_TRUE([userInfo[_WKTextManipulationTokenUserInfoVisibilityKey] boolValue]);
     }
 }