⚠ Archived content — this site is no longer maintained.   Current WebKit documentation is at docs.webkit.org.

Changeset 285862 in webkit


Ignore:
Timestamp:
Nov 16, 2021, 8:37:05 AM (5 years ago)
Author:
Wenson Hsieh
Message:

Add support for injecting and rendering text recognition blocks
​https://bugs.webkit.org/show_bug.cgi?id=233044

Reviewed by Aditya Keerthi.

Adds support for rendering text recognition blocks, which appear as opaque div elements over images. See below
for more details; no change in behavior, since nothing currently generates TextRecognitionBlockData yet.

  • dom/ImageOverlay.cpp:

(WebCore::ImageOverlay::imageOverlayDataDetectorClass):
(WebCore::ImageOverlay::imageOverlayBlockClass):
(WebCore::ImageOverlay::isDataDetectorResult):
(WebCore::ImageOverlay::updateSubtree):

Add support for creating text recognition block containers in the UA shadow root, if needed.

(WebCore::ImageOverlay::fitElementToQuad):

Factor out logic to adjust the width, height and transforms on a given element to fit the given quad, and return
rotated bounding rect info; we use this in three places below.

(WebCore::ImageOverlay::updateWithTextRecognitionResult):

Add support for adjusting the size and transforms on each of the block containers created above.

(WebCore::ImageOverlay::imageOverlayDataDetectorClassName): Deleted.

Rename this to just imageOverlayDataDetectorClass() to match the other static helper functions.

  • html/shadow/imageOverlay.css:

(div#image-overlay):
(div.image-overlay-line):
(div.image-overlay-line, div.image-overlay-block):
(div.image-overlay-block):
(div.image-overlay-line, .image-overlay-text):

Location:
trunk/Source/WebCore
Files:
3 edited

Legend:

Unmodified
Added
Removed

  • trunk/Source/WebCore/ChangeLog

    r285861 r285862  
     12021-11-16  Wenson Hsieh  <wenson_hsieh@apple.com>
     2
     3        Add support for injecting and rendering text recognition blocks
     4        https://bugs.webkit.org/show_bug.cgi?id=233044
     5
     6        Reviewed by Aditya Keerthi.
     7
     8        Adds support for rendering text recognition blocks, which appear as opaque div elements over images. See below
     9        for more details; no change in behavior, since nothing currently generates TextRecognitionBlockData yet.
     10
     11        * dom/ImageOverlay.cpp:
     12        (WebCore::ImageOverlay::imageOverlayDataDetectorClass):
     13        (WebCore::ImageOverlay::imageOverlayBlockClass):
     14        (WebCore::ImageOverlay::isDataDetectorResult):
     15        (WebCore::ImageOverlay::updateSubtree):
     16
     17        Add support for creating text recognition block containers in the UA shadow root, if needed.
     18
     19        (WebCore::ImageOverlay::fitElementToQuad):
     20
     21        Factor out logic to adjust the width, height and transforms on a given element to fit the given quad, and return
     22        rotated bounding rect info; we use this in three places below.
     23
     24        (WebCore::ImageOverlay::updateWithTextRecognitionResult):
     25
     26        Add support for adjusting the size and transforms on each of the block containers created above.
     27
     28        (WebCore::ImageOverlay::imageOverlayDataDetectorClassName): Deleted.
     29
     30        Rename this to just `imageOverlayDataDetectorClass()` to match the other static helper functions.
     31
     32        * html/shadow/imageOverlay.css:
     33        (div#image-overlay):
     34        (div.image-overlay-line):
     35        (div.image-overlay-line, div.image-overlay-block):
     36        (div.image-overlay-block):
     37        (div.image-overlay-line, .image-overlay-text):
     38
    1392021-11-16  Andreu Botella  <andreu@andreubotella.com>
    240

  • trunk/Source/WebCore/dom/ImageOverlay.cpp

    r285725 r285862  
    6464}
    6565
    66 static const AtomString& imageOverlayDataDetectorClassName()
     66static const AtomString& imageOverlayDataDetectorClass()
    6767{
    6868    static MainThreadNeverDestroyed<const AtomString> className("image-overlay-data-detector-result", AtomString::ConstructFromLiteral);
    … …  
    8181{
    8282    static MainThreadNeverDestroyed<const AtomString> className("image-overlay-text", AtomString::ConstructFromLiteral);
     83    return className;
     84}
     85
     86static const AtomString& imageOverlayBlockClass()
     87{
     88    static MainThreadNeverDestroyed<const AtomString> className("image-overlay-block", AtomString::ConstructFromLiteral);
    8389    return className;
    8490}
    … …  
    107113bool isDataDetectorResult(const HTMLElement& element)
    108114{
    109     return imageOverlayHost(element) && element.hasClass() && element.classNames().contains(imageOverlayDataDetectorClassName());
     115    return imageOverlayHost(element) && element.hasClass() && element.classNames().contains(imageOverlayDataDetectorClass());
    110116}
    111117
    … …  
    192198    Vector<LineElements> lines;
    193199    Vector<Ref<HTMLDivElement>> dataDetectors;
     200    Vector<Ref<HTMLDivElement>> blocks;
    194201};
    195202
    … …  
    230237
    231238    if (elements.root) {
    232         for (auto& lineOrDataDetector : childrenOfType<HTMLDivElement>(*elements.root)) {
    233             if (!lineOrDataDetector.hasClass())
     239        for (auto& childElement : childrenOfType<HTMLDivElement>(*elements.root)) {
     240            if (!childElement.hasClass())
    234241                continue;
    235242
    236             if (lineOrDataDetector.classList().contains(imageOverlayLineClass())) {
    237                 LineElements lineElements { lineOrDataDetector, { } };
    238                 for (auto& text : childrenOfType<HTMLDivElement>(lineOrDataDetector))
    239                     lineElements.children.append(text);
    240                 elements.lines.append(WTFMove(lineElements));
    241             } else if (lineOrDataDetector.classList().contains(imageOverlayDataDetectorClassName()))
    242                 elements.dataDetectors.append(lineOrDataDetector);
     243            auto& classes = childElement.classList();
     244            if (classes.contains(imageOverlayDataDetectorClass())) {
     245                elements.dataDetectors.append(childElement);
     246                continue;
     247            }
     248
     249            if (classes.contains(imageOverlayBlockClass())) {
     250                elements.blocks.append(childElement);
     251                continue;
     252            }
     253
     254            ASSERT(classes.contains(imageOverlayLineClass()));
     255            LineElements lineElements { childElement, { } };
     256            for (auto& text : childrenOfType<HTMLDivElement>(childElement))
     257                lineElements.children.append(text);
     258            elements.lines.append(WTFMove(lineElements));
    243259        }
    244260
    … …  
    248264
    249265            if (result.lines.size() != elements.lines.size())
     266                return false;
     267
     268            if (result.blocks.size() != elements.blocks.size())
    250269                return false;
    251270
    … …  
    260279                        return false;
    261280                }
     281            }
     282
     283            for (size_t index = 0; index < result.blocks.size(); ++index) {
     284                if (result.blocks[index].text != elements.blocks[index]->textContent())
     285                    return false;
    262286            }
    263287
    … …  
    311335        for (auto& dataDetector : result.dataDetectors) {
    312336            auto dataDetectorContainer = DataDetection::createElementForImageOverlay(document.get(), dataDetector);
    313             dataDetectorContainer->classList().add(imageOverlayDataDetectorClassName());
     337            dataDetectorContainer->classList().add(imageOverlayDataDetectorClass());
    314338            rootContainer->appendChild(dataDetectorContainer);
    315339            elements.dataDetectors.uncheckedAppend(WTFMove(dataDetectorContainer));
    316340        }
    317341#endif // ENABLE(DATA_DETECTION)
     342
     343        elements.blocks.reserveInitialCapacity(result.blocks.size());
     344        for (auto& block : result.blocks) {
     345            auto blockContainer = HTMLDivElement::create(document.get());
     346            blockContainer->classList().add(imageOverlayBlockClass());
     347            rootContainer->appendChild(blockContainer);
     348            blockContainer->appendChild(Text::create(document.get(), makeString('\n', block.text)));
     349            elements.blocks.uncheckedAppend(WTFMove(blockContainer));
     350        }
    318351
    319352        if (document->quirks().needsToForceUserSelectWhenInstallingImageOverlay())
    … …  
    329362
    330363    return elements;
     364}
     365
     366static RotatedRect fitElementToQuad(HTMLElement& container, const FloatQuad& quad)
     367{
     368    auto bounds = rotatedBoundingRectWithMinimumAngleOfRotation(quad, 0.01);
     369    container.setInlineStyleProperty(CSSPropertyWidth, bounds.size.width(), CSSUnitType::CSS_PX);
     370    container.setInlineStyleProperty(CSSPropertyHeight, bounds.size.height(), CSSUnitType::CSS_PX);
     371    container.setInlineStyleProperty(CSSPropertyTransform, makeString(
     372        "translate("_s,
     373        std::round(bounds.center.x() - (bounds.size.width() / 2)), "px, "_s,
     374        std::round(bounds.center.y() - (bounds.size.height() / 2)), "px) "_s,
     375        bounds.angleInRadians ? makeString("rotate("_s, bounds.angleInRadians, "rad) "_s) : emptyString()
     376    ));
     377    return bounds;
    331378}
    332379
    … …  
    363410            continue;
    364411
    365         auto lineBounds = rotatedBoundingRectWithMinimumAngleOfRotation(lineQuad, 0.01);
    366         lineContainer->setInlineStyleProperty(CSSPropertyWidth, lineBounds.size.width(), CSSUnitType::CSS_PX);
    367         lineContainer->setInlineStyleProperty(CSSPropertyHeight, lineBounds.size.height(), CSSUnitType::CSS_PX);
    368         lineContainer->setInlineStyleProperty(CSSPropertyTransform, makeString(
    369             "translate("_s,
    370             std::round(lineBounds.center.x() - (lineBounds.size.width() / 2)), "px, "_s,
    371             std::round(lineBounds.center.y() - (lineBounds.size.height() / 2)), "px) "_s,
    372             lineBounds.angleInRadians ? makeString("rotate("_s, lineBounds.angleInRadians, "rad) "_s) : emptyString()
    373         ));
    374 
     412        auto lineBounds = fitElementToQuad(lineContainer.get(), lineQuad);
    375413        auto offsetAlongHorizontalAxis = [&](const FloatPoint& quadPoint1, const FloatPoint& quadPoint2) {
    376414            auto intervalLength = lineBounds.size.width();
    … …  
    452490            continue;
    453491
     492        auto firstQuad = dataDetector.normalizedQuads.first();
     493        if (firstQuad.isEmpty())
     494            continue;
     495
    454496        // FIXME: We should come up with a way to coalesce the bounding quads into one or more rotated rects with the same angle of rotation.
    455         auto targetQuad = convertToContainerCoordinates(dataDetector.normalizedQuads.first());
    456         auto targetBounds = rotatedBoundingRectWithMinimumAngleOfRotation(targetQuad, 0.01);
    457         dataDetectorContainer->setInlineStyleProperty(CSSPropertyWidth, targetBounds.size.width(), CSSUnitType::CSS_PX);
    458         dataDetectorContainer->setInlineStyleProperty(CSSPropertyHeight, targetBounds.size.height(), CSSUnitType::CSS_PX);
    459         dataDetectorContainer->setInlineStyleProperty(CSSPropertyTransform, makeString(
    460             "translate("_s,
    461             std::round(targetBounds.center.x() - (targetBounds.size.width() / 2)), "px, "_s,
    462             std::round(targetBounds.center.y() - (targetBounds.size.height() / 2)), "px) "_s,
    463             targetBounds.angleInRadians ? makeString("rotate("_s, targetBounds.angleInRadians, "rad) "_s) : emptyString()
    464         ));
     497        fitElementToQuad(dataDetectorContainer.get(), convertToContainerCoordinates(firstQuad));
    465498    }
    466499#endif // ENABLE(DATA_DETECTION)
     500
     501    ASSERT(result.blocks.size() == elements.blocks.size());
     502    for (size_t index = 0; index < result.blocks.size(); ++index) {
     503        auto& block = result.blocks[index];
     504        if (block.normalizedQuad.isEmpty())
     505            continue;
     506
     507        auto blockContainer = elements.blocks[index];
     508        auto bounds = fitElementToQuad(blockContainer.get(), convertToContainerCoordinates(block.normalizedQuad));
     509        // FIXME: We'll need a smarter algorithm here that chooses the largest font size for the container without
     510        // vertically overflowing the container.
     511        blockContainer->setInlineStyleProperty(CSSPropertyFontSize, std::round(0.8 * bounds.size.height()), CSSUnitType::CSS_PX);
     512    }
    467513
    468514    if (RefPtr frame = document->frame())

  • trunk/Source/WebCore/html/shadow/imageOverlay.css

    r282769 r285862  
    3131    text-shadow: none;
    3232    text-align: center;
     33    font-family: system-ui;
     34}
     35
     36div.image-overlay-line {
    3337    white-space: nowrap;
    3438    line-height: 100%;
    35     font-family: system-ui;
    3639    font-size: 1024px; /* This large font size is chosen to minimize gaps when painting selection quads. */
    3740}
    3841
    39 div.image-overlay-line, .image-overlay-text {
     42div.image-overlay-line, div.image-overlay-block {
     43    pointer-events: auto;
     44}
     45
     46div.image-overlay-block {
     47    background-color: rgba(255, 255, 255, 0.75);
     48    border-radius: calc(clamp(2px, 0.1em, 12px));
     49    box-shadow: rgba(100, 100, 100, 0.2) 3px 4px 8px 4px;
     50    color: rgb(90, 90, 90);
     51    font-weight: bold;
     52    display: flex;
     53    justify-content: center;
     54    align-content: center;
     55    flex-direction: column;
     56    -webkit-backdrop-filter: blur(8px);
     57}
     58
     59div.image-overlay-line, .image-overlay-text, div.image-overlay-block {
    4060    position: absolute;
    4161    overflow: hidden;
    42 }
    43 
    44 div.image-overlay-line {
    45     pointer-events: auto;
    4662}
    4763
Note: See TracChangeset for help on using the changeset viewer.