Changeset 207153 in webkit


Ignore:
Timestamp:
Oct 11, 2016 11:38:49 AM (7 years ago)
Author:
commit-queue@webkit.org
Message:

Extend event stream to include interpolated events and add a force press test that uses that interpolation
https://bugs.webkit.org/show_bug.cgi?id=163161

Patch by Megan Gardner <Megan Gardner> on 2016-10-11
Reviewed by Simon Fraser.

Added functionality to the event stream to allow for interpolated events.
Can now do long press, as well as a better way to do drag and other time-based
events that require a large stream of descrete HID events.
Added a basic force touch test to demostrate this interpolation.
Also updated the script to allow for iPhone 7 specific tests, as force touch
needs to be on a device that had force touch.

  • Scripts/webkitpy/port/ios.py:

(IOSSimulatorPort):

  • TestRunnerShared/UIScriptContext/Bindings/UIScriptController.idl:
  • WebKitTestRunner/ios/HIDEventGenerator.mm:

(linearInterpolation):
(simpleCurveInterpolation):
(calculateNextCurveLocation):
(phaseFromString):
(interpolationFromString):
(-[HIDEventGenerator eventMaskFromEventInfo:]):
(-[HIDEventGenerator _createIOHIDEventWithInfo:]):
(-[HIDEventGenerator moveToPoints:touchCount:duration:]):
(-[HIDEventGenerator interpolatedEvents:]):
(-[HIDEventGenerator processEventsArray:withStartTime:]):
(-[HIDEventGenerator eventDispatchThreadEntry:]):
(simpleDragCurve): Deleted.
(calculateNextLocation): Deleted.

Location:
trunk
Files:
3 added
4 edited

Legend:

Unmodified
Added
Removed
  • trunk/Tools/ChangeLog

    r207152 r207153  
     12016-10-11  Megan Gardner  <megan_gardner@apple.com>
     2
     3        Extend event stream to include interpolated events and add a force press test that uses that interpolation
     4        https://bugs.webkit.org/show_bug.cgi?id=163161
     5
     6        Reviewed by Simon Fraser.
     7
     8        Added functionality to the event stream to allow for interpolated events.
     9        Can now do long press, as well as a better way to do drag and other time-based
     10        events that require a large stream of descrete HID events.
     11        Added a basic force touch test to demostrate this interpolation.
     12        Also updated the script to allow for iPhone 7 specific tests, as force touch
     13        needs to be on a device that had force touch.
     14
     15        * Scripts/webkitpy/port/ios.py:
     16        (IOSSimulatorPort):
     17        * TestRunnerShared/UIScriptContext/Bindings/UIScriptController.idl:
     18        * WebKitTestRunner/ios/HIDEventGenerator.mm:
     19        (linearInterpolation):
     20        (simpleCurveInterpolation):
     21        (calculateNextCurveLocation):
     22        (phaseFromString):
     23        (interpolationFromString):
     24        (-[HIDEventGenerator eventMaskFromEventInfo:]):
     25        (-[HIDEventGenerator _createIOHIDEventWithInfo:]):
     26        (-[HIDEventGenerator moveToPoints:touchCount:duration:]):
     27        (-[HIDEventGenerator interpolatedEvents:]):
     28        (-[HIDEventGenerator processEventsArray:withStartTime:]):
     29        (-[HIDEventGenerator eventDispatchThreadEntry:]):
     30        (simpleDragCurve): Deleted.
     31        (calculateNextLocation): Deleted.
     32
    1332016-10-11  Alex Christensen  <achristensen@webkit.org>
    234
  • trunk/Tools/Scripts/webkitpy/port/ios.py

    r206940 r207153  
    7575
    7676    DEFAULT_DEVICE_CLASS = 'iphone'
    77     CUSTOM_DEVICE_CLASSES = ['ipad']
     77    CUSTOM_DEVICE_CLASSES = ['ipad', 'iphone7']
    7878    SDK = 'iphonesimulator'
    7979
     
    8787        'x86_64': {
    8888            'iphone': 'iPhone 5s',
     89            'iphone7': 'iPhone 7',
    8990            'ipad': 'iPad Air'
    9091        },
  • trunk/Tools/TestRunnerShared/UIScriptContext/Bindings/UIScriptController.idl

    r206798 r207153  
    6868    //          },
    6969    //          {
     70    //              "interpolate" : "linear",
     71    //              "timestep" : 0.025,
     72    //              "startEvent" : {
     73    //                  "inputType" : "hand",
     74    //                  "timeOffset" : 0.025,
     75    //                  "touches" : [
     76    //                      {
     77    //                          "inputType" : "finger",
     78    //                          "phase" : "began",
     79    //                          "id" : 1,
     80    //                          "x" : 100,
     81    //                          "y" : 120,
     82    //                          "pressure" : 0
     83    //                      }
     84    //                  ]
     85    //              },
     86    //              "endEvent" : {
     87    //                  "inputType" : "hand",
     88    //                  "timeOffset" : 3.0,
     89    //                  "touches" : [
     90    //                      {
     91    //                          "inputType" : "finger",
     92    //                          "phase" : "stationary",
     93    //                          "id" : 1,
     94    //                          "x" : 20,
     95    //                          "y" : 40,
     96    //                          "pressure" : 500
     97    //                      }
     98    //                  ]
     99    //              }
     100    //          },
     101    //          {
    70102    //              "inputType" : "hand",
    71103    //              "timeOffset" : 0.002, // seconds relative to the first event
  • trunk/Tools/WebKitTestRunner/ios/HIDEventGenerator.mm

    r206798 r207153  
    4343NSString* const HIDEventTouchesKey = @"touches";
    4444NSString* const HIDEventPhaseKey = @"phase";
     45NSString* const HIDEventInterpolateKey = @"interpolate";
     46NSString* const HIDEventTimestepKey = @"timestep";
     47NSString* const HIDEventStartEventKey = @"startEvent";
     48NSString* const HIDEventEndEventKey = @"endEvent";
    4549NSString* const HIDEventTouchIDKey = @"id";
    4650NSString* const HIDEventPressureKey = @"pressure";
     
    5559NSString* const HIDEventInputTypeStylus = @"stylus";
    5660
     61NSString* const HIDEventInterpolationTypeLinear = @"linear";
     62NSString* const HIDEventInterpolationTypeSimpleCurve = @"simpleCurve";
     63
    5764NSString* const HIDEventPhaseBegan = @"began";
     65NSString* const HIDEventPhaseStationary = @"stationary";
    5866NSString* const HIDEventPhaseMoved = @"moved";
    5967NSString* const HIDEventPhaseEnded = @"ended";
     
    7078
    7179static int fingerIdentifiers[maxTouchCount] = { 2, 3, 4, 5, 1 };
     80
     81typedef enum {
     82    InterpolationTypeLinear,
     83    InterpolationTypeSimpleCurve,
     84} InterpolationType;
    7285
    7386typedef enum {
     
    99112}
    100113
    101 static double simpleDragCurve(double a, double b, double t)
     114static double linearInterpolation(double a, double b, double t)
     115{
     116    return (a + (b - a) * t );
     117}
     118
     119static double simpleCurveInterpolation(double a, double b, double t)
    102120{
    103121    return (a + (b - a) * sin(sin(t * M_PI / 2) * t * M_PI / 2));
    104122}
    105123
    106 static CGPoint calculateNextLocation(CGPoint a, CGPoint b, CFTimeInterval t)
    107 {
    108     return CGPointMake(simpleDragCurve(a.x, b.x, t), simpleDragCurve(a.y, b.y, t));
    109 }
     124
     125static CGPoint calculateNextCurveLocation(CGPoint a, CGPoint b, CFTimeInterval t)
     126{
     127    return CGPointMake(simpleCurveInterpolation(a.x, b.x, t), simpleCurveInterpolation(a.y, b.y, t));
     128}
     129
     130typedef double(*pressureInterpolationFunction)(double, double, CFTimeInterval);
     131static pressureInterpolationFunction interpolations[] = {
     132    linearInterpolation,
     133    simpleCurveInterpolation,
     134};
    110135
    111136static void delayBetweenMove(int eventIndex, double elapsed)
     
    188213    if ([string isEqualToString:HIDEventPhaseBegan])
    189214        return UITouchPhaseBegan;
     215   
     216    if ([string isEqualToString:HIDEventPhaseStationary])
     217        return UITouchPhaseStationary;
    190218
    191219    if ([string isEqualToString:HIDEventPhaseMoved])
     
    201229}
    202230
     231static InterpolationType interpolationFromString(NSString *string)
     232{
     233    if ([string isEqualToString:HIDEventInterpolationTypeLinear])
     234        return InterpolationTypeLinear;
     235   
     236    if ([string isEqualToString:HIDEventInterpolationTypeSimpleCurve])
     237        return InterpolationTypeSimpleCurve;
     238   
     239    return InterpolationTypeLinear;
     240}
     241
    203242- (IOHIDDigitizerEventMask)eventMaskFromEventInfo:(NSDictionary *)info
    204243{
     244    IOHIDDigitizerEventMask eventMask = 0;
    205245    NSArray *childEvents = info[HIDEventTouchesKey];
    206246    for (NSDictionary *touchInfo in childEvents) {
     
    208248        // If there are any new or ended events, mask includes touch.
    209249        if (phase == UITouchPhaseBegan || phase == UITouchPhaseEnded || phase == UITouchPhaseCancelled)
    210             return kIOHIDDigitizerEventTouch;
    211     }
    212    
    213     return 0;
    214 }
    215 
    216 // Returns 1 for all events where the fingers are on the glass (everything but enced and canceled).
     250            eventMask |= kIOHIDDigitizerEventTouch;
     251        // If there are any pressure readings, set mask must include attribute
     252        if ([touchInfo[HIDEventPressureKey] floatValue])
     253            eventMask |= kIOHIDDigitizerEventAttribute;
     254    }
     255   
     256    return eventMask;
     257}
     258
     259// Returns 1 for all events where the fingers are on the glass (everything but ended and canceled).
    217260- (CFIndex)touchFromEventInfo:(NSDictionary *)info
    218261{
     
    261304
    262305        UITouchPhase phase = phaseFromString(touchInfo[HIDEventPhaseKey]);
    263         if (phase != UITouchPhaseCancelled && phase != UITouchPhaseBegan && phase != UITouchPhaseEnded)
     306        if (phase != UITouchPhaseCancelled && phase != UITouchPhaseBegan && phase != UITouchPhaseEnded && phase != UITouchPhaseStationary)
    264307            childEventMask |= kIOHIDDigitizerEventPosition;
    265308
     
    269312        if (phase == UITouchPhaseCancelled)
    270313            childEventMask |= kIOHIDDigitizerEventCancel;
     314       
     315        if ([touchInfo[HIDEventPressureKey] floatValue])
     316            childEventMask |= kIOHIDDigitizerEventAttribute;
    271317
    272318        IOHIDEventRef subEvent = IOHIDEventCreateDigitizerFingerEvent(kCFAllocatorDefault, machTime,
     
    552598                startLocations[i] = _activePoints[i].point;
    553599
    554             nextLocations[i] = calculateNextLocation(startLocations[i], newLocations[i], interval);
     600            nextLocations[i] = calculateNextCurveLocation(startLocations[i], newLocations[i], interval);
    555601        }
    556602        [self _updateTouchPoints:nextLocations count:touchCount];
     
    921967}
    922968
     969- (NSArray *)interpolatedEvents:(NSDictionary *)interpolationsDictionary
     970{
     971    NSDictionary *startEvent = interpolationsDictionary[HIDEventStartEventKey];
     972    NSDictionary *endEvent = interpolationsDictionary[HIDEventEndEventKey];
     973    NSTimeInterval timeStep = [interpolationsDictionary[HIDEventTimestepKey] doubleValue];
     974    InterpolationType interpolationType = interpolationFromString(interpolationsDictionary[HIDEventInterpolateKey]);
     975   
     976    NSMutableArray *interpolatedEvents = [NSMutableArray arrayWithObject:startEvent];
     977   
     978    NSTimeInterval startTime = [startEvent[HIDEventTimeOffsetKey] doubleValue];
     979    NSTimeInterval endTime = [endEvent[HIDEventTimeOffsetKey] doubleValue];
     980    NSTimeInterval time = startTime + timeStep;
     981   
     982    NSArray *startTouches = startEvent[HIDEventTouchesKey];
     983    NSArray *endTouches = endEvent[HIDEventTouchesKey];
     984   
     985    while (time < endTime) {
     986        NSMutableDictionary *newEvent = [endEvent mutableCopy];
     987        double timeRatio = (time - startTime) / (endTime - startTime);
     988        newEvent[HIDEventTimeOffsetKey] = [NSNumber numberWithDouble:(time)];
     989       
     990        NSEnumerator *startEnumerator = [startTouches objectEnumerator];
     991        NSDictionary *startTouch;
     992        NSMutableArray *newTouches = [NSMutableArray arrayWithCapacity:[endTouches count]];
     993        while (startTouch = [startEnumerator nextObject])  {
     994            NSEnumerator *endEnumerator = [endTouches objectEnumerator];
     995            NSDictionary *endTouch = [endEnumerator nextObject];
     996            NSInteger startTouchID = [startTouch[HIDEventTouchIDKey] integerValue];
     997           
     998            while (endTouch && ([endTouch[HIDEventTouchIDKey] integerValue] != startTouchID))
     999                endTouch = [endEnumerator nextObject];
     1000           
     1001            if (endTouch) {
     1002                NSMutableDictionary *newTouch = [endTouch mutableCopy];
     1003               
     1004                if (newTouch[HIDEventXKey] != startTouch[HIDEventXKey])
     1005                    newTouch[HIDEventXKey] = @(interpolations[interpolationType]([startTouch[HIDEventXKey] doubleValue], [endTouch[HIDEventXKey] doubleValue], timeRatio));
     1006               
     1007                if (newTouch[HIDEventYKey] != startTouch[HIDEventYKey])
     1008                    newTouch[HIDEventYKey] = @(interpolations[interpolationType]([startTouch[HIDEventYKey] doubleValue], [endTouch[HIDEventYKey] doubleValue], timeRatio));
     1009               
     1010                if (newTouch[HIDEventPressureKey] != startTouch[HIDEventPressureKey])
     1011                    newTouch[HIDEventPressureKey] = @(interpolations[interpolationType]([startTouch[HIDEventPressureKey] doubleValue], [endTouch[HIDEventPressureKey] doubleValue], timeRatio));
     1012               
     1013                [newTouches addObject:newTouch];
     1014                [newTouch release];
     1015            } else
     1016                NSLog(@"Missing End Touch with ID: %ld", (long)startTouchID);
     1017        }
     1018       
     1019        newEvent[HIDEventTouchesKey] = newTouches;
     1020       
     1021        [interpolatedEvents addObject:newEvent];
     1022        [newEvent release];
     1023        time += timeStep;
     1024    }
     1025
     1026    return interpolatedEvents;
     1027}
     1028
     1029- (NSArray *)expandEvents:(NSArray *)events withStartTime:(CFAbsoluteTime)startTime
     1030{
     1031    NSMutableArray *expandedEvents = [NSMutableArray array];
     1032    for (NSDictionary *event in events) {
     1033        NSString *interpolate = event[HIDEventInterpolateKey];
     1034        // we have key events that we need to generate
     1035        if (interpolate) {
     1036            NSArray *newEvents = [self interpolatedEvents:event];
     1037            [expandedEvents addObjectsFromArray:[self expandEvents:newEvents withStartTime:startTime]];
     1038        } else
     1039            [expandedEvents addObject:event];
     1040    }
     1041    return expandedEvents;
     1042}
     1043
    9231044- (void)eventDispatchThreadEntry:(NSDictionary *)threadData
    9241045{
    9251046    NSDictionary *eventStream = threadData[@"eventInfo"];
    9261047    void (^completionBlock)() = threadData[@"completionBlock"];
    927 
     1048   
    9281049    NSArray *events = eventStream[TopLevelEventInfoKey];
    9291050    if (!events.count) {
     
    9311052        return;
    9321053    }
    933 
     1054   
    9341055    CFAbsoluteTime startTime = CFAbsoluteTimeGetCurrent();
    9351056   
    936     for (NSDictionary *eventInfo in events) {
     1057    NSArray *expandedEvents = [self expandEvents:events withStartTime:startTime];
     1058   
     1059    for (NSDictionary *eventInfo in expandedEvents) {
    9371060        NSTimeInterval eventRelativeTime = [eventInfo[HIDEventTimeOffsetKey] doubleValue];
    9381061        CFAbsoluteTime targetTime = startTime + eventRelativeTime;
     
    9411064        if (waitTime > 0)
    9421065            [NSThread sleepForTimeInterval:waitTime];
    943 
     1066       
    9441067        dispatch_async(dispatch_get_main_queue(), ^ {
    9451068            [self dispatchEventWithInfo:eventInfo];
    9461069        });
    9471070    }
    948 
     1071   
    9491072    dispatch_async(dispatch_get_main_queue(), ^ {
    9501073        [self _sendMarkerHIDEventWithCompletionBlock:completionBlock];
Note: See TracChangeset for help on using the changeset viewer.