Adding Gesture Recognition to RenderWidgetHostViewWin (web client)

content/browser/renderer_host/render_widget_host_view_win.cc

 // Copyright (c) 2012 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 #include "content/browser/renderer_host/render_widget_host_view_win.h"
 
 #include <algorithm>
 #include <peninputpanel_i.c>
 
 #include "base/bind.h"
@@ skipping 28 matching lines @@
 #include "content/public/browser/notification_service.h"
 #include "content/public/browser/notification_types.h"
 #include "content/public/common/content_switches.h"
 #include "content/public/common/page_zoom.h"
 #include "content/public/common/process_type.h"
 #include "skia/ext/skia_utils_win.h"
 #include "third_party/skia/include/core/SkRegion.h"
 #include "third_party/WebKit/Source/WebKit/chromium/public/WebCompositionUnderline.h"
 #include "third_party/WebKit/Source/WebKit/chromium/public/WebInputEvent.h"
 #include "third_party/WebKit/Source/WebKit/chromium/public/win/WebInputEventFactory.h"
+#include "ui/base/gestures/gesture_recognizer.h"
 #include "ui/base/ime/composition_text.h"
 #include "ui/base/l10n/l10n_util_win.h"
 #include "ui/base/text/text_elider.h"
 #include "ui/base/ui_base_switches.h"
 #include "ui/base/view_prop.h"
 #include "ui/base/win/hwnd_util.h"
 #include "ui/base/win/mouse_wheel_util.h"
 #include "ui/gfx/canvas.h"
 #include "ui/gfx/gdi_util.h"
 #include "ui/gfx/rect.h"
@@ skipping 267 matching lines @@
       is_loading_(false),
       text_input_type_(ui::TEXT_INPUT_TYPE_NONE),
       is_fullscreen_(false),
       ignore_mouse_movement_(true),
       composition_range_(ui::Range::InvalidRange()),
       touch_state_(this),
       pointer_down_context_(false),
       focus_on_editable_field_(false),
       received_focus_change_after_pointer_down_(false),
       touch_events_enabled_(false),
+      ALLOW_THIS_IN_INITIALIZER_LIST(
+          gesture_recognizer_(ui::GestureRecognizer::Create(this))),
       ALLOW_THIS_IN_INITIALIZER_LIST(sys_color_change_listener_(this)) {
   render_widget_host_->SetView(this);
   registrar_.Add(this,
                  content::NOTIFICATION_RENDERER_PROCESS_TERMINATED,
                  content::NotificationService::AllBrowserContextsAndSources());
   registrar_.Add(this,
                  content::NOTIFICATION_FOCUS_CHANGED_IN_PAGE,
                  content::NotificationService::AllBrowserContextsAndSources());
 }
 
@@ skipping 583 matching lines @@
   callback.Run(CopyFromCompositingSurface(size, output));
 }
 
 void RenderWidgetHostViewWin::SetBackground(const SkBitmap& background) {
   content::RenderWidgetHostViewBase::SetBackground(background);
   render_widget_host_->SetBackground(background);
 }
 
 void RenderWidgetHostViewWin::ProcessTouchAck(
     WebKit::WebInputEvent::Type type, bool processed) {
+  scoped_ptr<ui::GestureRecognizer::Gestures> gestures;
+  gestures.reset(gesture_recognizer_->AdvanceTouchQueue(this, processed));
+  ProcessGestures(gestures.get());
+
   if (type == WebKit::WebInputEvent::TouchStart)
     UpdateDesiredTouchMode(processed);
 }
 
 void RenderWidgetHostViewWin::SetToGestureMode() {
   if (base::win::GetVersion() < base::win::VERSION_WIN7)
     return;
   UnregisterTouchWindow(m_hWnd);
   // Single finger panning is consistent with other windows applications.
   const DWORD gesture_allow = GC_PAN_WITH_SINGLE_FINGER_VERTICALLY |
@@ skipping 35 matching lines @@
   // state. If we want touch mode, then we attempt to register for touch
   // events, and otherwise to unregister.
   if (touch_mode) {
     touch_mode = SetToTouchMode();
   }
   if (!touch_mode) {
     SetToGestureMode();
   }
 }
 
+template <class IINTERFACE, class PAYLOAD>

[rjkroege, 2012/05/09 16:48:15]

I have a imprecise style-guide-kind-of-feeling that these embedded helper
classes should be up at the top of the file and not mixed in with the exposed
methods.

[girard, 2012/05/09 22:49:19]

Moved to the top of the file (in the local anonymous namespace.)

+class WrappedObject: public IINTERFACE {
+public:
+  WrappedObject() {
+  }
+  WrappedObject(const PAYLOAD &copy) :
+      data_(copy) {
+  }
+  const PAYLOAD& data() {
+    return data_;
+  }
+protected:
+  PAYLOAD data_;
+};
+
+class LocalGestureEvent :
+    public WrappedObject<ui::GestureEvent, WebKit::WebGestureEvent> {
+public:
+  LocalGestureEvent(
+      ui::EventType type,
+      const gfx::Point& location,
+      int flags,
+      base::Time time,
+      float param_first,
+      float param_second,
+      unsigned int touch_id_bitfield) :
+    type_(type),
+    touch_ids_bitfield_(touch_id_bitfield) {
+    data_.x = location.x();
+    data_.y = location.y();
+    data_.deltaX = param_first;
+    data_.deltaY = param_second;
+    data_.type = ConvertToWebKit(type);
+  }
+
+  virtual int GetLowestTouchId() const OVERRIDE {
+    if (touch_ids_bitfield_ == 0)
+      return -1;
+    int i = -1;
+    // Find the index of the least significant 1 bit
+    while (!((1 << ++i) & touch_ids_bitfield_));
+    return i;
+  }
+
+  ui::EventType type() {
+    return type_;
+  }
+private:
+  // The set of indices of ones in the binary representation of
+  // |touch_ids_bitfield_| is the set of touch_ids associate with this gesture.
+  // This value is stored as a bitfield because the number of touch ids varies,
+  // but we currently don't need more than 32 touches at a time.
+  const unsigned int touch_ids_bitfield_;
+
+  ui::EventType type_;
+  static ui::EventType ConvertToUI( WebKit::WebGestureEvent::Type t)

[rjkroege, 2012/05/09 16:48:15]

we have this same switch statement in more than one place. gack.

You could move a static helper into GestureHelper and then use the same set of
switches everywhere?

[girard, 2012/05/09 22:49:19]

Agree that we should combine these operations together.

+  {
+    switch (t) {
+    case WebKit::WebGestureEvent::GestureScrollBegin:
+      return ui::ET_GESTURE_SCROLL_BEGIN;
+    case WebKit::WebGestureEvent::GestureScrollEnd:
+      return ui::ET_GESTURE_SCROLL_END;
+    case WebKit::WebGestureEvent::GestureScrollUpdate:
+      return ui::ET_GESTURE_SCROLL_UPDATE;
+//    case WebKit::WebGestureEvent::GestureFlingStart:

[rjkroege, 2012/05/09 16:48:15]

this will break fling. why do you want to do that?

[girard, 2012/05/09 22:49:19]

There is no ui::ET_GESTURE_FLING_START.  Is it coming, or would
ui::ET_SCROLL_FLING_START make sense?

+//    case WebKit::WebGestureEvent::GestureFlingCancel:
+    case WebKit::WebGestureEvent::GestureTap:
+      return ui::ET_GESTURE_TAP;
+    case WebKit::WebGestureEvent::GestureTapDown:
+      return ui::ET_GESTURE_TAP_DOWN;
+    case WebKit::WebGestureEvent::GestureDoubleTap:
+      return ui::ET_GESTURE_DOUBLE_TAP;
+    case WebKit::WebGestureEvent::GestureLongPress:
+      return ui::ET_GESTURE_LONG_PRESS;
+    case WebKit::WebGestureEvent::GesturePinchBegin:
+      return ui::ET_GESTURE_PINCH_BEGIN;
+    case WebKit::WebGestureEvent::GesturePinchEnd:
+      return ui::ET_GESTURE_PINCH_END;
+    case WebKit::WebGestureEvent::GesturePinchUpdate:
+      return ui::ET_GESTURE_PINCH_UPDATE;
+    default:
+      // Unexpected type mapping!
+      return ui::ET_UNKNOWN;
+    }
+  }
+
+  static WebKit::WebGestureEvent::Type ConvertToWebKit( ui::EventType t)
+  {
+    switch (t) {
+    case ui::ET_UNKNOWN:
+      return WebKit::WebGestureEvent::Undefined;
+    case ui::ET_GESTURE_SCROLL_BEGIN:
+      return WebKit::WebGestureEvent::GestureScrollBegin;
+    case ui::ET_GESTURE_SCROLL_END:
+      return WebKit::WebGestureEvent::GestureScrollEnd;
+    case ui::ET_GESTURE_SCROLL_UPDATE:
+      return WebKit::WebGestureEvent::GestureScrollUpdate;
+//      return WebKit::WebGestureEvent::GestureFlingStart:
+//      return WebKit::WebGestureEvent::GestureFlingCancel:
+    case ui::ET_GESTURE_TAP:
+      return WebKit::WebGestureEvent::GestureTap;
+    case ui::ET_GESTURE_TAP_DOWN:
+      return WebKit::WebGestureEvent::GestureTapDown;
+    case ui::ET_GESTURE_DOUBLE_TAP:
+      return WebKit::WebGestureEvent::GestureDoubleTap;
+    case ui::ET_GESTURE_LONG_PRESS:
+      return WebKit::WebGestureEvent::GestureLongPress;
+    case ui::ET_GESTURE_PINCH_BEGIN:
+      return WebKit::WebGestureEvent::GesturePinchBegin;
+    case ui::ET_GESTURE_PINCH_END:
+      return WebKit::WebGestureEvent::GesturePinchEnd;
+    case ui::ET_GESTURE_PINCH_UPDATE:
+      return WebKit::WebGestureEvent::GesturePinchUpdate;
+    default:
+      // Unexpected type mapping!

[rjkroege, 2012/05/09 16:48:15]

add DCHECK instead.

[girard, 2012/05/09 22:49:19]

Done.

+      return WebKit::WebGestureEvent::Undefined;
+    }
+  }
+};
+
+ui::GestureEvent* RenderWidgetHostViewWin::CreateGestureEvent(
+    ui::EventType type,
+    const gfx::Point& location,
+    int flags,
+    base::Time time,
+    float param_first,
+    float param_second,
+    unsigned int touch_id_bitfield) {
+
+  return new LocalGestureEvent(type, location, flags, time,
+      param_first, param_second, touch_id_bitfield);
+}
+
+class LocalTouchEvent :
+    public WrappedObject<ui::TouchEvent, WebKit::WebTouchEvent> {
+ public:
+  LocalTouchEvent(
+      ui::EventType type,
+      const gfx::Point& location,
+      int touch_id,
+      base::TimeDelta time_stamp) :
+        location_(location),
+        touch_id_(touch_id),
+        time_stamp_(time_stamp) {
+    data_.type = ConvertToWebKit(type);
+  }
+
+  LocalTouchEvent(const WebKit::WebTouchEvent& copy,
+      ui::EventType type,
+      base::TimeDelta time_stamp) :
+    WrappedObject<ui::TouchEvent, WebKit::WebTouchEvent>(copy),
+    time_stamp_(time_stamp) {
+    data_.type = ConvertToWebKit(type);
+    // This is an experiment!
+    touch_id_ = copy.touches[0].id;
+    location_ = gfx::Point(0,0);
+    data_.type = copy.type;
+  }
+
+  virtual ui::EventType GetEventType() const OVERRIDE {

[rjkroege, 2012/05/09 16:48:15]

Unfortunately, these would appear to have to be virtual. Add a TODO to fix this
post http://crbug.com/125937

[girard, 2012/05/11 14:21:49]

Done.

+    return ConvertToUI(data_.type);
+  }
+  virtual gfx::Point GetLocation() const OVERRIDE {
+    return location_;
+  }
+  virtual int GetTouchId() const OVERRIDE {
+    return touch_id_;
+  }
+  virtual int GetEventFlags() const OVERRIDE {
+    return 0;
+  }
+  virtual base::TimeDelta GetTimestamp() const OVERRIDE {
+    return time_stamp_;
+  }
+
+  // Returns a copy of this touch event. Used when queueing events for
+  // asynchronous gesture recognition.
+  virtual TouchEvent* Copy() const OVERRIDE {
+    return new LocalTouchEvent(
+        GetEventType(),GetLocation(),GetTouchId(),GetTimestamp());
+  }
+ private:
+  int touch_id_;
+  base::TimeDelta time_stamp_;
+  gfx::Point location_;
+
+  static ui::EventType ConvertToUI( WebKit::WebTouchEvent::Type t)
+  {
+    switch (t) {
+    case WebKit::WebInputEvent::Undefined:
+      return ui::ET_UNKNOWN;
+    case WebKit::WebInputEvent::TouchStart:
+      return ui::ET_TOUCH_PRESSED;
+    case WebKit::WebInputEvent::TouchMove:
+      //return ui::ET_TOUCH_STATIONARY;
+      return ui::ET_TOUCH_MOVED;
+    case WebKit::WebInputEvent::TouchEnd:
+      return ui::ET_TOUCH_RELEASED;
+    case WebKit::WebInputEvent::TouchCancel:
+      return ui::ET_TOUCH_CANCELLED;
+    default:
+      // Unexpected type mapping!
+      return ui::ET_UNKNOWN;
+    }
+  }
+
+  static WebKit::WebTouchEvent::Type ConvertToWebKit( ui::EventType t)
+  {
+    switch (t) {
+    case ui::ET_UNKNOWN:
+      return WebKit::WebInputEvent::Undefined;
+    case ui::ET_TOUCH_PRESSED:
+      return WebKit::WebInputEvent::TouchStart;
+    case ui::ET_TOUCH_MOVED:
+    case ui::ET_TOUCH_STATIONARY:
+      return WebKit::WebInputEvent::TouchMove;
+    case ui::ET_TOUCH_RELEASED:
+      return WebKit::WebInputEvent::TouchEnd;
+    case ui::ET_TOUCH_CANCELLED:
+      return WebKit::WebInputEvent::TouchCancel;
+    default:
+      // Unexpected ui type mapping!
+      return WebKit::WebInputEvent::Undefined;
+    }
+  }
+};
+
+ui::TouchEvent* RenderWidgetHostViewWin::CreateTouchEvent(
+    ui::EventType type,
+    const gfx::Point& location,
+    int touch_id,
+    base::TimeDelta time_stamp) {
+  return new LocalTouchEvent( type, location, touch_id, time_stamp);
+}
+
+bool RenderWidgetHostViewWin::DispatchLongPressGestureEvent(
+    ui::GestureEvent* event) {
+  return ForwardGestureEventToRenderer(WebKit::WebInputEvent::GestureLongPress,
+      event);
+}
+
+bool RenderWidgetHostViewWin::DispatchCancelTouchEvent(
+    ui::TouchEvent* event) {
+  if (!render_widget_host_)
+    return false;
+
+  WebKit::WebTouchEvent generatedEvent;
+  generatedEvent.changedTouchesLength = 1;
+  generatedEvent.changedTouches[0].id = event->GetTouchId();
+  generatedEvent.type = WebKit::WebInputEvent::TouchCancel;
+  render_widget_host_->ForwardTouchEvent(generatedEvent);
+  return true;
+}
+
 void RenderWidgetHostViewWin::SetHasHorizontalScrollbar(
     bool has_horizontal_scrollbar) {
 }
 
 void RenderWidgetHostViewWin::SetScrollOffsetPinning(
     bool is_pinned_to_left, bool is_pinned_to_right) {
 }
 
 ///////////////////////////////////////////////////////////////////////////////
 // RenderWidgetHostViewWin, private:
@@ skipping 887 matching lines @@
       static_cast<int>(WebKit::WebTouchEvent::touchesLengthCap));
   TOUCHINPUT points[WebKit::WebTouchEvent::touchesLengthCap];
 
   if (!total || !GetTouchInputInfo((HTOUCHINPUT)lparam, total,
                                    points, sizeof(TOUCHINPUT))) {
     return 0;
   }
 
   for (size_t start = 0; start < total;) {
     start += touch_state_.UpdateTouchPoints(points + start, total - start);
-    if (touch_state_.is_changed())
+    if (touch_state_.is_changed()) {
       render_widget_host_->ForwardTouchEvent(touch_state_.touch_event());

[sadrul, 2012/05/10 01:49:26]

Perhaps this should also check 'if (render_widget_host_->has_touch_handler())'
before forwarding the touch-event. If the touch-event is forwarded (i.e. host
has touch-handlers), then the events should be queued, otherwise they should be
processed immediately.

[girard, 2012/05/11 14:21:49]

Thanks!  Done.

+    }
+  }
+
+  // Send a copy of the touch events on to the gesture recognizer.
+  for (size_t index = 0; index < total; ++index) {
+    //bool hasAreaInfo =

[rjkroege, 2012/05/09 16:48:15]

remove dead code

[girard, 2012/05/09 22:49:19]

Done.

+    //    (points[index].dwMask & TOUCHINPUTMASKF_CONTACTAREA) != 0;
+    base::TimeDelta time_stamp =
+        base::TimeDelta::FromMilliseconds(points[index].dwTime);
+    DWORD flags = points[index].dwFlags;
+    ui::EventType eventType = ui::ET_TOUCH_STATIONARY;
+
+    if ((flags & TOUCHEVENTF_UP) != 0)
+      eventType = ui::ET_TOUCH_RELEASED;
+    else if ((flags & TOUCHEVENTF_MOVE) != 0) {
+      eventType = ui::ET_TOUCH_MOVED;
+      // Ignore this touch if there is a backlog.

[rjkroege, 2012/05/09 16:48:15]

I don't recall you answering me as to why there would be.

[girard, 2012/05/09 22:49:19]

On my machine, sending all messages on to the GR greatly degraded the operation
of the GR.  It would often reverse the direction of scrolls, or lag badly, or
miss gestures completely.

[sadrul, 2012/05/10 01:49:26]

Note: because this is dropping some of the touch-events, the GR can generate
unexpected gestures (or unexpected values for gesture parameters, e.g. fling
velocities might be off).

[girard, 2012/05/11 14:21:49]

After some more testing, I determined that this issue only came up when I was
running under a debugger.   As such I'm removing the entire "IsQueueEmpty"
subsystem.

+      if (!gesture_recognizer_->IsQueueEmpty(this))
+        continue;
+    }
+    else if ((flags & TOUCHEVENTF_DOWN) != 0)
+      eventType = ui::ET_TOUCH_PRESSED;
+
+    ui::TouchEvent *touchEvent = CreateTouchEvent(
+        eventType,
+        gfx::Point(
+        TOUCH_COORD_TO_PIXEL( points[index].x),
+        TOUCH_COORD_TO_PIXEL( points[index].y)),
+        points[index].dwID,
+        time_stamp);
+    GestureConsumer* consumer = this;
+    gesture_recognizer_->QueueTouchEventForGesture(
+      consumer, *touchEvent);
   }
 
   CloseTouchInputHandle((HTOUCHINPUT)lparam);
 
   return 0;
 }
 
+void RenderWidgetHostViewWin::ProcessGestures(
+    ui::GestureRecognizer::Gestures* gestures) {
+  if ((gestures == 0) || gestures->empty())
+    return;
+  for (ui::GestureRecognizer::Gestures::iterator g_it = gestures->begin();

[rjkroege, 2012/05/09 16:48:15]

given the switches above, can't you just send all the gestures. They are
WebGestureEvents already. I am puzzled.

[girard, 2012/05/09 22:49:19]

Good point.  This code can be removed.

[girard, 2012/05/09 22:49:19]

Done.

+      g_it != gestures->end();
+      ++g_it) {
+    LocalGestureEvent* gesture =
+        reinterpret_cast<LocalGestureEvent *>( *g_it);
+    switch (gesture->type()) {
+    case ui::ET_GESTURE_SCROLL_BEGIN:
+      ForwardGestureEventToRenderer(WebKit::WebInputEvent::GestureScrollBegin,
+          gesture);
+      break;
+    case ui::ET_GESTURE_SCROLL_END:
+      ForwardGestureEventToRenderer(WebKit::WebInputEvent::GestureScrollEnd,
+          gesture);
+      break;
+    case ui::ET_GESTURE_SCROLL_UPDATE:
+      ForwardGestureEventToRenderer(WebKit::WebInputEvent::GestureScrollUpdate,
+          gesture);
+      break;
+    case ui::ET_GESTURE_PINCH_BEGIN:
+      ForwardGestureEventToRenderer(WebKit::WebInputEvent::GesturePinchBegin,
+          gesture);
+      break;
+    case ui::ET_GESTURE_PINCH_END:
+      ForwardGestureEventToRenderer(WebKit::WebInputEvent::GesturePinchEnd,
+          gesture);
+      break;
+    case ui::ET_GESTURE_PINCH_UPDATE:
+      ForwardGestureEventToRenderer(WebKit::WebInputEvent::GesturePinchUpdate,
+          gesture);
+      break;
+    case ui::ET_GESTURE_LONG_PRESS:
+      ForwardGestureEventToRenderer(WebKit::WebInputEvent::GestureLongPress,
+          gesture);
+      break;
+    case ui::ET_GESTURE_THREE_FINGER_SWIPE:

[rjkroege, 2012/05/09 16:48:15]

why not forward it anyway? If no events are being generated, then, you could
simplify this code?

[girard, 2012/05/09 22:49:19]

Forward as what kind of WebGestureEvent? Are there plans to add
WebInputEvent::Type::GestureThreeFingerSwipe?

+      // TODO: Add this code when three finger swipes are added.
+      //ForwardGestureEventToRenderer(
+      //    WebKit::WebInputEvent::GestureThreeFingerSwipe, gesture);
+      break;
+    case ui::ET_GESTURE_TAP:
+      ForwardGestureEventToRenderer(WebKit::WebInputEvent::GestureTap,
+          gesture);
+      break;
+    case ui::ET_GESTURE_TAP_DOWN:
+      ForwardGestureEventToRenderer(WebKit::WebInputEvent::GestureTapDown,
+            gesture);
+      break;
+    case ui::ET_GESTURE_DOUBLE_TAP:
+      ForwardGestureEventToRenderer(
+            WebKit::WebInputEvent::GestureDoubleTap, gesture);
+      break;
+    }
+  }
+}
+
 LRESULT RenderWidgetHostViewWin::OnMouseActivate(UINT message,
                                                  WPARAM wparam,
                                                  LPARAM lparam,
                                                  BOOL& handled) {
   if (!render_widget_host_)
     return MA_NOACTIVATE;
 
   if (!IsActivatable())
     return MA_NOACTIVATE;
 
@@ skipping 523 matching lines @@
   GetClientRect(&ti.rect);
   SendMessage(tooltip_hwnd_, message, NULL, reinterpret_cast<LPARAM>(&ti));
 }
 
 void RenderWidgetHostViewWin::ResetTooltip() {
   if (::IsWindow(tooltip_hwnd_))
     ::DestroyWindow(tooltip_hwnd_);
   tooltip_hwnd_ = NULL;
 }
 
+bool RenderWidgetHostViewWin::ForwardGestureEventToRenderer(
+    WebKit::WebInputEvent::Type eventType,
+    ui::GestureEvent* gesture) {
+  if (!render_widget_host_)
+    return false;
+
+  LocalGestureEvent *local = reinterpret_cast<LocalGestureEvent *>( gesture);
+  const WebKit::WebGestureEvent& generatedEvent = local->data();
+  DCHECK(generatedEvent.type == eventType);
+  render_widget_host_->ForwardGestureEvent(generatedEvent);
+  return true;
+}
+
 void RenderWidgetHostViewWin::ForwardMouseEventToRenderer(UINT message,
                                                           WPARAM wparam,
                                                           LPARAM lparam) {
   if (!render_widget_host_)
     return;
 
   WebMouseEvent event(
       WebInputEventFactory::mouseEvent(m_hWnd, message, wparam, lparam));
 
   if (mouse_locked_) {
@@ skipping 232 matching lines @@
 void RenderWidgetHostViewWin::ResetPointerDownContext() {
   // If the default focus on the page is on an edit field and we did not
   // receive a focus change in the context of a pointer down message, it means
   // that the pointer down message occurred on the edit field and we should
   // display the on screen keyboard
   if (!received_focus_change_after_pointer_down_ && virtual_keyboard_)
     DisplayOnScreenKeyboardIfNeeded();
   received_focus_change_after_pointer_down_ = false;
   pointer_down_context_ = false;
 }