Making WaveVR/OculusVR controllers support haptic feedback (#2198)

* Making WaveVR controllers support haptic feedback.

* Making Oculus Quest support haptic feedback.

* Making Quest thumbRest always be false.

Author: daoshengmu

app/src/main/cpp/BrowserWorld.cpp

@@ -871,6 +871,7 @@ BrowserWorld::Draw() {
   m.device->ProcessEvents();
   m.context->Update();
   m.externalVR->PullBrowserState();
+  m.externalVR->SetHapticState(m.controllers);
 
   m.CheckExitImmersive();
   if (m.splashAnimation) {

app/src/main/cpp/Controller.cpp

@@ -58,6 +58,10 @@ Controller::operator=(const Controller& aController) {
   numButtons = aController.numButtons;
   memcpy(immersiveAxes, aController.immersiveAxes, sizeof(immersiveAxes));
   numAxes = aController.numAxes;
+  numHaptics = aController.numHaptics;
+  inputFrameID = aController.inputFrameID;
+  pulseDuration = aController.pulseDuration;
+  pulseIntensity = aController.pulseIntensity;
   leftHanded = aController.leftHanded;
   inDeadZone = aController.inDeadZone;
   lastHoverEvent = aController.lastHoverEvent;
@@ -91,6 +95,10 @@ Controller::Reset() {
   numButtons = 0;
   memset(immersiveAxes, 0, sizeof(immersiveAxes));
   numAxes = 0;
+  numHaptics = 0;
+  inputFrameID = 0;
+  pulseDuration = 0.0f;
+  pulseIntensity = 0.0f;
   leftHanded = false;
   inDeadZone = true;
   lastHoverEvent = 0.0;

app/src/main/cpp/Controller.h

@@ -51,6 +51,11 @@ struct Controller {
   uint32_t numButtons;
   float immersiveAxes[kControllerMaxAxes];
   uint32_t numAxes;
+  uint32_t numHaptics;
+  float inputFrameID;
+  float pulseDuration;
+  float pulseIntensity;
+
   bool leftHanded;
   bool inDeadZone;
   double lastHoverEvent;

app/src/main/cpp/ControllerContainer.cpp

@@ -341,6 +341,45 @@ ControllerContainer::SetAxes(const int32_t aControllerIndex, const float* aData,
   }
 }
 
+void
+ControllerContainer::SetHapticCount(const int32_t aControllerIndex, const uint32_t aNumHaptics) {
+  if (!m.Contains(aControllerIndex)) {
+    return;
+  }
+  m.list[aControllerIndex].numHaptics = aNumHaptics;
+}
+
+uint32_t
+ControllerContainer::GetHapticCount(const int32_t aControllerIndex) {
+  if (!m.Contains(aControllerIndex)) {
+    return 0;
+  }
+
+  return m.list[aControllerIndex].numHaptics;
+}
+
+void
+ControllerContainer::SetHapticFeedback(const int32_t aControllerIndex, const uint64_t aInputFrameID,
+                                        const float aPulseDuration, const float aPulseIntensity) {
+  if (!m.Contains(aControllerIndex)) {
+    return;
+  }
+  m.list[aControllerIndex].inputFrameID = aInputFrameID;
+  m.list[aControllerIndex].pulseDuration = aPulseDuration;
+  m.list[aControllerIndex].pulseIntensity = aPulseIntensity;
+}
+
+void
+ControllerContainer::GetHapticFeedback(const int32_t aControllerIndex, uint64_t & aInputFrameID,
+                                        float& aPulseDuration, float& aPulseIntensity) {
+  if (!m.Contains(aControllerIndex)) {
+    return;
+  }
+  aInputFrameID = m.list[aControllerIndex].inputFrameID;
+  aPulseDuration = m.list[aControllerIndex].pulseDuration;
+  aPulseIntensity = m.list[aControllerIndex].pulseIntensity;
+}
+
 void
 ControllerContainer::SetLeftHanded(const int32_t aControllerIndex, const bool aLeftHanded) {
   if (!m.Contains(aControllerIndex)) {

app/src/main/cpp/ControllerContainer.h

@@ -43,6 +43,10 @@ class ControllerContainer : public crow::ControllerDelegate  {
   void SetButtonCount(const int32_t aControllerIndex, const uint32_t aNumButtons) override;
   void SetButtonState(const int32_t aControllerIndex, const Button aWhichButton, const int32_t aImmersiveIndex, const bool aPressed, const bool aTouched, const float aImmersiveTrigger = -1.0f) override;
   void SetAxes(const int32_t aControllerIndex, const float* aData, const uint32_t aLength) override;
+  void SetHapticCount(const int32_t aControllerIndex, const uint32_t aNumHaptics) override;
+  uint32_t GetHapticCount(const int32_t aControllerIndex) override;
+  void SetHapticFeedback(const int32_t aControllerIndex, const uint64_t aInputFrameID, const float aPulseDuration, const float aPulseIntensity) override;
+  void GetHapticFeedback(const int32_t aControllerIndex, uint64_t &aInputFrameID, float& aPulseDuration, float& aPulseIntensity) override;
   void SetLeftHanded(const int32_t aControllerIndex, const bool aLeftHanded) override;
   void SetTouchPosition(const int32_t aControllerIndex, const float aTouchX, const float aTouchY) override;
   void EndTouch(const int32_t aControllerIndex) override;

app/src/main/cpp/ControllerDelegate.h

@@ -42,6 +42,10 @@ class ControllerDelegate {
   virtual void SetButtonCount(const int32_t aControllerIndex, const uint32_t aNumButtons) = 0;
   virtual void SetButtonState(const int32_t aControllerIndex, const Button aWhichButton, const int32_t aImmersiveIndex, const bool aPressed, const bool aTouched, const float aImmersiveTrigger = -1.0f) = 0;
   virtual void SetAxes(const int32_t aControllerIndex, const float* aData, const uint32_t aLength) = 0;
+  virtual void SetHapticCount(const int32_t aControllerIndex, const uint32_t aNumHaptics) = 0;
+  virtual uint32_t GetHapticCount(const int32_t aControllerIndex) = 0;
+  virtual void SetHapticFeedback(const int32_t aControllerIndex, const uint64_t aInputFrameID, const float aPulseDuration, const float aPulseIntensity) = 0;
+  virtual void GetHapticFeedback(const int32_t aControllerIndex, uint64_t & aInputFrameID, float& aPulseDuration, float& aPulseIntensity) = 0;
   virtual void SetLeftHanded(const int32_t aControllerIndex, const bool aLeftHanded) = 0;
   virtual void SetTouchPosition(const int32_t aControllerIndex, const float aTouchX, const float aTouchY) = 0;
   virtual void EndTouch(const int32_t aControllerIndex) = 0;

app/src/main/cpp/ExternalVR.cpp

@@ -412,6 +412,7 @@ ExternalVR::PushFramePoses(const vrb::Matrix& aHeadTransform, const std::vector<
     for (int i = 0; i< controller.numAxes; ++i) {
       immersiveController.axisValue[i] = controller.immersiveAxes[i];
     }
+    immersiveController.numHaptics = controller.numHaptics;
     immersiveController.hand = controller.leftHanded ? mozilla::gfx::ControllerHand::Left : mozilla::gfx::ControllerHand::Right;
 
     const uint16_t flags = GetControllerCapabilityFlags(controller.deviceCapabilities);
@@ -489,6 +490,26 @@ ExternalVR::GetFrameResult(int32_t& aSurfaceHandle, int32_t& aTextureWidth, int3
   aTextureHeight = (int32_t)m.browser.layerState[0].layer_stereo_immersive.textureSize.height;
 }
 
+void
+ExternalVR::SetHapticState(ControllerContainerPtr aControllerContainer) const {
+  const uint32_t count = aControllerContainer->GetControllerCount();
+  uint32_t i = 0, j = 0;
+  for (i = 0; i < count; ++i) {
+    for (j = 0; j < mozilla::gfx::kVRHapticsMaxCount; ++j) {
+      if (m.browser.hapticState[j].controllerIndex == i && m.browser.hapticState[j].inputFrameID) {
+        aControllerContainer->SetHapticFeedback(i, m.browser.hapticState[j].inputFrameID,
+                m.browser.hapticState[j].pulseDuration + m.browser.hapticState[j].pulseStart,
+                m.browser.hapticState[j].pulseIntensity);
+        break;
+      }
+    }
+    // All hapticState has already been reset to zero, so it can't be match.
+    if (j == mozilla::gfx::kVRHapticsMaxCount) {
+      aControllerContainer->SetHapticFeedback(i, 0, 0.0f, 0.0f);
+    }
+  }
+}
+
 void
 ExternalVR::StopPresenting() {
   m.system.displayState.presentingGeneration++;

app/src/main/cpp/ExternalVR.h

@@ -8,6 +8,7 @@
 
 #include "vrb/MacroUtils.h"
 #include "Controller.h"
+#include "ControllerContainer.h"
 #include "DeviceDelegate.h"
 #include "Device.h"
 #include <memory>
@@ -59,6 +60,7 @@ class ExternalVR : public ImmersiveDisplay {
                       int32_t& aTextureHeight,
                       device::EyeRect& aLeftEye,
                       device::EyeRect& aRightEye) const;
+  void SetHapticState(ControllerContainerPtr aControllerContainer) const;
   void StopPresenting();
   void SetSourceBrowser(VRBrowserType aBrowser);
   ExternalVR();

app/src/oculusvr/cpp/DeviceDelegateOculusVR.cpp

@@ -644,6 +644,9 @@ struct DeviceDelegateOculusVR::State {
     ovrInputTrackedRemoteCapabilities capabilities = {};
     vrb::Matrix transform = vrb::Matrix::Identity();
     ovrInputStateTrackedRemote inputState = {};
+    uint64_t inputFrameID = 0;
+    float remainingVibrateTime = 0.0f;
+    double lastHapticUpdateTimeStamp = 0.0f;
 
     bool Is6DOF() const {
       return (capabilities.ControllerCapabilities & ovrControllerCaps_HasPositionTracking) &&
@@ -692,7 +695,6 @@ struct DeviceDelegateOculusVR::State {
   device::CPULevel minCPULevel = device::CPULevel::Normal;
   device::DeviceType deviceType = device::UnknownType;
 
-
   void UpdatePerspective() {
     float fovX = vrapi_GetSystemPropertyFloat(&java, VRAPI_SYS_PROP_SUGGESTED_EYE_FOV_DEGREES_X);
     float fovY = vrapi_GetSystemPropertyFloat(&java, VRAPI_SYS_PROP_SUGGESTED_EYE_FOV_DEGREES_Y);
@@ -952,10 +954,13 @@ struct DeviceDelegateOculusVR::State {
             controller->CreateController(controllerState.index, int32_t(controllerState.hand),
                                          controllerName, beamTransform);
             controller->SetButtonCount(controllerState.index, 6);
+            controller->SetHapticCount(controllerState.index, 1);
           } else {
             // Oculus Go only has one kind of controller model.
             controller->CreateController(controllerState.index, 0, "Oculus Go Controller");
             controller->SetButtonCount(controllerState.index, 2);
+            // Oculus Go has no haptic feedback.
+            controller->SetHapticCount(controllerState.index, 0);
           }
           controllerState.created = true;
         }
@@ -1037,7 +1042,6 @@ struct DeviceDelegateOculusVR::State {
           const bool yTouched = (controllerState.inputState.Touches & ovrTouch_Y) != 0;
           const bool menuPressed = (controllerState.inputState.Buttons & ovrButton_Enter) != 0;
 
-
           controller->SetButtonState(controllerState.index, ControllerDelegate::BUTTON_X, 3, xPressed, xTouched);
           controller->SetButtonState(controllerState.index, ControllerDelegate::BUTTON_Y, 4, yPressed, yTouched);
 
@@ -1062,7 +1066,8 @@ struct DeviceDelegateOculusVR::State {
           VRB_WARN("Undefined hand type in DeviceDelegateOculusVR.");
         }
 
-        const bool thumbRest = (controllerState.inputState.Touches & ovrTouch_ThumbUp) != 0;
+        // This is always false in Oculus Browser.
+        const bool thumbRest = false;
         controller->SetButtonState(controllerState.index, ControllerDelegate::BUTTON_OTHERS, 5, thumbRest, thumbRest);
       } else {
         triggerPressed = (controllerState.inputState.Buttons & ovrButton_A) != 0;
@@ -1093,6 +1098,58 @@ struct DeviceDelegateOculusVR::State {
       controller->SetButtonState(controllerState.index, ControllerDelegate::BUTTON_TOUCHPAD, 0, trackpadPressed, trackpadTouched);
 
       controller->SetAxes(controllerState.index, axes, kNumAxes);
+      if (controller->GetHapticCount(controllerState.index)) {
+        UpdateHaptics(controllerState);
+      }
+    }
+  }
+
+  void UpdateHaptics(ControllerState& controllerState) {
+    vrb::RenderContextPtr renderContext = context.lock();
+    if (!renderContext) {
+      return;
+    }
+    if (!controller || !ovr) {
+      return;
+    }
+
+    uint64_t inputFrameID = 0;
+    float pulseDuration = 0.0f, pulseIntensity = 0.0f;
+    controller->GetHapticFeedback(controllerState.index, inputFrameID, pulseDuration, pulseIntensity);
+    if (inputFrameID > 0 && pulseIntensity > 0.0f && pulseDuration > 0) {
+      if (controllerState.inputFrameID != inputFrameID) {
+        // When there is a new input frame id from haptic vibration,
+        // that means we start a new session for a vibration.
+        controllerState.inputFrameID = inputFrameID;
+        controllerState.remainingVibrateTime = pulseDuration;
+        controllerState.lastHapticUpdateTimeStamp = renderContext->GetTimestamp();
+      } else {
+        // We are still running the previous vibration.
+        // So, it needs to reduce the delta time from the last vibration.
+        const double timeStamp = renderContext->GetTimestamp();
+        controllerState.remainingVibrateTime -= (timeStamp - controllerState.lastHapticUpdateTimeStamp);
+        controllerState.lastHapticUpdateTimeStamp = timeStamp;
+      }
+
+      if (controllerState.remainingVibrateTime > 0.0f && renderMode == device::RenderMode::Immersive) {
+        if (vrapi_SetHapticVibrationSimple(ovr, controllerState.deviceId, pulseIntensity > 1.0f ? 1.0f : pulseIntensity)
+            == ovrError_InvalidOperation) {
+          VRB_ERROR("vrapi_SetHapticVibrationBuffer failed.");
+        }
+      } else {
+        // The remaining time is zero or exiting the immersive mode, stop the vibration.
+        if (vrapi_SetHapticVibrationSimple(ovr, controllerState.deviceId, 0.0f) == ovrError_InvalidOperation) {
+          VRB_ERROR("vrapi_SetHapticVibrationBuffer failed.");
+        }
+        controllerState.remainingVibrateTime = 0.0f;
+      }
+    } else if (controllerState.remainingVibrateTime > 0.0f) {
+      // While the haptic feedback is terminated from the client side,
+      // but it still have remaining time, we need to ask for stopping vibration.
+      if (vrapi_SetHapticVibrationSimple(ovr, controllerState.deviceId, 0.0f) == ovrError_InvalidOperation) {
+        VRB_ERROR("vrapi_SetHapticVibrationBuffer failed.");
+      }
+      controllerState.remainingVibrateTime = 0.0f;
     }
   }
 

app/src/wavevr/cpp/DeviceDelegateWaveVR.cpp

@@ -45,6 +45,9 @@ struct DeviceDelegateWaveVR::State {
     int32_t gripPressedCount;
     vrb::Matrix transform;
     ElbowModel::HandEnum hand;
+    uint64_t inputFrameID;
+    float remainingVibrateTime;
+    double lastHapticUpdateTimeStamp;
     Controller()
         : index(-1)
           , type(WVR_DeviceType_Controller_Right)
@@ -55,6 +58,9 @@ struct DeviceDelegateWaveVR::State {
           , gripPressedCount(0)
           , transform(vrb::Matrix::Identity())
           , hand(ElbowModel::HandEnum::Right)
+          , inputFrameID(0)
+          , remainingVibrateTime(0.0f)
+          , lastHapticUpdateTimeStamp(0.0f)
     {}
   };
 
@@ -259,6 +265,7 @@ struct DeviceDelegateWaveVR::State {
     }
     delegate->CreateController(aController.index, aController.is6DoF ? 1 : 0, aController.is6DoF ? "HTC Vive Focus Plus Controller" : "HTC Vive Focus Controller", transform);
     delegate->SetLeftHanded(aController.index, aController.hand == ElbowModel::HandEnum::Left);
+    delegate->SetHapticCount(aController.index, 1);
     aController.created = true;
     aController.enabled = false;
   }
@@ -350,6 +357,62 @@ struct DeviceDelegateWaveVR::State {
         delegate->EndTouch(controller.index);
       }
       delegate->SetAxes(controller.index, immersiveAxes, kNumAxes);
+
+      UpdateHaptics(controller);
+    }
+  }
+
+  void UpdateHaptics(Controller& controller) {
+    vrb::RenderContextPtr renderContext = context.lock();
+    if (!renderContext) {
+      return;
+    }
+    if (!delegate) {
+      return;
+    }
+
+    uint64_t inputFrameID = 0;
+    float pulseDuration = 0.0f, pulseIntensity = 0.0f;
+    delegate->GetHapticFeedback(controller.index, inputFrameID, pulseDuration, pulseIntensity);
+    if (inputFrameID > 0 && pulseIntensity > 0.0f && pulseDuration > 0) {
+      if (controller.inputFrameID != inputFrameID) {
+        // When there is a new input frame id from haptic vibration,
+        // that means we start a new session for a vibration.
+        controller.inputFrameID = inputFrameID;
+        controller.remainingVibrateTime = pulseDuration;
+        controller.lastHapticUpdateTimeStamp = renderContext->GetTimestamp();
+      } else {
+        // We are still running the previous vibration.
+        // So, it needs to reduce the delta time from the last vibration.
+        const double timeStamp = renderContext->GetTimestamp();
+        controller.remainingVibrateTime -= (timeStamp - controller.lastHapticUpdateTimeStamp);
+        controller.lastHapticUpdateTimeStamp = timeStamp;
+      }
+
+      if (controller.remainingVibrateTime > 0.0f && renderMode == device::RenderMode::Immersive) {
+        // THe duration time unit needs to be transformed from milliseconds to microseconds.
+        // The gamepad extensions API does not yet have independent control
+        // of frequency and intensity. It only has vibration value (0.0 ~ 1.0).
+        // In this WaveVR SDK, the value makes more sense to be intensity because frequency can't
+        // < 1.0 here.
+        int intensity = ceil(pulseIntensity * 5);
+        intensity = intensity <= 5 ? intensity : 5;
+        WVR_TriggerVibration(controller.type, WVR_InputId_Max, controller.remainingVibrateTime * 1000.0f,
+                             1, WVR_Intensity(intensity));
+      } else {
+        // The remaining time is zero or exiting the immersive mode, stop the vibration.
+#if !defined(__arm__) // It will crash at WaveVR SDK arm32, let's skip it.
+        WVR_TriggerVibration(controller.type, WVR_InputId_Max, 0, 0, WVR_Intensity_Normal);
+#endif
+        controller.remainingVibrateTime = 0.0f;
+      }
+    } else if (controller.remainingVibrateTime > 0.0f) {
+      // While the haptic feedback is terminated from the client side,
+      // but it still have remaining time, we need to ask for stopping vibration.
+#if !defined(__arm__) // It will crash at WaveVR SDK arm32, let's skip it.
+      WVR_TriggerVibration(controller.type, WVR_InputId_Max, 0, 0, WVR_Intensity_Normal);
+#endif
+      controller.remainingVibrateTime = 0.0f;
     }
   }
 };