Skip to content
This repository was archived by the owner on Jul 22, 2024. It is now read-only.

Commit 15eae62

Browse files
keianhzobluemarvin
keianhzo
and
bluemarvin
authored
Pico Gaze Mode support (#2959)
* Pico Gaze Mode support * PR comments updates * Rebase fixes * UPdate gazeIndex update in container Co-authored-by: Randall E. Barker <[email protected]>
1 parent 25cca5a commit 15eae62

File tree

11 files changed

+181
-51
lines changed

11 files changed

+181
-51
lines changed

‎app/src/main/cpp/BrowserWorld.cpp

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -189,6 +189,7 @@ struct BrowserWorld::State {
189189
std::unordered_map<vrb::Node*, std::pair<Widget*, float>> depthSorting;
190190
std::function<void(device::Eye)> drawHandler;
191191
std::function<void()> frameEndHandler;
192+
bool wasInGazeMode = false;
192193

193194
State() : paused(true), glInitialized(false), modelsLoaded(false), env(nullptr), cylinderDensity(0.0f), nearClip(0.1f),
194195
farClip(300.0f), activity(nullptr), windowsInitialized(false), exitImmersiveRequested(false), loaderDelay(0) {
@@ -215,6 +216,7 @@ struct BrowserWorld::State {
215216
splashAnimation = SplashAnimation::Create(create);
216217
monitor = PerformanceMonitor::Create(create);
217218
monitor->AddPerformanceMonitorObserver(std::make_shared<PerformanceObserver>());
219+
wasInGazeMode = false;
218220
#if defined(WAVEVR)
219221
monitor->SetPerformanceDelta(15.0);
220222
#endif
@@ -224,6 +226,7 @@ struct BrowserWorld::State {
224226
bool CheckExitImmersive();
225227
void EnsureControllerFocused();
226228
void ChangeControllerFocus(const Controller& aController);
229+
void UpdateGazeModeState();
227230
void UpdateControllers(bool& aRelayoutWidgets);
228231
WidgetPtr GetWidget(int32_t aHandle) const;
229232
WidgetPtr FindWidget(const std::function<bool(const WidgetPtr&)>& aCondition) const;
@@ -336,6 +339,25 @@ ScaleScrollDelta(const float aValue, const double aStartTime, const double aCurr
336339
return aValue * scale;
337340
}
338341

342+
void
343+
BrowserWorld::State::UpdateGazeModeState() {
344+
bool isInGazeMode = device->IsInGazeMode();
345+
if (isInGazeMode != wasInGazeMode) {
346+
int32_t gazeIndex = device->GazeModeIndex();
347+
if (isInGazeMode && gazeIndex >= 0) {
348+
VRB_LOG("Gaze mode ON")
349+
controllers->SetEnabled(gazeIndex, true);
350+
controllers->SetVisible(gazeIndex, true);
351+
352+
} else {
353+
VRB_LOG("Gaze mode OFF")
354+
controllers->SetEnabled(gazeIndex, false);
355+
controllers->SetVisible(gazeIndex, false);
356+
}
357+
wasInGazeMode = isInGazeMode;
358+
}
359+
}
360+
339361
void
340362
BrowserWorld::State::UpdateControllers(bool& aRelayoutWidgets) {
341363
EnsureControllerFocused();
@@ -711,6 +733,7 @@ BrowserWorld::RegisterDeviceDelegate(DeviceDelegatePtr aDelegate) {
711733
m.leftCamera = m.device->GetCamera(device::Eye::Left);
712734
m.rightCamera = m.device->GetCamera(device::Eye::Right);
713735
ControllerDelegatePtr delegate = m.controllers;
736+
delegate->SetGazeModeIndex(m.device->GazeModeIndex());
714737
m.device->SetClipPlanes(m.nearClip, m.farClip);
715738
m.device->SetControllerDelegate(delegate);
716739
m.gestures = m.device->GetGestureDelegate();
@@ -893,6 +916,7 @@ BrowserWorld::StartFrame() {
893916
TickImmersive();
894917
} else {
895918
bool relayoutWidgets = false;
919+
m.UpdateGazeModeState();
896920
m.UpdateControllers(relayoutWidgets);
897921
if (relayoutWidgets) {
898922
UpdateVisibleWidgets();

‎app/src/main/cpp/ControllerContainer.cpp

Lines changed: 32 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ struct ControllerContainer::State {
3535
GeometryPtr beamModel;
3636
bool visible = false;
3737
vrb::Color pointerColor;
38+
int gazeIndex = -1;
3839

3940
void Initialize(vrb::CreationContextPtr& aContext) {
4041
context = aContext;
@@ -201,33 +202,36 @@ ControllerContainer::CreateController(const int32_t aControllerIndex, const int3
201202
controller.transform = Transform::Create(create);
202203
controller.pointer = Pointer::Create(create);
203204
controller.pointer->SetVisible(true);
204-
if ((m.models.size() >= aModelIndex) && m.models[aModelIndex]) {
205-
controller.transform->AddNode(m.models[aModelIndex]);
206-
controller.beamToggle = vrb::Toggle::Create(create);
207-
controller.beamToggle->ToggleAll(true);
208-
if (aBeamTransform.IsIdentity()) {
209-
controller.beamParent = controller.beamToggle;
205+
if (aControllerIndex != m.gazeIndex) {
206+
if ((m.models.size() >= aModelIndex) && m.models[aModelIndex]) {
207+
controller.transform->AddNode(m.models[aModelIndex]);
208+
controller.beamToggle = vrb::Toggle::Create(create);
209+
controller.beamToggle->ToggleAll(true);
210+
if (aBeamTransform.IsIdentity()) {
211+
controller.beamParent = controller.beamToggle;
212+
} else {
213+
vrb::TransformPtr beamTransform = Transform::Create(create);
214+
beamTransform->SetTransform(aBeamTransform);
215+
controller.beamParent = beamTransform;
216+
controller.beamToggle->AddNode(beamTransform);
217+
}
218+
controller.transform->AddNode(controller.beamToggle);
219+
if (m.beamModel && controller.beamParent) {
220+
controller.beamParent->AddNode(m.beamModel);
221+
}
210222
} else {
211-
vrb::TransformPtr beamTransform = Transform::Create(create);
212-
beamTransform->SetTransform(aBeamTransform);
213-
controller.beamParent = beamTransform;
214-
controller.beamToggle->AddNode(beamTransform);
215-
}
216-
controller.transform->AddNode(controller.beamToggle);
217-
if (m.beamModel && controller.beamParent) {
218-
controller.beamParent->AddNode(m.beamModel);
223+
VRB_ERROR("Failed to add controller model");
219224
}
220-
if (m.root) {
221-
m.root->AddNode(controller.transform);
222-
m.root->ToggleChild(*controller.transform, false);
223-
}
224-
if (m.pointerContainer) {
225-
m.pointerContainer->AddNode(controller.pointer->GetRoot());
226-
}
227-
m.updatePointerColor(controller);
228-
} else {
229-
VRB_ERROR("Failed to add controller model");
230225
}
226+
227+
if (m.root) {
228+
m.root->AddNode(controller.transform);
229+
m.root->ToggleChild(*controller.transform, false);
230+
}
231+
if (m.pointerContainer) {
232+
m.pointerContainer->AddNode(controller.pointer->GetRoot());
233+
}
234+
m.updatePointerColor(controller);
231235
}
232236

233237
void
@@ -458,6 +462,10 @@ ControllerContainer::SetVisible(const bool aVisible) {
458462
}
459463
}
460464

465+
void ControllerContainer::SetGazeModeIndex(const int32_t aControllerIndex) {
466+
m.gazeIndex = aControllerIndex;
467+
}
468+
461469
ControllerContainer::ControllerContainer(State& aState, vrb::CreationContextPtr& aContext) : m(aState) {
462470
m.Initialize(aContext);
463471
}

‎app/src/main/cpp/ControllerContainer.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ class ControllerContainer : public crow::ControllerDelegate {
5454
void SetScrolledDelta(const int32_t aControllerIndex, const float aScrollDeltaX, const float aScrollDeltaY) override;
5555
void SetPointerColor(const vrb::Color& color) const;
5656
void SetVisible(const bool aVisible);
57+
void SetGazeModeIndex(const int32_t aControllerIndex) override;
5758
protected:
5859
struct State;
5960
ControllerContainer(State& aState, vrb::CreationContextPtr& aContext);

‎app/src/main/cpp/ControllerDelegate.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ class ControllerDelegate {
5151
virtual void SetTouchPosition(const int32_t aControllerIndex, const float aTouchX, const float aTouchY) = 0;
5252
virtual void EndTouch(const int32_t aControllerIndex) = 0;
5353
virtual void SetScrolledDelta(const int32_t aControllerIndex, const float aScrollDeltaX, const float aScrollDeltaY) = 0;
54+
virtual void SetGazeModeIndex(const int32_t aControllerIndex) = 0;
5455
protected:
5556
ControllerDelegate() {}
5657
private:

‎app/src/main/cpp/DeviceDelegate.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,8 @@ class DeviceDelegate {
6666
virtual void StartFrame() = 0;
6767
virtual void BindEye(const device::Eye aWhich) = 0;
6868
virtual void EndFrame(bool aDiscard = false) = 0;
69+
virtual bool IsInGazeMode() const { return false; };
70+
virtual int32_t GazeModeIndex() const { return -1; };
6971
virtual VRLayerQuadPtr CreateLayerQuad(int32_t aWidth, int32_t aHeight,
7072
VRLayerSurface::SurfaceType aSurfaceType) { return nullptr; }
7173
virtual VRLayerQuadPtr CreateLayerQuad(const VRLayerSurfacePtr& aMoveLayer) { return nullptr; }

‎app/src/main/cpp/ExternalVR.cpp

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -414,12 +414,12 @@ ExternalVR::PushFramePoses(const vrb::Matrix& aHeadTransform, const std::vector<
414414
immersiveController.numButtons = controller.numButtons;
415415
immersiveController.buttonPressed = controller.immersivePressedState;
416416
immersiveController.buttonTouched = controller.immersiveTouchedState;
417-
for (int i = 0; i< controller.numButtons; ++i) {
418-
immersiveController.triggerValue[i] = controller.immersiveTriggerValues[i];
417+
for (int j = 0; j < controller.numButtons; ++j) {
418+
immersiveController.triggerValue[j] = controller.immersiveTriggerValues[j];
419419
}
420420
immersiveController.numAxes = controller.numAxes;
421-
for (int i = 0; i< controller.numAxes; ++i) {
422-
immersiveController.axisValue[i] = controller.immersiveAxes[i];
421+
for (int j = 0; j < controller.numAxes; ++j) {
422+
immersiveController.axisValue[j] = controller.immersiveAxes[j];
423423
}
424424
immersiveController.numHaptics = controller.numHaptics;
425425
immersiveController.hand = controller.leftHanded ? mozilla::gfx::ControllerHand::Left : mozilla::gfx::ControllerHand::Right;

‎app/src/picovr/cpp/DeviceDelegatePicoVR.cpp

Lines changed: 68 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -29,9 +29,10 @@ namespace crow {
2929

3030
static const vrb::Vector kAverageHeight(0.0f, 1.7f, 0.0f);
3131
// TODO: support different controllers & buttons
32-
static const int32_t kMaxControllerCount = 2;
32+
static const int32_t kMaxControllerCount = 3;
3333
static const int32_t kNumButtons = 6;
3434
static const int32_t kNumG2Buttons = 2;
35+
static const int32_t kNumGazeButtons = 2;
3536
static const int32_t kNumAxes = 2;
3637
static const int32_t k6DofHeadSet = 1;
3738
static const int32_t kButtonApp = 1;
@@ -93,6 +94,8 @@ struct DeviceDelegatePicoVR::State {
9394
float fov = (float) (51.0 * M_PI / 180.0);
9495
int32_t focusIndex = 0;
9596
bool recentered = false;
97+
bool isInGazeMode = false;
98+
int32_t gazeIndex = -1;
9699

97100
void Initialize() {
98101
vrb::RenderContextPtr localContext = context.lock();
@@ -103,6 +106,8 @@ struct DeviceDelegatePicoVR::State {
103106
UpdatePerspective();
104107
UpdateEyeTransform();
105108

109+
gazeIndex = VRBrowserPico::GetGazeIndex();
110+
106111
for (int32_t index = 0; index < kMaxControllerCount; index++) {
107112
controllers[index].index = index;
108113
if (index == 0) {
@@ -211,12 +216,21 @@ struct DeviceDelegatePicoVR::State {
211216

212217
vrb::Matrix transform = controller.transform;
213218
if (renderMode == device::RenderMode::StandAlone) {
214-
if (type == k6DofHeadSet) {
215-
transform.TranslateInPlace(headOffset);
219+
if (isInGazeMode) {
220+
if (i == gazeIndex) {
221+
vrb::Matrix head = vrb::Matrix::Rotation(orientation);
222+
head.PreMultiplyInPlace(vrb::Matrix::Position(headOffset));
223+
controller.transform = head;
224+
transform = controller.transform;
225+
}
216226
} else {
217-
vrb::Matrix head = vrb::Matrix::Rotation(orientation);
218-
head.PreMultiplyInPlace(vrb::Matrix::Position(headOffset));
219-
transform = elbow->GetTransform(controller.hand, head, transform);
227+
if (type == k6DofHeadSet) {
228+
transform.TranslateInPlace(headOffset);
229+
} else {
230+
vrb::Matrix head = vrb::Matrix::Rotation(orientation);
231+
head.PreMultiplyInPlace(vrb::Matrix::Position(headOffset));
232+
transform = elbow->GetTransform(controller.hand, head, transform);
233+
}
220234
}
221235
}
222236

@@ -227,6 +241,10 @@ struct DeviceDelegatePicoVR::State {
227241
}
228242
}
229243
}
244+
245+
int32_t GazeModeIndex() {
246+
return gazeIndex;
247+
}
230248
};
231249

232250
DeviceDelegatePicoVRPtr
@@ -315,18 +333,25 @@ DeviceDelegatePicoVR::SetControllerDelegate(ControllerDelegatePtr& aController)
315333
for (State::Controller& controller: m.controllers) {
316334
const int32_t index = controller.index;
317335

318-
if (m.type == k6DofHeadSet) {
319-
vrb::Matrix beam = vrb::Matrix::Rotation(vrb::Vector(1.0f, 0.0f, 0.0f), -vrb::PI_FLOAT / 11.5f);
320-
beam.TranslateInPlace(vrb::Vector(0.0f, 0.012f, -0.06f));
321-
m.controllerDelegate->CreateController(index, int32_t(controller.hand), controller.IsRightHand() ? "Pico Neo 2 (Right)" : "Pico Neo 2 (LEFT)", beam);
322-
m.controllerDelegate->SetButtonCount(index, kNumButtons);
323-
m.controllerDelegate->SetHapticCount(index, 1);
324-
} else {
325-
vrb::Matrix beam = vrb::Matrix::Rotation(vrb::Vector(1.0f, 0.0f, 0.0f), -vrb::PI_FLOAT / 11.5f);
326-
327-
m.controllerDelegate->CreateController(index, 0, "Pico G2 Controller", beam);
328-
m.controllerDelegate->SetButtonCount(index, kNumG2Buttons);
336+
if (index == m.gazeIndex) {
337+
vrb::Matrix beam = vrb::Matrix::Identity();
338+
m.controllerDelegate->CreateController(index, 0, "Pico Gaze Controller", beam);
339+
m.controllerDelegate->SetButtonCount(index, kNumGazeButtons);
329340
m.controllerDelegate->SetHapticCount(index, 0);
341+
342+
} else {
343+
if (m.type == k6DofHeadSet) {
344+
vrb::Matrix beam = vrb::Matrix::Rotation(vrb::Vector(1.0f, 0.0f, 0.0f), -vrb::PI_FLOAT / 11.5f);
345+
beam.TranslateInPlace(vrb::Vector(0.0f, 0.012f, -0.06f));
346+
m.controllerDelegate->CreateController(index, int32_t(controller.hand), controller.IsRightHand() ? "Pico Neo 2 (Right)" : "Pico Neo 2 (LEFT)", beam);
347+
m.controllerDelegate->SetButtonCount(index, kNumButtons);
348+
m.controllerDelegate->SetHapticCount(index, 1);
349+
} else {
350+
vrb::Matrix beam = vrb::Matrix::Rotation(vrb::Vector(1.0f, 0.0f, 0.0f), -vrb::PI_FLOAT / 11.5f);
351+
m.controllerDelegate->CreateController(index, 0, "Pico G2 Controller", beam);
352+
m.controllerDelegate->SetButtonCount(index, kNumG2Buttons);
353+
m.controllerDelegate->SetHapticCount(index, 0);
354+
}
330355
}
331356
controller.created = true;
332357
}
@@ -377,6 +402,22 @@ DeviceDelegatePicoVR::StartFrame() {
377402

378403
m.cameras[0]->SetHeadTransform(head);
379404
m.cameras[1]->SetHeadTransform(head);
405+
406+
407+
// Update te gaze mode state based on controllers availability
408+
m.isInGazeMode = true;
409+
for (int32_t i = 0; i < m.controllers.size(); ++i) {
410+
if (i != m.gazeIndex && m.controllers[i].enabled) {
411+
m.isInGazeMode = false;
412+
break;
413+
}
414+
}
415+
416+
if (m.isInGazeMode) {
417+
m.controllers[m.gazeIndex].enabled = m.isInGazeMode;
418+
m.controllers[m.gazeIndex].transform = GetHeadTransform();
419+
}
420+
380421
m.UpdateControllers();
381422
}
382423

@@ -392,6 +433,16 @@ DeviceDelegatePicoVR::EndFrame(const bool aDiscard) {
392433

393434
}
394435

436+
bool
437+
DeviceDelegatePicoVR::IsInGazeMode() const {
438+
return m.isInGazeMode;
439+
}
440+
441+
int32_t
442+
DeviceDelegatePicoVR::GazeModeIndex() const {
443+
return m.gazeIndex;
444+
}
445+
395446
void
396447
DeviceDelegatePicoVR::Pause() {
397448
m.paused = true;

‎app/src/picovr/cpp/DeviceDelegatePicoVR.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,8 @@ class DeviceDelegatePicoVR : public DeviceDelegate {
4040
void StartFrame() override;
4141
void BindEye(const device::Eye aWhich) override;
4242
void EndFrame(const bool aDiscard) override;
43+
bool IsInGazeMode() const override;
44+
int32_t GazeModeIndex() const override;
4345
// Custom methods
4446
void Pause();
4547
void Resume();

‎app/src/picovr/cpp/VRBrowserPico.cpp

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,8 @@ VRBrowserPico::InitializeJava(JNIEnv* aEnv, jobject aActivity) {
3939
return;
4040
}
4141

42+
sGetGazeIndex = FindJNIMethodID(sEnv, sBrowserClass, kGetGazeIndex, kGetGazeIndexSignature);
43+
4244
sUpdateHaptics = FindJNIMethodID(sEnv, sBrowserClass, sUpdateHapticsName, sUpdateHapticsSignature);
4345
sCancelAllHaptics = FindJNIMethodID(sEnv, sBrowserClass, sCancelAllHapticsName, sCancelAllHapticsSignature);
4446
}
@@ -73,4 +75,13 @@ VRBrowserPico::CancelAllHaptics() {
7375
CheckJNIException(sEnv, __FUNCTION__);
7476
}
7577

78+
int32_t
79+
VRBrowserPico::GetGazeIndex() {
80+
if (!ValidateMethodID(sEnv, sActivity, sGetGazeIndex, __FUNCTION__)) { return -1; }
81+
jint jGazeIndex = (jint) sEnv->CallIntMethod(sActivity, sGetGazeIndex);
82+
CheckJNIException(sEnv, __FUNCTION__);
83+
84+
return (int32_t )jGazeIndex;
85+
}
86+
7687
} // namespace crow

‎app/src/picovr/cpp/VRBrowserPico.h

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -11,13 +11,21 @@
1111
#include <jni.h>
1212
#include <functional>
1313

14+
namespace {
15+
const char *kGetGazeIndex = "getGazeIndex";
16+
const char *kGetGazeIndexSignature = "()I";
17+
18+
jmethodID sGetGazeIndex = nullptr;
19+
}
20+
1421
namespace crow {
1522

16-
namespace VRBrowserPico {
17-
void InitializeJava(JNIEnv* aEnv, jobject aActivity);
18-
void ShutdownJava();
19-
void UpdateHaptics(jint aControllerIndex, jfloat aIntensity, jfloat aDuration);
20-
void CancelAllHaptics();
21-
} // namespace VRBrowser;
23+
namespace VRBrowserPico {
24+
void InitializeJava(JNIEnv* aEnv, jobject aActivity);
25+
void ShutdownJava();
26+
void UpdateHaptics(jint aControllerIndex, jfloat aIntensity, jfloat aDuration);
27+
void CancelAllHaptics();
28+
int32_t GetGazeIndex();
29+
} // namespace VRBrowser;
2230

2331
} // namespace crow

0 commit comments

Comments
 (0)