diff --git a/core/js/bici.js b/core/js/bici.js index 2c87110..628d3f3 100644 --- a/core/js/bici.js +++ b/core/js/bici.js @@ -37,7 +37,7 @@ let coreFiles = `M4, pca, loadImage, webgl, webcam, trackHead, help, midi, numberString, pen, aiScriptPanel, keyEvent, matchCurves, glyphs, chalktalk, codeArea, math, shape, shader, diagram, sliders, widgets, webrtc-client, video-ui, implicit, - mediapipe, gesture, shadowHand, tracking, main`.split(','); + mediapipe, gesture, shadowHand, tracking, grabableObjects, main`.split(','); let project, slideData; let coreLoaded = false; diff --git a/core/js/gesture.js b/core/js/gesture.js index 6098921..10fd411 100644 --- a/core/js/gesture.js +++ b/core/js/gesture.js @@ -176,7 +176,6 @@ class PinchGesture extends HandGesture { this.state[h] = newState; } - _onStart(hand) { this.updateState(hand); super._onStart(hand); @@ -193,8 +192,67 @@ class PinchGesture extends HandGesture { } } +class RotationGesture extends HandGesture { + constructor(id, maxDistance = 0.15, activationThreshold = 1, activeCooldown = 33) { + let detectSpread = (hand) => { + const scaleFac = Math.min(1, .1 / (.2 + hand.landmarks[4].z)); + let distances = getFingerThumbDistances([1, 2, 3, 4], hand); + return distances.every((d) => d > maxDistance * scaleFac); + } + super(id, activationThreshold, activeCooldown, detectSpread); + } + + /** + * Orientation of the object being decided by the orientation of the palm in space + * @param {*} hand Hand data returned from mediapipe + */ + updateState(hand) { + const h = hand.handedness; + + // Figuring out plane of hand. Used to figure out axis of rotation + const pWrist = hand.landmarks[0]; + const pThumbTip = hand.landmarks[4]; + const pPinkyTip = hand.landmarks[20]; + + const vecThumbToLast = normalize(subtract(pointToArray(pPinkyTip), pointToArray(pThumbTip))); + const palmNormal = normalize( + cross( + subtract(pointToArray(pPinkyTip), pointToArray(pWrist)), + subtract(pointToArray(pThumbTip), pointToArray(pWrist)) + ) + ); + // Vector perpendicular to the above two vectors. Forms a basis. + const vecPerp = normalize(cross(palmNormal, vecThumbToLast)); + + this.state[h].rotationMatrix = [ + vecThumbToLast[0], vecPerp[0], palmNormal[0], 0, + vecThumbToLast[1], vecPerp[1], palmNormal[1], 0, + vecThumbToLast[2], vecPerp[2], palmNormal[2], 0, + 0, 0, 0, 1 + ]; + + this.state[h].refFinger = pThumbTip; + } + + _onStart(hand) { + this.updateState(hand); + super._onStart(hand); + } + + _onEnd(hand) { + super._onEnd(hand); + this.state[hand.handedness] = {}; + } + + _onActive(hand) { + this.updateState(hand); + super._onActive(hand); + } +} + +let pointToArray = p => [ p.x, p.y, p.z ]; + function getFingerThumbDistances(fingers, hand) { - let pointToArray = p => [ p.x, p.y, p.z ]; let distances = []; const thumbPt = pointToArray(hand.landmarks[4]); diff --git a/core/js/grabableObjects.js b/core/js/grabableObjects.js new file mode 100644 index 0000000..a5dd918 --- /dev/null +++ b/core/js/grabableObjects.js @@ -0,0 +1,122 @@ +class GrabableObject { + // Supports cubic hitboxes for now + constructor(mesh, color, x, y, z, lx, ly, lz) { + this.pos = {x: x, y: y, z: z}; + this.bounds = {lx: lx, ly: ly, lz: lz}; + // Default scaling and rotations + this.scale = 1; + this.aim = identity(); + this.mesh = mesh; + this.color = color; + } + + setScale(scale) { + this.scale = scale; + } + + drawObject() { + // ToDo: Replace using matrix stack in utils + let m = + mxm( + mxm( + mxm( + move(this.pos.x * 2 - 1, this.pos.y * 2 - 1, this.pos.z * 2 - 1), + scale(0.4 + 0.3 * this.pos.z) + ), + scale(this.scale) + ), + this.aim + ); + + drawObj(this.mesh, m, this.color); + } + + getEffectiveSize() { + const scale = this.scale * (0.4 + 0.3 * this.pos.z); + return { + lx: this.bounds.lx * scale, + ly: this.bounds.ly * scale, + lz: this.bounds.lz * scale + } + } + + isHitting(obj2) { + const size1 = this.getEffectiveSize(); + const size2 = obj2.getEffectiveSize(); + + return this.pos.x - size1.lx/2 < obj2.pos.x + size2.lx/2 + && this.pos.x + size1.lx/2 > obj2.pos.x - size2.lx/2 + && this.pos.y - size1.ly/2 < obj2.pos.y + size2.ly/2 + && this.pos.y + size1.ly/2 > obj2.pos.y - size2.ly/2 + && this.pos.z - size1.lz/2 < obj2.pos.z + size2.lz/2 + && this.pos.z + size1.lz/2 > obj2.pos.z - size2.lz/2; + } +} + +class ObjectTracker { + constructor() { + this.objs = []; + this.currSelection = null; + } + + addObj(mesh, color, x, y, z, lx, ly, lz) { + let obj = new GrabableObject(mesh, color, x, y, z, lx, ly, lz); + this.objs.push(obj); + } + + drawObjs() { + for (const obj of this.objs) { + obj.drawObject(); + } + } + + onPinch(x, y, z, handedness) { + if (this.currSelection) { + return; + } + + for (const obj of this.objs) { + let dx = Math.abs(obj.pos.x - x); + let dy = Math.abs(obj.pos.y - y); + if (norm([dx, dy, 0]) <= 0.3) { + this.currSelection = {object: obj, hand: handedness} + break; + } + } + } + + onLeave(handedness) { + if (this.currSelection?.hand === handedness) { + this.currSelection = null; + } + } + + onDrag(x, y, z, handedness) { + if (this.currSelection?.hand === handedness) { + const prevPos = {...this.currSelection.object.pos}; + this.currSelection.object.pos = {x: x, y: y, z: z}; + for (const obj of this.objs) { + if (obj === this.currSelection.object) { + continue; + } + if (this.currSelection.object.isHitting(obj)) { + this.currSelection.object.pos = prevPos; + } + } + } + } + + rescale(vector, otherHand) { + const holderHand = otherHand == "left" ? "right" : "left"; + if (this.currSelection?.hand === holderHand) { + this.currSelection.object.scale = 10 * norm(vector);; + } + } + + onRotate(rMatrix, otherHand) { + const holderHand = otherHand == "left" ? "right" : "left"; + if (this.currSelection?.hand === holderHand) { + this.currSelection.object.aim = rMatrix; + } + } +} \ No newline at end of file diff --git a/core/js/main.js b/core/js/main.js index a90ca5a..ea4c5b5 100644 --- a/core/js/main.js +++ b/core/js/main.js @@ -69,6 +69,29 @@ let canvas3D_up = (x,y,z,id='id') => { scene.onUp(xToScene(x), yToScene(y), zToScene(z), id); } +let canvas3D_rescale = (state, otherHand, id) => { + if (! canvas3D.isDown) + canvas3D.isDown = {}; + + if (scene && scene.rescale && canvas3D.isDown[id]) { + scene.rescale( + [state.left.x, state.left.y, state.left.z], + [state.right.x, state.right.y, state.right.z], + otherHand + ); + } +} + +let canvas3D_rotate = (rMatrix, handedness, id='id') => { + if (!canvas3D.isDown) + canvas3D.isDown = {}; + + if (scene && scene.onRotate && canvas3D.isDown[id]) { + console.log("rotating"); + scene.onRotate(rMatrix, handedness, id); + } +} + canvas3D.addEventListener('mousemove', event => canvas3D_move(event.clientX, event.clientY, 0, "mouse")); canvas3D.addEventListener('mousedown', event => canvas3D_down(event.clientX, event.clientY, 0, "mouse")); canvas3D.addEventListener('mouseup' , event => canvas3D_up (event.clientX, event.clientY, 0, "mouse")); diff --git a/core/js/tracking.js b/core/js/tracking.js index 61c9d28..8a20c89 100644 --- a/core/js/tracking.js +++ b/core/js/tracking.js @@ -645,7 +645,7 @@ let initializeGestureTracking = () => { } }; - indexPinch.onActive = ({state, id}, hand) => { + indexPinch.onActive = ({state, isActive, id}, hand) => { const h = hand.handedness; const {x, y, z} = toScreen(state[h], h); @@ -662,8 +662,11 @@ let initializeGestureTracking = () => { const eventId = `${id}.${state[h].pointer}`; if(state[h].pointer === 'head') { canvas3D_move(headX, headY, 0, eventId) + } else if ((isActive.left??false) && (isActive.right??false)) { + // ToDo: Find an appropriate place to check for two handed gestures + canvas3D_rescale(state, h, eventId); } else { - canvas3D_move(x, y, z, eventId) + canvas3D_move(x, y, z, eventId); } }; @@ -735,10 +738,35 @@ let initializeGestureTracking = () => { } + // --- Palm rotation gesture to manipulate objects on the 3D canvas --- + let palmRotate = new RotationGesture("palmRotate"); + + palmRotate.onStart = ({state, id}, hand) => { + const h = hand.handedness; + const eventId = `${id}.${h}`; + let {x, y, z} = toScreen(state[h].refFinger, h); + canvas3D_down(x, y, z, eventId); + }; + + palmRotate.onActive = ({state, id}, hand) => { + const h = hand.handedness; + const eventId = `${id}.${h}`; + canvas3D_rotate(state[h].rotationMatrix, h, eventId); + }; + + palmRotate.onEnd = ({state, id}, hand) => { + const h = hand.handedness; + const eventId = `${id}.${h}`; + let {x, y, z} = toScreen(state[h].refFinger, h); + canvas3D_up(x, y, z, eventId); + }; + + // Add all active gestures to the current tracker const gestureTracker = new GestureTracker(); gestureTracker.add(indexPinch); gestureTracker.add(middlePinch); gestureTracker.add(spreadGesture); + gestureTracker.add(palmRotate); window.gestureTracker = gestureTracker; } diff --git a/projects/0105/scenes/scene1.js b/projects/0105/scenes/scene1.js index 61a246f..70e7f1f 100644 --- a/projects/0105/scenes/scene1.js +++ b/projects/0105/scenes/scene1.js @@ -1,31 +1,48 @@ +let sceneTracker = new ObjectTracker(); +sceneTracker.addObj(Shape.cubeMesh(), [1, 0, 0], 0, 0, 0, 1, 1, 1); +sceneTracker.addObj(Shape.cubeMesh(), [0, 0, 1], 1, 1, 1, 1, 1, 1); + function Scene() { - let x = 0; - let y = 0; - let z = 0; - let ball = Shape.sphereMesh(30,15); this.vertexShader = Shader.defaultVertexShader; this.fragmentShader = Shader.shinyFragmentShader; this.update = () => { - drawObj(ball, mxm(move(2*x-1,2*y-1,2*z-1),scale(.4+.3*z)),[1,0,0]); + sceneTracker.drawObjs(); } - this.onDrag = (_x,_y,_z) => { + this.onDown = (_x, _y, _z, id) => { + let [x, y, z] = getNormalizedPoint(_x, _y, _z); + let h = getHandedness(id); + sceneTracker.onPinch(x, y, z, h); + } -/* -let X = canvas3D_x(), Y = canvas3D_y(), W = canvas3D.width; + this.onUp = (_x, _y, _z, id) => { + let h = getHandedness(id); + sceneTracker.onLeave(h); + } -_x = (_x + 1) / 2 * W + X; -_x = 2 * (2 * _x - 1.5 * X - X) / W - 1; + this.onRotate = (rMatrix, handedness, id) => { + sceneTracker.onRotate(rMatrix, handedness, id); + } -_y = (1 - _y) / 2 * W + Y; -_y = 1 - 2 * (2 * _y - 1.5 * Y - 40 - Y) / W; -*/ + this.rescale = (left, right, otherHand) => { + let vector = subtract(right, left); + sceneTracker.rescale(vector, otherHand); + } + this.onDrag = (_x,_y,_z, id) => { + let [x, y, z] = getNormalizedPoint(_x, _y, _z); + let h = getHandedness(id); + sceneTracker.onDrag(x, y, z, h); + } + + let getNormalizedPoint = (_x, _y, _z) => { _x = 2 * _x + 1 - canvas3D_x() / canvas3D.width; _y = 2 * _y + 1 - canvas3D_y() / canvas3D.width - .1; - x = (_x + 1) / 2; - y = (_y + 1) / 2; - z = (_z + 1) / 2; + return [(_x + 1) / 2, (_y + 1) / 2, (_z + 1) / 2]; + } + + let getHandedness = (id) => { + return id.split(".")[1]; } }