kenperlin · arnavgupta829 · Feb 5, 2026
diff --git a/core/js/bici.js b/core/js/bici.js
@@ -37,7 +37,7 @@ let coreFiles = `M4, pca, loadImage, webgl, webcam, trackHead, help,
 	        midi, numberString, pen, aiScriptPanel, keyEvent, matchCurves,
 	        glyphs, chalktalk, codeArea, math, shape, shader,
 	        diagram, sliders, widgets, webrtc-client, video-ui, implicit,
-		mediapipe, gesture, shadowHand, tracking, main`.split(',');
+		mediapipe, gesture, shadowHand, tracking, grabableObjects, main`.split(',');
 
 let project, slideData;
 let coreLoaded = false;

diff --git a/core/js/gesture.js b/core/js/gesture.js
@@ -176,7 +176,6 @@ class PinchGesture extends HandGesture {
     this.state[h] = newState;
   }
 
-
   _onStart(hand) {
     this.updateState(hand);
     super._onStart(hand);
@@ -193,8 +192,67 @@ class PinchGesture extends HandGesture {
   }
 }
 
+class RotationGesture extends HandGesture {
+  constructor(id, maxDistance = 0.15, activationThreshold = 1, activeCooldown = 33) {
+    let detectSpread = (hand) => {
+      const scaleFac = Math.min(1, .1 / (.2 + hand.landmarks[4].z));
+      let distances = getFingerThumbDistances([1, 2, 3, 4], hand);
+      return distances.every((d) => d > maxDistance * scaleFac);
+    }
+    super(id, activationThreshold, activeCooldown, detectSpread);
+  }
+
+  /**
+   * Orientation of the object being decided by the orientation of the palm in space
+   * @param {*} hand Hand data returned from mediapipe
+   */
+  updateState(hand) {
+    const h = hand.handedness;
+
+    // Figuring out plane of hand. Used to figure out axis of rotation
+    const pWrist = hand.landmarks[0];
+    const pThumbTip = hand.landmarks[4];
+    const pPinkyTip = hand.landmarks[20];
+
+    const vecThumbToLast = normalize(subtract(pointToArray(pPinkyTip), pointToArray(pThumbTip)));
+    const palmNormal = normalize(
+      cross(
+        subtract(pointToArray(pPinkyTip), pointToArray(pWrist)),
+        subtract(pointToArray(pThumbTip), pointToArray(pWrist))
+      )
+    );
+    // Vector perpendicular to the above two vectors. Forms a basis.
+    const vecPerp = normalize(cross(palmNormal, vecThumbToLast));
+
+    this.state[h].rotationMatrix = [
+      vecThumbToLast[0], vecPerp[0], palmNormal[0], 0,
+      vecThumbToLast[1], vecPerp[1], palmNormal[1], 0,
+      vecThumbToLast[2], vecPerp[2], palmNormal[2], 0,
+      0, 0, 0, 1
+    ];
+
+    this.state[h].refFinger = pThumbTip;
+  }
+
+  _onStart(hand) {
+    this.updateState(hand);
+    super._onStart(hand);
+  }
+
+  _onEnd(hand) {
+    super._onEnd(hand);
+    this.state[hand.handedness] = {};
+  }
+
+  _onActive(hand) {
+    this.updateState(hand);
+    super._onActive(hand);
+  }
+}
+
+let pointToArray = p => [ p.x, p.y, p.z ];
+
 function getFingerThumbDistances(fingers, hand) {
-  let pointToArray = p => [ p.x, p.y, p.z ];
   let distances = [];
   const thumbPt = pointToArray(hand.landmarks[4]);
 

diff --git a/core/js/grabableObjects.js b/core/js/grabableObjects.js
@@ -0,0 +1,122 @@
+class GrabableObject {
+  // Supports cubic hitboxes for now
+  constructor(mesh, color, x, y, z, lx, ly, lz) {
+    this.pos = {x: x, y: y, z: z};
+    this.bounds = {lx: lx, ly: ly, lz: lz};
+    // Default scaling and rotations
+    this.scale = 1;
+    this.aim = identity();
+    this.mesh = mesh;
+    this.color = color;
+  }
+
+  setScale(scale) {
+    this.scale = scale;
+  }
+
+  drawObject() {
+    // ToDo: Replace using matrix stack in utils
+    let m = 
+      mxm(
+        mxm(
+          mxm(
+            move(this.pos.x * 2 - 1, this.pos.y * 2 - 1, this.pos.z * 2 - 1),
+            scale(0.4 + 0.3 * this.pos.z)
+          ),
+          scale(this.scale)
+        ), 
+        this.aim
+      );
+
+    drawObj(this.mesh, m, this.color);
+  }
+
+  getEffectiveSize() {
+    const scale = this.scale * (0.4 + 0.3 * this.pos.z);
+    return {
+      lx: this.bounds.lx * scale,
+      ly: this.bounds.ly * scale,
+      lz: this.bounds.lz * scale
+    }
+  }
+
+  isHitting(obj2) {
+    const size1 = this.getEffectiveSize();
+    const size2 = obj2.getEffectiveSize();
+
+    return this.pos.x - size1.lx/2 < obj2.pos.x + size2.lx/2
+      && this.pos.x + size1.lx/2 > obj2.pos.x - size2.lx/2
+      && this.pos.y - size1.ly/2 < obj2.pos.y + size2.ly/2
+      && this.pos.y + size1.ly/2 > obj2.pos.y - size2.ly/2
+      && this.pos.z - size1.lz/2 < obj2.pos.z + size2.lz/2
+      && this.pos.z + size1.lz/2 > obj2.pos.z - size2.lz/2;
+  }
+}
+
+class ObjectTracker {
+  constructor() {
+    this.objs = [];
+    this.currSelection = null;
+  }
+
+  addObj(mesh, color, x, y, z, lx, ly, lz) {
+    let obj = new GrabableObject(mesh, color, x, y, z, lx, ly, lz);
+    this.objs.push(obj);
+  }
+
+  drawObjs() {
+    for (const obj of this.objs) {
+      obj.drawObject();
+    }
+  }
+
+  onPinch(x, y, z, handedness) {
+    if (this.currSelection) {
+      return;
+    }
+
+    for (const obj of this.objs) {
+      let dx = Math.abs(obj.pos.x - x);
+      let dy = Math.abs(obj.pos.y - y);
+      if (norm([dx, dy, 0]) <= 0.3) {
+        this.currSelection = {object: obj, hand: handedness}
+        break;
+      }
+    }
+  }
+
+  onLeave(handedness) {
+    if (this.currSelection?.hand === handedness) {
+      this.currSelection = null;
+    }
+  }
+
+  onDrag(x, y, z, handedness) {
+    if (this.currSelection?.hand === handedness) {
+      const prevPos = {...this.currSelection.object.pos};
+      this.currSelection.object.pos = {x: x, y: y, z: z};
+      for (const obj of this.objs) {
+        if (obj === this.currSelection.object) {
+          continue;
+        }
+        if (this.currSelection.object.isHitting(obj)) {
+          this.currSelection.object.pos = prevPos;
+        }
+      }
+    }
+  }
+
+  rescale(vector, otherHand) {
+    const holderHand = otherHand == "left" ? "right" : "left";
+    if (this.currSelection?.hand === holderHand) {
+      this.currSelection.object.scale = 10 * norm(vector);;
+    }
+  }
+
+  onRotate(rMatrix, otherHand) {
+    const holderHand = otherHand == "left" ? "right" : "left";
+    if (this.currSelection?.hand === holderHand) {
+      this.currSelection.object.aim = rMatrix;
+    }
+  }
+}
diff --git a/core/js/main.js b/core/js/main.js
@@ -69,6 +69,29 @@ let canvas3D_up = (x,y,z,id='id') => {
       scene.onUp(xToScene(x), yToScene(y), zToScene(z), id);
 }
 
+let canvas3D_rescale = (state, otherHand, id) => {
+   if (! canvas3D.isDown)
+      canvas3D.isDown = {};
+
+   if (scene && scene.rescale && canvas3D.isDown[id]) {
+      scene.rescale(
+         [state.left.x, state.left.y, state.left.z], 
+         [state.right.x, state.right.y, state.right.z],
+         otherHand
+      );
+   }
+}
+
+let canvas3D_rotate = (rMatrix, handedness, id='id') => {
+   if (!canvas3D.isDown)
+      canvas3D.isDown = {};
+
+   if (scene && scene.onRotate && canvas3D.isDown[id]) {
+      console.log("rotating");
+      scene.onRotate(rMatrix, handedness, id);
+   }
+}
+
 canvas3D.addEventListener('mousemove', event => canvas3D_move(event.clientX, event.clientY, 0, "mouse"));
 canvas3D.addEventListener('mousedown', event => canvas3D_down(event.clientX, event.clientY, 0, "mouse"));
 canvas3D.addEventListener('mouseup'  , event => canvas3D_up  (event.clientX, event.clientY, 0, "mouse"));

diff --git a/core/js/tracking.js b/core/js/tracking.js
@@ -645,7 +645,7 @@ let initializeGestureTracking = () => {
       }
    };
 
-   indexPinch.onActive = ({state, id}, hand) => {
+   indexPinch.onActive = ({state, isActive, id}, hand) => {
 
       const h = hand.handedness;
       const {x, y, z} = toScreen(state[h], h);
@@ -662,8 +662,11 @@ let initializeGestureTracking = () => {
       const eventId = `${id}.${state[h].pointer}`;
       if(state[h].pointer === 'head') {
          canvas3D_move(headX, headY, 0, eventId)
+      } else if ((isActive.left??false) && (isActive.right??false)) { 
+         // ToDo: Find an appropriate place to check for two handed gestures
+         canvas3D_rescale(state, h, eventId);
       } else {
-         canvas3D_move(x, y, z, eventId)
+         canvas3D_move(x, y, z, eventId);
       }
    };
 
@@ -735,10 +738,35 @@ let initializeGestureTracking = () => {
 
    }
 
+   // --- Palm rotation gesture to manipulate objects on the 3D canvas ---
+   let palmRotate = new RotationGesture("palmRotate");
+
+   palmRotate.onStart = ({state, id}, hand) => {
+      const h = hand.handedness;
+      const eventId = `${id}.${h}`;
+      let {x, y, z} = toScreen(state[h].refFinger, h);
+      canvas3D_down(x, y, z, eventId);
+   };
+
+   palmRotate.onActive = ({state, id}, hand) => {
+      const h = hand.handedness;
+      const eventId = `${id}.${h}`;
+      canvas3D_rotate(state[h].rotationMatrix, h, eventId);
+   };
+
+   palmRotate.onEnd = ({state, id}, hand) => {
+      const h = hand.handedness;
+      const eventId = `${id}.${h}`;
+      let {x, y, z} = toScreen(state[h].refFinger, h);
+      canvas3D_up(x, y, z, eventId);
+   };
+
+   // Add all active gestures to the current tracker
    const gestureTracker = new GestureTracker();
    gestureTracker.add(indexPinch);
    gestureTracker.add(middlePinch);
    gestureTracker.add(spreadGesture);
+   gestureTracker.add(palmRotate);
 
    window.gestureTracker = gestureTracker;   
 }

diff --git a/projects/0105/scenes/scene1.js b/projects/0105/scenes/scene1.js
@@ -1,31 +1,48 @@
+let sceneTracker = new ObjectTracker();
+sceneTracker.addObj(Shape.cubeMesh(), [1, 0, 0], 0, 0, 0, 1, 1, 1);
+sceneTracker.addObj(Shape.cubeMesh(), [0, 0, 1], 1, 1, 1, 1, 1, 1);
+
 function Scene() {
-   let x = 0;
-   let y = 0;
-   let z = 0;
-   let ball = Shape.sphereMesh(30,15);
    this.vertexShader = Shader.defaultVertexShader;
    this.fragmentShader = Shader.shinyFragmentShader;
    this.update = () => {
-      drawObj(ball, mxm(move(2*x-1,2*y-1,2*z-1),scale(.4+.3*z)),[1,0,0]);
+      sceneTracker.drawObjs();
    }
-   this.onDrag = (_x,_y,_z) => {
 
+   this.onDown = (_x, _y, _z, id) => {
+      let [x, y, z] = getNormalizedPoint(_x, _y, _z);
+      let h = getHandedness(id);
+      sceneTracker.onPinch(x, y, z, h);
+   }
 
-/*
-let X = canvas3D_x(), Y = canvas3D_y(), W = canvas3D.width;
+   this.onUp = (_x, _y, _z, id) => {
+      let h = getHandedness(id);
+      sceneTracker.onLeave(h);
+   }
 
-_x = (_x + 1) / 2 * W + X;
-_x = 2 * (2 * _x - 1.5 * X - X) / W - 1;
+   this.onRotate = (rMatrix, handedness, id) => {
+      sceneTracker.onRotate(rMatrix, handedness, id);
+   }
 
-_y = (1 - _y) / 2 * W + Y;
-_y = 1 - 2 * (2 * _y - 1.5 * Y - 40 - Y) / W;
-*/
+   this.rescale = (left, right, otherHand) => {
+      let vector = subtract(right, left);
+      sceneTracker.rescale(vector, otherHand);
+   }
 
+   this.onDrag = (_x,_y,_z, id) => {
+      let [x, y, z] = getNormalizedPoint(_x, _y, _z);
+      let h = getHandedness(id);
+      sceneTracker.onDrag(x, y, z, h);
+   }
+
+   let getNormalizedPoint = (_x, _y, _z) => {
       _x = 2 * _x + 1 - canvas3D_x() / canvas3D.width;
       _y = 2 * _y + 1 - canvas3D_y() / canvas3D.width - .1;
 
-      x = (_x + 1) / 2;
-      y = (_y + 1) / 2;
-      z = (_z + 1) / 2;
+      return [(_x + 1) / 2, (_y + 1) / 2, (_z + 1) / 2];
+   }
+
+   let getHandedness = (id) => {
+      return id.split(".")[1];
    }
 }