I added a getting started section to the readme

.gitignore

@@ -10,6 +10,7 @@ app/bin/*
 build/*
 *.pbxuser
 *.mode1v3
+*.perspectivev3
 
 # old skool
 .svn

README.md

@@ -22,6 +22,35 @@ We want to recognize:
 - Individual fingers
 - Finger-based hand gestures (e.g. peace sign, etc)
 
+Getting Started
+---------------
+
+Right now, you'll want to be on Chrome, on OSX, then:
+
+- Plug in your Kinect to your computer via USB
+- Download this code from http://github.com/doug/depthjs
+- Install the chrome extension
+  - Bring up the extensions management page by clicking the wrench icon and choosing *Tools* > *Extensions*.
+  - If *Developer mode* has a + by it, click the + to add developer information to the page. The + changes to a -, and more buttons and information appear.
+  - Click the *Load unpacked extension* button. A file dialog appears.
+  - In the file dialog, navigate to your *depthjs/chrome-extension-mac* and click *OK*.
+- Open a new web page (it only affects new pages)  
+- Have fun!
+
+What the camera sees...
+
+- When the extension starts up, it opens 3 windows, blob, first, and second
+- first & second show the limits of the depth of field the camera is paying attention to, you should try to make it so that your hand is in both first & second and not much else
+- blob shows you everything the camera can see as well as a blue circle around where it thinks your hand is.
+
+After opening a new page in Chrome,
+
+- Pull your hand back so that both first & second windows are blank and there is no circle in the blob window
+- Then bring your hand forward until it gets an outline in the blob window, and pause for a second until the blue circle appears
+- You should then see a blue circle on the web site that will track your movements
+- To click, just close your hand into a fist!
+
+
 Components
 ----------
 DepthJS is very modular. The Kinect driver and computer vision are written on top of Open Frameworks and OpenCV in C++. This component can output the raw RGB image, the raw depth map (filtered for the hand), as well as the high-level events that the computer vision recognizes. The three outputs are pumped out on three separate 0MQ TCP sockets. Next, a Torando web server (written in Python) takes the 0MQ data and wraps it into a WebSocket, which is what enables the web browser extension to receive the data. Finally a pure javascript-based extension connects to the WebSocket server to receive the events. Event handlers may be placed globally, in content scripts injected into each web page, or pushed via the content script to local DOM elements written by 3rd parties.

chrome-extension-mac/content_script/event_handlers.js

@@ -30,16 +30,27 @@ console.log("DepthJS: Loading event handlers");
 DepthJS.state = null;
 DepthJS.lastRegisterTime = null;
 
+DepthJS.trigger = function(element, name) {
+  var event = document.createEvent("Events")
+  event.initEvent(name, true, true); //true for can bubble, true for cancelable
+  element.dispatchEvent(event);
+}
+
+DepthJS.trigger(window, "depthjs-loading");
+
 DepthJS.eventHandlers.onSwipeLeft = function() {
+  DepthJS.trigger(window, "swipeLeft");
    // history.back();
 };
 
 DepthJS.eventHandlers.onSwipeRight = function() {
+  DepthJS.trigger(window, "swipeRight");
   // We interpret as "forward".
   // history.forward();
 };
 
 DepthJS.eventHandlers.onSwipeDown = function() {
+  DepthJS.trigger(window, "swipeDown");
   // We interpret as "scroll down 75% of window".
   // var scrollAmount = Math.floor($(window).height() * 0.75);
   // $("html, body").animate({
@@ -48,6 +59,7 @@ DepthJS.eventHandlers.onSwipeDown = function() {
 };
 
 DepthJS.eventHandlers.onSwipeUp = function() {
+  DepthJS.trigger(window, "swipeUp");
   // We interpret as "scroll up 75% of window".
   // var scrollAmount = Math.floor($(window).height() * 0.75);
   // $("html, body").animate({
@@ -56,12 +68,14 @@ DepthJS.eventHandlers.onSwipeUp = function() {
 };
 
 DepthJS.eventHandlers.onHandPointer = function(){
+  DepthJS.trigger(window, "handPointer");
   if (DepthJS.verbose) console.log("DepthJS. Hand Pointer");
   DepthJS.eventHandlers.onUnregister();
   DepthJS.state = "selectorBox";
 };
 
 DepthJS.eventHandlers.onHandOpen = function(){
+  DepthJS.trigger(window, "handOpen");
   if (DepthJS.verbose) console.log("DepthJS. Hand Open");
   DepthJS.eventHandlers.onUnregister();
   DepthJS.state = "panner";
@@ -83,11 +97,12 @@ DepthJS.eventHandlers.onSelectorBoxMode = function() {
 // POINTER -----------------------------------------------------------------------------------------
 DepthJS.eventHandlers.onRegister = function(data) {
   if (DepthJS.verbose) console.log("DepthJS: User registered their hand");
-  $(window).trigger("touchstart");
+  DepthJS.trigger(window, "touchStart");
   if (data.mode == "twohands") {
     console.log("Ignoring in two hands for now");
     return;
   }
+  data.mode = "theforce";
   if (data.mode == "theforce") {
     DepthJS.registerMode = "selectorBox";
   } else if (data.mode == "twohands") {
@@ -109,6 +124,7 @@ DepthJS.eventHandlers.onUnregister = function() {
   DepthJS.selectorBox.hide();
   DepthJS.selectorBoxPopup.hide();
   DepthJS.depthose.hide();
+  DepthJS.trigger(window, "touchStop");  
 };
 
 DepthJS.eventHandlers.onHandClick = function() {
@@ -167,6 +183,8 @@ DepthJS.eventHandlers.onMove = function(data) {
   } else if (DepthJS.state == "selectorBoxPopup") {
     DepthJS.selectorBoxPopup.move(accumulatedX, accumulatedY, accumulatedZ);
   } else {
+    //console.debug("setting the force")
+    DepthJS.eventHandlers.onRegister({mode:"theforce"});
     if (DepthJS.verbose) console.log("Ignoring move in state " + DepthJS.state);
   }
 };

chrome-extension-mac/content_script/init.js

@@ -9,15 +9,15 @@ $(function() {
 
     // Let us know its running
     console.log("Finished initing, sticking in logo");
-    $("<img src='https://github.com/doug/depthjs/raw/master/chrome-extension/logo_128x128.png'>").css({
-      position: "fixed",
-      width: "32px",
-      height: "32px",
-      bottom: "20px",
-      left: "20px"
-    }).appendTo("body");
-    console.log($("img"));
-
+    $("<img/>").attr("src", chrome.extension.getURL("logo_128x128.png"))
+               .css({
+                  position: "fixed",
+                  width: "32px",
+                  height: "32px",
+                  bottom: "20px",
+                  left: "20px"
+               })
+               .appendTo("body");
 
     var lastTime = null;
     function reloadChecker() {

chrome-extension-mac/content_script/root.js

@@ -54,7 +54,7 @@ function print() {
   });
 
   var alphabeticalKeys = _.keys(counts).sort();
-  console.log("------" + (new Date() + ""));
+  // console.log("------" + (new Date() + ""));
   _.each(alphabeticalKeys, function(type) {
     console.log(["   " + counts[type] + " " + type + "; last = ", lastByType[type]]);
   });

chrome-extension-mac/content_script/selector_box.js

@@ -46,36 +46,48 @@ DepthJS.selectorBox.move = function(x, y) {
   if (x != $box.css("left") || y != $box.css("top")) {
     $box.css({left: x, top: y});
   }
+  DepthJS.selectorBox.handleHover()
 };
 
-DepthJS.selectorBox.activate = function() {
-  if (DepthJS.verbose) console.log("DepthJS: Activating underneath selectorBox");
-  // Lame code for now...
-
-  var $intersectingLinks = $("a").filter(function() {
-    var $a = $(this);
-    var ax = $a.offset().left + $(window).scrollLeft();
-    var aw = $a.width();
-    var ay = $a.offset().top + $(window).scrollTop();
-    var ah = $a.height();
-
-    var $box = DepthJS.selectorBox.$box;
-    var bx = $box.position().left;
-    var by = $box.position().top;
-    var bw = $box.width();
-    var bh = $box.height();
-
-    if (by > ay + ah || // box-top is lower than link-bottom
-        by + bh < ay || // box-bottom is higher than link-top
-        bx > ax + aw || // box-left is right of link right
-        bx + bw < aw) { // box-right is left of link left
-      return false;
+DepthJS.selectorBox.elementAtCursor = function() {
+  var $box = DepthJS.selectorBox.$box;
+  var x = $box.position().left + $box.width() / 2;
+  var y = $box.position().top + $box.height() / 2;
+
+  $box.hide();
+  var element = document.elementFromPoint(x, y);
+  $box.show();
+  return $(element).closestMatching('a,.hoverable');
+}
+
+$.fn.closestMatching = function(selector) {
+  if ($(this).is(selector)) return $(this)[0];
+
+  var parents = $(this).parents();
+  for (var i = 0; i < parents.length; i++) {
+    if ($(parents[i]).is(selector)) return $(parents[i])[0];
+  }
+  return undefined;
+}
+
+DepthJS.selectorBox.handleHover = function() {
+  var lastElement = $('.depthjs-hover')[0];
+  var element = DepthJS.selectorBox.elementAtCursor();
+
+  if (element == lastElement) { // same element
+    // do nothing
+  } else { 
+    if (lastElement) {
+      DepthJS.trigger($(lastElement).removeClass('depthjs-hover')[0], 'hoverOut');
     }
-    return true;
-  });
+    if (element) {
+      DepthJS.trigger($(element).addClass("depthjs-hover")[0], "hoverOver");
+    }
+  }
+};
 
-  if (DepthJS.verbose) console.log("Got " + $intersectingLinks.length + " links");
-  if (DepthJS.verbose) console.log($intersectingLinks);
+DepthJS.selectorBox.activate = function() {
+  var $intersectingLinks = DepthJS.selectorBox.elementAtCursor();
   if ($intersectingLinks.length > 0) {
     DepthJS.selectorBoxPopup.$links = $intersectingLinks;
     DepthJS.selectorBoxPopup.activate();

new_cv/NewCV/FreenectDevice.h

@@ -0,0 +1,104 @@
+/*
+ *  FreenectDevice.h
+ *  NewCV
+ *
+ *  Created by Roy Shilkrot on 3/3/11.
+ *  Copyright 2011 MIT. All rights reserved.
+ *
+ */
+#include "libfreenect.hpp"
+#include <pthread.h>
+#include <cv.hpp>
+#include <highgui.h>
+#include <ml.h>
+
+using namespace cv;
+
+#include <iostream>
+using namespace std;
+
+class Mutex {
+public:
+	Mutex() {
+		pthread_mutex_init( &m_mutex, NULL );
+	}
+	void lock() {
+		pthread_mutex_lock( &m_mutex );
+	}
+	void unlock() {
+		pthread_mutex_unlock( &m_mutex );
+	}
+private:
+	pthread_mutex_t m_mutex;
+};
+
+class MyFreenectDevice : public Freenect::FreenectDevice {
+public:
+	MyFreenectDevice(freenect_context *_ctx, int _index)
+	: Freenect::FreenectDevice(_ctx, _index), m_buffer_depth(FREENECT_DEPTH_11BIT_SIZE),m_buffer_rgb(FREENECT_VIDEO_RGB_SIZE), m_gamma(2048), m_new_rgb_frame(false), m_new_depth_frame(false),
+	depthMat(Size(640,480),CV_16UC1), rgbMat(Size(640,480),CV_8UC3,Scalar(0)), ownMat(Size(640,480),CV_8UC3,Scalar(0))
+	{
+		for( unsigned int i = 0 ; i < 2048 ; i++) {
+			float v = i/2048.0;
+			v = std::pow(v, 3)* 6;
+			m_gamma[i] = v*6*256;
+		}
+	}
+	// Do not call directly even in child
+	void VideoCallback(void* _rgb, uint32_t timestamp) {
+//		std::cout << "RGB callback" << std::endl;
+		m_rgb_mutex.lock();
+		uint8_t* rgb = static_cast<uint8_t*>(_rgb);
+		rgbMat.data = rgb;
+		m_new_rgb_frame = true;
+		m_rgb_mutex.unlock();
+	};
+	// Do not call directly even in child
+	void DepthCallback(void* _depth, uint32_t timestamp) {
+//		std::cout << "Depth callback" << std::endl;
+		m_depth_mutex.lock();
+		uint16_t* depth = static_cast<uint16_t*>(_depth);
+		depthMat.data = (uchar*) depth;
+		m_new_depth_frame = true;
+		m_depth_mutex.unlock();
+	}
+
+	bool getVideo(Mat& output) {
+		m_rgb_mutex.lock();
+		if(m_new_rgb_frame) {
+			cv::cvtColor(rgbMat, output, CV_RGB2BGR);
+			m_new_rgb_frame = false;
+			m_rgb_mutex.unlock();
+			return true;
+		} else {
+			m_rgb_mutex.unlock();
+			return false;
+		}
+	}
+
+	bool getDepth(Mat& output) {
+		m_depth_mutex.lock();
+		if(m_new_depth_frame) {
+			depthMat.copyTo(output);
+			m_new_depth_frame = false;
+			m_depth_mutex.unlock();
+			return true;
+		} else {
+			m_depth_mutex.unlock();
+			return false;
+		}
+	}
+
+private:
+	std::vector<uint8_t> m_buffer_depth;
+	std::vector<uint8_t> m_buffer_rgb;
+	std::vector<uint16_t> m_gamma;
+	Mat depthMat;
+	Mat rgbMat;
+	Mat ownMat;
+	Mutex m_rgb_mutex;
+	Mutex m_depth_mutex;
+	bool m_new_rgb_frame;
+	bool m_new_depth_frame;
+};
+