From 16954ca25db2353f80c4ae1c996392cfc4f695fd Mon Sep 17 00:00:00 2001
From: jonnew <jpnewman@mit.edu>
Date: Wed, 26 Oct 2016 17:21:27 -0400
Subject: [PATCH 1/3] Fix spike waveform transpose issue in Python3

- Implements the same fix given in OpenEphys.py in main directory, which
  I guess is for Pytho2.7
---
 Python3/OpenEphys.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Python3/OpenEphys.py b/Python3/OpenEphys.py
index 31bbfbb..a1ff732 100644
--- a/Python3/OpenEphys.py
+++ b/Python3/OpenEphys.py
@@ -219,7 +219,7 @@ def loadSpikes(filepath):
         sampleFreq = np.fromfile(f, np.dtype('<u2'),1)
         
         waveforms = np.fromfile(f, np.dtype('<u2'), numChannels*numSamples)
-        wv = np.reshape(waveforms, (numSamples, numChannels))
+        wv = np.reshape(waveforms, (numChannels, numSamples))
         
         gain[currentSpike,:] = np.fromfile(f, np.float32, numChannels)
         thresh[currentSpike,:] = np.fromfile(f, np.dtype('<u2'), numChannels)
@@ -229,7 +229,7 @@ def loadSpikes(filepath):
         #print wv.shape        
         
         for ch in range(numChannels):
-            spikes[currentSpike,:,ch] = (np.float64(wv[:,ch])-32768)/(gain[currentSpike,ch]/1000)
+            spikes[currentSpike,:,ch] = (np.float64(wv[ch])-32768)/(gain[currentSpike,ch]/1000)
         
         currentSpike += 1
         

From 96a8ca45401b6ef7572390a2cbae3aaa641f39fd Mon Sep 17 00:00:00 2001
From: jonnew <jpnewman@mit.edu>
Date: Wed, 26 Oct 2016 19:44:56 -0400
Subject: [PATCH 2/3] Four order of magnitude speedup in loadSpike

- Make use of numpy structured arrays to get rid while loop
- Single call to np.fromfile
- On my machine with a nice SSD, for a 40 MB .spike file, the load time
  went from 15.1 sec to about 7 milliseconds.
- This could probably be applied to other data types (continuous and
  events) but I'm not interested in doing it right now.
- I'm only editing one version of the OpenEphys.py file...
---
 Python3/OpenEphys.py | 57 ++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 52 insertions(+), 5 deletions(-)

diff --git a/Python3/OpenEphys.py b/Python3/OpenEphys.py
index a1ff732..2efb6bd 100644
--- a/Python3/OpenEphys.py
+++ b/Python3/OpenEphys.py
@@ -2,7 +2,7 @@
 """
 Created on Sun Aug  3 15:18:38 2014
 
-@author: Dan Denman and Josh Siegle
+@author: Dan Denman and Josh Siegle. Speedups from Jon Newman.
 
 Loads .continuous, .events, and .spikes files saved from the Open Ephys GUI
 
@@ -17,6 +17,7 @@
 import scipy.signal
 import scipy.io
 import time
+import math
 import struct
 from copy import deepcopy
 
@@ -174,8 +175,8 @@ def loadContinuous(filepath, dtype = float):
     ch['recordingNumber'] = recordingNumbers[0:recordNumber]
     f.close()
     return ch
-    
-def loadSpikes(filepath):
+
+def loadSpikesOld(filepath):
     
     # doesn't quite work...spikes are transposed in a weird way    
     
@@ -219,7 +220,7 @@ def loadSpikes(filepath):
         sampleFreq = np.fromfile(f, np.dtype('<u2'),1)
         
         waveforms = np.fromfile(f, np.dtype('<u2'), numChannels*numSamples)
-        wv = np.reshape(waveforms, (numChannels, numSamples))
+        wv = np.reshape(waveforms, (numSamples, numChannels))
         
         gain[currentSpike,:] = np.fromfile(f, np.float32, numChannels)
         thresh[currentSpike,:] = np.fromfile(f, np.dtype('<u2'), numChannels)
@@ -229,7 +230,7 @@ def loadSpikes(filepath):
         #print wv.shape        
         
         for ch in range(numChannels):
-            spikes[currentSpike,:,ch] = (np.float64(wv[ch])-32768)/(gain[currentSpike,ch]/1000)
+            spikes[currentSpike,:,ch] = (np.float64(wv[:,ch])-32768)/(gain[currentSpike,ch]/1000)
         
         currentSpike += 1
         
@@ -242,6 +243,52 @@ def loadSpikes(filepath):
     data['sortedId'] = sortedId[:currentSpike]
 
     return data
+
+def loadSpikes(filepath):
+    
+        data = { }
+        
+        f = open(filepath,'rb')
+        header = readHeader(f)
+        
+        if float(header[' version']) < 0.4:
+            raise Exception('Loader is only compatible with .spikes files with version 0.4 or higher')
+         
+        data['header'] = header 
+        numChannels = int(header['num_channels'])
+        numSamples = 40 # **NOT CURRENTLY WRITTEN TO HEADER**
+
+        # Record schema
+        dt = [('type', '<u1'),
+              ('timestamp', '<i8'),
+              ('softtime', '<i8'),
+              ('source', '<u2'),
+              ('numchan', '<u2'),
+              ('numsamp', '<u2'),
+              ('sortid', '<u2'),
+              ('electrodeid', '<u2'),
+              ('channel', '<u2'),
+              ('color', '<u1',(3)), # The color of a spike...
+              ('pcproj', np.float32 ,(2)),
+              ('sampfreq', '<u2'),
+              ('waveform', '<u2', (numChannels, numSamples)),
+              ('gain', np.float32, (numChannels)),
+              ('thresh', '<u2', (numChannels)),
+              ('recordnum', '<u2')]
+
+        record_type = np.dtype(dt); 
+        record_bytes = record_type.itemsize
+        bytes_left = os.fstat(f.fileno()).st_size - f.tell()
+        records_to_read = math.floor(bytes_left/record_bytes)
+
+        if records_to_read > MAX_NUMBER_OF_SPIKES:
+            print("Warning: only loading {} spikes".format(MAX_NUMBER_OF_SPIKES))
+            records_to_read = MAX_NUMBER_OF_SPIKES
+
+        data['data'] = np.zeros(records_to_read, dtype=record_type)
+        data['data'] = np.fromfile(f, record_type, records_to_read)
+
+        return data
     
 def loadEvents(filepath):
 

From 277aaf27866e2a8988d18546736c02f3e771a03c Mon Sep 17 00:00:00 2001
From: jonnew <jpnewman@mit.edu>
Date: Wed, 26 Oct 2016 19:48:07 -0400
Subject: [PATCH 3/3] Four order of magnitude speedup in loadSpike

- Make use of numpy structured arrays to get rid while loop
- Single call to np.fromfile
- On my machine with a nice SSD, for a 40 MB .spike file, the load time
  went from 15.1 sec to about 7 milliseconds.
- This could probably be applied to other data types (continuous and
  events) but I'm not interested in doing it right now.
- I'm only editing one version of the OpenEphys.py file...
- In keeping with just adding new functions to preserve backwards
  compatibility (instead of versioning...) I named the routine
  loadSpikeFast
---
 Python3/OpenEphys.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Python3/OpenEphys.py b/Python3/OpenEphys.py
index 2efb6bd..67d1b48 100644
--- a/Python3/OpenEphys.py
+++ b/Python3/OpenEphys.py
@@ -176,7 +176,7 @@ def loadContinuous(filepath, dtype = float):
     f.close()
     return ch
 
-def loadSpikesOld(filepath):
+def loadSpikes(filepath):
     
     # doesn't quite work...spikes are transposed in a weird way    
     
@@ -244,7 +244,7 @@ def loadSpikesOld(filepath):
 
     return data
 
-def loadSpikes(filepath):
+def loadSpikesFast(filepath):
     
         data = { }