init

Author: tariqdaouda

CHANGELOG.rst

@@ -0,0 +1,3 @@
+1.0.0:
+=======
+The begining of a new era

DESCRIPTION.rst

@@ -0,0 +1,4 @@
+Mariana:
+=======
+
+A machine learning framework for buildung deep neural networks

MANIFEST.in

@@ -0,0 +1,2 @@
+include *.rst
+include LICENCE

Mariana/NeuralNet.py

@@ -0,0 +1,199 @@
+import numpy
+import theano
+import cPickle
+import theano.tensor as tt
+
+class Layer(object) :
+
+	def __init__(self, name, inputs, nbInputs, nbOutputs, activation = None) :
+		"A generic definition a layer"
+		self.reset(name, inputs, nbInputs, nbOutputs, activation)
+
+	def reset(self, name, inputs, nbInputs, nbOutputs, activation = None) :
+		"resets everything to new parameters"
+
+		self.nbInputs = nbInputs
+		self.inputs = inputs
+		self.nbOutputs = nbOutputs
+		
+		self.name = name
+		self.activation = activation
+		
+		initWeights = numpy.random.random((nbInputs, nbOutputs)) 
+		initWeights = (initWeights/sum(initWeights))
+		initWeights = numpy.asarray(initWeights, dtype=theano.config.floatX)
+		self.W = theano.shared(value = initWeights, name = self.name + "_W")
+			
+		initBias = numpy.zeros((nbOutputs,), dtype=theano.config.floatX)
+		self.b = theano.shared(value = initBias, name = self.name + "_b")
+
+		self.params = [self.W, self.b]
+		
+		if self.activation is None :
+			self.outputs = tt.dot(self.inputs, self.W) + self.b
+		else :
+			self.outputs = self.activation(tt.dot(self.inputs, self.W) + self.b)
+
+	def serialize(self) :
+		"returns a dict {name, nbInputs, nbOutputs, activation, W, b}"
+		return {
+				"name" : self.name,
+				"nbInputs" : self.nbInputs,
+				"nbOutputs" : self.nbOutputs,
+				"activation" : self.activation,
+				"W" : self.W.get_value(borrow = True),
+				"b" : self.b.get_value(borrow = True)
+			}
+
+	def __str__ (self) :
+		if self.nbOutputs < 21:
+			o = " O"*self.nbOutputs
+		else :
+			o = " O O O ... O O O"
+		return "%s: [%s ](%s x %s)" % (self.name, o, self.nbInputs, self.nbOutputs)
+
+class NeuralNet(object) :
+
+	def __init__(self, name, nbInputs, costFct, lr = 0.01, momentum = 0, l1 = 0., l2 = 0.) :
+		"A neural network"
+		self.reset(name, nbInputs, costFct, lr, momentum, l1, l2)
+
+	def reset(self, name, nbInputs, costFct, lr, momentum, l1, l2) :
+		self.name = name
+		self.nbInputs = nbInputs
+		self.costFct = costFct
+		
+		self.inputs = tt.matrix(name = self.name + "_X")
+		self.y = tt.ivector(name = self.name + "_Y")
+		self.layers = []
+		self.layersDct = {}
+		self.params = []
+		self.lr = lr
+		self.momentum = momentum
+		self.l1 = l1
+		self.l2 = l2
+		
+		self._mustInitUpdates = True
+
+	def stackLayer(self, name, nbOutputs, activation) :
+		"adds a layer to the stack and returns it"
+		if name in self.layersDct :
+			raise KeyError("There's already a layer by the name '%s'" % name)
+
+		if len(self.layers) < 1 :
+			layer = Layer(self.name + "_" + name, self.inputs, self.nbInputs, nbOutputs, activation)
+		else :
+			priorLayer = self.layers[-1]
+			layer = Layer(self.name + "_" + name, priorLayer.outputs, priorLayer.nbOutputs, nbOutputs, activation)
+
+		self.layersDct[name] = (layer, len(self.layers))
+		self.layers.append(layer)
+		return layer
+
+	def popLayer(self) :
+		"removes the last layer from the stack and returns it"
+		layer = self.layers.pop()
+		del(self.layersDct[layer.name])
+
+		self._mustInitUpdates = True
+
+		return layer
+
+	def _initUpdates(self) :
+		self.outputs = self.layers[-1].outputs
+		cost = self.costFct(self.y, self.outputs)
+		self.updates = []
+		for layer in self.layers :
+			self.params.extend(layer.params)
+			for param in layer.params :
+				# gparam = tt.grad(cost, param) + self.momentun * ()
+				gparam = tt.grad(cost, param)
+				momentum_param = theano.shared(param.get_value()*0., broadcastable=param.broadcastable)
+				self.updates.append((momentum_param, self.momentum * momentum_param + (1-self.momentum)*gparam))
+				self.updates.append((param, param - self.lr * momentum_param))
+				
+				
+		L1 =  self.l1 * sum([abs(l.W).sum() for l in self.layers])
+		L2 = self.l2 * sum([(l.W**2).sum() for l in self.layers])
+		cost = self.costFct(self.y, self.outputs) + L1 + L2
+
+		self.theano_train = theano.function(inputs = [self.inputs, self.y], outputs = [cost, self.outputs], updates = self.updates)
+		self.theano_test = theano.function(inputs = [self.inputs, self.y], outputs = [cost, self.outputs])
+		self.theano_propagate = theano.function(inputs = [self.inputs], outputs = self.outputs)
+		self.theano_prediction = theano.function(inputs = [self.inputs], outputs = tt.argmax(self.outputs, axis = 1))
+		
+		self._mustInitUpdates = False
+
+	def train(self, x, y) :
+		if self._mustInitUpdates :
+			self._initUpdates()
+		# print x.shape
+		return self.theano_train(x, y)
+
+	def test(self, x, y) :
+		"same function for both test and validation"
+		if self._mustInitUpdates :
+			self._initUpdates()
+		return self.theano_test(x, y)
+
+	def propagate(self, x) :
+		if self._mustInitUpdates :
+			self._initUpdates()
+		return self.theano_propagate(x)
+
+	def predict(self, x) :
+		if self._mustInitUpdates :
+			self._initUpdates()
+		return self.theano_prediction(x)
+
+	def save(self, filename) :
+		"save the whole model"
+		fil = open(filename + '.mdl', 'wb')
+		
+		model = {
+			"name" : self.name,
+			"nbInputs" : self.nbInputs,
+			"costFct" : self.costFct,
+			"lr" : self.lr,
+			"momentum" : self.momentum,
+			"l1" : self.l1,
+			"l2" : self.l2,
+			"layers" : []
+		}
+
+		layers = []
+		for layer in self.layers :
+			params = []
+			layers.append(layer.serialize())
+
+		model["layers"] = layers
+		cPickle.dump(model, fil, -1)
+		fil.close()
+
+	@classmethod
+	def load(cls, filename) :
+		"load a previously saved model"
+		fil = open(filename)
+		model = cPickle.load(fil)
+		nn = NeuralNet( model["name"], model["nbInputs"], model["costFct"], model["lr"], model["momentum"], model["l1"], model["l2"])
+		for layer in model.layers :
+			l = nn.stackLayer(layer["name"], layer["nbOutputs"], layer["activation"])
+			l.W = layer["W"]
+			l.b = layer["b"]
+		fil.close()
+		return nn
+
+	def __getitem__(self, layerName) :
+		return self.layersDct[layerName]
+	
+	def __str__(self) :
+		ls = []
+		s = "<Net: %s (inputs: %s, cost: %s, lr: %s, momentum: %s, l1: %s, l2: %s)>" % (self.name, self.nbInputs, self.costFct.__name__, self.lr, self.momentum, self.l1, self.l2)
+		for l in self.layers :
+			strl = str(l)
+			ls.append( ' '* (len(s)/2) + 'X' )
+			ls.append(' '*( (len(s)-len(strl))/2) + strl)
+			prevLen = strl
+			
+		s += "\n\n%s" % ('\n'.join(ls))
+		return s

Mariana/OLDNeuralNet.py

@@ -0,0 +1,132 @@
+import numpy
+import theano
+import theano.tensor as tt
+
+class Layer(object) :
+
+	def __init__(self, name, inputs, nbInputs, nbOutputs, activation = None) :
+		
+		self.nbInputs = nbInputs
+		self.nbOutputs = nbOutputs
+		
+		self.inputs = inputs
+		self.name = name
+		self.activation = activation
+		
+		#simply init with small random weights, should use glorot et al. 2010
+		tmpW = numpy.random.random((nbInputs, nbOutputs)) 
+		tmpW = (tmpW/sum(tmpW)) #*0.001
+		initW = numpy.asarray(tmpW, dtype=theano.config.floatX)
+		self.W = theano.shared(value = initW, name = self.name + "_W")
+		
+		initB = numpy.zeros((nbOutputs,), dtype=theano.config.floatX)
+		self.b = theano.shared(value = initB, name = self.name + "_b")
+		
+		self.params = [self.W, self.b]
+		
+		if self.activation is None :
+			self.outputs = tt.dot(self.inputs, self.W) + self.b
+		else :
+			self.outputs = self.activation(tt.dot(self.inputs, self.W) + self.b)
+
+	def __str__ (self) :
+		return "Layer: %s" % self.name
+
+class NeuralNet(object) :
+
+	def __init__(self, name, nbInputs, costFct, lr = 0.01, l1 = 0., l2 = 0.) :
+		self.name = name
+		self.nbInputs = nbInputs
+		self.costFct = costFct
+		self.inputs = tt.lmatrix(name = self.name + "_X")
+		self.y = tt.lvector(name = self.name + "_Y")
+		self.layers = []
+		self.layersDct = {}
+		self.params = []
+		self.lr = lr
+		self.l1 = l1
+		self.l2 = l2
+		
+		self._mustInitUpdates = True
+
+	def stackLayer(self, name, nbOutputs, activation) :
+		if name in self.layersDct :
+			raise KeyError("There's already a layer by the name %s" % name)
+
+		if len(self.layers) < 1 :
+			layer = Layer(self.name + "_" + name, self.inputs, self.nbInputs, nbOutputs, activation)
+		else :
+			priorLayer = self.layers[-1]
+			layer = Layer(self.name + "_" + name, priorLayer.outputs, priorLayer.nbOutputs, nbOutputs, activation)
+
+		self.layersDct[name] = layer
+		self.layers.append(layer)
+		self.params.extend(layer.params)
+	
+	def _initUpdates(self) :
+		self.outputs = self.layers[-1].outputs
+		self.cost = self.costFct(self.y, self.outputs)
+
+		self.updates = []
+		self.gs = []
+		for param in self.params :
+			gparam = tt.grad(self.cost, param)
+			self.updates.append((param, param - self.lr * gparam))
+			self.gs.append(gparam)
+			
+		self.theano_train = theano.function(inputs = [self.inputs, self.y], outputs = [self.cost, self.outputs], updates = self.updates)
+		self.theano_test = theano.function(inputs = [self.inputs, self.y], outputs = [self.cost, self.outputs])
+		self.theano_propagate = theano.function(inputs = [self.inputs], outputs = self.outputs)
+		#~ self.theano_predict = theano.function(inputs = [self.inputs], outputs = tt.argmax(self.outputs))
+		
+		self._mustInitUpdates = False
+
+	def train(self, x, y) :
+		if self._mustInitUpdates :
+			self._initUpdates()
+		
+		return self.theano_train(x, y)
+
+	def test(self, x, y) :
+		"same function for both test and validation"
+		if self._mustInitUpdates :
+			self._initUpdates()
+		return self.theano_test(x, y)
+
+	def propagate(self, x) :
+		if self._mustInitUpdates :
+			self._initUpdates()
+		return self.theano_propagate(x)
+
+	def predict(self, x) :
+		if self._mustInitUpdates :
+			self._initUpdates()
+		return tt.argmax(self.theano_predict(x))
+
+	def L1(self) :
+		return sum([abs(l.W).sum() for l in self.layers])
+
+	def L2(self) :
+		return sum([(l.W**2).sum() for l in self.layers])
+
+	def __getitem__(self, layerName) :
+		return self.layersDct[layerName]
+	
+	def __str__(self) :
+		ls = []
+		for l in self.layers :
+			ls.append(l.name)
+		ls = " -> ".join(ls)
+		s = "Net: %s (%s)" % (self.name, ls)
+		return s
+
+def negLogLikelihood(y, outputs) :
+	"""cost fct for softmax"""
+	cost = -tt.mean(tt.log(outputs)[tt.arange(y.shape[0]), y])
+	return cost
+
+def meanSquaredError(y, outputs) :
+	"""cost fct"""
+	cost = -tt.mean( tt.dot(outputs, y) **2 )
+	#~ cost = -tt.mean(outputs - y)
+	return cost

Mariana/V2/layers.py

@@ -0,0 +1,15 @@
+import layers as L
+
+class Model(object) :
+	
+	def __init__(self,name) :
+		self.name = name
+		self.layers = {}
+		self.inputLayers = {}
+
+	def _registerLayer(self, layer) :
+		if layer.name in layers :
+			raise KeyError("The model '%s' has already a layer called '%s'" % (self.name, layer.name)
+		self.layers[layer.name] = layer
+		if layer.__class__ is L.InputLayer :
+			self.inputLayers[layer.name] = layer

Mariana/V2/model.py

@@ -0,0 +1,25 @@
+import json
+import layers as L
+
+class MarianaParser(object) :
+	"parse a json entry to create network"
+	def __init__(self) :
+		pass
+
+	def parseFile(self, jsonFile) :
+		f = open(jsonFile)
+		self.json = json.load(f)
+		f.close()
+		model = M.model(self.json['name'])
+
+		for jl in jsonLayers :
+			if jl['function'] == 'input' :
+				layers.append(L.InputLayer(model = model, **jl))
+			else :
+				if jl['function'] == 'hidden' :
+					layers.append(L.HiddenLayer(model = model, **jl))
+				elif jl['function'] == 'output' :
+					layers.append(L.OutputLayer(model = model, **jl))
+				else :
+					raise ValueError("Unknown layer function '%s'" % jl['function'])
+		return model